aspera-cli 4.24.1 → 4.25.0.pre

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (99) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/CHANGELOG.md +1064 -745
  4. data/CONTRIBUTING.md +43 -100
  5. data/README.md +1281 -720
  6. data/bin/ascli +20 -1
  7. data/bin/asession +23 -27
  8. data/lib/aspera/agent/base.rb +10 -21
  9. data/lib/aspera/agent/connect.rb +2 -3
  10. data/lib/aspera/agent/desktop.rb +2 -2
  11. data/lib/aspera/agent/direct.rb +49 -32
  12. data/lib/aspera/agent/factory.rb +31 -0
  13. data/lib/aspera/api/aoc.rb +134 -76
  14. data/lib/aspera/api/cos_node.rb +3 -2
  15. data/lib/aspera/api/faspex.rb +213 -0
  16. data/lib/aspera/api/node.rb +107 -94
  17. data/lib/aspera/ascmd.rb +1 -2
  18. data/lib/aspera/ascp/installation.rb +73 -58
  19. data/lib/aspera/ascp/management.rb +119 -23
  20. data/lib/aspera/assert.rb +39 -11
  21. data/lib/aspera/cli/error.rb +4 -2
  22. data/lib/aspera/cli/extended_value.rb +91 -67
  23. data/lib/aspera/cli/formatter.rb +62 -27
  24. data/lib/aspera/cli/hints.rb +8 -0
  25. data/lib/aspera/cli/info.rb +4 -4
  26. data/lib/aspera/cli/main.rb +76 -84
  27. data/lib/aspera/cli/manager.rb +352 -248
  28. data/lib/aspera/cli/plugins/alee.rb +5 -4
  29. data/lib/aspera/cli/plugins/aoc.rb +175 -195
  30. data/lib/aspera/cli/plugins/ats.rb +4 -4
  31. data/lib/aspera/cli/plugins/base.rb +343 -0
  32. data/lib/aspera/cli/plugins/basic_auth.rb +45 -0
  33. data/lib/aspera/cli/plugins/config.rb +283 -269
  34. data/lib/aspera/cli/plugins/console.rb +27 -22
  35. data/lib/aspera/cli/plugins/cos.rb +3 -3
  36. data/lib/aspera/cli/plugins/factory.rb +78 -0
  37. data/lib/aspera/cli/plugins/faspex.rb +49 -46
  38. data/lib/aspera/cli/plugins/faspex5.rb +113 -225
  39. data/lib/aspera/cli/plugins/faspio.rb +19 -18
  40. data/lib/aspera/cli/plugins/httpgw.rb +14 -13
  41. data/lib/aspera/cli/plugins/node.rb +162 -149
  42. data/lib/aspera/cli/plugins/oauth.rb +48 -0
  43. data/lib/aspera/cli/plugins/orchestrator.rb +129 -45
  44. data/lib/aspera/cli/plugins/preview.rb +30 -50
  45. data/lib/aspera/cli/plugins/server.rb +21 -21
  46. data/lib/aspera/cli/plugins/shares.rb +45 -47
  47. data/lib/aspera/cli/sync_actions.rb +50 -39
  48. data/lib/aspera/cli/transfer_agent.rb +35 -49
  49. data/lib/aspera/cli/transfer_progress.rb +6 -6
  50. data/lib/aspera/cli/version.rb +3 -3
  51. data/lib/aspera/cli/wizard.rb +70 -55
  52. data/lib/aspera/colors.rb +6 -0
  53. data/lib/aspera/command_line_builder.rb +59 -61
  54. data/lib/aspera/command_line_converter.rb +2 -1
  55. data/lib/aspera/coverage.rb +2 -2
  56. data/lib/aspera/data_repository.rb +1 -1
  57. data/lib/aspera/environment.rb +51 -41
  58. data/lib/aspera/faspex_gw.rb +7 -5
  59. data/lib/aspera/faspex_postproc.rb +1 -1
  60. data/lib/aspera/keychain/factory.rb +1 -2
  61. data/lib/aspera/keychain/macos_security.rb +1 -1
  62. data/lib/aspera/log.rb +37 -9
  63. data/lib/aspera/markdown.rb +31 -0
  64. data/lib/aspera/nagios.rb +7 -6
  65. data/lib/aspera/oauth/base.rb +25 -28
  66. data/lib/aspera/oauth/factory.rb +9 -9
  67. data/lib/aspera/oauth/url_json.rb +2 -1
  68. data/lib/aspera/oauth/web.rb +2 -2
  69. data/lib/aspera/preview/file_types.rb +23 -37
  70. data/lib/aspera/products/connect.rb +7 -6
  71. data/lib/aspera/products/desktop.rb +1 -4
  72. data/lib/aspera/products/other.rb +9 -1
  73. data/lib/aspera/products/transferd.rb +0 -1
  74. data/lib/aspera/rest.rb +168 -113
  75. data/lib/aspera/rest_error_analyzer.rb +4 -4
  76. data/lib/aspera/ssh.rb +7 -4
  77. data/lib/aspera/ssl.rb +41 -0
  78. data/lib/aspera/sync/args.schema.yaml +46 -3
  79. data/lib/aspera/sync/conf.schema.yaml +307 -123
  80. data/lib/aspera/sync/database.rb +2 -1
  81. data/lib/aspera/sync/operations.rb +135 -79
  82. data/lib/aspera/temp_file_manager.rb +17 -5
  83. data/lib/aspera/transfer/error.rb +16 -7
  84. data/lib/aspera/transfer/parameters.rb +35 -22
  85. data/lib/aspera/transfer/resumer.rb +74 -0
  86. data/lib/aspera/transfer/spec.rb +5 -5
  87. data/lib/aspera/transfer/spec.schema.yaml +170 -59
  88. data/lib/aspera/transfer/spec_doc.rb +49 -43
  89. data/lib/aspera/uri_reader.rb +2 -2
  90. data/lib/aspera/web_auth.rb +6 -6
  91. data/lib/transferd_pb.rb +2 -2
  92. data.tar.gz.sig +0 -0
  93. metadata +26 -11
  94. metadata.gz.sig +0 -0
  95. data/lib/aspera/cli/basic_auth_plugin.rb +0 -43
  96. data/lib/aspera/cli/plugin.rb +0 -333
  97. data/lib/aspera/cli/plugin_factory.rb +0 -81
  98. data/lib/aspera/resumer.rb +0 -77
  99. data/lib/aspera/transfer/error_info.rb +0 -91
@@ -1,9 +1,12 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require 'aspera/oauth/jwt'
4
+ require 'aspera/assert'
5
+ require 'aspera/cli/plugins/factory'
4
6
 
5
7
  module Aspera
6
8
  module Cli
9
+ # The wizard detects applications and generates a config
7
10
  class Wizard
8
11
  WIZARD_RESULT_KEYS = %i[preset_value test_args].freeze
9
12
  DEFAULT_PRIV_KEY_FILENAME = 'my_private_key.pem' # pragma: allowlist secret
@@ -13,13 +16,17 @@ module Aspera
13
16
  def initialize(parent, main_folder)
14
17
  @parent = parent
15
18
  @main_folder = main_folder
16
- # wizard options
17
- options.declare(:override, 'Wizard: override existing value', values: :bool, default: :no)
18
- options.declare(:default, 'Wizard: set as default configuration for specified plugin (also: update)', values: :bool, default: true)
19
- options.declare(:test_mode, 'Wizard: skip private key check step', values: :bool, default: false)
19
+ # Wizard options
20
+ options.declare(:override, 'Wizard: override existing value', allowed: Allowed::TYPES_BOOLEAN, default: false)
21
+ options.declare(:default, 'Wizard: set as default configuration for specified plugin (also: update)', allowed: Allowed::TYPES_BOOLEAN, default: true)
20
22
  options.declare(:key_path, 'Wizard: path to private key for JWT')
21
23
  end
22
24
 
25
+ # @return false if in test mode to avoid interactive input
26
+ def required
27
+ !ENV['ASCLI_WIZ_TEST']
28
+ end
29
+
23
30
  def options
24
31
  @parent.options
25
32
  end
@@ -32,6 +39,10 @@ module Aspera
32
39
  @parent.config
33
40
  end
34
41
 
42
+ def check_email(email)
43
+ Aspera.assert(email =~ /\A[\w+\-.]+@[a-z\d\-.]+\.[a-z]+\z/i, type: ParameterError){"Username shall be an email: #{email}"}
44
+ end
45
+
35
46
  # Find a plugin, and issue the "require"
36
47
  # @return [Hash] plugin info: { product:, name:, url:, version: }
37
48
  def identify_plugins_for_url
@@ -40,19 +51,19 @@ module Aspera
40
51
  check_only = check_only.to_sym unless check_only.nil?
41
52
  found_apps = []
42
53
  my_self_plugin_sym = self.class.name.split('::').last.downcase.to_sym
43
- PluginFactory.instance.plugin_list.each do |plugin_name_sym|
44
- # no detection for internal plugin
54
+ Plugins::Factory.instance.plugin_list.each do |plugin_name_sym|
55
+ # No detection for internal plugin
45
56
  next if plugin_name_sym.eql?(my_self_plugin_sym)
46
57
  next if check_only && !check_only.eql?(plugin_name_sym)
47
- # load plugin class
48
- detect_plugin_class = PluginFactory.instance.plugin_class(plugin_name_sym)
49
- # requires detection method
50
- next unless detect_plugin_class.respond_to?(:detect)
58
+ # Load plugin class
59
+ plugin_klass = Plugins::Factory.instance.plugin_class(plugin_name_sym)
60
+ # Requires detection method
61
+ next unless plugin_klass.respond_to?(:detect)
51
62
  detection_info = nil
52
63
  begin
53
64
  Log.log.debug{"detecting #{plugin_name_sym} at #{app_url}"}
54
65
  formatter.long_operation_running("#{plugin_name_sym}\r")
55
- detection_info = detect_plugin_class.detect(app_url)
66
+ detection_info = plugin_klass.detect(app_url)
56
67
  rescue OpenSSL::SSL::SSLError => e
57
68
  Log.log.warn(e.message)
58
69
  Log.log.warn('Use option --insecure=yes to allow unchecked certificate') if e.message.include?('cert')
@@ -63,8 +74,8 @@ module Aspera
63
74
  next if detection_info.nil?
64
75
  Aspera.assert_type(detection_info, Hash)
65
76
  Aspera.assert_type(detection_info[:url], String) if detection_info.key?(:url)
66
- app_name = detect_plugin_class.respond_to?(:application_name) ? detect_plugin_class.application_name : detect_plugin_class.name.split('::').last
67
- # if there is a redirect, then the detector can override the url.
77
+ app_name = plugin_klass.respond_to?(:application_name) ? plugin_klass.application_name : plugin_klass.name.split('::').last
78
+ # If there is a redirect, then the detector can override the url.
68
79
  found_apps.push({product: plugin_name_sym, name: app_name, url: app_url, version: 'unknown'}.merge(detection_info))
69
80
  end
70
81
  raise "No known application found at #{app_url}" if found_apps.empty?
@@ -72,6 +83,42 @@ module Aspera
72
83
  return found_apps
73
84
  end
74
85
 
86
+ # To be called in public wizard method to get private key
87
+ # @param user [String] User's email
88
+ # @param url [String] Instance URL
89
+ # @param page [String] URL of page to enter pub key
90
+ # @return [String] Private key path (can contain ~ for home)
91
+ def ask_private_key(user:, url:, page:)
92
+ # Lets see if path to priv key is provided
93
+ private_key_path = options.get_option(:key_path)
94
+ # Give a chance to provide
95
+ if private_key_path.nil?
96
+ formatter.display_status('Path to private RSA key (leave empty to generate):')
97
+ private_key_path = options.get_option(:key_path, mandatory: true).to_s
98
+ end
99
+ # Else generate path
100
+ private_key_path = File.join(@main_folder, DEFAULT_PRIV_KEY_FILENAME) if private_key_path.empty?
101
+ if File.exist?(File.expand_path(private_key_path))
102
+ formatter.display_status('Using existing key:')
103
+ else
104
+ formatter.display_status("Generating #{OAuth::Jwt::DEFAULT_PRIV_KEY_LENGTH} bit RSA key...")
105
+ OAuth::Jwt.generate_rsa_private_key(path: private_key_path)
106
+ formatter.display_status('Created key:')
107
+ end
108
+ formatter.display_status(private_key_path)
109
+ private_key_pem = File.read(File.expand_path(private_key_path))
110
+ pub_key_pem = OpenSSL::PKey::RSA.new(private_key_pem).public_key.to_s
111
+ options.set_option(:private_key, private_key_pem)
112
+ formatter.display_status("Please Log in as user #{user.red} at: #{url.red}")
113
+ formatter.display_status("Navigate to: #{page}")
114
+ formatter.display_status("Check or update the value to (#{'including BEGIN/END lines'.red}):".blink)
115
+ formatter.display_status(pub_key_pem, hide_secrets: false)
116
+ formatter.display_status('Once updated or validated, press [Enter].')
117
+ Environment.instance.open_uri(url)
118
+ $stdin.gets if required
119
+ private_key_path
120
+ end
121
+
75
122
  # Wizard function, creates configuration
76
123
  # @param apps [Array] list of detected apps
77
124
  def find(apps)
@@ -88,49 +135,18 @@ module Aspera
88
135
  Log.dump(:identification, identification)
89
136
  wiz_url = identification[:url]
90
137
  formatter.display_status("Using: #{identification[:name]} at #{wiz_url}".bold)
91
- # set url for instantiation of plugin
138
+ # Set url for instantiation of plugin
92
139
  options.add_option_preset({url: wiz_url}, 'wizard')
93
- # instantiate plugin: command line options will be known and wizard can be called
94
- wiz_plugin_class = PluginFactory.instance.plugin_class(identification[:product])
95
- Aspera.assert(wiz_plugin_class.respond_to?(:wizard), type: Cli::BadArgument) do
140
+ # Instantiate plugin: command line options will be known, e.g. private_key, and wizard can be called
141
+ plugin_instance = Plugins::Factory.instance.plugin_class(identification[:product]).new(context: @parent.context)
142
+ Aspera.assert(plugin_instance.respond_to?(:wizard), type: Cli::BadArgument) do
96
143
  "Detected: #{identification[:product]}, but this application has no wizard"
97
144
  end
98
- # instantiate plugin: command line options will be known, e.g. private_key
99
- plugin_instance = wiz_plugin_class.new(context: @parent.context)
100
- wiz_params = {
101
- object: plugin_instance
102
- }
103
- # is private key needed ?
104
- if options.known_options.key?(:private_key) &&
105
- (!wiz_plugin_class.respond_to?(:private_key_required?) || wiz_plugin_class.private_key_required?(wiz_url))
106
- # lets see if path to priv key is provided
107
- private_key_path = options.get_option(:key_path)
108
- # give a chance to provide
109
- if private_key_path.nil?
110
- formatter.display_status('Path to private RSA key (leave empty to generate):')
111
- private_key_path = options.get_option(:key_path, mandatory: true).to_s
112
- end
113
- # else generate path
114
- private_key_path = File.join(@main_folder, DEFAULT_PRIV_KEY_FILENAME) if private_key_path.empty?
115
- if File.exist?(private_key_path)
116
- formatter.display_status('Using existing key:')
117
- else
118
- formatter.display_status("Generating #{OAuth::Jwt::DEFAULT_PRIV_KEY_LENGTH} bit RSA key...")
119
- OAuth::Jwt.generate_rsa_private_key(path: private_key_path)
120
- formatter.display_status('Created key:')
121
- end
122
- formatter.display_status(private_key_path)
123
- private_key_pem = File.read(private_key_path)
124
- options.set_option(:private_key, private_key_pem)
125
- wiz_params[:private_key_path] = private_key_path
126
- wiz_params[:pub_key_pem] = OpenSSL::PKey::RSA.new(private_key_pem).public_key.to_s
127
- end
128
- Log.dump(:wiz_params, wiz_params)
129
- # finally, call the wizard
130
- wizard_result = wiz_plugin_class.wizard(**wiz_params)
145
+ # Call the wizard
146
+ wizard_result = plugin_instance.wizard(self, wiz_url)
131
147
  Log.log.debug{"wizard result: #{wizard_result}"}
132
148
  Aspera.assert(WIZARD_RESULT_KEYS.eql?(wizard_result.keys.sort)){"missing or extra keys in wizard result: #{wizard_result.keys}"}
133
- # get preset name from user or default
149
+ # Get preset name from user or default
134
150
  if wiz_preset_name.empty?
135
151
  elements = [
136
152
  identification[:product],
@@ -139,18 +155,17 @@ module Aspera
139
155
  elements.push(options.get_option(:username, mandatory: true)) unless wizard_result[:preset_value].key?(:link) rescue nil
140
156
  wiz_preset_name = elements.join('_').strip.downcase.gsub(/[^a-z0-9]/, '_').squeeze('_')
141
157
  end
142
- # test mode does not change conf file
143
- return Main.result_single_object(wizard_result) if options.get_option(:test_mode)
144
158
  # Write configuration file
145
159
  formatter.display_status("Preparing preset: #{wiz_preset_name}")
146
- # init defaults if necessary
160
+ # Init defaults if necessary
147
161
  option_override = options.get_option(:override, mandatory: true)
148
162
  option_default = options.get_option(:default, mandatory: true)
149
163
  config.defaults_set(identification[:product], wiz_preset_name, wizard_result[:preset_value].stringify_keys, option_default, option_override)
150
164
  test_args = wizard_result[:test_args]
151
165
  test_args = "-P#{wiz_preset_name} #{test_args}" unless option_default
152
166
  # TODO: actually test the command
153
- return Main.result_status("You can test with:\n#{Info::CMD_NAME} #{identification[:product]} #{test_args}")
167
+ test_cmd = "#{Info::CMD_NAME} #{identification[:product]} #{test_args}"
168
+ return Main.result_status("You can test with:\n#{test_cmd.red}")
154
169
  end
155
170
  end
156
171
  end
data/lib/aspera/colors.rb CHANGED
@@ -57,10 +57,16 @@ class String
57
57
  define_method(name){self}
58
58
  end
59
59
  end
60
+
60
61
  # Transform capitalized to snake case
61
62
  def capital_to_snake
62
63
  return gsub(/([a-z\d])([A-Z])/, '\1_\2')
63
64
  .gsub(/([A-Z]+)([A-Z][a-z])/, '\1_\2')
64
65
  .downcase
65
66
  end
67
+
68
+ # Transform snake case to capitalized
69
+ def snake_to_capital
70
+ split('_').map(&:capitalize).join
71
+ end
66
72
  end
@@ -4,72 +4,69 @@ require 'aspera/log'
4
4
  require 'aspera/assert'
5
5
  require 'yaml'
6
6
  module Aspera
7
- # helper class to build command line from a parameter list (key-value hash)
8
- # constructor takes hash: { 'param1':'value1', ...}
9
- # process_param is called repeatedly with all known parameters
10
- # add_env_args is called to get resulting param list and env var (also checks that all params were used)
7
+ # Helper class to build command line from a parameter list (key-value hash)
8
+ # Constructor takes hash: `{ 'param1':'value1', ...}`
9
+ # `process_param` is called repeatedly with all known parameters
10
+ # `add_env_args` is called to get resulting param list and env var (also checks that all params were used)
11
11
  class CommandLineBuilder
12
- # description [String] Description
13
- # type [String,Array] Accepted type(s) for non-enum
14
- # default [String] Default value if not specified
15
- # enum [Array] Set with list of values for enum types accepted in transfer spec
16
- # items [Array]
17
- # properties [Array]
18
- # x-cli-envvar [String] Name of env var
19
- # x-cli-option [String] Command line option (starts with "-")
20
- # x-cli-switch [Bool] true if option has no arg, else by default option has a value
21
- # x-cli-special [Bool] true if special handling (defered)
22
- # x-cli-convert [String,Hash] Method name for Convert object or Conversion for enum ts to arg
23
- # x-agents [Array] Supported agents (for doc only), if not specified: all
24
- # x-ts-name [Bool,String] (async) true if same name in transfer spec, else real name in transfer spec, else ignored
25
- # x-ts-convert [String] (async) Method name for Convert object
26
- # x-deprecation [String] Deprecation message for doc
27
- PROPERTY_KEYS = %w[
28
- description
29
- type
30
- default
31
- enum
32
- items
33
- properties
34
- required
35
- $comment
36
- x-cli-envvar
37
- x-cli-option
38
- x-cli-switch
39
- x-cli-special
40
- x-cli-convert
41
- x-agents
42
- x-ts-name
43
- x-deprecation
12
+ # Supported keys in JSON schema
13
+ PROPERTY_KEYS = [
14
+ 'description', # [String] Description
15
+ 'type', # [String,Array] Accepted type(s) for non-enum
16
+ 'default', # [String] Default value if not specified
17
+ 'enum', # [Array] Set with list of values for enum types accepted in transfer spec
18
+ 'items', # [Array]
19
+ 'properties', # [Array]
20
+ 'required', # [Array]
21
+ '$comment', # [String]
22
+ 'x-cli-envvar', # [String] Name of env var
23
+ 'x-cli-option', # [String] Command line option (starts with "-")
24
+ 'x-cli-short', # [String] Command line option (starts with "-")
25
+ 'x-cli-switch', # [Bool] `true` if option has no arg, else by default option has a value
26
+ 'x-cli-special', # [Bool] `true` if special handling (deferred)
27
+ 'x-cli-convert', # [String,Hash] Method name for Convert object or Conversion for enum ts to arg
28
+ 'x-agents', # [Array] Supported agents (for doc only), if not specified: all
29
+ 'x-ts-name', # [Bool,String] (async) true if same name in transfer spec, else real name in transfer spec, else ignored
30
+ 'x-ts-convert', # [String] (async) Name of methods to convert value from transfer spec to `conf` API.
31
+ 'x-deprecation' # [String] Deprecation message for doc
44
32
  ].freeze
45
33
 
46
- CLI_AGENT = 'direct'
47
-
48
- private_constant :PROPERTY_KEYS, :CLI_AGENT
34
+ private_constant :PROPERTY_KEYS
49
35
 
50
36
  class << self
51
- # @return true if given agent supports that field
37
+ # Called by provider of definition before constructor of this class so that schema has all mandatory fields
38
+ def read_schema(folder, name, ascp: false)
39
+ schema = YAML.load_file(File.join(folder, "#{name}.schema.yaml"))
40
+ validate_schema(schema, ascp: ascp)
41
+ end
42
+
43
+ # @param agent [Symbol] Transfer agent name
44
+ # @param properties [Hash] Transfer spec parameter information
45
+ # @return [Boolean] `true` if given agent supports that field
52
46
  def supported_by_agent(agent, properties)
53
- !properties.key?('x-agents') || properties['x-agents'].include?(agent)
47
+ !properties.key?('x-agents') || properties['x-agents'].include?(agent.to_s)
54
48
  end
55
49
 
56
- # fill default values
57
- def adjust_properties_defaults(properties)
58
- properties.each do |name, info|
50
+ private
51
+
52
+ # Fill default values for some fields in the schema
53
+ # @param schema [Hash] The JSON schema
54
+ # @param ascp [Bool] `true` if ascp
55
+ def validate_schema(schema, ascp: false)
56
+ direct_props = %w[x-cli-option x-cli-envvar x-cli-special].freeze
57
+ schema['properties'].each do |name, info|
59
58
  Aspera.assert_type(info, Hash){"#{info.class} for #{name}"}
60
59
  unsupported_keys = info.keys - PROPERTY_KEYS
61
60
  Aspera.assert(unsupported_keys.empty?){"Unsupported definition keys: #{unsupported_keys}"}
62
- # by default : string, unless it's without arg
63
- info['type'] ||= info['x-cli-switch'] ? 'boolean' : 'string'
64
- # add default cli option name if not present, and if supported in "direct".
65
- info['x-cli-option'] = "--#{name.to_s.tr('_', '-')}" if !info.key?('x-cli-option') && !info['x-cli-envvar'] && (info.key?('x-cli-switch') || supported_by_agent(CLI_AGENT, info))
61
+ Aspera.assert(info.key?('type') || info.key?('enum')){"Missing type for #{name} in #{schema['description']}"}
62
+ Aspera.assert(info['type'].eql?('boolean')){"switch must be bool: #{name}"} if info['x-cli-switch'] && !info['x-cli-special']
63
+ info['x-cli-option'] = "--#{name.to_s.tr('_', '-')}" if info['x-cli-option'].eql?(true) || (info['x-cli-switch'].eql?(true) && !info.key?('x-cli-option'))
64
+ Aspera.assert(direct_props.any?{ |i| info.key?(i)}, type: :warn){name} if ascp && supported_by_agent(:direct, info)
66
65
  info.freeze
66
+ validate_schema(info, ascp: ascp) if info['type'].eql?('object') && info['properties']
67
+ validate_schema(info['items'], ascp: ascp) if info['type'].eql?('array') && info['items'] && info['items']['properties']
67
68
  end
68
- end
69
-
70
- # Called by provider of definition before constructor of this class so that schema has all mandatory fields
71
- def read_schema(source_path, name)
72
- YAML.load_file(File.join(File.dirname(source_path), "#{name}.schema.yaml"))
69
+ schema
73
70
  end
74
71
  end
75
72
 
@@ -98,10 +95,10 @@ module Aspera
98
95
  # Add processed parameters to env and args, warns about unused parameters
99
96
  # @param [Hash] env_args with :env and :args
100
97
  def add_env_args(env_args)
101
- Log.log.debug{"add_env_args: ENV=#{@result[:env]}, ARGS=#{@result[:args]}"}
98
+ Log.dump(:env_args, @result)
102
99
  # warn about non translated arguments
103
100
  @object.each_pair do |name, value|
104
- Log.log.warn{raise "Unknown transfer spec parameter: #{name} = \"#{value}\""} unless @processed_parameters.include?(name)
101
+ Log.log.warn{"Unknown transfer spec parameter: #{name} = \"#{value}\""} unless @processed_parameters.include?(name)
105
102
  end
106
103
  # set result
107
104
  env_args[:env].merge!(@result[:env])
@@ -109,9 +106,10 @@ module Aspera
109
106
  return
110
107
  end
111
108
 
112
- # add options directly to command line
113
- def add_command_line_options(options)
114
- return if options.nil?
109
+ # Add options directly to command line
110
+ def add_command_line_options(*options)
111
+ options = options.first if options.first.is_a?(Array) && options.length.eql?(1)
112
+ Aspera.assert_type(options, Array)
115
113
  options.each{ |o| @result[:args].push(o.to_s)}
116
114
  end
117
115
 
@@ -178,7 +176,7 @@ module Aspera
178
176
  parameter_value = converted_value
179
177
  end
180
178
 
181
- return unless self.class.supported_by_agent(CLI_AGENT, properties)
179
+ return unless self.class.supported_by_agent(:direct, properties)
182
180
 
183
181
  if read
184
182
  # just get value (deferred)
@@ -198,13 +196,13 @@ module Aspera
198
196
  else Aspera.error_unexpected_value(parameter_value){name}
199
197
  end
200
198
  # add_param = !add_param if properties[:add_on_false]
201
- add_command_line_options([properties['x-cli-option']]) if add_param
199
+ add_command_line_options(properties['x-cli-option']) if add_param
202
200
  else
203
201
  # transform into command line option with value
204
202
  # parameter_value=parameter_value.to_s if parameter_value.is_a?(Integer)
205
203
  parameter_value = [parameter_value] unless parameter_value.is_a?(Array)
206
204
  # if transfer_spec value is an array, applies option many times
207
- parameter_value.each{ |v| add_command_line_options([properties['x-cli-option'], v])}
205
+ parameter_value.each{ |v| add_command_line_options(properties['x-cli-option'], v)}
208
206
  end
209
207
  end
210
208
  end
@@ -23,8 +23,9 @@ module Aspera
23
23
  end
24
24
  end
25
25
 
26
+ # Kbps to bps
26
27
  def kbps_to_bps(value)
27
- 1000 * value
28
+ 1000 * value.to_i
28
29
  end
29
30
  end
30
31
  end
@@ -5,10 +5,10 @@ if ENV.key?('ENABLE_COVERAGE')
5
5
  require 'simplecov'
6
6
  require 'securerandom'
7
7
  # compute development top folder based on this source location
8
- development_root = 3.times.inject(File.realpath(__FILE__)){ |p, _| File.dirname(p)}
8
+ development_root = File.dirname(File.realpath(__FILE__), 3)
9
9
  SimpleCov.root(development_root)
10
10
  SimpleCov.enable_for_subprocesses if SimpleCov.respond_to?(:enable_for_subprocesses)
11
- # keep cache data for 1 day (must be longer that time to run the whole test suite)
11
+ # keep cache data for 1 day (must be longer than time to run the whole test suite)
12
12
  SimpleCov.merge_timeout(86400)
13
13
  SimpleCov.command_name(SecureRandom.uuid)
14
14
  SimpleCov.at_exit do
@@ -19,7 +19,7 @@ module Aspera
19
19
  # @return [String] decoded data
20
20
  def item(name)
21
21
  index = ELEMENTS.index(name)
22
- raise ArgumentError, "unknown data item #{name} (#{name.class})" unless index
22
+ raise ParameterError, "Unknown data item #{name} (#{name.class})" unless index
23
23
  raw_data = data(START_INDEX + index)
24
24
  case name
25
25
  when :dsa, :rsa
@@ -5,7 +5,9 @@ require 'aspera/log'
5
5
  require 'aspera/assert'
6
6
  require 'rbconfig'
7
7
  require 'singleton'
8
+ require 'open3'
8
9
  require 'English'
10
+ require 'shellwords'
9
11
 
10
12
  # cspell:words MEBI mswin bccwin
11
13
 
@@ -38,6 +40,8 @@ module Aspera
38
40
  WINDOWS_FILENAME_INVALID_CHARACTERS = '<>:"/\\|?*'
39
41
  REPLACE_CHARACTER = '_'
40
42
 
43
+ RB_EXT = '.rb'
44
+
41
45
  class << self
42
46
  def ruby_version
43
47
  return RbConfig::CONFIG['RUBY_PROGRAM_VERSION']
@@ -54,9 +58,9 @@ module Aspera
54
58
  end
55
59
 
56
60
  # Generate log line for external program with arguments
57
- # @param env [Hash, nil] environment variables
58
- # @param exec [String] path to executable
59
- # @param args [Array, nil] arguments
61
+ # @param exec [String] Path to executable
62
+ # @param args [Array, nil] Arguments
63
+ # @param env [Hash, nil] Environment variables
60
64
  # @return [String] log line with environment, program and arguments
61
65
  def log_spawn(exec:, args: nil, env: nil)
62
66
  [
@@ -69,65 +73,70 @@ module Aspera
69
73
 
70
74
  # Start process in background
71
75
  # caller can call Process.wait on returned value
72
- # @param exec [String] path to executable
73
- # @param args [Array, nil] arguments for executable
74
- # @param env [Hash, nil] environment variables
75
- # @param options [Hash, nil] spawn options
76
+ # @param exec [String] Path to executable
77
+ # @param args [Array, nil] Arguments for executable
78
+ # @param env [Hash, nil] Environment variables
79
+ # @param kwargs [Hash] Options for `Process.spawn`
76
80
  # @return [String] PID of process
77
81
  # @raise [Exception] if problem
78
- def secure_spawn(exec:, args: nil, env: nil, **options)
82
+ def secure_spawn(exec:, args: nil, env: nil, **kwargs)
79
83
  Aspera.assert_type(exec, String)
80
84
  Aspera.assert_type(args, Array, NilClass)
81
85
  Aspera.assert_type(env, Hash, NilClass)
82
- Aspera.assert_type(options, Hash, NilClass)
83
86
  Log.log.debug{log_spawn(exec: exec, args: args, env: env)}
84
- # start ascp in separate process
85
87
  spawn_args = []
86
88
  spawn_args.push(env) unless env.nil?
87
89
  spawn_args.push([exec, exec])
88
90
  spawn_args.concat(args) unless args.nil?
89
- opts = {close_others: true}
90
- opts.merge!(options) unless options.nil?
91
- ascp_pid = Process.spawn(*spawn_args, **opts)
92
- Log.log.debug{"pid: #{ascp_pid}"}
93
- return ascp_pid
91
+ kwargs[:close_others] = true unless kwargs.key?(:close_others)
92
+ # Start separate process in background
93
+ pid = Process.spawn(*spawn_args, **kwargs)
94
+ Log.dump(:pid, pid)
95
+ return pid
94
96
  end
95
97
 
96
- # start process and wait for completion
97
- # @param env [Hash, nil] environment variables
98
- # @param exec [String] path to executable
99
- # @param args [Array, nil] arguments
100
- # @return [String] PID of process
101
- def secure_execute(exec:, args: nil, env: nil, **system_args)
98
+ # Start process (not in shell) and wait for completion.
99
+ # By default, sets `exception: true` in `kwargs`
100
+ # @param exec [String] Path to executable
101
+ # @param args [Array, nil] Arguments
102
+ # @param env [Hash, nil] Environment variables
103
+ # @param kwargs [Hash] Arguments for `Kernel.system`
104
+ # @return `nil`
105
+ # @raise [RuntimeError] if problem
106
+ def secure_execute(exec:, args: nil, env: nil, **kwargs)
102
107
  Aspera.assert_type(exec, String)
103
108
  Aspera.assert_type(args, Array, NilClass)
104
109
  Aspera.assert_type(env, Hash, NilClass)
105
110
  Log.log.debug{log_spawn(exec: exec, args: args, env: env)}
106
- # start in separate process
111
+ Log.dump(:kwargs, kwargs, level: :trace1)
107
112
  spawn_args = []
108
113
  spawn_args.push(env) unless env.nil?
109
- # ensure no shell expansion
114
+ # Ensure no shell expansion
110
115
  spawn_args.push([exec, exec])
111
116
  spawn_args.concat(args) unless args.nil?
112
- kwargs = {exception: true}
113
- kwargs.merge!(system_args)
117
+ # By default: exception on error
118
+ kwargs[:exception] = true unless kwargs.key?(:exception)
119
+ # Start in separate process
114
120
  Kernel.system(*spawn_args, **kwargs)
115
121
  nil
116
122
  end
117
123
 
118
124
  # Execute process and capture stdout
119
- # @param exec [String] path to executable
120
- # @param args [Array] arguments to executable
121
- # @param opts [Hash] options to capture3
125
+ # @param exec [String] path to executable
126
+ # @param args [Array] arguments to executable
127
+ # @param kwargs [Hash] options to Open3.capture3
122
128
  # @return stdout of executable or raise exception
123
- def secure_capture(exec:, args: [], exception: true, **opts)
129
+ def secure_capture(exec:, args: [], env: nil, exception: true, **kwargs)
124
130
  Aspera.assert_type(exec, String)
125
131
  Aspera.assert_type(args, Array)
126
- Aspera.assert_type(opts, Hash)
127
- Log.log.debug{log_spawn(exec: exec, args: args)}
128
- Log.dump(:opts, opts, level: :trace2)
129
- Log.dump(:ENV, ENV.to_h, level: :trace1)
130
- stdout, stderr, status = Open3.capture3(exec, *args, **opts)
132
+ Log.log.debug{log_spawn(exec: exec, args: args, env: env)}
133
+ Log.dump(:kwargs, kwargs, level: :trace2)
134
+ # Log.dump(:ENV, ENV.to_h, level: :trace1)
135
+ capture_args = []
136
+ capture_args.push(env) unless env.nil?
137
+ capture_args.push(exec)
138
+ capture_args.concat(args)
139
+ stdout, stderr, status = Open3.capture3(*capture_args, **kwargs)
131
140
  Log.log.debug{"status=#{status}, stderr=#{stderr}"}
132
141
  Log.log.trace1{"stdout=#{stdout}"}
133
142
  raise "process failed: #{status.exitstatus} (#{stderr})" if !status.success? && exception
@@ -186,7 +195,7 @@ module Aspera
186
195
  end
187
196
  end
188
197
  attr_accessor :url_method, :file_illegal_characters
189
- attr_reader :os, :cpu, :executable_extension, :default_gui_mode
198
+ attr_reader :os, :cpu, :default_gui_mode
190
199
 
191
200
  def initialize
192
201
  initialize_fields
@@ -218,7 +227,7 @@ module Aspera
218
227
  CPU_ARM64
219
228
  else Aspera.error_unexpected_value(RbConfig::CONFIG['host_cpu']){'host_cpu'}
220
229
  end
221
- @executable_extension = @os.eql?(OS_WINDOWS) ? 'exe' : nil
230
+ @executable_extension = @os.eql?(OS_WINDOWS) ? '.exe' : nil
222
231
  # :text or :graphical depending on the environment
223
232
  @default_gui_mode =
224
233
  if [Environment::OS_WINDOWS, Environment::OS_MACOS].include?(os) ||
@@ -239,8 +248,10 @@ module Aspera
239
248
  "#{@os}-#{@cpu}"
240
249
  end
241
250
 
242
- # executable file extension for current OS
243
- def exe_file(name)
251
+ # Add executable file extension (e.g. ".exe") for current OS
252
+ # @param name [String,nil] Path or file name
253
+ # @return [String] Executable name with extension
254
+ def exe_file(name = nil)
244
255
  return name unless @executable_extension
245
256
  return "#{name}#{@executable_extension}"
246
257
  end
@@ -314,9 +325,8 @@ module Aspera
314
325
  # Windows does not allow file name:
315
326
  # - with control characters anywhere
316
327
  # - ending with space or dot
317
- filename = filename
318
- .gsub(/[\x00-\x1F\x7F]/, safe_char)
319
- .sub(/[. ]+\z/, safe_char)
328
+ filename = filename.gsub(/[\x00-\x1F\x7F]/, safe_char)
329
+ filename = filename.chop while filename.end_with?(' ', '.')
320
330
  if @file_illegal_characters&.size.to_i >= 2
321
331
  # replace all illegal characters with safe_char
322
332
  filename = filename.tr(@file_illegal_characters[1..-1], safe_char)
@@ -8,10 +8,12 @@ require 'json'
8
8
  module Aspera
9
9
  # Simulate the Faspex 4 /send API and creates a package on Aspera on Cloud or Faspex 5
10
10
  class Faspex4GWServlet < WEBrick::HTTPServlet::AbstractServlet
11
+ AOC_API = 'Aspera::Api::AoC'
12
+ FX_API = 'Aspera::Api::Faspex'
11
13
  # @param app_api [Rest] API object
12
14
  # @param app_context [String] workspace id (aoc only)
13
15
  def initialize(server, app_api, app_context)
14
- Aspera.assert_values(app_api.class.name, ['Aspera::Api::AoC', 'Aspera::Rest'])
16
+ Aspera.assert_values(app_api.class.name, [AOC_API, FX_API])
15
17
  super(server)
16
18
  @app_api = app_api
17
19
  @app_context = app_context
@@ -49,11 +51,11 @@ module Aspera
49
51
  transfer_spec = @app_api.call(
50
52
  operation: 'POST',
51
53
  subpath: "packages/#{package['id']}/transfer_spec/upload",
52
- query: {transfer_type: Cli::Plugins::Faspex5::TRANSFER_CONNECT},
54
+ query: {transfer_type: Api::Faspex::TRANSFER_CONNECT},
53
55
  content_type: Rest::MIME_JSON,
54
56
  body: {paths: [{'destination'=>'/'}]},
55
57
  headers: {'Accept' => Rest::MIME_JSON}
56
- )[:data]
58
+ )
57
59
  transfer_spec.delete('authentication')
58
60
  # but we place it in a Faspex package creation response
59
61
  return {
@@ -72,9 +74,9 @@ module Aspera
72
74
  # compare string, as class is not yet known here
73
75
  faspex_package_create_result =
74
76
  case @app_api.class.name
75
- when 'Aspera::Api::AoC'
77
+ when AOC_API
76
78
  faspex4_send_to_aoc(faspex_pkg_parameters)
77
- when 'Aspera::Rest'
79
+ when FX_API
78
80
  faspex4_send_to_faspex5(faspex_pkg_parameters)
79
81
  else Aspera.error_unexpected_value(@app_api.class.name)
80
82
  end