aspera-cli 4.24.1 → 4.24.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (87) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/CHANGELOG.md +15 -2
  4. data/README.md +745 -436
  5. data/bin/ascli +20 -1
  6. data/bin/asession +23 -27
  7. data/lib/aspera/agent/base.rb +10 -21
  8. data/lib/aspera/agent/connect.rb +2 -3
  9. data/lib/aspera/agent/desktop.rb +2 -2
  10. data/lib/aspera/agent/direct.rb +49 -32
  11. data/lib/aspera/agent/factory.rb +31 -0
  12. data/lib/aspera/api/aoc.rb +79 -49
  13. data/lib/aspera/api/faspex.rb +212 -0
  14. data/lib/aspera/api/node.rb +99 -84
  15. data/lib/aspera/ascp/installation.rb +22 -21
  16. data/lib/aspera/ascp/management.rb +119 -23
  17. data/lib/aspera/assert.rb +14 -8
  18. data/lib/aspera/cli/extended_value.rb +15 -15
  19. data/lib/aspera/cli/formatter.rb +7 -5
  20. data/lib/aspera/cli/hints.rb +8 -0
  21. data/lib/aspera/cli/info.rb +4 -4
  22. data/lib/aspera/cli/main.rb +55 -70
  23. data/lib/aspera/cli/manager.rb +7 -4
  24. data/lib/aspera/cli/plugins/alee.rb +2 -1
  25. data/lib/aspera/cli/plugins/aoc.rb +110 -186
  26. data/lib/aspera/cli/plugins/ats.rb +4 -4
  27. data/lib/aspera/cli/plugins/base.rb +335 -0
  28. data/lib/aspera/cli/plugins/basic_auth.rb +45 -0
  29. data/lib/aspera/cli/plugins/config.rb +249 -220
  30. data/lib/aspera/cli/plugins/console.rb +15 -15
  31. data/lib/aspera/cli/plugins/cos.rb +2 -2
  32. data/lib/aspera/cli/plugins/factory.rb +78 -0
  33. data/lib/aspera/cli/plugins/faspex.rb +17 -20
  34. data/lib/aspera/cli/plugins/faspex5.rb +79 -193
  35. data/lib/aspera/cli/plugins/faspio.rb +14 -13
  36. data/lib/aspera/cli/plugins/httpgw.rb +13 -12
  37. data/lib/aspera/cli/plugins/node.rb +34 -32
  38. data/lib/aspera/cli/plugins/oauth.rb +48 -0
  39. data/lib/aspera/cli/plugins/orchestrator.rb +15 -13
  40. data/lib/aspera/cli/plugins/preview.rb +4 -4
  41. data/lib/aspera/cli/plugins/server.rb +15 -13
  42. data/lib/aspera/cli/plugins/shares.rb +18 -15
  43. data/lib/aspera/cli/sync_actions.rb +1 -1
  44. data/lib/aspera/cli/transfer_agent.rb +24 -20
  45. data/lib/aspera/cli/transfer_progress.rb +6 -6
  46. data/lib/aspera/cli/version.rb +3 -3
  47. data/lib/aspera/cli/wizard.rb +65 -53
  48. data/lib/aspera/colors.rb +6 -0
  49. data/lib/aspera/command_line_builder.rb +45 -50
  50. data/lib/aspera/command_line_converter.rb +2 -1
  51. data/lib/aspera/coverage.rb +1 -1
  52. data/lib/aspera/data_repository.rb +1 -1
  53. data/lib/aspera/environment.rb +10 -7
  54. data/lib/aspera/faspex_gw.rb +6 -4
  55. data/lib/aspera/faspex_postproc.rb +1 -1
  56. data/lib/aspera/keychain/macos_security.rb +1 -1
  57. data/lib/aspera/log.rb +37 -9
  58. data/lib/aspera/nagios.rb +1 -1
  59. data/lib/aspera/oauth/base.rb +17 -10
  60. data/lib/aspera/oauth/factory.rb +8 -8
  61. data/lib/aspera/oauth/web.rb +2 -2
  62. data/lib/aspera/products/connect.rb +4 -3
  63. data/lib/aspera/products/desktop.rb +1 -4
  64. data/lib/aspera/products/other.rb +9 -1
  65. data/lib/aspera/products/transferd.rb +0 -1
  66. data/lib/aspera/rest.rb +126 -83
  67. data/lib/aspera/ssh.rb +3 -3
  68. data/lib/aspera/sync/args.schema.yaml +46 -3
  69. data/lib/aspera/sync/conf.schema.yaml +130 -94
  70. data/lib/aspera/sync/operations.rb +16 -16
  71. data/lib/aspera/temp_file_manager.rb +17 -5
  72. data/lib/aspera/transfer/error.rb +16 -7
  73. data/lib/aspera/transfer/parameters.rb +34 -20
  74. data/lib/aspera/transfer/resumer.rb +74 -0
  75. data/lib/aspera/transfer/spec.rb +4 -3
  76. data/lib/aspera/transfer/spec.schema.yaml +132 -51
  77. data/lib/aspera/transfer/spec_doc.rb +41 -35
  78. data/lib/aspera/uri_reader.rb +1 -1
  79. data/lib/aspera/web_auth.rb +6 -6
  80. data.tar.gz.sig +0 -0
  81. metadata +9 -7
  82. metadata.gz.sig +0 -0
  83. data/lib/aspera/cli/basic_auth_plugin.rb +0 -43
  84. data/lib/aspera/cli/plugin.rb +0 -333
  85. data/lib/aspera/cli/plugin_factory.rb +0 -81
  86. data/lib/aspera/resumer.rb +0 -77
  87. data/lib/aspera/transfer/error_info.rb +0 -91
@@ -4,7 +4,7 @@ require 'aspera/cli/manager'
4
4
  require 'aspera/cli/formatter'
5
5
  require 'aspera/cli/plugins/config'
6
6
  require 'aspera/cli/extended_value'
7
- require 'aspera/cli/plugin_factory'
7
+ require 'aspera/cli/plugins/factory'
8
8
  require 'aspera/cli/transfer_agent'
9
9
  require 'aspera/cli/version'
10
10
  require 'aspera/cli/info'
@@ -46,29 +46,13 @@ module Aspera
46
46
  STATUS_FIELD = 'status'
47
47
  COMMAND_CONFIG = :config
48
48
  COMMAND_HELP = :help
49
- # types that go to result of type = text
49
+ # Types that go to result of type = text
50
50
  SCALAR_TYPES = [String, Integer, Symbol].freeze
51
51
  USER_INTERFACES = %i[text graphical].freeze
52
52
 
53
53
  private_constant :COMMAND_CONFIG, :COMMAND_HELP, :SCALAR_TYPES, :USER_INTERFACES
54
54
 
55
55
  class << self
56
- # early debug for parser
57
- # Note: does not accept shortcuts
58
- def early_debug_setup(argv)
59
- Log.instance.program_name = Info::CMD_NAME
60
- argv.each do |arg|
61
- case arg
62
- when '--' then break
63
- when /^--log-level=(.*)/ then Log.instance.level = Regexp.last_match(1).to_sym
64
- when /^--logger=(.*)/ then Log.instance.logger_type = Regexp.last_match(1).to_sym
65
- end
66
- rescue => e
67
- $stderr.puts("Error: #{e}") # rubocop:disable Style/StderrPuts
68
- Process.exit(1)
69
- end
70
- end
71
-
72
56
  def result_special(how); {type: :special, data: how}; end
73
57
 
74
58
  # Expect some list, but nothing to display
@@ -81,13 +65,13 @@ module Aspera
81
65
  # @param status [String] The status
82
66
  def result_status(status); return {type: :status, data: status}; end
83
67
 
84
- # text result coming from command result
68
+ # Text result coming from command result
85
69
  def result_text(data); return {type: :text, data: data}; end
86
70
 
87
71
  def result_success; return result_status('complete'); end
88
72
 
89
73
  # Process statuses of finished transfer sessions
90
- # raise exception if there is one error
74
+ # @raise exception if there is one error
91
75
  # else returns an empty status
92
76
  def result_transfer(statuses)
93
77
  worst = TransferAgent.session_status(statuses)
@@ -98,10 +82,10 @@ module Aspera
98
82
  # Used when one command executes several transfer jobs (each job being possibly multi session)
99
83
  # @param status_table [Array] [{STATUS_FIELD=>[status array],...},...]
100
84
  # @return a status object suitable as command result
101
- # each element has a key STATUS_FIELD which contains the result of possibly multiple sessions
85
+ # Each element has a key STATUS_FIELD which contains the result of possibly multiple sessions
102
86
  def result_transfer_multiple(status_table)
103
87
  global_status = :success
104
- # transform status array into string and find if there was problem
88
+ # Transform status array into string and find if there was problem
105
89
  status_table.each do |item|
106
90
  worst = TransferAgent.session_status(item[STATUS_FIELD])
107
91
  global_status = worst unless worst.eql?(:success)
@@ -153,7 +137,7 @@ module Aspera
153
137
  end
154
138
  end
155
139
 
156
- # minimum initialization, no exception raised
140
+ # Minimum initialization, no exception raised
157
141
  def initialize(argv)
158
142
  @argv = argv
159
143
  Log.dump(:argv, @argv, level: :trace2)
@@ -163,16 +147,16 @@ module Aspera
163
147
  @context = Context.new
164
148
  end
165
149
 
166
- # this is the main function called by initial script just after constructor
150
+ # This is the main function called by initial script just after constructor
167
151
  def process_command_line
168
- # catch exception information , if any
152
+ # Catch exception information , if any
169
153
  exception_info = nil
170
- # false if command shall not be executed (e.g. --show-config)
154
+ # False if command shall not be executed (e.g. --show-config)
171
155
  execute_command = true
172
- # catch exceptions
156
+ # Catch exceptions
173
157
  begin
174
158
  init_agents_options_plugins
175
- # help requested without command ? (plugins must be known here)
159
+ # Help requested without command ? (plugins must be known here)
176
160
  show_usage if @option_help && @context.options.command_or_arg_empty?
177
161
  generate_bash_completion if @bash_completion
178
162
  @context.config.periodic_check_newer_gem_version
@@ -180,35 +164,35 @@ module Aspera
180
164
  if @option_show_config && @context.options.command_or_arg_empty?
181
165
  COMMAND_CONFIG
182
166
  else
183
- @context.options.get_next_command(PluginFactory.instance.plugin_list.unshift(COMMAND_HELP))
167
+ @context.options.get_next_command(Plugins::Factory.instance.plugin_list.unshift(COMMAND_HELP))
184
168
  end
185
- # command will not be executed, but we need manual
169
+ # Command will not be executed, but we need manual
186
170
  @context.options.fail_on_missing_mandatory = false if @option_help || @option_show_config
187
- # main plugin is not dynamically instantiated
171
+ # Main plugin is not dynamically instantiated
188
172
  case command_sym
189
173
  when COMMAND_HELP
190
174
  show_usage
191
175
  when COMMAND_CONFIG
192
176
  command_plugin = @context.config
193
177
  else
194
- # get plugin, set options, etc
178
+ # Get plugin, set options, etc
195
179
  command_plugin = get_plugin_instance_with_options(command_sym)
196
- # parse plugin specific options
180
+ # Parse plugin specific options
197
181
  @context.options.parse_options!
198
182
  end
199
- # help requested for current plugin
183
+ # Help requested for current plugin
200
184
  show_usage(all: false) if @option_help
201
185
  if @option_show_config
202
186
  @context.formatter.display_results(type: :single_object, data: @context.options.known_options(only_defined: true).stringify_keys)
203
187
  execute_command = false
204
188
  end
205
- # locking for single execution (only after "per plugin" option, in case lock port is there)
189
+ # Locking for single execution (only after "per plugin" option, in case lock port is there)
206
190
  lock_port = @context.options.get_option(:lock_port)
207
191
  if !lock_port.nil?
208
192
  begin
209
- # no need to close later, will be freed on process exit. must save in member else it is garbage collected
193
+ # No need to close later, will be freed on process exit. must save in member else it is garbage collected
210
194
  Log.log.debug{"Opening lock port #{lock_port}"}
211
- # loopback address, could also be 'localhost'
195
+ # Loopback address, could also be 'localhost'
212
196
  @tcp_server = TCPServer.new('127.0.0.1', lock_port)
213
197
  rescue StandardError => e
214
198
  execute_command = false
@@ -221,11 +205,11 @@ module Aspera
221
205
  Log.log.debug{"Wrote pid #{Process.pid} to #{pid_file}"}
222
206
  at_exit{File.delete(pid_file)}
223
207
  end
224
- # execute and display (if not exclusive execution)
208
+ # Execute and display (if not exclusive execution)
225
209
  @context.formatter.display_results(**command_plugin.execute_action) if execute_command
226
- # save config file if command modified it
210
+ # Save config file if command modified it
227
211
  @context.config.save_config_file_if_needed
228
- # finish
212
+ # Finish
229
213
  @context.transfer.shutdown
230
214
  rescue Net::SSH::AuthenticationFailed => e; exception_info = {e: e, t: 'SSH', security: true}
231
215
  rescue OpenSSL::SSL::SSLError => e; exception_info = {e: e, t: 'SSL'}
@@ -238,7 +222,7 @@ module Aspera
238
222
  rescue StandardError => e; exception_info = {e: e, t: "Other(#{e.class.name})", debug: true}
239
223
  rescue Interrupt => e; exception_info = {e: e, t: 'Interruption', debug: true}
240
224
  end
241
- # cleanup file list files
225
+ # Cleanup file list files
242
226
  TempFileManager.instance.cleanup
243
227
  # 1- processing of error condition
244
228
  unless exception_info.nil?
@@ -252,15 +236,15 @@ module Aspera
252
236
  if execute_command || @option_show_config
253
237
  @context.options.final_errors.each do |msg|
254
238
  @context.formatter.display_message(:error, "#{Formatter::ERROR_FLASH} Argument: #{msg}")
255
- # add code as exception if there is not already an error
239
+ # Add code as exception if there is not already an error
256
240
  exception_info = {e: Exception.new(msg), t: 'UnusedArg'} if exception_info.nil?
257
241
  end
258
242
  end
259
243
  # 3- in case of error, fail the process status
260
244
  unless exception_info.nil?
261
- # show stack trace in debug mode
245
+ # Show stack trace in debug mode
262
246
  raise exception_info[:e] if Log.log.debug?
263
- # else give hint and exit
247
+ # Else give hint and exit
264
248
  @context.formatter.display_message(:error, 'Use --log-level=debug to get more details.') if exception_info[:debug]
265
249
  Process.exit(1)
266
250
  end
@@ -269,24 +253,24 @@ module Aspera
269
253
 
270
254
  def init_agents_options_plugins
271
255
  init_agents_and_options
272
- # find plugins, shall be after parse! ?
273
- PluginFactory.instance.add_plugins_from_lookup_folders
256
+ # Find plugins, shall be after parse! ?
257
+ Plugins::Factory.instance.add_plugins_from_lookup_folders
274
258
  end
275
259
 
276
260
  def show_usage(all: true, exit: true)
277
- # display main plugin options (+config)
261
+ # Display main plugin options (+config)
278
262
  @context.formatter.display_message(:error, @context.options.parser)
279
263
  if all
280
264
  @context.only_manual
281
- # list plugins that have a "require" field, i.e. all but main plugin
282
- PluginFactory.instance.plugin_list.each do |plugin_name_sym|
283
- # config was already included in the global options
265
+ # List plugins that have a "require" field, i.e. all but main plugin
266
+ Plugins::Factory.instance.plugin_list.each do |plugin_name_sym|
267
+ # Config was already included in the global options
284
268
  next if plugin_name_sym.eql?(COMMAND_CONFIG)
285
- # override main option parser with a brand new, to avoid having global options
269
+ # Override main option parser with a brand new, to avoid having global options
286
270
  @context.options = Manager.new(Info::CMD_NAME)
287
- @context.options.parser.banner = '' # remove default banner
271
+ @context.options.parser.banner = '' # Remove default banner
288
272
  get_plugin_instance_with_options(plugin_name_sym)
289
- # display generated help for plugin options
273
+ # Display generated help for plugin options
290
274
  @context.formatter.display_message(:error, @context.options.parser.help)
291
275
  end
292
276
  end
@@ -297,34 +281,34 @@ module Aspera
297
281
 
298
282
  # This can throw exception if there is a problem with the environment, needs to be caught by execute method
299
283
  def init_agents_and_options
300
- # create formatter, in case there is an exception, it is used to display.
284
+ # Create formatter, in case there is an exception, it is used to display.
301
285
  @context.formatter = Formatter.new
302
- # create command line manager with arguments
286
+ # Create command line manager with arguments
303
287
  @context.options = Manager.new(Info::CMD_NAME, @argv)
304
- # formatter adds options
288
+ # Formatter adds options
305
289
  @context.formatter.declare_options(@context.options)
306
290
  ExtendedValue.instance.default_decoder = @context.options.get_option(:struct_parser)
307
- # compare $0 with expected name
291
+ # Compare $0 with expected name
308
292
  current_prog_name = File.basename($PROGRAM_NAME)
309
293
  @context.formatter.display_message(
310
294
  :error,
311
295
  "#{Formatter::WARNING_FLASH} Please use '#{Info::CMD_NAME}' instead of '#{current_prog_name}'"
312
296
  ) unless current_prog_name.eql?(Info::CMD_NAME)
313
- # declare and parse global options
297
+ # Declare and parse global options
314
298
  declare_global_options
315
- # do not display config commands if help is asked
299
+ # Do not display config commands if help is asked
316
300
  @context.man_header = false
317
- # the Config plugin adds the @preset parser, so declare before TransferAgent which may use it
301
+ # The Config plugin adds the @preset parser, so declare before TransferAgent which may use it
318
302
  @context.config = Plugins::Config.new(context: @context)
319
303
  @context.man_header = true
320
- # data persistency is set in config
304
+ # Data persistency is set in config
321
305
  Aspera.assert(@context.persistency){'missing persistency object'}
322
- # the TransferAgent plugin may use the @preset parser
306
+ # The TransferAgent plugin may use the @preset parser
323
307
  @context.transfer = TransferAgent.new(@context.options, @context.config)
324
- # add commands for config plugin after all options have been added
308
+ # Add commands for config plugin after all options have been added
325
309
  @context.config.add_manual_header(false)
326
310
  @context.validate
327
- # set banner when all environment is created so that additional extended value modifiers are known, e.g. @preset
311
+ # Set banner when all environment is created so that additional extended value modifiers are known, e.g. @preset
328
312
  @context.options.parser.banner = app_banner
329
313
  end
330
314
 
@@ -362,7 +346,7 @@ module Aspera
362
346
  END_OF_BANNER
363
347
  end
364
348
 
365
- # define header for manual
349
+ # Define header for manual
366
350
  def declare_global_options
367
351
  Log.log.debug('declare_global_options')
368
352
  @context.options.declare(:help, 'Show this message', values: :none, short: 'h'){@option_help = true}
@@ -385,25 +369,26 @@ module Aspera
385
369
  @context.options.declare(:once_only, 'Process only new items (some commands)', values: :bool, default: false)
386
370
  @context.options.declare(:log_secrets, 'Show passwords in logs', values: :bool, handler: {o: SecretHider.instance, m: :log_secrets})
387
371
  @context.options.declare(:clean_temp, 'Cleanup temporary files on exit', values: :bool, handler: {o: TempFileManager.instance, m: :cleanup_on_exit})
372
+ @context.options.declare(:temp_folder, 'Temporary folder', handler: {o: TempFileManager.instance, m: :global_temp})
388
373
  @context.options.declare(:pid_file, 'Write process identifier to file, delete on exit', types: String)
389
- # parse declared options
374
+ # Parse declared options
390
375
  @context.options.parse_options!
391
376
  end
392
377
 
393
378
  # @return the plugin instance, based on name
394
- # also loads the plugin options, and default values from conf file
379
+ # Also loads the plugin options, and default values from conf file
395
380
  # @param plugin_name_sym : symbol for plugin name
396
381
  def get_plugin_instance_with_options(plugin_name_sym)
397
382
  Log.log.debug{"get_plugin_instance_with_options(#{plugin_name_sym})"}
398
- # load default params only if no param already loaded before plugin instantiation
383
+ # Load default params only if no param already loaded before plugin instantiation
399
384
  @context.config.add_plugin_default_preset(plugin_name_sym)
400
- command_plugin = PluginFactory.instance.create(plugin_name_sym, context: @context)
385
+ command_plugin = Plugins::Factory.instance.create(plugin_name_sym, context: @context)
401
386
  return command_plugin
402
387
  end
403
388
 
404
389
  def generate_bash_completion
405
390
  if @context.options.get_next_argument('', multiple: true, mandatory: false).nil?
406
- PluginFactory.instance.plugin_list.each{ |p| puts p}
391
+ Plugins::Factory.instance.plugin_list.each{ |p| puts p}
407
392
  else
408
393
  Log.log.warn('only first level completion so far')
409
394
  end
@@ -141,6 +141,7 @@ module Aspera
141
141
  @ask_missing_mandatory = false # STDIN.isatty
142
142
  # ask optional options if not provided and in interactive
143
143
  @ask_missing_optional = false
144
+ # get_option fails if a mandatory parameter is asked
144
145
  @fail_on_missing_mandatory = true
145
146
  # Array of [key(sym), value]
146
147
  # those must be set before parse
@@ -182,7 +183,7 @@ module Aspera
182
183
  declare(:interactive, 'Use interactive input of missing params', values: :bool, handler: {o: self, m: :ask_missing_mandatory})
183
184
  declare(:ask_options, 'Ask even optional options', values: :bool, handler: {o: self, m: :ask_missing_optional})
184
185
  declare(:struct_parser, 'Default parser when expected value is a struct', values: %i[json ruby])
185
- # do not parse options yet, let's wait for option `-h` to be overriden
186
+ # do not parse options yet, let's wait for option `-h` to be overridden
186
187
  end
187
188
 
188
189
  # @param descr [String] description for help
@@ -254,7 +255,7 @@ module Aspera
254
255
  Log.log.trace1{"(#{attributes[:read_write]}) get #{option_symbol}=#{result}"}
255
256
  result = default if result.nil?
256
257
  # do not fail for manual generation if option mandatory but not set
257
- result = '' if result.nil? && mandatory && !@fail_on_missing_mandatory
258
+ result = :skip_missing_mandatory if result.nil? && mandatory && !@fail_on_missing_mandatory
258
259
  # Log.log.debug{"interactive=#{@ask_missing_mandatory}"}
259
260
  if result.nil?
260
261
  if !@ask_missing_mandatory
@@ -436,6 +437,8 @@ module Aspera
436
437
  @declared_options.each_key do |option_symbol|
437
438
  v = get_option(option_symbol)
438
439
  result[option_symbol] = v unless only_defined && v.nil?
440
+ rescue => e
441
+ result[option_symbol] = e.to_s
439
442
  end
440
443
  return result
441
444
  end
@@ -536,10 +539,10 @@ module Aspera
536
539
  return true if value == 'true'
537
540
  return false if value == 'false'
538
541
  Integer(value)
539
- rescue ArgumentError
542
+ rescue ::ArgumentError
540
543
  begin
541
544
  Float(value)
542
- rescue ArgumentError
545
+ rescue ::ArgumentError
543
546
  evaluate_extended_value(value, nil)
544
547
  end
545
548
  end
@@ -2,11 +2,12 @@
2
2
 
3
3
  require 'aspera/api/alee'
4
4
  require 'aspera/nagios'
5
+ require 'aspera/cli/plugins/basic_auth'
5
6
 
6
7
  module Aspera
7
8
  module Cli
8
9
  module Plugins
9
- class Alee < Cli::BasicAuthPlugin
10
+ class Alee < BasicAuth
10
11
  ACTIONS = %i[health entitlement].freeze
11
12
 
12
13
  def execute_action