aspera-cli 4.15.0 → 4.16.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (80) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/BUGS.md +29 -3
  4. data/CHANGELOG.md +292 -228
  5. data/CONTRIBUTING.md +69 -18
  6. data/README.md +1102 -952
  7. data/bin/ascli +13 -31
  8. data/bin/asession +3 -1
  9. data/examples/dascli +2 -2
  10. data/lib/aspera/aoc.rb +28 -33
  11. data/lib/aspera/ascmd.rb +3 -6
  12. data/lib/aspera/assert.rb +45 -0
  13. data/lib/aspera/cli/extended_value.rb +5 -5
  14. data/lib/aspera/cli/formatter.rb +26 -13
  15. data/lib/aspera/cli/hints.rb +4 -3
  16. data/lib/aspera/cli/main.rb +16 -3
  17. data/lib/aspera/cli/manager.rb +45 -36
  18. data/lib/aspera/cli/plugin.rb +20 -13
  19. data/lib/aspera/cli/plugins/aoc.rb +103 -73
  20. data/lib/aspera/cli/plugins/ats.rb +4 -3
  21. data/lib/aspera/cli/plugins/config.rb +114 -119
  22. data/lib/aspera/cli/plugins/cos.rb +2 -2
  23. data/lib/aspera/cli/plugins/faspex.rb +23 -19
  24. data/lib/aspera/cli/plugins/faspex5.rb +75 -43
  25. data/lib/aspera/cli/plugins/node.rb +28 -15
  26. data/lib/aspera/cli/plugins/orchestrator.rb +4 -2
  27. data/lib/aspera/cli/plugins/preview.rb +9 -7
  28. data/lib/aspera/cli/plugins/server.rb +6 -3
  29. data/lib/aspera/cli/plugins/shares.rb +30 -26
  30. data/lib/aspera/cli/sync_actions.rb +9 -9
  31. data/lib/aspera/cli/transfer_agent.rb +21 -14
  32. data/lib/aspera/cli/transfer_progress.rb +2 -3
  33. data/lib/aspera/cli/version.rb +1 -1
  34. data/lib/aspera/command_line_builder.rb +13 -11
  35. data/lib/aspera/cos_node.rb +3 -2
  36. data/lib/aspera/coverage.rb +22 -0
  37. data/lib/aspera/data_repository.rb +33 -2
  38. data/lib/aspera/environment.rb +4 -2
  39. data/lib/aspera/fasp/{agent_aspera.rb → agent_alpha.rb} +29 -39
  40. data/lib/aspera/fasp/agent_base.rb +17 -7
  41. data/lib/aspera/fasp/agent_direct.rb +88 -84
  42. data/lib/aspera/fasp/agent_httpgw.rb +4 -3
  43. data/lib/aspera/fasp/agent_node.rb +3 -2
  44. data/lib/aspera/fasp/agent_trsdk.rb +79 -37
  45. data/lib/aspera/fasp/installation.rb +51 -12
  46. data/lib/aspera/fasp/management.rb +11 -6
  47. data/lib/aspera/fasp/parameters.rb +53 -47
  48. data/lib/aspera/fasp/resume_policy.rb +7 -5
  49. data/lib/aspera/fasp/sync.rb +273 -0
  50. data/lib/aspera/fasp/transfer_spec.rb +10 -8
  51. data/lib/aspera/fasp/uri.rb +2 -2
  52. data/lib/aspera/faspex_gw.rb +11 -8
  53. data/lib/aspera/faspex_postproc.rb +6 -5
  54. data/lib/aspera/id_generator.rb +3 -1
  55. data/lib/aspera/json_rpc.rb +10 -8
  56. data/lib/aspera/keychain/encrypted_hash.rb +46 -11
  57. data/lib/aspera/keychain/macos_security.rb +15 -13
  58. data/lib/aspera/log.rb +4 -3
  59. data/lib/aspera/nagios.rb +7 -2
  60. data/lib/aspera/node.rb +17 -16
  61. data/lib/aspera/node_simulator.rb +214 -0
  62. data/lib/aspera/oauth.rb +22 -19
  63. data/lib/aspera/persistency_action_once.rb +13 -14
  64. data/lib/aspera/persistency_folder.rb +3 -2
  65. data/lib/aspera/preview/file_types.rb +53 -267
  66. data/lib/aspera/preview/generator.rb +7 -5
  67. data/lib/aspera/preview/terminal.rb +14 -5
  68. data/lib/aspera/preview/utils.rb +8 -7
  69. data/lib/aspera/proxy_auto_config.rb +6 -3
  70. data/lib/aspera/rest.rb +29 -13
  71. data/lib/aspera/rest_error_analyzer.rb +1 -0
  72. data/lib/aspera/rest_errors_aspera.rb +2 -0
  73. data/lib/aspera/secret_hider.rb +5 -2
  74. data/lib/aspera/ssh.rb +10 -8
  75. data/lib/aspera/temp_file_manager.rb +1 -1
  76. data/lib/aspera/web_server_simple.rb +2 -1
  77. data.tar.gz.sig +0 -0
  78. metadata +96 -45
  79. metadata.gz.sig +0 -0
  80. data/lib/aspera/sync.rb +0 -219
@@ -5,6 +5,7 @@ require 'aspera/cli/error'
5
5
  require 'aspera/colors'
6
6
  require 'aspera/secret_hider'
7
7
  require 'aspera/log'
8
+ require 'aspera/assert'
8
9
  require 'io/console'
9
10
  require 'optparse'
10
11
 
@@ -20,7 +21,7 @@ module Aspera
20
21
  @option_name = option_name
21
22
  @has_writer = @object.respond_to?(writer_method)
22
23
  Log.log.debug{"AttrAccessor: #{@option_name}: #{@object.class}.#{@method}: writer=#{@has_writer}"}
23
- raise "internal error: #{object} does not respond to #{method_name}" unless @object.respond_to?(@method)
24
+ assert(@object.respond_to?(@method)) {"#{object} does not respond to #{method_name}"}
24
25
  end
25
26
 
26
27
  def value
@@ -54,12 +55,13 @@ module Aspera
54
55
  # option name separator in code (symbol)
55
56
  OPTION_SEP_SYMBOL = '_'
56
57
  SOURCE_USER = 'cmdline' # cspell:disable-line
58
+ TYPE_INTEGER = [Integer].freeze
57
59
 
58
- private_constant :FALSE_VALUES, :TRUE_VALUES, :BOOLEAN_VALUES, :OPTION_SEP_LINE, :OPTION_SEP_SYMBOL, :SOURCE_USER
60
+ private_constant :FALSE_VALUES, :TRUE_VALUES, :BOOLEAN_VALUES, :OPTION_SEP_LINE, :OPTION_SEP_SYMBOL, :SOURCE_USER, :TYPE_INTEGER
59
61
 
60
62
  class << self
61
63
  def enum_to_bool(enum)
62
- raise "Value not valid for boolean: [#{enum}]/#{enum.class}" unless BOOLEAN_VALUES.include?(enum)
64
+ assert_values(enum, BOOLEAN_VALUES){'boolean'}
63
65
  return TRUE_VALUES.include?(enum)
64
66
  end
65
67
 
@@ -73,14 +75,17 @@ module Aspera
73
75
  matching_exact = allowed_values.select{|i| i.to_s.eql?(short_value)}
74
76
  return matching_exact.first if matching_exact.length == 1
75
77
  matching = allowed_values.select{|i| i.to_s.start_with?(short_value)}
76
- raise Cli::BadArgument, bad_arg_message_multi("unknown value for #{descr}: #{short_value}", allowed_values) if matching.empty?
77
- raise Cli::BadArgument, bad_arg_message_multi("ambiguous shortcut for #{descr}: #{short_value}", matching) unless matching.length.eql?(1)
78
+ multi_choice_assert(!matching.empty?,"unknown value for #{descr}: #{short_value}", allowed_values)
79
+ multi_choice_assert(matching.length.eql?(1),"ambiguous shortcut for #{descr}: #{short_value}", matching)
78
80
  return enum_to_bool(matching.first) if allowed_values.eql?(BOOLEAN_VALUES)
79
81
  return matching.first
80
82
  end
81
83
 
82
- def bad_arg_message_multi(error_msg, choices)
83
- return [error_msg, 'Use:'].concat(choices.map{|c|"- #{c}"}.sort).join("\n")
84
+ # Generates error message with list of allowed values
85
+ # @param error_msg [String] error message
86
+ # @param choices [Array] list of allowed values
87
+ def multi_choice_assert(assertion,error_msg, choices)
88
+ raise Cli::BadArgument, [error_msg, 'Use:'].concat(choices.map{|c|"- #{c}"}.sort).join("\n") unless assertion
84
89
  end
85
90
 
86
91
  # change option name with dash to name with underscore
@@ -92,9 +97,13 @@ module Aspera
92
97
  return "--#{name.to_s.gsub(OPTION_SEP_SYMBOL, OPTION_SEP_LINE)}"
93
98
  end
94
99
 
100
+ # @param what [Symbol] :option or :argument
101
+ # @param descr [String] description for help
102
+ # @param value [Object] value to check
103
+ # @param type_list [NilClass, Class, Array[Class]] accepted value type(s)
95
104
  def validate_type(what, descr, value, type_list)
96
105
  return nil if type_list.nil?
97
- raise 'internal error: types must be a Class Array' unless type_list.is_a?(Array) && type_list.all?(Class)
106
+ assert(type_list.is_a?(Array) && type_list.all?(Class)){'types must be a Class Array'}
98
107
  raise Cli::BadArgument,
99
108
  "#{what.to_s.capitalize} #{descr} is a #{value.class} but must be #{type_list.length > 1 ? 'one of ' : ''}#{type_list.map(&:name).join(',')}" unless \
100
109
  type_list.any?{|t|value.is_a?(t)}
@@ -172,10 +181,14 @@ module Aspera
172
181
  # @param default [Object] default value
173
182
  # @return value, list or nil
174
183
  def get_next_argument(descr, expected: :single, mandatory: true, type: nil, aliases: nil, default: nil)
175
- unless type.nil?
176
- type = [type] unless type.is_a?(Array)
177
- raise "INTERNAL ERROR: type must be Array of Class: #{type}" unless type.all?(Class)
178
- descr = "#{descr} (#{type})"
184
+ assert(%i[single multiple].include?(expected) || (expected.is_a?(Array) && expected.all?(Symbol))){'expected must be single, multiple, or array of symbol'}
185
+ assert(type.nil? || type.is_a?(Class) || (type.is_a?(Array) && type.all?(Class))){'type must be Class or Array of Class'}
186
+ assert(aliases.nil? || (aliases.is_a?(Hash) && aliases.keys.all?(Symbol) && aliases.values.all?(Symbol))){'aliases must be Hash'}
187
+ allowed_types = type
188
+ unless allowed_types.nil?
189
+ allowed_types = [allowed_types] unless allowed_types.is_a?(Array)
190
+ descr = "#{descr} (#{allowed_types.join(', ')})"
191
+ Log.log.debug{">>>> #{descr}=#{allowed_types}"}
179
192
  end
180
193
  result =
181
194
  if !@unprocessed_cmd_line_arguments.empty?
@@ -193,23 +206,21 @@ module Aspera
193
206
  when Array
194
207
  allowed_values = [].concat(expected)
195
208
  allowed_values.concat(aliases.keys) unless aliases.nil?
196
- raise "internal error: only symbols allowed: #{allowed_values}" unless allowed_values.all?(Symbol)
197
209
  self.class.get_from_list(@unprocessed_cmd_line_arguments.shift, descr, allowed_values)
198
- else
199
- raise 'Internal error: expected: must be single, multiple, or value array'
210
+ else error_unexpected_value(expected)
200
211
  end
201
212
  elsif !default.nil? then default
202
213
  # no value provided, either get value interactively, or exception
203
214
  elsif mandatory then get_interactive(:argument, descr, expected: expected)
204
215
  end
205
- if result.is_a?(String) && type.eql?([Integer])
206
- str_result = result
207
- result = Integer(str_result, exception: false)
208
- raise Cli::BadArgument, "Invalid integer: #{str_result}" if result.nil?
216
+ if result.is_a?(String) && allowed_types.eql?(TYPE_INTEGER)
217
+ int_result = Integer(result, exception: false)
218
+ raise Cli::BadArgument, "Invalid integer: #{result}" if int_result.nil?
219
+ result = int_result
209
220
  end
210
221
  Log.log.debug{"#{descr}=#{result}"}
211
- result = aliases[result] if !aliases.nil? && aliases.key?(result)
212
- self.class.validate_type(:argument, descr, result, type) unless result.nil? && !mandatory
222
+ result = aliases[result] if aliases&.key?(result)
223
+ self.class.validate_type(:argument, descr, result, allowed_types) unless result.nil? && !mandatory
213
224
  return result
214
225
  end
215
226
 
@@ -221,7 +232,7 @@ module Aspera
221
232
  # @param mandatory [Boolean] if true, raise error if option not set
222
233
  def get_option(option_symbol, mandatory: false, default: nil)
223
234
  attributes = @declared_options[option_symbol]
224
- raise "INTERNAL ERROR: option not declared: #{option_symbol}" unless attributes
235
+ assert(attributes){"option not declared: #{option_symbol}"}
225
236
  result = nil
226
237
  case attributes[:read_write]
227
238
  when :accessor
@@ -284,12 +295,10 @@ module Aspera
284
295
  # @param types [Class, Array] accepted value type(s)
285
296
  # @param block [Proc] block to execute when option is found
286
297
  def declare(option_symbol, description, handler: nil, default: nil, values: nil, short: nil, coerce: nil, types: nil, deprecation: nil, &block)
287
- raise "INTERNAL ERROR: #{option_symbol} already declared" if @declared_options.key?(option_symbol)
288
- # raise "INTERNAL ERROR: #{option_symbol} clash with another option" if
289
- # @declared_options.keys.map(&:to_s).any?{|k|k.start_with?(option_symbol.to_s) || option_symbol.to_s.start_with?(k)}
290
- raise "INTERNAL ERROR: #{option_symbol} ends with dot" unless description[-1] != '.'
291
- raise "INTERNAL ERROR: #{option_symbol} description does not start with capital" unless description[0] == description[0].upcase
292
- raise "INTERNAL ERROR: #{option_symbol} shall use :types" if ['hash', 'extended value'].any?{|s|description.downcase.include?(s) }
298
+ assert(!@declared_options.key?(option_symbol)){"#{option_symbol} already declared"}
299
+ assert(description[-1] != '.'){"#{option_symbol} ends with dot"}
300
+ assert(description[0] == description[0].upcase){"#{option_symbol} description does not start with capital"}
301
+ assert(!['hash', 'extended value'].any?{|s|description.downcase.include?(s) }){"#{option_symbol} shall use :types"}
293
302
  opt = @declared_options[option_symbol] = {
294
303
  read_write: handler.nil? ? :value : :accessor,
295
304
  # by default passwords and secrets are sensitive, else specify when declaring the option
@@ -297,7 +306,7 @@ module Aspera
297
306
  }
298
307
  if !types.nil?
299
308
  types = [types] unless types.is_a?(Array)
300
- raise "INTERNAL ERROR: types must be Array of Class: #{types}" unless types.all?(Class)
309
+ assert(types.all?(Class)){"types must be Array of Class: #{types}"}
301
310
  opt[:types] = types
302
311
  description = "#{description} (#{types.map(&:name).join(', ')})"
303
312
  end
@@ -307,8 +316,8 @@ module Aspera
307
316
  end
308
317
  Log.log.debug{"declare: #{option_symbol}: #{opt[:read_write]}".green}
309
318
  if opt[:read_write].eql?(:accessor)
310
- raise 'internal error' unless handler.is_a?(Hash)
311
- raise 'internal error' unless handler.keys.sort.eql?(%i[m o])
319
+ assert_type(handler, Hash)
320
+ assert(handler.keys.sort.eql?(%i[m o]))
312
321
  Log.log.debug{"set attr obj #{option_symbol} (#{handler[:o]},#{handler[:m]})"}
313
322
  opt[:accessor] = AttrAccessor.new(handler[:o], handler[:m], option_symbol)
314
323
  end
@@ -347,11 +356,11 @@ module Aspera
347
356
  set_option(option_symbol, time_string, SOURCE_USER)
348
357
  end
349
358
  when :none
350
- raise "internal error: missing block for #{option_symbol}" if block.nil?
359
+ assert(!block.nil?){"missing block for #{option_symbol}"}
351
360
  on_args.push(symbol_to_option(option_symbol, nil))
352
361
  on_args.push("-#{short}") if short.is_a?(String)
353
362
  @parser.on(*on_args, &block)
354
- else raise "internal error: Unknown type for values: #{values} / #{values.class}"
363
+ else error_unexpected_value(values)
355
364
  end
356
365
  Log.log.debug{"on_args=#{on_args}"}
357
366
  end
@@ -359,8 +368,8 @@ module Aspera
359
368
  # Adds each of the keys of specified hash as an option
360
369
  # @param preset_hash [Hash] hash of options to add
361
370
  def add_option_preset(preset_hash, op: :push)
371
+ assert_type(preset_hash, Hash)
362
372
  Log.log.debug{"add_option_preset=#{preset_hash}"}
363
- raise "internal error: default expects Hash: #{preset_hash.class}" unless preset_hash.is_a?(Hash)
364
373
  # incremental override
365
374
  preset_hash.each{|k, v|@unprocessed_defaults.send(op, [k.to_sym, v])}
366
375
  end
@@ -460,8 +469,8 @@ module Aspera
460
469
 
461
470
  def get_interactive(type, descr, expected: :single)
462
471
  if !@ask_missing_mandatory
463
- raise Cli::BadArgument, self.class.bad_arg_message_multi("missing: #{descr}", expected) if expected.is_a?(Array)
464
- raise Cli::BadArgument, "missing argument (#{expected}): #{descr}"
472
+ raise Cli::BadArgument, "missing argument (#{expected}): #{descr}" unless expected.is_a?(Array)
473
+ self.class.multi_choice_assert(false,"missing: #{descr}", expected)
465
474
  end
466
475
  result = nil
467
476
  sensitive = type.eql?(:option) && @declared_options[descr.to_sym].is_a?(Hash) && @declared_options[descr.to_sym][:sensitive]
@@ -1,6 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require 'aspera/cli/extended_value'
4
+ require 'aspera/assert'
4
5
 
5
6
  module Aspera
6
7
  module Cli
@@ -33,11 +34,11 @@ module Aspera
33
34
  end
34
35
 
35
36
  def initialize(env)
36
- raise 'env must be Hash' unless env.is_a?(Hash)
37
+ assert_type(env, Hash)
37
38
  @agents = env
38
39
  # check presence in descendant of mandatory method and constant
39
- raise StandardError, "Missing method 'execute_action' in #{self.class}" unless respond_to?(:execute_action)
40
- raise StandardError, 'ACTIONS shall be redefined by subclass' unless self.class.constants.include?(:ACTIONS)
40
+ assert(respond_to?(:execute_action)){"Missing method 'execute_action' in #{self.class}"}
41
+ assert(self.class.constants.include?(:ACTIONS)){'ACTIONS shall be redefined by subclass'}
41
42
  # manual header for all plugins
42
43
  options.parser.separator('')
43
44
  options.parser.separator("COMMAND: #{self.class.name.split('::').last.downcase}")
@@ -75,6 +76,7 @@ module Aspera
75
76
  # @param id_result [String] key in result hash to use as identifier
76
77
  # @param fields [Array] fields to display
77
78
  def do_bulk_operation(command:, descr:, values: Hash, id_result: 'id', fields: :default)
79
+ assert(block_given?){'missing block'}
78
80
  is_bulk = options.get_option(:bulk)
79
81
  case values
80
82
  when :identifier
@@ -82,7 +84,6 @@ module Aspera
82
84
  when Class
83
85
  values = value_create_modify(command: command, type: values, bulk: is_bulk)
84
86
  end
85
- raise 'Internal error: missing block' unless block_given?
86
87
  # if not bulk, there is a single value
87
88
  params = is_bulk ? values : [values]
88
89
  Log.log.warn('Empty list given for bulk operation') if params.empty?
@@ -147,6 +148,7 @@ module Aspera
147
148
  return {type: :single_object, data: rest_api.read(one_res_path)[:data], fields: display_fields}
148
149
  when :list
149
150
  resp = rest_api.read(res_class_path, old_query_read_delete)
151
+ return Main.result_empty if resp[:http].code == '204'
150
152
  data = resp[:data]
151
153
  # TODO: not generic : which application is this for ?
152
154
  if resp[:http]['Content-Type'].start_with?('application/vnd.api+json')
@@ -180,10 +182,10 @@ module Aspera
180
182
  end
181
183
 
182
184
  # implement generic rest operations on given resource path
183
- def entity_action(rest_api, res_class_path, **opts)
185
+ def entity_action(rest_api, res_class_path, **opts, &block)
184
186
  # res_name=res_class_path.gsub(%r{^.*/},'').gsub(%r{s$},'').gsub('_',' ')
185
187
  command = options.get_next_command(ALL_OPS)
186
- return entity_command(command, rest_api, res_class_path, **opts)
188
+ return entity_command(command, rest_api, res_class_path, **opts, &block)
187
189
  end
188
190
 
189
191
  # query parameters in URL suitable for REST list/GET and delete/DELETE
@@ -223,22 +225,27 @@ module Aspera
223
225
 
224
226
  # Retrieves an extended value from command line, used for creation or modification of entities
225
227
  # @param command [Symbol] command name for error message
226
- # @param type [Class] expected type of value, either a Class, an Array of Class, or :bulk_hash
228
+ # @param type [Class] expected type of value, either a Class, an Array of Class
229
+ # @param bulk [Boolean] if true, value must be an Array of <type>
227
230
  # @param default [Object] default value if not provided
228
231
  # TODO: when deprecation of `value` is completed: remove line with :value
229
- def value_create_modify(command:, type: Hash, bulk: false, default: nil)
232
+ def value_create_modify(command:, description: nil, type: Hash, bulk: false, default: nil)
230
233
  value = options.get_option(:value)
231
234
  Log.log.warn("option `value` is deprecated. Use positional parameter for #{command}") unless value.nil?
232
- value = options.get_next_argument("parameters for #{command}", mandatory: default.nil?) if value.nil?
235
+ value = options.get_next_argument(
236
+ "parameters for #{command}#{description.nil? ? '' : " (#{description})"}", mandatory: default.nil?,
237
+ type: bulk ? Array : type) if value.nil?
233
238
  value = default if value.nil?
234
239
  unless type.nil?
235
240
  type = [type] unless type.is_a?(Array)
236
- raise "Internal error, check types must be a Class, not #{type.map(&:class).join(',')}" unless type.all?(Class)
241
+ assert(type.all?(Class)){"check types must be a Class, not #{type.map(&:class).join(',')}"}
237
242
  if bulk
238
- raise Cli::BadArgument, "Value must be an Array of #{type.join(',')}" unless value.is_a?(Array)
239
- raise Cli::BadArgument, "Value must be a #{type.join(',')}, not #{value.map{|i| i.class.name}.uniq.join(',')}" unless value.all?{|v|type.include?(v.class)}
243
+ assert_type(value, Array, exception_class: Cli::BadArgument)
244
+ value.each do |v|
245
+ assert_values(v.class, type, exception_class: Cli::BadArgument)
246
+ end
240
247
  else
241
- raise Cli::BadArgument, "Value must be a #{type.join(',')}, not #{value.class.name}" unless type.include?(value.class)
248
+ assert_values(value.class, type, exception_class: Cli::BadArgument)
242
249
  end
243
250
  end
244
251
  return value