aspera-cli 4.15.0 → 4.16.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (80) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/BUGS.md +29 -3
  4. data/CHANGELOG.md +292 -228
  5. data/CONTRIBUTING.md +69 -18
  6. data/README.md +1102 -952
  7. data/bin/ascli +13 -31
  8. data/bin/asession +3 -1
  9. data/examples/dascli +2 -2
  10. data/lib/aspera/aoc.rb +28 -33
  11. data/lib/aspera/ascmd.rb +3 -6
  12. data/lib/aspera/assert.rb +45 -0
  13. data/lib/aspera/cli/extended_value.rb +5 -5
  14. data/lib/aspera/cli/formatter.rb +26 -13
  15. data/lib/aspera/cli/hints.rb +4 -3
  16. data/lib/aspera/cli/main.rb +16 -3
  17. data/lib/aspera/cli/manager.rb +45 -36
  18. data/lib/aspera/cli/plugin.rb +20 -13
  19. data/lib/aspera/cli/plugins/aoc.rb +103 -73
  20. data/lib/aspera/cli/plugins/ats.rb +4 -3
  21. data/lib/aspera/cli/plugins/config.rb +114 -119
  22. data/lib/aspera/cli/plugins/cos.rb +2 -2
  23. data/lib/aspera/cli/plugins/faspex.rb +23 -19
  24. data/lib/aspera/cli/plugins/faspex5.rb +75 -43
  25. data/lib/aspera/cli/plugins/node.rb +28 -15
  26. data/lib/aspera/cli/plugins/orchestrator.rb +4 -2
  27. data/lib/aspera/cli/plugins/preview.rb +9 -7
  28. data/lib/aspera/cli/plugins/server.rb +6 -3
  29. data/lib/aspera/cli/plugins/shares.rb +30 -26
  30. data/lib/aspera/cli/sync_actions.rb +9 -9
  31. data/lib/aspera/cli/transfer_agent.rb +21 -14
  32. data/lib/aspera/cli/transfer_progress.rb +2 -3
  33. data/lib/aspera/cli/version.rb +1 -1
  34. data/lib/aspera/command_line_builder.rb +13 -11
  35. data/lib/aspera/cos_node.rb +3 -2
  36. data/lib/aspera/coverage.rb +22 -0
  37. data/lib/aspera/data_repository.rb +33 -2
  38. data/lib/aspera/environment.rb +4 -2
  39. data/lib/aspera/fasp/{agent_aspera.rb → agent_alpha.rb} +29 -39
  40. data/lib/aspera/fasp/agent_base.rb +17 -7
  41. data/lib/aspera/fasp/agent_direct.rb +88 -84
  42. data/lib/aspera/fasp/agent_httpgw.rb +4 -3
  43. data/lib/aspera/fasp/agent_node.rb +3 -2
  44. data/lib/aspera/fasp/agent_trsdk.rb +79 -37
  45. data/lib/aspera/fasp/installation.rb +51 -12
  46. data/lib/aspera/fasp/management.rb +11 -6
  47. data/lib/aspera/fasp/parameters.rb +53 -47
  48. data/lib/aspera/fasp/resume_policy.rb +7 -5
  49. data/lib/aspera/fasp/sync.rb +273 -0
  50. data/lib/aspera/fasp/transfer_spec.rb +10 -8
  51. data/lib/aspera/fasp/uri.rb +2 -2
  52. data/lib/aspera/faspex_gw.rb +11 -8
  53. data/lib/aspera/faspex_postproc.rb +6 -5
  54. data/lib/aspera/id_generator.rb +3 -1
  55. data/lib/aspera/json_rpc.rb +10 -8
  56. data/lib/aspera/keychain/encrypted_hash.rb +46 -11
  57. data/lib/aspera/keychain/macos_security.rb +15 -13
  58. data/lib/aspera/log.rb +4 -3
  59. data/lib/aspera/nagios.rb +7 -2
  60. data/lib/aspera/node.rb +17 -16
  61. data/lib/aspera/node_simulator.rb +214 -0
  62. data/lib/aspera/oauth.rb +22 -19
  63. data/lib/aspera/persistency_action_once.rb +13 -14
  64. data/lib/aspera/persistency_folder.rb +3 -2
  65. data/lib/aspera/preview/file_types.rb +53 -267
  66. data/lib/aspera/preview/generator.rb +7 -5
  67. data/lib/aspera/preview/terminal.rb +14 -5
  68. data/lib/aspera/preview/utils.rb +8 -7
  69. data/lib/aspera/proxy_auto_config.rb +6 -3
  70. data/lib/aspera/rest.rb +29 -13
  71. data/lib/aspera/rest_error_analyzer.rb +1 -0
  72. data/lib/aspera/rest_errors_aspera.rb +2 -0
  73. data/lib/aspera/secret_hider.rb +5 -2
  74. data/lib/aspera/ssh.rb +10 -8
  75. data/lib/aspera/temp_file_manager.rb +1 -1
  76. data/lib/aspera/web_server_simple.rb +2 -1
  77. data.tar.gz.sig +0 -0
  78. metadata +96 -45
  79. metadata.gz.sig +0 -0
  80. data/lib/aspera/sync.rb +0 -219
@@ -5,6 +5,7 @@ require 'aspera/cli/error'
5
5
  require 'aspera/colors'
6
6
  require 'aspera/secret_hider'
7
7
  require 'aspera/log'
8
+ require 'aspera/assert'
8
9
  require 'io/console'
9
10
  require 'optparse'
10
11
 
@@ -20,7 +21,7 @@ module Aspera
20
21
  @option_name = option_name
21
22
  @has_writer = @object.respond_to?(writer_method)
22
23
  Log.log.debug{"AttrAccessor: #{@option_name}: #{@object.class}.#{@method}: writer=#{@has_writer}"}
23
- raise "internal error: #{object} does not respond to #{method_name}" unless @object.respond_to?(@method)
24
+ assert(@object.respond_to?(@method)) {"#{object} does not respond to #{method_name}"}
24
25
  end
25
26
 
26
27
  def value
@@ -54,12 +55,13 @@ module Aspera
54
55
  # option name separator in code (symbol)
55
56
  OPTION_SEP_SYMBOL = '_'
56
57
  SOURCE_USER = 'cmdline' # cspell:disable-line
58
+ TYPE_INTEGER = [Integer].freeze
57
59
 
58
- private_constant :FALSE_VALUES, :TRUE_VALUES, :BOOLEAN_VALUES, :OPTION_SEP_LINE, :OPTION_SEP_SYMBOL, :SOURCE_USER
60
+ private_constant :FALSE_VALUES, :TRUE_VALUES, :BOOLEAN_VALUES, :OPTION_SEP_LINE, :OPTION_SEP_SYMBOL, :SOURCE_USER, :TYPE_INTEGER
59
61
 
60
62
  class << self
61
63
  def enum_to_bool(enum)
62
- raise "Value not valid for boolean: [#{enum}]/#{enum.class}" unless BOOLEAN_VALUES.include?(enum)
64
+ assert_values(enum, BOOLEAN_VALUES){'boolean'}
63
65
  return TRUE_VALUES.include?(enum)
64
66
  end
65
67
 
@@ -73,14 +75,17 @@ module Aspera
73
75
  matching_exact = allowed_values.select{|i| i.to_s.eql?(short_value)}
74
76
  return matching_exact.first if matching_exact.length == 1
75
77
  matching = allowed_values.select{|i| i.to_s.start_with?(short_value)}
76
- raise Cli::BadArgument, bad_arg_message_multi("unknown value for #{descr}: #{short_value}", allowed_values) if matching.empty?
77
- raise Cli::BadArgument, bad_arg_message_multi("ambiguous shortcut for #{descr}: #{short_value}", matching) unless matching.length.eql?(1)
78
+ multi_choice_assert(!matching.empty?,"unknown value for #{descr}: #{short_value}", allowed_values)
79
+ multi_choice_assert(matching.length.eql?(1),"ambiguous shortcut for #{descr}: #{short_value}", matching)
78
80
  return enum_to_bool(matching.first) if allowed_values.eql?(BOOLEAN_VALUES)
79
81
  return matching.first
80
82
  end
81
83
 
82
- def bad_arg_message_multi(error_msg, choices)
83
- return [error_msg, 'Use:'].concat(choices.map{|c|"- #{c}"}.sort).join("\n")
84
+ # Generates error message with list of allowed values
85
+ # @param error_msg [String] error message
86
+ # @param choices [Array] list of allowed values
87
+ def multi_choice_assert(assertion,error_msg, choices)
88
+ raise Cli::BadArgument, [error_msg, 'Use:'].concat(choices.map{|c|"- #{c}"}.sort).join("\n") unless assertion
84
89
  end
85
90
 
86
91
  # change option name with dash to name with underscore
@@ -92,9 +97,13 @@ module Aspera
92
97
  return "--#{name.to_s.gsub(OPTION_SEP_SYMBOL, OPTION_SEP_LINE)}"
93
98
  end
94
99
 
100
+ # @param what [Symbol] :option or :argument
101
+ # @param descr [String] description for help
102
+ # @param value [Object] value to check
103
+ # @param type_list [NilClass, Class, Array[Class]] accepted value type(s)
95
104
  def validate_type(what, descr, value, type_list)
96
105
  return nil if type_list.nil?
97
- raise 'internal error: types must be a Class Array' unless type_list.is_a?(Array) && type_list.all?(Class)
106
+ assert(type_list.is_a?(Array) && type_list.all?(Class)){'types must be a Class Array'}
98
107
  raise Cli::BadArgument,
99
108
  "#{what.to_s.capitalize} #{descr} is a #{value.class} but must be #{type_list.length > 1 ? 'one of ' : ''}#{type_list.map(&:name).join(',')}" unless \
100
109
  type_list.any?{|t|value.is_a?(t)}
@@ -172,10 +181,14 @@ module Aspera
172
181
  # @param default [Object] default value
173
182
  # @return value, list or nil
174
183
  def get_next_argument(descr, expected: :single, mandatory: true, type: nil, aliases: nil, default: nil)
175
- unless type.nil?
176
- type = [type] unless type.is_a?(Array)
177
- raise "INTERNAL ERROR: type must be Array of Class: #{type}" unless type.all?(Class)
178
- descr = "#{descr} (#{type})"
184
+ assert(%i[single multiple].include?(expected) || (expected.is_a?(Array) && expected.all?(Symbol))){'expected must be single, multiple, or array of symbol'}
185
+ assert(type.nil? || type.is_a?(Class) || (type.is_a?(Array) && type.all?(Class))){'type must be Class or Array of Class'}
186
+ assert(aliases.nil? || (aliases.is_a?(Hash) && aliases.keys.all?(Symbol) && aliases.values.all?(Symbol))){'aliases must be Hash'}
187
+ allowed_types = type
188
+ unless allowed_types.nil?
189
+ allowed_types = [allowed_types] unless allowed_types.is_a?(Array)
190
+ descr = "#{descr} (#{allowed_types.join(', ')})"
191
+ Log.log.debug{">>>> #{descr}=#{allowed_types}"}
179
192
  end
180
193
  result =
181
194
  if !@unprocessed_cmd_line_arguments.empty?
@@ -193,23 +206,21 @@ module Aspera
193
206
  when Array
194
207
  allowed_values = [].concat(expected)
195
208
  allowed_values.concat(aliases.keys) unless aliases.nil?
196
- raise "internal error: only symbols allowed: #{allowed_values}" unless allowed_values.all?(Symbol)
197
209
  self.class.get_from_list(@unprocessed_cmd_line_arguments.shift, descr, allowed_values)
198
- else
199
- raise 'Internal error: expected: must be single, multiple, or value array'
210
+ else error_unexpected_value(expected)
200
211
  end
201
212
  elsif !default.nil? then default
202
213
  # no value provided, either get value interactively, or exception
203
214
  elsif mandatory then get_interactive(:argument, descr, expected: expected)
204
215
  end
205
- if result.is_a?(String) && type.eql?([Integer])
206
- str_result = result
207
- result = Integer(str_result, exception: false)
208
- raise Cli::BadArgument, "Invalid integer: #{str_result}" if result.nil?
216
+ if result.is_a?(String) && allowed_types.eql?(TYPE_INTEGER)
217
+ int_result = Integer(result, exception: false)
218
+ raise Cli::BadArgument, "Invalid integer: #{result}" if int_result.nil?
219
+ result = int_result
209
220
  end
210
221
  Log.log.debug{"#{descr}=#{result}"}
211
- result = aliases[result] if !aliases.nil? && aliases.key?(result)
212
- self.class.validate_type(:argument, descr, result, type) unless result.nil? && !mandatory
222
+ result = aliases[result] if aliases&.key?(result)
223
+ self.class.validate_type(:argument, descr, result, allowed_types) unless result.nil? && !mandatory
213
224
  return result
214
225
  end
215
226
 
@@ -221,7 +232,7 @@ module Aspera
221
232
  # @param mandatory [Boolean] if true, raise error if option not set
222
233
  def get_option(option_symbol, mandatory: false, default: nil)
223
234
  attributes = @declared_options[option_symbol]
224
- raise "INTERNAL ERROR: option not declared: #{option_symbol}" unless attributes
235
+ assert(attributes){"option not declared: #{option_symbol}"}
225
236
  result = nil
226
237
  case attributes[:read_write]
227
238
  when :accessor
@@ -284,12 +295,10 @@ module Aspera
284
295
  # @param types [Class, Array] accepted value type(s)
285
296
  # @param block [Proc] block to execute when option is found
286
297
  def declare(option_symbol, description, handler: nil, default: nil, values: nil, short: nil, coerce: nil, types: nil, deprecation: nil, &block)
287
- raise "INTERNAL ERROR: #{option_symbol} already declared" if @declared_options.key?(option_symbol)
288
- # raise "INTERNAL ERROR: #{option_symbol} clash with another option" if
289
- # @declared_options.keys.map(&:to_s).any?{|k|k.start_with?(option_symbol.to_s) || option_symbol.to_s.start_with?(k)}
290
- raise "INTERNAL ERROR: #{option_symbol} ends with dot" unless description[-1] != '.'
291
- raise "INTERNAL ERROR: #{option_symbol} description does not start with capital" unless description[0] == description[0].upcase
292
- raise "INTERNAL ERROR: #{option_symbol} shall use :types" if ['hash', 'extended value'].any?{|s|description.downcase.include?(s) }
298
+ assert(!@declared_options.key?(option_symbol)){"#{option_symbol} already declared"}
299
+ assert(description[-1] != '.'){"#{option_symbol} ends with dot"}
300
+ assert(description[0] == description[0].upcase){"#{option_symbol} description does not start with capital"}
301
+ assert(!['hash', 'extended value'].any?{|s|description.downcase.include?(s) }){"#{option_symbol} shall use :types"}
293
302
  opt = @declared_options[option_symbol] = {
294
303
  read_write: handler.nil? ? :value : :accessor,
295
304
  # by default passwords and secrets are sensitive, else specify when declaring the option
@@ -297,7 +306,7 @@ module Aspera
297
306
  }
298
307
  if !types.nil?
299
308
  types = [types] unless types.is_a?(Array)
300
- raise "INTERNAL ERROR: types must be Array of Class: #{types}" unless types.all?(Class)
309
+ assert(types.all?(Class)){"types must be Array of Class: #{types}"}
301
310
  opt[:types] = types
302
311
  description = "#{description} (#{types.map(&:name).join(', ')})"
303
312
  end
@@ -307,8 +316,8 @@ module Aspera
307
316
  end
308
317
  Log.log.debug{"declare: #{option_symbol}: #{opt[:read_write]}".green}
309
318
  if opt[:read_write].eql?(:accessor)
310
- raise 'internal error' unless handler.is_a?(Hash)
311
- raise 'internal error' unless handler.keys.sort.eql?(%i[m o])
319
+ assert_type(handler, Hash)
320
+ assert(handler.keys.sort.eql?(%i[m o]))
312
321
  Log.log.debug{"set attr obj #{option_symbol} (#{handler[:o]},#{handler[:m]})"}
313
322
  opt[:accessor] = AttrAccessor.new(handler[:o], handler[:m], option_symbol)
314
323
  end
@@ -347,11 +356,11 @@ module Aspera
347
356
  set_option(option_symbol, time_string, SOURCE_USER)
348
357
  end
349
358
  when :none
350
- raise "internal error: missing block for #{option_symbol}" if block.nil?
359
+ assert(!block.nil?){"missing block for #{option_symbol}"}
351
360
  on_args.push(symbol_to_option(option_symbol, nil))
352
361
  on_args.push("-#{short}") if short.is_a?(String)
353
362
  @parser.on(*on_args, &block)
354
- else raise "internal error: Unknown type for values: #{values} / #{values.class}"
363
+ else error_unexpected_value(values)
355
364
  end
356
365
  Log.log.debug{"on_args=#{on_args}"}
357
366
  end
@@ -359,8 +368,8 @@ module Aspera
359
368
  # Adds each of the keys of specified hash as an option
360
369
  # @param preset_hash [Hash] hash of options to add
361
370
  def add_option_preset(preset_hash, op: :push)
371
+ assert_type(preset_hash, Hash)
362
372
  Log.log.debug{"add_option_preset=#{preset_hash}"}
363
- raise "internal error: default expects Hash: #{preset_hash.class}" unless preset_hash.is_a?(Hash)
364
373
  # incremental override
365
374
  preset_hash.each{|k, v|@unprocessed_defaults.send(op, [k.to_sym, v])}
366
375
  end
@@ -460,8 +469,8 @@ module Aspera
460
469
 
461
470
  def get_interactive(type, descr, expected: :single)
462
471
  if !@ask_missing_mandatory
463
- raise Cli::BadArgument, self.class.bad_arg_message_multi("missing: #{descr}", expected) if expected.is_a?(Array)
464
- raise Cli::BadArgument, "missing argument (#{expected}): #{descr}"
472
+ raise Cli::BadArgument, "missing argument (#{expected}): #{descr}" unless expected.is_a?(Array)
473
+ self.class.multi_choice_assert(false,"missing: #{descr}", expected)
465
474
  end
466
475
  result = nil
467
476
  sensitive = type.eql?(:option) && @declared_options[descr.to_sym].is_a?(Hash) && @declared_options[descr.to_sym][:sensitive]
@@ -1,6 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require 'aspera/cli/extended_value'
4
+ require 'aspera/assert'
4
5
 
5
6
  module Aspera
6
7
  module Cli
@@ -33,11 +34,11 @@ module Aspera
33
34
  end
34
35
 
35
36
  def initialize(env)
36
- raise 'env must be Hash' unless env.is_a?(Hash)
37
+ assert_type(env, Hash)
37
38
  @agents = env
38
39
  # check presence in descendant of mandatory method and constant
39
- raise StandardError, "Missing method 'execute_action' in #{self.class}" unless respond_to?(:execute_action)
40
- raise StandardError, 'ACTIONS shall be redefined by subclass' unless self.class.constants.include?(:ACTIONS)
40
+ assert(respond_to?(:execute_action)){"Missing method 'execute_action' in #{self.class}"}
41
+ assert(self.class.constants.include?(:ACTIONS)){'ACTIONS shall be redefined by subclass'}
41
42
  # manual header for all plugins
42
43
  options.parser.separator('')
43
44
  options.parser.separator("COMMAND: #{self.class.name.split('::').last.downcase}")
@@ -75,6 +76,7 @@ module Aspera
75
76
  # @param id_result [String] key in result hash to use as identifier
76
77
  # @param fields [Array] fields to display
77
78
  def do_bulk_operation(command:, descr:, values: Hash, id_result: 'id', fields: :default)
79
+ assert(block_given?){'missing block'}
78
80
  is_bulk = options.get_option(:bulk)
79
81
  case values
80
82
  when :identifier
@@ -82,7 +84,6 @@ module Aspera
82
84
  when Class
83
85
  values = value_create_modify(command: command, type: values, bulk: is_bulk)
84
86
  end
85
- raise 'Internal error: missing block' unless block_given?
86
87
  # if not bulk, there is a single value
87
88
  params = is_bulk ? values : [values]
88
89
  Log.log.warn('Empty list given for bulk operation') if params.empty?
@@ -147,6 +148,7 @@ module Aspera
147
148
  return {type: :single_object, data: rest_api.read(one_res_path)[:data], fields: display_fields}
148
149
  when :list
149
150
  resp = rest_api.read(res_class_path, old_query_read_delete)
151
+ return Main.result_empty if resp[:http].code == '204'
150
152
  data = resp[:data]
151
153
  # TODO: not generic : which application is this for ?
152
154
  if resp[:http]['Content-Type'].start_with?('application/vnd.api+json')
@@ -180,10 +182,10 @@ module Aspera
180
182
  end
181
183
 
182
184
  # implement generic rest operations on given resource path
183
- def entity_action(rest_api, res_class_path, **opts)
185
+ def entity_action(rest_api, res_class_path, **opts, &block)
184
186
  # res_name=res_class_path.gsub(%r{^.*/},'').gsub(%r{s$},'').gsub('_',' ')
185
187
  command = options.get_next_command(ALL_OPS)
186
- return entity_command(command, rest_api, res_class_path, **opts)
188
+ return entity_command(command, rest_api, res_class_path, **opts, &block)
187
189
  end
188
190
 
189
191
  # query parameters in URL suitable for REST list/GET and delete/DELETE
@@ -223,22 +225,27 @@ module Aspera
223
225
 
224
226
  # Retrieves an extended value from command line, used for creation or modification of entities
225
227
  # @param command [Symbol] command name for error message
226
- # @param type [Class] expected type of value, either a Class, an Array of Class, or :bulk_hash
228
+ # @param type [Class] expected type of value, either a Class, an Array of Class
229
+ # @param bulk [Boolean] if true, value must be an Array of <type>
227
230
  # @param default [Object] default value if not provided
228
231
  # TODO: when deprecation of `value` is completed: remove line with :value
229
- def value_create_modify(command:, type: Hash, bulk: false, default: nil)
232
+ def value_create_modify(command:, description: nil, type: Hash, bulk: false, default: nil)
230
233
  value = options.get_option(:value)
231
234
  Log.log.warn("option `value` is deprecated. Use positional parameter for #{command}") unless value.nil?
232
- value = options.get_next_argument("parameters for #{command}", mandatory: default.nil?) if value.nil?
235
+ value = options.get_next_argument(
236
+ "parameters for #{command}#{description.nil? ? '' : " (#{description})"}", mandatory: default.nil?,
237
+ type: bulk ? Array : type) if value.nil?
233
238
  value = default if value.nil?
234
239
  unless type.nil?
235
240
  type = [type] unless type.is_a?(Array)
236
- raise "Internal error, check types must be a Class, not #{type.map(&:class).join(',')}" unless type.all?(Class)
241
+ assert(type.all?(Class)){"check types must be a Class, not #{type.map(&:class).join(',')}"}
237
242
  if bulk
238
- raise Cli::BadArgument, "Value must be an Array of #{type.join(',')}" unless value.is_a?(Array)
239
- raise Cli::BadArgument, "Value must be a #{type.join(',')}, not #{value.map{|i| i.class.name}.uniq.join(',')}" unless value.all?{|v|type.include?(v.class)}
243
+ assert_type(value, Array, exception_class: Cli::BadArgument)
244
+ value.each do |v|
245
+ assert_values(v.class, type, exception_class: Cli::BadArgument)
246
+ end
240
247
  else
241
- raise Cli::BadArgument, "Value must be a #{type.join(',')}, not #{value.class.name}" unless type.include?(value.class)
248
+ assert_values(value.class, type, exception_class: Cli::BadArgument)
242
249
  end
243
250
  end
244
251
  return value