aspera-cli 4.11.0 → 4.12.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (67) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/BUGS.md +0 -1
  4. data/CHANGELOG.md +71 -52
  5. data/CONTRIBUTING.md +31 -6
  6. data/README.md +404 -259
  7. data/bin/asession +2 -2
  8. data/docs/test_env.conf +1 -0
  9. data/examples/aoc.rb +2 -2
  10. data/examples/dascli +11 -11
  11. data/examples/faspex4.rb +7 -7
  12. data/examples/node.rb +1 -1
  13. data/examples/proxy.pac +2 -2
  14. data/examples/server.rb +3 -3
  15. data/lib/aspera/aoc.rb +105 -40
  16. data/lib/aspera/cli/extended_value.rb +4 -4
  17. data/lib/aspera/cli/{formater.rb → formatter.rb} +7 -7
  18. data/lib/aspera/cli/listener/progress.rb +1 -1
  19. data/lib/aspera/cli/listener/progress_multi.rb +2 -2
  20. data/lib/aspera/cli/main.rb +18 -18
  21. data/lib/aspera/cli/manager.rb +5 -5
  22. data/lib/aspera/cli/plugin.rb +23 -20
  23. data/lib/aspera/cli/plugins/aoc.rb +75 -112
  24. data/lib/aspera/cli/plugins/ats.rb +6 -6
  25. data/lib/aspera/cli/plugins/config.rb +84 -83
  26. data/lib/aspera/cli/plugins/cos.rb +1 -1
  27. data/lib/aspera/cli/plugins/faspex.rb +38 -38
  28. data/lib/aspera/cli/plugins/faspex5.rb +187 -43
  29. data/lib/aspera/cli/plugins/node.rb +30 -37
  30. data/lib/aspera/cli/plugins/orchestrator.rb +7 -4
  31. data/lib/aspera/cli/plugins/preview.rb +10 -9
  32. data/lib/aspera/cli/plugins/server.rb +1 -1
  33. data/lib/aspera/cli/plugins/shares.rb +67 -43
  34. data/lib/aspera/cli/transfer_agent.rb +16 -16
  35. data/lib/aspera/cli/version.rb +2 -1
  36. data/lib/aspera/command_line_builder.rb +70 -66
  37. data/lib/aspera/cos_node.rb +9 -9
  38. data/lib/aspera/fasp/agent_base.rb +3 -1
  39. data/lib/aspera/fasp/agent_connect.rb +23 -23
  40. data/lib/aspera/fasp/agent_direct.rb +13 -14
  41. data/lib/aspera/fasp/agent_httpgw.rb +20 -19
  42. data/lib/aspera/fasp/agent_node.rb +13 -15
  43. data/lib/aspera/fasp/agent_trsdk.rb +1 -1
  44. data/lib/aspera/fasp/installation.rb +5 -5
  45. data/lib/aspera/fasp/listener.rb +1 -1
  46. data/lib/aspera/fasp/parameters.rb +49 -41
  47. data/lib/aspera/fasp/parameters.yaml +311 -212
  48. data/lib/aspera/fasp/resume_policy.rb +2 -2
  49. data/lib/aspera/fasp/transfer_spec.rb +0 -13
  50. data/lib/aspera/faspex_gw.rb +80 -161
  51. data/lib/aspera/faspex_postproc.rb +77 -0
  52. data/lib/aspera/log.rb +7 -7
  53. data/lib/aspera/nagios.rb +6 -6
  54. data/lib/aspera/node.rb +24 -19
  55. data/lib/aspera/oauth.rb +50 -47
  56. data/lib/aspera/proxy_auto_config.js +22 -22
  57. data/lib/aspera/proxy_auto_config.rb +3 -3
  58. data/lib/aspera/rest.rb +12 -10
  59. data/lib/aspera/rest_error_analyzer.rb +5 -5
  60. data/lib/aspera/secret_hider.rb +4 -3
  61. data/lib/aspera/ssh.rb +4 -4
  62. data/lib/aspera/sync.rb +37 -36
  63. data/lib/aspera/web_auth.rb +7 -59
  64. data/lib/aspera/web_server_simple.rb +76 -0
  65. data.tar.gz.sig +0 -0
  66. metadata +6 -4
  67. metadata.gz.sig +0 -0
@@ -1,7 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require 'aspera/cli/manager'
4
- require 'aspera/cli/formater'
4
+ require 'aspera/cli/formatter'
5
5
  require 'aspera/cli/plugins/config'
6
6
  require 'aspera/cli/extended_value'
7
7
  require 'aspera/cli/transfer_agent'
@@ -81,13 +81,13 @@ module Aspera
81
81
 
82
82
  def option_ui=(value); OpenApplication.instance.url_method = value; end
83
83
 
84
- # called everytime a new REST HTTP session is opened
84
+ # called every time a new REST HTTP session is opened
85
85
  # @param http [Net::HTTP] the newly created http session object
86
86
  def http_parameters=(http)
87
87
  if @option_insecure
88
88
  url = http.inspect.gsub(/^[^ ]* /, 'https://').gsub(/ [^ ]*$/, '')
89
89
  if !@ssl_warned_urls.include?(url)
90
- @plugin_env[:formater].display_message(:error, "#{WARNING_FLASH} ignoring certificate for: #{url}. Do not use unsafe certificates in production.")
90
+ @formatter.display_message(:error, "#{WARNING_FLASH} ignoring certificate for: #{url}. Do not use unsafe certificates in production.")
91
91
  @ssl_warned_urls.push(url)
92
92
  end
93
93
  http.verify_mode = SELF_SIGNED_CERT
@@ -113,7 +113,7 @@ module Aspera
113
113
  early_debug_setup(argv)
114
114
  # compare $0 with expected name
115
115
  current_prog_name = File.basename($PROGRAM_NAME)
116
- @plugin_env[:formater].display_message(:error, "#{'WARNING'.bg_red.blink.gray} Please use '#{PROGRAM_NAME}' instead of '#{current_prog_name}'") \
116
+ @formatter.display_message(:error, "#{'WARNING'.bg_red.blink.gray} Please use '#{PROGRAM_NAME}' instead of '#{current_prog_name}'") \
117
117
  unless current_prog_name.eql?(PROGRAM_NAME)
118
118
  @option_help = false
119
119
  @bash_completion = false
@@ -128,7 +128,7 @@ module Aspera
128
128
  # give command line arguments to option manager
129
129
  @plugin_env[:options] = @opt_mgr = Manager.new(PROGRAM_NAME, argv: argv)
130
130
  # formatter adds options
131
- @plugin_env[:formater] = Formater.new(@plugin_env[:options])
131
+ @formatter = @plugin_env[:formatter] = Formatter.new(@plugin_env[:options])
132
132
  Rest.user_agent = PROGRAM_NAME
133
133
  Rest.session_cb = lambda{|http|self.http_parameters = http}
134
134
  # declare and parse global options
@@ -191,7 +191,7 @@ module Aspera
191
191
  @opt_mgr.add_opt_switch(:bash_comp, 'generate bash completion for command') { @bash_completion = true }
192
192
  @opt_mgr.add_opt_switch(:show_config, 'Display parameters used for the provided action.') { @option_show_config = true }
193
193
  @opt_mgr.add_opt_switch(:rest_debug, '-r', 'more debug for HTTP calls') { @option_rest_debug = true }
194
- @opt_mgr.add_opt_switch(:version, '-v', 'display version') { @plugin_env[:formater].display_message(:data, Aspera::Cli::VERSION); Process.exit(0) } # rubocop:disable Style/Semicolon, Layout/LineLength
194
+ @opt_mgr.add_opt_switch(:version, '-v', 'display version') { @formatter.display_message(:data, Aspera::Cli::VERSION); Process.exit(0) } # rubocop:disable Style/Semicolon, Layout/LineLength
195
195
  @opt_mgr.add_opt_switch(:warnings, '-w', 'check for language warnings') { $VERBOSE = true }
196
196
  # handler must be set before declaration
197
197
  @opt_mgr.set_obj_attr(:log_level, Log.instance, :level)
@@ -203,7 +203,7 @@ module Aspera
203
203
  @opt_mgr.set_obj_attr(:cache_tokens, self, :option_cache_tokens)
204
204
  @opt_mgr.add_opt_list(:ui, OpenApplication.user_interfaces, 'method to start browser')
205
205
  @opt_mgr.add_opt_list(:log_level, Log.levels, 'Log level')
206
- @opt_mgr.add_opt_list(:logger, Log.logtypes, 'log method')
206
+ @opt_mgr.add_opt_list(:logger, Log::LOG_TYPES, 'logging method')
207
207
  @opt_mgr.add_opt_simple(:lock_port, 'prevent dual execution of a command, e.g. in cron')
208
208
  @opt_mgr.add_opt_simple(:http_options, 'options for http socket (extended value)')
209
209
  @opt_mgr.add_opt_boolean(:insecure, 'do not validate HTTPS certificate')
@@ -223,7 +223,7 @@ module Aspera
223
223
  env ||= @plugin_env
224
224
  Log.log.debug{"get_plugin_instance_with_options(#{plugin_name_sym})"}
225
225
  require @plugin_env[:config].plugins[plugin_name_sym][:require_stanza]
226
- # load default params only if no param already loaded before plugin instanciation
226
+ # load default params only if no param already loaded before plugin instantiation
227
227
  env[:config].add_plugin_default_preset(plugin_name_sym)
228
228
  command_plugin = Plugins::Config.plugin_class(plugin_name_sym).new(env)
229
229
  Log.log.debug{"got #{command_plugin.class}"}
@@ -243,7 +243,7 @@ module Aspera
243
243
  def exit_with_usage(all_plugins)
244
244
  Log.log.debug('exit_with_usage'.bg_red)
245
245
  # display main plugin options
246
- @plugin_env[:formater].display_message(:error, @opt_mgr.parser)
246
+ @formatter.display_message(:error, @opt_mgr.parser)
247
247
  if all_plugins
248
248
  # list plugins that have a "require" field, i.e. all but main plugin
249
249
  @plugin_env[:config].plugins.each_key do |plugin_name_sym|
@@ -255,7 +255,7 @@ module Aspera
255
255
  plugin_env[:options].parser.banner = '' # remove default banner
256
256
  get_plugin_instance_with_options(plugin_name_sym, plugin_env)
257
257
  # display generated help for plugin options
258
- @plugin_env[:formater].display_message(:error, plugin_env[:options].parser.help)
258
+ @formatter.display_message(:error, plugin_env[:options].parser.help)
259
259
  end
260
260
  end
261
261
  Process.exit(0)
@@ -300,7 +300,7 @@ module Aspera
300
300
  end
301
301
  # command will not be executed, but we need manual
302
302
  @opt_mgr.fail_on_missing_mandatory = false if @option_help || @option_show_config
303
- # main plugin is not dynamically instanciated
303
+ # main plugin is not dynamically instantiated
304
304
  case command_sym
305
305
  when :help
306
306
  exit_with_usage(true)
@@ -315,7 +315,7 @@ module Aspera
315
315
  # help requested for current plugin
316
316
  exit_with_usage(false) if @option_help
317
317
  if @option_show_config
318
- @plugin_env[:formater].display_results({type: :single_object, data: @opt_mgr.declared_options(only_defined: true)})
318
+ @formatter.display_results({type: :single_object, data: @opt_mgr.declared_options(only_defined: true)})
319
319
  execute_command = false
320
320
  end
321
321
  # locking for single execution (only after "per plugin" option, in case lock port is there)
@@ -331,7 +331,7 @@ module Aspera
331
331
  end
332
332
  end
333
333
  # execute and display (if not exclusive execution)
334
- @plugin_env[:formater].display_results(command_plugin.execute_action) if execute_command
334
+ @formatter.display_results(command_plugin.execute_action) if execute_command
335
335
  # finish
336
336
  @plugin_env[:transfer].shutdown
337
337
  rescue Net::SSH::AuthenticationFailed => e; exception_info = {e: e, t: 'SSH', security: true}
@@ -349,17 +349,17 @@ module Aspera
349
349
  # 1- processing of error condition
350
350
  unless exception_info.nil?
351
351
  Log.log.warn(exception_info[:e].message) if Aspera::Log.instance.logger_type.eql?(:syslog) && exception_info[:security]
352
- @plugin_env[:formater].display_message(:error, "#{ERROR_FLASH} #{exception_info[:t]}: #{exception_info[:e].message}")
353
- @plugin_env[:formater].display_message(:error, 'Use option -h to get help.') if exception_info[:usage]
352
+ @formatter.display_message(:error, "#{ERROR_FLASH} #{exception_info[:t]}: #{exception_info[:e].message}")
353
+ @formatter.display_message(:error, 'Use option -h to get help.') if exception_info[:usage]
354
354
  if exception_info[:e].is_a?(Fasp::Error) && exception_info[:e].message.eql?('Remote host is not who we expected')
355
- @plugin_env[:formater].display_message(:error, "For this specific error, refer to:\n"\
355
+ @formatter.display_message(:error, "For this specific error, refer to:\n"\
356
356
  "#{SRC_URL}#error-remote-host-is-not-who-we-expected\nAdd this to arguments:\n--ts=@json:'{\"sshfp\":null}'")
357
357
  end
358
358
  end
359
359
  # 2- processing of command not processed (due to exception or bad command line)
360
360
  if execute_command || @option_show_config
361
361
  @opt_mgr.final_errors.each do |msg|
362
- @plugin_env[:formater].display_message(:error, "#{ERROR_FLASH} Argument: #{msg}")
362
+ @formatter.display_message(:error, "#{ERROR_FLASH} Argument: #{msg}")
363
363
  # add code as exception if there is not already an error
364
364
  exception_info = {e: Exception.new(msg), t: 'UnusedArg'} if exception_info.nil?
365
365
  end
@@ -369,7 +369,7 @@ module Aspera
369
369
  # show stack trace in debug mode
370
370
  raise exception_info[:e] if Log.instance.level.eql?(:debug)
371
371
  # else give hint and exit
372
- @plugin_env[:formater].display_message(:error, 'Use --log-level=debug to get more details.') if exception_info[:debug]
372
+ @formatter.display_message(:error, 'Use --log-level=debug to get more details.') if exception_info[:debug]
373
373
  Process.exit(1)
374
374
  end
375
375
  return nil
@@ -47,7 +47,7 @@ module Aspera
47
47
  BOOLEAN_SIMPLE = %i[no yes].freeze
48
48
  FALSE_VALUES = [BOOLEAN_SIMPLE.first, false].freeze
49
49
  TRUE_VALUES = [BOOLEAN_SIMPLE.last, true].freeze
50
- BOOLEAN_VALUES = [].concat(TRUE_VALUES, FALSE_VALUES).freeze
50
+ BOOLEAN_VALUES = [TRUE_VALUES, FALSE_VALUES].flatten.freeze
51
51
 
52
52
  # option name separator on command line
53
53
  OPTION_SEP_LINE = '-'
@@ -73,7 +73,7 @@ module Aspera
73
73
  return matching_exact.first if matching_exact.length == 1
74
74
  matching = allowed_values.select{|i| i.to_s.start_with?(shortval)}
75
75
  raise CliBadArgument, bad_arg_message_multi("unknown value for #{descr}: #{shortval}", allowed_values) if matching.empty?
76
- raise CliBadArgument, bad_arg_message_multi("ambigous shortcut for #{descr}: #{shortval}", matching) unless matching.length.eql?(1)
76
+ raise CliBadArgument, bad_arg_message_multi("ambiguous shortcut for #{descr}: #{shortval}", matching) unless matching.length.eql?(1)
77
77
  return enum_to_bool(matching.first) if allowed_values.eql?(BOOLEAN_VALUES)
78
78
  return matching.first
79
79
  end
@@ -104,10 +104,10 @@ module Aspera
104
104
  # those must be set before parse, parse consumes those defined only
105
105
  @unprocessed_defaults = []
106
106
  @unprocessed_env = []
107
- # NOTE: was initially inherited but it is prefered to have specific methods
107
+ # NOTE: was initially inherited but it is preferred to have specific methods
108
108
  @parser = OptionParser.new
109
109
  @parser.program_name = program_name
110
- # options can also be provided by env vars : --param-name -> ASLMCLI_PARAM_NAME
110
+ # options can also be provided by env vars : --param-name -> ASCLI_PARAM_NAME
111
111
  env_prefix = program_name.upcase + OPTION_SEP_NAME
112
112
  ENV.each do |k, v|
113
113
  if k.start_with?(env_prefix)
@@ -186,7 +186,7 @@ module Aspera
186
186
  def declare_option(option_symbol, type)
187
187
  Log.log.debug{"declare_option: #{option_symbol}: #{type}: skip=#{@declared_options.key?(option_symbol)}".green}
188
188
  if @declared_options.key?(option_symbol)
189
- raise "INTERNAL ERROR: option #{option_symbol} already declared. only accessor can be redeclared and ignored" \
189
+ raise "INTERNAL ERROR: option #{option_symbol} already declared. only accessor can be re-declared and ignored" \
190
190
  unless @declared_options[option_symbol][:type].eql?(:accessor)
191
191
  return
192
192
  end
@@ -8,7 +8,7 @@ module Aspera
8
8
  GLOBAL_OPS = %i[create list].freeze
9
9
  # operations with id
10
10
  INSTANCE_OPS = %i[modify delete show].freeze
11
- ALL_OPS = [].concat(GLOBAL_OPS, INSTANCE_OPS).freeze
11
+ ALL_OPS = [GLOBAL_OPS, INSTANCE_OPS].flatten.freeze
12
12
  # max number of items for list command
13
13
  MAX_ITEMS = 'max'
14
14
  # max number of pages for list command
@@ -16,8 +16,6 @@ module Aspera
16
16
  # used when all resources are selected
17
17
  VAL_ALL = 'ALL'
18
18
 
19
- # AGENTS=%i[options transfer config formater persistency].freeze
20
-
21
19
  # global for inherited classes
22
20
  @@options_created = false # rubocop:disable Style/ClassVars
23
21
 
@@ -96,9 +94,10 @@ module Aspera
96
94
  # @param res_class_path [String] sub path in URL to resource relative to base url
97
95
  # @param display_fields [Array] fields to display by default
98
96
  # @param id_default [String] default identifier to use for existing entity commands (show, modify)
99
- # @param item_list_key [String] result is in a subkey of the json
97
+ # @param item_list_key [String] result is in a sub key of the json
98
+ # @param id_as_arg [String] if set, the id is provided as url argument ?<id_as_arg>=<id>
100
99
  # @return result suitable for CLI result
101
- def entity_command(command, rest_api, res_class_path, display_fields: nil, id_default: nil, item_list_key: false)
100
+ def entity_command(command, rest_api, res_class_path, display_fields: nil, id_default: nil, item_list_key: false, id_as_arg: false)
102
101
  if INSTANCE_OPS.include?(command)
103
102
  begin
104
103
  one_res_id = instance_identifier
@@ -107,13 +106,14 @@ module Aspera
107
106
  one_res_id = id_default
108
107
  end
109
108
  one_res_path = "#{res_class_path}/#{one_res_id}"
109
+ one_res_path = "#{res_class_path}?#{id_as_arg}=#{one_res_id}" if id_as_arg
110
110
  end
111
111
  # parameters mandatory for create/modify
112
112
  if %i[create modify].include?(command)
113
113
  parameters = options.get_option(:value, is_type: :mandatory)
114
114
  end
115
115
  # parameters optional for list
116
- if [:list].include?(command)
116
+ if %i[list delete].include?(command)
117
117
  parameters = options.get_option(:value)
118
118
  end
119
119
  case command
@@ -124,7 +124,7 @@ module Aspera
124
124
  end
125
125
  when :delete
126
126
  return do_bulk_operation(one_res_id, 'deleted') do |one_id|
127
- rest_api.delete("#{res_class_path}/#{one_id}")
127
+ rest_api.delete("#{res_class_path}/#{one_id}", parameters)
128
128
  {'id' => one_id}
129
129
  end
130
130
  when :show
@@ -134,6 +134,7 @@ module Aspera
134
134
  data = resp[:data]
135
135
  # TODO: not generic : which application is this for ?
136
136
  if resp[:http]['Content-Type'].start_with?('application/vnd.api+json')
137
+ Log.log.debug{'is vnd.api'}
137
138
  data = data[res_class_path]
138
139
  end
139
140
  if item_list_key
@@ -142,12 +143,19 @@ module Aspera
142
143
  if !total_count.nil?
143
144
  count_msg = "Items: #{item_list.length}/#{total_count}"
144
145
  count_msg = count_msg.bg_red unless item_list.length.eql?(total_count.to_i)
145
- self.format.display_status(count_msg)
146
+ formatter.display_status(count_msg)
146
147
  end
147
148
  data = item_list
148
149
  end
149
- return {type: :object_list, data: data, fields: display_fields} if data.empty? || data.first.is_a?(Hash)
150
- return {type: :value_list, data: data, name: 'id'}
150
+ case data
151
+ when Hash
152
+ return {type: :single_object, data: data, fields: display_fields}
153
+ when Array
154
+ return {type: :object_list, data: data, fields: display_fields} if data.empty? || data.first.is_a?(Hash)
155
+ return {type: :value_list, data: data, name: 'id'}
156
+ else
157
+ raise "An error occurred: unexpected result type for list: #{data.class}"
158
+ end
151
159
  when :modify
152
160
  property = options.get_option(:property)
153
161
  parameters = {property => parameters} unless property.nil?
@@ -168,7 +176,8 @@ module Aspera
168
176
  # query for list operation
169
177
  def option_url_query(default)
170
178
  query = options.get_option(:query)
171
- query = default if query.nil?
179
+ # dup default, as it could be frozen
180
+ query = default.dup if query.nil?
172
181
  Log.log.debug{"Query=#{query}".bg_red}
173
182
  begin
174
183
  # check it is suitable
@@ -180,15 +189,9 @@ module Aspera
180
189
  end
181
190
 
182
191
  # shortcuts helpers for plugin environment
183
- def options; return @agents[:options]; end
184
-
185
- def transfer; return @agents[:transfer]; end
186
-
187
- def config; return @agents[:config]; end
188
-
189
- def format; return @agents[:formater]; end
190
-
191
- def persistency; return @agents[:persistency]; end
192
+ %i[options transfer config formatter persistency].each do |name|
193
+ define_method(name){@agents[name]}
194
+ end
192
195
  end # Plugin
193
196
  end # Cli
194
197
  end # Aspera