aspera-cli 4.15.0 → 4.16.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (80) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/BUGS.md +29 -3
  4. data/CHANGELOG.md +292 -228
  5. data/CONTRIBUTING.md +69 -18
  6. data/README.md +1102 -952
  7. data/bin/ascli +13 -31
  8. data/bin/asession +3 -1
  9. data/examples/dascli +2 -2
  10. data/lib/aspera/aoc.rb +28 -33
  11. data/lib/aspera/ascmd.rb +3 -6
  12. data/lib/aspera/assert.rb +45 -0
  13. data/lib/aspera/cli/extended_value.rb +5 -5
  14. data/lib/aspera/cli/formatter.rb +26 -13
  15. data/lib/aspera/cli/hints.rb +4 -3
  16. data/lib/aspera/cli/main.rb +16 -3
  17. data/lib/aspera/cli/manager.rb +45 -36
  18. data/lib/aspera/cli/plugin.rb +20 -13
  19. data/lib/aspera/cli/plugins/aoc.rb +103 -73
  20. data/lib/aspera/cli/plugins/ats.rb +4 -3
  21. data/lib/aspera/cli/plugins/config.rb +114 -119
  22. data/lib/aspera/cli/plugins/cos.rb +2 -2
  23. data/lib/aspera/cli/plugins/faspex.rb +23 -19
  24. data/lib/aspera/cli/plugins/faspex5.rb +75 -43
  25. data/lib/aspera/cli/plugins/node.rb +28 -15
  26. data/lib/aspera/cli/plugins/orchestrator.rb +4 -2
  27. data/lib/aspera/cli/plugins/preview.rb +9 -7
  28. data/lib/aspera/cli/plugins/server.rb +6 -3
  29. data/lib/aspera/cli/plugins/shares.rb +30 -26
  30. data/lib/aspera/cli/sync_actions.rb +9 -9
  31. data/lib/aspera/cli/transfer_agent.rb +21 -14
  32. data/lib/aspera/cli/transfer_progress.rb +2 -3
  33. data/lib/aspera/cli/version.rb +1 -1
  34. data/lib/aspera/command_line_builder.rb +13 -11
  35. data/lib/aspera/cos_node.rb +3 -2
  36. data/lib/aspera/coverage.rb +22 -0
  37. data/lib/aspera/data_repository.rb +33 -2
  38. data/lib/aspera/environment.rb +4 -2
  39. data/lib/aspera/fasp/{agent_aspera.rb → agent_alpha.rb} +29 -39
  40. data/lib/aspera/fasp/agent_base.rb +17 -7
  41. data/lib/aspera/fasp/agent_direct.rb +88 -84
  42. data/lib/aspera/fasp/agent_httpgw.rb +4 -3
  43. data/lib/aspera/fasp/agent_node.rb +3 -2
  44. data/lib/aspera/fasp/agent_trsdk.rb +79 -37
  45. data/lib/aspera/fasp/installation.rb +51 -12
  46. data/lib/aspera/fasp/management.rb +11 -6
  47. data/lib/aspera/fasp/parameters.rb +53 -47
  48. data/lib/aspera/fasp/resume_policy.rb +7 -5
  49. data/lib/aspera/fasp/sync.rb +273 -0
  50. data/lib/aspera/fasp/transfer_spec.rb +10 -8
  51. data/lib/aspera/fasp/uri.rb +2 -2
  52. data/lib/aspera/faspex_gw.rb +11 -8
  53. data/lib/aspera/faspex_postproc.rb +6 -5
  54. data/lib/aspera/id_generator.rb +3 -1
  55. data/lib/aspera/json_rpc.rb +10 -8
  56. data/lib/aspera/keychain/encrypted_hash.rb +46 -11
  57. data/lib/aspera/keychain/macos_security.rb +15 -13
  58. data/lib/aspera/log.rb +4 -3
  59. data/lib/aspera/nagios.rb +7 -2
  60. data/lib/aspera/node.rb +17 -16
  61. data/lib/aspera/node_simulator.rb +214 -0
  62. data/lib/aspera/oauth.rb +22 -19
  63. data/lib/aspera/persistency_action_once.rb +13 -14
  64. data/lib/aspera/persistency_folder.rb +3 -2
  65. data/lib/aspera/preview/file_types.rb +53 -267
  66. data/lib/aspera/preview/generator.rb +7 -5
  67. data/lib/aspera/preview/terminal.rb +14 -5
  68. data/lib/aspera/preview/utils.rb +8 -7
  69. data/lib/aspera/proxy_auto_config.rb +6 -3
  70. data/lib/aspera/rest.rb +29 -13
  71. data/lib/aspera/rest_error_analyzer.rb +1 -0
  72. data/lib/aspera/rest_errors_aspera.rb +2 -0
  73. data/lib/aspera/secret_hider.rb +5 -2
  74. data/lib/aspera/ssh.rb +10 -8
  75. data/lib/aspera/temp_file_manager.rb +1 -1
  76. data/lib/aspera/web_server_simple.rb +2 -1
  77. data.tar.gz.sig +0 -0
  78. metadata +96 -45
  79. metadata.gz.sig +0 -0
  80. data/lib/aspera/sync.rb +0 -219
@@ -12,6 +12,8 @@ require 'aspera/persistency_action_once'
12
12
  require 'aspera/open_application'
13
13
  require 'aspera/nagios'
14
14
  require 'aspera/id_generator'
15
+ require 'aspera/log'
16
+ require 'aspera/assert'
15
17
  require 'xmlsimple'
16
18
  require 'json'
17
19
  require 'cgi'
@@ -75,10 +77,10 @@ module Aspera
75
77
  }
76
78
  end
77
79
 
78
- # extract elements from anonymous faspex link
80
+ # extract elements from faspex public link
79
81
  def get_link_data(public_url)
80
82
  public_uri = URI.parse(public_url)
81
- raise Cli::BadArgument, 'Public link does not match Faspex format' unless (m = public_uri.path.match(%r{^(.*)/(external.*)$}))
83
+ assert((m = public_uri.path.match(%r{^(.*)/(external.*)$})), exception_class: Cli::BadArgument){'Public link does not match Faspex format'}
82
84
  base = m[1]
83
85
  subpath = m[2]
84
86
  port_add = public_uri.port.eql?(public_uri.default_port) ? '' : ":#{public_uri.port}"
@@ -91,16 +93,13 @@ module Aspera
91
93
  return result
92
94
  end
93
95
 
94
- # get Fasp::Uri::SCHEME URI from entry in xml, and fix problems..
96
+ # get Fasp::Uri::SCHEME URI from entry in xml, and fix problems.
95
97
  def get_fasp_uri_from_entry(entry, raise_no_link: true)
96
98
  unless entry.key?('link')
97
99
  raise Cli::BadArgument, 'package has no link (deleted?)' if raise_no_link
98
100
  return nil
99
101
  end
100
102
  result = entry['link'].find{|e| e['rel'].eql?('package')}['href']
101
- # tags in the end of URL is not well % encoded... there are "=" that should be %3D
102
- # TODO: enter ticket to Faspex ?
103
- # ##XXif m=result.match(/(=+)$/);result.gsub!(/=+$/,"#{"%3D"*m[1].length}");end
104
103
  return result
105
104
  end
106
105
 
@@ -162,9 +161,9 @@ module Aspera
162
161
  max_pages = nil
163
162
  result = []
164
163
  if !mailbox_query.nil?
165
- raise 'query: must be Hash or nil' unless mailbox_query.is_a?(Hash)
166
- raise "query: supported params: #{ATOM_EXT_PARAMS}" unless (mailbox_query.keys - ATOM_EXT_PARAMS).empty?
167
- raise 'query: startIndex and page are exclusive' if mailbox_query.key?('startIndex') && mailbox_query.key?('page')
164
+ assert_type(mailbox_query, Hash){'query'}
165
+ assert((mailbox_query.keys - ATOM_EXT_PARAMS).empty?){"query: supported params: #{ATOM_EXT_PARAMS}"}
166
+ assert(!(mailbox_query.key?('startIndex') && mailbox_query.key?('page'))){'query: startIndex and page are exclusive'}
168
167
  max_items = mailbox_query[MAX_ITEMS]
169
168
  mailbox_query.delete(MAX_ITEMS)
170
169
  max_pages = mailbox_query[MAX_PAGES]
@@ -271,7 +270,7 @@ module Aspera
271
270
  end
272
271
  return nagios.result
273
272
  when :package
274
- command_pkg = options.get_next_command(%i[send recv list show])
273
+ command_pkg = options.get_next_command(%i[send receive list show], aliases: {recv: :receive})
275
274
  case command_pkg
276
275
  when :show
277
276
  delivery_id = instance_identifier
@@ -284,7 +283,7 @@ module Aspera
284
283
  }
285
284
  when :send
286
285
  delivery_info = options.get_option(:delivery_info, mandatory: true)
287
- raise Cli::BadArgument, 'delivery_info must be hash, refer to doc' unless delivery_info.is_a?(Hash)
286
+ assert_type(delivery_info, Hash, exception_class: Cli::BadArgument){'delivery_info'}
288
287
  # actual parameter to faspex API
289
288
  package_create_params = {'delivery' => delivery_info}
290
289
  public_link_url = options.get_option(:link)
@@ -294,7 +293,7 @@ module Aspera
294
293
  first_source = delivery_info['sources'].first
295
294
  first_source['paths'].push(*transfer.source_list)
296
295
  source_id = instance_identifier(as_option: :remote_source) do |field, value|
297
- raise Cli::BadArgument, 'only name as selector, or give id' unless field.eql?('name')
296
+ assert(field.eql?('name'), exception_class: Cli::BadArgument){'only name as selector, or give id'}
298
297
  source_list = api_v3.call({operation: 'GET', subpath: 'source_shares', headers: {'Accept' => 'application/json'}})[:data]['items']
299
298
  self.class.get_source_id_by_name(value, source_list)
300
299
  end
@@ -318,7 +317,7 @@ module Aspera
318
317
  end
319
318
  # Log.log.debug{Log.dump('transfer_spec',transfer_spec)}
320
319
  return Main.result_transfer(transfer.start(transfer_spec))
321
- when :recv
320
+ when :receive
322
321
  link_url = options.get_option(:link)
323
322
  # list of faspex ID/URI to download
324
323
  pkg_id_uri = nil
@@ -341,8 +340,13 @@ module Aspera
341
340
  delivery_id = instance_identifier
342
341
  raise 'empty id' if delivery_id.empty?
343
342
  recipient = options.get_option(:recipient)
344
- if ExtendedValue::ALL.eql?(delivery_id)
343
+ if delivery_id.eql?(ExtendedValue::ALL)
345
344
  pkg_id_uri = mailbox_filtered_entries.map{|i|{id: i[PACKAGE_MATCH_FIELD], uri: self.class.get_fasp_uri_from_entry(i, raise_no_link: false)}}
345
+ elsif delivery_id.eql?(ExtendedValue::INIT)
346
+ assert(skip_ids_persistency){'Only with option once_only'}
347
+ skip_ids_persistency.data.clear.concat(mailbox_filtered_entries.map{|i|{id: i[PACKAGE_MATCH_FIELD]}})
348
+ skip_ids_persistency.save
349
+ return Main.result_status("Initialized skip for #{skip_ids_persistency.data.count} package(s)")
346
350
  elsif !recipient.nil? && recipient.start_with?('*')
347
351
  found_package_link = mailbox_filtered_entries(stop_at_id: delivery_id).find{|p|p[PACKAGE_MATCH_FIELD].eql?(delivery_id)}['link'].first['href']
348
352
  raise "Not Found. Dropbox and Workgroup packages can use the link option with #{Fasp::Uri::SCHEME}" if found_package_link.nil?
@@ -423,17 +427,17 @@ module Aspera
423
427
  return {type: :object_list, data: source_list}
424
428
  else # :info :node
425
429
  source_id = instance_identifier do |field, value|
426
- raise Cli::BadArgument, 'only name as selector, or give id' unless field.eql?('name')
430
+ assert(field.eql?('name'), exception_class: Cli::BadArgument){'only name as selector, or give id'}
427
431
  self.class.get_source_id_by_name(value, source_list)
428
432
  end.to_i
429
433
  source_name = source_list.find{|i|i['id'].eql?(source_id)}['name']
430
434
  source_hash = options.get_option(:storage, mandatory: true)
431
435
  # check value of option
432
- raise Cli::Error, 'storage option must be a Hash' unless source_hash.is_a?(Hash)
436
+ assert_type(source_hash, Hash, exception_class: Cli::Error){'storage option'}
433
437
  source_hash.each do |name, storage|
434
- raise Cli::Error, "storage '#{name}' must be a Hash" unless storage.is_a?(Hash)
438
+ assert_type(storage, Hash, exception_class: Cli::Error){"storage '#{name}'"}
435
439
  [KEY_NODE, KEY_PATH].each do |key|
436
- raise Cli::Error, "storage '#{name}' must have a '#{key}'" unless storage.key?(key)
440
+ assert(storage.key?(key), exception_class: Cli::Error){"storage '#{name}' must have a '#{key}'"}
437
441
  end
438
442
  end
439
443
  if !source_hash.key?(source_name)
@@ -446,8 +450,8 @@ module Aspera
446
450
  return {data: source_info, type: :single_object}
447
451
  when :node
448
452
  node_config = ExtendedValue.instance.evaluate(source_info[KEY_NODE])
449
- raise Cli::Error, "bad type for: \"#{source_info[KEY_NODE]}\"" unless node_config.is_a?(Hash)
450
453
  Log.log.debug{"node=#{node_config}"}
454
+ assert_type(node_config, Hash, exception_class: Cli::Error){source_info[KEY_NODE]}
451
455
  api_node = Rest.new({
452
456
  base_url: node_config['url'],
453
457
  auth: {
@@ -3,10 +3,12 @@
3
3
  # spellchecker: ignore workgroups mypackages passcode
4
4
 
5
5
  require 'aspera/cli/basic_auth_plugin'
6
+ require 'aspera/cli/extended_value'
6
7
  require 'aspera/persistency_action_once'
7
8
  require 'aspera/id_generator'
8
9
  require 'aspera/nagios'
9
10
  require 'aspera/environment'
11
+ require 'aspera/assert'
10
12
  require 'securerandom'
11
13
  require 'tty-spinner'
12
14
 
@@ -19,17 +21,17 @@ module Aspera
19
21
  API_DETECT = 'api/v5/configuration/ping'
20
22
  # list of supported mailbox types (to list packages)
21
23
  API_LIST_MAILBOX_TYPES = %w[inbox inbox_history inbox_all inbox_all_history outbox outbox_history pending pending_history all].freeze
22
- PACKAGE_ALL_INIT = 'INIT'
23
24
  PACKAGE_SEND_FROM_REMOTE_SOURCE = 'remote_source'
24
25
  # Faspex API v5: get transfer spec for connect
25
26
  TRANSFER_CONNECT = 'connect'
26
27
  ADMIN_RESOURCES = %i[
27
28
  accounts contacts jobs workgroups shared_inboxes nodes oauth_clients registrations saml_configs
28
- metadata_profiles email_notifications
29
+ metadata_profiles email_notifications alternate_addresses
29
30
  ].freeze
30
31
  JOB_RUNNING = %w[queued working].freeze
31
32
  STANDARD_PATH = '/aspera/faspex'
32
- private_constant(*%i[JOB_RUNNING RECIPIENT_TYPES PACKAGE_TERMINATED API_DETECT API_LIST_MAILBOX_TYPES PACKAGE_SEND_FROM_REMOTE_SOURCE])
33
+ PER_PAGE_DEFAULT = 100
34
+ private_constant(*%i[JOB_RUNNING RECIPIENT_TYPES PACKAGE_TERMINATED API_DETECT API_LIST_MAILBOX_TYPES PACKAGE_SEND_FROM_REMOTE_SOURCE PER_PAGE_DEFAULT])
33
35
  class << self
34
36
  def application_name
35
37
  'Faspex'
@@ -102,7 +104,7 @@ module Aspera
102
104
  options.declare(:auth, 'OAuth type of authentication', values: %i[boot].concat(Oauth::STD_AUTH_TYPES), default: :jwt)
103
105
  options.declare(:private_key, 'OAuth JWT RSA private key PEM value (prefix file path with @file:)')
104
106
  options.declare(:passphrase, 'OAuth JWT RSA private key passphrase')
105
- options.declare(:box, "Package inbox, either shared inbox name or one of #{API_LIST_MAILBOX_TYPES} or #{ExtendedValue::ALL}", default: 'inbox')
107
+ options.declare(:box, "Package inbox, either shared inbox name or one of: #{API_LIST_MAILBOX_TYPES.join(', ')} or #{ExtendedValue::ALL}", default: 'inbox')
106
108
  options.declare(:shared_folder, 'Send package with files from shared folder')
107
109
  options.declare(:group_type, 'Type of shared box', values: %i[shared_inboxes workgroups], default: :shared_inboxes)
108
110
  options.parse_options!
@@ -168,14 +170,14 @@ module Aspera
168
170
  headers: {typ: 'JWT'}
169
171
  }
170
172
  }})
171
- else raise 'Unexpected case for option: auth'
173
+ else error_unexpected_value(auth_type)
172
174
  end
173
175
  end
174
176
 
175
177
  # if recipient is just an email, then convert to expected API hash : name and type
176
178
  def normalize_recipients(parameters)
177
179
  return unless parameters.key?('recipients')
178
- raise 'Field recipients must be an Array' unless parameters['recipients'].is_a?(Array)
180
+ assert_type(parameters['recipients'], Array){'recipients'}
179
181
  recipient_types = RECIPIENT_TYPES
180
182
  if parameters.key?('recipient_types')
181
183
  recipient_types = parameters['recipient_types']
@@ -236,27 +238,25 @@ module Aspera
236
238
  spinner.spin
237
239
  sleep(0.5)
238
240
  end
239
- raise 'internal error'
241
+ error_unreachable_line
240
242
  end
241
243
 
242
- # get a (full or partial) list of all entities of a given type
244
+ # Get a (full or partial) list of all entities of a given type
243
245
  # @param type [String] the type of entity to list (just a name)
244
246
  # @param query [Hash,nil] additional query parameters
245
- # @param path [String] optional prefix to add to the path (nil or empty string: no prefix)
247
+ # @param real_path [String] real path if it's n ot just the type
246
248
  # @param item_list_key [String] key in the result to get the list of items
247
- def list_entities(type:, path: nil, query: nil, item_list_key: nil)
248
- query = {} if query.nil?
249
+ def list_entities(type:, real_path: nil, query: {}, item_list_key: nil)
249
250
  type = type.to_s if type.is_a?(Symbol)
251
+ assert_type(type, String)
250
252
  item_list_key = type if item_list_key.nil?
251
- raise "internal error: Invalid type #{type.class}" unless type.is_a?(String)
252
- full_path = type
253
- full_path = "#{path}/#{full_path}" unless path.nil? || path.empty?
253
+ full_path = real_path.nil? ? type : real_path
254
254
  result = []
255
255
  offset = 0
256
256
  max_items = query.delete(MAX_ITEMS)
257
257
  remain_pages = query.delete(MAX_PAGES)
258
258
  # merge default parameters, by default 100 per page
259
- query = {'limit'=> 100}.merge(query)
259
+ query = {'limit'=> PER_PAGE_DEFAULT}.merge(query)
260
260
  loop do
261
261
  query['offset'] = offset
262
262
  page_result = @api_v5.read(full_path, query)[:data]
@@ -275,33 +275,36 @@ module Aspera
275
275
  end
276
276
 
277
277
  # lookup an entity id from its name
278
- def lookup_entity_by_field(type:, value:, field: 'name', query: :default, path: nil, item_list_key: nil)
279
- query = {'q'=> value} if query.eql?(:default)
280
- found = list_entities(type: type, path: path, query: query, item_list_key: item_list_key).select{|i|i[field].eql?(value)}
278
+ def lookup_entity_by_field(type:, value:, field: 'name', query: :default, real_path: nil, item_list_key: nil)
279
+ if query.eql?(:default)
280
+ assert(field.eql?('name')){'Default query is on name only'}
281
+ query = {'q'=> value}
282
+ end
283
+ found = list_entities(type: type, real_path: real_path, query: query, item_list_key: item_list_key).select{|i|i[field].eql?(value)}
281
284
  case found.length
282
285
  when 0 then raise "No #{type} with #{field} = #{value}"
283
286
  when 1 then return found.first
284
- else raise "Found #{found.length} #{path} with #{field} = #{value}"
287
+ else raise "Found #{found.length} #{real_path} with #{field} = #{value}"
285
288
  end
286
289
  end
287
290
 
288
291
  # list all packages with optional filter
289
- def list_packages_with_filter
292
+ def list_packages_with_filter(query: {})
290
293
  filter = options.get_next_argument('filter', mandatory: false, type: Proc, default: ->(_x){true})
291
294
  # translate box name to API prefix (with ending slash)
292
295
  box = options.get_option(:box)
293
- api_path =
296
+ real_path =
294
297
  case box
295
- when ExtendedValue::ALL then '' # only admin can list all packages globally
296
- when *API_LIST_MAILBOX_TYPES then box
298
+ when ExtendedValue::ALL then 'packages' # only admin can list all packages globally
299
+ when *API_LIST_MAILBOX_TYPES then "#{box}/packages"
297
300
  else
298
301
  group_type = options.get_option(:group_type)
299
- "#{group_type}/#{lookup_entity_by_field(type: group_type, value: box)['id']}"
302
+ "#{group_type}/#{lookup_entity_by_field(type: group_type, value: box)['id']}/packages"
300
303
  end
301
304
  return list_entities(
302
305
  type: 'packages',
303
- query: query_read_delete(default: {}),
304
- path: api_path).select(&filter)
306
+ query: query_read_delete(default: query),
307
+ real_path: real_path).select(&filter)
305
308
  end
306
309
 
307
310
  def package_receive(package_ids)
@@ -315,25 +318,32 @@ module Aspera
315
318
  id: IdGenerator.from_list([
316
319
  'faspex_recv',
317
320
  options.get_option(:url, mandatory: true),
318
- options.get_option(:username, mandatory: true)]))
321
+ options.get_option(:username, mandatory: true),
322
+ options.get_option(:box, mandatory: true)
323
+ ]))
319
324
  end
325
+ packages = []
320
326
  case package_ids
321
- when PACKAGE_ALL_INIT
322
- raise 'Only with option once_only' unless skip_ids_persistency
327
+ when ExtendedValue::INIT
328
+ assert(skip_ids_persistency){'Only with option once_only'}
323
329
  skip_ids_persistency.data.clear.concat(list_packages_with_filter.map{|p|p['id']})
324
330
  skip_ids_persistency.save
325
331
  return Main.result_status("Initialized skip for #{skip_ids_persistency.data.count} package(s)")
326
332
  when ExtendedValue::ALL
327
333
  # TODO: if packages have same name, they will overwrite ?
328
- package_ids = list_packages_with_filter.map{|p|p['id']}
329
- Log.log.debug{Log.dump(:package_ids, package_ids)}
330
- Log.log.debug{Log.dump(:skip_ids, skip_ids_persistency.data)}
331
- package_ids.reject!{|i|skip_ids_persistency.data.include?(i)} if skip_ids_persistency
332
- Log.log.debug{Log.dump(:package_ids, package_ids)}
334
+ packages = list_packages_with_filter(query: {'status' => 'completed'})
335
+ Log.log.trace1{Log.dump(:package_ids, packages.map{|p|p['id']})}
336
+ Log.log.trace1{Log.dump(:skip_ids, skip_ids_persistency.data)}
337
+ packages.reject!{|p|skip_ids_persistency.data.include?(p['id'])} if skip_ids_persistency
338
+ Log.log.trace1{Log.dump(:package_ids, packages.map{|p|p['id']})}
339
+ else
340
+ # a single id was provided, or a list of ids
341
+ package_ids = [package_ids] unless package_ids.is_a?(Array)
342
+ assert_type(package_ids, Array){'Expecting a single package id or a list of ids'}
343
+ assert(package_ids.all?(String)){'Package id shall be String'}
344
+ # packages = package_ids.map{|pkg_id|@api_v5.read("packages/#{pkg_id}")[:data]}
345
+ packages = package_ids.map{|pkg_id|{'id'=>pkg_id}}
333
346
  end
334
- # a single id was provided
335
- # TODO: check package_ids is a list of strings
336
- package_ids = [package_ids] if package_ids.is_a?(String)
337
347
  result_transfer = []
338
348
  param_file_list = {}
339
349
  begin
@@ -352,7 +362,8 @@ module Aspera
352
362
  else # shared inbox / workgroup
353
363
  download_params[:recipient_workgroup_id] = lookup_entity_by_field(type: options.get_option(:group_type), value: box)['id']
354
364
  end
355
- package_ids.each do |pkg_id|
365
+ packages.each do |package|
366
+ pkg_id = package['id']
356
367
  formatter.display_status("Receiving package #{pkg_id}")
357
368
  # TODO: allow from sent as well ?
358
369
  transfer_spec = @api_v5.call(
@@ -406,7 +417,8 @@ module Aspera
406
417
  when :delete
407
418
  ids = package_id
408
419
  ids = [ids] unless ids.is_a?(Array)
409
- raise 'Package identifier must be a single id or an Array' unless ids.is_a?(Array) && ids.all?(String)
420
+ assert_type(ids, Array){'Package identifier'}
421
+ assert(ids.all?(String)){'Package id shall be String'}
410
422
  # API returns 204, empty on success
411
423
  @api_v5.call({operation: 'DELETE', subpath: 'packages', headers: {'Accept' => 'application/json'}, json_params: {ids: ids}})
412
424
  return Main.result_status('Package(s) deleted')
@@ -523,11 +535,14 @@ module Aspera
523
535
  id_as_arg = false
524
536
  display_fields = nil
525
537
  adm_api = @api_v5
538
+ special_query = :default
526
539
  available_commands = [].concat(Plugin::ALL_OPS)
527
540
  case res_type
528
541
  when :metadata_profiles
529
542
  res_path = 'configuration/metadata_profiles'
530
543
  list_key = 'profiles'
544
+ when :alternate_addresses
545
+ res_path = 'configuration/alternate_addresses'
531
546
  when :email_notifications
532
547
  list_key = false
533
548
  id_as_arg = 'type'
@@ -538,11 +553,26 @@ module Aspera
538
553
  adm_api = Rest.new(@api_v5.params.merge({base_url: auth_api_url}))
539
554
  when :shared_inboxes, :workgroups
540
555
  available_commands.push(:members, :saml_groups, :invite_external_collaborator)
556
+ special_query = {'all': true}
557
+ when :nodes
558
+ available_commands.push(:shared_folders)
541
559
  end
542
560
  res_command = options.get_next_command(available_commands)
543
561
  case res_command
544
562
  when *Plugin::ALL_OPS
545
- return entity_command(res_command, adm_api, res_path, item_list_key: list_key, display_fields: display_fields, id_as_arg: id_as_arg)
563
+ return entity_command(res_command, adm_api, res_path, item_list_key: list_key, display_fields: display_fields, id_as_arg: id_as_arg) do |field, value|
564
+ lookup_entity_by_field(
565
+ type: res_type, real_path: res_path, field: field, value: value, query: special_query)['id']
566
+ end
567
+ when :shared_folders
568
+ node_id = instance_identifier do |field, value|
569
+ lookup_entity_by_field(type: res_type.to_s, field: field, value: value)['id']
570
+ end
571
+ sh_path = "#{res_path}/#{node_id}/shared_folders"
572
+ return entity_action(adm_api, sh_path, item_list_key: 'shared_folders') do |field, value|
573
+ lookup_entity_by_field(
574
+ type: 'shared_folders', real_path: sh_path, field: field, value: value)['id']
575
+ end
546
576
  when :invite_external_collaborator
547
577
  shared_inbox_id = instance_identifier { |field, value| lookup_entity_by_field(type: res_type.to_s, field: field, value: value)['id']}
548
578
  creation_payload = value_create_modify(command: res_command, type: [Hash, String])
@@ -550,7 +580,9 @@ module Aspera
550
580
  res_path = "#{res_type}/#{shared_inbox_id}/external_collaborator"
551
581
  result = adm_api.create(res_path, creation_payload)[:data]
552
582
  formatter.display_status(result['message'])
553
- result = lookup_entity_by_field(type: 'members', path: "#{res_type}/#{shared_inbox_id}", value: creation_payload['email_address'], query: {})
583
+ result = lookup_entity_by_field(
584
+ type: 'members', real_path: "#{res_type}/#{shared_inbox_id}/members", value: creation_payload['email_address'],
585
+ query: {})
554
586
  return {type: :single_object, data: result}
555
587
  when :members, :saml_groups
556
588
  res_id = instance_identifier { |field, value| lookup_entity_by_field(type: res_type.to_s, field: field, value: value)['id']}
@@ -607,7 +639,7 @@ module Aspera
607
639
  end
608
640
  when :gateway
609
641
  require 'aspera/faspex_gw'
610
- url = value_create_modify(command: command, type: String)
642
+ url = value_create_modify(command: command, description: 'listening url (e.g. https://localhost:12345)', type: String)
611
643
  uri = URI.parse(url)
612
644
  server = WebServerSimple.new(uri)
613
645
  server.mount(uri.path, Faspex4GWServlet, @api_v5, nil)
@@ -622,7 +654,7 @@ module Aspera
622
654
  require 'aspera/faspex_postproc' # cspell:disable-line
623
655
  parameters = value_create_modify(command: command)
624
656
  parameters = parameters.symbolize_keys
625
- raise 'Missing key: url' unless parameters.key?(:url)
657
+ assert(parameters.key?(:url)){'Missing key: url'}
626
658
  uri = URI.parse(parameters[:url])
627
659
  parameters[:processing] ||= {}
628
660
  parameters[:processing][:root] = uri.path
@@ -9,8 +9,9 @@ require 'aspera/hash_ext'
9
9
  require 'aspera/id_generator'
10
10
  require 'aspera/node'
11
11
  require 'aspera/aoc'
12
- require 'aspera/sync'
13
12
  require 'aspera/oauth'
13
+ require 'aspera/node_simulator'
14
+ require 'aspera/assert'
14
15
  require 'base64'
15
16
  require 'zlib'
16
17
 
@@ -303,7 +304,7 @@ module Aspera
303
304
  save_to_file: File.join(transfer.destination_folder(Fasp::TransferSpec::DIRECTION_RECEIVE), file_name))
304
305
  return Main.result_status("downloaded: #{file_name}")
305
306
  end
306
- raise 'INTERNAL ERROR'
307
+ error_unreachable_line
307
308
  end
308
309
 
309
310
  # common API to node and Shares
@@ -412,7 +413,7 @@ module Aspera
412
413
  url: apifid[:api].params[:base_url],
413
414
  root_id: apifid[:file_id]
414
415
  }
415
- raise 'No auth for node' if apifid[:api].params[:auth].nil?
416
+ assert_values(apifid[:api].params[:auth][:type], %i[basic oauth2])
416
417
  case apifid[:api].params[:auth][:type]
417
418
  when :basic
418
419
  result[:username] = apifid[:api].params[:auth][:username]
@@ -420,7 +421,7 @@ module Aspera
420
421
  when :oauth2
421
422
  result[:username] = apifid[:api].params[:headers][Aspera::Node::HEADER_X_ASPERA_ACCESS_KEY]
422
423
  result[:password] = apifid[:api].oauth_token
423
- else raise 'internal error: unknown auth type'
424
+ else error_unreachable_line
424
425
  end
425
426
  return {type: :single_object, data: result} if command_repo.eql?(:node_info)
426
427
  # check format of bearer token
@@ -463,10 +464,11 @@ module Aspera
463
464
  return execute_sync_action do |sync_direction, _local_path, remote_path|
464
465
  # Gen4 API
465
466
  # direction is push pull, bidi
467
+ assert_values(sync_direction, %i[push pull bidi])
466
468
  ts_direction = case sync_direction
467
469
  when :push, :bidi then Fasp::TransferSpec::DIRECTION_SEND
468
470
  when :pull then Fasp::TransferSpec::DIRECTION_RECEIVE
469
- else raise "internal error: bad direction: #{sync_direction} (#{sync_direction.class})"
471
+ else error_unreachable_line
470
472
  end
471
473
  # remote is specified by option to_folder
472
474
  apifid = @api_node.resolve_api_fid(top_file_id, remote_path)
@@ -536,12 +538,7 @@ module Aspera
536
538
  subpath: "files/#{apifid[:file_id]}/preview",
537
539
  headers: {'Accept' => 'image/png'}
538
540
  )
539
- require 'aspera/preview/terminal'
540
- terminal_options = options.get_option(:query, default: {}).symbolize_keys
541
- allowed_options = Preview::Terminal.method(:build).parameters.select{|i|i[0].eql?(:key)}.map{|i|i[1]}
542
- unknown_options = terminal_options.keys - allowed_options
543
- raise "invalid options: #{unknown_options.join(', ')}, use #{allowed_options.join(', ')}" unless unknown_options.empty?
544
- return Main.result_status(Preview::Terminal.build(result[:http].body, **terminal_options))
541
+ return Main.result_picture_in_terminal(options, result[:http].body)
545
542
  when :permission
546
543
  apifid = apifid_from_next_arg(top_file_id)
547
544
  command_perm = options.get_next_command(%i[list create delete])
@@ -574,11 +571,11 @@ module Aspera
574
571
  # notify application of creation
575
572
  the_app[:api].permissions_send_event(created_data: created_data, app_info: the_app) unless the_app.nil?
576
573
  return { type: :single_object, data: created_data}
577
- else raise "internal error:shall not reach here (#{command_perm})"
574
+ else error_unreachable_line
578
575
  end
579
- else raise "INTERNAL ERROR: no case for #{command_repo}"
576
+ else error_unreachable_line
580
577
  end # command_repo
581
- # raise 'INTERNAL ERROR: missing return'
578
+ error_unreachable_line
582
579
  end # execute_command_gen4
583
580
 
584
581
  # This is older API
@@ -683,7 +680,8 @@ module Aspera
683
680
  central
684
681
  asperabrowser
685
682
  basic_token
686
- bearer_token].concat(COMMON_ACTIONS).freeze
683
+ bearer_token
684
+ simulator].concat(COMMON_ACTIONS).freeze
687
685
 
688
686
  def execute_action(command=nil, prefix_path=nil)
689
687
  command ||= options.get_next_command(ACTIONS)
@@ -906,6 +904,21 @@ module Aspera
906
904
  token_info = options.get_next_argument('user and group identification', type: Hash)
907
905
  access_key = options.get_option(:username, mandatory: true)
908
906
  return Main.result_status(Aspera::Node.bearer_token(payload: token_info, access_key: access_key, private_key: private_key))
907
+ when :simulator
908
+ require 'aspera/node_simulator'
909
+ parameters = value_create_modify(command: command)
910
+ parameters = parameters.symbolize_keys
911
+ raise 'Missing key: url' unless parameters.key?(:url)
912
+ uri = URI.parse(parameters[:url])
913
+ server = WebServerSimple.new(uri, certificate: parameters[:certificate])
914
+ server.mount(uri.path, NodeSimulatorServlet, parameters[:credentials], transfer)
915
+ # on ctrl-c, tell server main loop to exit
916
+ trap('INT') { server.shutdown }
917
+ formatter.display_status("Node Simulator listening on #{uri.port}")
918
+ Log.log.info{"Listening on #{uri.port}"}
919
+ # this is blocking until server exits
920
+ server.start
921
+ return Main.result_status('Simulator terminated')
909
922
  end # case command
910
923
  raise 'ERROR: shall not reach this line'
911
924
  end # execute_action
@@ -2,6 +2,8 @@
2
2
 
3
3
  require 'aspera/cli/basic_auth_plugin'
4
4
  require 'aspera/nagios'
5
+ require 'aspera/log'
6
+ require 'aspera/assert'
5
7
  require 'xmlsimple'
6
8
 
7
9
  module Aspera
@@ -95,7 +97,7 @@ module Aspera
95
97
  call_args[:url_params][:format] = format
96
98
  when :ext
97
99
  call_args[:subpath] = "#{call_args[:subpath]}.#{format}"
98
- else raise 'unexpected'
100
+ else error_unexpected_value(call_type)
99
101
  end
100
102
  end
101
103
  result = @api_orch.call(call_args)
@@ -205,7 +207,7 @@ module Aspera
205
207
  result[:data] = call_ao('initiate', id: wf_id, args: call_params, accept: override_accept)[:data]
206
208
  return result
207
209
  end # wf command
208
- else raise "ERROR, unknown command: [#{command}]"
210
+ else error_unexpected_value(command)
209
211
  end # case command
210
212
  end # execute_action
211
213
  end # Orchestrator
@@ -13,6 +13,8 @@ require 'aspera/node'
13
13
  require 'aspera/hash_ext'
14
14
  require 'aspera/timer_limiter'
15
15
  require 'aspera/id_generator'
16
+ require 'aspera/log'
17
+ require 'aspera/assert'
16
18
  require 'securerandom'
17
19
 
18
20
  module Aspera
@@ -94,7 +96,7 @@ module Aspera
94
96
  end
95
97
 
96
98
  options.parse_options!
97
- raise 'skip_folder shall be an Array, use @json:[...]' unless @option_skip_folders.is_a?(Array)
99
+ assert_type(@option_skip_folders, Array){'skip_folder'}
98
100
  @tmp_folder = File.join(options.get_option(:temp_folder, mandatory: true), "#{TMP_DIR_PREFIX}.#{SecureRandom.uuid}")
99
101
  FileUtils.mkdir_p(@tmp_folder)
100
102
  Log.log.debug{"tmpdir: #{@tmp_folder}"}
@@ -104,7 +106,7 @@ module Aspera
104
106
  @skip_types = []
105
107
  value.split(',').each do |v|
106
108
  s = v.to_sym
107
- raise "not supported: #{v}" unless Aspera::Preview::FileTypes::CONVERSION_TYPES.include?(s)
109
+ assert_values(s, Aspera::Preview::FileTypes::CONVERSION_TYPES){'skip_types'}
108
110
  @skip_types.push(s)
109
111
  end
110
112
  end
@@ -211,7 +213,7 @@ module Aspera
211
213
  end
212
214
 
213
215
  def do_transfer(direction, folder_id, source_filename, destination='/')
214
- raise 'Internal ERROR' if destination.nil? && direction.eql?(Fasp::TransferSpec::DIRECTION_RECEIVE)
216
+ assert(!(destination.nil? && direction.eql?(Fasp::TransferSpec::DIRECTION_RECEIVE)))
215
217
  t_spec = @api_node.transfer_spec_gen4(folder_id, direction, {
216
218
  'paths' => [{'source' => source_filename}],
217
219
  'tags' => {Fasp::TransferSpec::TAG_RESERVED => {PREV_GEN_TAG => true}}
@@ -420,13 +422,13 @@ module Aspera
420
422
  raise Cli::Error, "Folder #{@option_previews_folder} does not exist on node. " \
421
423
  'Please create it in the storage root, or specify an alternate name.' if @previews_folder_entry.nil?
422
424
  else
423
- raise 'only local storage allowed in this mode' unless @access_key_self['storage']['type'].eql?('local')
425
+ assert(@access_key_self['storage']['type'].eql?('local')){'only local storage allowed in this mode'}
424
426
  @local_storage_root = @access_key_self['storage']['path']
425
427
  # TODO: option to override @local_storage_root='xxx'
426
428
  @local_storage_root = @local_storage_root[PVCL_LOCAL_STORAGE.length..-1] if @local_storage_root.start_with?(PVCL_LOCAL_STORAGE)
427
429
  # TODO: windows could have "C:" ?
428
- raise "not local storage: #{@local_storage_root}" unless @local_storage_root.start_with?('/')
429
- raise Cli::Error, "Local storage root folder #{@local_storage_root} does not exist." unless File.directory?(@local_storage_root)
430
+ assert(@local_storage_root.start_with?('/')){"not local storage: #{@local_storage_root}"}
431
+ assert(File.directory?(@local_storage_root), exception_class: Cli::Error){"Local storage root folder #{@local_storage_root} does not exist."}
430
432
  @local_preview_folder = File.join(@local_storage_root, @option_previews_folder)
431
433
  raise Cli::Error, "Folder #{@local_preview_folder} does not exist locally. " \
432
434
  'Please create it, or specify an alternate name.' unless File.directory?(@local_preview_folder)
@@ -435,7 +437,7 @@ module Aspera
435
437
  Log.log.debug{"marker file: #{marker_file}"}
436
438
  if File.exist?(marker_file)
437
439
  ak = File.read(marker_file).chomp
438
- raise "mismatch access key in #{marker_file}: contains #{ak}, using #{@access_key_self['id']}" unless @access_key_self['id'].eql?(ak)
440
+ assert(@access_key_self['id'].eql?(ak)){"mismatch access key in #{marker_file}: contains #{ak}, using #{@access_key_self['id']}"}
439
441
  else
440
442
  File.write(marker_file, @access_key_self['id'])
441
443
  end
@@ -7,6 +7,8 @@ require 'aspera/fasp/transfer_spec'
7
7
  require 'aspera/ascmd'
8
8
  require 'aspera/ssh'
9
9
  require 'aspera/nagios'
10
+ require 'aspera/log'
11
+ require 'aspera/assert'
10
12
  require 'tempfile'
11
13
  require 'open3'
12
14
 
@@ -148,8 +150,9 @@ module Aspera
148
150
  end
149
151
  ssh_key_list = options.get_option(:ssh_keys)
150
152
  if !ssh_key_list.nil?
151
- raise 'Expecting single value or array for ssh_keys' unless ssh_key_list.is_a?(Array) || ssh_key_list.is_a?(String)
152
153
  ssh_key_list = [ssh_key_list] if ssh_key_list.is_a?(String)
154
+ assert_type(ssh_key_list, Array){'ssh_keys'}
155
+ assert(ssh_key_list.all?(String))
153
156
  ssh_key_list.map!{|p|File.expand_path(p)}
154
157
  Log.log.debug{"SSH keys=#{ssh_key_list}"}
155
158
  if !ssh_key_list.empty?
@@ -225,7 +228,7 @@ module Aspera
225
228
  else
226
229
  nagios.add_critical('transfer', statuses.reject{|i|i.eql?(:success)}.first.to_s)
227
230
  end
228
- else raise 'ERROR'
231
+ else error_unexpected_value(command_nagios)
229
232
  end
230
233
  return nagios.result
231
234
  when *TRANSFER_COMMANDS
@@ -248,7 +251,7 @@ module Aspera
248
251
  rescue Aspera::AsCmd::Error => e
249
252
  raise Cli::BadArgument, e.extended_message
250
253
  end
251
- else raise 'internal error: unexpected action'
254
+ else error_unreachable_line
252
255
  end
253
256
  end # execute_action
254
257
  end # Server