aspera-cli 4.21.1 → 4.22.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (105) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/BUGS.md +1 -1
  4. data/CHANGELOG.md +52 -22
  5. data/CONTRIBUTING.md +69 -148
  6. data/README.md +929 -668
  7. data/bin/ascli +5 -14
  8. data/bin/asession +1 -3
  9. data/examples/get_proto_file.rb +4 -3
  10. data/examples/proxy.pac +20 -20
  11. data/lib/aspera/agent/base.rb +11 -5
  12. data/lib/aspera/agent/connect.rb +30 -28
  13. data/lib/aspera/agent/{alpha.rb → desktop.rb} +35 -31
  14. data/lib/aspera/agent/direct.rb +141 -121
  15. data/lib/aspera/agent/httpgw.rb +22 -26
  16. data/lib/aspera/agent/node.rb +14 -11
  17. data/lib/aspera/agent/transferd.rb +30 -19
  18. data/lib/aspera/api/alee.rb +1 -1
  19. data/lib/aspera/api/aoc.rb +6 -6
  20. data/lib/aspera/api/cos_node.rb +2 -2
  21. data/lib/aspera/api/httpgw.rb +7 -3
  22. data/lib/aspera/api/node.rb +10 -8
  23. data/lib/aspera/ascmd.rb +3 -3
  24. data/lib/aspera/ascp/installation.rb +53 -72
  25. data/lib/aspera/ascp/management.rb +1 -1
  26. data/lib/aspera/assert.rb +11 -2
  27. data/lib/aspera/cli/error.rb +2 -2
  28. data/lib/aspera/cli/extended_value.rb +46 -21
  29. data/lib/aspera/cli/formatter.rb +55 -48
  30. data/lib/aspera/cli/hints.rb +1 -1
  31. data/lib/aspera/cli/info.rb +1 -0
  32. data/lib/aspera/cli/main.rb +192 -170
  33. data/lib/aspera/cli/manager.rb +18 -18
  34. data/lib/aspera/cli/plugin.rb +23 -20
  35. data/lib/aspera/cli/plugin_factory.rb +1 -1
  36. data/lib/aspera/cli/plugins/alee.rb +1 -1
  37. data/lib/aspera/cli/plugins/aoc.rb +247 -159
  38. data/lib/aspera/cli/plugins/ats.rb +19 -17
  39. data/lib/aspera/cli/plugins/config.rb +76 -113
  40. data/lib/aspera/cli/plugins/console.rb +5 -3
  41. data/lib/aspera/cli/plugins/faspex.rb +39 -35
  42. data/lib/aspera/cli/plugins/faspex5.rb +111 -84
  43. data/lib/aspera/cli/plugins/faspio.rb +13 -1
  44. data/lib/aspera/cli/plugins/httpgw.rb +13 -1
  45. data/lib/aspera/cli/plugins/node.rb +312 -182
  46. data/lib/aspera/cli/plugins/orchestrator.rb +34 -40
  47. data/lib/aspera/cli/plugins/preview.rb +3 -3
  48. data/lib/aspera/cli/plugins/server.rb +6 -6
  49. data/lib/aspera/cli/plugins/shares.rb +5 -5
  50. data/lib/aspera/cli/sync_actions.rb +19 -18
  51. data/lib/aspera/cli/transfer_agent.rb +5 -5
  52. data/lib/aspera/cli/transfer_progress.rb +2 -2
  53. data/lib/aspera/cli/version.rb +1 -1
  54. data/lib/aspera/command_line_builder.rb +116 -95
  55. data/lib/aspera/coverage.rb +8 -5
  56. data/lib/aspera/environment.rb +26 -17
  57. data/lib/aspera/faspex_gw.rb +14 -14
  58. data/lib/aspera/faspex_postproc.rb +10 -11
  59. data/lib/aspera/hash_ext.rb +4 -14
  60. data/lib/aspera/json_rpc.rb +1 -1
  61. data/lib/aspera/keychain/encrypted_hash.rb +47 -34
  62. data/lib/aspera/keychain/factory.rb +41 -0
  63. data/lib/aspera/keychain/hashicorp_vault.rb +71 -0
  64. data/lib/aspera/keychain/macos_security.rb +19 -11
  65. data/lib/aspera/log.rb +28 -34
  66. data/lib/aspera/nagios.rb +6 -6
  67. data/lib/aspera/node_simulator.rb +8 -8
  68. data/lib/aspera/oauth/base.rb +14 -7
  69. data/lib/aspera/oauth/factory.rb +5 -6
  70. data/lib/aspera/oauth/url_json.rb +6 -6
  71. data/lib/aspera/persistency_action_once.rb +6 -4
  72. data/lib/aspera/persistency_folder.rb +2 -2
  73. data/lib/aspera/preview/generator.rb +13 -10
  74. data/lib/aspera/preview/options.rb +16 -16
  75. data/lib/aspera/preview/terminal.rb +4 -4
  76. data/lib/aspera/preview/utils.rb +15 -17
  77. data/lib/aspera/products/connect.rb +35 -1
  78. data/lib/aspera/products/{alpha.rb → desktop.rb} +3 -3
  79. data/lib/aspera/products/transferd.rb +9 -2
  80. data/lib/aspera/proxy_auto_config.rb +2 -2
  81. data/lib/aspera/rest.rb +56 -47
  82. data/lib/aspera/rest_errors_aspera.rb +1 -1
  83. data/lib/aspera/secret_hider.rb +12 -5
  84. data/lib/aspera/ssh.rb +4 -4
  85. data/lib/aspera/temp_file_manager.rb +5 -4
  86. data/lib/aspera/transfer/convert.rb +29 -0
  87. data/lib/aspera/transfer/error_info.rb +66 -66
  88. data/lib/aspera/transfer/parameters.rb +13 -68
  89. data/lib/aspera/transfer/spec.rb +5 -6
  90. data/lib/aspera/transfer/spec.schema.yaml +753 -0
  91. data/lib/aspera/transfer/spec_doc.rb +62 -0
  92. data/lib/aspera/transfer/sync.rb +23 -72
  93. data/lib/aspera/transfer/sync_instance.schema.yaml +13 -0
  94. data/lib/aspera/transfer/sync_session.schema.yaml +79 -0
  95. data/lib/aspera/transfer/uri.rb +6 -6
  96. data/lib/aspera/uri_reader.rb +18 -1
  97. data/lib/aspera/web_auth.rb +1 -1
  98. data/lib/aspera/web_server_simple.rb +53 -44
  99. data.tar.gz.sig +0 -0
  100. metadata +28 -165
  101. metadata.gz.sig +0 -0
  102. data/examples/build_exec +0 -74
  103. data/examples/build_exec_rubyc +0 -40
  104. data/examples/build_package.sh +0 -28
  105. data/lib/aspera/transfer/spec.yaml +0 -718
@@ -52,7 +52,7 @@ module Aspera
52
52
  result = api.call(
53
53
  operation: 'POST',
54
54
  headers: {
55
- 'Content-type' => 'text/plain',
55
+ 'Content-type' => Rest::MIME_TEXT,
56
56
  'Accept' => 'application/xrds+xml'
57
57
  }
58
58
  )
@@ -74,7 +74,9 @@ module Aspera
74
74
  return nil
75
75
  end
76
76
 
77
- def wizard(object:, private_key_path: nil, pub_key_pem: nil)
77
+ # @param object [Plugin] An instance of this class
78
+ # @return [Hash] :preset_value, :test_args
79
+ def wizard(object:)
78
80
  options = object.options
79
81
  return {
80
82
  preset_value: {
@@ -89,7 +91,7 @@ module Aspera
89
91
  # extract elements from faspex public link
90
92
  def get_link_data(public_url)
91
93
  public_uri = URI.parse(public_url)
92
- Aspera.assert((m = public_uri.path.match(%r{^(.*)/(external.*)$})), exception_class: Cli::BadArgument){'Public link does not match Faspex format'}
94
+ Aspera.assert(m = public_uri.path.match(%r{^(.*)/(external.*)$}), exception_class: Cli::BadArgument){'Public link does not match Faspex format'}
93
95
  base = m[1]
94
96
  subpath = m[2]
95
97
  port_add = public_uri.port.eql?(public_uri.default_port) ? '' : ":#{public_uri.port}"
@@ -108,15 +110,15 @@ module Aspera
108
110
  raise Cli::BadArgument, 'package has no link (deleted?)' if raise_no_link
109
111
  return nil
110
112
  end
111
- result = entry['link'].find{|e| e['rel'].eql?('package')}['href']
113
+ result = entry['link'].find{ |e| e['rel'].eql?('package')}['href']
112
114
  return result
113
115
  end
114
116
 
115
117
  # @return [Integer] identifier of source
116
118
  def get_source_id_by_name(source_name, source_list)
117
- match_source = source_list.find { |i| i['name'].eql?(source_name) }
119
+ match_source = source_list.find{ |i| i['name'].eql?(source_name)}
118
120
  return match_source['id'] unless match_source.nil?
119
- raise Cli::Error, %Q(No such Faspex source: "#{source_name}" in [#{source_list.map{|i| %Q("#{i['name']}")}.join(', ')}])
121
+ raise Cli::Error, %Q(No such Faspex source: "#{source_name}" in [#{source_list.map{ |i| %Q("#{i['name']}")}.join(', ')}])
120
122
  end
121
123
  end
122
124
 
@@ -199,14 +201,14 @@ module Aspera
199
201
  package[PACKAGE_MATCH_FIELD] =
200
202
  case mailbox
201
203
  when :inbox, :archive
202
- recipient = package['to'].find{|i|recipient_names.include?(i['name'])}
204
+ recipient = package['to'].find{ |i| recipient_names.include?(i['name'])}
203
205
  recipient.nil? ? nil : recipient['recipient_delivery_id']
204
206
  else # :sent
205
207
  package['delivery_id']
206
208
  end
207
209
  # add special key
208
210
  package['items'] = package['link'].is_a?(Array) ? package['link'].length : 0
209
- package['metadata'] = package['metadata']['field'].each_with_object({}){|i, m| m[i['name']] = i['content'] }
211
+ package['metadata'] = package['metadata']['field'].each_with_object({}){ |i, m| m[i['name']] = i['content']}
210
212
  # if we look for a specific package
211
213
  stop_condition = true if !stop_at_id.nil? && stop_at_id.eql?(package[PACKAGE_MATCH_FIELD])
212
214
  # keep only those for the specified recipient
@@ -220,13 +222,13 @@ module Aspera
220
222
  result = result.slice(0, max_items) if result.count > max_items
221
223
  break
222
224
  end
223
- link = box_data['link'].find{|i|i['rel'].eql?('next')}
225
+ link = box_data['link'].find{ |i| i['rel'].eql?('next')}
224
226
  Log.log.debug{"link: #{link}"}
225
227
  # no next link
226
228
  break if link.nil?
227
229
  # replace parameters with the ones from next link
228
230
  params = CGI.parse(URI.parse(link['href']).query)
229
- mailbox_query = params.keys.each_with_object({}){|i, m| m[i] = params[i].first }
231
+ mailbox_query = params.keys.each_with_object({}){ |i, m| m[i] = params[i].first}
230
232
  Log.log.debug{"query: #{mailbox_query}"}
231
233
  break if !max_pages.nil? && (mailbox_query['page'].to_i > max_pages)
232
234
  end
@@ -250,11 +252,12 @@ module Aspera
250
252
  # pkg_created=api_public_link.create(create_path,package_create_params)
251
253
  # so extract data from javascript
252
254
  package_creation_data = api_public_link.call(
253
- operation: 'POST',
254
- subpath: create_path,
255
- headers: {'Accept' => 'text/javascript'},
256
- body: package_create_params,
257
- body_type: :json)[:http].body
255
+ operation: 'POST',
256
+ subpath: create_path,
257
+ content_type: Rest::MIME_JSON,
258
+ body: package_create_params,
259
+ headers: {'Accept' => 'text/javascript'}
260
+ )[:http].body
258
261
  # get arguments of function call
259
262
  package_creation_data.delete!("\n") # one line
260
263
  package_creation_data.gsub!(/^[^"]+\("\{/, '{') # delete header
@@ -288,7 +291,7 @@ module Aspera
288
291
  case command_pkg
289
292
  when :show
290
293
  delivery_id = instance_identifier
291
- return {type: :single_object, data: mailbox_filtered_entries(stop_at_id: delivery_id).find{|p|p[PACKAGE_MATCH_FIELD].eql?(delivery_id)} }
294
+ return Main.result_single_object(mailbox_filtered_entries(stop_at_id: delivery_id).find{ |p| p[PACKAGE_MATCH_FIELD].eql?(delivery_id)})
292
295
  when :list
293
296
  return {
294
297
  type: :object_list,
@@ -350,14 +353,14 @@ module Aspera
350
353
  raise 'empty id' if delivery_id.empty?
351
354
  recipient = options.get_option(:recipient)
352
355
  if delivery_id.eql?(SpecialValues::ALL)
353
- pkg_id_uri = mailbox_filtered_entries.map{|i|{id: i[PACKAGE_MATCH_FIELD], uri: self.class.get_fasp_uri_from_entry(i, raise_no_link: false)}}
356
+ pkg_id_uri = mailbox_filtered_entries.map{ |i| {id: i[PACKAGE_MATCH_FIELD], uri: self.class.get_fasp_uri_from_entry(i, raise_no_link: false)}}
354
357
  elsif delivery_id.eql?(SpecialValues::INIT)
355
358
  Aspera.assert(skip_ids_persistency){'Only with option once_only'}
356
- skip_ids_persistency.data.clear.concat(mailbox_filtered_entries.map{|i|{id: i[PACKAGE_MATCH_FIELD]}})
359
+ skip_ids_persistency.data.clear.concat(mailbox_filtered_entries.map{ |i| {id: i[PACKAGE_MATCH_FIELD]}})
357
360
  skip_ids_persistency.save
358
361
  return Main.result_status("Initialized skip for #{skip_ids_persistency.data.count} package(s)")
359
362
  elsif !recipient.nil? && recipient.start_with?('*')
360
- found_package_link = mailbox_filtered_entries(stop_at_id: delivery_id).find{|p|p[PACKAGE_MATCH_FIELD].eql?(delivery_id)}['link'].first['href']
363
+ found_package_link = mailbox_filtered_entries(stop_at_id: delivery_id).find{ |p| p[PACKAGE_MATCH_FIELD].eql?(delivery_id)}['link'].first['href']
361
364
  raise "Not Found. Dropbox and Workgroup packages can use the link option with #{Transfer::Uri::SCHEME}" if found_package_link.nil?
362
365
  pkg_id_uri = [{id: delivery_id, uri: found_package_link}]
363
366
  else
@@ -397,7 +400,7 @@ module Aspera
397
400
  # prune packages already downloaded
398
401
  # TODO : remove ids from skip not present in inbox to avoid growing too big
399
402
  # skip_ids_data.select!{|id|pkg_id_uri.select{|p|p[:id].eql?(id)}}
400
- pkg_id_uri.reject!{|i|skip_ids_data.include?(i[:id])}
403
+ pkg_id_uri.reject!{ |i| skip_ids_data.include?(i[:id])}
401
404
  Log.log.debug{Log.dump(:pkg_id_uri, pkg_id_uri)}
402
405
  return Main.result_status('no new package') if pkg_id_uri.empty?
403
406
  result_transfer = []
@@ -413,12 +416,13 @@ module Aspera
413
416
  xml_payload =
414
417
  %Q(<?xml version="1.0" encoding="UTF-8"?><url-list xmlns="http://schemas.asperasoft.com/xml/url-list"><url href="#{sanitized}"/></url-list>)
415
418
  transfer_spec['token'] = api_v3.call(
416
- operation: 'POST',
417
- subpath: 'issue-token',
418
- headers: {'Accept' => 'text/plain', 'Content-Type' => 'application/vnd.aspera.url-list+xml'},
419
- query: {'direction' => 'down'},
420
- body: xml_payload,
421
- body_type: :text)[:http].body
419
+ operation: 'POST',
420
+ subpath: 'issue-token',
421
+ query: {'direction' => 'down'},
422
+ content_type: Rest::MIME_TEXT,
423
+ body: xml_payload,
424
+ headers: {'Accept' => Rest::MIME_TEXT, 'Content-Type' => 'application/vnd.aspera.url-list+xml'}
425
+ )[:http].body
422
426
  end
423
427
  transfer_spec['direction'] = Transfer::Spec::DIRECTION_RECEIVE
424
428
  statuses = transfer.start(transfer_spec)
@@ -435,13 +439,13 @@ module Aspera
435
439
  source_list = api_v3.read('source_shares')['items']
436
440
  case command_source
437
441
  when :list
438
- return {type: :object_list, data: source_list}
442
+ return Main.result_object_list(source_list)
439
443
  else # :info :node
440
444
  source_id = instance_identifier do |field, value|
441
445
  Aspera.assert(field.eql?('name'), exception_class: Cli::BadArgument){'only name as selector, or give id'}
442
446
  self.class.get_source_id_by_name(value, source_list)
443
447
  end.to_i
444
- selected_source = source_list.find{|i|i['id'].eql?(source_id)}
448
+ selected_source = source_list.find{ |i| i['id'].eql?(source_id)}
445
449
  raise 'No such source' if selected_source.nil?
446
450
  source_name = selected_source['name']
447
451
  source_hash = options.get_option(:storage, mandatory: true)
@@ -460,7 +464,7 @@ module Aspera
460
464
  Log.log.debug{Log.dump(:source_info, source_info)}
461
465
  case command_source
462
466
  when :info
463
- return {data: source_info, type: :single_object}
467
+ return Main.result_single_object(source_info)
464
468
  when :node
465
469
  node_config = ExtendedValue.instance.evaluate(source_info[KEY_NODE])
466
470
  Log.log.debug{"node=#{node_config}"}
@@ -472,18 +476,18 @@ module Aspera
472
476
  username: node_config['username'],
473
477
  password: node_config['password']})
474
478
  command = options.get_next_command(Node::COMMANDS_FASPEX)
475
- return Node.new(**init_params, api: api_node).execute_action(command, source_info[KEY_PATH])
479
+ return Node.new(**init_params, api: api_node, prefix_path: source_info[KEY_PATH]).execute_action(command)
476
480
  end
477
481
  end
478
482
  when :me
479
483
  my_info = api_v3.read('me')
480
- return {data: my_info, type: :single_object}
484
+ return Main.result_single_object(my_info)
481
485
  when :dropbox
482
486
  command_pkg = options.get_next_command([:list])
483
487
  case command_pkg
484
488
  when :list
485
489
  dropbox_list = api_v3.read('dropboxes')
486
- return {type: :object_list, data: dropbox_list['items'], fields: %w[name id description can_read can_write]}
490
+ return Main.result_object_list(dropbox_list['items'], fields: %w[name id description can_read can_write])
487
491
  end
488
492
  when :v4
489
493
  command = options.get_next_command(%i[package dropbox dmembership workgroup wmembership user metadata_profile])
@@ -512,7 +516,7 @@ module Aspera
512
516
  # add missing entries
513
517
  users.each do |u|
514
518
  unless u['emails'].nil?
515
- email = u['emails'].find{|i|i['primary'].eql?('true')}
519
+ email = u['emails'].find{ |i| i['primary'].eql?('true')}
516
520
  u['email'] = email['value'] unless email.nil?
517
521
  end
518
522
  if u['email'].nil?
@@ -522,11 +526,11 @@ module Aspera
522
526
  u['first_name'], u['last_name'] = u['displayName'].split(' ', 2)
523
527
  u['x'] = true
524
528
  end
525
- return {type: :object_list, data: users}
529
+ return Main.result_object_list(users)
526
530
  when :login_methods
527
531
  login_meths = api_v3.call(operation: 'GET', subpath: 'login/new', headers: {'Accept' => 'application/xrds+xml'})[:http].body
528
532
  login_methods = XmlSimple.xml_in(login_meths, {'ForceArray' => false})
529
- return {type: :object_list, data: login_methods['XRD']['Service']}
533
+ return Main.result_object_list(login_methods['XRD']['Service'])
530
534
  end
531
535
  end
532
536
  end
@@ -74,6 +74,10 @@ module Aspera
74
74
  return nil
75
75
  end
76
76
 
77
+ # @param object [Plugin] An instance of this class
78
+ # @param private_key_path [String] path to private key
79
+ # @param pub_key_pem [String] PEM of public key
80
+ # @return [Hash] :preset_value, :test_args
77
81
  def wizard(object:, private_key_path:, pub_key_pem:)
78
82
  options = object.options
79
83
  formatter = object.formatter
@@ -183,7 +187,7 @@ module Aspera
183
187
  else Aspera.error_unexpected_value(auth_type)
184
188
  end
185
189
  # in case user wants to use HTTPGW tell transfer agent how to get address
186
- transfer.httpgw_url_cb = lambda { @api_v5.read('account')['gateway_url'] }
190
+ transfer.httpgw_url_cb = lambda{@api_v5.read('account')['gateway_url']}
187
191
  end
188
192
 
189
193
  # if recipient is just an email, then convert to expected API hash : name and type
@@ -228,10 +232,8 @@ module Aspera
228
232
  config.progress_bar&.event(:transfer, session_id: id, info: status['bytes_written'].to_i)
229
233
  end
230
234
  if status_list.include?(status['upload_status'])
231
- # if status['upload_status'].eql?('completed')
232
235
  config.progress_bar&.event(:end, session_id: id)
233
236
  return status
234
- # end
235
237
  end
236
238
  sleep(1.0)
237
239
  end
@@ -302,7 +304,7 @@ module Aspera
302
304
  Aspera.assert(field.eql?('name')){'Default query is on name only'}
303
305
  query = {'q'=> value}
304
306
  end
305
- found = list_entities(type: type, real_path: real_path, query: query, item_list_key: item_list_key).select{|i|i[field].eql?(value)}
307
+ found = list_entities(type: type, real_path: real_path, query: query, item_list_key: item_list_key).select{ |i| i[field].eql?(value)}
306
308
  case found.length
307
309
  when 0 then raise "No #{type} with #{field} = #{value}"
308
310
  when 1 then return found.first
@@ -348,28 +350,28 @@ module Aspera
348
350
  case package_ids
349
351
  when SpecialValues::INIT
350
352
  Aspera.assert(skip_ids_persistency){'Only with option once_only'}
351
- skip_ids_persistency.data.clear.concat(list_packages_with_filter.map{|p|p['id']})
353
+ skip_ids_persistency.data.clear.concat(list_packages_with_filter.map{ |p| p['id']})
352
354
  skip_ids_persistency.save
353
355
  return Main.result_status("Initialized skip for #{skip_ids_persistency.data.count} package(s)")
354
356
  when SpecialValues::ALL
355
357
  # TODO: if packages have same name, they will overwrite ?
356
358
  packages = list_packages_with_filter(query: {'status' => 'completed'})
357
- Log.log.trace1{Log.dump(:package_ids, packages.map{|p|p['id']})}
359
+ Log.log.trace1{Log.dump(:package_ids, packages.map{ |p| p['id']})}
358
360
  Log.log.trace1{Log.dump(:skip_ids, skip_ids_persistency.data)}
359
- packages.reject!{|p|skip_ids_persistency.data.include?(p['id'])} if skip_ids_persistency
360
- Log.log.trace1{Log.dump(:package_ids, packages.map{|p|p['id']})}
361
+ packages.reject!{ |p| skip_ids_persistency.data.include?(p['id'])} if skip_ids_persistency
362
+ Log.log.trace1{Log.dump(:package_ids, packages.map{ |p| p['id']})}
361
363
  else
362
364
  # a single id was provided, or a list of ids
363
365
  package_ids = [package_ids] unless package_ids.is_a?(Array)
364
366
  Aspera.assert_type(package_ids, Array){'Expecting a single package id or a list of ids'}
365
367
  Aspera.assert(package_ids.all?(String)){'Package id shall be String'}
366
368
  # packages = package_ids.map{|pkg_id|@api_v5.read("packages/#{pkg_id}")}
367
- packages = package_ids.map{|pkg_id|{'id'=>pkg_id}}
369
+ packages = package_ids.map{ |pkg_id| {'id'=>pkg_id}}
368
370
  end
369
371
  result_transfer = []
370
372
  param_file_list = {}
371
373
  begin
372
- param_file_list['paths'] = transfer.source_list.map{|source|{'path'=>source}}
374
+ param_file_list['paths'] = transfer.source_list.map{ |source| {'path'=>source}}
373
375
  rescue Cli::BadArgument
374
376
  # paths is optional
375
377
  end
@@ -389,12 +391,12 @@ module Aspera
389
391
  formatter.display_status("Receiving package #{pkg_id}")
390
392
  # TODO: allow from sent as well ?
391
393
  transfer_spec = @api_v5.call(
392
- operation: 'POST',
393
- subpath: "packages/#{pkg_id}/transfer_spec/download",
394
- headers: {'Accept' => 'application/json'},
395
- query: download_params,
396
- body: param_file_list,
397
- body_type: :json
394
+ operation: 'POST',
395
+ subpath: "packages/#{pkg_id}/transfer_spec/download",
396
+ query: download_params,
397
+ content_type: Rest::MIME_JSON,
398
+ body: param_file_list,
399
+ headers: {'Accept' => Rest::MIME_JSON}
398
400
  )[:data]
399
401
  # delete flag for Connect Client
400
402
  transfer_spec.delete('authentication')
@@ -409,45 +411,64 @@ module Aspera
409
411
  return Main.result_transfer_multiple(result_transfer)
410
412
  end
411
413
 
412
- # browse a folder
414
+ # Browse a folder
413
415
  # @param browse_endpoint [String] the endpoint to browse
414
416
  def browse_folder(browse_endpoint)
415
- folders_to_process = [options.get_next_argument('folder path', mandatory: false, default: '/')]
417
+ folders_to_process = [options.get_next_argument('folder path', default: '/')]
416
418
  query = query_read_delete(default: {})
417
- query['filters'] = {} unless query.key?('filters')
418
- filters = query.delete('filters')
419
- filters['basenames'] = [] unless filters.key?('basenames')
419
+ filters = query.delete('filters'){{}}
420
+ Aspera.assert_type(filters, Hash)
421
+ filters['basenames'] ||= []
420
422
  Aspera.assert_type(filters, Hash){'filters'}
421
- max_items = query.delete('max')
423
+ max_items = query.delete(MAX_ITEMS)
422
424
  recursive = query.delete('recursive')
425
+ use_paging = query.delete('paging'){true}
426
+ if use_paging
427
+ browse_endpoint = "#{browse_endpoint}/page"
428
+ query['per_page'] ||= 500
429
+ else
430
+ query['offset'] ||= 0
431
+ query['limit'] ||= 500
432
+ end
423
433
  all_items = []
434
+ total_count = nil
424
435
  until folders_to_process.empty?
425
436
  path = folders_to_process.shift
426
437
  loop do
427
438
  response = @api_v5.call(
428
- operation: 'POST',
429
- subpath: browse_endpoint,
430
- headers: {'Accept' => 'application/json'},
431
- query: query,
432
- body: {'path' => path, 'filters' => filters},
433
- body_type: :json)
439
+ operation: 'POST',
440
+ subpath: browse_endpoint,
441
+ query: query,
442
+ content_type: Rest::MIME_JSON,
443
+ body: {'path' => path, 'filters' => filters},
444
+ headers: {'Accept' => Rest::MIME_JSON}
445
+ )
434
446
  all_items.concat(response[:data]['items'])
435
- if recursive
436
- folders_to_process.concat(response[:data]['items'].select{|i|i['type'].eql?('directory')}.map{|i|i['path']})
437
- end
438
447
  if !max_items.nil? && (all_items.count >= max_items)
439
448
  all_items = all_items.slice(0, max_items) if all_items.count > max_items
440
449
  break
441
450
  end
442
- iteration_token = response[:http][HEADER_ITERATION_TOKEN]
443
- break if iteration_token.nil? || iteration_token.empty?
444
- query['iteration_token'] = iteration_token
451
+ if recursive
452
+ folders_to_process.concat(response[:data]['items'].select{ |i| i['type'].eql?('directory')}.map{ |i| i['path']})
453
+ end
454
+ if use_paging
455
+ iteration_token = response[:http][HEADER_ITERATION_TOKEN]
456
+ break if iteration_token.nil? || iteration_token.empty?
457
+ query['iteration_token'] = iteration_token
458
+ else
459
+ if total_count.nil?
460
+ total_count = response[:data]['total_count']
461
+ end
462
+ break if response[:data]['item_count'].eql?(0)
463
+ query['offset'] += response[:data]['item_count']
464
+ end
445
465
  formatter.long_operation_running(all_items.count)
446
466
  end
447
467
  query.delete('iteration_token')
448
468
  end
449
469
  formatter.long_operation_terminated
450
- return {type: :object_list, data: all_items}
470
+
471
+ return Main.result_object_list(all_items, total: total_count)
451
472
  end
452
473
 
453
474
  def package_action
@@ -458,7 +479,7 @@ module Aspera
458
479
  end
459
480
  case command
460
481
  when :show
461
- return {type: :single_object, data: @api_v5.read("packages/#{package_id}")}
482
+ return Main.result_single_object(@api_v5.read("packages/#{package_id}"))
462
483
  when :browse
463
484
  location = case options.get_option(:box)
464
485
  when 'inbox' then 'received'
@@ -467,8 +488,9 @@ module Aspera
467
488
  end
468
489
  return browse_folder("packages/#{package_id}/files/#{location}")
469
490
  when :status
470
- status = wait_package_status(package_id, status_list: nil)
471
- return {type: :single_object, data: status}
491
+ status_list = options.get_next_argument('list of states, or nothing', mandatory: false, validation: Array)
492
+ status = wait_package_status(package_id, status_list: status_list)
493
+ return Main.result_single_object(status)
472
494
  when :delete
473
495
  ids = package_id
474
496
  ids = [ids] unless ids.is_a?(Array)
@@ -476,11 +498,12 @@ module Aspera
476
498
  Aspera.assert(ids.all?(String)){"Package id(s) shall be String, but have: #{ids.map(&:class).uniq.join(', ')}"}
477
499
  # API returns 204, empty on success
478
500
  @api_v5.call(
479
- operation: 'DELETE',
480
- subpath: 'packages',
481
- headers: {'Accept' => 'application/json'},
482
- body: {ids: ids},
483
- body_type: :json)
501
+ operation: 'DELETE',
502
+ subpath: 'packages',
503
+ content_type: Rest::MIME_JSON,
504
+ body: {ids: ids},
505
+ headers: {'Accept' => Rest::MIME_JSON}
506
+ )
484
507
  return Main.result_status('Package(s) deleted')
485
508
  when :receive
486
509
  return package_receive(package_id)
@@ -499,12 +522,12 @@ module Aspera
499
522
  if shared_folder.nil?
500
523
  # send from local files
501
524
  transfer_spec = @api_v5.call(
502
- operation: 'POST',
503
- subpath: "packages/#{package['id']}/transfer_spec/upload",
504
- headers: {'Accept' => 'application/json'},
505
- query: {transfer_type: TRANSFER_CONNECT},
506
- body: {paths: transfer.source_list},
507
- body_type: :json
525
+ operation: 'POST',
526
+ subpath: "packages/#{package['id']}/transfer_spec/upload",
527
+ query: {transfer_type: TRANSFER_CONNECT},
528
+ content_type: Rest::MIME_JSON,
529
+ body: {paths: transfer.source_list},
530
+ headers: {'Accept' => Rest::MIME_JSON}
508
531
  )[:data]
509
532
  # well, we asked a TS for connect, but we actually want a generic one
510
533
  transfer_spec.delete('authentication')
@@ -525,7 +548,7 @@ module Aspera
525
548
  formatter.display_status("Package #{package['id']}")
526
549
  result = wait_package_status(package['id'])
527
550
  end
528
- return {type: :single_object, data: result}
551
+ return Main.result_single_object(result)
529
552
  end
530
553
  when :list
531
554
  return {
@@ -600,9 +623,13 @@ module Aspera
600
623
 
601
624
  end
602
625
  when :browse
603
- return browse_folder("#{res_path}/#{instance_identifier}/browse")
626
+ node_id = instance_identifier do |field, value|
627
+ lookup_entity_by_field(
628
+ type: res_type, value: value, field: field, real_path: res_path, item_list_key: list_key, query: res_id_query)['id']
629
+ end
630
+ return browse_folder("#{res_path}/#{node_id}/browse")
604
631
  when :invite_external_collaborator
605
- shared_inbox_id = instance_identifier { |field, value| lookup_entity_by_field(type: res_type.to_s, field: field, value: value, query: res_id_query)['id']}
632
+ shared_inbox_id = instance_identifier{ |field, value| lookup_entity_by_field(type: res_type.to_s, field: field, value: value, query: res_id_query)['id']}
606
633
  creation_payload = value_create_modify(command: res_command, type: [Hash, String])
607
634
  creation_payload = {'email_address' => creation_payload} if creation_payload.is_a?(String)
608
635
  res_path = "#{res_type}/#{shared_inbox_id}/external_collaborator"
@@ -613,9 +640,9 @@ module Aspera
613
640
  real_path: "#{res_type}/#{shared_inbox_id}/members",
614
641
  value: creation_payload['email_address'],
615
642
  query: {})
616
- return {type: :single_object, data: result}
643
+ return Main.result_single_object(result)
617
644
  when :members, :saml_groups
618
- res_id = instance_identifier { |field, value| lookup_entity_by_field(type: res_type.to_s, field: field, value: value, query: res_id_query)['id']}
645
+ res_id = instance_identifier{ |field, value| lookup_entity_by_field(type: res_type.to_s, field: field, value: value, query: res_id_query)['id']}
619
646
  res_prefix = "#{res_type}/#{res_id}"
620
647
  res_path = "#{res_prefix}/#{res_command}"
621
648
  list_key = res_command.to_s
@@ -638,7 +665,7 @@ module Aspera
638
665
  end
639
666
  end
640
667
  access = options.get_next_argument('level', mandatory: false, accept_list: %i[submit_only standard shared_inbox_admin], default: :standard)
641
- options.unshift_next_argument({user: users.map{|u|{id: u, access: access}}})
668
+ options.unshift_next_argument({user: users.map{ |u| {id: u, access: access}}})
642
669
  end
643
670
  return entity_command(sub_command, adm_api, res_path, item_list_key: list_key) do |field, value|
644
671
  lookup_entity_by_field(
@@ -648,7 +675,7 @@ module Aspera
648
675
  query: {type: Rest.array_params(%w{local_user saml_user self_registered_user external_user})})['id']
649
676
  end
650
677
  when :reset_password
651
- contact_id = instance_identifier { |field, value| lookup_entity_by_field(type: res_type.to_s, field: field, value: value, query: res_id_query)['id']}
678
+ contact_id = instance_identifier{ |field, value| lookup_entity_by_field(type: res_type.to_s, field: field, value: value, query: res_id_query)['id']}
652
679
  adm_api.create("#{res_type}/#{contact_id}/reset_password", {})
653
680
  return Main.result_status('password reset, user shall check email')
654
681
  end
@@ -668,35 +695,37 @@ module Aspera
668
695
  delete_data = value_create_modify(command: command, default: {})
669
696
  delete_data = @api_v5.read('configuration').slice('days_before_deleting_package_records') if delete_data.empty?
670
697
  res = @api_v5.create('internal/packages/clean_deleted', delete_data)
671
- return {type: :single_object, data: res}
698
+ return Main.result_single_object(res)
672
699
  when :events
673
700
  event_type = options.get_next_command(%i[application webhook])
674
701
  case event_type
675
702
  when :application
676
- return {type: :object_list, data: list_entities(type: 'application_events', query: query_read_delete),
677
- fields: %w[event_type created_at application user.name]}
703
+ return Main.result_object_list(
704
+ list_entities(type: 'application_events', query: query_read_delete),
705
+ fields: %w[event_type created_at application user.name])
678
706
  when :webhook
679
- return {type: :object_list, data: list_entities(type: 'all_webhooks_events', query: query_read_delete, item_list_key: 'events')}
707
+ return Main.result_object_list(
708
+ list_entities(type: 'all_webhooks_events', query: query_read_delete, item_list_key: 'events'))
680
709
  end
681
710
  when :configuration
682
711
  conf_path = 'configuration'
683
712
  conf_cmd = options.get_next_command(%i[show modify])
684
713
  case conf_cmd
685
714
  when :show
686
- return { type: :single_object, data: @api_v5.read(conf_path) }
715
+ return Main.result_single_object(@api_v5.read(conf_path))
687
716
  when :modify
688
- return { type: :single_object, data: @api_v5.update(conf_path, value_create_modify(command: conf_cmd)) }
717
+ return Main.result_single_object(@api_v5.update(conf_path, value_create_modify(command: conf_cmd)))
689
718
  end
690
719
  when :smtp
691
720
  smtp_path = 'configuration/smtp'
692
721
  smtp_cmd = options.get_next_command(%i[show create modify delete test])
693
722
  case smtp_cmd
694
723
  when :show
695
- return { type: :single_object, data: @api_v5.read(smtp_path) }
724
+ return Main.result_single_object(@api_v5.read(smtp_path))
696
725
  when :create
697
- return { type: :single_object, data: @api_v5.create(smtp_path, value_create_modify(command: smtp_cmd)) }
726
+ return Main.result_single_object(@api_v5.create(smtp_path, value_create_modify(command: smtp_cmd)))
698
727
  when :modify
699
- return { type: :single_object, data: @api_v5.update(smtp_path, value_create_modify(command: smtp_cmd)) }
728
+ return Main.result_single_object(@api_v5.update(smtp_path, value_create_modify(command: smtp_cmd)))
700
729
  when :delete
701
730
  @api_v5.delete(smtp_path)
702
731
  return Main.result_status('SMTP configuration deleted')
@@ -706,7 +735,7 @@ fields: %w[event_type created_at application user.name]}
706
735
  creation = @api_v5.create(File.join(smtp_path, 'test'), test_data)
707
736
  result = wait_for_job(creation['job_id'])
708
737
  result['serialized_args'] = JSON.parse(result['serialized_args']) rescue result['serialized_args']
709
- return { type: :single_object, data: result }
738
+ return Main.result_single_object(result)
710
739
  end
711
740
  end
712
741
  end
@@ -718,7 +747,7 @@ fields: %w[event_type created_at application user.name]}
718
747
  set_api unless command.eql?(:postprocessing)
719
748
  case command
720
749
  when :version
721
- return { type: :single_object, data: @api_v5.read('version') }
750
+ return Main.result_single_object(@api_v5.read('version'))
722
751
  when :health
723
752
  nagios = Nagios.new
724
753
  begin
@@ -733,33 +762,33 @@ fields: %w[event_type created_at application user.name]}
733
762
  when :user
734
763
  case options.get_next_command(%i[account profile])
735
764
  when :account
736
- return { type: :single_object, data: @api_v5.read('account') }
765
+ return Main.result_single_object(@api_v5.read('account'))
737
766
  when :profile
738
767
  case options.get_next_command(%i[show modify])
739
768
  when :show
740
- return { type: :single_object, data: @api_v5.read('account/preferences') }
769
+ return Main.result_single_object(@api_v5.read('account/preferences'))
741
770
  when :modify
742
771
  @api_v5.update('account/preferences', options.get_next_argument('modified parameters', validation: Hash))
743
772
  return Main.result_status('modified')
744
773
  end
745
774
  end
746
775
  when :bearer_token
747
- return {type: :text, data: @api_v5.oauth.token}
776
+ return Main.result_text(@api_v5.oauth.authorization)
748
777
  when :packages
749
778
  return package_action
750
779
  when :shared_folders
751
780
  all_shared_folders = @api_v5.read('shared_folders')['shared_folders']
752
781
  case options.get_next_command(%i[list browse])
753
782
  when :list
754
- return {type: :object_list, data: all_shared_folders}
783
+ return Main.result_object_list(all_shared_folders)
755
784
  when :browse
756
785
  shared_folder_id = instance_identifier do |field, value|
757
- matches = all_shared_folders.select{|i|i[field].eql?(value)}
786
+ matches = all_shared_folders.select{ |i| i[field].eql?(value)}
758
787
  raise "no match for #{field} = #{value}" if matches.empty?
759
788
  raise "multiple matches for #{field} = #{value}" if matches.length > 1
760
789
  matches.first['id']
761
790
  end
762
- node = all_shared_folders.find{|i|i['id'].eql?(shared_folder_id)}
791
+ node = all_shared_folders.find{ |i| i['id'].eql?(shared_folder_id)}
763
792
  raise "No such shared folder id #{shared_folder_id}" if node.nil?
764
793
  return browse_folder("nodes/#{node['node_id']}/shared_folders/#{shared_folder_id}/browse")
765
794
  end
@@ -786,22 +815,20 @@ fields: %w[event_type created_at application user.name]}
786
815
  end
787
816
  when :gateway
788
817
  require 'aspera/faspex_gw'
789
- url = value_create_modify(command: command, description: 'listening url (e.g. https://localhost:12345)', type: String)
790
- uri = URI.parse(url)
791
- server = WebServerSimple.new(uri)
818
+ parameters = value_create_modify(command: command, default: {}).symbolize_keys
819
+ uri = URI.parse(parameters.delete(:url){WebServerSimple::DEFAULT_URL})
820
+ server = WebServerSimple.new(uri, **parameters.slice(*WebServerSimple::PARAMS))
821
+ Aspera.assert(parameters.except(*WebServerSimple::PARAMS).empty?)
792
822
  server.mount(uri.path, Faspex4GWServlet, @api_v5, nil)
793
823
  server.start
794
824
  return Main.result_status('Gateway terminated')
795
825
  when :postprocessing
796
826
  require 'aspera/faspex_postproc' # cspell:disable-line
797
- parameters = value_create_modify(command: command)
798
- parameters = parameters.symbolize_keys
799
- Aspera.assert(parameters.key?(:url)){'Missing key: url'}
800
- uri = URI.parse(parameters[:url])
801
- parameters[:processing] ||= {}
802
- parameters[:processing][:root] = uri.path
803
- server = WebServerSimple.new(uri, certificate: parameters[:certificate])
804
- server.mount(uri.path, Faspex4PostProcServlet, parameters[:processing])
827
+ parameters = value_create_modify(command: command, default: {}).symbolize_keys
828
+ uri = URI.parse(parameters.delete(:url){WebServerSimple::DEFAULT_URL})
829
+ parameters[:root] = uri.path
830
+ server = WebServerSimple.new(uri, **parameters.slice(*WebServerSimple::PARAMS))
831
+ server.mount(uri.path, Faspex4PostProcServlet, parameters.except(*WebServerSimple::PARAMS))
805
832
  server.start
806
833
  return Main.result_status('Gateway terminated')
807
834
  end