aspera-cli 4.21.2 → 4.22.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (97) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/BUGS.md +1 -1
  4. data/CHANGELOG.md +34 -16
  5. data/CONTRIBUTING.md +6 -10
  6. data/README.md +805 -574
  7. data/examples/get_proto_file.rb +1 -1
  8. data/lib/aspera/agent/base.rb +9 -5
  9. data/lib/aspera/agent/connect.rb +30 -28
  10. data/lib/aspera/agent/desktop.rb +29 -25
  11. data/lib/aspera/agent/direct.rb +137 -125
  12. data/lib/aspera/agent/httpgw.rb +22 -26
  13. data/lib/aspera/agent/node.rb +14 -11
  14. data/lib/aspera/agent/transferd.rb +6 -2
  15. data/lib/aspera/api/aoc.rb +6 -6
  16. data/lib/aspera/api/cos_node.rb +1 -1
  17. data/lib/aspera/api/httpgw.rb +7 -3
  18. data/lib/aspera/api/node.rb +6 -4
  19. data/lib/aspera/ascmd.rb +3 -3
  20. data/lib/aspera/ascp/installation.rb +15 -16
  21. data/lib/aspera/ascp/management.rb +1 -1
  22. data/lib/aspera/assert.rb +11 -2
  23. data/lib/aspera/cli/error.rb +2 -2
  24. data/lib/aspera/cli/extended_value.rb +38 -19
  25. data/lib/aspera/cli/formatter.rb +48 -48
  26. data/lib/aspera/cli/hints.rb +1 -1
  27. data/lib/aspera/cli/main.rb +190 -168
  28. data/lib/aspera/cli/manager.rb +15 -15
  29. data/lib/aspera/cli/plugin.rb +23 -20
  30. data/lib/aspera/cli/plugin_factory.rb +1 -1
  31. data/lib/aspera/cli/plugins/alee.rb +1 -1
  32. data/lib/aspera/cli/plugins/aoc.rb +144 -107
  33. data/lib/aspera/cli/plugins/ats.rb +19 -17
  34. data/lib/aspera/cli/plugins/config.rb +67 -83
  35. data/lib/aspera/cli/plugins/console.rb +5 -3
  36. data/lib/aspera/cli/plugins/faspex.rb +39 -35
  37. data/lib/aspera/cli/plugins/faspex5.rb +104 -80
  38. data/lib/aspera/cli/plugins/faspio.rb +13 -1
  39. data/lib/aspera/cli/plugins/httpgw.rb +13 -1
  40. data/lib/aspera/cli/plugins/node.rb +306 -179
  41. data/lib/aspera/cli/plugins/orchestrator.rb +34 -40
  42. data/lib/aspera/cli/plugins/preview.rb +3 -3
  43. data/lib/aspera/cli/plugins/server.rb +6 -6
  44. data/lib/aspera/cli/plugins/shares.rb +5 -5
  45. data/lib/aspera/cli/sync_actions.rb +19 -18
  46. data/lib/aspera/cli/transfer_agent.rb +5 -5
  47. data/lib/aspera/cli/transfer_progress.rb +2 -2
  48. data/lib/aspera/cli/version.rb +1 -1
  49. data/lib/aspera/command_line_builder.rb +116 -95
  50. data/lib/aspera/coverage.rb +4 -3
  51. data/lib/aspera/environment.rb +6 -6
  52. data/lib/aspera/faspex_gw.rb +14 -14
  53. data/lib/aspera/faspex_postproc.rb +7 -6
  54. data/lib/aspera/hash_ext.rb +2 -2
  55. data/lib/aspera/json_rpc.rb +1 -1
  56. data/lib/aspera/keychain/encrypted_hash.rb +47 -34
  57. data/lib/aspera/keychain/factory.rb +41 -0
  58. data/lib/aspera/keychain/hashicorp_vault.rb +71 -0
  59. data/lib/aspera/keychain/macos_security.rb +19 -11
  60. data/lib/aspera/log.rb +28 -34
  61. data/lib/aspera/nagios.rb +6 -6
  62. data/lib/aspera/node_simulator.rb +8 -8
  63. data/lib/aspera/oauth/base.rb +8 -6
  64. data/lib/aspera/oauth/factory.rb +5 -6
  65. data/lib/aspera/oauth/url_json.rb +6 -6
  66. data/lib/aspera/persistency_action_once.rb +6 -4
  67. data/lib/aspera/persistency_folder.rb +2 -2
  68. data/lib/aspera/preview/generator.rb +1 -1
  69. data/lib/aspera/preview/options.rb +16 -16
  70. data/lib/aspera/preview/terminal.rb +3 -3
  71. data/lib/aspera/preview/utils.rb +11 -13
  72. data/lib/aspera/products/connect.rb +1 -1
  73. data/lib/aspera/products/desktop.rb +1 -1
  74. data/lib/aspera/products/transferd.rb +1 -1
  75. data/lib/aspera/proxy_auto_config.rb +2 -2
  76. data/lib/aspera/rest.rb +52 -43
  77. data/lib/aspera/rest_errors_aspera.rb +1 -1
  78. data/lib/aspera/secret_hider.rb +5 -5
  79. data/lib/aspera/ssh.rb +4 -4
  80. data/lib/aspera/transfer/convert.rb +29 -0
  81. data/lib/aspera/transfer/error_info.rb +66 -66
  82. data/lib/aspera/transfer/parameters.rb +13 -68
  83. data/lib/aspera/transfer/spec.rb +5 -6
  84. data/lib/aspera/transfer/spec.schema.yaml +753 -0
  85. data/lib/aspera/transfer/spec_doc.rb +62 -0
  86. data/lib/aspera/transfer/sync.rb +23 -72
  87. data/lib/aspera/transfer/sync_instance.schema.yaml +13 -0
  88. data/lib/aspera/transfer/sync_session.schema.yaml +79 -0
  89. data/lib/aspera/transfer/uri.rb +6 -6
  90. data/lib/aspera/uri_reader.rb +1 -1
  91. data/lib/aspera/web_auth.rb +1 -1
  92. data/lib/aspera/web_server_simple.rb +53 -44
  93. data.tar.gz.sig +1 -2
  94. metadata +37 -4
  95. metadata.gz.sig +0 -0
  96. data/examples/build_package.sh +0 -28
  97. data/lib/aspera/transfer/spec.yaml +0 -718
@@ -52,7 +52,7 @@ module Aspera
52
52
  result = api.call(
53
53
  operation: 'POST',
54
54
  headers: {
55
- 'Content-type' => 'text/plain',
55
+ 'Content-type' => Rest::MIME_TEXT,
56
56
  'Accept' => 'application/xrds+xml'
57
57
  }
58
58
  )
@@ -74,7 +74,9 @@ module Aspera
74
74
  return nil
75
75
  end
76
76
 
77
- def wizard(object:, private_key_path: nil, pub_key_pem: nil)
77
+ # @param object [Plugin] An instance of this class
78
+ # @return [Hash] :preset_value, :test_args
79
+ def wizard(object:)
78
80
  options = object.options
79
81
  return {
80
82
  preset_value: {
@@ -89,7 +91,7 @@ module Aspera
89
91
  # extract elements from faspex public link
90
92
  def get_link_data(public_url)
91
93
  public_uri = URI.parse(public_url)
92
- Aspera.assert((m = public_uri.path.match(%r{^(.*)/(external.*)$})), exception_class: Cli::BadArgument){'Public link does not match Faspex format'}
94
+ Aspera.assert(m = public_uri.path.match(%r{^(.*)/(external.*)$}), exception_class: Cli::BadArgument){'Public link does not match Faspex format'}
93
95
  base = m[1]
94
96
  subpath = m[2]
95
97
  port_add = public_uri.port.eql?(public_uri.default_port) ? '' : ":#{public_uri.port}"
@@ -108,15 +110,15 @@ module Aspera
108
110
  raise Cli::BadArgument, 'package has no link (deleted?)' if raise_no_link
109
111
  return nil
110
112
  end
111
- result = entry['link'].find{|e| e['rel'].eql?('package')}['href']
113
+ result = entry['link'].find{ |e| e['rel'].eql?('package')}['href']
112
114
  return result
113
115
  end
114
116
 
115
117
  # @return [Integer] identifier of source
116
118
  def get_source_id_by_name(source_name, source_list)
117
- match_source = source_list.find { |i| i['name'].eql?(source_name) }
119
+ match_source = source_list.find{ |i| i['name'].eql?(source_name)}
118
120
  return match_source['id'] unless match_source.nil?
119
- raise Cli::Error, %Q(No such Faspex source: "#{source_name}" in [#{source_list.map{|i| %Q("#{i['name']}")}.join(', ')}])
121
+ raise Cli::Error, %Q(No such Faspex source: "#{source_name}" in [#{source_list.map{ |i| %Q("#{i['name']}")}.join(', ')}])
120
122
  end
121
123
  end
122
124
 
@@ -199,14 +201,14 @@ module Aspera
199
201
  package[PACKAGE_MATCH_FIELD] =
200
202
  case mailbox
201
203
  when :inbox, :archive
202
- recipient = package['to'].find{|i|recipient_names.include?(i['name'])}
204
+ recipient = package['to'].find{ |i| recipient_names.include?(i['name'])}
203
205
  recipient.nil? ? nil : recipient['recipient_delivery_id']
204
206
  else # :sent
205
207
  package['delivery_id']
206
208
  end
207
209
  # add special key
208
210
  package['items'] = package['link'].is_a?(Array) ? package['link'].length : 0
209
- package['metadata'] = package['metadata']['field'].each_with_object({}){|i, m| m[i['name']] = i['content'] }
211
+ package['metadata'] = package['metadata']['field'].each_with_object({}){ |i, m| m[i['name']] = i['content']}
210
212
  # if we look for a specific package
211
213
  stop_condition = true if !stop_at_id.nil? && stop_at_id.eql?(package[PACKAGE_MATCH_FIELD])
212
214
  # keep only those for the specified recipient
@@ -220,13 +222,13 @@ module Aspera
220
222
  result = result.slice(0, max_items) if result.count > max_items
221
223
  break
222
224
  end
223
- link = box_data['link'].find{|i|i['rel'].eql?('next')}
225
+ link = box_data['link'].find{ |i| i['rel'].eql?('next')}
224
226
  Log.log.debug{"link: #{link}"}
225
227
  # no next link
226
228
  break if link.nil?
227
229
  # replace parameters with the ones from next link
228
230
  params = CGI.parse(URI.parse(link['href']).query)
229
- mailbox_query = params.keys.each_with_object({}){|i, m| m[i] = params[i].first }
231
+ mailbox_query = params.keys.each_with_object({}){ |i, m| m[i] = params[i].first}
230
232
  Log.log.debug{"query: #{mailbox_query}"}
231
233
  break if !max_pages.nil? && (mailbox_query['page'].to_i > max_pages)
232
234
  end
@@ -250,11 +252,12 @@ module Aspera
250
252
  # pkg_created=api_public_link.create(create_path,package_create_params)
251
253
  # so extract data from javascript
252
254
  package_creation_data = api_public_link.call(
253
- operation: 'POST',
254
- subpath: create_path,
255
- headers: {'Accept' => 'text/javascript'},
256
- body: package_create_params,
257
- body_type: :json)[:http].body
255
+ operation: 'POST',
256
+ subpath: create_path,
257
+ content_type: Rest::MIME_JSON,
258
+ body: package_create_params,
259
+ headers: {'Accept' => 'text/javascript'}
260
+ )[:http].body
258
261
  # get arguments of function call
259
262
  package_creation_data.delete!("\n") # one line
260
263
  package_creation_data.gsub!(/^[^"]+\("\{/, '{') # delete header
@@ -288,7 +291,7 @@ module Aspera
288
291
  case command_pkg
289
292
  when :show
290
293
  delivery_id = instance_identifier
291
- return {type: :single_object, data: mailbox_filtered_entries(stop_at_id: delivery_id).find{|p|p[PACKAGE_MATCH_FIELD].eql?(delivery_id)} }
294
+ return Main.result_single_object(mailbox_filtered_entries(stop_at_id: delivery_id).find{ |p| p[PACKAGE_MATCH_FIELD].eql?(delivery_id)})
292
295
  when :list
293
296
  return {
294
297
  type: :object_list,
@@ -350,14 +353,14 @@ module Aspera
350
353
  raise 'empty id' if delivery_id.empty?
351
354
  recipient = options.get_option(:recipient)
352
355
  if delivery_id.eql?(SpecialValues::ALL)
353
- pkg_id_uri = mailbox_filtered_entries.map{|i|{id: i[PACKAGE_MATCH_FIELD], uri: self.class.get_fasp_uri_from_entry(i, raise_no_link: false)}}
356
+ pkg_id_uri = mailbox_filtered_entries.map{ |i| {id: i[PACKAGE_MATCH_FIELD], uri: self.class.get_fasp_uri_from_entry(i, raise_no_link: false)}}
354
357
  elsif delivery_id.eql?(SpecialValues::INIT)
355
358
  Aspera.assert(skip_ids_persistency){'Only with option once_only'}
356
- skip_ids_persistency.data.clear.concat(mailbox_filtered_entries.map{|i|{id: i[PACKAGE_MATCH_FIELD]}})
359
+ skip_ids_persistency.data.clear.concat(mailbox_filtered_entries.map{ |i| {id: i[PACKAGE_MATCH_FIELD]}})
357
360
  skip_ids_persistency.save
358
361
  return Main.result_status("Initialized skip for #{skip_ids_persistency.data.count} package(s)")
359
362
  elsif !recipient.nil? && recipient.start_with?('*')
360
- found_package_link = mailbox_filtered_entries(stop_at_id: delivery_id).find{|p|p[PACKAGE_MATCH_FIELD].eql?(delivery_id)}['link'].first['href']
363
+ found_package_link = mailbox_filtered_entries(stop_at_id: delivery_id).find{ |p| p[PACKAGE_MATCH_FIELD].eql?(delivery_id)}['link'].first['href']
361
364
  raise "Not Found. Dropbox and Workgroup packages can use the link option with #{Transfer::Uri::SCHEME}" if found_package_link.nil?
362
365
  pkg_id_uri = [{id: delivery_id, uri: found_package_link}]
363
366
  else
@@ -397,7 +400,7 @@ module Aspera
397
400
  # prune packages already downloaded
398
401
  # TODO : remove ids from skip not present in inbox to avoid growing too big
399
402
  # skip_ids_data.select!{|id|pkg_id_uri.select{|p|p[:id].eql?(id)}}
400
- pkg_id_uri.reject!{|i|skip_ids_data.include?(i[:id])}
403
+ pkg_id_uri.reject!{ |i| skip_ids_data.include?(i[:id])}
401
404
  Log.log.debug{Log.dump(:pkg_id_uri, pkg_id_uri)}
402
405
  return Main.result_status('no new package') if pkg_id_uri.empty?
403
406
  result_transfer = []
@@ -413,12 +416,13 @@ module Aspera
413
416
  xml_payload =
414
417
  %Q(<?xml version="1.0" encoding="UTF-8"?><url-list xmlns="http://schemas.asperasoft.com/xml/url-list"><url href="#{sanitized}"/></url-list>)
415
418
  transfer_spec['token'] = api_v3.call(
416
- operation: 'POST',
417
- subpath: 'issue-token',
418
- headers: {'Accept' => 'text/plain', 'Content-Type' => 'application/vnd.aspera.url-list+xml'},
419
- query: {'direction' => 'down'},
420
- body: xml_payload,
421
- body_type: :text)[:http].body
419
+ operation: 'POST',
420
+ subpath: 'issue-token',
421
+ query: {'direction' => 'down'},
422
+ content_type: Rest::MIME_TEXT,
423
+ body: xml_payload,
424
+ headers: {'Accept' => Rest::MIME_TEXT, 'Content-Type' => 'application/vnd.aspera.url-list+xml'}
425
+ )[:http].body
422
426
  end
423
427
  transfer_spec['direction'] = Transfer::Spec::DIRECTION_RECEIVE
424
428
  statuses = transfer.start(transfer_spec)
@@ -435,13 +439,13 @@ module Aspera
435
439
  source_list = api_v3.read('source_shares')['items']
436
440
  case command_source
437
441
  when :list
438
- return {type: :object_list, data: source_list}
442
+ return Main.result_object_list(source_list)
439
443
  else # :info :node
440
444
  source_id = instance_identifier do |field, value|
441
445
  Aspera.assert(field.eql?('name'), exception_class: Cli::BadArgument){'only name as selector, or give id'}
442
446
  self.class.get_source_id_by_name(value, source_list)
443
447
  end.to_i
444
- selected_source = source_list.find{|i|i['id'].eql?(source_id)}
448
+ selected_source = source_list.find{ |i| i['id'].eql?(source_id)}
445
449
  raise 'No such source' if selected_source.nil?
446
450
  source_name = selected_source['name']
447
451
  source_hash = options.get_option(:storage, mandatory: true)
@@ -460,7 +464,7 @@ module Aspera
460
464
  Log.log.debug{Log.dump(:source_info, source_info)}
461
465
  case command_source
462
466
  when :info
463
- return {data: source_info, type: :single_object}
467
+ return Main.result_single_object(source_info)
464
468
  when :node
465
469
  node_config = ExtendedValue.instance.evaluate(source_info[KEY_NODE])
466
470
  Log.log.debug{"node=#{node_config}"}
@@ -472,18 +476,18 @@ module Aspera
472
476
  username: node_config['username'],
473
477
  password: node_config['password']})
474
478
  command = options.get_next_command(Node::COMMANDS_FASPEX)
475
- return Node.new(**init_params, api: api_node).execute_action(command, source_info[KEY_PATH])
479
+ return Node.new(**init_params, api: api_node, prefix_path: source_info[KEY_PATH]).execute_action(command)
476
480
  end
477
481
  end
478
482
  when :me
479
483
  my_info = api_v3.read('me')
480
- return {data: my_info, type: :single_object}
484
+ return Main.result_single_object(my_info)
481
485
  when :dropbox
482
486
  command_pkg = options.get_next_command([:list])
483
487
  case command_pkg
484
488
  when :list
485
489
  dropbox_list = api_v3.read('dropboxes')
486
- return {type: :object_list, data: dropbox_list['items'], fields: %w[name id description can_read can_write]}
490
+ return Main.result_object_list(dropbox_list['items'], fields: %w[name id description can_read can_write])
487
491
  end
488
492
  when :v4
489
493
  command = options.get_next_command(%i[package dropbox dmembership workgroup wmembership user metadata_profile])
@@ -512,7 +516,7 @@ module Aspera
512
516
  # add missing entries
513
517
  users.each do |u|
514
518
  unless u['emails'].nil?
515
- email = u['emails'].find{|i|i['primary'].eql?('true')}
519
+ email = u['emails'].find{ |i| i['primary'].eql?('true')}
516
520
  u['email'] = email['value'] unless email.nil?
517
521
  end
518
522
  if u['email'].nil?
@@ -522,11 +526,11 @@ module Aspera
522
526
  u['first_name'], u['last_name'] = u['displayName'].split(' ', 2)
523
527
  u['x'] = true
524
528
  end
525
- return {type: :object_list, data: users}
529
+ return Main.result_object_list(users)
526
530
  when :login_methods
527
531
  login_meths = api_v3.call(operation: 'GET', subpath: 'login/new', headers: {'Accept' => 'application/xrds+xml'})[:http].body
528
532
  login_methods = XmlSimple.xml_in(login_meths, {'ForceArray' => false})
529
- return {type: :object_list, data: login_methods['XRD']['Service']}
533
+ return Main.result_object_list(login_methods['XRD']['Service'])
530
534
  end
531
535
  end
532
536
  end
@@ -74,6 +74,10 @@ module Aspera
74
74
  return nil
75
75
  end
76
76
 
77
+ # @param object [Plugin] An instance of this class
78
+ # @param private_key_path [String] path to private key
79
+ # @param pub_key_pem [String] PEM of public key
80
+ # @return [Hash] :preset_value, :test_args
77
81
  def wizard(object:, private_key_path:, pub_key_pem:)
78
82
  options = object.options
79
83
  formatter = object.formatter
@@ -183,7 +187,7 @@ module Aspera
183
187
  else Aspera.error_unexpected_value(auth_type)
184
188
  end
185
189
  # in case user wants to use HTTPGW tell transfer agent how to get address
186
- transfer.httpgw_url_cb = lambda { @api_v5.read('account')['gateway_url'] }
190
+ transfer.httpgw_url_cb = lambda{@api_v5.read('account')['gateway_url']}
187
191
  end
188
192
 
189
193
  # if recipient is just an email, then convert to expected API hash : name and type
@@ -300,7 +304,7 @@ module Aspera
300
304
  Aspera.assert(field.eql?('name')){'Default query is on name only'}
301
305
  query = {'q'=> value}
302
306
  end
303
- found = list_entities(type: type, real_path: real_path, query: query, item_list_key: item_list_key).select{|i|i[field].eql?(value)}
307
+ found = list_entities(type: type, real_path: real_path, query: query, item_list_key: item_list_key).select{ |i| i[field].eql?(value)}
304
308
  case found.length
305
309
  when 0 then raise "No #{type} with #{field} = #{value}"
306
310
  when 1 then return found.first
@@ -346,28 +350,28 @@ module Aspera
346
350
  case package_ids
347
351
  when SpecialValues::INIT
348
352
  Aspera.assert(skip_ids_persistency){'Only with option once_only'}
349
- skip_ids_persistency.data.clear.concat(list_packages_with_filter.map{|p|p['id']})
353
+ skip_ids_persistency.data.clear.concat(list_packages_with_filter.map{ |p| p['id']})
350
354
  skip_ids_persistency.save
351
355
  return Main.result_status("Initialized skip for #{skip_ids_persistency.data.count} package(s)")
352
356
  when SpecialValues::ALL
353
357
  # TODO: if packages have same name, they will overwrite ?
354
358
  packages = list_packages_with_filter(query: {'status' => 'completed'})
355
- Log.log.trace1{Log.dump(:package_ids, packages.map{|p|p['id']})}
359
+ Log.log.trace1{Log.dump(:package_ids, packages.map{ |p| p['id']})}
356
360
  Log.log.trace1{Log.dump(:skip_ids, skip_ids_persistency.data)}
357
- packages.reject!{|p|skip_ids_persistency.data.include?(p['id'])} if skip_ids_persistency
358
- Log.log.trace1{Log.dump(:package_ids, packages.map{|p|p['id']})}
361
+ packages.reject!{ |p| skip_ids_persistency.data.include?(p['id'])} if skip_ids_persistency
362
+ Log.log.trace1{Log.dump(:package_ids, packages.map{ |p| p['id']})}
359
363
  else
360
364
  # a single id was provided, or a list of ids
361
365
  package_ids = [package_ids] unless package_ids.is_a?(Array)
362
366
  Aspera.assert_type(package_ids, Array){'Expecting a single package id or a list of ids'}
363
367
  Aspera.assert(package_ids.all?(String)){'Package id shall be String'}
364
368
  # packages = package_ids.map{|pkg_id|@api_v5.read("packages/#{pkg_id}")}
365
- packages = package_ids.map{|pkg_id|{'id'=>pkg_id}}
369
+ packages = package_ids.map{ |pkg_id| {'id'=>pkg_id}}
366
370
  end
367
371
  result_transfer = []
368
372
  param_file_list = {}
369
373
  begin
370
- param_file_list['paths'] = transfer.source_list.map{|source|{'path'=>source}}
374
+ param_file_list['paths'] = transfer.source_list.map{ |source| {'path'=>source}}
371
375
  rescue Cli::BadArgument
372
376
  # paths is optional
373
377
  end
@@ -387,12 +391,12 @@ module Aspera
387
391
  formatter.display_status("Receiving package #{pkg_id}")
388
392
  # TODO: allow from sent as well ?
389
393
  transfer_spec = @api_v5.call(
390
- operation: 'POST',
391
- subpath: "packages/#{pkg_id}/transfer_spec/download",
392
- headers: {'Accept' => 'application/json'},
393
- query: download_params,
394
- body: param_file_list,
395
- body_type: :json
394
+ operation: 'POST',
395
+ subpath: "packages/#{pkg_id}/transfer_spec/download",
396
+ query: download_params,
397
+ content_type: Rest::MIME_JSON,
398
+ body: param_file_list,
399
+ headers: {'Accept' => Rest::MIME_JSON}
396
400
  )[:data]
397
401
  # delete flag for Connect Client
398
402
  transfer_spec.delete('authentication')
@@ -407,45 +411,64 @@ module Aspera
407
411
  return Main.result_transfer_multiple(result_transfer)
408
412
  end
409
413
 
410
- # browse a folder
414
+ # Browse a folder
411
415
  # @param browse_endpoint [String] the endpoint to browse
412
416
  def browse_folder(browse_endpoint)
413
- folders_to_process = [options.get_next_argument('folder path', mandatory: false, default: '/')]
417
+ folders_to_process = [options.get_next_argument('folder path', default: '/')]
414
418
  query = query_read_delete(default: {})
415
- query['filters'] = {} unless query.key?('filters')
416
- filters = query.delete('filters')
417
- filters['basenames'] = [] unless filters.key?('basenames')
419
+ filters = query.delete('filters'){{}}
420
+ Aspera.assert_type(filters, Hash)
421
+ filters['basenames'] ||= []
418
422
  Aspera.assert_type(filters, Hash){'filters'}
419
- max_items = query.delete('max')
423
+ max_items = query.delete(MAX_ITEMS)
420
424
  recursive = query.delete('recursive')
425
+ use_paging = query.delete('paging'){true}
426
+ if use_paging
427
+ browse_endpoint = "#{browse_endpoint}/page"
428
+ query['per_page'] ||= 500
429
+ else
430
+ query['offset'] ||= 0
431
+ query['limit'] ||= 500
432
+ end
421
433
  all_items = []
434
+ total_count = nil
422
435
  until folders_to_process.empty?
423
436
  path = folders_to_process.shift
424
437
  loop do
425
438
  response = @api_v5.call(
426
- operation: 'POST',
427
- subpath: browse_endpoint,
428
- headers: {'Accept' => 'application/json'},
429
- query: query,
430
- body: {'path' => path, 'filters' => filters},
431
- body_type: :json)
439
+ operation: 'POST',
440
+ subpath: browse_endpoint,
441
+ query: query,
442
+ content_type: Rest::MIME_JSON,
443
+ body: {'path' => path, 'filters' => filters},
444
+ headers: {'Accept' => Rest::MIME_JSON}
445
+ )
432
446
  all_items.concat(response[:data]['items'])
433
- if recursive
434
- folders_to_process.concat(response[:data]['items'].select{|i|i['type'].eql?('directory')}.map{|i|i['path']})
435
- end
436
447
  if !max_items.nil? && (all_items.count >= max_items)
437
448
  all_items = all_items.slice(0, max_items) if all_items.count > max_items
438
449
  break
439
450
  end
440
- iteration_token = response[:http][HEADER_ITERATION_TOKEN]
441
- break if iteration_token.nil? || iteration_token.empty?
442
- query['iteration_token'] = iteration_token
451
+ if recursive
452
+ folders_to_process.concat(response[:data]['items'].select{ |i| i['type'].eql?('directory')}.map{ |i| i['path']})
453
+ end
454
+ if use_paging
455
+ iteration_token = response[:http][HEADER_ITERATION_TOKEN]
456
+ break if iteration_token.nil? || iteration_token.empty?
457
+ query['iteration_token'] = iteration_token
458
+ else
459
+ if total_count.nil?
460
+ total_count = response[:data]['total_count']
461
+ end
462
+ break if response[:data]['item_count'].eql?(0)
463
+ query['offset'] += response[:data]['item_count']
464
+ end
443
465
  formatter.long_operation_running(all_items.count)
444
466
  end
445
467
  query.delete('iteration_token')
446
468
  end
447
469
  formatter.long_operation_terminated
448
- return {type: :object_list, data: all_items}
470
+
471
+ return Main.result_object_list(all_items, total: total_count)
449
472
  end
450
473
 
451
474
  def package_action
@@ -456,7 +479,7 @@ module Aspera
456
479
  end
457
480
  case command
458
481
  when :show
459
- return {type: :single_object, data: @api_v5.read("packages/#{package_id}")}
482
+ return Main.result_single_object(@api_v5.read("packages/#{package_id}"))
460
483
  when :browse
461
484
  location = case options.get_option(:box)
462
485
  when 'inbox' then 'received'
@@ -467,7 +490,7 @@ module Aspera
467
490
  when :status
468
491
  status_list = options.get_next_argument('list of states, or nothing', mandatory: false, validation: Array)
469
492
  status = wait_package_status(package_id, status_list: status_list)
470
- return {type: :single_object, data: status}
493
+ return Main.result_single_object(status)
471
494
  when :delete
472
495
  ids = package_id
473
496
  ids = [ids] unless ids.is_a?(Array)
@@ -475,11 +498,12 @@ module Aspera
475
498
  Aspera.assert(ids.all?(String)){"Package id(s) shall be String, but have: #{ids.map(&:class).uniq.join(', ')}"}
476
499
  # API returns 204, empty on success
477
500
  @api_v5.call(
478
- operation: 'DELETE',
479
- subpath: 'packages',
480
- headers: {'Accept' => 'application/json'},
481
- body: {ids: ids},
482
- body_type: :json)
501
+ operation: 'DELETE',
502
+ subpath: 'packages',
503
+ content_type: Rest::MIME_JSON,
504
+ body: {ids: ids},
505
+ headers: {'Accept' => Rest::MIME_JSON}
506
+ )
483
507
  return Main.result_status('Package(s) deleted')
484
508
  when :receive
485
509
  return package_receive(package_id)
@@ -498,12 +522,12 @@ module Aspera
498
522
  if shared_folder.nil?
499
523
  # send from local files
500
524
  transfer_spec = @api_v5.call(
501
- operation: 'POST',
502
- subpath: "packages/#{package['id']}/transfer_spec/upload",
503
- headers: {'Accept' => 'application/json'},
504
- query: {transfer_type: TRANSFER_CONNECT},
505
- body: {paths: transfer.source_list},
506
- body_type: :json
525
+ operation: 'POST',
526
+ subpath: "packages/#{package['id']}/transfer_spec/upload",
527
+ query: {transfer_type: TRANSFER_CONNECT},
528
+ content_type: Rest::MIME_JSON,
529
+ body: {paths: transfer.source_list},
530
+ headers: {'Accept' => Rest::MIME_JSON}
507
531
  )[:data]
508
532
  # well, we asked a TS for connect, but we actually want a generic one
509
533
  transfer_spec.delete('authentication')
@@ -524,7 +548,7 @@ module Aspera
524
548
  formatter.display_status("Package #{package['id']}")
525
549
  result = wait_package_status(package['id'])
526
550
  end
527
- return {type: :single_object, data: result}
551
+ return Main.result_single_object(result)
528
552
  end
529
553
  when :list
530
554
  return {
@@ -605,7 +629,7 @@ module Aspera
605
629
  end
606
630
  return browse_folder("#{res_path}/#{node_id}/browse")
607
631
  when :invite_external_collaborator
608
- shared_inbox_id = instance_identifier { |field, value| lookup_entity_by_field(type: res_type.to_s, field: field, value: value, query: res_id_query)['id']}
632
+ shared_inbox_id = instance_identifier{ |field, value| lookup_entity_by_field(type: res_type.to_s, field: field, value: value, query: res_id_query)['id']}
609
633
  creation_payload = value_create_modify(command: res_command, type: [Hash, String])
610
634
  creation_payload = {'email_address' => creation_payload} if creation_payload.is_a?(String)
611
635
  res_path = "#{res_type}/#{shared_inbox_id}/external_collaborator"
@@ -616,9 +640,9 @@ module Aspera
616
640
  real_path: "#{res_type}/#{shared_inbox_id}/members",
617
641
  value: creation_payload['email_address'],
618
642
  query: {})
619
- return {type: :single_object, data: result}
643
+ return Main.result_single_object(result)
620
644
  when :members, :saml_groups
621
- res_id = instance_identifier { |field, value| lookup_entity_by_field(type: res_type.to_s, field: field, value: value, query: res_id_query)['id']}
645
+ res_id = instance_identifier{ |field, value| lookup_entity_by_field(type: res_type.to_s, field: field, value: value, query: res_id_query)['id']}
622
646
  res_prefix = "#{res_type}/#{res_id}"
623
647
  res_path = "#{res_prefix}/#{res_command}"
624
648
  list_key = res_command.to_s
@@ -641,7 +665,7 @@ module Aspera
641
665
  end
642
666
  end
643
667
  access = options.get_next_argument('level', mandatory: false, accept_list: %i[submit_only standard shared_inbox_admin], default: :standard)
644
- options.unshift_next_argument({user: users.map{|u|{id: u, access: access}}})
668
+ options.unshift_next_argument({user: users.map{ |u| {id: u, access: access}}})
645
669
  end
646
670
  return entity_command(sub_command, adm_api, res_path, item_list_key: list_key) do |field, value|
647
671
  lookup_entity_by_field(
@@ -651,7 +675,7 @@ module Aspera
651
675
  query: {type: Rest.array_params(%w{local_user saml_user self_registered_user external_user})})['id']
652
676
  end
653
677
  when :reset_password
654
- contact_id = instance_identifier { |field, value| lookup_entity_by_field(type: res_type.to_s, field: field, value: value, query: res_id_query)['id']}
678
+ contact_id = instance_identifier{ |field, value| lookup_entity_by_field(type: res_type.to_s, field: field, value: value, query: res_id_query)['id']}
655
679
  adm_api.create("#{res_type}/#{contact_id}/reset_password", {})
656
680
  return Main.result_status('password reset, user shall check email')
657
681
  end
@@ -671,35 +695,37 @@ module Aspera
671
695
  delete_data = value_create_modify(command: command, default: {})
672
696
  delete_data = @api_v5.read('configuration').slice('days_before_deleting_package_records') if delete_data.empty?
673
697
  res = @api_v5.create('internal/packages/clean_deleted', delete_data)
674
- return {type: :single_object, data: res}
698
+ return Main.result_single_object(res)
675
699
  when :events
676
700
  event_type = options.get_next_command(%i[application webhook])
677
701
  case event_type
678
702
  when :application
679
- return {type: :object_list, data: list_entities(type: 'application_events', query: query_read_delete),
680
- fields: %w[event_type created_at application user.name]}
703
+ return Main.result_object_list(
704
+ list_entities(type: 'application_events', query: query_read_delete),
705
+ fields: %w[event_type created_at application user.name])
681
706
  when :webhook
682
- return {type: :object_list, data: list_entities(type: 'all_webhooks_events', query: query_read_delete, item_list_key: 'events')}
707
+ return Main.result_object_list(
708
+ list_entities(type: 'all_webhooks_events', query: query_read_delete, item_list_key: 'events'))
683
709
  end
684
710
  when :configuration
685
711
  conf_path = 'configuration'
686
712
  conf_cmd = options.get_next_command(%i[show modify])
687
713
  case conf_cmd
688
714
  when :show
689
- return { type: :single_object, data: @api_v5.read(conf_path) }
715
+ return Main.result_single_object(@api_v5.read(conf_path))
690
716
  when :modify
691
- return { type: :single_object, data: @api_v5.update(conf_path, value_create_modify(command: conf_cmd)) }
717
+ return Main.result_single_object(@api_v5.update(conf_path, value_create_modify(command: conf_cmd)))
692
718
  end
693
719
  when :smtp
694
720
  smtp_path = 'configuration/smtp'
695
721
  smtp_cmd = options.get_next_command(%i[show create modify delete test])
696
722
  case smtp_cmd
697
723
  when :show
698
- return { type: :single_object, data: @api_v5.read(smtp_path) }
724
+ return Main.result_single_object(@api_v5.read(smtp_path))
699
725
  when :create
700
- return { type: :single_object, data: @api_v5.create(smtp_path, value_create_modify(command: smtp_cmd)) }
726
+ return Main.result_single_object(@api_v5.create(smtp_path, value_create_modify(command: smtp_cmd)))
701
727
  when :modify
702
- return { type: :single_object, data: @api_v5.update(smtp_path, value_create_modify(command: smtp_cmd)) }
728
+ return Main.result_single_object(@api_v5.update(smtp_path, value_create_modify(command: smtp_cmd)))
703
729
  when :delete
704
730
  @api_v5.delete(smtp_path)
705
731
  return Main.result_status('SMTP configuration deleted')
@@ -709,7 +735,7 @@ fields: %w[event_type created_at application user.name]}
709
735
  creation = @api_v5.create(File.join(smtp_path, 'test'), test_data)
710
736
  result = wait_for_job(creation['job_id'])
711
737
  result['serialized_args'] = JSON.parse(result['serialized_args']) rescue result['serialized_args']
712
- return { type: :single_object, data: result }
738
+ return Main.result_single_object(result)
713
739
  end
714
740
  end
715
741
  end
@@ -721,7 +747,7 @@ fields: %w[event_type created_at application user.name]}
721
747
  set_api unless command.eql?(:postprocessing)
722
748
  case command
723
749
  when :version
724
- return { type: :single_object, data: @api_v5.read('version') }
750
+ return Main.result_single_object(@api_v5.read('version'))
725
751
  when :health
726
752
  nagios = Nagios.new
727
753
  begin
@@ -736,33 +762,33 @@ fields: %w[event_type created_at application user.name]}
736
762
  when :user
737
763
  case options.get_next_command(%i[account profile])
738
764
  when :account
739
- return { type: :single_object, data: @api_v5.read('account') }
765
+ return Main.result_single_object(@api_v5.read('account'))
740
766
  when :profile
741
767
  case options.get_next_command(%i[show modify])
742
768
  when :show
743
- return { type: :single_object, data: @api_v5.read('account/preferences') }
769
+ return Main.result_single_object(@api_v5.read('account/preferences'))
744
770
  when :modify
745
771
  @api_v5.update('account/preferences', options.get_next_argument('modified parameters', validation: Hash))
746
772
  return Main.result_status('modified')
747
773
  end
748
774
  end
749
775
  when :bearer_token
750
- return {type: :text, data: @api_v5.oauth.authorization}
776
+ return Main.result_text(@api_v5.oauth.authorization)
751
777
  when :packages
752
778
  return package_action
753
779
  when :shared_folders
754
780
  all_shared_folders = @api_v5.read('shared_folders')['shared_folders']
755
781
  case options.get_next_command(%i[list browse])
756
782
  when :list
757
- return {type: :object_list, data: all_shared_folders}
783
+ return Main.result_object_list(all_shared_folders)
758
784
  when :browse
759
785
  shared_folder_id = instance_identifier do |field, value|
760
- matches = all_shared_folders.select{|i|i[field].eql?(value)}
786
+ matches = all_shared_folders.select{ |i| i[field].eql?(value)}
761
787
  raise "no match for #{field} = #{value}" if matches.empty?
762
788
  raise "multiple matches for #{field} = #{value}" if matches.length > 1
763
789
  matches.first['id']
764
790
  end
765
- node = all_shared_folders.find{|i|i['id'].eql?(shared_folder_id)}
791
+ node = all_shared_folders.find{ |i| i['id'].eql?(shared_folder_id)}
766
792
  raise "No such shared folder id #{shared_folder_id}" if node.nil?
767
793
  return browse_folder("nodes/#{node['node_id']}/shared_folders/#{shared_folder_id}/browse")
768
794
  end
@@ -789,22 +815,20 @@ fields: %w[event_type created_at application user.name]}
789
815
  end
790
816
  when :gateway
791
817
  require 'aspera/faspex_gw'
792
- url = value_create_modify(command: command, description: 'listening url (e.g. https://localhost:12345)', type: String)
793
- uri = URI.parse(url)
794
- server = WebServerSimple.new(uri)
818
+ parameters = value_create_modify(command: command, default: {}).symbolize_keys
819
+ uri = URI.parse(parameters.delete(:url){WebServerSimple::DEFAULT_URL})
820
+ server = WebServerSimple.new(uri, **parameters.slice(*WebServerSimple::PARAMS))
821
+ Aspera.assert(parameters.except(*WebServerSimple::PARAMS).empty?)
795
822
  server.mount(uri.path, Faspex4GWServlet, @api_v5, nil)
796
823
  server.start
797
824
  return Main.result_status('Gateway terminated')
798
825
  when :postprocessing
799
826
  require 'aspera/faspex_postproc' # cspell:disable-line
800
- parameters = value_create_modify(command: command)
801
- parameters = parameters.symbolize_keys
802
- Aspera.assert(parameters.key?(:url)){'Missing key: url'}
803
- uri = URI.parse(parameters[:url])
804
- parameters[:processing] ||= {}
805
- parameters[:processing][:root] = uri.path
806
- server = WebServerSimple.new(uri, certificate: parameters[:certificate])
807
- server.mount(uri.path, Faspex4PostProcServlet, parameters[:processing])
827
+ parameters = value_create_modify(command: command, default: {}).symbolize_keys
828
+ uri = URI.parse(parameters.delete(:url){WebServerSimple::DEFAULT_URL})
829
+ parameters[:root] = uri.path
830
+ server = WebServerSimple.new(uri, **parameters.slice(*WebServerSimple::PARAMS))
831
+ server.mount(uri.path, Faspex4PostProcServlet, parameters.except(*WebServerSimple::PARAMS))
808
832
  server.start
809
833
  return Main.result_status('Gateway terminated')
810
834
  end