aspera-cli 4.24.2 → 4.25.0.pre2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (73) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/CHANGELOG.md +1067 -758
  4. data/CONTRIBUTING.md +93 -120
  5. data/README.md +817 -510
  6. data/lib/aspera/agent/direct.rb +14 -12
  7. data/lib/aspera/agent/transferd.rb +4 -4
  8. data/lib/aspera/api/aoc.rb +71 -43
  9. data/lib/aspera/api/cos_node.rb +3 -2
  10. data/lib/aspera/api/faspex.rb +6 -5
  11. data/lib/aspera/api/node.rb +10 -12
  12. data/lib/aspera/ascmd.rb +1 -2
  13. data/lib/aspera/ascp/installation.rb +55 -41
  14. data/lib/aspera/ascp/management.rb +9 -5
  15. data/lib/aspera/assert.rb +28 -6
  16. data/lib/aspera/cli/error.rb +4 -2
  17. data/lib/aspera/cli/extended_value.rb +94 -62
  18. data/lib/aspera/cli/formatter.rb +55 -22
  19. data/lib/aspera/cli/main.rb +21 -14
  20. data/lib/aspera/cli/manager.rb +349 -248
  21. data/lib/aspera/cli/plugins/alee.rb +3 -3
  22. data/lib/aspera/cli/plugins/aoc.rb +94 -51
  23. data/lib/aspera/cli/plugins/base.rb +62 -49
  24. data/lib/aspera/cli/plugins/config.rb +85 -96
  25. data/lib/aspera/cli/plugins/console.rb +15 -9
  26. data/lib/aspera/cli/plugins/cos.rb +1 -1
  27. data/lib/aspera/cli/plugins/faspex.rb +34 -27
  28. data/lib/aspera/cli/plugins/faspex5.rb +47 -44
  29. data/lib/aspera/cli/plugins/faspio.rb +7 -6
  30. data/lib/aspera/cli/plugins/httpgw.rb +3 -2
  31. data/lib/aspera/cli/plugins/node.rb +132 -120
  32. data/lib/aspera/cli/plugins/oauth.rb +1 -1
  33. data/lib/aspera/cli/plugins/orchestrator.rb +116 -33
  34. data/lib/aspera/cli/plugins/preview.rb +26 -46
  35. data/lib/aspera/cli/plugins/server.rb +9 -10
  36. data/lib/aspera/cli/plugins/shares.rb +77 -43
  37. data/lib/aspera/cli/sync_actions.rb +49 -38
  38. data/lib/aspera/cli/transfer_agent.rb +16 -34
  39. data/lib/aspera/cli/version.rb +1 -1
  40. data/lib/aspera/cli/wizard.rb +8 -5
  41. data/lib/aspera/command_line_builder.rb +20 -17
  42. data/lib/aspera/coverage.rb +6 -2
  43. data/lib/aspera/environment.rb +71 -84
  44. data/lib/aspera/faspex_gw.rb +1 -1
  45. data/lib/aspera/faspex_postproc.rb +1 -1
  46. data/lib/aspera/keychain/factory.rb +1 -2
  47. data/lib/aspera/keychain/macos_security.rb +2 -2
  48. data/lib/aspera/log.rb +2 -1
  49. data/lib/aspera/markdown.rb +31 -0
  50. data/lib/aspera/nagios.rb +6 -5
  51. data/lib/aspera/oauth/base.rb +17 -27
  52. data/lib/aspera/oauth/factory.rb +1 -1
  53. data/lib/aspera/oauth/url_json.rb +2 -1
  54. data/lib/aspera/preview/file_types.rb +23 -37
  55. data/lib/aspera/preview/terminal.rb +95 -29
  56. data/lib/aspera/preview/utils.rb +6 -5
  57. data/lib/aspera/products/connect.rb +3 -3
  58. data/lib/aspera/rest.rb +51 -39
  59. data/lib/aspera/rest_error_analyzer.rb +4 -4
  60. data/lib/aspera/ssh.rb +5 -2
  61. data/lib/aspera/ssl.rb +41 -0
  62. data/lib/aspera/sync/conf.schema.yaml +182 -34
  63. data/lib/aspera/sync/database.rb +2 -1
  64. data/lib/aspera/sync/operations.rb +128 -72
  65. data/lib/aspera/transfer/parameters.rb +3 -4
  66. data/lib/aspera/transfer/spec.rb +2 -3
  67. data/lib/aspera/transfer/spec.schema.yaml +49 -19
  68. data/lib/aspera/transfer/spec_doc.rb +14 -14
  69. data/lib/aspera/uri_reader.rb +1 -1
  70. data/lib/transferd_pb.rb +2 -2
  71. data.tar.gz.sig +0 -0
  72. metadata +33 -6
  73. metadata.gz.sig +0 -0
@@ -23,6 +23,7 @@ module Aspera
23
23
  'Faspex'
24
24
  end
25
25
 
26
+ # @return [Hash,NilClass]
26
27
  def detect(address_or_url)
27
28
  # add scheme if missing
28
29
  address_or_url = "https://#{address_or_url}" unless address_or_url.match?(%r{^[a-z]{1,6}://})
@@ -33,7 +34,7 @@ module Aspera
33
34
  # Faspex is always HTTPS
34
35
  next unless base_url.start_with?('https://')
35
36
  api = Rest.new(base_url: base_url, redirect_max: 1)
36
- response = api.call(operation: 'GET', subpath: Api::Faspex::PATH_API_DETECT)[:http]
37
+ response = api.read(Api::Faspex::PATH_API_DETECT, ret: :resp)
37
38
  next unless response.code.start_with?('2') && response.body.strip.empty?
38
39
  # end is at -1, and subtract 1 for "/"
39
40
  url_length = -2 - Api::Faspex::PATH_API_DETECT.length
@@ -52,7 +53,7 @@ module Aspera
52
53
  end
53
54
 
54
55
  # @param wizard [Wizard] The wizard object
55
- # @param app_url [Wizard] The wizard object
56
+ # @param app_url [String] Tested URL
56
57
  # @return [Hash] :preset_value, :test_args
57
58
  def wizard(wizard, app_url)
58
59
  client_id = options.get_option(:client_id)
@@ -93,7 +94,7 @@ module Aspera
93
94
  super
94
95
  options.declare(:box, "Package inbox, either shared inbox name or one of: #{Api::Faspex::API_LIST_MAILBOX_TYPES.join(', ')} or #{SpecialValues::ALL}", default: 'inbox')
95
96
  options.declare(:shared_folder, 'Send package with files from shared folder')
96
- options.declare(:group_type, 'Type of shared box', values: %i[shared_inboxes workgroups], default: :shared_inboxes)
97
+ options.declare(:group_type, 'Type of shared box', allowed: %i[shared_inboxes workgroups], default: :shared_inboxes)
97
98
  options.parse_options!
98
99
  end
99
100
 
@@ -222,8 +223,7 @@ module Aspera
222
223
  else
223
224
  # a single id was provided, or a list of ids
224
225
  package_ids = [package_ids] unless package_ids.is_a?(Array)
225
- Aspera.assert_type(package_ids, Array){'Expecting a single package id or a list of ids'}
226
- Aspera.assert(package_ids.all?(String)){'Package id shall be String'}
226
+ Aspera.assert_array_all(package_ids, String){'Package id(s)'}
227
227
  # packages = package_ids.map{|pkg_id|@api_v5.read("packages/#{pkg_id}")}
228
228
  packages = package_ids.map{ |pkg_id| {'id'=>pkg_id}}
229
229
  end
@@ -256,7 +256,7 @@ module Aspera
256
256
  content_type: Rest::MIME_JSON,
257
257
  body: param_file_list,
258
258
  headers: {'Accept' => Rest::MIME_JSON}
259
- )[:data]
259
+ )
260
260
  # delete flag for Connect Client
261
261
  transfer_spec.delete('authentication')
262
262
  statuses = transfer.start(transfer_spec)
@@ -294,28 +294,29 @@ module Aspera
294
294
  until folders_to_process.empty?
295
295
  path = folders_to_process.shift
296
296
  loop do
297
- response = @api_v5.call(
297
+ data, http = @api_v5.call(
298
298
  operation: 'POST',
299
299
  subpath: browse_endpoint,
300
300
  query: query,
301
301
  content_type: Rest::MIME_JSON,
302
302
  body: {'path' => path, 'filters' => filters},
303
- headers: {'Accept' => Rest::MIME_JSON}
303
+ headers: {'Accept' => Rest::MIME_JSON},
304
+ ret: :both
304
305
  )
305
- all_items.concat(response[:data]['items'])
306
+ all_items.concat(data['items'])
306
307
  if !max_items.nil? && (all_items.count >= max_items)
307
308
  all_items = all_items.slice(0, max_items) if all_items.count > max_items
308
309
  break
309
310
  end
310
- folders_to_process.concat(response[:data]['items'].select{ |i| i['type'].eql?('directory')}.map{ |i| i['path']}) if recursive
311
+ folders_to_process.concat(data['items'].select{ |i| i['type'].eql?('directory')}.map{ |i| i['path']}) if recursive
311
312
  if use_paging
312
- iteration_token = response[:http][Api::Faspex::HEADER_ITERATION_TOKEN]
313
+ iteration_token = http[Api::Faspex::HEADER_ITERATION_TOKEN]
313
314
  break if iteration_token.nil? || iteration_token.empty?
314
315
  query['iteration_token'] = iteration_token
315
316
  else
316
- total_count = response[:data]['total_count'] if total_count.nil?
317
- break if response[:data]['item_count'].eql?(0)
318
- query['offset'] += response[:data]['item_count']
317
+ total_count = data['total_count'] if total_count.nil?
318
+ break if data['item_count'].eql?(0)
319
+ query['offset'] += data['item_count']
319
320
  end
320
321
  formatter.long_operation_running(all_items.count)
321
322
  end
@@ -349,8 +350,7 @@ module Aspera
349
350
  when :delete
350
351
  ids = package_id
351
352
  ids = [ids] unless ids.is_a?(Array)
352
- Aspera.assert_type(ids, Array){'Package identifier'}
353
- Aspera.assert(ids.all?(String)){"Package id(s) shall be String, but have: #{ids.map(&:class).uniq.join(', ')}"}
353
+ Aspera.assert_array_all(ids, String){'Package id(s)'}
354
354
  # API returns 204, empty on success
355
355
  @api_v5.call(
356
356
  operation: 'DELETE',
@@ -372,29 +372,32 @@ module Aspera
372
372
  }]
373
373
  end
374
374
  normalize_recipients(parameters)
375
+ # User specified content prot in tspec, but faspex requires in package creation
376
+ # `transfer_spec/upload` will set `content_protection`
377
+ if transfer.user_transfer_spec['content_protection'] && !parameters.key?('ear_enabled')
378
+ transfer.user_transfer_spec.delete('content_protection')
379
+ parameters['ear_enabled'] = true
380
+ end
375
381
  package = @api_v5.create('packages', parameters)
376
382
  shared_folder = options.get_option(:shared_folder)
377
383
  if shared_folder.nil?
378
384
  # send from local files
379
- transfer_spec = @api_v5.call(
380
- operation: 'POST',
381
- subpath: "packages/#{package['id']}/transfer_spec/upload",
382
- query: {transfer_type: Api::Faspex::TRANSFER_CONNECT},
383
- content_type: Rest::MIME_JSON,
384
- body: {paths: transfer.source_list},
385
- headers: {'Accept' => Rest::MIME_JSON}
386
- )[:data]
385
+ transfer_spec = @api_v5.create(
386
+ "packages/#{package['id']}/transfer_spec/upload",
387
+ {paths: transfer.source_list},
388
+ query: {transfer_type: Api::Faspex::TRANSFER_CONNECT}
389
+ )
387
390
  # well, we asked a TS for connect, but we actually want a generic one
388
391
  transfer_spec.delete('authentication')
389
392
  return Main.result_transfer(transfer.start(transfer_spec))
390
393
  else
391
394
  # send from remote shared folder
392
- if (m = shared_folder.match(REGEX_LOOKUP_ID_BY_FIELD))
395
+ if (m = Base.percent_selector(shared_folder))
393
396
  shared_folder = lookup_entity_by_field(
394
397
  api: @api_v5,
395
398
  entity: 'shared_folders',
396
- field: m[1],
397
- value: ExtendedValue.instance.evaluate(m[2])
399
+ field: m[:field],
400
+ value: m[:value]
398
401
  )['id']
399
402
  end
400
403
  transfer_request = {shared_folder_id: shared_folder, paths: transfer.source_list}
@@ -512,20 +515,20 @@ module Aspera
512
515
  users = options.get_next_argument('user id, %name:, or Array')
513
516
  users = [users] unless users.is_a?(Array)
514
517
  users = users.map do |user|
515
- if (m = user.match(REGEX_LOOKUP_ID_BY_FIELD))
518
+ if (m = Base.percent_selector(user))
516
519
  lookup_entity_by_field(
517
520
  api: @api_v5,
518
521
  entity: 'accounts',
519
- field: m[1],
520
- value: ExtendedValue.instance.evaluate(m[2]),
521
- query: Rest.php_style({type: %w{local_user saml_user self_registered_user external_user}})
522
+ field: m[:field],
523
+ value: m[:value],
524
+ query: Rest.php_style({type: ACCOUNT_TYPES})
522
525
  )['id']
523
526
  else
524
527
  # it's the user id (not member id...)
525
528
  user
526
529
  end
527
530
  end
528
- access = options.get_next_argument('level', mandatory: false, accept_list: %i[submit_only standard shared_inbox_admin], default: :standard)
531
+ access = options.get_next_argument('level', mandatory: false, accept_list: SHARED_INBOX_MEMBER_LEVELS, default: :standard)
529
532
  options.unshift_next_argument({user: users.map{ |u| {id: u, access: access}}})
530
533
  end
531
534
  return entity_execute(
@@ -536,10 +539,10 @@ module Aspera
536
539
  ) do |field, value|
537
540
  lookup_entity_by_field(
538
541
  api: @api_v5,
539
- entity: 'accounts',
542
+ entity: 'contacts',
540
543
  field: field,
541
544
  value: value,
542
- query: Rest.php_style({type: %w{local_user saml_user self_registered_user external_user}})
545
+ query: Rest.php_style({type: %w[user]})
543
546
  )['id']
544
547
  end
545
548
  when :reset_password
@@ -552,12 +555,8 @@ module Aspera
552
555
  end
553
556
 
554
557
  def execute_admin
555
- command = options.get_next_command(%i[configuration smtp resource events clean_deleted].concat(Api::Faspex::ADMIN_RESOURCES).freeze)
558
+ command = options.get_next_command(%i[configuration smtp events clean_deleted].concat(Api::Faspex::ADMIN_RESOURCES).freeze)
556
559
  case command
557
- when :resource
558
- # resource will be deprecated
559
- Log.log.warn('resource command is deprecated (4.18), directly use the specific command instead')
560
- return execute_resource(options.get_next_command(Api::Faspex::ADMIN_RESOURCES))
561
560
  when *Api::Faspex::ADMIN_RESOURCES
562
561
  return execute_resource(command)
563
562
  when :clean_deleted
@@ -630,16 +629,16 @@ module Aspera
630
629
  when :health
631
630
  nagios = Nagios.new
632
631
  begin
633
- http_res = Rest.new(base_url: options.get_option(:url, mandatory: true))
634
- .call(operation: 'GET', subpath: 'health', headers: {'Accept' => Rest::MIME_JSON})
635
- http_res[:data].each do |k, v|
632
+ data, http = Rest.new(base_url: options.get_option(:url, mandatory: true))
633
+ .read('health', ret: :both)
634
+ data.each do |k, v|
636
635
  nagios.add_ok(k, v.to_s)
637
636
  end
638
- nagios.add_ok('version', http_res[:http]['X-IBM-Aspera']) if http_res[:http]['X-IBM-Aspera']
637
+ nagios.add_ok('version', http['X-IBM-Aspera']) if http['X-IBM-Aspera']
639
638
  rescue StandardError => e
640
639
  nagios.add_critical('core', e.to_s)
641
640
  end
642
- return nagios.result
641
+ Main.result_object_list(nagios.status_list)
643
642
  when :user
644
643
  case options.get_next_command(%i[account profile])
645
644
  when :account
@@ -718,6 +717,10 @@ module Aspera
718
717
  return Main.result_status('Gateway terminated')
719
718
  end
720
719
  end
720
+ SHARED_INBOX_MEMBER_LEVELS = %i[submit_only standard shared_inbox_admin].freeze
721
+ ACCOUNT_TYPES = %w{local_user saml_user self_registered_user external_user}.freeze
722
+ CONTACT_TYPES = %w{workgroup shared_inbox distribution_list user external_user}.freeze
723
+ private_constant :SHARED_INBOX_MEMBER_LEVELS, :ACCOUNT_TYPES, :CONTACT_TYPES
721
724
  end
722
725
  end
723
726
  end
@@ -13,11 +13,12 @@ module Aspera
13
13
  'faspio Gateway'
14
14
  end
15
15
 
16
+ # @return [Hash,NilClass]
16
17
  def detect(base_url)
17
18
  api = Rest.new(base_url: base_url)
18
- ping_result = api.call(operation: 'GET', subpath: 'ping', headers: {'Accept' => Rest::MIME_JSON})
19
- server_type = ping_result[:http]['Server']
20
- return unless ping_result[:data].is_a?(Hash) && ping_result[:data].empty?
19
+ data, http = api.read('ping', ret: :both)
20
+ server_type = http['Server']
21
+ return unless data.is_a?(Hash) && data.empty?
21
22
  return unless server_type.is_a?(String) && server_type.include?('faspio')
22
23
  return {
23
24
  version: server_type.gsub(%r{^.*/}, ''),
@@ -27,7 +28,7 @@ module Aspera
27
28
  end
28
29
 
29
30
  # @param wizard [Wizard] The wizard object
30
- # @param app_url [Wizard] The wizard object
31
+ # @param app_url [String] Tested URL
31
32
  # @return [Hash] :preset_value, :test_args
32
33
  def wizard(wizard, app_url)
33
34
  return {
@@ -42,7 +43,7 @@ module Aspera
42
43
 
43
44
  def initialize(**_)
44
45
  super
45
- options.declare(:auth, 'OAuth type of authentication', values: %i[jwt basic])
46
+ options.declare(:auth, 'OAuth type of authentication', allowed: %i[jwt basic])
46
47
  options.declare(:client_id, 'OAuth client identifier')
47
48
  options.declare(:private_key, 'OAuth JWT RSA private key PEM value (prefix file path with @file:)')
48
49
  options.declare(:passphrase, 'OAuth JWT RSA private key passphrase')
@@ -88,7 +89,7 @@ module Aspera
88
89
  rescue StandardError => e
89
90
  nagios.add_critical('api', e.to_s)
90
91
  end
91
- return nagios.result
92
+ Main.result_object_list(nagios.status_list)
92
93
  when :bridges
93
94
  return entity_execute(api: api, entity: 'bridges')
94
95
  end
@@ -14,6 +14,7 @@ module Aspera
14
14
  'HTTP Gateway'
15
15
  end
16
16
 
17
+ # @return [Hash,NilClass]
17
18
  def detect(base_url)
18
19
  api = Api::Httpgw.new(url: base_url)
19
20
  api_info = api.info
@@ -26,7 +27,7 @@ module Aspera
26
27
  end
27
28
 
28
29
  # @param wizard [Wizard] The wizard object
29
- # @param app_url [Wizard] The wizard object
30
+ # @param app_url [String] Tested URL
30
31
  # @return [Hash] :preset_value, :test_args
31
32
  def wizard(wizard, app_url)
32
33
  return {
@@ -56,7 +57,7 @@ module Aspera
56
57
  rescue StandardError => e
57
58
  nagios.add_critical('api', e.to_s)
58
59
  end
59
- return nagios.result
60
+ Main.result_object_list(nagios.status_list)
60
61
  when :info
61
62
  api_v1 = Api::Httpgw.new(url: base_url)
62
63
  return Main.result_single_object(api_v1.info)