aspera-cli 4.24.2 → 4.25.0.pre

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (65) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/CHANGELOG.md +1064 -758
  4. data/CONTRIBUTING.md +43 -100
  5. data/README.md +671 -419
  6. data/lib/aspera/api/aoc.rb +71 -43
  7. data/lib/aspera/api/cos_node.rb +3 -2
  8. data/lib/aspera/api/faspex.rb +6 -5
  9. data/lib/aspera/api/node.rb +10 -12
  10. data/lib/aspera/ascmd.rb +1 -2
  11. data/lib/aspera/ascp/installation.rb +53 -39
  12. data/lib/aspera/assert.rb +25 -3
  13. data/lib/aspera/cli/error.rb +4 -2
  14. data/lib/aspera/cli/extended_value.rb +84 -60
  15. data/lib/aspera/cli/formatter.rb +55 -22
  16. data/lib/aspera/cli/main.rb +21 -14
  17. data/lib/aspera/cli/manager.rb +348 -247
  18. data/lib/aspera/cli/plugins/alee.rb +3 -3
  19. data/lib/aspera/cli/plugins/aoc.rb +70 -14
  20. data/lib/aspera/cli/plugins/base.rb +57 -49
  21. data/lib/aspera/cli/plugins/config.rb +69 -84
  22. data/lib/aspera/cli/plugins/console.rb +13 -8
  23. data/lib/aspera/cli/plugins/cos.rb +1 -1
  24. data/lib/aspera/cli/plugins/faspex.rb +32 -26
  25. data/lib/aspera/cli/plugins/faspex5.rb +45 -43
  26. data/lib/aspera/cli/plugins/faspio.rb +5 -5
  27. data/lib/aspera/cli/plugins/httpgw.rb +1 -1
  28. data/lib/aspera/cli/plugins/node.rb +131 -120
  29. data/lib/aspera/cli/plugins/oauth.rb +1 -1
  30. data/lib/aspera/cli/plugins/orchestrator.rb +114 -32
  31. data/lib/aspera/cli/plugins/preview.rb +26 -46
  32. data/lib/aspera/cli/plugins/server.rb +6 -8
  33. data/lib/aspera/cli/plugins/shares.rb +27 -32
  34. data/lib/aspera/cli/sync_actions.rb +49 -38
  35. data/lib/aspera/cli/transfer_agent.rb +16 -34
  36. data/lib/aspera/cli/version.rb +1 -1
  37. data/lib/aspera/cli/wizard.rb +8 -5
  38. data/lib/aspera/command_line_builder.rb +20 -17
  39. data/lib/aspera/coverage.rb +1 -1
  40. data/lib/aspera/environment.rb +41 -34
  41. data/lib/aspera/faspex_gw.rb +1 -1
  42. data/lib/aspera/keychain/factory.rb +1 -2
  43. data/lib/aspera/markdown.rb +31 -0
  44. data/lib/aspera/nagios.rb +6 -5
  45. data/lib/aspera/oauth/base.rb +17 -27
  46. data/lib/aspera/oauth/factory.rb +1 -1
  47. data/lib/aspera/oauth/url_json.rb +2 -1
  48. data/lib/aspera/preview/file_types.rb +23 -37
  49. data/lib/aspera/products/connect.rb +3 -3
  50. data/lib/aspera/rest.rb +51 -39
  51. data/lib/aspera/rest_error_analyzer.rb +4 -4
  52. data/lib/aspera/ssh.rb +5 -2
  53. data/lib/aspera/ssl.rb +41 -0
  54. data/lib/aspera/sync/conf.schema.yaml +182 -34
  55. data/lib/aspera/sync/database.rb +2 -1
  56. data/lib/aspera/sync/operations.rb +125 -69
  57. data/lib/aspera/transfer/parameters.rb +3 -4
  58. data/lib/aspera/transfer/spec.rb +2 -3
  59. data/lib/aspera/transfer/spec.schema.yaml +48 -18
  60. data/lib/aspera/transfer/spec_doc.rb +14 -14
  61. data/lib/aspera/uri_reader.rb +1 -1
  62. data/lib/transferd_pb.rb +2 -2
  63. data.tar.gz.sig +0 -0
  64. metadata +19 -6
  65. metadata.gz.sig +3 -2
@@ -33,7 +33,7 @@ module Aspera
33
33
  # Faspex is always HTTPS
34
34
  next unless base_url.start_with?('https://')
35
35
  api = Rest.new(base_url: base_url, redirect_max: 1)
36
- response = api.call(operation: 'GET', subpath: Api::Faspex::PATH_API_DETECT)[:http]
36
+ response = api.read(Api::Faspex::PATH_API_DETECT, ret: :resp)
37
37
  next unless response.code.start_with?('2') && response.body.strip.empty?
38
38
  # end is at -1, and subtract 1 for "/"
39
39
  url_length = -2 - Api::Faspex::PATH_API_DETECT.length
@@ -93,7 +93,7 @@ module Aspera
93
93
  super
94
94
  options.declare(:box, "Package inbox, either shared inbox name or one of: #{Api::Faspex::API_LIST_MAILBOX_TYPES.join(', ')} or #{SpecialValues::ALL}", default: 'inbox')
95
95
  options.declare(:shared_folder, 'Send package with files from shared folder')
96
- options.declare(:group_type, 'Type of shared box', values: %i[shared_inboxes workgroups], default: :shared_inboxes)
96
+ options.declare(:group_type, 'Type of shared box', allowed: %i[shared_inboxes workgroups], default: :shared_inboxes)
97
97
  options.parse_options!
98
98
  end
99
99
 
@@ -222,8 +222,7 @@ module Aspera
222
222
  else
223
223
  # a single id was provided, or a list of ids
224
224
  package_ids = [package_ids] unless package_ids.is_a?(Array)
225
- Aspera.assert_type(package_ids, Array){'Expecting a single package id or a list of ids'}
226
- Aspera.assert(package_ids.all?(String)){'Package id shall be String'}
225
+ Aspera.assert_array_all(package_ids, String){'Package id(s)'}
227
226
  # packages = package_ids.map{|pkg_id|@api_v5.read("packages/#{pkg_id}")}
228
227
  packages = package_ids.map{ |pkg_id| {'id'=>pkg_id}}
229
228
  end
@@ -256,7 +255,7 @@ module Aspera
256
255
  content_type: Rest::MIME_JSON,
257
256
  body: param_file_list,
258
257
  headers: {'Accept' => Rest::MIME_JSON}
259
- )[:data]
258
+ )
260
259
  # delete flag for Connect Client
261
260
  transfer_spec.delete('authentication')
262
261
  statuses = transfer.start(transfer_spec)
@@ -294,28 +293,29 @@ module Aspera
294
293
  until folders_to_process.empty?
295
294
  path = folders_to_process.shift
296
295
  loop do
297
- response = @api_v5.call(
296
+ data, http = @api_v5.call(
298
297
  operation: 'POST',
299
298
  subpath: browse_endpoint,
300
299
  query: query,
301
300
  content_type: Rest::MIME_JSON,
302
301
  body: {'path' => path, 'filters' => filters},
303
- headers: {'Accept' => Rest::MIME_JSON}
302
+ headers: {'Accept' => Rest::MIME_JSON},
303
+ ret: :both
304
304
  )
305
- all_items.concat(response[:data]['items'])
305
+ all_items.concat(data['items'])
306
306
  if !max_items.nil? && (all_items.count >= max_items)
307
307
  all_items = all_items.slice(0, max_items) if all_items.count > max_items
308
308
  break
309
309
  end
310
- folders_to_process.concat(response[:data]['items'].select{ |i| i['type'].eql?('directory')}.map{ |i| i['path']}) if recursive
310
+ folders_to_process.concat(data['items'].select{ |i| i['type'].eql?('directory')}.map{ |i| i['path']}) if recursive
311
311
  if use_paging
312
- iteration_token = response[:http][Api::Faspex::HEADER_ITERATION_TOKEN]
312
+ iteration_token = http[Api::Faspex::HEADER_ITERATION_TOKEN]
313
313
  break if iteration_token.nil? || iteration_token.empty?
314
314
  query['iteration_token'] = iteration_token
315
315
  else
316
- total_count = response[:data]['total_count'] if total_count.nil?
317
- break if response[:data]['item_count'].eql?(0)
318
- query['offset'] += response[:data]['item_count']
316
+ total_count = data['total_count'] if total_count.nil?
317
+ break if data['item_count'].eql?(0)
318
+ query['offset'] += data['item_count']
319
319
  end
320
320
  formatter.long_operation_running(all_items.count)
321
321
  end
@@ -349,8 +349,7 @@ module Aspera
349
349
  when :delete
350
350
  ids = package_id
351
351
  ids = [ids] unless ids.is_a?(Array)
352
- Aspera.assert_type(ids, Array){'Package identifier'}
353
- Aspera.assert(ids.all?(String)){"Package id(s) shall be String, but have: #{ids.map(&:class).uniq.join(', ')}"}
352
+ Aspera.assert_array_all(ids, String){'Package id(s)'}
354
353
  # API returns 204, empty on success
355
354
  @api_v5.call(
356
355
  operation: 'DELETE',
@@ -372,29 +371,32 @@ module Aspera
372
371
  }]
373
372
  end
374
373
  normalize_recipients(parameters)
374
+ # User specified content prot in tspec, but faspex requires in package creation
375
+ # `transfer_spec/upload` will set `content_protection`
376
+ if transfer.user_transfer_spec['content_protection'] && !parameters.key?('ear_enabled')
377
+ transfer.user_transfer_spec.delete('content_protection')
378
+ parameters['ear_enabled'] = true
379
+ end
375
380
  package = @api_v5.create('packages', parameters)
376
381
  shared_folder = options.get_option(:shared_folder)
377
382
  if shared_folder.nil?
378
383
  # send from local files
379
- transfer_spec = @api_v5.call(
380
- operation: 'POST',
381
- subpath: "packages/#{package['id']}/transfer_spec/upload",
382
- query: {transfer_type: Api::Faspex::TRANSFER_CONNECT},
383
- content_type: Rest::MIME_JSON,
384
- body: {paths: transfer.source_list},
385
- headers: {'Accept' => Rest::MIME_JSON}
386
- )[:data]
384
+ transfer_spec = @api_v5.create(
385
+ "packages/#{package['id']}/transfer_spec/upload",
386
+ {paths: transfer.source_list},
387
+ query: {transfer_type: Api::Faspex::TRANSFER_CONNECT}
388
+ )
387
389
  # well, we asked a TS for connect, but we actually want a generic one
388
390
  transfer_spec.delete('authentication')
389
391
  return Main.result_transfer(transfer.start(transfer_spec))
390
392
  else
391
393
  # send from remote shared folder
392
- if (m = shared_folder.match(REGEX_LOOKUP_ID_BY_FIELD))
394
+ if (m = Base.percent_selector(shared_folder))
393
395
  shared_folder = lookup_entity_by_field(
394
396
  api: @api_v5,
395
397
  entity: 'shared_folders',
396
- field: m[1],
397
- value: ExtendedValue.instance.evaluate(m[2])
398
+ field: m[:field],
399
+ value: m[:value]
398
400
  )['id']
399
401
  end
400
402
  transfer_request = {shared_folder_id: shared_folder, paths: transfer.source_list}
@@ -512,20 +514,20 @@ module Aspera
512
514
  users = options.get_next_argument('user id, %name:, or Array')
513
515
  users = [users] unless users.is_a?(Array)
514
516
  users = users.map do |user|
515
- if (m = user.match(REGEX_LOOKUP_ID_BY_FIELD))
517
+ if (m = Base.percent_selector(user))
516
518
  lookup_entity_by_field(
517
519
  api: @api_v5,
518
520
  entity: 'accounts',
519
- field: m[1],
520
- value: ExtendedValue.instance.evaluate(m[2]),
521
- query: Rest.php_style({type: %w{local_user saml_user self_registered_user external_user}})
521
+ field: m[:field],
522
+ value: m[:value],
523
+ query: Rest.php_style({type: ACCOUNT_TYPES})
522
524
  )['id']
523
525
  else
524
526
  # it's the user id (not member id...)
525
527
  user
526
528
  end
527
529
  end
528
- access = options.get_next_argument('level', mandatory: false, accept_list: %i[submit_only standard shared_inbox_admin], default: :standard)
530
+ access = options.get_next_argument('level', mandatory: false, accept_list: SHARED_INBOX_MEMBER_LEVELS, default: :standard)
529
531
  options.unshift_next_argument({user: users.map{ |u| {id: u, access: access}}})
530
532
  end
531
533
  return entity_execute(
@@ -536,10 +538,10 @@ module Aspera
536
538
  ) do |field, value|
537
539
  lookup_entity_by_field(
538
540
  api: @api_v5,
539
- entity: 'accounts',
541
+ entity: 'contacts',
540
542
  field: field,
541
543
  value: value,
542
- query: Rest.php_style({type: %w{local_user saml_user self_registered_user external_user}})
544
+ query: Rest.php_style({type: %w[user]})
543
545
  )['id']
544
546
  end
545
547
  when :reset_password
@@ -552,12 +554,8 @@ module Aspera
552
554
  end
553
555
 
554
556
  def execute_admin
555
- command = options.get_next_command(%i[configuration smtp resource events clean_deleted].concat(Api::Faspex::ADMIN_RESOURCES).freeze)
557
+ command = options.get_next_command(%i[configuration smtp events clean_deleted].concat(Api::Faspex::ADMIN_RESOURCES).freeze)
556
558
  case command
557
- when :resource
558
- # resource will be deprecated
559
- Log.log.warn('resource command is deprecated (4.18), directly use the specific command instead')
560
- return execute_resource(options.get_next_command(Api::Faspex::ADMIN_RESOURCES))
561
559
  when *Api::Faspex::ADMIN_RESOURCES
562
560
  return execute_resource(command)
563
561
  when :clean_deleted
@@ -630,16 +628,16 @@ module Aspera
630
628
  when :health
631
629
  nagios = Nagios.new
632
630
  begin
633
- http_res = Rest.new(base_url: options.get_option(:url, mandatory: true))
634
- .call(operation: 'GET', subpath: 'health', headers: {'Accept' => Rest::MIME_JSON})
635
- http_res[:data].each do |k, v|
631
+ data, http = Rest.new(base_url: options.get_option(:url, mandatory: true))
632
+ .read('health', ret: :both)
633
+ data.each do |k, v|
636
634
  nagios.add_ok(k, v.to_s)
637
635
  end
638
- nagios.add_ok('version', http_res[:http]['X-IBM-Aspera']) if http_res[:http]['X-IBM-Aspera']
636
+ nagios.add_ok('version', http['X-IBM-Aspera']) if http['X-IBM-Aspera']
639
637
  rescue StandardError => e
640
638
  nagios.add_critical('core', e.to_s)
641
639
  end
642
- return nagios.result
640
+ Main.result_object_list(nagios.status_list)
643
641
  when :user
644
642
  case options.get_next_command(%i[account profile])
645
643
  when :account
@@ -718,6 +716,10 @@ module Aspera
718
716
  return Main.result_status('Gateway terminated')
719
717
  end
720
718
  end
719
+ SHARED_INBOX_MEMBER_LEVELS = %i[submit_only standard shared_inbox_admin].freeze
720
+ ACCOUNT_TYPES = %w{local_user saml_user self_registered_user external_user}.freeze
721
+ CONTACT_TYPES = %w{workgroup shared_inbox distribution_list user external_user}.freeze
722
+ private_constant :SHARED_INBOX_MEMBER_LEVELS, :ACCOUNT_TYPES, :CONTACT_TYPES
721
723
  end
722
724
  end
723
725
  end
@@ -15,9 +15,9 @@ module Aspera
15
15
 
16
16
  def detect(base_url)
17
17
  api = Rest.new(base_url: base_url)
18
- ping_result = api.call(operation: 'GET', subpath: 'ping', headers: {'Accept' => Rest::MIME_JSON})
19
- server_type = ping_result[:http]['Server']
20
- return unless ping_result[:data].is_a?(Hash) && ping_result[:data].empty?
18
+ data, http = api.read('ping', ret: :both)
19
+ server_type = http['Server']
20
+ return unless data.is_a?(Hash) && data.empty?
21
21
  return unless server_type.is_a?(String) && server_type.include?('faspio')
22
22
  return {
23
23
  version: server_type.gsub(%r{^.*/}, ''),
@@ -42,7 +42,7 @@ module Aspera
42
42
 
43
43
  def initialize(**_)
44
44
  super
45
- options.declare(:auth, 'OAuth type of authentication', values: %i[jwt basic])
45
+ options.declare(:auth, 'OAuth type of authentication', allowed: %i[jwt basic])
46
46
  options.declare(:client_id, 'OAuth client identifier')
47
47
  options.declare(:private_key, 'OAuth JWT RSA private key PEM value (prefix file path with @file:)')
48
48
  options.declare(:passphrase, 'OAuth JWT RSA private key passphrase')
@@ -88,7 +88,7 @@ module Aspera
88
88
  rescue StandardError => e
89
89
  nagios.add_critical('api', e.to_s)
90
90
  end
91
- return nagios.result
91
+ Main.result_object_list(nagios.status_list)
92
92
  when :bridges
93
93
  return entity_execute(api: api, entity: 'bridges')
94
94
  end
@@ -56,7 +56,7 @@ module Aspera
56
56
  rescue StandardError => e
57
57
  nagios.add_critical('api', e.to_s)
58
58
  end
59
- return nagios.result
59
+ Main.result_object_list(nagios.status_list)
60
60
  when :info
61
61
  api_v1 = Api::Httpgw.new(url: base_url)
62
62
  return Main.result_single_object(api_v1.info)