aspera-cli 4.23.0 → 4.24.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (110) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/CHANGELOG.md +37 -1
  4. data/CONTRIBUTING.md +86 -29
  5. data/README.md +2109 -1300
  6. data/bin/ascli +2 -1
  7. data/bin/asession +4 -4
  8. data/lib/aspera/agent/base.rb +4 -0
  9. data/lib/aspera/agent/connect.rb +20 -18
  10. data/lib/aspera/agent/desktop.rb +14 -11
  11. data/lib/aspera/agent/direct.rb +39 -31
  12. data/lib/aspera/agent/httpgw.rb +2 -2
  13. data/lib/aspera/agent/node.rb +9 -11
  14. data/lib/aspera/agent/transferd.rb +18 -11
  15. data/lib/aspera/api/aoc.rb +44 -31
  16. data/lib/aspera/api/cos_node.rb +7 -5
  17. data/lib/aspera/api/httpgw.rb +15 -18
  18. data/lib/aspera/api/node.rb +104 -22
  19. data/lib/aspera/ascmd.rb +22 -16
  20. data/lib/aspera/ascp/installation.rb +37 -40
  21. data/lib/aspera/ascp/management.rb +5 -4
  22. data/lib/aspera/assert.rb +54 -23
  23. data/lib/aspera/cli/basic_auth_plugin.rb +8 -7
  24. data/lib/aspera/cli/error.rb +1 -1
  25. data/lib/aspera/cli/extended_value.rb +28 -29
  26. data/lib/aspera/cli/formatter.rb +191 -168
  27. data/lib/aspera/cli/hints.rb +29 -3
  28. data/lib/aspera/cli/main.rb +138 -107
  29. data/lib/aspera/cli/manager.rb +50 -30
  30. data/lib/aspera/cli/plugin.rb +148 -77
  31. data/lib/aspera/cli/plugin_factory.rb +2 -2
  32. data/lib/aspera/cli/plugins/aoc.rb +189 -70
  33. data/lib/aspera/cli/plugins/ats.rb +15 -13
  34. data/lib/aspera/cli/plugins/config.rb +100 -214
  35. data/lib/aspera/cli/plugins/console.rb +49 -18
  36. data/lib/aspera/cli/plugins/cos.rb +4 -4
  37. data/lib/aspera/cli/plugins/faspex.rb +45 -51
  38. data/lib/aspera/cli/plugins/faspex5.rb +164 -165
  39. data/lib/aspera/cli/plugins/faspio.rb +6 -5
  40. data/lib/aspera/cli/plugins/httpgw.rb +2 -2
  41. data/lib/aspera/cli/plugins/node.rb +144 -162
  42. data/lib/aspera/cli/plugins/orchestrator.rb +10 -14
  43. data/lib/aspera/cli/plugins/preview.rb +26 -29
  44. data/lib/aspera/cli/plugins/server.rb +28 -28
  45. data/lib/aspera/cli/plugins/shares.rb +40 -28
  46. data/lib/aspera/cli/sync_actions.rb +101 -80
  47. data/lib/aspera/cli/transfer_agent.rb +51 -50
  48. data/lib/aspera/cli/transfer_progress.rb +29 -20
  49. data/lib/aspera/cli/version.rb +1 -1
  50. data/lib/aspera/cli/wizard.rb +157 -0
  51. data/lib/aspera/colors.rb +13 -8
  52. data/lib/aspera/command_line_builder.rb +28 -22
  53. data/lib/aspera/command_line_converter.rb +31 -0
  54. data/lib/aspera/environment.rb +145 -101
  55. data/lib/aspera/faspex_gw.rb +1 -1
  56. data/lib/aspera/faspex_postproc.rb +3 -2
  57. data/lib/aspera/hash_ext.rb +1 -1
  58. data/lib/aspera/id_generator.rb +10 -10
  59. data/lib/aspera/keychain/base.rb +18 -0
  60. data/lib/aspera/keychain/encrypted_hash.rb +6 -12
  61. data/lib/aspera/keychain/factory.rb +9 -3
  62. data/lib/aspera/keychain/hashicorp_vault.rb +9 -6
  63. data/lib/aspera/keychain/macos_security.rb +13 -13
  64. data/lib/aspera/log.rb +91 -19
  65. data/lib/aspera/nagios.rb +5 -6
  66. data/lib/aspera/node_simulator.rb +12 -7
  67. data/lib/aspera/oauth/base.rb +5 -3
  68. data/lib/aspera/oauth/factory.rb +24 -18
  69. data/lib/aspera/oauth/jwt.rb +13 -1
  70. data/lib/aspera/oauth/url_json.rb +3 -3
  71. data/lib/aspera/oauth/web.rb +5 -3
  72. data/lib/aspera/persistency_folder.rb +2 -2
  73. data/lib/aspera/preview/file_types.rb +4 -3
  74. data/lib/aspera/preview/generator.rb +25 -12
  75. data/lib/aspera/preview/terminal.rb +10 -7
  76. data/lib/aspera/preview/utils.rb +11 -9
  77. data/lib/aspera/products/connect.rb +1 -1
  78. data/lib/aspera/products/desktop.rb +1 -1
  79. data/lib/aspera/products/other.rb +2 -2
  80. data/lib/aspera/products/transferd.rb +8 -6
  81. data/lib/aspera/proxy_auto_config.rb +1 -1
  82. data/lib/aspera/rest.rb +29 -22
  83. data/lib/aspera/rest_call_error.rb +1 -1
  84. data/lib/aspera/resumer.rb +1 -1
  85. data/lib/aspera/secret_hider.rb +46 -40
  86. data/lib/aspera/ssh.rb +13 -3
  87. data/lib/aspera/sync/args.schema.yaml +102 -0
  88. data/lib/aspera/sync/conf.schema.yaml +701 -0
  89. data/lib/aspera/sync/database.rb +83 -0
  90. data/lib/aspera/sync/operations.rb +296 -0
  91. data/lib/aspera/temp_file_manager.rb +3 -2
  92. data/lib/aspera/transfer/error.rb +1 -1
  93. data/lib/aspera/transfer/error_info.rb +1 -2
  94. data/lib/aspera/transfer/faux_file.rb +11 -10
  95. data/lib/aspera/transfer/parameters.rb +6 -5
  96. data/lib/aspera/transfer/spec.rb +15 -1
  97. data/lib/aspera/transfer/spec.schema.yaml +316 -293
  98. data/lib/aspera/transfer/spec_doc.rb +34 -16
  99. data/lib/aspera/transfer/uri.rb +5 -5
  100. data/lib/aspera/uri_reader.rb +14 -10
  101. data/lib/aspera/web_auth.rb +2 -2
  102. data/lib/aspera/web_server_simple.rb +2 -2
  103. data.tar.gz.sig +0 -0
  104. metadata +15 -13
  105. metadata.gz.sig +0 -0
  106. data/lib/aspera/transfer/async_conf.schema.yaml +0 -716
  107. data/lib/aspera/transfer/convert.rb +0 -29
  108. data/lib/aspera/transfer/sync.rb +0 -232
  109. data/lib/aspera/transfer/sync_instance.schema.yaml +0 -20
  110. data/lib/aspera/transfer/sync_session.schema.yaml +0 -86
@@ -44,7 +44,8 @@ module Aspera
44
44
  application
45
45
  client_registration_token
46
46
  client_access_key
47
- kms_profile].freeze
47
+ kms_profile
48
+ ].freeze
48
49
  # query to list fully received packages
49
50
  PACKAGE_RECEIVED_BASE_QUERY = {
50
51
  'archived' => false,
@@ -68,12 +69,12 @@ module Aspera
68
69
  # only org provided ?
69
70
  base_url = "#{base_url}.#{Api::AoC::SAAS_DOMAIN_PROD}" unless base_url.include?('.')
70
71
  # AoC is only https
71
- return nil unless base_url.start_with?('https://')
72
+ return unless base_url.start_with?('https://')
72
73
  res_http = Rest.new(base_url: base_url, redirect_max: 0).call(operation: 'GET', subpath: 'auth/ping', return_error: true)[:http]
73
- return nil if res_http['Location'].nil?
74
+ return if res_http['Location'].nil?
74
75
  redirect_uri = URI.parse(res_http['Location'])
75
76
  od = Api::AoC.split_org_domain(URI.parse(base_url))
76
- return nil unless redirect_uri.path.end_with?("oauth2/#{od[:organization]}/login")
77
+ return unless redirect_uri.path.end_with?("oauth2/#{od[:organization]}/login")
77
78
  # either in standard domain, or product name in page
78
79
  return {
79
80
  version: Api::AoC.saas_url?(base_url) ? 'SaaS' : 'Self-managed',
@@ -151,14 +152,14 @@ module Aspera
151
152
  formatter.display_status('We will use web authentication to bootstrap.')
152
153
  auto_set_pub_key = true
153
154
  auto_set_jwt = true
154
- raise 'TODO'
155
+ Aspera.error_not_implemented
155
156
  # aoc_api.oauth.grant_method = :web
156
157
  # aoc_api.oauth.scope = Api::AoC::SCOPE_FILES_ADMIN
157
158
  # aoc_api.oauth.specific_parameters[:redirect_uri] = REDIRECT_LOCALHOST
158
159
  end
159
160
  myself = object.aoc_api.read('self')
160
161
  if auto_set_pub_key
161
- Aspera.assert(myself['public_key'].empty?, exception_class: Cli::Error){'Public key is already set in profile (use --override=yes)'} unless option_override
162
+ Aspera.assert(myself['public_key'].empty?, type: Cli::Error){'Public key is already set in profile (use --override=yes)'} unless option_override
162
163
  formatter.display_status('Updating profile with the public key.')
163
164
  aoc_api.update("users/#{myself['id']}", {'public_key' => pub_key_pem})
164
165
  end
@@ -182,6 +183,38 @@ module Aspera
182
183
  test_args: 'user profile show'
183
184
  }
184
185
  end
186
+
187
+ # @param base [String] Base folder path
188
+ # @return [String] Folder path that does jot exist, with possible .<number> extension
189
+ def next_available_folder(base, always: false)
190
+ counter = always ? 1 : 0
191
+ loop do
192
+ result = counter.zero? ? base : "#{base}.#{counter}"
193
+ return result unless Dir.exist?(result)
194
+ counter += 1
195
+ end
196
+ end
197
+
198
+ # Get folder path that does not exist
199
+ # If it exists, an extension is added
200
+ # or a sequential number if extension == :seq
201
+ # @param folder [String] base folder
202
+ def unique_folder(folder, extension: nil, always: false)
203
+ case extension
204
+ when nil
205
+ folder
206
+ when :seq
207
+ # reuse helper
208
+ next_available_folder(folder, always: always)
209
+ else
210
+ if Dir.exist?(folder) || always
211
+ # NOTE: it might already exist
212
+ "#{folder}.#{Environment.instance.sanitized_filename(extension)}"
213
+ else
214
+ folder
215
+ end
216
+ end
217
+ end
185
218
  end
186
219
 
187
220
  def initialize(**_)
@@ -208,7 +241,8 @@ module Aspera
208
241
  def api_from_options(aoc_base_path)
209
242
  create_values = OPTIONS_NEW.each_with_object({
210
243
  subpath: aoc_base_path,
211
- secret_finder: config}) do |i, m|
244
+ secret_finder: config
245
+ }) do |i, m|
212
246
  m[i] = options.get_option(i) unless options.get_option(i).nil?
213
247
  end
214
248
  create_values[:scope] = Api::AoC::SCOPE_FILES_USER if create_values[:scope].nil?
@@ -256,7 +290,7 @@ module Aspera
256
290
  # @return identifier
257
291
  def get_resource_id_from_args(resource_class_path)
258
292
  return instance_identifier do |field, value|
259
- Aspera.assert(field.eql?('name'), exception_class: Cli::BadArgument){'only selection by name is supported'}
293
+ Aspera.assert(field.eql?('name'), type: Cli::BadArgument){'only selection by name is supported'}
260
294
  aoc_api.lookup_by_name(resource_class_path, value)['id']
261
295
  end
262
296
  end
@@ -269,7 +303,7 @@ module Aspera
269
303
  # Call block with same query using paging and response information
270
304
  # block must return a hash with :data and :http keys
271
305
  # @return [Hash] {items: , total: }
272
- def api_call_paging(base_query={})
306
+ def api_call_paging(base_query = {})
273
307
  Aspera.assert_type(base_query, Hash){'query'}
274
308
  Aspera.assert(block_given?)
275
309
  # set default large page if user does not specify own parameters. AoC Caps to 1000 anyway
@@ -305,7 +339,7 @@ module Aspera
305
339
 
306
340
  # read using the query and paging
307
341
  # @return [Hash] {data: , total: }
308
- def api_read_all(resource_class_path, base_query={})
342
+ def api_read_all(resource_class_path, base_query = {})
309
343
  return api_call_paging(base_query) do |query|
310
344
  aoc_api.call(operation: 'GET', subpath: resource_class_path, headers: {'Accept' => Rest::MIME_JSON}, query: query)
311
345
  end
@@ -331,7 +365,7 @@ module Aspera
331
365
  def resolve_dropbox_name_default_ws_id(query)
332
366
  if query.key?('dropbox_name')
333
367
  # convenience: specify name instead of id
334
- raise 'Use field dropbox_name or dropbox_id, not both' if query.key?('dropbox_id')
368
+ raise BadArgument, 'Use field dropbox_name or dropbox_id, not both' if query.key?('dropbox_id')
335
369
  # TODO : craft a query that looks for dropbox only in current workspace
336
370
  query['dropbox_id'] = aoc_api.lookup_by_name('dropboxes', query.delete('dropbox_name'))['id']
337
371
  end
@@ -340,6 +374,8 @@ module Aspera
340
374
  query['exclude_dropbox_packages'] = !query.key?('dropbox_id') unless query.key?('exclude_dropbox_packages')
341
375
  end
342
376
 
377
+ # List all packages according to `query` option.
378
+ # @param <none>
343
379
  # @return [Hash] {items,total} with all packages according to combination of user's query and default query
344
380
  def list_all_packages_with_query
345
381
  query = query_read_delete(default: {})
@@ -361,7 +397,7 @@ module Aspera
361
397
  **workspace_id_hash(name: true)
362
398
  )
363
399
  file_id = top_node_api.read("access_keys/#{top_node_api.app_info[:node_info]['access_key']}")['root_file_id'] if file_id.nil?
364
- node_plugin = Node.new(**init_params, api: top_node_api)
400
+ node_plugin = Node.new(context: context, api: top_node_api)
365
401
  case command_repo
366
402
  when *Node::COMMANDS_GEN4
367
403
  return node_plugin.execute_command_gen4(command_repo, file_id)
@@ -400,8 +436,9 @@ module Aspera
400
436
  return Main.result_transfer(transfer.start(server_apfid[:api].transfer_spec_gen4(
401
437
  server_apfid[:file_id],
402
438
  client_direction,
403
- add_ts)))
404
- else Aspera.error_unreachable_line
439
+ add_ts
440
+ )))
441
+ else Aspera.error_unexpected_value(command_repo){'command'}
405
442
  end
406
443
  Aspera.error_unreachable_line
407
444
  end
@@ -468,12 +505,12 @@ module Aspera
468
505
  return Main.result_single_object(object, fields: fields)
469
506
  when :modify
470
507
  changes = options.get_next_argument('properties', validation: Hash)
471
- return do_bulk_operation(command: command, descr: 'identifier', values: res_id) do |one_id|
508
+ return do_bulk_operation(command: command, values: res_id) do |one_id|
472
509
  aoc_api.update("#{resource_class_path}/#{one_id}", changes)
473
510
  {'id' => one_id}
474
511
  end
475
512
  when :delete
476
- return do_bulk_operation(command: command, descr: 'identifier', values: res_id) do |one_id|
513
+ return do_bulk_operation(command: command, values: res_id) do |one_id|
477
514
  aoc_api.delete("#{resource_class_path}/#{one_id}")
478
515
  {'id' => one_id}
479
516
  end
@@ -510,7 +547,7 @@ module Aspera
510
547
  when :list
511
548
  return result_list('admin/auth_providers')
512
549
  when :update
513
- raise 'not implemented'
550
+ Aspera.error_not_implemented
514
551
  end
515
552
  when :subscription
516
553
  org = aoc_api.read('organization')
@@ -612,12 +649,16 @@ module Aspera
612
649
  # cspell:enable
613
650
  result = bss_graphql.create(
614
651
  nil,
615
- {query: graphql_query,
616
- variables: {
617
- organization_id: org['id'],
618
- aggregate: aggregate,
619
- startDate: start_date,
620
- endDate: end_date}})['data']
652
+ {
653
+ query: graphql_query,
654
+ variables: {
655
+ organization_id: org['id'],
656
+ aggregate: aggregate,
657
+ startDate: start_date,
658
+ endDate: end_date
659
+ }
660
+ }
661
+ )['data']
621
662
  return Main.result_single_object(result['aoc'])
622
663
  end
623
664
  when :ats
@@ -625,13 +666,13 @@ module Aspera
625
666
  base_url: "#{aoc_api.base_url}/admin/ats/pub/v1",
626
667
  auth: {scope: Api::AoC::SCOPE_FILES_ADMIN_USER}
627
668
  }))
628
- return Ats.new(**init_params).execute_action_gen(ats_api)
669
+ return Ats.new(context: context).execute_action_gen(ats_api)
629
670
  when :analytics
630
671
  analytics_api = Rest.new(**aoc_api.params.deep_merge({
631
672
  base_url: "#{aoc_api.base_url.gsub('/api/v1', '')}/analytics/v2",
632
673
  auth: {scope: Api::AoC::SCOPE_FILES_ADMIN_USER}
633
674
  }))
634
- command_analytics = options.get_next_command(%i[application_events transfers])
675
+ command_analytics = options.get_next_command(%i[application_events transfers files])
635
676
  case command_analytics
636
677
  when :application_events
637
678
  event_type = command_analytics.to_s
@@ -639,15 +680,15 @@ module Aspera
639
680
  return Main.result_object_list(events)
640
681
  when :transfers
641
682
  event_type = command_analytics.to_s
642
- filter_resource = options.get_next_argument('resource', accept_list: %i[organizations users nodes])
643
- filter_id = options.get_next_argument('identifier', mandatory: false) ||
644
- case filter_resource
683
+ event_resource_type = options.get_next_argument('resource', accept_list: %i[organizations users nodes])
684
+ event_resource_id = options.get_next_argument("#{event_resource_type} identifier", mandatory: false) ||
685
+ case event_resource_type
645
686
  when :organizations then aoc_api.current_user_info['organization_id']
646
687
  when :users then aoc_api.current_user_info['id']
647
- when :nodes then aoc_api.current_user_info['id'] # TODO: consistent ? # rubocop:disable Lint/DuplicateBranch
688
+ when :nodes then aoc_api.current_user_info['read_only_home_node_id']
648
689
  else Aspera.error_unreachable_line
649
690
  end
650
- filter = options.get_option(:query) || {}
691
+ filter = query_read_delete(default: {})
651
692
  filter['limit'] ||= 100
652
693
  if options.get_option(:once_only, mandatory: true)
653
694
  aoc_api.context = :files
@@ -659,18 +700,17 @@ module Aspera
659
700
  'aoc_ana_date',
660
701
  options.get_option(:url, mandatory: true),
661
702
  aoc_api.workspace[:name],
662
- filter_resource.to_s,
663
- filter_id
664
- ]))
703
+ event_resource_type.to_s,
704
+ event_resource_id
705
+ ])
706
+ )
665
707
  start_date_time = saved_date.first
666
708
  stop_date_time = Time.now.utc.strftime('%FT%T.%LZ')
667
- # Log.log().error("start: #{start_date_time}")
668
- # Log.log().error("end: #{stop_date_time}")
669
709
  saved_date[0] = stop_date_time
670
710
  filter['start_time'] = start_date_time unless start_date_time.nil?
671
711
  filter['stop_time'] = stop_date_time
672
712
  end
673
- events = analytics_api.read("#{filter_resource}/#{filter_id}/#{event_type}", query_read_delete(default: filter))[event_type]
713
+ events = analytics_api.read("#{event_resource_type}/#{event_resource_id}/#{event_type}", filter)[event_type]
674
714
  start_date_persistency&.save
675
715
  if !options.get_option(:notify_to).nil?
676
716
  events.each do |tr_event|
@@ -678,6 +718,22 @@ module Aspera
678
718
  end
679
719
  end
680
720
  return Main.result_object_list(events)
721
+ when :files
722
+ event_type = command_analytics.to_s
723
+ event_resource_type = options.get_next_argument('resource', accept_list: %i[organizations users nodes])
724
+ event_resource_id = instance_identifier(description: "#{event_resource_type} identifier")
725
+ event_resource_id =
726
+ case event_resource_type
727
+ when :organizations then aoc_api.current_user_info['organization_id']
728
+ when :users then aoc_api.current_user_info['id']
729
+ when :nodes then aoc_api.current_user_info['read_only_home_node_id']
730
+ else Aspera.error_unreachable_line
731
+ end if event_resource_id.empty?
732
+ event_uuid = instance_identifier(description: 'event uuid')
733
+ filter = query_read_delete(default: {})
734
+ filter['limit'] ||= 100
735
+ events = analytics_api.read("#{event_resource_type}/#{event_resource_id}/transfers/#{event_uuid}/#{event_type}", filter)[event_type]
736
+ return Main.result_object_list(events)
681
737
  end
682
738
  when :usage_reports
683
739
  aoc_api.context = :files
@@ -701,20 +757,21 @@ module Aspera
701
757
  when :private then 'shared_folder_auth_link'
702
758
  else Aspera.error_unreachable_line
703
759
  end
704
- case options.get_next_command(%i[create delete list])
760
+ command = options.get_next_command(%i[create delete list show modify])
761
+ case command
705
762
  when :create
706
- creation_params = {
763
+ entity_data = {
707
764
  purpose: purpose_local,
708
765
  user_selected_name: nil
709
766
  }
710
767
  case link_type
711
768
  when :private
712
- creation_params[:data] = shared_data
769
+ entity_data[:data] = shared_data
713
770
  when :public
714
- creation_params[:expires_at] = nil
715
- creation_params[:password_enabled] = false
771
+ entity_data[:expires_at] = nil
772
+ entity_data[:password_enabled] = false
716
773
  shared_data[:name] = ''
717
- creation_params[:data] = {
774
+ entity_data[:data] = {
718
775
  aoc: true,
719
776
  url_token_data: {
720
777
  data: shared_data,
@@ -722,11 +779,17 @@ module Aspera
722
779
  }
723
780
  }
724
781
  end
725
- result_create_short_link = aoc_api.create('short_links', creation_params)
782
+ custom_data = value_create_modify(command: command, default: {})
783
+ if (pass = custom_data.delete('password'))
784
+ entity_data[:data][:url_token_data][:password] = pass
785
+ entity_data[:password_enabled] = true
786
+ end
787
+ entity_data.deep_merge!(custom_data)
788
+ result_create_short_link = aoc_api.create('short_links', entity_data)
726
789
  # public: Creation: permission on node
727
790
  yield(result_create_short_link['resource_id']) if block_given? && link_type.eql?(:public)
728
791
  return Main.result_single_object(result_create_short_link)
729
- when :list
792
+ when :list, :show
730
793
  query = if link_type.eql?(:private)
731
794
  shared_data
732
795
  else
@@ -744,7 +807,31 @@ module Aspera
744
807
  # embed: 'updated_by_user',
745
808
  sort: '-created_at'
746
809
  }
747
- return result_list('short_links', fields: Formatter.all_but('data'), base_query: list_params)
810
+ return result_list('short_links', fields: Formatter.all_but('data'), base_query: list_params) if command.eql?(:list)
811
+ one_id = instance_identifier
812
+ found = api_read_all('short_links', list_params)[:items].find{ |item| item['id'].eql?(one_id)}
813
+ raise Cli::BadIdentifier.new('Short link', one_id) if found.nil?
814
+ return Main.result_single_object(found, fields: Formatter.all_but('data'))
815
+ when :modify
816
+ raise Cli::BadArgument, 'Only public links can be modified' unless link_type.eql?(:public)
817
+ node_file = shared_data.slice(:node_id, :file_id)
818
+ entity_data = {
819
+ data: {
820
+ url_token_data: {
821
+ data: node_file
822
+ }
823
+ },
824
+ json_query: node_file
825
+ }
826
+ one_id = instance_identifier
827
+ custom_data = value_create_modify(command: command, default: {})
828
+ if (pass = custom_data.delete('password'))
829
+ entity_data[:data][:url_token_data][:password] = pass
830
+ entity_data[:password_enabled] = true
831
+ end
832
+ entity_data.deep_merge!(custom_data)
833
+ aoc_api.update("short_links/#{one_id}", entity_data)
834
+ return Main.result_status('modified')
748
835
  when :delete
749
836
  one_id = instance_identifier
750
837
  shared_data.delete(:workspace_id)
@@ -763,7 +850,7 @@ module Aspera
763
850
 
764
851
  # @return persistency object if option `once_only` is used.
765
852
  def package_persistency
766
- return nil unless options.get_option(:once_only, mandatory: true)
853
+ return unless options.get_option(:once_only, mandatory: true)
767
854
  # TODO: add query info to id
768
855
  PersistencyActionOnce.new(
769
856
  manager: persistency,
@@ -771,8 +858,9 @@ module Aspera
771
858
  id: IdGenerator.from_list(
772
859
  ['aoc_recv',
773
860
  options.get_option(:url, mandatory: true),
774
- aoc_api.workspace[:id]
775
- ].concat(aoc_api.additional_persistence_ids)))
861
+ aoc_api.workspace[:id]].concat(aoc_api.additional_persistence_ids)
862
+ )
863
+ )
776
864
  end
777
865
 
778
866
  def reject_packages_from_persistency(all_packages, skip_ids_persistency)
@@ -842,7 +930,7 @@ module Aspera
842
930
  end
843
931
  end
844
932
  when :packages
845
- package_command = options.get_next_command(%i[shared_inboxes send receive list show delete].concat(Node::NODE4_READ_ACTIONS), aliases: {recv: :receive})
933
+ package_command = options.get_next_command(%i[shared_inboxes send receive list show delete modify].concat(Node::NODE4_READ_ACTIONS), aliases: {recv: :receive})
846
934
  case package_command
847
935
  when :shared_inboxes
848
936
  case options.get_next_command(%i[list show short_link])
@@ -864,7 +952,7 @@ module Aspera
864
952
  package_data = value_create_modify(command: package_command)
865
953
  new_user_option = options.get_option(:new_user_option)
866
954
  option_validate = options.get_option(:validate_metadata)
867
- # works for both normal user auth and link auth
955
+ # Works for both normal user auth and link auth.
868
956
  workspace_id_hash(hash: package_data, string: true) unless package_data.key?('workspace_id')
869
957
  if !aoc_api.public_link.nil?
870
958
  aoc_api.assert_public_link_types(%w[send_package_to_user send_package_to_dropbox])
@@ -873,7 +961,7 @@ module Aspera
873
961
  # enforce workspace id from link (should be already ok, but in case user wanted to override)
874
962
  package_data['workspace_id'] = aoc_api.public_link['data']['workspace_id']
875
963
  end
876
-
964
+ package_data['encryption_at_rest'] = true if transfer.option_transfer_spec['content_protection'].eql?('encrypt')
877
965
  # transfer may raise an error
878
966
  created_package = aoc_api.create_package_simple(package_data, option_validate, new_user_option)
879
967
  Main.result_transfer(transfer.start(created_package[:spec], rest_token: created_package[:node]))
@@ -883,21 +971,21 @@ module Aspera
883
971
  ids_to_download = nil
884
972
  if !aoc_api.public_link.nil?
885
973
  aoc_api.assert_public_link_types(['view_received_package'])
886
- # set the package id from link
974
+ # Set the package id from link
887
975
  ids_to_download = aoc_api.public_link['data']['package_id']
888
976
  end
889
- # get from command line unless it was a public link
977
+ # Get from command line unless it was a public link
890
978
  ids_to_download ||= instance_identifier
891
979
  skip_ids_persistency = package_persistency
892
980
  case ids_to_download
893
- when SpecialValues::ALL, SpecialValues::INIT
981
+ when SpecialValues::INIT
982
+ all_packages = list_all_packages_with_query[:items]
983
+ Aspera.assert(skip_ids_persistency){'INIT requires option once_only'}
984
+ skip_ids_persistency.data.clear.concat(all_packages.map{ |e| e['id']})
985
+ skip_ids_persistency.save
986
+ return Main.result_status("Initialized skip for #{skip_ids_persistency.data.count} package(s)")
987
+ when SpecialValues::ALL
894
988
  all_packages = list_all_packages_with_query[:items]
895
- if ids_to_download.eql?(SpecialValues::INIT)
896
- Aspera.assert(skip_ids_persistency){'INIT requires option once_only'}
897
- skip_ids_persistency.data.clear.concat(all_packages.map{ |e| e['id']})
898
- skip_ids_persistency.save
899
- return Main.result_status("Initialized skip for #{skip_ids_persistency.data.count} package(s)")
900
- end
901
989
  # remove from list the ones already downloaded
902
990
  reject_packages_from_persistency(all_packages, skip_ids_persistency)
903
991
  ids_to_download = all_packages.map{ |e| e['id']}
@@ -908,7 +996,14 @@ module Aspera
908
996
  end
909
997
  # download all files, or specified list only
910
998
  ts_paths = transfer.ts_source_paths(default: ['.'])
911
- per_package_field = options.get_option(:package_folder)
999
+ per_package_def = options.get_option(:package_folder)
1000
+ unless per_package_def.nil?
1001
+ raise Cli::BadArgument, "Invalid package folder option : #{per_package_def}" unless per_package_def =~ /\A([^+]+)(?:\+([^?]+)(\?)?)?\z/
1002
+ per_package_field1 = Regexp.last_match(1)
1003
+ per_package_field2 = Regexp.last_match(2)
1004
+ per_package_sub_always = Regexp.last_match(3).nil?
1005
+ end
1006
+ # get value outside of loop
912
1007
  destination_folder = transfer.destination_folder(Transfer::Spec::DIRECTION_RECEIVE)
913
1008
  result_transfer = []
914
1009
  ids_to_download.each do |package_id|
@@ -916,16 +1011,30 @@ module Aspera
916
1011
  package_node_api = aoc_api.node_api_from(
917
1012
  node_id: package_info['node_id'],
918
1013
  package_info: package_info,
919
- **workspace_id_hash(name: true))
1014
+ **workspace_id_hash(name: true)
1015
+ )
920
1016
  transfer_spec = package_node_api.transfer_spec_gen4(
921
1017
  package_info['contents_file_id'],
922
1018
  Transfer::Spec::DIRECTION_RECEIVE,
923
- {'paths'=> ts_paths})
924
- transfer.option_transfer_spec['destination_root'] = File.join(destination_folder, package_info[per_package_field]) unless per_package_field.nil?
925
- formatter.display_status(%Q{Downloading package: [#{package_info['id']}] "#{package_info['name']}" to [#{transfer.option_transfer_spec['destination_root']}]})
1019
+ {'paths'=> ts_paths}
1020
+ )
1021
+ unless per_package_def.nil?
1022
+ # folder based on first field
1023
+ folder = File.join(
1024
+ destination_folder,
1025
+ Environment.instance.sanitized_filename(package_info[per_package_field1])
1026
+ )
1027
+ transfer.option_transfer_spec['destination_root'] = self.class.unique_folder(
1028
+ folder,
1029
+ extension: per_package_field2.eql?('seq') ? :seq : package_info[per_package_field2],
1030
+ always: per_package_sub_always
1031
+ )
1032
+ end
1033
+ formatter.display_status(%Q{Downloading package: [#{package_info['id']}] "#{package_info['name']}" to [#{destination_folder}]})
926
1034
  statuses = transfer.start(
927
1035
  transfer_spec,
928
- rest_token: package_node_api)
1036
+ rest_token: package_node_api
1037
+ )
929
1038
  result_transfer.push({'package' => package_id, Main::STATUS_FIELD => statuses})
930
1039
  # update skip list only if all transfer sessions completed
931
1040
  if skip_ids_persistency && TransferAgent.session_status(statuses).eql?(:success)
@@ -946,10 +1055,15 @@ module Aspera
946
1055
  display_fields += ['workspace_id'] if aoc_api.workspace[:id].nil?
947
1056
  return Main.result_object_list(result[:items], fields: display_fields, total: result[:total])
948
1057
  when :delete
949
- return do_bulk_operation(command: package_command, descr: 'identifier', values: instance_identifier) do |id|
1058
+ return do_bulk_operation(command: package_command, values: instance_identifier) do |id|
950
1059
  Aspera.assert_values(id.class, [String, Integer]){'identifier'}
951
1060
  aoc_api.delete("packages/#{id}")
952
1061
  end
1062
+ when :modify
1063
+ id = instance_identifier
1064
+ package_data = value_create_modify(command: package_command)
1065
+ aoc_api.update("packages/#{id}", package_data)
1066
+ return Main.result_status('modified')
953
1067
  when *Node::NODE4_READ_ACTIONS
954
1068
  package_id = instance_identifier
955
1069
  package_info = aoc_api.read("packages/#{package_id}")
@@ -964,7 +1078,8 @@ module Aspera
964
1078
  folder_dest = options.get_next_argument('path', validation: String)
965
1079
  home_node_api = aoc_api.node_api_from(
966
1080
  node_id: aoc_api.home[:node_id],
967
- **workspace_id_hash(name: true))
1081
+ **workspace_id_hash(name: true)
1082
+ )
968
1083
  shared_apfid = home_node_api.resolve_api_fid(aoc_api.home[:file_id], folder_dest)
969
1084
  return short_link_command(
970
1085
  purpose_public: 'view_shared_file',
@@ -1002,12 +1117,16 @@ module Aspera
1002
1117
  command_automation = options.get_next_command(%i[workflows instances])
1003
1118
  case command_automation
1004
1119
  when :instances
1005
- return entity_action(aoc_api, 'workflow_instances')
1120
+ return entity_execute(api: aoc_api, entity: 'workflow_instances')
1006
1121
  when :workflows
1007
1122
  wf_command = options.get_next_command(%i[action launch].concat(Plugin::ALL_OPS))
1008
1123
  case wf_command
1009
1124
  when *Plugin::ALL_OPS
1010
- return entity_command(wf_command, automation_api, 'workflows')
1125
+ return entity_execute(
1126
+ api: automation_api,
1127
+ entity: 'workflows',
1128
+ command: wf_command
1129
+ )
1011
1130
  when :launch
1012
1131
  wf_id = instance_identifier
1013
1132
  data = automation_api.create("workflows/#{wf_id}/launch", {})
@@ -19,6 +19,8 @@ module Aspera
19
19
  private_constant :CLOUD_TABLE
20
20
  def initialize(**_)
21
21
  super
22
+ @ats_api_pub = nil
23
+ @ats_api_pub_v1_cache = nil
22
24
  options.declare(:ibm_api_key, 'IBM API key, see https://cloud.ibm.com/iam/apikeys')
23
25
  options.declare(:instance, 'ATS instance in ibm cloud')
24
26
  options.declare(:ats_key, 'ATS key identifier (ats_xxx)')
@@ -45,7 +47,8 @@ module Aspera
45
47
  auth: {
46
48
  type: :basic,
47
49
  username: options.get_option(:ats_key, mandatory: true),
48
- password: options.get_option(:ats_secret, mandatory: true)}
50
+ password: options.get_option(:ats_secret, mandatory: true)
51
+ }
49
52
  )
50
53
  end
51
54
 
@@ -82,9 +85,7 @@ module Aspera
82
85
  # specific one do not have s3 end point in id
83
86
  params['transfer_server_id'] = server_data2['id']
84
87
  end
85
- if !params['storage'].key?('authentication_endpoint')
86
- params['storage']['endpoint'] = server_data2['s3_authentication_endpoint']
87
- end
88
+ params['storage']['endpoint'] = server_data2['s3_authentication_endpoint'] if !params['storage'].key?('authentication_endpoint')
88
89
  end
89
90
  end
90
91
  res = ats_api_pub_v1.create('access_keys', params)
@@ -120,9 +121,10 @@ module Aspera
120
121
  type: :basic,
121
122
  username: access_key_id,
122
123
  password: config.lookup_secret(url: node_url, username: access_key_id)
123
- })
124
+ }
125
+ )
124
126
  command = options.get_next_command(Node::COMMANDS_GEN4)
125
- return Node.new(**init_params, api: api_node).execute_command_gen4(command, ak_data['root_file_id'])
127
+ return Node.new(context: context, api: api_node).execute_command_gen4(command, ak_data['root_file_id'])
126
128
  when :cluster
127
129
  ats_url = ats_api_pub_v1.base_url
128
130
  api_ak_auth = Rest.new(
@@ -131,7 +133,8 @@ module Aspera
131
133
  type: :basic,
132
134
  username: access_key_id,
133
135
  password: config.lookup_secret(url: ats_url, username: access_key_id)
134
- })
136
+ }
137
+ )
135
138
  return Main.result_single_object(api_ak_auth.read('servers'))
136
139
  else Aspera.error_unexpected_value(command)
137
140
  end
@@ -150,13 +153,13 @@ module Aspera
150
153
  else
151
154
  server_id = instance_identifier
152
155
  server_data = @ats_api_pub.all_servers.find{ |i| i['id'].eql?(server_id)}
153
- raise 'no such server id' if server_data.nil?
156
+ raise BadIdentifier.new('server', server_id) if server_data.nil?
154
157
  end
155
158
  return Main.result_single_object(server_data)
156
159
  end
157
160
  end
158
161
 
159
- def ats_api_v2_auth_ibm(rest_add_headers={})
162
+ def ats_api_v2_auth_ibm(rest_add_headers = {})
160
163
  return Rest.new(
161
164
  base_url: "#{Api::Ats::SERVICE_BASE_URL}/v2",
162
165
  headers: rest_add_headers,
@@ -168,14 +171,13 @@ module Aspera
168
171
  grant_type: 'urn:ibm:params:oauth:grant-type:apikey',
169
172
  response_type: 'cloud_iam',
170
173
  apikey: options.get_option(:ibm_api_key, mandatory: true)
171
- })
174
+ }
175
+ )
172
176
  end
173
177
 
174
178
  def execute_action_api_key
175
179
  command = options.get_next_command(%i[instances create list show delete])
176
- if %i[show delete].include?(command)
177
- concerned_id = instance_identifier
178
- end
180
+ concerned_id = instance_identifier if %i[show delete].include?(command)
179
181
  rest_add_header = {}
180
182
  if !command.eql?(:instances)
181
183
  instance = options.get_option(:instance)