aspera-cli 4.27.3 → 4.27.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (79) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/CHANGELOG.md +118 -0
  4. data/CONTRIBUTING.md +5 -2
  5. data/bin/ascli +1 -0
  6. data/docs/README.md +224 -42
  7. data/lib/aspera/agent/base.rb +7 -2
  8. data/lib/aspera/agent/connect.rb +0 -4
  9. data/lib/aspera/agent/desktop.rb +0 -4
  10. data/lib/aspera/agent/direct.rb +49 -21
  11. data/lib/aspera/agent/node.rb +6 -5
  12. data/lib/aspera/agent/transferd.rb +2 -2
  13. data/lib/aspera/api/faspex.rb +2 -0
  14. data/lib/aspera/api/httpgw.rb +1 -2
  15. data/lib/aspera/api/node.rb +3 -3
  16. data/lib/aspera/ascp/installation.rb +1 -1
  17. data/lib/aspera/cli/async_transfer_store.rb +10 -9
  18. data/lib/aspera/cli/bootstrapper.rb +3 -1
  19. data/lib/aspera/cli/command_registry.rb +69 -7
  20. data/lib/aspera/cli/command_spec.rb +1 -1
  21. data/lib/aspera/cli/extended_value.rb +4 -3
  22. data/lib/aspera/cli/formatter.rb +10 -8
  23. data/lib/aspera/cli/http.rb +8 -20
  24. data/lib/aspera/cli/option_types.rb +3 -1
  25. data/lib/aspera/cli/option_value.rb +16 -19
  26. data/lib/aspera/cli/options.schema.yaml +86 -10
  27. data/lib/aspera/cli/parser.rb +30 -18
  28. data/lib/aspera/cli/plugins/aoc.rb +133 -156
  29. data/lib/aspera/cli/plugins/ats.rb +1 -7
  30. data/lib/aspera/cli/plugins/base.rb +40 -34
  31. data/lib/aspera/cli/plugins/config.rb +24 -11
  32. data/lib/aspera/cli/plugins/console.rb +1 -1
  33. data/lib/aspera/cli/plugins/faspex5.rb +30 -11
  34. data/lib/aspera/cli/plugins/faspio.rb +1 -1
  35. data/lib/aspera/cli/plugins/node.rb +26 -19
  36. data/lib/aspera/cli/plugins/orchestrator.rb +73 -44
  37. data/lib/aspera/cli/plugins/preview.rb +21 -19
  38. data/lib/aspera/cli/plugins/server.rb +3 -4
  39. data/lib/aspera/cli/plugins/shares.rb +21 -24
  40. data/lib/aspera/cli/preset_actions.rb +28 -18
  41. data/lib/aspera/cli/preset_manager.rb +34 -19
  42. data/lib/aspera/cli/prompt.rb +2 -1
  43. data/lib/aspera/cli/result.rb +33 -22
  44. data/lib/aspera/cli/runner.rb +1 -5
  45. data/lib/aspera/cli/sync_actions.rb +17 -16
  46. data/lib/aspera/cli/transfer_actions.rb +14 -3
  47. data/lib/aspera/cli/transfer_agent.rb +6 -4
  48. data/lib/aspera/cli/transfer_progress.rb +290 -55
  49. data/lib/aspera/cli/version.rb +1 -1
  50. data/lib/aspera/cli/wizard.rb +1 -1
  51. data/lib/aspera/coverage.rb +0 -1
  52. data/lib/aspera/environment.rb +29 -5
  53. data/lib/aspera/keychain/encrypted_hash.rb +1 -1
  54. data/lib/aspera/keychain/factory.rb +2 -1
  55. data/lib/aspera/log.rb +25 -2
  56. data/lib/aspera/node_emulator.rb +759 -0
  57. data/lib/aspera/oauth/base.rb +2 -1
  58. data/lib/aspera/oauth/factory.rb +6 -3
  59. data/lib/aspera/oauth/json_credentials.rb +34 -0
  60. data/lib/aspera/oauth/jwt.rb +3 -4
  61. data/lib/aspera/oauth.rb +1 -0
  62. data/lib/aspera/persistency_folder.rb +1 -3
  63. data/lib/aspera/preview/generator.rb +4 -1
  64. data/lib/aspera/preview/options.schema.yaml +119 -0
  65. data/lib/aspera/rest/aspera_errors.rb +12 -0
  66. data/lib/aspera/rest/client.rb +28 -19
  67. data/lib/aspera/rest/list.rb +14 -8
  68. data/lib/aspera/rest/parameters.rb +2 -2
  69. data/lib/aspera/schema/IBM Aspera Faspex API-5.0-enhanced.yaml +0 -20
  70. data/lib/aspera/schema/IBM Aspera Orchestrator API-v1.yaml +1784 -0
  71. data/lib/aspera/schema/IBM Aspera on Cloud API-0.2.6-enhanced.yaml +1230 -137
  72. data/lib/aspera/schema/IBM_Aspera_Shares.yaml +14 -13
  73. data/lib/aspera/schema/reader.rb +12 -18
  74. data/lib/aspera/schema/registry.rb +6 -1
  75. data.tar.gz.sig +0 -0
  76. metadata +5 -3
  77. metadata.gz.sig +0 -0
  78. data/lib/aspera/node_simulator.rb +0 -345
  79. data/lib/aspera/preview/options.rb +0 -45
@@ -8,6 +8,7 @@ require 'aspera/cli/extended_value'
8
8
  require 'aspera/cli/special_values'
9
9
  require 'aspera/cli/version'
10
10
  require 'aspera/cli/formatter'
11
+ require 'aspera/cli/http'
11
12
  require 'aspera/cli/info'
12
13
  require 'aspera/cli/wizard'
13
14
  require 'aspera/cli/sync_actions'
@@ -99,16 +100,15 @@ module Aspera
99
100
  option :cache_tokens, description: 'Save and reuse OAuth tokens', allowed: Type::BOOLEAN, default: true
100
101
  option :expand_mounts, description: 'Commands: list commands of sub-trees provided by another plugin', allowed: Type::BOOLEAN, default: false
101
102
  option :no_default, description: 'Do not load default configuration for plugin', allowed: Type::NONE, short: 'N', on_set: -> { presets.use_plugin_defaults = false }
103
+ use_options Wizard
104
+ # `on_set` callbacks of HTTP options are methods of `http_config`
105
+ use_options Http, target: -> { http_config }
102
106
 
103
107
  def initialize(**_)
104
108
  super
105
109
  @vault_instance = nil
106
110
  @sdk_default_location = false
107
- # Declare wizard options (Wizard#initialize calls options.declare internally)
108
111
  @wizard = Wizard.new(self, context.main_folder)
109
- # HTTP options: declare metadata (class method), then bind to the instance
110
- Http.declare_options(options)
111
- context.http_config.bind_options(options)
112
112
  # Values set through `on_set` callbacks are used below (sdk_folder)
113
113
  options.parse_options!
114
114
  set_sdk_dir
@@ -186,7 +186,8 @@ module Aspera
186
186
  {name: :preset_name, mandatory: false, default: ''}],
187
187
  action: lambda do |url:, plugin_name: nil, preset_name: '', **|
188
188
  apps = @wizard.identify_plugins_for_url(url: url, plugin_name: plugin_name).freeze
189
- @wizard.find(apps, preset_name: preset_name)
189
+ # The product wizard prompts for missing mandatory options (username, password, ...)
190
+ options.with_interactive { @wizard.find(apps, preset_name: preset_name) }
190
191
  end
191
192
  )
192
193
  command :coffee, description: 'Show a coffee image', action: ->(**) { Result::Image.new(COFFEE_IMAGE_URL) }
@@ -396,8 +397,7 @@ module Aspera
396
397
  Result::ObjectList.new(builder.rows, fields: builder.columns)
397
398
  end
398
399
  )
399
- command :admin, description: 'Manage sync database (admin operations)'
400
- SyncActions.register_sync_admin_commands(self, :admin)
400
+ sync_admin_commands :admin
401
401
  command :translate, description: 'Translate async-style arguments to sync config format',
402
402
  arguments: [{name: :async_arguments, multiple: true}],
403
403
  action: ->(async_arguments:, **) { Result::SingleObject.new(Sync::Operations.args_to_conf(async_arguments)) }
@@ -493,7 +493,7 @@ module Aspera
493
493
 
494
494
  def action_download(file_url:, file_dest: nil, **)
495
495
  file_url = file_url.chomp
496
- file_dest = File.join(transfer.destination_folder(Transfer::Spec::DIRECTION_RECEIVE), file_url.gsub(%r{.*/}, '')) if file_dest.nil?
496
+ file_dest = File.join(transfer.destination_folder(Transfer::Spec::DIRECTION_RECEIVE), file_url.rpartition('/').last) if file_dest.nil?
497
497
  Log.log.info("Downloading: #{file_url}")
498
498
  Rest::Client.new(base_url: file_url).call(operation: 'GET', save_to: file_dest)
499
499
  Result::Status.new("Saved to: #{file_dest}")
@@ -569,12 +569,12 @@ module Aspera
569
569
  Aspera.assert_values(plugin_name.to_sym, plugin_names, type: Cli::BadArgument)
570
570
  plugin_names = [plugin_name.to_sym]
571
571
  end
572
- prefix = Array(command_path).map(&:to_sym)
573
- Aspera.assert(prefix.empty? || plugin_names.length.eql?(1), type: Cli::BadArgument) { 'command path requires a plugin name' }
572
+ words = Array(command_path).map(&:to_sym)
573
+ Aspera.assert(words.empty? || plugin_names.length.eql?(1), type: Cli::BadArgument) { 'command path requires a plugin name' }
574
574
  expand_mounts = options.get_option(:expand_mounts)
575
575
  commands = plugin_names.flat_map do |name|
576
576
  reg = Plugins::Factory.instance.plugin_class(name).command_registry
577
- Aspera.assert(prefix.empty? || reg[prefix], type: Cli::BadArgument) { "no such command: #{name} #{prefix.join(' ')}" }
577
+ prefix = listed_command_path(name, reg, words)
578
578
  paths = prefix.empty? || reg.children_of(prefix).any? ? reg.leaf_paths(prefix, expand_mounts: expand_mounts) : [prefix]
579
579
  paths.map do |path|
580
580
  mount = expand_mounts ? nil : reg.mount_at(path)
@@ -592,6 +592,19 @@ module Aspera
592
592
  Result::ObjectList.new(commands, fields: %w[syntax description])
593
593
  end
594
594
 
595
+ # Command path to list, from words that may include positional arguments, as on a command line.
596
+ # e.g. `faspex5 packages receive ALL` lists `faspex5 packages receive`.
597
+ # @param name [Symbol] plugin name
598
+ # @param reg [CommandRegistry] registry of plugin
599
+ # @param words [Array<Symbol>] sub-commands (or aliases), possibly mixed with their arguments
600
+ # @return [Array<Symbol>] command path
601
+ # @raise [Cli::BadArgument] if a word is neither a sub-command nor an expected argument
602
+ def listed_command_path(name, reg, words)
603
+ path, unknown = reg.command_path(words)
604
+ raise Cli::BadArgument, Parser.multi_choice_assert_msg("no such command: #{[name, *path, unknown].join(' ')}", reg.children_of(path).keys) unless unknown.nil?
605
+ path
606
+ end
607
+
595
608
  # Build syntax by interleaving each path segment with the arguments declared on that node
596
609
  # @param name [String, Symbol] plugin name
597
610
  # @param reg [CommandRegistry] registry of plugin
@@ -63,7 +63,7 @@ module Aspera
63
63
  username: options.get_option(:username, mandatory: true),
64
64
  password: options.get_option(:password, mandatory: true)
65
65
  },
66
- test_args: 'transfer list'
66
+ test_args: 'transfer current list'
67
67
  }
68
68
  end
69
69
 
@@ -16,6 +16,8 @@ require 'aspera/nagios'
16
16
  require 'aspera/environment'
17
17
  require 'aspera/assert'
18
18
  require 'securerandom'
19
+ require 'digest'
20
+ require 'json'
19
21
  require 'aspera/rainbow'
20
22
  using Rainbow
21
23
 
@@ -143,7 +145,7 @@ module Aspera
143
145
  end
144
146
  if status_list.include?(status['upload_status'])
145
147
  progress_bar&.event(:session_end, session_id: id)
146
- progress_bar&.event(:end)
148
+ progress_bar&.event(:end, info: !status['upload_status'].eql?('failed'))
147
149
  return status
148
150
  end
149
151
  sleep(1.0)
@@ -195,13 +197,25 @@ module Aspera
195
197
  # Build query to get package recipients based on package info in case of shared inbox or workgroup recipient
196
198
  # @param package_id [String] the package id to get info from
197
199
  def recipient_query(package_id)
200
+ # With a public link, the recipient context is already encoded in the auth token: no extra query params needed
201
+ return {} if api_v5.pub_link_context
198
202
  package_info = api_v5.read("packages/#{package_id}")
199
203
  base_query = {}
200
- base_query['recipient_workgroup_id'] = package_info['recipients'].first['id'] if WORKGROUP_TYPES.include?(package_info['recipients'].first['recipient_type'])
201
- base_query['recipient_user_id'] = package_info['recipients'].first['id'] if package_info['recipients'].first['recipient_type'].eql?('user')
204
+ first_recipient = package_info['recipients']&.first
205
+ if first_recipient
206
+ base_query['recipient_workgroup_id'] = first_recipient['id'] if WORKGROUP_TYPES.include?(first_recipient['recipient_type'])
207
+ base_query['recipient_user_id'] = first_recipient['id'] if first_recipient['recipient_type'].eql?('user')
208
+ end
202
209
  base_query
203
210
  end
204
211
 
212
+ # @return [String] user part of the persistency id of received packages:
213
+ # the username, or for a public link (no username), a digest of the link context (unique per link)
214
+ def persistency_user_id
215
+ return options.get_option(:username, mandatory: true) if api_v5.pub_link_context.nil?
216
+ Digest::SHA256.hexdigest(JSON.generate(api_v5.pub_link_context))[0, 16]
217
+ end
218
+
205
219
  def package_receive(package_ids)
206
220
  # prepare persistency if needed
207
221
  skip_ids_persistency = nil
@@ -213,7 +227,7 @@ module Aspera
213
227
  id: IdGenerator.from_list(
214
228
  'faspex_recv',
215
229
  options.get_option(:url, mandatory: true),
216
- options.get_option(:username, mandatory: true),
230
+ persistency_user_id,
217
231
  options.get_option(:box, mandatory: true)
218
232
  )
219
233
  )
@@ -229,7 +243,7 @@ module Aspera
229
243
  # TODO: if packages have same name, they will overwrite ?
230
244
  packages, max_items = list_packages_with_filter(query: {'status' => 'completed'}) # no filter: all completed packages
231
245
  Log.dump(:package_ids, level: :trace1) { packages.map { |p| p['id'] } }
232
- Log.dump(:skip_ids, skip_ids_persistency.data, level: :trace1)
246
+ Log.dump(:skip_ids, skip_ids_persistency&.data, level: :trace1)
233
247
  packages.reject! { |p| skip_ids_persistency.data.include?(p['id']) } if skip_ids_persistency
234
248
  # Apply `max` after once_only filtering so we get the N first not-yet-downloaded packages
235
249
  packages = packages[0, max_items] if max_items
@@ -240,11 +254,13 @@ module Aspera
240
254
  Aspera.assert_array_all(package_ids, String) { 'Package id(s)' }
241
255
  # packages = package_ids.map{|pkg_id|api_v5.read("packages/#{pkg_id}")}
242
256
  packages = package_ids.map { |pkg_id| {'id'=>pkg_id} }
257
+ # The package of a public link is received only once, like with `ALL`
258
+ packages.reject! { |p| skip_ids_persistency.data.include?(p['id']) } if skip_ids_persistency && api_v5.pub_link_context
243
259
  end
244
260
  result_transfer = []
245
261
  param_file_list = {}
246
262
  begin
247
- param_file_list['paths'] = transfer.ts_source_paths
263
+ param_file_list['paths'] = transfer.ts_source_paths.map { |p| {'path' => p['source']} }
248
264
  rescue Cli::MissingArgument
249
265
  # paths is optional
250
266
  end
@@ -305,7 +321,8 @@ module Aspera
305
321
  )
306
322
  # well, we asked a TS for connect, but we actually want a generic one
307
323
  transfer_spec.delete('authentication')
308
- return Runner.result_transfer(transfer.start(transfer_spec))
324
+ Runner.result_transfer(transfer.start(transfer_spec))
325
+ return Result::SingleObject.new(package)
309
326
  else
310
327
  # send from remote shared folder
311
328
  if (m = Parser.percent_selector(shared_folder))
@@ -493,7 +510,7 @@ module Aspera
493
510
  arguments: [{name: :package, type: Hash, schema: Schema::Registry.req_body(Schema::Registry::FASPEX, 'packages.post')}],
494
511
  action: ->(package:, **) { package_send(package) }
495
512
  command :show, description: 'Show a package', setup: :setup_package_id, arguments: PACKAGE_ID_ARG,
496
- action: ->(package_id:, **) { Result::SingleObject.new(api_v5.read("packages/#{package_id}")) }
513
+ action: ->(package_id:, **) { Result::SingleObject.new(api_v5.read("packages/#{package_id}", query_read_delete)) }
497
514
  command :browse, description: 'Browse package files', setup: :setup_package_id,
498
515
  arguments: PACKAGE_ID_ARG + [{name: :folder_path, mandatory: false, default: '/'}],
499
516
  action: ->(folder_path:, package_id:, **) { browse_folder("packages/#{package_id}/files/#{Api::Faspex.box_type(options.get_option(:box))}", recipient_query(package_id), folder_path: folder_path) }
@@ -833,9 +850,11 @@ module Aspera
833
850
  res_path = "#{res_instance_path}/members"
834
851
  api_v5.lookup_entity_by_field(entity: res_path, field: field, value: value, query: Rest.php_style({type: %w[user]}))['user_id']
835
852
  end
836
- define_method(:"lookup_#{res}_saml_groups_id") do |field, value, res_instance_path:, **|
837
- res_path = "#{res_instance_path}/saml_groups"
838
- api_v5.lookup_entity_by_field(entity: res_path, field: field, value: value, query: Rest.php_style({type: %w[user]}))['user_id']
853
+ # Like members (`user_id`), a SAML group of the resource is identified by `group_id`; its name is `group_name`
854
+ define_method(:"lookup_#{res}_saml_groups_id") do |field, value, saml_groups_path:, **|
855
+ field = 'group_name' if field.eql?('name')
856
+ query = field.eql?('group_name') ? {'q' => value} : {}
857
+ api_v5.lookup_entity_by_field(entity: saml_groups_path, field: field, value: value, items_key: 'groups', query: query)['group_id']
839
858
  end
840
859
  end
841
860
 
@@ -33,7 +33,7 @@ module Aspera
33
33
  preset_value: {
34
34
  url: app_url
35
35
  },
36
- test_args: 'info'
36
+ test_args: 'bridges list'
37
37
  }
38
38
  end
39
39
 
@@ -11,7 +11,6 @@ require 'aspera/hash_ext'
11
11
  require 'aspera/id_generator'
12
12
  require 'aspera/api/node'
13
13
  require 'aspera/oauth'
14
- require 'aspera/node_simulator'
15
14
  require 'aspera/rest/list'
16
15
  require 'aspera/assert'
17
16
  require 'base64'
@@ -356,8 +355,7 @@ module Aspera
356
355
  Sync::Operations::DIRECTIONS.each do |dir|
357
356
  command dir, description: "#{dir.capitalize}-sync (Gen3)", transfer_paths: :send, arguments: SyncActions::PATH_AND_INFO_ARGS
358
357
  end
359
- command :admin, description: 'Manage sync database (admin operations)'
360
- SyncActions.register_sync_admin_commands(self, :admin)
358
+ sync_admin_commands :admin
361
359
  end
362
360
  # access_keys sub-tree
363
361
  command :access_keys, description: 'Manage access keys'
@@ -387,8 +385,7 @@ module Aspera
387
385
  Sync::Operations::DIRECTIONS.each do |dir|
388
386
  command dir, description: "#{dir.capitalize}-sync", transfer_paths: :send, arguments: SyncActions::PATH_AND_INFO_ARGS
389
387
  end
390
- command :admin, description: 'Manage sync database (admin operations)'
391
- SyncActions.register_sync_admin_commands(self, :admin)
388
+ sync_admin_commands :admin
392
389
  end
393
390
  end
394
391
  commands_under %i[access_keys do permission] do
@@ -591,8 +588,9 @@ module Aspera
591
588
  Result::Text.new(Api::Node.bearer_token(payload: token, access_key: access_key, private_key: private_key))
592
589
  end
593
590
  )
594
- command :simulator, description: 'Start node simulator',
595
- arguments: [{name: :parameters, type: Hash, mandatory: false, default: {}, schema: 'opts:components.schemas.NodeSimulatorOptions'}]
591
+ command :emulator, description: 'Start node emulator',
592
+ aliases: [:simulator],
593
+ arguments: [{name: :parameters, type: Hash, mandatory: false, default: {}, schema: Schema::Registry::NODE_EMULATOR_OPTIONS}]
596
594
  command :telemetry, description: 'Report telemetry to external system',
597
595
  arguments: [{name: :parameters, type: Hash, mandatory: false, default: {}, schema: 'opts:components.schemas.NodeTelemetryOptions'}]
598
596
 
@@ -655,7 +653,7 @@ module Aspera
655
653
  transfer_spec = api_node.create(
656
654
  'files/sync_setup',
657
655
  {transfer_requests: [{transfer_request: request_transfer_spec}]}
658
- )['transfer_specs'].first['transfer_spec']
656
+ )['transfer_specs'].first&.fetch('transfer_spec')
659
657
  transfer_spec.delete_if { |_k, v| v.nil? }
660
658
  Log.dump(:ts, transfer_spec)
661
659
  transfer_spec
@@ -663,7 +661,7 @@ module Aspera
663
661
  end
664
662
 
665
663
  Sync::Operations::DIRECTIONS.each do |dir|
666
- define_method(:"action_sync_#{dir}") { |path:, sync_info: {}, **| run_sync_transfer(dir, path: path, sync_info: sync_info, &sync_gen3_block) }
664
+ define_action_method([:sync, dir]) { |path:, sync_info: {}, **| run_sync_transfer(dir, path: path, sync_info: sync_info, &sync_gen3_block) }
667
665
  end
668
666
 
669
667
  def action_upload(**)
@@ -674,7 +672,7 @@ module Aspera
674
672
  api_node.add_tspec_info(request_transfer_spec) if api_node.respond_to?(:add_tspec_info)
675
673
  Api::Node.add_public_key(request_transfer_spec)
676
674
  setup_payload = {transfer_requests: [{transfer_request: request_transfer_spec}]}
677
- transfer_spec = api_node.create('files/upload_setup', setup_payload)['transfer_specs'].first['transfer_spec']
675
+ transfer_spec = api_node.create('files/upload_setup', setup_payload)['transfer_specs'].first&.fetch('transfer_spec')
678
676
  Api::Node.add_private_key(transfer_spec)
679
677
  transfer_spec.delete('paths')
680
678
  Runner.result_transfer(transfer.start(transfer_spec))
@@ -688,7 +686,7 @@ module Aspera
688
686
  api_node.add_tspec_info(request_transfer_spec) if api_node.respond_to?(:add_tspec_info)
689
687
  Api::Node.add_public_key(request_transfer_spec)
690
688
  setup_payload = {transfer_requests: [{transfer_request: request_transfer_spec}]}
691
- transfer_spec = api_node.create('files/download_setup', setup_payload)['transfer_specs'].first['transfer_spec']
689
+ transfer_spec = api_node.create('files/download_setup', setup_payload)['transfer_specs'].first&.fetch('transfer_spec')
692
690
  Api::Node.add_private_key(transfer_spec)
693
691
  Runner.result_transfer(transfer.start(transfer_spec))
694
692
  end
@@ -824,7 +822,7 @@ module Aspera
824
822
  end
825
823
 
826
824
  Sync::Operations::DIRECTIONS.each do |dir|
827
- define_method(:"action_access_keys_do_sync_#{dir}") do |path:, sync_info: {}, do_root_file_id:, **|
825
+ define_action_method([:access_keys, :do, :sync, dir]) do |path:, sync_info: {}, do_root_file_id:, **|
828
826
  run_sync_transfer(dir, path: path, sync_info: sync_info, &sync_gen4_block(do_root_file_id))
829
827
  end
830
828
  end
@@ -1015,7 +1013,7 @@ module Aspera
1015
1013
 
1016
1014
  def action_async_counters(async_id:, **)
1017
1015
  Integer(async_id)
1018
- resp = api_node.create('async/counters', {'syncs' => [async_id]})['sync_counters'].first[async_id].last
1016
+ resp = api_node.create('async/counters', {'syncs' => [async_id]})['sync_counters'].first&.[](async_id)&.last
1019
1017
  return Result::Empty.new if resp.nil?
1020
1018
  Result::SingleObject.new(resp)
1021
1019
  end
@@ -1131,14 +1129,23 @@ module Aspera
1131
1129
  return Result::Status.new('done')
1132
1130
  end
1133
1131
 
1134
- def action_simulator(parameters: {}, **)
1135
- require 'aspera/node_simulator'
1136
- parameters = parameters.symbolize_keys
1137
- uri = URI.parse(parameters.delete(:url) { WebServerSimple::DEFAULT_URL })
1132
+ def action_emulator(parameters: {}, **)
1133
+ require 'aspera/node_emulator'
1134
+ # missing parameters take the default of the schema
1135
+ defaults = Schema::Registry.instance.reader(Schema::Registry::NODE_EMULATOR_OPTIONS).current['properties'].each_with_object({}) do |(name, property), result|
1136
+ result[name.to_sym] = property['default'] if property.key?('default')
1137
+ end
1138
+ parameters = defaults.merge(parameters.symbolize_keys)
1139
+ uri = URI.parse(parameters.delete(:url))
1140
+ config = parameters.except(*WebServerSimple::PARAMS)
1141
+ docroot = config.delete(:docroot)
1142
+ retention_sec = config.delete(:retention_sec)
1143
+ Aspera.assert(config[:username].nil? == config[:password].nil?, type: Cli::BadArgument) { 'Parameters username and password must be set together' }
1144
+ Log.log.warn('No username and password: emulator accepts requests without authentication') if config[:username].nil?
1138
1145
  server = WebServerSimple.new(uri, **parameters.slice(*WebServerSimple::PARAMS))
1139
- server.mount(uri.path, NodeSimulatorServlet, parameters.except(*WebServerSimple::PARAMS), NodeSimulator.new)
1146
+ server.mount(uri.path, NodeEmulatorServlet, config, NodeEmulator.new(docroot: docroot, retention_sec: retention_sec))
1140
1147
  server.start
1141
- return Result::Status.new('Simulator terminated')
1148
+ return Result::Status.new('Emulator terminated')
1142
1149
  end
1143
1150
 
1144
1151
  def action_telemetry(parameters: {}, **)
@@ -52,36 +52,37 @@ module Aspera
52
52
  username: options.get_option(:username, mandatory: true),
53
53
  password: options.get_option(:password, mandatory: true)
54
54
  },
55
- test_args: 'workflow list'
55
+ test_args: 'workflows list'
56
56
  }
57
57
  end
58
58
 
59
+ option :apikey, description: 'API key (exclusive with username/password)'
60
+ option :auth_style, description: 'Authentication style', allowed: %i[basic query token], default: :token
61
+ option :ret_style, description: 'Method to specify the expected response format ("Accept")', allowed: %i[header query path], default: :query
59
62
  option :result, description: "Specify result value as: 'work_step:parameter'", deprecation: {last: '4.27.2', message: 'use keys `step` and `variable` of argument `execution` of `workflows start`'}
60
63
  option :synchronous, description: 'Wait for completion', allowed: Type::BOOLEAN, deprecation: {last: '4.27.2', message: 'use key `synchronous` of argument `execution` of `workflows start`'}
61
- option :ret_style, description: 'How return type is requested in api', allowed: %i[header arg ext], default: :arg
62
- option :auth_style, description: 'Authentication type', allowed: %i[arg_pass head_basic apikey], default: :head_basic
63
64
 
64
- # Call orchestrator API, it's a bit special
65
+ # Call orchestrator API (handles ret_style negotiation and XML parsing)
65
66
  # @param endpoint [String] the endpoint to call
66
- # @param ret_style [Symbol] the return style, :header, :arg, :ext(extension)
67
+ # @param body [Hash] the body to pass (JSON); also implies POST
67
68
  # @param format [String] the format to request, 'json', 'xml', nil
68
- # @param args [Hash] the arguments to pass
69
+ # @param query [Hash] the arguments to pass as query parameters
69
70
  # @param xml_arrays [Boolean] if true, force arrays in xml parsing
70
71
  # @param http [Boolean] if true, returns the HttpResponse, else
71
- def call_ao(endpoint, ret_style: nil, format: 'json', args: nil, xml_arrays: true, http: false)
72
- # calls are all GET
73
- call_args = {operation: 'GET', subpath: "api/#{endpoint}", ret: :both, query: {}}
74
- ret_style = options.get_option(:ret_style, mandatory: true) if ret_style.nil?
75
- call_args[:query].merge!(args) unless args.nil?
72
+ def call_ao(endpoint, body: nil, format: 'json', query: nil, xml_arrays: true, http: false)
73
+ call_args = {operation: body.nil? ? 'GET' : 'POST', subpath: "api/#{endpoint}", ret: :both, query: {}}
74
+ call_args.merge!(body: body, content_type: Mime::JSON) unless body.nil? # rubocop:disable Performance/RedundantMerge
75
+ ret_style = options.get_option(:ret_style, mandatory: true)
76
+ call_args[:query].merge!(query) unless query.nil?
76
77
  unless format.nil?
77
78
  case ret_style
78
79
  when :header
79
80
  call_args[:headers] = {'Accept' => "application/#{format}"}
80
- when :arg
81
+ when :query
81
82
  call_args[:query][:format] = format
82
- when :ext
83
+ when :path
83
84
  call_args[:subpath] = "#{call_args[:subpath]}.#{format}"
84
- else Aspera.error_unexpected_value(ret_style)
85
+ else Aspera.error_unexpected_value(ret_style) { 'ret_style' }
85
86
  end
86
87
  end
87
88
  add_query = query_read_delete
@@ -99,13 +100,18 @@ module Aspera
99
100
 
100
101
  command :health, description: 'Check Orchestrator API health'
101
102
  command :info, description: 'Check that Orchestrator responds (ping)', action: ->(**) { Result::SingleObject.new(call_ao('remote_node_ping', format: 'xml', xml_arrays: false)) }
102
- command :processes, description: 'Show Orchestrator background process status', action: ->(**) { Result::ObjectList.new(call_ao('processes_status', format: 'xml')['process']) }
103
+ command :processes, description: 'Show Orchestrator background process status', action: ->(**) { Result::ObjectList.new(call_ao('processes_status', format: 'xml')['node'].flat_map { |n| n['process'] }) }
103
104
  command :monitors, description: 'Show Orchestrator monitor snapshot', action: ->(**) { Result::SingleObject.new(call_ao('monitor_snapshot')['monitor']) }
104
- command :plugins, description: 'Show Orchestrator plugin versions', action: ->(**) { Result::ObjectList.new(call_ao('plugin_version')['Plugin']) }
105
- command :workflows, description: 'Manage workflows'
106
105
  command :workorders, description: 'Manage work orders'
107
106
  command :workstep, description: 'Manage work steps'
108
107
 
108
+ commands_under :plugins do
109
+ command :list, description: 'Show Orchestrator plugin versions', action: ->(**) { Result::ObjectList.new(call_ao('plugin_version')['Plugin']) }
110
+ command :reload_set, description: 'Reload a set of plugins',
111
+ arguments: [{name: :plugin_set, type: Hash, schema: 'opts:components.schemas.OrchestratorReloadPluginSet'}],
112
+ action: ->(plugin_set:, **) { Result::SingleObject.new(call_ao('reload_plugin_set', body: plugin_set.transform_keys(&:to_sym))) }
113
+ end
114
+
109
115
  commands_under :workflows do
110
116
  command :list, description: 'List all workflows'
111
117
  command :status, description: 'Show running status of a workflow',
@@ -120,9 +126,18 @@ module Aspera
120
126
  command :start, description: 'Start a workflow: create a work order (sync or async)',
121
127
  arguments: [
122
128
  {name: :workflow_id, type: :identifier},
123
- {name: :parameters, type: Hash, mandatory: false, default: {}},
129
+ {name: :parameters, type: Hash, mandatory: false, default: {}, schema: 'opts:components.schemas.OrchestratorInitiateParameters'},
124
130
  {name: :execution, type: Hash, mandatory: false, default: {}, schema: 'opts:components.schemas.OrchestratorWorkflowStart'}
125
131
  ]
132
+ command :import, description: 'Import a workflow',
133
+ arguments: [{name: :workflow, type: Hash, schema: 'opts:components.schemas.OrchestratorImportWorkflow'}],
134
+ action: ->(workflow:, **) { Result::SingleObject.new(call_ao('import_workflow', body: workflow.transform_keys(&:to_sym))) }
135
+ command :publish, description: 'Publish a workflow',
136
+ arguments: [{name: :workflow_id, type: :identifier}],
137
+ action: ->(workflow_id:, **) { Result::SingleObject.new(call_ao('publish_workflow', body: {workflow_id: workflow_id.to_i})) }
138
+ command :import_with_constraints, description: 'Import a workflow with constraint resolution',
139
+ arguments: [{name: :payload, type: Hash, schema: 'opts:components.schemas.OrchestratorImportWithConstraints'}],
140
+ action: ->(payload:, **) { Result::SingleObject.new(call_ao('import_with_constraints', body: payload.transform_keys(&:to_sym))) }
126
141
  command :export, description: 'Export a workflow',
127
142
  arguments: [{name: :workflow_id, type: :identifier}],
128
143
  action: ->(workflow_id:, **) { Result::Text.new(call_ao("export_workflow/#{workflow_id}", format: nil, http: true).body) }
@@ -164,28 +179,43 @@ module Aspera
164
179
  # @return [Rest::Client]
165
180
  def api_orch
166
181
  return @api_orch if @api_orch
182
+ base_url = options.get_option(:url, mandatory: true)
183
+ style = options.get_option(:auth_style, mandatory: true)
184
+ apikey = options.get_option(:apikey)
185
+ username = options.get_option(:username)
186
+ password = options.get_option(:password)
187
+ Aspera.assert(apikey.nil? || (username.nil? && password.nil?), type: Cli::BadArgument) { 'apikey and username/password are mutually exclusive' }
188
+ Aspera.assert(!apikey.nil? || (!username.nil? && !password.nil?), type: Cli::BadArgument) { 'provide either apikey or username and password' }
189
+
167
190
  auth_params =
168
- case options.get_option(:auth_style, mandatory: true)
169
- when :arg_pass
191
+ case style
192
+ when :basic
193
+ Aspera.assert(apikey.nil?, type: Cli::BadArgument) { 'basic auth style cannot be used with apikey, use token or query' }
170
194
  {
171
- type: :url,
172
- url_query: {
173
- 'login' => options.get_option(:username, mandatory: true),
174
- 'password' => options.get_option(:password, mandatory: true)
175
- }
195
+ type: :basic,
196
+ username: username,
197
+ password: password
176
198
  }
177
- when :head_basic
199
+ when :query
200
+ if apikey
201
+ {type: :url, url_query: {'apikey' => apikey}}
202
+ else
203
+ {type: :url, url_query: {'login' => username, 'password' => password}}
204
+ end
205
+ when :token
178
206
  {
179
- type: :basic,
180
- username: options.get_option(:username, mandatory: true),
181
- password: options.get_option(:password, mandatory: true)
207
+ type: :oauth2,
208
+ grant_method: :json_credentials,
209
+ base_url: base_url,
210
+ path_token: 'api/login',
211
+ token_field: 'token',
212
+ json: apikey ? {apikey: apikey} : {username: username, password: password}
182
213
  }
183
- when :apikey
184
- Aspera.error_not_implemented
214
+ else Aspera.error_unexpected_value(style)
185
215
  end
186
216
  @api_orch = Rest::Client.new(
187
- base_url: options.get_option(:url, mandatory: true),
188
- auth: auth_params
217
+ base_url: base_url,
218
+ auth: auth_params
189
219
  )
190
220
  end
191
221
 
@@ -225,23 +255,22 @@ module Aspera
225
255
  Aspera.assert(fields.length == 2, type: Cli::BadArgument) { "Expects: work_step:result_name : #{result_location}" }
226
256
  execution['step'], execution['variable'] = fields
227
257
  end
228
- call_params = {format: :json}
229
- # get external parameters if any
230
- parameters.each do |name, value|
231
- call_params["external_parameters[#{name}]"] = value
232
- end
258
+ # POST /api/initiate with JSON body {workflow_id:, external_parameters:}
259
+ json_body = {workflow_id: workflow_id.to_i, external_parameters: parameters}
260
+ # control params passed as query string (not part of the spec requestBody)
261
+ query_params = {}
233
262
  Aspera.assert_type(execution['synchronous'], NilClass, *BoolValue::TYPES, type: Cli::BadArgument) { 'synchronous' }
234
- call_params['synchronous'] = true if execution['synchronous'].eql?(true)
263
+ query_params[:synchronous] = true if execution['synchronous'].eql?(true)
235
264
  # expected result for synchro call ?
236
265
  if execution.key?('step') || execution.key?('variable')
237
266
  Aspera.assert(execution['step'].is_a?(String) && execution['variable'].is_a?(String), type: Cli::BadArgument) { 'Both step and variable are required' }
238
- call_params['explicit_output_step'] = execution['step']
239
- call_params['explicit_output_variable'] = execution['variable']
267
+ query_params[:explicit_output_step] = execution['step']
268
+ query_params[:explicit_output_variable] = execution['variable']
240
269
  # implicitly, call is synchronous
241
- call_params['synchronous'] = true
270
+ query_params[:synchronous] = true
242
271
  end
243
- result_data = call_ao("initiate/#{workflow_id}", args: call_params)
244
- call_params['synchronous'] ? Result::Text.new(result_data) : Result::SingleObject.new(result_data)
272
+ result_data = call_ao('initiate', body: json_body, query: query_params.empty? ? nil : query_params)
273
+ query_params[:synchronous] ? Result::Text.new(result_data) : Result::SingleObject.new(result_data)
245
274
  end
246
275
  end
247
276
  end
@@ -3,11 +3,11 @@
3
3
  # cspell:ignore trevents
4
4
  require 'aspera/cli/plugins/basic_auth'
5
5
  require 'aspera/preview/generator'
6
- require 'aspera/preview/options'
7
6
  require 'aspera/preview/utils'
8
7
  require 'aspera/preview/file_types'
9
8
  require 'aspera/preview/terminal'
10
9
  require 'aspera/transfer/spec'
10
+ require 'aspera/schema/registry'
11
11
  require 'aspera/persistency_action_once'
12
12
  require 'aspera/temp_file_manager'
13
13
  require 'aspera/api/node'
@@ -60,29 +60,30 @@ module Aspera
60
60
  option :detect_mime, description: 'Detect Mime type by analyzing file', allowed: Type::BOOLEAN, default: false
61
61
  option :overwrite, description: 'When to overwrite result file', allowed: %i[always never mtime], default: :mtime
62
62
  option :root_url, description: "How to read and write files on storage (#{REMOTE_ACCESS}, or #{UriReader.file_url('<folder>')})", default: REMOTE_ACCESS
63
- # Generator-specific options (bound to @gen_options in initialize)
64
- Aspera::Preview::Options::DESCRIPTIONS.each do |opt|
65
- values = if opt.key?(:values)
66
- opt[:values]
67
- elsif BoolValue.symbol?(opt[:default])
68
- BoolValue::TYPES
69
- end
70
- option(opt[:name], description: opt[:description].capitalize, allowed: values, default: opt[:default])
63
+ # Generator options, defined in schema: values set in @gen_options
64
+ Schema::Registry.instance.reader(Schema::Registry::PREVIEW_OPTIONS).current['properties'].each do |name, property|
65
+ key = name.to_sym
66
+ option(
67
+ key,
68
+ description: property['description'].lines.first.strip.delete_suffix('.'),
69
+ allowed: property.key?('enum') ? property['enum'].map(&:to_sym) : {'integer' => Type::INTEGER, 'number' => Type::FLOAT, 'boolean' => Type::BOOLEAN}[property['type']],
70
+ default: property['default'],
71
+ # `Hash` value: validated with schema
72
+ schema: ("#{Schema::Registry::PREVIEW_OPTIONS}:properties.#{name}" if property['type'].eql?('object')),
73
+ on_set: ->(value) { @gen_options[key] = value }
74
+ )
71
75
  end
72
76
 
73
77
  def initialize(**_)
78
+ # Generator options, see `options.schema.yaml`
79
+ # Created before `super`, which declares options: values are set through `on_set`.
80
+ @gen_options = Aspera::Preview::Generator::Options.new
74
81
  super
75
- # Generator configuration populated from CLI options.
76
- @gen_options = Aspera::Preview::Options.new
77
82
  # Used to rate-limit periodic progress logging and checkpoint persistence.
78
83
  @periodic = TimerLimiter.new(LOG_LIMITER_SEC)
79
84
  # Optional callback used to filter entries before generation.
80
85
  @filter_block = nil
81
86
  @access_remote = true
82
- # Bind generator-specific options to @gen_options
83
- Aspera::Preview::Options::DESCRIPTIONS.each do |opt|
84
- options.on_set(opt[:name], @gen_options.method(:"#{opt[:name]}="))
85
- end
86
87
  # Values set through `on_set` callbacks are used below
87
88
  options.parse_options!
88
89
  @option_skip_types = options.get_option(:skip_types)
@@ -98,9 +99,8 @@ module Aspera
98
99
  @preview_formats_to_generate = Aspera::Preview::Generator::PREVIEW_FORMATS.clone
99
100
  skip = options.get_option(:skip_format)
100
101
  @preview_formats_to_generate.delete(skip) if skip
102
+ # Created by `tools_cleanup`, only for commands that use it
101
103
  @tmp_folder = File.join(TempFileManager.instance.global_temp, "#{TMP_DIR_PREFIX}.#{SecureRandom.uuid}")
102
- FileUtils.mkdir_p(@tmp_folder)
103
- Log.log.debug { "tmpdir: #{@tmp_folder}" }
104
104
  end
105
105
 
106
106
  # Process legacy transfer events and trigger preview generation for completed downloads.
@@ -438,7 +438,7 @@ module Aspera
438
438
  Log.log.debug { "root: #{node_info['docroot']}" }
439
439
  # Default storage url to local file if not provided
440
440
  option_root_url = options.get_option(:root_url, mandatory: true)
441
- option_root_url = UriReader.file_url(@access_key_self['storage']['path']) if option_root_url.eql?(REMOTE_ACCESS) && @access_key_self['storage']['type'].eql?('local')
441
+ option_root_url = UriReader.file_url(@access_key_self['storage']['path']) if option_root_url.eql?(REMOTE_ACCESS) && @access_key_self['storage']&.fetch('type').eql?('local')
442
442
  @access_remote = !UriReader.file?(option_root_url)
443
443
  Log.log.debug { "remote: #{@access_remote}" }
444
444
  # TODO: can the `previews` folder parameter be read from Node API ?
@@ -506,10 +506,12 @@ module Aspera
506
506
  private
507
507
 
508
508
  # Check tools and set option flag.
509
- # Clean up the temporary folder after each command.
509
+ # Create the temporary folder for the command, and clean it up after.
510
510
  def tools_cleanup
511
511
  Aspera::Preview::FileTypes.instance.detect_mime = options.get_option(:detect_mime, mandatory: true)
512
512
  Aspera::Preview::Utils.check_tools(@option_skip_types)
513
+ FileUtils.mkdir_p(@tmp_folder)
514
+ Log.log.debug { "tmpdir: #{@tmp_folder}" }
513
515
  yield
514
516
  ensure
515
517
  Log.log.debug { "cleaning up temp folder #{@tmp_folder}" }