aspera-cli 4.27.3 → 4.27.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/CHANGELOG.md +83 -0
- data/CONTRIBUTING.md +5 -2
- data/bin/ascli +1 -0
- data/docs/README.md +224 -42
- data/lib/aspera/agent/base.rb +7 -2
- data/lib/aspera/agent/connect.rb +0 -4
- data/lib/aspera/agent/desktop.rb +0 -4
- data/lib/aspera/agent/direct.rb +49 -21
- data/lib/aspera/agent/node.rb +6 -5
- data/lib/aspera/agent/transferd.rb +2 -2
- data/lib/aspera/api/faspex.rb +2 -0
- data/lib/aspera/api/httpgw.rb +1 -2
- data/lib/aspera/api/node.rb +1 -1
- data/lib/aspera/ascp/installation.rb +1 -1
- data/lib/aspera/cli/async_transfer_store.rb +10 -9
- data/lib/aspera/cli/bootstrapper.rb +3 -1
- data/lib/aspera/cli/command_registry.rb +69 -7
- data/lib/aspera/cli/command_spec.rb +1 -1
- data/lib/aspera/cli/extended_value.rb +4 -3
- data/lib/aspera/cli/formatter.rb +10 -8
- data/lib/aspera/cli/http.rb +8 -20
- data/lib/aspera/cli/option_types.rb +3 -1
- data/lib/aspera/cli/option_value.rb +16 -19
- data/lib/aspera/cli/options.schema.yaml +86 -10
- data/lib/aspera/cli/parser.rb +30 -18
- data/lib/aspera/cli/plugins/aoc.rb +133 -156
- data/lib/aspera/cli/plugins/ats.rb +1 -7
- data/lib/aspera/cli/plugins/base.rb +40 -34
- data/lib/aspera/cli/plugins/config.rb +24 -11
- data/lib/aspera/cli/plugins/console.rb +1 -1
- data/lib/aspera/cli/plugins/faspex5.rb +30 -11
- data/lib/aspera/cli/plugins/faspio.rb +1 -1
- data/lib/aspera/cli/plugins/node.rb +26 -19
- data/lib/aspera/cli/plugins/orchestrator.rb +73 -44
- data/lib/aspera/cli/plugins/preview.rb +21 -19
- data/lib/aspera/cli/plugins/server.rb +3 -4
- data/lib/aspera/cli/plugins/shares.rb +21 -24
- data/lib/aspera/cli/preset_actions.rb +28 -18
- data/lib/aspera/cli/preset_manager.rb +34 -19
- data/lib/aspera/cli/prompt.rb +2 -1
- data/lib/aspera/cli/result.rb +33 -22
- data/lib/aspera/cli/runner.rb +1 -5
- data/lib/aspera/cli/sync_actions.rb +17 -16
- data/lib/aspera/cli/transfer_actions.rb +14 -3
- data/lib/aspera/cli/transfer_agent.rb +6 -4
- data/lib/aspera/cli/transfer_progress.rb +290 -55
- data/lib/aspera/cli/version.rb +1 -1
- data/lib/aspera/cli/wizard.rb +1 -1
- data/lib/aspera/coverage.rb +0 -1
- data/lib/aspera/environment.rb +29 -5
- data/lib/aspera/keychain/encrypted_hash.rb +1 -1
- data/lib/aspera/keychain/factory.rb +2 -1
- data/lib/aspera/log.rb +25 -2
- data/lib/aspera/node_emulator.rb +759 -0
- data/lib/aspera/oauth/base.rb +2 -1
- data/lib/aspera/oauth/factory.rb +6 -3
- data/lib/aspera/oauth/json_credentials.rb +34 -0
- data/lib/aspera/oauth/jwt.rb +3 -4
- data/lib/aspera/oauth.rb +1 -0
- data/lib/aspera/persistency_folder.rb +1 -3
- data/lib/aspera/preview/generator.rb +4 -1
- data/lib/aspera/preview/options.schema.yaml +119 -0
- data/lib/aspera/rest/aspera_errors.rb +12 -0
- data/lib/aspera/rest/client.rb +28 -19
- data/lib/aspera/rest/list.rb +14 -8
- data/lib/aspera/rest/parameters.rb +2 -2
- data/lib/aspera/schema/IBM Aspera Faspex API-5.0-enhanced.yaml +0 -20
- data/lib/aspera/schema/IBM Aspera Orchestrator API-v1.yaml +1784 -0
- data/lib/aspera/schema/IBM Aspera on Cloud API-0.2.6-enhanced.yaml +1230 -137
- data/lib/aspera/schema/IBM_Aspera_Shares.yaml +14 -13
- data/lib/aspera/schema/reader.rb +12 -18
- data/lib/aspera/schema/registry.rb +6 -1
- data.tar.gz.sig +0 -0
- metadata +5 -3
- metadata.gz.sig +0 -0
- data/lib/aspera/node_simulator.rb +0 -345
- data/lib/aspera/preview/options.rb +0 -45
|
@@ -8,6 +8,7 @@ require 'aspera/cli/extended_value'
|
|
|
8
8
|
require 'aspera/cli/special_values'
|
|
9
9
|
require 'aspera/cli/version'
|
|
10
10
|
require 'aspera/cli/formatter'
|
|
11
|
+
require 'aspera/cli/http'
|
|
11
12
|
require 'aspera/cli/info'
|
|
12
13
|
require 'aspera/cli/wizard'
|
|
13
14
|
require 'aspera/cli/sync_actions'
|
|
@@ -99,16 +100,15 @@ module Aspera
|
|
|
99
100
|
option :cache_tokens, description: 'Save and reuse OAuth tokens', allowed: Type::BOOLEAN, default: true
|
|
100
101
|
option :expand_mounts, description: 'Commands: list commands of sub-trees provided by another plugin', allowed: Type::BOOLEAN, default: false
|
|
101
102
|
option :no_default, description: 'Do not load default configuration for plugin', allowed: Type::NONE, short: 'N', on_set: -> { presets.use_plugin_defaults = false }
|
|
103
|
+
use_options Wizard
|
|
104
|
+
# `on_set` callbacks of HTTP options are methods of `http_config`
|
|
105
|
+
use_options Http, target: -> { http_config }
|
|
102
106
|
|
|
103
107
|
def initialize(**_)
|
|
104
108
|
super
|
|
105
109
|
@vault_instance = nil
|
|
106
110
|
@sdk_default_location = false
|
|
107
|
-
# Declare wizard options (Wizard#initialize calls options.declare internally)
|
|
108
111
|
@wizard = Wizard.new(self, context.main_folder)
|
|
109
|
-
# HTTP options: declare metadata (class method), then bind to the instance
|
|
110
|
-
Http.declare_options(options)
|
|
111
|
-
context.http_config.bind_options(options)
|
|
112
112
|
# Values set through `on_set` callbacks are used below (sdk_folder)
|
|
113
113
|
options.parse_options!
|
|
114
114
|
set_sdk_dir
|
|
@@ -186,7 +186,8 @@ module Aspera
|
|
|
186
186
|
{name: :preset_name, mandatory: false, default: ''}],
|
|
187
187
|
action: lambda do |url:, plugin_name: nil, preset_name: '', **|
|
|
188
188
|
apps = @wizard.identify_plugins_for_url(url: url, plugin_name: plugin_name).freeze
|
|
189
|
-
|
|
189
|
+
# The product wizard prompts for missing mandatory options (username, password, ...)
|
|
190
|
+
options.with_interactive { @wizard.find(apps, preset_name: preset_name) }
|
|
190
191
|
end
|
|
191
192
|
)
|
|
192
193
|
command :coffee, description: 'Show a coffee image', action: ->(**) { Result::Image.new(COFFEE_IMAGE_URL) }
|
|
@@ -396,8 +397,7 @@ module Aspera
|
|
|
396
397
|
Result::ObjectList.new(builder.rows, fields: builder.columns)
|
|
397
398
|
end
|
|
398
399
|
)
|
|
399
|
-
|
|
400
|
-
SyncActions.register_sync_admin_commands(self, :admin)
|
|
400
|
+
sync_admin_commands :admin
|
|
401
401
|
command :translate, description: 'Translate async-style arguments to sync config format',
|
|
402
402
|
arguments: [{name: :async_arguments, multiple: true}],
|
|
403
403
|
action: ->(async_arguments:, **) { Result::SingleObject.new(Sync::Operations.args_to_conf(async_arguments)) }
|
|
@@ -493,7 +493,7 @@ module Aspera
|
|
|
493
493
|
|
|
494
494
|
def action_download(file_url:, file_dest: nil, **)
|
|
495
495
|
file_url = file_url.chomp
|
|
496
|
-
file_dest = File.join(transfer.destination_folder(Transfer::Spec::DIRECTION_RECEIVE), file_url.
|
|
496
|
+
file_dest = File.join(transfer.destination_folder(Transfer::Spec::DIRECTION_RECEIVE), file_url.rpartition('/').last) if file_dest.nil?
|
|
497
497
|
Log.log.info("Downloading: #{file_url}")
|
|
498
498
|
Rest::Client.new(base_url: file_url).call(operation: 'GET', save_to: file_dest)
|
|
499
499
|
Result::Status.new("Saved to: #{file_dest}")
|
|
@@ -569,12 +569,12 @@ module Aspera
|
|
|
569
569
|
Aspera.assert_values(plugin_name.to_sym, plugin_names, type: Cli::BadArgument)
|
|
570
570
|
plugin_names = [plugin_name.to_sym]
|
|
571
571
|
end
|
|
572
|
-
|
|
573
|
-
Aspera.assert(
|
|
572
|
+
words = Array(command_path).map(&:to_sym)
|
|
573
|
+
Aspera.assert(words.empty? || plugin_names.length.eql?(1), type: Cli::BadArgument) { 'command path requires a plugin name' }
|
|
574
574
|
expand_mounts = options.get_option(:expand_mounts)
|
|
575
575
|
commands = plugin_names.flat_map do |name|
|
|
576
576
|
reg = Plugins::Factory.instance.plugin_class(name).command_registry
|
|
577
|
-
|
|
577
|
+
prefix = listed_command_path(name, reg, words)
|
|
578
578
|
paths = prefix.empty? || reg.children_of(prefix).any? ? reg.leaf_paths(prefix, expand_mounts: expand_mounts) : [prefix]
|
|
579
579
|
paths.map do |path|
|
|
580
580
|
mount = expand_mounts ? nil : reg.mount_at(path)
|
|
@@ -592,6 +592,19 @@ module Aspera
|
|
|
592
592
|
Result::ObjectList.new(commands, fields: %w[syntax description])
|
|
593
593
|
end
|
|
594
594
|
|
|
595
|
+
# Command path to list, from words that may include positional arguments, as on a command line.
|
|
596
|
+
# e.g. `faspex5 packages receive ALL` lists `faspex5 packages receive`.
|
|
597
|
+
# @param name [Symbol] plugin name
|
|
598
|
+
# @param reg [CommandRegistry] registry of plugin
|
|
599
|
+
# @param words [Array<Symbol>] sub-commands (or aliases), possibly mixed with their arguments
|
|
600
|
+
# @return [Array<Symbol>] command path
|
|
601
|
+
# @raise [Cli::BadArgument] if a word is neither a sub-command nor an expected argument
|
|
602
|
+
def listed_command_path(name, reg, words)
|
|
603
|
+
path, unknown = reg.command_path(words)
|
|
604
|
+
raise Cli::BadArgument, Parser.multi_choice_assert_msg("no such command: #{[name, *path, unknown].join(' ')}", reg.children_of(path).keys) unless unknown.nil?
|
|
605
|
+
path
|
|
606
|
+
end
|
|
607
|
+
|
|
595
608
|
# Build syntax by interleaving each path segment with the arguments declared on that node
|
|
596
609
|
# @param name [String, Symbol] plugin name
|
|
597
610
|
# @param reg [CommandRegistry] registry of plugin
|
|
@@ -16,6 +16,8 @@ require 'aspera/nagios'
|
|
|
16
16
|
require 'aspera/environment'
|
|
17
17
|
require 'aspera/assert'
|
|
18
18
|
require 'securerandom'
|
|
19
|
+
require 'digest'
|
|
20
|
+
require 'json'
|
|
19
21
|
require 'aspera/rainbow'
|
|
20
22
|
using Rainbow
|
|
21
23
|
|
|
@@ -143,7 +145,7 @@ module Aspera
|
|
|
143
145
|
end
|
|
144
146
|
if status_list.include?(status['upload_status'])
|
|
145
147
|
progress_bar&.event(:session_end, session_id: id)
|
|
146
|
-
progress_bar&.event(:end)
|
|
148
|
+
progress_bar&.event(:end, info: !status['upload_status'].eql?('failed'))
|
|
147
149
|
return status
|
|
148
150
|
end
|
|
149
151
|
sleep(1.0)
|
|
@@ -195,13 +197,25 @@ module Aspera
|
|
|
195
197
|
# Build query to get package recipients based on package info in case of shared inbox or workgroup recipient
|
|
196
198
|
# @param package_id [String] the package id to get info from
|
|
197
199
|
def recipient_query(package_id)
|
|
200
|
+
# With a public link, the recipient context is already encoded in the auth token: no extra query params needed
|
|
201
|
+
return {} if api_v5.pub_link_context
|
|
198
202
|
package_info = api_v5.read("packages/#{package_id}")
|
|
199
203
|
base_query = {}
|
|
200
|
-
|
|
201
|
-
|
|
204
|
+
first_recipient = package_info['recipients']&.first
|
|
205
|
+
if first_recipient
|
|
206
|
+
base_query['recipient_workgroup_id'] = first_recipient['id'] if WORKGROUP_TYPES.include?(first_recipient['recipient_type'])
|
|
207
|
+
base_query['recipient_user_id'] = first_recipient['id'] if first_recipient['recipient_type'].eql?('user')
|
|
208
|
+
end
|
|
202
209
|
base_query
|
|
203
210
|
end
|
|
204
211
|
|
|
212
|
+
# @return [String] user part of the persistency id of received packages:
|
|
213
|
+
# the username, or for a public link (no username), a digest of the link context (unique per link)
|
|
214
|
+
def persistency_user_id
|
|
215
|
+
return options.get_option(:username, mandatory: true) if api_v5.pub_link_context.nil?
|
|
216
|
+
Digest::SHA256.hexdigest(JSON.generate(api_v5.pub_link_context))[0, 16]
|
|
217
|
+
end
|
|
218
|
+
|
|
205
219
|
def package_receive(package_ids)
|
|
206
220
|
# prepare persistency if needed
|
|
207
221
|
skip_ids_persistency = nil
|
|
@@ -213,7 +227,7 @@ module Aspera
|
|
|
213
227
|
id: IdGenerator.from_list(
|
|
214
228
|
'faspex_recv',
|
|
215
229
|
options.get_option(:url, mandatory: true),
|
|
216
|
-
|
|
230
|
+
persistency_user_id,
|
|
217
231
|
options.get_option(:box, mandatory: true)
|
|
218
232
|
)
|
|
219
233
|
)
|
|
@@ -229,7 +243,7 @@ module Aspera
|
|
|
229
243
|
# TODO: if packages have same name, they will overwrite ?
|
|
230
244
|
packages, max_items = list_packages_with_filter(query: {'status' => 'completed'}) # no filter: all completed packages
|
|
231
245
|
Log.dump(:package_ids, level: :trace1) { packages.map { |p| p['id'] } }
|
|
232
|
-
Log.dump(:skip_ids, skip_ids_persistency
|
|
246
|
+
Log.dump(:skip_ids, skip_ids_persistency&.data, level: :trace1)
|
|
233
247
|
packages.reject! { |p| skip_ids_persistency.data.include?(p['id']) } if skip_ids_persistency
|
|
234
248
|
# Apply `max` after once_only filtering so we get the N first not-yet-downloaded packages
|
|
235
249
|
packages = packages[0, max_items] if max_items
|
|
@@ -240,11 +254,13 @@ module Aspera
|
|
|
240
254
|
Aspera.assert_array_all(package_ids, String) { 'Package id(s)' }
|
|
241
255
|
# packages = package_ids.map{|pkg_id|api_v5.read("packages/#{pkg_id}")}
|
|
242
256
|
packages = package_ids.map { |pkg_id| {'id'=>pkg_id} }
|
|
257
|
+
# The package of a public link is received only once, like with `ALL`
|
|
258
|
+
packages.reject! { |p| skip_ids_persistency.data.include?(p['id']) } if skip_ids_persistency && api_v5.pub_link_context
|
|
243
259
|
end
|
|
244
260
|
result_transfer = []
|
|
245
261
|
param_file_list = {}
|
|
246
262
|
begin
|
|
247
|
-
param_file_list['paths'] = transfer.ts_source_paths
|
|
263
|
+
param_file_list['paths'] = transfer.ts_source_paths.map { |p| {'path' => p['source']} }
|
|
248
264
|
rescue Cli::MissingArgument
|
|
249
265
|
# paths is optional
|
|
250
266
|
end
|
|
@@ -305,7 +321,8 @@ module Aspera
|
|
|
305
321
|
)
|
|
306
322
|
# well, we asked a TS for connect, but we actually want a generic one
|
|
307
323
|
transfer_spec.delete('authentication')
|
|
308
|
-
|
|
324
|
+
Runner.result_transfer(transfer.start(transfer_spec))
|
|
325
|
+
return Result::SingleObject.new(package)
|
|
309
326
|
else
|
|
310
327
|
# send from remote shared folder
|
|
311
328
|
if (m = Parser.percent_selector(shared_folder))
|
|
@@ -493,7 +510,7 @@ module Aspera
|
|
|
493
510
|
arguments: [{name: :package, type: Hash, schema: Schema::Registry.req_body(Schema::Registry::FASPEX, 'packages.post')}],
|
|
494
511
|
action: ->(package:, **) { package_send(package) }
|
|
495
512
|
command :show, description: 'Show a package', setup: :setup_package_id, arguments: PACKAGE_ID_ARG,
|
|
496
|
-
action: ->(package_id:, **) { Result::SingleObject.new(api_v5.read("packages/#{package_id}")) }
|
|
513
|
+
action: ->(package_id:, **) { Result::SingleObject.new(api_v5.read("packages/#{package_id}", query_read_delete)) }
|
|
497
514
|
command :browse, description: 'Browse package files', setup: :setup_package_id,
|
|
498
515
|
arguments: PACKAGE_ID_ARG + [{name: :folder_path, mandatory: false, default: '/'}],
|
|
499
516
|
action: ->(folder_path:, package_id:, **) { browse_folder("packages/#{package_id}/files/#{Api::Faspex.box_type(options.get_option(:box))}", recipient_query(package_id), folder_path: folder_path) }
|
|
@@ -833,9 +850,11 @@ module Aspera
|
|
|
833
850
|
res_path = "#{res_instance_path}/members"
|
|
834
851
|
api_v5.lookup_entity_by_field(entity: res_path, field: field, value: value, query: Rest.php_style({type: %w[user]}))['user_id']
|
|
835
852
|
end
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
|
|
853
|
+
# Like members (`user_id`), a SAML group of the resource is identified by `group_id`; its name is `group_name`
|
|
854
|
+
define_method(:"lookup_#{res}_saml_groups_id") do |field, value, saml_groups_path:, **|
|
|
855
|
+
field = 'group_name' if field.eql?('name')
|
|
856
|
+
query = field.eql?('group_name') ? {'q' => value} : {}
|
|
857
|
+
api_v5.lookup_entity_by_field(entity: saml_groups_path, field: field, value: value, items_key: 'groups', query: query)['group_id']
|
|
839
858
|
end
|
|
840
859
|
end
|
|
841
860
|
|
|
@@ -11,7 +11,6 @@ require 'aspera/hash_ext'
|
|
|
11
11
|
require 'aspera/id_generator'
|
|
12
12
|
require 'aspera/api/node'
|
|
13
13
|
require 'aspera/oauth'
|
|
14
|
-
require 'aspera/node_simulator'
|
|
15
14
|
require 'aspera/rest/list'
|
|
16
15
|
require 'aspera/assert'
|
|
17
16
|
require 'base64'
|
|
@@ -356,8 +355,7 @@ module Aspera
|
|
|
356
355
|
Sync::Operations::DIRECTIONS.each do |dir|
|
|
357
356
|
command dir, description: "#{dir.capitalize}-sync (Gen3)", transfer_paths: :send, arguments: SyncActions::PATH_AND_INFO_ARGS
|
|
358
357
|
end
|
|
359
|
-
|
|
360
|
-
SyncActions.register_sync_admin_commands(self, :admin)
|
|
358
|
+
sync_admin_commands :admin
|
|
361
359
|
end
|
|
362
360
|
# access_keys sub-tree
|
|
363
361
|
command :access_keys, description: 'Manage access keys'
|
|
@@ -387,8 +385,7 @@ module Aspera
|
|
|
387
385
|
Sync::Operations::DIRECTIONS.each do |dir|
|
|
388
386
|
command dir, description: "#{dir.capitalize}-sync", transfer_paths: :send, arguments: SyncActions::PATH_AND_INFO_ARGS
|
|
389
387
|
end
|
|
390
|
-
|
|
391
|
-
SyncActions.register_sync_admin_commands(self, :admin)
|
|
388
|
+
sync_admin_commands :admin
|
|
392
389
|
end
|
|
393
390
|
end
|
|
394
391
|
commands_under %i[access_keys do permission] do
|
|
@@ -591,8 +588,9 @@ module Aspera
|
|
|
591
588
|
Result::Text.new(Api::Node.bearer_token(payload: token, access_key: access_key, private_key: private_key))
|
|
592
589
|
end
|
|
593
590
|
)
|
|
594
|
-
command :
|
|
595
|
-
|
|
591
|
+
command :emulator, description: 'Start node emulator',
|
|
592
|
+
aliases: [:simulator],
|
|
593
|
+
arguments: [{name: :parameters, type: Hash, mandatory: false, default: {}, schema: Schema::Registry::NODE_EMULATOR_OPTIONS}]
|
|
596
594
|
command :telemetry, description: 'Report telemetry to external system',
|
|
597
595
|
arguments: [{name: :parameters, type: Hash, mandatory: false, default: {}, schema: 'opts:components.schemas.NodeTelemetryOptions'}]
|
|
598
596
|
|
|
@@ -655,7 +653,7 @@ module Aspera
|
|
|
655
653
|
transfer_spec = api_node.create(
|
|
656
654
|
'files/sync_setup',
|
|
657
655
|
{transfer_requests: [{transfer_request: request_transfer_spec}]}
|
|
658
|
-
)['transfer_specs'].first
|
|
656
|
+
)['transfer_specs'].first&.fetch('transfer_spec')
|
|
659
657
|
transfer_spec.delete_if { |_k, v| v.nil? }
|
|
660
658
|
Log.dump(:ts, transfer_spec)
|
|
661
659
|
transfer_spec
|
|
@@ -663,7 +661,7 @@ module Aspera
|
|
|
663
661
|
end
|
|
664
662
|
|
|
665
663
|
Sync::Operations::DIRECTIONS.each do |dir|
|
|
666
|
-
|
|
664
|
+
define_action_method([:sync, dir]) { |path:, sync_info: {}, **| run_sync_transfer(dir, path: path, sync_info: sync_info, &sync_gen3_block) }
|
|
667
665
|
end
|
|
668
666
|
|
|
669
667
|
def action_upload(**)
|
|
@@ -674,7 +672,7 @@ module Aspera
|
|
|
674
672
|
api_node.add_tspec_info(request_transfer_spec) if api_node.respond_to?(:add_tspec_info)
|
|
675
673
|
Api::Node.add_public_key(request_transfer_spec)
|
|
676
674
|
setup_payload = {transfer_requests: [{transfer_request: request_transfer_spec}]}
|
|
677
|
-
transfer_spec = api_node.create('files/upload_setup', setup_payload)['transfer_specs'].first
|
|
675
|
+
transfer_spec = api_node.create('files/upload_setup', setup_payload)['transfer_specs'].first&.fetch('transfer_spec')
|
|
678
676
|
Api::Node.add_private_key(transfer_spec)
|
|
679
677
|
transfer_spec.delete('paths')
|
|
680
678
|
Runner.result_transfer(transfer.start(transfer_spec))
|
|
@@ -688,7 +686,7 @@ module Aspera
|
|
|
688
686
|
api_node.add_tspec_info(request_transfer_spec) if api_node.respond_to?(:add_tspec_info)
|
|
689
687
|
Api::Node.add_public_key(request_transfer_spec)
|
|
690
688
|
setup_payload = {transfer_requests: [{transfer_request: request_transfer_spec}]}
|
|
691
|
-
transfer_spec = api_node.create('files/download_setup', setup_payload)['transfer_specs'].first
|
|
689
|
+
transfer_spec = api_node.create('files/download_setup', setup_payload)['transfer_specs'].first&.fetch('transfer_spec')
|
|
692
690
|
Api::Node.add_private_key(transfer_spec)
|
|
693
691
|
Runner.result_transfer(transfer.start(transfer_spec))
|
|
694
692
|
end
|
|
@@ -824,7 +822,7 @@ module Aspera
|
|
|
824
822
|
end
|
|
825
823
|
|
|
826
824
|
Sync::Operations::DIRECTIONS.each do |dir|
|
|
827
|
-
|
|
825
|
+
define_action_method([:access_keys, :do, :sync, dir]) do |path:, sync_info: {}, do_root_file_id:, **|
|
|
828
826
|
run_sync_transfer(dir, path: path, sync_info: sync_info, &sync_gen4_block(do_root_file_id))
|
|
829
827
|
end
|
|
830
828
|
end
|
|
@@ -1015,7 +1013,7 @@ module Aspera
|
|
|
1015
1013
|
|
|
1016
1014
|
def action_async_counters(async_id:, **)
|
|
1017
1015
|
Integer(async_id)
|
|
1018
|
-
resp = api_node.create('async/counters', {'syncs' => [async_id]})['sync_counters'].first[async_id
|
|
1016
|
+
resp = api_node.create('async/counters', {'syncs' => [async_id]})['sync_counters'].first&.[](async_id)&.last
|
|
1019
1017
|
return Result::Empty.new if resp.nil?
|
|
1020
1018
|
Result::SingleObject.new(resp)
|
|
1021
1019
|
end
|
|
@@ -1131,14 +1129,23 @@ module Aspera
|
|
|
1131
1129
|
return Result::Status.new('done')
|
|
1132
1130
|
end
|
|
1133
1131
|
|
|
1134
|
-
def
|
|
1135
|
-
require 'aspera/
|
|
1136
|
-
parameters
|
|
1137
|
-
|
|
1132
|
+
def action_emulator(parameters: {}, **)
|
|
1133
|
+
require 'aspera/node_emulator'
|
|
1134
|
+
# missing parameters take the default of the schema
|
|
1135
|
+
defaults = Schema::Registry.instance.reader(Schema::Registry::NODE_EMULATOR_OPTIONS).current['properties'].each_with_object({}) do |(name, property), result|
|
|
1136
|
+
result[name.to_sym] = property['default'] if property.key?('default')
|
|
1137
|
+
end
|
|
1138
|
+
parameters = defaults.merge(parameters.symbolize_keys)
|
|
1139
|
+
uri = URI.parse(parameters.delete(:url))
|
|
1140
|
+
config = parameters.except(*WebServerSimple::PARAMS)
|
|
1141
|
+
docroot = config.delete(:docroot)
|
|
1142
|
+
retention_sec = config.delete(:retention_sec)
|
|
1143
|
+
Aspera.assert(config[:username].nil? == config[:password].nil?, type: Cli::BadArgument) { 'Parameters username and password must be set together' }
|
|
1144
|
+
Log.log.warn('No username and password: emulator accepts requests without authentication') if config[:username].nil?
|
|
1138
1145
|
server = WebServerSimple.new(uri, **parameters.slice(*WebServerSimple::PARAMS))
|
|
1139
|
-
server.mount(uri.path,
|
|
1146
|
+
server.mount(uri.path, NodeEmulatorServlet, config, NodeEmulator.new(docroot: docroot, retention_sec: retention_sec))
|
|
1140
1147
|
server.start
|
|
1141
|
-
return Result::Status.new('
|
|
1148
|
+
return Result::Status.new('Emulator terminated')
|
|
1142
1149
|
end
|
|
1143
1150
|
|
|
1144
1151
|
def action_telemetry(parameters: {}, **)
|
|
@@ -52,36 +52,37 @@ module Aspera
|
|
|
52
52
|
username: options.get_option(:username, mandatory: true),
|
|
53
53
|
password: options.get_option(:password, mandatory: true)
|
|
54
54
|
},
|
|
55
|
-
test_args: '
|
|
55
|
+
test_args: 'workflows list'
|
|
56
56
|
}
|
|
57
57
|
end
|
|
58
58
|
|
|
59
|
+
option :apikey, description: 'API key (exclusive with username/password)'
|
|
60
|
+
option :auth_style, description: 'Authentication style', allowed: %i[basic query token], default: :token
|
|
61
|
+
option :ret_style, description: 'Method to specify the expected response format ("Accept")', allowed: %i[header query path], default: :query
|
|
59
62
|
option :result, description: "Specify result value as: 'work_step:parameter'", deprecation: {last: '4.27.2', message: 'use keys `step` and `variable` of argument `execution` of `workflows start`'}
|
|
60
63
|
option :synchronous, description: 'Wait for completion', allowed: Type::BOOLEAN, deprecation: {last: '4.27.2', message: 'use key `synchronous` of argument `execution` of `workflows start`'}
|
|
61
|
-
option :ret_style, description: 'How return type is requested in api', allowed: %i[header arg ext], default: :arg
|
|
62
|
-
option :auth_style, description: 'Authentication type', allowed: %i[arg_pass head_basic apikey], default: :head_basic
|
|
63
64
|
|
|
64
|
-
# Call orchestrator API
|
|
65
|
+
# Call orchestrator API (handles ret_style negotiation and XML parsing)
|
|
65
66
|
# @param endpoint [String] the endpoint to call
|
|
66
|
-
# @param
|
|
67
|
+
# @param body [Hash] the body to pass (JSON); also implies POST
|
|
67
68
|
# @param format [String] the format to request, 'json', 'xml', nil
|
|
68
|
-
# @param
|
|
69
|
+
# @param query [Hash] the arguments to pass as query parameters
|
|
69
70
|
# @param xml_arrays [Boolean] if true, force arrays in xml parsing
|
|
70
71
|
# @param http [Boolean] if true, returns the HttpResponse, else
|
|
71
|
-
def call_ao(endpoint,
|
|
72
|
-
|
|
73
|
-
call_args
|
|
74
|
-
ret_style = options.get_option(:ret_style, mandatory: true)
|
|
75
|
-
call_args[:query].merge!(
|
|
72
|
+
def call_ao(endpoint, body: nil, format: 'json', query: nil, xml_arrays: true, http: false)
|
|
73
|
+
call_args = {operation: body.nil? ? 'GET' : 'POST', subpath: "api/#{endpoint}", ret: :both, query: {}}
|
|
74
|
+
call_args.merge!(body: body, content_type: Mime::JSON) unless body.nil? # rubocop:disable Performance/RedundantMerge
|
|
75
|
+
ret_style = options.get_option(:ret_style, mandatory: true)
|
|
76
|
+
call_args[:query].merge!(query) unless query.nil?
|
|
76
77
|
unless format.nil?
|
|
77
78
|
case ret_style
|
|
78
79
|
when :header
|
|
79
80
|
call_args[:headers] = {'Accept' => "application/#{format}"}
|
|
80
|
-
when :
|
|
81
|
+
when :query
|
|
81
82
|
call_args[:query][:format] = format
|
|
82
|
-
when :
|
|
83
|
+
when :path
|
|
83
84
|
call_args[:subpath] = "#{call_args[:subpath]}.#{format}"
|
|
84
|
-
else Aspera.error_unexpected_value(ret_style)
|
|
85
|
+
else Aspera.error_unexpected_value(ret_style) { 'ret_style' }
|
|
85
86
|
end
|
|
86
87
|
end
|
|
87
88
|
add_query = query_read_delete
|
|
@@ -99,13 +100,18 @@ module Aspera
|
|
|
99
100
|
|
|
100
101
|
command :health, description: 'Check Orchestrator API health'
|
|
101
102
|
command :info, description: 'Check that Orchestrator responds (ping)', action: ->(**) { Result::SingleObject.new(call_ao('remote_node_ping', format: 'xml', xml_arrays: false)) }
|
|
102
|
-
command :processes, description: 'Show Orchestrator background process status', action: ->(**) { Result::ObjectList.new(call_ao('processes_status', format: 'xml')['process']) }
|
|
103
|
+
command :processes, description: 'Show Orchestrator background process status', action: ->(**) { Result::ObjectList.new(call_ao('processes_status', format: 'xml')['node'].flat_map { |n| n['process'] }) }
|
|
103
104
|
command :monitors, description: 'Show Orchestrator monitor snapshot', action: ->(**) { Result::SingleObject.new(call_ao('monitor_snapshot')['monitor']) }
|
|
104
|
-
command :plugins, description: 'Show Orchestrator plugin versions', action: ->(**) { Result::ObjectList.new(call_ao('plugin_version')['Plugin']) }
|
|
105
|
-
command :workflows, description: 'Manage workflows'
|
|
106
105
|
command :workorders, description: 'Manage work orders'
|
|
107
106
|
command :workstep, description: 'Manage work steps'
|
|
108
107
|
|
|
108
|
+
commands_under :plugins do
|
|
109
|
+
command :list, description: 'Show Orchestrator plugin versions', action: ->(**) { Result::ObjectList.new(call_ao('plugin_version')['Plugin']) }
|
|
110
|
+
command :reload_set, description: 'Reload a set of plugins',
|
|
111
|
+
arguments: [{name: :plugin_set, type: Hash, schema: 'opts:components.schemas.OrchestratorReloadPluginSet'}],
|
|
112
|
+
action: ->(plugin_set:, **) { Result::SingleObject.new(call_ao('reload_plugin_set', body: plugin_set.transform_keys(&:to_sym))) }
|
|
113
|
+
end
|
|
114
|
+
|
|
109
115
|
commands_under :workflows do
|
|
110
116
|
command :list, description: 'List all workflows'
|
|
111
117
|
command :status, description: 'Show running status of a workflow',
|
|
@@ -120,9 +126,18 @@ module Aspera
|
|
|
120
126
|
command :start, description: 'Start a workflow: create a work order (sync or async)',
|
|
121
127
|
arguments: [
|
|
122
128
|
{name: :workflow_id, type: :identifier},
|
|
123
|
-
{name: :parameters, type: Hash, mandatory: false, default: {}},
|
|
129
|
+
{name: :parameters, type: Hash, mandatory: false, default: {}, schema: 'opts:components.schemas.OrchestratorInitiateParameters'},
|
|
124
130
|
{name: :execution, type: Hash, mandatory: false, default: {}, schema: 'opts:components.schemas.OrchestratorWorkflowStart'}
|
|
125
131
|
]
|
|
132
|
+
command :import, description: 'Import a workflow',
|
|
133
|
+
arguments: [{name: :workflow, type: Hash, schema: 'opts:components.schemas.OrchestratorImportWorkflow'}],
|
|
134
|
+
action: ->(workflow:, **) { Result::SingleObject.new(call_ao('import_workflow', body: workflow.transform_keys(&:to_sym))) }
|
|
135
|
+
command :publish, description: 'Publish a workflow',
|
|
136
|
+
arguments: [{name: :workflow_id, type: :identifier}],
|
|
137
|
+
action: ->(workflow_id:, **) { Result::SingleObject.new(call_ao('publish_workflow', body: {workflow_id: workflow_id.to_i})) }
|
|
138
|
+
command :import_with_constraints, description: 'Import a workflow with constraint resolution',
|
|
139
|
+
arguments: [{name: :payload, type: Hash, schema: 'opts:components.schemas.OrchestratorImportWithConstraints'}],
|
|
140
|
+
action: ->(payload:, **) { Result::SingleObject.new(call_ao('import_with_constraints', body: payload.transform_keys(&:to_sym))) }
|
|
126
141
|
command :export, description: 'Export a workflow',
|
|
127
142
|
arguments: [{name: :workflow_id, type: :identifier}],
|
|
128
143
|
action: ->(workflow_id:, **) { Result::Text.new(call_ao("export_workflow/#{workflow_id}", format: nil, http: true).body) }
|
|
@@ -164,28 +179,43 @@ module Aspera
|
|
|
164
179
|
# @return [Rest::Client]
|
|
165
180
|
def api_orch
|
|
166
181
|
return @api_orch if @api_orch
|
|
182
|
+
base_url = options.get_option(:url, mandatory: true)
|
|
183
|
+
style = options.get_option(:auth_style, mandatory: true)
|
|
184
|
+
apikey = options.get_option(:apikey)
|
|
185
|
+
username = options.get_option(:username)
|
|
186
|
+
password = options.get_option(:password)
|
|
187
|
+
Aspera.assert(apikey.nil? || (username.nil? && password.nil?), type: Cli::BadArgument) { 'apikey and username/password are mutually exclusive' }
|
|
188
|
+
Aspera.assert(!apikey.nil? || (!username.nil? && !password.nil?), type: Cli::BadArgument) { 'provide either apikey or username and password' }
|
|
189
|
+
|
|
167
190
|
auth_params =
|
|
168
|
-
case
|
|
169
|
-
when :
|
|
191
|
+
case style
|
|
192
|
+
when :basic
|
|
193
|
+
Aspera.assert(apikey.nil?, type: Cli::BadArgument) { 'basic auth style cannot be used with apikey, use token or query' }
|
|
170
194
|
{
|
|
171
|
-
type:
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
'password' => options.get_option(:password, mandatory: true)
|
|
175
|
-
}
|
|
195
|
+
type: :basic,
|
|
196
|
+
username: username,
|
|
197
|
+
password: password
|
|
176
198
|
}
|
|
177
|
-
when :
|
|
199
|
+
when :query
|
|
200
|
+
if apikey
|
|
201
|
+
{type: :url, url_query: {'apikey' => apikey}}
|
|
202
|
+
else
|
|
203
|
+
{type: :url, url_query: {'login' => username, 'password' => password}}
|
|
204
|
+
end
|
|
205
|
+
when :token
|
|
178
206
|
{
|
|
179
|
-
type:
|
|
180
|
-
|
|
181
|
-
|
|
207
|
+
type: :oauth2,
|
|
208
|
+
grant_method: :json_credentials,
|
|
209
|
+
base_url: base_url,
|
|
210
|
+
path_token: 'api/login',
|
|
211
|
+
token_field: 'token',
|
|
212
|
+
json: apikey ? {apikey: apikey} : {username: username, password: password}
|
|
182
213
|
}
|
|
183
|
-
|
|
184
|
-
Aspera.error_not_implemented
|
|
214
|
+
else Aspera.error_unexpected_value(style)
|
|
185
215
|
end
|
|
186
216
|
@api_orch = Rest::Client.new(
|
|
187
|
-
base_url:
|
|
188
|
-
auth:
|
|
217
|
+
base_url: base_url,
|
|
218
|
+
auth: auth_params
|
|
189
219
|
)
|
|
190
220
|
end
|
|
191
221
|
|
|
@@ -225,23 +255,22 @@ module Aspera
|
|
|
225
255
|
Aspera.assert(fields.length == 2, type: Cli::BadArgument) { "Expects: work_step:result_name : #{result_location}" }
|
|
226
256
|
execution['step'], execution['variable'] = fields
|
|
227
257
|
end
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
end
|
|
258
|
+
# POST /api/initiate with JSON body {workflow_id:, external_parameters:}
|
|
259
|
+
json_body = {workflow_id: workflow_id.to_i, external_parameters: parameters}
|
|
260
|
+
# control params passed as query string (not part of the spec requestBody)
|
|
261
|
+
query_params = {}
|
|
233
262
|
Aspera.assert_type(execution['synchronous'], NilClass, *BoolValue::TYPES, type: Cli::BadArgument) { 'synchronous' }
|
|
234
|
-
|
|
263
|
+
query_params[:synchronous] = true if execution['synchronous'].eql?(true)
|
|
235
264
|
# expected result for synchro call ?
|
|
236
265
|
if execution.key?('step') || execution.key?('variable')
|
|
237
266
|
Aspera.assert(execution['step'].is_a?(String) && execution['variable'].is_a?(String), type: Cli::BadArgument) { 'Both step and variable are required' }
|
|
238
|
-
|
|
239
|
-
|
|
267
|
+
query_params[:explicit_output_step] = execution['step']
|
|
268
|
+
query_params[:explicit_output_variable] = execution['variable']
|
|
240
269
|
# implicitly, call is synchronous
|
|
241
|
-
|
|
270
|
+
query_params[:synchronous] = true
|
|
242
271
|
end
|
|
243
|
-
result_data = call_ao(
|
|
244
|
-
|
|
272
|
+
result_data = call_ao('initiate', body: json_body, query: query_params.empty? ? nil : query_params)
|
|
273
|
+
query_params[:synchronous] ? Result::Text.new(result_data) : Result::SingleObject.new(result_data)
|
|
245
274
|
end
|
|
246
275
|
end
|
|
247
276
|
end
|
|
@@ -3,11 +3,11 @@
|
|
|
3
3
|
# cspell:ignore trevents
|
|
4
4
|
require 'aspera/cli/plugins/basic_auth'
|
|
5
5
|
require 'aspera/preview/generator'
|
|
6
|
-
require 'aspera/preview/options'
|
|
7
6
|
require 'aspera/preview/utils'
|
|
8
7
|
require 'aspera/preview/file_types'
|
|
9
8
|
require 'aspera/preview/terminal'
|
|
10
9
|
require 'aspera/transfer/spec'
|
|
10
|
+
require 'aspera/schema/registry'
|
|
11
11
|
require 'aspera/persistency_action_once'
|
|
12
12
|
require 'aspera/temp_file_manager'
|
|
13
13
|
require 'aspera/api/node'
|
|
@@ -60,29 +60,30 @@ module Aspera
|
|
|
60
60
|
option :detect_mime, description: 'Detect Mime type by analyzing file', allowed: Type::BOOLEAN, default: false
|
|
61
61
|
option :overwrite, description: 'When to overwrite result file', allowed: %i[always never mtime], default: :mtime
|
|
62
62
|
option :root_url, description: "How to read and write files on storage (#{REMOTE_ACCESS}, or #{UriReader.file_url('<folder>')})", default: REMOTE_ACCESS
|
|
63
|
-
# Generator
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
63
|
+
# Generator options, defined in schema: values set in @gen_options
|
|
64
|
+
Schema::Registry.instance.reader(Schema::Registry::PREVIEW_OPTIONS).current['properties'].each do |name, property|
|
|
65
|
+
key = name.to_sym
|
|
66
|
+
option(
|
|
67
|
+
key,
|
|
68
|
+
description: property['description'].lines.first.strip.delete_suffix('.'),
|
|
69
|
+
allowed: property.key?('enum') ? property['enum'].map(&:to_sym) : {'integer' => Type::INTEGER, 'number' => Type::FLOAT, 'boolean' => Type::BOOLEAN}[property['type']],
|
|
70
|
+
default: property['default'],
|
|
71
|
+
# `Hash` value: validated with schema
|
|
72
|
+
schema: ("#{Schema::Registry::PREVIEW_OPTIONS}:properties.#{name}" if property['type'].eql?('object')),
|
|
73
|
+
on_set: ->(value) { @gen_options[key] = value }
|
|
74
|
+
)
|
|
71
75
|
end
|
|
72
76
|
|
|
73
77
|
def initialize(**_)
|
|
78
|
+
# Generator options, see `options.schema.yaml`
|
|
79
|
+
# Created before `super`, which declares options: values are set through `on_set`.
|
|
80
|
+
@gen_options = Aspera::Preview::Generator::Options.new
|
|
74
81
|
super
|
|
75
|
-
# Generator configuration populated from CLI options.
|
|
76
|
-
@gen_options = Aspera::Preview::Options.new
|
|
77
82
|
# Used to rate-limit periodic progress logging and checkpoint persistence.
|
|
78
83
|
@periodic = TimerLimiter.new(LOG_LIMITER_SEC)
|
|
79
84
|
# Optional callback used to filter entries before generation.
|
|
80
85
|
@filter_block = nil
|
|
81
86
|
@access_remote = true
|
|
82
|
-
# Bind generator-specific options to @gen_options
|
|
83
|
-
Aspera::Preview::Options::DESCRIPTIONS.each do |opt|
|
|
84
|
-
options.on_set(opt[:name], @gen_options.method(:"#{opt[:name]}="))
|
|
85
|
-
end
|
|
86
87
|
# Values set through `on_set` callbacks are used below
|
|
87
88
|
options.parse_options!
|
|
88
89
|
@option_skip_types = options.get_option(:skip_types)
|
|
@@ -98,9 +99,8 @@ module Aspera
|
|
|
98
99
|
@preview_formats_to_generate = Aspera::Preview::Generator::PREVIEW_FORMATS.clone
|
|
99
100
|
skip = options.get_option(:skip_format)
|
|
100
101
|
@preview_formats_to_generate.delete(skip) if skip
|
|
102
|
+
# Created by `tools_cleanup`, only for commands that use it
|
|
101
103
|
@tmp_folder = File.join(TempFileManager.instance.global_temp, "#{TMP_DIR_PREFIX}.#{SecureRandom.uuid}")
|
|
102
|
-
FileUtils.mkdir_p(@tmp_folder)
|
|
103
|
-
Log.log.debug { "tmpdir: #{@tmp_folder}" }
|
|
104
104
|
end
|
|
105
105
|
|
|
106
106
|
# Process legacy transfer events and trigger preview generation for completed downloads.
|
|
@@ -438,7 +438,7 @@ module Aspera
|
|
|
438
438
|
Log.log.debug { "root: #{node_info['docroot']}" }
|
|
439
439
|
# Default storage url to local file if not provided
|
|
440
440
|
option_root_url = options.get_option(:root_url, mandatory: true)
|
|
441
|
-
option_root_url = UriReader.file_url(@access_key_self['storage']['path']) if option_root_url.eql?(REMOTE_ACCESS) && @access_key_self['storage']
|
|
441
|
+
option_root_url = UriReader.file_url(@access_key_self['storage']['path']) if option_root_url.eql?(REMOTE_ACCESS) && @access_key_self['storage']&.fetch('type').eql?('local')
|
|
442
442
|
@access_remote = !UriReader.file?(option_root_url)
|
|
443
443
|
Log.log.debug { "remote: #{@access_remote}" }
|
|
444
444
|
# TODO: can the `previews` folder parameter be read from Node API ?
|
|
@@ -506,10 +506,12 @@ module Aspera
|
|
|
506
506
|
private
|
|
507
507
|
|
|
508
508
|
# Check tools and set option flag.
|
|
509
|
-
#
|
|
509
|
+
# Create the temporary folder for the command, and clean it up after.
|
|
510
510
|
def tools_cleanup
|
|
511
511
|
Aspera::Preview::FileTypes.instance.detect_mime = options.get_option(:detect_mime, mandatory: true)
|
|
512
512
|
Aspera::Preview::Utils.check_tools(@option_skip_types)
|
|
513
|
+
FileUtils.mkdir_p(@tmp_folder)
|
|
514
|
+
Log.log.debug { "tmpdir: #{@tmp_folder}" }
|
|
513
515
|
yield
|
|
514
516
|
ensure
|
|
515
517
|
Log.log.debug { "cleaning up temp folder #{@tmp_folder}" }
|