aspera-cli 4.26.2 → 4.27.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/CHANGELOG.md +281 -21
- data/CONTRIBUTING.md +98 -7
- data/bin/ascli +3 -3
- data/docs/README.md +10757 -0
- data/docs/test-mcp-with-ai.md +202 -0
- data/lib/aspera/agent/base.rb +20 -6
- data/lib/aspera/agent/connect.rb +53 -16
- data/lib/aspera/agent/desktop.rb +52 -17
- data/lib/aspera/agent/direct.rb +26 -24
- data/lib/aspera/agent/httpgw.rb +1 -1
- data/lib/aspera/agent/node.rb +32 -0
- data/lib/aspera/agent/transferd.rb +39 -3
- data/lib/aspera/api/alee.rb +1 -1
- data/lib/aspera/api/aoc.rb +14 -16
- data/lib/aspera/api/cos_node.rb +1 -1
- data/lib/aspera/api/faspex.rb +15 -14
- data/lib/aspera/api/httpgw.rb +4 -4
- data/lib/aspera/api/node.rb +26 -23
- data/lib/aspera/api/queries/bss_subscription_account.graphql +18 -0
- data/lib/aspera/api/queries/bss_subscription_usage.graphql +18 -0
- data/lib/aspera/ascmd.rb +21 -19
- data/lib/aspera/ascp/installation.rb +37 -38
- data/lib/aspera/ascp/management.rb +2 -3
- data/lib/aspera/cli/ascp_actions.rb +155 -0
- data/lib/aspera/cli/async_transfer_store.rb +81 -0
- data/lib/aspera/cli/bootstrapper.rb +197 -0
- data/lib/aspera/cli/command_registry.rb +145 -0
- data/lib/aspera/cli/command_spec.rb +138 -0
- data/lib/aspera/cli/context.rb +16 -3
- data/lib/aspera/cli/error.rb +16 -0
- data/lib/aspera/cli/extended_value.rb +2 -1
- data/lib/aspera/cli/formatter.rb +41 -27
- data/lib/aspera/cli/gem_checker.rb +65 -0
- data/lib/aspera/cli/hints.rb +2 -2
- data/lib/aspera/cli/http.rb +70 -19
- data/lib/aspera/cli/info.rb +2 -0
- data/lib/aspera/cli/mailer.rb +97 -0
- data/lib/aspera/cli/mcp_tool.rb +198 -0
- data/lib/aspera/cli/options.schema.yaml +364 -7
- data/lib/aspera/cli/{manager.rb → parser.rb} +334 -142
- data/lib/aspera/cli/plugins/alee.rb +20 -22
- data/lib/aspera/cli/plugins/aoc.rb +1166 -853
- data/lib/aspera/cli/plugins/ats.rb +200 -161
- data/lib/aspera/cli/plugins/base.rb +457 -100
- data/lib/aspera/cli/plugins/basic_auth.rb +14 -4
- data/lib/aspera/cli/plugins/config.rb +428 -754
- data/lib/aspera/cli/plugins/console.rb +106 -64
- data/lib/aspera/cli/plugins/cos.rb +44 -32
- data/lib/aspera/cli/plugins/factory.rb +4 -4
- data/lib/aspera/cli/plugins/faspex.rb +294 -257
- data/lib/aspera/cli/plugins/faspex5.rb +585 -357
- data/lib/aspera/cli/plugins/faspio.rb +49 -51
- data/lib/aspera/cli/plugins/httpgw.rb +18 -25
- data/lib/aspera/cli/plugins/mcp.rb +279 -0
- data/lib/aspera/cli/plugins/node.rb +998 -794
- data/lib/aspera/cli/plugins/oauth.rb +7 -10
- data/lib/aspera/cli/plugins/orchestrator.rb +111 -134
- data/lib/aspera/cli/plugins/preview.rb +180 -139
- data/lib/aspera/cli/plugins/server.rb +124 -79
- data/lib/aspera/cli/plugins/shares.rb +301 -107
- data/lib/aspera/cli/preset_actions.rb +139 -0
- data/lib/aspera/cli/preset_manager.rb +13 -12
- data/lib/aspera/cli/result.rb +61 -11
- data/lib/aspera/cli/runner.rb +275 -141
- data/lib/aspera/cli/secret_finder.rb +40 -0
- data/lib/aspera/cli/special_values.rb +1 -0
- data/lib/aspera/cli/sync_actions.rb +83 -59
- data/lib/aspera/cli/terminal_formatter.rb +1 -1
- data/lib/aspera/cli/transfer_actions.rb +83 -0
- data/lib/aspera/cli/transfer_agent.rb +116 -51
- data/lib/aspera/cli/transfer_progress.rb +3 -3
- data/lib/aspera/cli/vault_manager.rb +57 -0
- data/lib/aspera/cli/version.rb +1 -1
- data/lib/aspera/cli/wizard.rb +21 -20
- data/lib/aspera/command_line_builder.rb +22 -22
- data/lib/aspera/data_repository.rb +2 -1
- data/lib/aspera/dot_container.rb +4 -4
- data/lib/aspera/environment.rb +18 -11
- data/lib/aspera/exec_spec.rb +13 -0
- data/lib/aspera/faspex_gw.rb +1 -1
- data/lib/aspera/faspex_postproc.rb +4 -3
- data/lib/aspera/graphql.rb +35 -0
- data/lib/aspera/hash_ext.rb +6 -0
- data/lib/aspera/json_rpc/client.rb +62 -0
- data/lib/aspera/json_rpc/version.rb +7 -0
- data/lib/aspera/keychain/encrypted_hash.rb +1 -1
- data/lib/aspera/keychain/factory.rb +3 -3
- data/lib/aspera/keychain/macos_security.rb +3 -3
- data/lib/aspera/link_header.rb +82 -0
- data/lib/aspera/log.rb +17 -3
- data/lib/aspera/markdown.rb +85 -2
- data/lib/aspera/node_simulator.rb +3 -1
- data/lib/aspera/oauth/base.rb +5 -5
- data/lib/aspera/oauth/boot.rb +2 -2
- data/lib/aspera/oauth/factory.rb +7 -5
- data/lib/aspera/oauth/jwt.rb +4 -4
- data/lib/aspera/oauth/web.rb +2 -2
- data/lib/aspera/persistency_action_once.rb +8 -8
- data/lib/aspera/persistency_folder.rb +10 -3
- data/lib/aspera/preview/file_types.rb +3 -3
- data/lib/aspera/preview/generator.rb +3 -3
- data/lib/aspera/preview/utils.rb +1 -1
- data/lib/aspera/products/connect.rb +1 -1
- data/lib/aspera/products/other.rb +1 -1
- data/lib/aspera/products/transferd.rb +1 -1
- data/lib/aspera/proxy_auto_config.rb +7 -6
- data/lib/aspera/rest.rb +10 -36
- data/lib/aspera/rest_error_analyzer.rb +3 -3
- data/lib/aspera/rest_errors_aspera.rb +0 -10
- data/lib/aspera/rest_list.rb +3 -3
- data/lib/aspera/schema/IBM Aspera on Cloud API-0.2.6-enhanced.yaml +1853 -510
- data/lib/aspera/schema/documentation.rb +57 -26
- data/lib/aspera/schema/reader.rb +59 -10
- data/lib/aspera/schema/registry.rb +39 -6
- data/lib/aspera/secret_hider.rb +7 -2
- data/lib/aspera/ssl.rb +4 -4
- data/lib/aspera/sync/conf.schema.yaml +2 -2
- data/lib/aspera/sync/operations.rb +10 -13
- data/lib/aspera/temp_file_manager.rb +1 -1
- data/lib/aspera/timer_limiter.rb +1 -1
- data/lib/aspera/transfer/faux_file.rb +24 -11
- data/lib/aspera/transfer/parameters.rb +23 -21
- data/lib/aspera/transfer/result.rb +74 -0
- data/lib/aspera/transfer/resumer.rb +6 -6
- data/lib/aspera/transfer/spec.rb +18 -1
- data/lib/aspera/transfer/spec.schema.yaml +12 -3
- data/lib/aspera/uri_reader.rb +2 -2
- data/lib/aspera/web_auth.rb +2 -2
- data/lib/aspera/web_server_simple.rb +8 -5
- data/lib/aspera/yaml.rb +2 -1
- data.tar.gz.sig +0 -0
- metadata +25 -3
- metadata.gz.sig +0 -0
- data/lib/aspera/json_rpc.rb +0 -52
|
@@ -48,13 +48,31 @@ module Aspera
|
|
|
48
48
|
|
|
49
49
|
attr_accessor :option_skip_types, :option_previews_folder, :option_folder_reset_cache, :option_skip_folders, :option_overwrite
|
|
50
50
|
|
|
51
|
+
# DSL option declarations (Pattern 1 and Pattern 2 - handler: Symbol resolved to self)
|
|
52
|
+
option :skip_format, description: 'Skip this preview format', allowed: Aspera::Preview::Generator::PREVIEW_FORMATS
|
|
53
|
+
# TODO: use the same option as in `node` plugin
|
|
54
|
+
option :folder_reset_cache, description: 'Force detection of generated preview by refresh cache', allowed: %i[no header read], handler: :option_folder_reset_cache, default: :no
|
|
55
|
+
option :skip_types, description: 'Skip generation for those types of files', allowed: Allowed::TYPES_SYMBOL_ARRAY + Aspera::Preview::FileTypes::CONVERSION_TYPES, handler: :option_skip_types
|
|
56
|
+
option :previews_folder, description: 'Preview folder in storage root', handler: :option_previews_folder, default: DEFAULT_PREVIEWS_FOLDER
|
|
57
|
+
option :skip_folders, description: 'List of folder to skip', allowed: Allowed::TYPES_STRING_ARRAY, handler: :option_skip_folders
|
|
58
|
+
option :base, description: 'Basename of output for for test'
|
|
59
|
+
option :scan_path, description: 'Subpath in folder id to start scan in (default=/)'
|
|
60
|
+
option :scan_id, description: 'Folder id in storage to start scan in, default is access key main folder id'
|
|
61
|
+
option :mimemagic, description: 'Use Mime type detection of gem mimemagic', allowed: Allowed::TYPES_BOOLEAN, default: false
|
|
62
|
+
option :overwrite, description: 'When to overwrite result file', allowed: %i[always never mtime], handler: :option_overwrite, default: :mtime
|
|
63
|
+
option :root_url, description: "How to read and write files on storage (<empty>, #{REMOTE_ACCESS}, or #{UriReader.file_url('<folder>')})", default: ''
|
|
64
|
+
# Generator-specific options (Category C - bound to @gen_options via set_handler in initialize)
|
|
65
|
+
Aspera::Preview::Options::DESCRIPTIONS.each do |opt|
|
|
66
|
+
values = if opt.key?(:values)
|
|
67
|
+
opt[:values]
|
|
68
|
+
elsif BoolValue.symbol?(opt[:default])
|
|
69
|
+
BoolValue::TYPES
|
|
70
|
+
end
|
|
71
|
+
option(opt[:name], description: opt[:description].capitalize, allowed: values, default: opt[:default])
|
|
72
|
+
end
|
|
73
|
+
|
|
51
74
|
def initialize(**_)
|
|
52
75
|
super
|
|
53
|
-
@option_skip_types = []
|
|
54
|
-
@option_skip_folders = []
|
|
55
|
-
@option_previews_folder = nil
|
|
56
|
-
@option_overwrite = nil
|
|
57
|
-
@option_folder_reset_cache = nil
|
|
58
76
|
# Generator configuration populated from CLI options.
|
|
59
77
|
@gen_options = Aspera::Preview::Options.new
|
|
60
78
|
# Used to rate-limit periodic progress logging and checkpoint persistence.
|
|
@@ -62,40 +80,9 @@ module Aspera
|
|
|
62
80
|
# Optional callback used to filter entries before generation.
|
|
63
81
|
@filter_block = nil
|
|
64
82
|
@access_remote = true
|
|
65
|
-
# Bind
|
|
66
|
-
options.declare(
|
|
67
|
-
:skip_format, 'Skip this preview format',
|
|
68
|
-
allowed: Aspera::Preview::Generator::PREVIEW_FORMATS
|
|
69
|
-
)
|
|
70
|
-
# TODO: use the same option as in `node` plugin
|
|
71
|
-
options.declare(
|
|
72
|
-
:folder_reset_cache, 'Force detection of generated preview by refresh cache',
|
|
73
|
-
allowed: %i[no header read],
|
|
74
|
-
handler: {o: self, m: :option_folder_reset_cache},
|
|
75
|
-
default: :no
|
|
76
|
-
)
|
|
77
|
-
options.declare(:skip_types, 'Skip generation for those types of files', handler: {o: self, m: :option_skip_types}, allowed: Allowed::TYPES_SYMBOL_ARRAY + Aspera::Preview::FileTypes::CONVERSION_TYPES)
|
|
78
|
-
options.declare(:previews_folder, 'Preview folder in storage root', handler: {o: self, m: :option_previews_folder}, default: DEFAULT_PREVIEWS_FOLDER)
|
|
79
|
-
options.declare(:skip_folders, 'List of folder to skip', handler: {o: self, m: :option_skip_folders}, allowed: Allowed::TYPES_STRING_ARRAY)
|
|
80
|
-
options.declare(:base, 'Basename of output for for test')
|
|
81
|
-
options.declare(:scan_path, 'Subpath in folder id to start scan in (default=/)')
|
|
82
|
-
options.declare(:scan_id, 'Folder id in storage to start scan in, default is access key main folder id')
|
|
83
|
-
options.declare(:mimemagic, 'Use Mime type detection of gem mimemagic', allowed: Allowed::TYPES_BOOLEAN, default: false)
|
|
84
|
-
options.declare(:overwrite, 'When to overwrite result file', handler: {o: self, m: :option_overwrite}, allowed: %i[always never mtime], default: :mtime)
|
|
85
|
-
options.declare(
|
|
86
|
-
:root_url,
|
|
87
|
-
"How to read and write files on storage (<empty>, #{REMOTE_ACCESS}, or #{UriReader.file_url('<folder>')})",
|
|
88
|
-
allowed: Allowed::TYPES_STRING,
|
|
89
|
-
default: ''
|
|
90
|
-
)
|
|
91
|
-
# Declare generator-specific options and apply their default values.
|
|
83
|
+
# Bind generator-specific options to @gen_options (Category C: set_handler after object creation)
|
|
92
84
|
Aspera::Preview::Options::DESCRIPTIONS.each do |opt|
|
|
93
|
-
|
|
94
|
-
opt[:values]
|
|
95
|
-
elsif BoolValue.symbol?(opt[:default])
|
|
96
|
-
BoolValue::TYPES
|
|
97
|
-
end
|
|
98
|
-
options.declare(opt[:name], opt[:description].capitalize, allowed: values, handler: {o: @gen_options, m: opt[:name]}, default: opt[:default])
|
|
85
|
+
options.set_handler(opt[:name], object: @gen_options, method: opt[:name])
|
|
99
86
|
end
|
|
100
87
|
|
|
101
88
|
options.parse_options!
|
|
@@ -112,7 +99,7 @@ module Aspera
|
|
|
112
99
|
# Process legacy transfer events and trigger preview generation for completed downloads.
|
|
113
100
|
#
|
|
114
101
|
# @param iteration_persistency [PersistencyActionOnce, nil] stores the last processed event id
|
|
115
|
-
# @return [
|
|
102
|
+
# @return [nil]
|
|
116
103
|
def process_trevents(iteration_persistency)
|
|
117
104
|
events_filter = {
|
|
118
105
|
'access_key' => @access_key_self['id'],
|
|
@@ -152,12 +139,13 @@ module Aspera
|
|
|
152
139
|
iteration_persistency.save
|
|
153
140
|
end
|
|
154
141
|
end
|
|
142
|
+
nil
|
|
155
143
|
end
|
|
156
144
|
|
|
157
145
|
# Process recent Node API file events since the last persisted checkpoint.
|
|
158
146
|
#
|
|
159
147
|
# @param iteration_persistency [PersistencyActionOnce, nil] stores the last processed event id
|
|
160
|
-
# @return [
|
|
148
|
+
# @return [nil]
|
|
161
149
|
def process_events(iteration_persistency)
|
|
162
150
|
# Restrict the event stream to file-related changes for the current access key.
|
|
163
151
|
events_filter = {
|
|
@@ -188,6 +176,7 @@ module Aspera
|
|
|
188
176
|
iteration_persistency.save
|
|
189
177
|
end
|
|
190
178
|
end
|
|
179
|
+
nil
|
|
191
180
|
end
|
|
192
181
|
|
|
193
182
|
# Transfer a file to or from the configured Node storage using a tagged transfer spec.
|
|
@@ -278,7 +267,7 @@ module Aspera
|
|
|
278
267
|
# Remote entries must include `parent_file_id`.
|
|
279
268
|
#
|
|
280
269
|
# @param entry [Hash] local or remote file entry to preview
|
|
281
|
-
# @return [
|
|
270
|
+
# @return [nil]
|
|
282
271
|
def generate_preview(entry)
|
|
283
272
|
# prepare generic information
|
|
284
273
|
gen_infos = @preview_formats_to_generate.map do |preview_format|
|
|
@@ -406,113 +395,165 @@ module Aspera
|
|
|
406
395
|
end
|
|
407
396
|
end
|
|
408
397
|
|
|
409
|
-
|
|
398
|
+
# --- DSL ---
|
|
399
|
+
|
|
400
|
+
# Commands that connect to the Node API share a common setup.
|
|
401
|
+
# :check, :test and :show work without Node API authentication.
|
|
402
|
+
command :scan, description: 'Scan all files and generate previews', setup: :setup_node_api
|
|
403
|
+
command :events, description: 'Process file events and generate previews', setup: :setup_node_api
|
|
404
|
+
command :trevents, description: 'Process transfer events and generate previews', setup: :setup_node_api
|
|
405
|
+
command :check, description: 'Check required tools are installed'
|
|
406
|
+
command :test, description: 'Test preview generation for a source file',
|
|
407
|
+
arguments: [{name: :source_file, type: String}, {name: :format, allowed: Aspera::Preview::Generator::PREVIEW_FORMATS, mandatory: false, default: :png}]
|
|
408
|
+
command :show, description: 'Generate and display preview of a source file',
|
|
409
|
+
arguments: [{name: :source_file, type: String}]
|
|
410
|
+
|
|
411
|
+
# --- setup ---
|
|
410
412
|
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
413
|
+
# Build the Node API and set up storage references for scan/events/trevents.
|
|
414
|
+
# @return [Hash] empty ctx (state stored in instance variables)
|
|
415
|
+
def setup_node_api(**)
|
|
416
|
+
@api_node = Api::Node.new(**basic_auth_params)
|
|
417
|
+
@transfer_server_address = URI.parse(@api_node.base_url).host
|
|
418
|
+
# Get current access key information
|
|
419
|
+
@access_key_self = @api_node.read('access_keys/self')
|
|
420
|
+
# TODO: check events is activated here:
|
|
421
|
+
# note that docroot is good to look at as well
|
|
422
|
+
node_info = @api_node.read('info')
|
|
423
|
+
Log.log.debug{"root: #{node_info['docroot']}"}
|
|
424
|
+
# Default storage url to local file if not provided
|
|
425
|
+
option_root_url = options.get_option(:root_url, mandatory: true)
|
|
426
|
+
option_root_url = UriReader.file_url(@access_key_self['storage']['path']) if option_root_url.empty? && @access_key_self['storage']['type'].eql?('local')
|
|
427
|
+
@access_remote = !UriReader.file?(option_root_url)
|
|
428
|
+
Log.log.debug{"remote: #{@access_remote}"}
|
|
429
|
+
# TODO: can the `previews` folder parameter be read from Node API ?
|
|
430
|
+
@option_skip_folders.push("/#{@option_previews_folder}")
|
|
431
|
+
if @access_remote
|
|
432
|
+
# NOTE: the filter "name", it's why we take the first one
|
|
433
|
+
@previews_folder_entry = @api_node.read_folder_content(@access_key_self['root_file_id'], {name: @option_previews_folder}).first
|
|
434
|
+
raise Cli::Error, "Folder #{@option_previews_folder} does not exist on node. " \
|
|
435
|
+
'Please create it in the storage root, or specify an alternate name.' if @previews_folder_entry.nil?
|
|
436
|
+
else
|
|
437
|
+
@local_storage_root = UriReader.file_path(option_root_url)
|
|
438
|
+
# TODO: Windows could have "C:" ?
|
|
439
|
+
Aspera.assert(@local_storage_root.start_with?('/')){"not local storage: #{@local_storage_root}"}
|
|
440
|
+
Aspera.assert(File.directory?(@local_storage_root), type: Cli::Error){"Local storage root folder #{@local_storage_root} does not exist."}
|
|
441
|
+
@local_preview_folder = File.join(@local_storage_root, @option_previews_folder)
|
|
442
|
+
Aspera.assert(File.directory?(@local_preview_folder), type: Cli::Error){"Folder #{@local_preview_folder} does not exist locally. Please create it, or specify an alternate name."}
|
|
443
|
+
# Protection to avoid clash of file id for two different access keys
|
|
444
|
+
marker_file = File.join(@local_preview_folder, AK_MARKER_FILE)
|
|
445
|
+
Log.log.debug{"marker file: #{marker_file}"}
|
|
446
|
+
if File.exist?(marker_file)
|
|
447
|
+
ak = File.read(marker_file).chomp
|
|
448
|
+
Aspera.assert(@access_key_self['id'].eql?(ak)){"mismatch access key in #{marker_file}: contains #{ak}, using #{@access_key_self['id']}"}
|
|
435
449
|
else
|
|
436
|
-
|
|
437
|
-
# TODO: Windows could have "C:" ?
|
|
438
|
-
Aspera.assert(@local_storage_root.start_with?('/')){"not local storage: #{@local_storage_root}"}
|
|
439
|
-
Aspera.assert(File.directory?(@local_storage_root), type: Cli::Error){"Local storage root folder #{@local_storage_root} does not exist."}
|
|
440
|
-
@local_preview_folder = File.join(@local_storage_root, @option_previews_folder)
|
|
441
|
-
Aspera.assert(File.directory?(@local_preview_folder), type: Cli::Error){"Folder #{@local_preview_folder} does not exist locally. Please create it, or specify an alternate name."}
|
|
442
|
-
# Protection to avoid clash of file id for two different access keys
|
|
443
|
-
marker_file = File.join(@local_preview_folder, AK_MARKER_FILE)
|
|
444
|
-
Log.log.debug{"marker file: #{marker_file}"}
|
|
445
|
-
if File.exist?(marker_file)
|
|
446
|
-
ak = File.read(marker_file).chomp
|
|
447
|
-
Aspera.assert(@access_key_self['id'].eql?(ak)){"mismatch access key in #{marker_file}: contains #{ak}, using #{@access_key_self['id']}"}
|
|
448
|
-
else
|
|
449
|
-
File.write(marker_file, @access_key_self['id'])
|
|
450
|
-
end
|
|
450
|
+
File.write(marker_file, @access_key_self['id'])
|
|
451
451
|
end
|
|
452
452
|
end
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
@filter_block = Api::Node.file_matcher_from_argument(options)
|
|
473
|
-
scan_folder_files(folder_info, scan_path)
|
|
474
|
-
return Result::Status.new('scan finished')
|
|
475
|
-
when :events, :trevents
|
|
476
|
-
@filter_block = Api::Node.file_matcher_from_argument(options)
|
|
477
|
-
iteration_persistency = nil
|
|
478
|
-
if options.get_option(:once_only, mandatory: true)
|
|
479
|
-
iteration_persistency = PersistencyActionOnce.new(
|
|
480
|
-
manager: persistency,
|
|
481
|
-
data: [],
|
|
482
|
-
id: IdGenerator.from_list(
|
|
483
|
-
'preview_iteration',
|
|
484
|
-
command.to_s,
|
|
485
|
-
options.get_option(:url, mandatory: true),
|
|
486
|
-
options.get_option(:username, mandatory: true)
|
|
487
|
-
)
|
|
488
|
-
)
|
|
453
|
+
{}
|
|
454
|
+
end
|
|
455
|
+
|
|
456
|
+
# --- handlers ---
|
|
457
|
+
|
|
458
|
+
def action_scan
|
|
459
|
+
check_tools_and_mimemagic
|
|
460
|
+
scan_path = options.get_option(:scan_path)
|
|
461
|
+
scan_id = options.get_option(:scan_id)
|
|
462
|
+
folder_info =
|
|
463
|
+
if scan_id.nil?
|
|
464
|
+
{
|
|
465
|
+
'id' => @access_key_self['root_file_id'],
|
|
466
|
+
'name' => '/',
|
|
467
|
+
'type' => 'folder',
|
|
468
|
+
'path' => '/'
|
|
469
|
+
}
|
|
470
|
+
else
|
|
471
|
+
@api_node.read("files/#{scan_id}")
|
|
489
472
|
end
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
when :check
|
|
494
|
-
return Result::Status.new('Tools validated')
|
|
495
|
-
when :test
|
|
496
|
-
source = options.get_next_argument('source file')
|
|
497
|
-
format = options.get_next_argument('format', accept_list: Aspera::Preview::Generator::PREVIEW_FORMATS, default: :png)
|
|
498
|
-
generated_file_path = preview_filename(format, options.get_option(:base))
|
|
499
|
-
Aspera::Preview::Generator.new(source, generated_file_path, @gen_options, @tmp_folder).generate
|
|
500
|
-
return Result::Status.new("generated: #{generated_file_path}")
|
|
501
|
-
when :show
|
|
502
|
-
source = options.get_next_argument('source file')
|
|
503
|
-
# terminal_options = options.get_next_argument('options', validation: Hash, default: {}).symbolize_keys
|
|
504
|
-
generated_file_path = preview_filename(:png, options.get_option(:base))
|
|
505
|
-
Aspera::Preview::Generator.new(source, generated_file_path, @gen_options, @tmp_folder).generate
|
|
506
|
-
formatter.display_status("generated: #{generated_file_path}")
|
|
507
|
-
# formatter.display_status(Aspera::Preview::Terminal.build(File.read(generated_file_path), **terminal_options))
|
|
508
|
-
# return Result::Status.new("generated: #{generated_file_path}")
|
|
509
|
-
return Result::Image.new(UriReader.file_url(generated_file_path))
|
|
510
|
-
else Aspera.error_unexpected_value(command)
|
|
511
|
-
end
|
|
473
|
+
@filter_block = Api::Node.file_matcher_from_argument(options)
|
|
474
|
+
scan_folder_files(folder_info, scan_path)
|
|
475
|
+
Result::Status.new('scan finished')
|
|
512
476
|
ensure
|
|
477
|
+
cleanup_tmp_folder
|
|
478
|
+
end
|
|
479
|
+
|
|
480
|
+
def action_events
|
|
481
|
+
check_tools_and_mimemagic
|
|
482
|
+
run_event_loop(:events)
|
|
483
|
+
ensure
|
|
484
|
+
cleanup_tmp_folder
|
|
485
|
+
end
|
|
486
|
+
|
|
487
|
+
def action_trevents
|
|
488
|
+
check_tools_and_mimemagic
|
|
489
|
+
run_event_loop(:trevents)
|
|
490
|
+
ensure
|
|
491
|
+
cleanup_tmp_folder
|
|
492
|
+
end
|
|
493
|
+
|
|
494
|
+
def action_check
|
|
495
|
+
check_tools_and_mimemagic
|
|
496
|
+
Result::Status.new('Tools validated')
|
|
497
|
+
ensure
|
|
498
|
+
cleanup_tmp_folder
|
|
499
|
+
end
|
|
500
|
+
|
|
501
|
+
def action_test(source_file:, format:, **)
|
|
502
|
+
check_tools_and_mimemagic
|
|
503
|
+
generated_file_path = preview_filename(format, options.get_option(:base))
|
|
504
|
+
Aspera::Preview::Generator.new(source_file, generated_file_path, @gen_options, @tmp_folder).generate
|
|
505
|
+
Result::Status.new("generated: #{generated_file_path}")
|
|
506
|
+
ensure
|
|
507
|
+
cleanup_tmp_folder
|
|
508
|
+
end
|
|
509
|
+
|
|
510
|
+
def action_show(source_file:, **)
|
|
511
|
+
check_tools_and_mimemagic
|
|
512
|
+
# terminal_options = options.get_next_argument('options', validation: Hash, default: {}).symbolize_keys
|
|
513
|
+
generated_file_path = preview_filename(:png, options.get_option(:base))
|
|
514
|
+
Aspera::Preview::Generator.new(source_file, generated_file_path, @gen_options, @tmp_folder).generate
|
|
515
|
+
formatter.display_status("generated: #{generated_file_path}")
|
|
516
|
+
# formatter.display_status(Aspera::Preview::Terminal.build(File.read(generated_file_path), **terminal_options))
|
|
517
|
+
# Result::Status.new("generated: #{generated_file_path}")
|
|
518
|
+
Result::Image.new(UriReader.file_url(generated_file_path))
|
|
519
|
+
ensure
|
|
520
|
+
cleanup_tmp_folder
|
|
521
|
+
end
|
|
522
|
+
|
|
523
|
+
private
|
|
524
|
+
|
|
525
|
+
# Check tools and set mimemagic flag (required for all commands).
|
|
526
|
+
def check_tools_and_mimemagic
|
|
527
|
+
Aspera::Preview::FileTypes.instance.use_mimemagic = options.get_option(:mimemagic, mandatory: true)
|
|
528
|
+
Aspera::Preview::Utils.check_tools(@option_skip_types)
|
|
529
|
+
end
|
|
530
|
+
|
|
531
|
+
# Clean up the temporary folder after each command.
|
|
532
|
+
def cleanup_tmp_folder
|
|
513
533
|
Log.log.debug{"cleaning up temp folder #{@tmp_folder}"}
|
|
514
534
|
FileUtils.rm_rf(@tmp_folder)
|
|
515
535
|
end
|
|
536
|
+
|
|
537
|
+
# Shared event-loop body for :events and :trevents.
|
|
538
|
+
# @param command [:events, :trevents]
|
|
539
|
+
def run_event_loop(command)
|
|
540
|
+
@filter_block = Api::Node.file_matcher_from_argument(options)
|
|
541
|
+
iteration_persistency = nil
|
|
542
|
+
if options.get_option(:once_only, mandatory: true)
|
|
543
|
+
iteration_persistency = PersistencyActionOnce.new(
|
|
544
|
+
manager: persistency,
|
|
545
|
+
data: [],
|
|
546
|
+
id: IdGenerator.from_list(
|
|
547
|
+
'preview_iteration',
|
|
548
|
+
command.to_s,
|
|
549
|
+
options.get_option(:url, mandatory: true),
|
|
550
|
+
options.get_option(:username, mandatory: true)
|
|
551
|
+
)
|
|
552
|
+
)
|
|
553
|
+
end
|
|
554
|
+
send(:"process_#{command}", iteration_persistency)
|
|
555
|
+
Result::Status.new("#{command} finished")
|
|
556
|
+
end
|
|
516
557
|
end
|
|
517
558
|
end
|
|
518
559
|
end
|
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
# cspell:ignore ascmd zmode zuid zgid fasping
|
|
4
4
|
require 'aspera/cli/plugins/basic_auth'
|
|
5
5
|
require 'aspera/cli/sync_actions'
|
|
6
|
+
require 'aspera/transfer/result'
|
|
6
7
|
require 'aspera/transfer/spec'
|
|
7
8
|
require 'aspera/ascmd'
|
|
8
9
|
require 'aspera/ssh'
|
|
@@ -21,26 +22,17 @@ module Aspera
|
|
|
21
22
|
LOCAL_SCHEME = 'local'
|
|
22
23
|
HTTPS_SCHEME = 'https'
|
|
23
24
|
URI_SCHEMES = [SSH_SCHEME, LOCAL_SCHEME, HTTPS_SCHEME].freeze
|
|
24
|
-
ASCMD_ALIASES = {
|
|
25
|
-
browse: :ls,
|
|
26
|
-
delete: :rm,
|
|
27
|
-
rename: :mv
|
|
28
|
-
}.freeze
|
|
29
|
-
TRANSFER_COMMANDS = %i[sync upload download].freeze
|
|
30
25
|
|
|
31
|
-
private_constant :SSH_SCHEME, :URI_SCHEMES
|
|
26
|
+
private_constant :SSH_SCHEME, :URI_SCHEMES
|
|
32
27
|
|
|
33
28
|
class LocalExecutor
|
|
34
29
|
def execute(ascmd_path, input:)
|
|
35
30
|
Environment.secure_execute(ascmd_path, mode: :capture, stdin_data: input, binmode: true, exception: false).first
|
|
36
31
|
end
|
|
37
32
|
end
|
|
33
|
+
application_name 'HSTS Fasp/SSH'
|
|
38
34
|
|
|
39
35
|
class << self
|
|
40
|
-
def application_name
|
|
41
|
-
'HSTS Fasp/SSH'
|
|
42
|
-
end
|
|
43
|
-
|
|
44
36
|
# @return [Hash,NilClass]
|
|
45
37
|
def detect(address_or_url)
|
|
46
38
|
urls = if address_or_url.match?(%r{^[a-z]{1,6}://})
|
|
@@ -87,10 +79,8 @@ module Aspera
|
|
|
87
79
|
def initialize(**_)
|
|
88
80
|
super
|
|
89
81
|
@connection_type = :ssh
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
options.declare(:ssh_options, 'SSH options', allowed: Hash, default: {})
|
|
93
|
-
SyncActions.declare_options(options)
|
|
82
|
+
@ascmd_executor = nil
|
|
83
|
+
@server_transfer_spec = nil
|
|
94
84
|
options.parse_options!
|
|
95
85
|
@ssh_opts = options.get_option(:ssh_options).symbolize_keys
|
|
96
86
|
end
|
|
@@ -169,77 +159,132 @@ module Aspera
|
|
|
169
159
|
return server_transfer_spec
|
|
170
160
|
end
|
|
171
161
|
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
162
|
+
Sync::Operations::DIRECTIONS.each do |dir|
|
|
163
|
+
define_method(:"action_sync_#{dir}"){ |**| run_sync_transfer(dir){@server_transfer_spec}}
|
|
164
|
+
end
|
|
165
|
+
|
|
166
|
+
# --- DSL ---
|
|
167
|
+
|
|
168
|
+
option :ssh_keys, description: 'SSH key path list', allowed: Allowed::TYPES_STRING_ARRAY
|
|
169
|
+
option :passphrase, description: 'SSH private key passphrase'
|
|
170
|
+
option :ssh_options, description: 'SSH options', allowed: Hash
|
|
171
|
+
|
|
172
|
+
# root_setup runs once before any command argument is consumed, populating
|
|
173
|
+
# @server_transfer_spec and @ascmd_executor so that condition methods work.
|
|
174
|
+
root_setup :setup_server
|
|
175
|
+
|
|
176
|
+
command :health, description: 'Check transfer health'
|
|
177
|
+
command(:upload, description: 'Upload files to server', transfer_paths: :send, action: lambda do
|
|
178
|
+
@server_transfer_spec['direction'] = Transfer::Spec.transfer_type_to_direction(:upload)
|
|
179
|
+
Runner.result_transfer(transfer.start(@server_transfer_spec))
|
|
180
|
+
end)
|
|
181
|
+
command(:download, description: 'Download files from server', transfer_paths: :receive, action: lambda do
|
|
182
|
+
@server_transfer_spec['direction'] = Transfer::Spec.transfer_type_to_direction(:download)
|
|
183
|
+
Runner.result_transfer(transfer.start(@server_transfer_spec))
|
|
184
|
+
end)
|
|
185
|
+
command :sync, description: 'Synchronize files with server'
|
|
186
|
+
commands_under(:sync) do
|
|
187
|
+
Sync::Operations::DIRECTIONS.each do |dir|
|
|
188
|
+
command(dir, description: "#{dir.capitalize}-sync with server", transfer_paths: :send)
|
|
189
|
+
end
|
|
190
|
+
command :admin, description: 'Manage sync database (admin operations)'
|
|
191
|
+
SyncActions.register_sync_admin_commands(self, %i[sync admin])
|
|
192
|
+
end
|
|
193
|
+
|
|
194
|
+
commands_under(:health) do
|
|
195
|
+
command :transfer, description: 'Check FASP transfer health'
|
|
196
|
+
end
|
|
197
|
+
|
|
198
|
+
# AsCmd operations (available only when SSH/local - not WSS)
|
|
199
|
+
ASCMD_ARGS = [{name: :command_arguments, multiple: true, mandatory: false, default: nil}].freeze
|
|
200
|
+
private_constant :ASCMD_ARGS
|
|
201
|
+
|
|
202
|
+
command :ls, description: 'List files on server', condition: :ascmd_available?, aliases: [:browse], arguments: ASCMD_ARGS
|
|
203
|
+
command :rm, description: 'Delete file(s) on server', condition: :ascmd_available?, aliases: [:delete], arguments: ASCMD_ARGS
|
|
204
|
+
command :mv, description: 'Rename/move file(s) on server', condition: :ascmd_available?, aliases: [:rename], arguments: ASCMD_ARGS
|
|
205
|
+
command :cp, description: 'Copy file(s) on server', condition: :ascmd_available?, arguments: ASCMD_ARGS
|
|
206
|
+
command :mkdir, description: 'Create directory on server', condition: :ascmd_available?, arguments: ASCMD_ARGS
|
|
207
|
+
command :df, description: 'Show disk usage on server', condition: :ascmd_available?, arguments: ASCMD_ARGS
|
|
208
|
+
command :du, description: 'Show file sizes on server', condition: :ascmd_available?, arguments: ASCMD_ARGS
|
|
209
|
+
command :md5sum, description: 'Compute MD5 checksums on server', condition: :ascmd_available?, arguments: ASCMD_ARGS
|
|
210
|
+
command :info, description: 'Show server system information', condition: :ascmd_available?, arguments: ASCMD_ARGS
|
|
211
|
+
|
|
212
|
+
# Generate ascmd handlers - convention: action_<op>
|
|
213
|
+
%i[rm mv cp mkdir].each do |op|
|
|
214
|
+
define_action_method([op]) do |command_arguments:, **|
|
|
215
|
+
execute_ascmd(op, command_arguments){Result::Success.new}
|
|
180
216
|
end
|
|
181
217
|
end
|
|
182
218
|
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
219
|
+
%i[du md5sum info].each do |op|
|
|
220
|
+
define_action_method([op]) do |command_arguments:, **|
|
|
221
|
+
execute_ascmd(op, command_arguments){ |r| Result::SingleObject.new(r.stringify_keys)}
|
|
222
|
+
end
|
|
223
|
+
end
|
|
224
|
+
|
|
225
|
+
# --- conditions ---
|
|
226
|
+
|
|
227
|
+
def ascmd_available?
|
|
228
|
+
!@ascmd_executor.nil?
|
|
229
|
+
end
|
|
230
|
+
|
|
231
|
+
# --- setup ---
|
|
187
232
|
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
233
|
+
# Build the transfer spec and ascmd executor from CLI options.
|
|
234
|
+
# Stores results in instance variables; returns empty context hash.
|
|
235
|
+
def setup_server(**)
|
|
236
|
+
@server_transfer_spec = options_to_base_transfer_spec
|
|
237
|
+
@ascmd_executor = case @connection_type
|
|
191
238
|
when :local then LocalExecutor.new
|
|
192
|
-
when :wss
|
|
193
|
-
when :ssh
|
|
239
|
+
when :wss then nil
|
|
240
|
+
when :ssh then Ssh.new(@server_transfer_spec['remote_host'], @server_transfer_spec['remote_user'], @ssh_opts)
|
|
194
241
|
else Aspera.error_unexpected_value(@connection_type){'connection type'}
|
|
195
242
|
end
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
end
|
|
242
|
-
else Aspera.error_unreachable_line
|
|
243
|
+
{}
|
|
244
|
+
end
|
|
245
|
+
|
|
246
|
+
# --- health ---
|
|
247
|
+
|
|
248
|
+
def action_health_transfer
|
|
249
|
+
nagios = Nagios.new
|
|
250
|
+
probe_ts = @server_transfer_spec.merge({
|
|
251
|
+
'direction' => 'send',
|
|
252
|
+
'cookie' => 'aspera.sync', # hide in console
|
|
253
|
+
'resume_policy' => 'none',
|
|
254
|
+
'paths' => [{'source' => 'faux:///pingfile?1k', 'destination' => '.fasping'}]
|
|
255
|
+
})
|
|
256
|
+
statuses = transfer.start(probe_ts)
|
|
257
|
+
if statuses.is_a?(Transfer::Result::Success)
|
|
258
|
+
nagios.add_ok('transfer', 'ok')
|
|
259
|
+
else
|
|
260
|
+
nagios.add_critical('transfer', statuses.is_a?(Transfer::Result::Error) ? statuses.exception.message : statuses.to_s)
|
|
261
|
+
end
|
|
262
|
+
Result::ObjectList.new(nagios.status_list)
|
|
263
|
+
end
|
|
264
|
+
|
|
265
|
+
# --- ascmd handlers ---
|
|
266
|
+
|
|
267
|
+
def action_ls(command_arguments:, **)
|
|
268
|
+
execute_ascmd(:ls, command_arguments) do |result|
|
|
269
|
+
Result::ObjectList.new(result.map(&:stringify_keys), fields: %w[zmode zuid zgid size mtime name])
|
|
270
|
+
end
|
|
271
|
+
end
|
|
272
|
+
|
|
273
|
+
def action_df(command_arguments:, **)
|
|
274
|
+
execute_ascmd(:df, command_arguments) do |result|
|
|
275
|
+
Result::ObjectList.new(result.map(&:stringify_keys))
|
|
276
|
+
end
|
|
277
|
+
end
|
|
278
|
+
|
|
279
|
+
private
|
|
280
|
+
|
|
281
|
+
def execute_ascmd(op, command_arguments)
|
|
282
|
+
ascmd = AsCmd.new(@ascmd_executor)
|
|
283
|
+
begin
|
|
284
|
+
result = ascmd.execute_single(op, command_arguments)
|
|
285
|
+
yield(result)
|
|
286
|
+
rescue AsCmd::Error => e
|
|
287
|
+
raise Cli::BadArgument, e.extended_message
|
|
243
288
|
end
|
|
244
289
|
end
|
|
245
290
|
end
|