aspera-cli 4.26.2 → 4.27.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/CHANGELOG.md +310 -21
- data/CONTRIBUTING.md +99 -14
- data/TODO.md +50 -0
- data/bin/ascli +3 -3
- data/docs/README.md +10977 -0
- data/docs/test-mcp-with-ai.md +202 -0
- data/lib/aspera/agent/base.rb +29 -7
- data/lib/aspera/agent/connect.rb +58 -21
- data/lib/aspera/agent/desktop.rb +53 -18
- data/lib/aspera/agent/direct.rb +71 -40
- data/lib/aspera/agent/factory.rb +3 -3
- data/lib/aspera/agent/httpgw.rb +55 -14
- data/lib/aspera/agent/node.rb +34 -2
- data/lib/aspera/agent/transferd.rb +48 -12
- data/lib/aspera/api/alee.rb +1 -1
- data/lib/aspera/api/aoc.rb +42 -44
- data/lib/aspera/api/cos_node.rb +10 -9
- data/lib/aspera/api/faspex.rb +16 -15
- data/lib/aspera/api/httpgw.rb +23 -23
- data/lib/aspera/api/node.rb +51 -70
- data/lib/aspera/api/queries/bss_subscription_account.graphql +18 -0
- data/lib/aspera/api/queries/bss_subscription_usage.graphql +18 -0
- data/lib/aspera/ascmd.rb +32 -30
- data/lib/aspera/ascp/installation.rb +75 -69
- data/lib/aspera/ascp/management.rb +7 -8
- data/lib/aspera/assert.rb +12 -13
- data/lib/aspera/cli/ascp_actions.rb +155 -0
- data/lib/aspera/cli/async_transfer_store.rb +125 -0
- data/lib/aspera/cli/bootstrapper.rb +197 -0
- data/lib/aspera/cli/command_registry.rb +145 -0
- data/lib/aspera/cli/command_spec.rb +141 -0
- data/lib/aspera/cli/context.rb +19 -6
- data/lib/aspera/cli/error.rb +16 -0
- data/lib/aspera/cli/extended_value.rb +27 -26
- data/lib/aspera/cli/formatter.rb +95 -59
- data/lib/aspera/cli/gem_checker.rb +65 -0
- data/lib/aspera/cli/hints.rb +3 -3
- data/lib/aspera/cli/http.rb +74 -30
- data/lib/aspera/cli/info.rb +2 -0
- data/lib/aspera/cli/mailer.rb +97 -0
- data/lib/aspera/cli/mcp_tool.rb +225 -0
- data/lib/aspera/cli/option_declarator.rb +77 -0
- data/lib/aspera/cli/options.schema.yaml +634 -7
- data/lib/aspera/cli/parser.rb +1217 -0
- data/lib/aspera/cli/plugins/alee.rb +20 -22
- data/lib/aspera/cli/plugins/aoc.rb +1178 -881
- data/lib/aspera/cli/plugins/ats.rb +205 -162
- data/lib/aspera/cli/plugins/base.rb +703 -175
- data/lib/aspera/cli/plugins/basic_auth.rb +7 -9
- data/lib/aspera/cli/plugins/config.rb +434 -757
- data/lib/aspera/cli/plugins/console.rb +107 -65
- data/lib/aspera/cli/plugins/cos.rb +46 -33
- data/lib/aspera/cli/plugins/factory.rb +6 -6
- data/lib/aspera/cli/plugins/faspex5.rb +627 -388
- data/lib/aspera/cli/plugins/faspio.rb +54 -51
- data/lib/aspera/cli/plugins/httpgw.rb +18 -25
- data/lib/aspera/cli/plugins/mcp.rb +279 -0
- data/lib/aspera/cli/plugins/node.rb +988 -840
- data/lib/aspera/cli/plugins/oauth.rb +7 -10
- data/lib/aspera/cli/plugins/orchestrator.rb +112 -135
- data/lib/aspera/cli/plugins/preview.rb +195 -157
- data/lib/aspera/cli/plugins/server.rb +141 -91
- data/lib/aspera/cli/plugins/shares.rb +343 -110
- data/lib/aspera/cli/preset_actions.rb +159 -0
- data/lib/aspera/cli/preset_manager.rb +82 -39
- data/lib/aspera/cli/result.rb +78 -26
- data/lib/aspera/cli/runner.rb +323 -153
- data/lib/aspera/cli/secret_finder.rb +40 -0
- data/lib/aspera/cli/special_values.rb +1 -0
- data/lib/aspera/cli/sync_actions.rb +89 -64
- data/lib/aspera/cli/terminal_formatter.rb +3 -3
- data/lib/aspera/cli/transfer_actions.rb +90 -0
- data/lib/aspera/cli/transfer_agent.rb +129 -64
- data/lib/aspera/cli/transfer_progress.rb +9 -9
- data/lib/aspera/cli/vault_manager.rb +79 -0
- data/lib/aspera/cli/version.rb +1 -1
- data/lib/aspera/cli/wizard.rb +28 -30
- data/lib/aspera/colors.rb +3 -3
- data/lib/aspera/command_line_builder.rb +34 -34
- data/lib/aspera/command_line_converter.rb +1 -1
- data/lib/aspera/coverage.rb +1 -2
- data/lib/aspera/data_repository.rb +2 -1
- data/lib/aspera/dot_container.rb +12 -12
- data/lib/aspera/environment.rb +30 -23
- data/lib/aspera/exec_spec.rb +13 -0
- data/lib/aspera/faspex_gw.rb +5 -5
- data/lib/aspera/faspex_postproc.rb +16 -10
- data/lib/aspera/graphql.rb +37 -0
- data/lib/aspera/hash_ext.rb +8 -2
- data/lib/aspera/json_rpc/client.rb +62 -0
- data/lib/aspera/json_rpc/version.rb +7 -0
- data/lib/aspera/keychain/base.rb +10 -3
- data/lib/aspera/keychain/encrypted_hash.rb +92 -17
- data/lib/aspera/keychain/factory.rb +20 -8
- data/lib/aspera/keychain/hashicorp_vault.rb +1 -1
- data/lib/aspera/keychain/macos_security.rb +23 -25
- data/lib/aspera/keychain/one_password_api.rb +86 -0
- data/lib/aspera/keychain/one_password_base.rb +34 -0
- data/lib/aspera/keychain/one_password_cli.rb +98 -0
- data/lib/aspera/link_header.rb +82 -0
- data/lib/aspera/log.rb +29 -15
- data/lib/aspera/markdown.rb +90 -7
- data/lib/aspera/nagios.rb +8 -8
- data/lib/aspera/node_simulator.rb +35 -27
- data/lib/aspera/oauth/base.rb +9 -10
- data/lib/aspera/oauth/boot.rb +5 -5
- data/lib/aspera/oauth/factory.rb +12 -10
- data/lib/aspera/oauth/jwt.rb +9 -11
- data/lib/aspera/oauth/web.rb +6 -6
- data/lib/aspera/persistency_action_once.rb +12 -12
- data/lib/aspera/persistency_folder.rb +18 -11
- data/lib/aspera/preview/file_types.rb +5 -5
- data/lib/aspera/preview/generator.rb +52 -49
- data/lib/aspera/preview/options.rb +3 -2
- data/lib/aspera/preview/terminal.rb +5 -5
- data/lib/aspera/preview/utils.rb +66 -19
- data/lib/aspera/products/connect.rb +2 -2
- data/lib/aspera/products/desktop.rb +1 -1
- data/lib/aspera/products/other.rb +3 -3
- data/lib/aspera/products/transferd.rb +4 -4
- data/lib/aspera/proxy_auto_config.rb +15 -14
- data/lib/aspera/rest.rb +159 -151
- data/lib/aspera/rest_error_analyzer.rb +6 -6
- data/lib/aspera/rest_errors_aspera.rb +0 -10
- data/lib/aspera/rest_list.rb +11 -10
- data/lib/aspera/schema/IBM Aspera Node API-4.4.6.yaml +6232 -0
- data/lib/aspera/schema/IBM Aspera faspio Gateway API-1.0.0.yaml +249 -0
- data/lib/aspera/schema/IBM Aspera on Cloud API-0.2.6-enhanced.yaml +1853 -510
- data/lib/aspera/schema/IBM_Aspera_Shares.yaml +4653 -0
- data/lib/aspera/schema/documentation.rb +65 -28
- data/lib/aspera/schema/reader.rb +136 -13
- data/lib/aspera/schema/registry.rb +60 -8
- data/lib/aspera/secret_hider.rb +12 -7
- data/lib/aspera/ssh.rb +64 -31
- data/lib/aspera/ssl.rb +5 -5
- data/lib/aspera/sync/conf.schema.yaml +2 -2
- data/lib/aspera/sync/database.rb +2 -2
- data/lib/aspera/sync/operations.rb +22 -25
- data/lib/aspera/temp_file_manager.rb +25 -5
- data/lib/aspera/timer_limiter.rb +1 -1
- data/lib/aspera/transfer/faux_file.rb +24 -11
- data/lib/aspera/transfer/parameters.rb +36 -34
- data/lib/aspera/transfer/result.rb +74 -0
- data/lib/aspera/transfer/resumer.rb +10 -10
- data/lib/aspera/transfer/spec.rb +18 -1
- data/lib/aspera/transfer/spec.schema.yaml +12 -3
- data/lib/aspera/uri_reader.rb +52 -15
- data/lib/aspera/web_auth.rb +8 -7
- data/lib/aspera/web_server_simple.rb +15 -12
- data/lib/aspera/yaml.rb +5 -4
- data.tar.gz.sig +0 -0
- metadata +34 -6
- metadata.gz.sig +0 -0
- data/lib/aspera/cli/manager.rb +0 -766
- data/lib/aspera/cli/plugins/faspex.rb +0 -539
- data/lib/aspera/json_rpc.rb +0 -52
- data/lib/aspera/transfer/uri.rb +0 -56
|
@@ -48,13 +48,30 @@ module Aspera
|
|
|
48
48
|
|
|
49
49
|
attr_accessor :option_skip_types, :option_previews_folder, :option_folder_reset_cache, :option_skip_folders, :option_overwrite
|
|
50
50
|
|
|
51
|
+
# DSL option declarations (Pattern 1 and Pattern 2 - handler: Symbol resolved to self)
|
|
52
|
+
option :skip_format, description: 'Skip this preview format', allowed: Aspera::Preview::Generator::PREVIEW_FORMATS
|
|
53
|
+
# TODO: use the same option as in `node` plugin
|
|
54
|
+
option :folder_reset_cache, description: 'Force detection of generated preview by refresh cache', allowed: %i[no header read], handler: :option_folder_reset_cache, default: :no
|
|
55
|
+
option :skip_types, description: 'Skip generation for those types of files', allowed: Type::SYMBOL_ARRAY + Aspera::Preview::FileTypes::CONVERSION_TYPES, handler: :option_skip_types
|
|
56
|
+
option :previews_folder, description: 'Preview folder in storage root', handler: :option_previews_folder, default: DEFAULT_PREVIEWS_FOLDER
|
|
57
|
+
option :skip_folders, description: 'List of folder to skip', allowed: Type::STRING_ARRAY, handler: :option_skip_folders
|
|
58
|
+
option :base, description: 'Basename of output for for test'
|
|
59
|
+
option :filter, description: 'File name filter: String (glob), Regexp, or Proc', allowed: [String, Regexp, Proc, NilClass]
|
|
60
|
+
option :mimemagic, description: 'Use Mime type detection of gem mimemagic', allowed: Type::BOOLEAN, default: false
|
|
61
|
+
option :overwrite, description: 'When to overwrite result file', allowed: %i[always never mtime], handler: :option_overwrite, default: :mtime
|
|
62
|
+
option :root_url, description: "How to read and write files on storage (#{REMOTE_ACCESS}, or #{UriReader.file_url('<folder>')})", default: REMOTE_ACCESS
|
|
63
|
+
# Generator-specific options (Category C - bound to @gen_options via set_handler in initialize)
|
|
64
|
+
Aspera::Preview::Options::DESCRIPTIONS.each do |opt|
|
|
65
|
+
values = if opt.key?(:values)
|
|
66
|
+
opt[:values]
|
|
67
|
+
elsif BoolValue.symbol?(opt[:default])
|
|
68
|
+
BoolValue::TYPES
|
|
69
|
+
end
|
|
70
|
+
option(opt[:name], description: opt[:description].capitalize, allowed: values, default: opt[:default])
|
|
71
|
+
end
|
|
72
|
+
|
|
51
73
|
def initialize(**_)
|
|
52
74
|
super
|
|
53
|
-
@option_skip_types = []
|
|
54
|
-
@option_skip_folders = []
|
|
55
|
-
@option_previews_folder = nil
|
|
56
|
-
@option_overwrite = nil
|
|
57
|
-
@option_folder_reset_cache = nil
|
|
58
75
|
# Generator configuration populated from CLI options.
|
|
59
76
|
@gen_options = Aspera::Preview::Options.new
|
|
60
77
|
# Used to rate-limit periodic progress logging and checkpoint persistence.
|
|
@@ -62,40 +79,9 @@ module Aspera
|
|
|
62
79
|
# Optional callback used to filter entries before generation.
|
|
63
80
|
@filter_block = nil
|
|
64
81
|
@access_remote = true
|
|
65
|
-
# Bind
|
|
66
|
-
options.declare(
|
|
67
|
-
:skip_format, 'Skip this preview format',
|
|
68
|
-
allowed: Aspera::Preview::Generator::PREVIEW_FORMATS
|
|
69
|
-
)
|
|
70
|
-
# TODO: use the same option as in `node` plugin
|
|
71
|
-
options.declare(
|
|
72
|
-
:folder_reset_cache, 'Force detection of generated preview by refresh cache',
|
|
73
|
-
allowed: %i[no header read],
|
|
74
|
-
handler: {o: self, m: :option_folder_reset_cache},
|
|
75
|
-
default: :no
|
|
76
|
-
)
|
|
77
|
-
options.declare(:skip_types, 'Skip generation for those types of files', handler: {o: self, m: :option_skip_types}, allowed: Allowed::TYPES_SYMBOL_ARRAY + Aspera::Preview::FileTypes::CONVERSION_TYPES)
|
|
78
|
-
options.declare(:previews_folder, 'Preview folder in storage root', handler: {o: self, m: :option_previews_folder}, default: DEFAULT_PREVIEWS_FOLDER)
|
|
79
|
-
options.declare(:skip_folders, 'List of folder to skip', handler: {o: self, m: :option_skip_folders}, allowed: Allowed::TYPES_STRING_ARRAY)
|
|
80
|
-
options.declare(:base, 'Basename of output for for test')
|
|
81
|
-
options.declare(:scan_path, 'Subpath in folder id to start scan in (default=/)')
|
|
82
|
-
options.declare(:scan_id, 'Folder id in storage to start scan in, default is access key main folder id')
|
|
83
|
-
options.declare(:mimemagic, 'Use Mime type detection of gem mimemagic', allowed: Allowed::TYPES_BOOLEAN, default: false)
|
|
84
|
-
options.declare(:overwrite, 'When to overwrite result file', handler: {o: self, m: :option_overwrite}, allowed: %i[always never mtime], default: :mtime)
|
|
85
|
-
options.declare(
|
|
86
|
-
:root_url,
|
|
87
|
-
"How to read and write files on storage (<empty>, #{REMOTE_ACCESS}, or #{UriReader.file_url('<folder>')})",
|
|
88
|
-
allowed: Allowed::TYPES_STRING,
|
|
89
|
-
default: ''
|
|
90
|
-
)
|
|
91
|
-
# Declare generator-specific options and apply their default values.
|
|
82
|
+
# Bind generator-specific options to @gen_options (Category C: set_handler after object creation)
|
|
92
83
|
Aspera::Preview::Options::DESCRIPTIONS.each do |opt|
|
|
93
|
-
|
|
94
|
-
opt[:values]
|
|
95
|
-
elsif BoolValue.symbol?(opt[:default])
|
|
96
|
-
BoolValue::TYPES
|
|
97
|
-
end
|
|
98
|
-
options.declare(opt[:name], opt[:description].capitalize, allowed: values, handler: {o: @gen_options, m: opt[:name]}, default: opt[:default])
|
|
84
|
+
options.set_handler(opt[:name], object: @gen_options, method: opt[:name])
|
|
99
85
|
end
|
|
100
86
|
|
|
101
87
|
options.parse_options!
|
|
@@ -106,13 +92,13 @@ module Aspera
|
|
|
106
92
|
@preview_formats_to_generate.delete(skip) if skip
|
|
107
93
|
@tmp_folder = File.join(TempFileManager.instance.global_temp, "#{TMP_DIR_PREFIX}.#{SecureRandom.uuid}")
|
|
108
94
|
FileUtils.mkdir_p(@tmp_folder)
|
|
109
|
-
Log.log.debug{"tmpdir: #{@tmp_folder}"}
|
|
95
|
+
Log.log.debug { "tmpdir: #{@tmp_folder}" }
|
|
110
96
|
end
|
|
111
97
|
|
|
112
98
|
# Process legacy transfer events and trigger preview generation for completed downloads.
|
|
113
99
|
#
|
|
114
100
|
# @param iteration_persistency [PersistencyActionOnce, nil] stores the last processed event id
|
|
115
|
-
# @return [
|
|
101
|
+
# @return [nil]
|
|
116
102
|
def process_trevents(iteration_persistency)
|
|
117
103
|
events_filter = {
|
|
118
104
|
'access_key' => @access_key_self['id'],
|
|
@@ -124,7 +110,7 @@ module Aspera
|
|
|
124
110
|
events = @api_node.read('events', events_filter)
|
|
125
111
|
rescue RestCallError => e
|
|
126
112
|
if e.message.include?('Invalid iteration_token')
|
|
127
|
-
Log.log.warn{"Retrying without iteration token: #{e}"}
|
|
113
|
+
Log.log.warn { "Retrying without iteration token: #{e}" }
|
|
128
114
|
events_filter.delete('iteration_token')
|
|
129
115
|
retry
|
|
130
116
|
end
|
|
@@ -145,19 +131,20 @@ module Aspera
|
|
|
145
131
|
end
|
|
146
132
|
# Periodically log progress and persist the latest processed event.
|
|
147
133
|
next unless @periodic.trigger? || event.equal?(events.last)
|
|
148
|
-
Log.log.debug{"Processed event #{event['id']}"}
|
|
134
|
+
Log.log.debug { "Processed event #{event['id']}" }
|
|
149
135
|
# Save a checkpoint to avoid replaying the full batch after a failure.
|
|
150
136
|
if !iteration_persistency.nil?
|
|
151
137
|
iteration_persistency.data[0] = event['id'].to_s
|
|
152
138
|
iteration_persistency.save
|
|
153
139
|
end
|
|
154
140
|
end
|
|
141
|
+
nil
|
|
155
142
|
end
|
|
156
143
|
|
|
157
144
|
# Process recent Node API file events since the last persisted checkpoint.
|
|
158
145
|
#
|
|
159
146
|
# @param iteration_persistency [PersistencyActionOnce, nil] stores the last processed event id
|
|
160
|
-
# @return [
|
|
147
|
+
# @return [nil]
|
|
161
148
|
def process_events(iteration_persistency)
|
|
162
149
|
# Restrict the event stream to file-related changes for the current access key.
|
|
163
150
|
events_filter = {
|
|
@@ -173,7 +160,7 @@ module Aspera
|
|
|
173
160
|
if event.dig('data', 'type').eql?('file')
|
|
174
161
|
file_entry = @api_node.read("files/#{event['data']['id']}") rescue nil
|
|
175
162
|
if !file_entry.nil? &&
|
|
176
|
-
@option_skip_folders.none?{ |d| file_entry['path'].start_with?(d)}
|
|
163
|
+
@option_skip_folders.none? { |d| file_entry['path'].start_with?(d) }
|
|
177
164
|
file_entry['parent_file_id'] = event['data']['parent_file_id']
|
|
178
165
|
Log.log.error('TODO'.red) if event['types'].include?('file.deleted')
|
|
179
166
|
generate_preview(file_entry) if event['types'].include?('file.deleted')
|
|
@@ -181,13 +168,14 @@ module Aspera
|
|
|
181
168
|
end
|
|
182
169
|
# Periodically log progress and persist the latest processed event.
|
|
183
170
|
next unless @periodic.trigger? || event.equal?(events.last)
|
|
184
|
-
Log.log.debug{"Processing event #{event['id']}"}
|
|
171
|
+
Log.log.debug { "Processing event #{event['id']}" }
|
|
185
172
|
# Save a checkpoint to avoid replaying the full batch after a failure.
|
|
186
173
|
if !iteration_persistency.nil?
|
|
187
174
|
iteration_persistency.data[0] = event['id'].to_s
|
|
188
175
|
iteration_persistency.save
|
|
189
176
|
end
|
|
190
177
|
end
|
|
178
|
+
nil
|
|
191
179
|
end
|
|
192
180
|
|
|
193
181
|
# Transfer a file to or from the configured Node storage using a tagged transfer spec.
|
|
@@ -215,7 +203,11 @@ module Aspera
|
|
|
215
203
|
# @param entry [Hash] file entry containing at least the relative path
|
|
216
204
|
# @return [String] local directory where previews for this entry are stored
|
|
217
205
|
def get_infos_local(gen_infos, entry)
|
|
218
|
-
|
|
206
|
+
# Resolve to a real path and ensure it stays within the storage root (prevents path traversal via API response)
|
|
207
|
+
candidate = File.expand_path(File.join(@local_storage_root, entry['path']))
|
|
208
|
+
resolved_root = File.realpath(@local_storage_root)
|
|
209
|
+
Aspera.assert(candidate.start_with?("#{resolved_root}/")) { 'Preview entry path traversal attempt detected' }
|
|
210
|
+
local_original_filepath = candidate
|
|
219
211
|
original_mtime = File.mtime(local_original_filepath)
|
|
220
212
|
# Output directory for previews generated from the local source file.
|
|
221
213
|
local_entry_preview_dir = File.join(@local_preview_folder, entry_preview_folder_name(entry))
|
|
@@ -278,7 +270,7 @@ module Aspera
|
|
|
278
270
|
# Remote entries must include `parent_file_id`.
|
|
279
271
|
#
|
|
280
272
|
# @param entry [Hash] local or remote file entry to preview
|
|
281
|
-
# @return [
|
|
273
|
+
# @return [nil]
|
|
282
274
|
def generate_preview(entry)
|
|
283
275
|
# prepare generic information
|
|
284
276
|
gen_infos = @preview_formats_to_generate.map do |preview_format|
|
|
@@ -327,14 +319,14 @@ module Aspera
|
|
|
327
319
|
# download original file to temp folder
|
|
328
320
|
do_transfer(Transfer::Spec::DIRECTION_RECEIVE, entry['parent_file_id'], entry['name'], @tmp_folder)
|
|
329
321
|
end
|
|
330
|
-
Log.log.debug{"source: #{entry['id']}: #{entry['path']}"}
|
|
322
|
+
Log.log.debug { "source: #{entry['id']}: #{entry['path']}" }
|
|
331
323
|
gen_infos.each do |gen_info|
|
|
332
324
|
gen_info[:generator].generate
|
|
333
325
|
rescue => e
|
|
334
|
-
Log.log.error{"Ignoring: #{e.class} #{e.message}"}
|
|
326
|
+
Log.log.error { "Ignoring: #{e.class} #{e.message}" }
|
|
335
327
|
Log.log.debug(e.backtrace.join("\n").red)
|
|
336
328
|
# in case of any error, place a standard error image
|
|
337
|
-
FileUtils.cp(gen_info[:generator].error_asset,
|
|
329
|
+
FileUtils.cp(gen_info[:generator].error_asset, gen_info[:dst])
|
|
338
330
|
end
|
|
339
331
|
if @access_remote
|
|
340
332
|
# upload
|
|
@@ -346,7 +338,7 @@ module Aspera
|
|
|
346
338
|
# force read file updated previews
|
|
347
339
|
@api_node.read("files/#{entry['id']}") if @option_folder_reset_cache.eql?(:read)
|
|
348
340
|
rescue StandardError => e
|
|
349
|
-
Log.log.error{"Ignore: #{e.message}"}
|
|
341
|
+
Log.log.error { "Ignore: #{e.message}" }
|
|
350
342
|
Log.log.debug(e.backtrace.join("\n").red)
|
|
351
343
|
end
|
|
352
344
|
|
|
@@ -358,20 +350,20 @@ module Aspera
|
|
|
358
350
|
# canonical path: start with / and ends with /
|
|
359
351
|
top_path = "/#{top_path.split('/').reject(&:empty?).join('/')}/"
|
|
360
352
|
end
|
|
361
|
-
Log.log.debug{"scan: #{top_entry} : #{top_path}".green}
|
|
353
|
+
Log.log.debug { "scan: #{top_entry} : #{top_path}".green }
|
|
362
354
|
# don't use recursive call, use list instead
|
|
363
355
|
entries_to_process = [top_entry]
|
|
364
356
|
until entries_to_process.empty?
|
|
365
357
|
entry = entries_to_process.shift
|
|
366
358
|
# process this entry only if it is within the top_path
|
|
367
359
|
entry_path_with_slash = entry['path']
|
|
368
|
-
Log.log.debug{"processing entry #{entry_path_with_slash}"} if @periodic.trigger?
|
|
360
|
+
Log.log.debug { "processing entry #{entry_path_with_slash}" } if @periodic.trigger?
|
|
369
361
|
entry_path_with_slash = "#{entry_path_with_slash}/" unless entry_path_with_slash.end_with?('/')
|
|
370
362
|
if !top_path.nil? && !top_path.start_with?(entry_path_with_slash) && !entry_path_with_slash.start_with?(top_path)
|
|
371
|
-
Log.log.debug{"#{entry['path']} folder (skip start)".bg_red}
|
|
363
|
+
Log.log.debug { "#{entry['path']} folder (skip start)".bg_red }
|
|
372
364
|
next
|
|
373
365
|
end
|
|
374
|
-
Log.log.debug{"item:#{entry}"}
|
|
366
|
+
Log.log.debug { "item:#{entry}" }
|
|
375
367
|
begin
|
|
376
368
|
case entry['type']
|
|
377
369
|
when 'file'
|
|
@@ -384,9 +376,9 @@ module Aspera
|
|
|
384
376
|
Log.log.debug('Ignoring link.')
|
|
385
377
|
when 'folder'
|
|
386
378
|
if @option_skip_folders.include?(entry['path'])
|
|
387
|
-
Log.log.debug{"#{entry['path']} folder (skip list)".bg_red}
|
|
379
|
+
Log.log.debug { "#{entry['path']} folder (skip list)".bg_red }
|
|
388
380
|
else
|
|
389
|
-
Log.log.debug{"#{entry['path']} folder".green}
|
|
381
|
+
Log.log.debug { "#{entry['path']} folder".green }
|
|
390
382
|
# get folder content
|
|
391
383
|
folder_entries = @api_node.read_folder_content(entry['id'])
|
|
392
384
|
# process all items in current folder
|
|
@@ -398,120 +390,166 @@ module Aspera
|
|
|
398
390
|
end
|
|
399
391
|
end
|
|
400
392
|
else
|
|
401
|
-
Log.log.warn{"unknown entry type: #{entry['type']}"}
|
|
393
|
+
Log.log.warn { "unknown entry type: #{entry['type']}" }
|
|
402
394
|
end
|
|
403
395
|
rescue StandardError => e
|
|
404
|
-
Log.log.warn{"An error occurred: #{e}, ignoring"}
|
|
396
|
+
Log.log.warn { "An error occurred: #{e}, ignoring" }
|
|
405
397
|
end
|
|
406
398
|
end
|
|
407
399
|
end
|
|
408
400
|
|
|
409
|
-
|
|
401
|
+
# --- DSL ---
|
|
402
|
+
|
|
403
|
+
# scan, events and trevents connect to the Node API (setup: :setup_node_api).
|
|
404
|
+
# `check`, `test` and `show` work without Node API authentication.
|
|
405
|
+
command :scan, description: 'Scan a file or folder and generate previews', setup: :setup_node_api, arguments: [{name: :path, type: String, description: 'File or folder from which to start scanning', mandatory: false, default: '/'}]
|
|
406
|
+
command :events, description: 'Process file events and generate previews', setup: :setup_node_api
|
|
407
|
+
command :trevents, description: 'Process transfer events and generate previews', setup: :setup_node_api
|
|
408
|
+
command :check, description: 'Check required tools are installed'
|
|
409
|
+
command :show, description: 'Generate and display preview of a source file', arguments: [{name: :source_file, type: String, description: 'Local file to preview'}]
|
|
410
|
+
command :test, description: 'Test preview generation for a source file',
|
|
411
|
+
arguments: [
|
|
412
|
+
{name: :source_file, type: String, description: 'Local file to preview'},
|
|
413
|
+
{name: :format, description: 'Preview output format', allowed: Aspera::Preview::Generator::PREVIEW_FORMATS, mandatory: false, default: :png}
|
|
414
|
+
]
|
|
415
|
+
|
|
416
|
+
# --- setup ---
|
|
410
417
|
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
418
|
+
# Build the Node API and set up storage references for scan/events/trevents.
|
|
419
|
+
# @return [Hash] empty ctx (state stored in instance variables)
|
|
420
|
+
def setup_node_api(**)
|
|
421
|
+
@api_node = Api::Node.new(**basic_auth_params)
|
|
422
|
+
@transfer_server_address = URI.parse(@api_node.base_url).host
|
|
423
|
+
# Get current access key information
|
|
424
|
+
@access_key_self = @api_node.read('access_keys/self')
|
|
425
|
+
# TODO: check events is activated here:
|
|
426
|
+
# note that docroot is good to look at as well
|
|
427
|
+
node_info = @api_node.read('info')
|
|
428
|
+
Log.log.debug { "root: #{node_info['docroot']}" }
|
|
429
|
+
# Default storage url to local file if not provided
|
|
430
|
+
option_root_url = options.get_option(:root_url, mandatory: true)
|
|
431
|
+
option_root_url = UriReader.file_url(@access_key_self['storage']['path']) if option_root_url.eql?(REMOTE_ACCESS) && @access_key_self['storage']['type'].eql?('local')
|
|
432
|
+
@access_remote = !UriReader.file?(option_root_url)
|
|
433
|
+
Log.log.debug { "remote: #{@access_remote}" }
|
|
434
|
+
# TODO: can the `previews` folder parameter be read from Node API ?
|
|
435
|
+
@option_skip_folders.push("/#{@option_previews_folder}")
|
|
436
|
+
if @access_remote
|
|
437
|
+
# NOTE: the filter "name", it's why we take the first one
|
|
438
|
+
@previews_folder_entry = @api_node.read_folder_content(@access_key_self['root_file_id'], {name: @option_previews_folder}).first
|
|
439
|
+
raise Cli::Error, "Folder #{@option_previews_folder} does not exist on node. " \
|
|
440
|
+
'Please create it in the storage root, or specify an alternate name.' if @previews_folder_entry.nil?
|
|
441
|
+
else
|
|
442
|
+
@local_storage_root = UriReader.file_path(option_root_url)
|
|
443
|
+
# TODO: Windows could have "C:" ?
|
|
444
|
+
Aspera.assert(@local_storage_root.start_with?('/')) { "not local storage: #{@local_storage_root}" }
|
|
445
|
+
Aspera.assert(File.directory?(@local_storage_root), type: Cli::Error) { "Local storage root folder #{@local_storage_root} does not exist." }
|
|
446
|
+
@local_preview_folder = File.join(@local_storage_root, @option_previews_folder)
|
|
447
|
+
Aspera.assert(File.directory?(@local_preview_folder), type: Cli::Error) { "Folder #{@local_preview_folder} does not exist locally. Please create it, or specify an alternate name." }
|
|
448
|
+
# Protection to avoid clash of file id for two different access keys
|
|
449
|
+
marker_file = File.join(@local_preview_folder, AK_MARKER_FILE)
|
|
450
|
+
Log.log.debug { "marker file: #{marker_file}" }
|
|
451
|
+
if File.exist?(marker_file)
|
|
452
|
+
ak = File.read(marker_file).chomp
|
|
453
|
+
Aspera.assert(@access_key_self['id'].eql?(ak)) { "mismatch access key in #{marker_file}: contains #{ak}, using #{@access_key_self['id']}" }
|
|
435
454
|
else
|
|
436
|
-
|
|
437
|
-
# TODO: Windows could have "C:" ?
|
|
438
|
-
Aspera.assert(@local_storage_root.start_with?('/')){"not local storage: #{@local_storage_root}"}
|
|
439
|
-
Aspera.assert(File.directory?(@local_storage_root), type: Cli::Error){"Local storage root folder #{@local_storage_root} does not exist."}
|
|
440
|
-
@local_preview_folder = File.join(@local_storage_root, @option_previews_folder)
|
|
441
|
-
Aspera.assert(File.directory?(@local_preview_folder), type: Cli::Error){"Folder #{@local_preview_folder} does not exist locally. Please create it, or specify an alternate name."}
|
|
442
|
-
# Protection to avoid clash of file id for two different access keys
|
|
443
|
-
marker_file = File.join(@local_preview_folder, AK_MARKER_FILE)
|
|
444
|
-
Log.log.debug{"marker file: #{marker_file}"}
|
|
445
|
-
if File.exist?(marker_file)
|
|
446
|
-
ak = File.read(marker_file).chomp
|
|
447
|
-
Aspera.assert(@access_key_self['id'].eql?(ak)){"mismatch access key in #{marker_file}: contains #{ak}, using #{@access_key_self['id']}"}
|
|
448
|
-
else
|
|
449
|
-
File.write(marker_file, @access_key_self['id'])
|
|
450
|
-
end
|
|
455
|
+
File.write(marker_file, @access_key_self['id'])
|
|
451
456
|
end
|
|
452
457
|
end
|
|
458
|
+
{}
|
|
459
|
+
end
|
|
460
|
+
|
|
461
|
+
# --- handlers ---
|
|
462
|
+
|
|
463
|
+
def action_scan(path:, **)
|
|
464
|
+
check_tools_and_mimemagic
|
|
465
|
+
apifid =
|
|
466
|
+
if (selector = Parser.percent_selector(path))
|
|
467
|
+
Aspera.assert_values(selector[:field], ['id'], type: BadArgument) { 'file id' }
|
|
468
|
+
Api::NodeFileId.new(@api_node, selector[:value].to_s.empty? ? @access_key_self['root_file_id'] : selector[:value])
|
|
469
|
+
else
|
|
470
|
+
@api_node.resolve_api_fid(@access_key_self['root_file_id'], path)
|
|
471
|
+
end
|
|
472
|
+
@filter_block = Base.file_matcher(options.get_option(:filter))
|
|
473
|
+
scan_folder_files(@api_node.read("files/#{apifid.file_id}"))
|
|
474
|
+
Result::Status.new('scan finished')
|
|
475
|
+
ensure
|
|
476
|
+
cleanup_tmp_folder
|
|
477
|
+
end
|
|
478
|
+
|
|
479
|
+
def action_events(**)
|
|
480
|
+
check_tools_and_mimemagic
|
|
481
|
+
run_event_loop(:events, filter: options.get_option(:filter))
|
|
482
|
+
ensure
|
|
483
|
+
cleanup_tmp_folder
|
|
484
|
+
end
|
|
485
|
+
|
|
486
|
+
def action_trevents(**)
|
|
487
|
+
check_tools_and_mimemagic
|
|
488
|
+
run_event_loop(:trevents, filter: options.get_option(:filter))
|
|
489
|
+
ensure
|
|
490
|
+
cleanup_tmp_folder
|
|
491
|
+
end
|
|
492
|
+
|
|
493
|
+
def action_check
|
|
494
|
+
check_tools_and_mimemagic
|
|
495
|
+
Result::Status.new('Tools validated')
|
|
496
|
+
ensure
|
|
497
|
+
cleanup_tmp_folder
|
|
498
|
+
end
|
|
499
|
+
|
|
500
|
+
def action_test(source_file:, format:, **)
|
|
501
|
+
check_tools_and_mimemagic
|
|
502
|
+
generated_file_path = preview_filename(format, options.get_option(:base))
|
|
503
|
+
Aspera::Preview::Generator.new(source_file, generated_file_path, @gen_options, @tmp_folder).generate
|
|
504
|
+
Result::Status.new("generated: #{generated_file_path}")
|
|
505
|
+
ensure
|
|
506
|
+
cleanup_tmp_folder
|
|
507
|
+
end
|
|
508
|
+
|
|
509
|
+
def action_show(source_file:, **)
|
|
510
|
+
check_tools_and_mimemagic
|
|
511
|
+
generated_file_path = preview_filename(:png, options.get_option(:base))
|
|
512
|
+
Aspera::Preview::Generator.new(source_file, generated_file_path, @gen_options, @tmp_folder).generate
|
|
513
|
+
formatter.display_status("generated: #{generated_file_path}")
|
|
514
|
+
Result::Image.new(UriReader.file_url(generated_file_path))
|
|
515
|
+
ensure
|
|
516
|
+
cleanup_tmp_folder
|
|
517
|
+
end
|
|
518
|
+
|
|
519
|
+
private
|
|
520
|
+
|
|
521
|
+
# Check tools and set mimemagic flag (required for all commands).
|
|
522
|
+
def check_tools_and_mimemagic
|
|
453
523
|
Aspera::Preview::FileTypes.instance.use_mimemagic = options.get_option(:mimemagic, mandatory: true)
|
|
454
|
-
# check tools that are anyway required for all cases
|
|
455
524
|
Aspera::Preview::Utils.check_tools(@option_skip_types)
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
iteration_persistency = PersistencyActionOnce.new(
|
|
480
|
-
manager: persistency,
|
|
481
|
-
data: [],
|
|
482
|
-
id: IdGenerator.from_list(
|
|
483
|
-
'preview_iteration',
|
|
484
|
-
command.to_s,
|
|
485
|
-
options.get_option(:url, mandatory: true),
|
|
486
|
-
options.get_option(:username, mandatory: true)
|
|
487
|
-
)
|
|
525
|
+
end
|
|
526
|
+
|
|
527
|
+
# Clean up the temporary folder after each command.
|
|
528
|
+
def cleanup_tmp_folder
|
|
529
|
+
Log.log.debug { "cleaning up temp folder #{@tmp_folder}" }
|
|
530
|
+
FileUtils.rm_rf(@tmp_folder)
|
|
531
|
+
end
|
|
532
|
+
|
|
533
|
+
# Shared event-loop body for :events and :trevents.
|
|
534
|
+
# @param command [:events, :trevents]
|
|
535
|
+
# @param filter [String, Regexp, Proc, nil] optional filter expression
|
|
536
|
+
def run_event_loop(command, filter: nil)
|
|
537
|
+
@filter_block = Base.file_matcher(filter)
|
|
538
|
+
iteration_persistency = nil
|
|
539
|
+
if options.get_option(:once_only, mandatory: true)
|
|
540
|
+
iteration_persistency = PersistencyActionOnce.new(
|
|
541
|
+
manager: persistency,
|
|
542
|
+
data: [],
|
|
543
|
+
id: IdGenerator.from_list(
|
|
544
|
+
'preview_iteration',
|
|
545
|
+
command.to_s,
|
|
546
|
+
options.get_option(:url, mandatory: true),
|
|
547
|
+
options.get_option(:username, mandatory: true)
|
|
488
548
|
)
|
|
489
|
-
|
|
490
|
-
# call processing method specified by command line command
|
|
491
|
-
send(:"process_#{command}", iteration_persistency)
|
|
492
|
-
return Result::Status.new("#{command} finished")
|
|
493
|
-
when :check
|
|
494
|
-
return Result::Status.new('Tools validated')
|
|
495
|
-
when :test
|
|
496
|
-
source = options.get_next_argument('source file')
|
|
497
|
-
format = options.get_next_argument('format', accept_list: Aspera::Preview::Generator::PREVIEW_FORMATS, default: :png)
|
|
498
|
-
generated_file_path = preview_filename(format, options.get_option(:base))
|
|
499
|
-
Aspera::Preview::Generator.new(source, generated_file_path, @gen_options, @tmp_folder).generate
|
|
500
|
-
return Result::Status.new("generated: #{generated_file_path}")
|
|
501
|
-
when :show
|
|
502
|
-
source = options.get_next_argument('source file')
|
|
503
|
-
# terminal_options = options.get_next_argument('options', validation: Hash, default: {}).symbolize_keys
|
|
504
|
-
generated_file_path = preview_filename(:png, options.get_option(:base))
|
|
505
|
-
Aspera::Preview::Generator.new(source, generated_file_path, @gen_options, @tmp_folder).generate
|
|
506
|
-
formatter.display_status("generated: #{generated_file_path}")
|
|
507
|
-
# formatter.display_status(Aspera::Preview::Terminal.build(File.read(generated_file_path), **terminal_options))
|
|
508
|
-
# return Result::Status.new("generated: #{generated_file_path}")
|
|
509
|
-
return Result::Image.new(UriReader.file_url(generated_file_path))
|
|
510
|
-
else Aspera.error_unexpected_value(command)
|
|
549
|
+
)
|
|
511
550
|
end
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
FileUtils.rm_rf(@tmp_folder)
|
|
551
|
+
send(:"process_#{command}", iteration_persistency)
|
|
552
|
+
Result::Status.new("#{command} finished")
|
|
515
553
|
end
|
|
516
554
|
end
|
|
517
555
|
end
|