aspera-cli 4.28.0 → 4.29.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/BUGS.md +5 -10
- data/CHANGELOG.md +217 -2
- data/CODE_OF_CONDUCT.md +18 -0
- data/CONTRIBUTING.md +135 -56
- data/GOVERNANCE.md +30 -0
- data/LICENSE +201 -0
- data/README.md +122 -26
- data/SECURITY.md +84 -0
- data/bin/ascli +8 -5
- data/bin/asession +9 -6
- data/docs/README.md +1895 -632
- data/docs/test-mcp-with-ai.md +8 -8
- data/lib/aspera/agent/base.rb +23 -2
- data/lib/aspera/agent/connect.rb +16 -12
- data/lib/aspera/agent/desktop.rb +10 -7
- data/lib/aspera/agent/direct.rb +20 -19
- data/lib/aspera/agent/factory.rb +6 -3
- data/lib/aspera/agent/httpgw.rb +7 -5
- data/lib/aspera/agent/node.rb +15 -7
- data/lib/aspera/agent/transferd.rb +167 -43
- data/lib/aspera/api/alee.rb +4 -1
- data/lib/aspera/api/aoc.rb +14 -11
- data/lib/aspera/api/ats.rb +4 -2
- data/lib/aspera/api/cos_node.rb +7 -4
- data/lib/aspera/api/faspex.rb +16 -9
- data/lib/aspera/api/httpgw.rb +9 -6
- data/lib/aspera/api/node.rb +29 -22
- data/lib/aspera/ascmd.rb +8 -5
- data/lib/aspera/ascp/installation.rb +95 -67
- data/lib/aspera/ascp/log.schema.yaml +1084 -0
- data/lib/aspera/ascp/log_analyzer/advice.rb +97 -0
- data/lib/aspera/ascp/log_analyzer/diagnostics.rb +344 -0
- data/lib/aspera/ascp/log_analyzer/lines.rb +75 -0
- data/lib/aspera/ascp/log_analyzer/log_file.rb +593 -0
- data/lib/aspera/ascp/log_analyzer/records.rb +379 -0
- data/lib/aspera/ascp/log_analyzer/rules.rb +172 -0
- data/lib/aspera/ascp/log_analyzer/session.rb +782 -0
- data/lib/aspera/ascp/log_analyzer.rb +370 -0
- data/lib/aspera/ascp/log_locations.rb +163 -0
- data/lib/aspera/ascp/management.rb +16 -8
- data/lib/aspera/ascp/session_errors.rb +217 -0
- data/lib/aspera/assert.rb +4 -0
- data/lib/aspera/cli/ascp_actions.rb +12 -19
- data/lib/aspera/cli/ascp_logs_actions.rb +188 -0
- data/lib/aspera/cli/async_transfer_store.rb +7 -4
- data/lib/aspera/cli/bootstrapper.rb +38 -33
- data/lib/aspera/cli/command_line.rb +63 -8
- data/lib/aspera/cli/command_registry.rb +9 -1
- data/lib/aspera/cli/command_spec.rb +47 -7
- data/lib/aspera/cli/completion/ascli.bash +19 -5
- data/lib/aspera/cli/completion/ascli.fish +13 -2
- data/lib/aspera/cli/completion/ascli.ps1 +31 -9
- data/lib/aspera/cli/completion/ascli.zsh +22 -6
- data/lib/aspera/cli/completion_actions.rb +303 -69
- data/lib/aspera/cli/context.rb +9 -4
- data/lib/aspera/cli/deprecation.rb +5 -2
- data/lib/aspera/cli/error.rb +6 -0
- data/lib/aspera/cli/extended_value.rb +66 -31
- data/lib/aspera/cli/formatter.rb +43 -30
- data/lib/aspera/cli/gem_checker.rb +9 -6
- data/lib/aspera/cli/hints.rb +22 -10
- data/lib/aspera/cli/http.rb +38 -10
- data/lib/aspera/cli/info.rb +12 -7
- data/lib/aspera/cli/mailer.rb +6 -3
- data/lib/aspera/cli/mcp/http_servlet.rb +7 -8
- data/lib/aspera/cli/mcp/secrets.rb +8 -5
- data/lib/aspera/cli/mcp/tool.rb +20 -17
- data/lib/aspera/cli/option_declarator.rb +10 -4
- data/lib/aspera/cli/option_registry.rb +6 -3
- data/lib/aspera/cli/option_types.rb +17 -2
- data/lib/aspera/cli/option_value.rb +77 -18
- data/lib/aspera/cli/options.schema.yaml +641 -80
- data/lib/aspera/cli/parser.rb +71 -35
- data/lib/aspera/cli/plugins/alee.rb +7 -4
- data/lib/aspera/cli/plugins/aoc.rb +248 -37
- data/lib/aspera/cli/plugins/ats.rb +10 -9
- data/lib/aspera/cli/plugins/base.rb +50 -18
- data/lib/aspera/cli/plugins/basic_auth.rb +5 -2
- data/lib/aspera/cli/plugins/config.rb +210 -104
- data/lib/aspera/cli/plugins/console.rb +6 -3
- data/lib/aspera/cli/plugins/cos.rb +8 -4
- data/lib/aspera/cli/plugins/factory.rb +21 -7
- data/lib/aspera/cli/plugins/faspex5.rb +31 -26
- data/lib/aspera/cli/plugins/faspio.rb +7 -5
- data/lib/aspera/cli/plugins/httpgw.rb +7 -4
- data/lib/aspera/cli/plugins/mcp.rb +42 -22
- data/lib/aspera/cli/plugins/node.rb +39 -38
- data/lib/aspera/cli/plugins/oauth.rb +23 -6
- data/lib/aspera/cli/plugins/orchestrator.rb +287 -83
- data/lib/aspera/cli/plugins/preview.rb +72 -45
- data/lib/aspera/cli/plugins/server.rb +66 -14
- data/lib/aspera/cli/plugins/shares.rb +87 -18
- data/lib/aspera/cli/plugins/transferd.rb +228 -0
- data/lib/aspera/cli/poll.rb +236 -0
- data/lib/aspera/cli/preset_actions.rb +144 -30
- data/lib/aspera/cli/preset_manager.rb +113 -35
- data/lib/aspera/cli/prompt.rb +157 -9
- data/lib/aspera/cli/result.rb +12 -8
- data/lib/aspera/cli/runner.rb +89 -48
- data/lib/aspera/cli/secret_finder.rb +4 -1
- data/lib/aspera/cli/security.rb +89 -9
- data/lib/aspera/cli/special_values.rb +4 -1
- data/lib/aspera/cli/sync_actions.rb +15 -10
- data/lib/aspera/cli/table_fitter.rb +3 -0
- data/lib/aspera/cli/terminal_formatter.rb +9 -6
- data/lib/aspera/cli/transfer_actions.rb +11 -8
- data/lib/aspera/cli/transfer_agent.rb +43 -20
- data/lib/aspera/cli/transfer_progress.rb +14 -45
- data/lib/aspera/cli/vault_actions.rb +45 -0
- data/lib/aspera/cli/vault_manager.rb +188 -42
- data/lib/aspera/cli/version.rb +4 -1
- data/lib/aspera/cli/web_secret.rb +12 -10
- data/lib/aspera/cli/wizard.rb +42 -39
- data/lib/aspera/command_line_builder.rb +9 -5
- data/lib/aspera/command_line_converter.rb +4 -1
- data/lib/aspera/coverage.rb +103 -33
- data/lib/aspera/data_repository.rb +4 -1
- data/lib/aspera/dot_container.rb +4 -1
- data/lib/aspera/environment.rb +23 -12
- data/lib/aspera/exec_spec.rb +3 -0
- data/lib/aspera/faspex_gw.rb +13 -9
- data/lib/aspera/faspex_postproc.rb +15 -12
- data/lib/aspera/formatter_interface.rb +4 -1
- data/lib/aspera/graphql.rb +5 -1
- data/lib/aspera/hash_ext.rb +3 -0
- data/lib/aspera/id_generator.rb +20 -5
- data/lib/aspera/inflate.rb +43 -0
- data/lib/aspera/json_rpc/client.rb +6 -5
- data/lib/aspera/json_rpc/version.rb +3 -0
- data/lib/aspera/line_logger.rb +4 -1
- data/lib/aspera/link_header.rb +4 -1
- data/lib/aspera/log.rb +18 -5
- data/lib/aspera/markdown.rb +5 -2
- data/lib/aspera/mime.rb +3 -0
- data/lib/aspera/nagios.rb +6 -4
- data/lib/aspera/node_emulator.rb +15 -13
- data/lib/aspera/oauth/client/base.rb +124 -73
- data/lib/aspera/oauth/client/boot.rb +7 -4
- data/lib/aspera/oauth/client/factory.rb +41 -18
- data/lib/aspera/oauth/client/generic.rb +5 -1
- data/lib/aspera/oauth/client/json_credentials.rb +7 -3
- data/lib/aspera/oauth/client/jwt.rb +6 -3
- data/lib/aspera/oauth/client/shares_web.rb +94 -0
- data/lib/aspera/oauth/client/url_json.rb +7 -3
- data/lib/aspera/oauth/client/web.rb +40 -13
- data/lib/aspera/oauth/server/pkce.rb +6 -5
- data/lib/aspera/orchestrator_workflow_checker.rb +129 -2
- data/lib/aspera/persistency_action_once.rb +5 -3
- data/lib/aspera/persistency_folder.rb +6 -3
- data/lib/aspera/preview/file_types.rb +6 -3
- data/lib/aspera/preview/generator.rb +9 -5
- data/lib/aspera/preview/options.schema.yaml +3 -3
- data/lib/aspera/preview/terminal.rb +16 -6
- data/lib/aspera/preview/utils.rb +6 -4
- data/lib/aspera/products/connect.rb +5 -2
- data/lib/aspera/products/desktop.rb +17 -2
- data/lib/aspera/products/other.rb +4 -2
- data/lib/aspera/products/transferd.rb +6 -2
- data/lib/aspera/products/transferd_service.rb +610 -0
- data/lib/aspera/proxy_auto_config.js +2 -2
- data/lib/aspera/proxy_auto_config.rb +5 -2
- data/lib/aspera/rainbow.rb +3 -0
- data/lib/aspera/rest/aspera_errors.rb +4 -1
- data/lib/aspera/rest/call_error.rb +3 -0
- data/lib/aspera/rest/client.rb +45 -34
- data/lib/aspera/rest/error_analyzer.rb +5 -2
- data/lib/aspera/rest/list.rb +8 -4
- data/lib/aspera/rest/parameters.rb +3 -0
- data/lib/aspera/rest/util.rb +8 -5
- data/lib/aspera/rest.rb +9 -6
- data/lib/aspera/schema/IBM Aspera Faspex API-5.0-enhanced.yaml +9 -9
- data/lib/aspera/schema/IBM Aspera Orchestrator API-v1-enhanced.yaml +118 -30
- data/lib/aspera/schema/IBM Aspera on Cloud API-0.2.6-enhanced.yaml +11 -11
- data/lib/aspera/schema/IBM Aspera on Cloud Automation API-1.0.5-enhanced.yaml +2437 -1437
- data/lib/aspera/schema/async_tables.yaml +11 -10
- data/lib/aspera/schema/documentation.rb +8 -2
- data/lib/aspera/schema/reader.rb +7 -0
- data/lib/aspera/schema/registry.rb +44 -28
- data/lib/aspera/schema/transferd-config.schema.json +333 -0
- data/lib/aspera/schema/validator.rb +4 -1
- data/lib/aspera/sealed_hash.rb +150 -0
- data/lib/aspera/secret/hider.rb +213 -0
- data/lib/aspera/secret/reference.rb +150 -0
- data/lib/aspera/secret/resolver.rb +212 -0
- data/lib/aspera/secret/store/base.rb +153 -0
- data/lib/aspera/secret/store/encrypted_hash.rb +329 -0
- data/lib/aspera/secret/store/factory.rb +103 -0
- data/lib/aspera/secret/store/hashicorp_vault.rb +390 -0
- data/lib/aspera/secret/store/keeper.rb +141 -0
- data/lib/aspera/secret/store/macos_security.rb +284 -0
- data/lib/aspera/secret/store/one_password_api.rb +99 -0
- data/lib/aspera/secret/store/one_password_base.rb +42 -0
- data/lib/aspera/secret/store/one_password_cli.rb +112 -0
- data/lib/aspera/secret/store/secret_service.rb +125 -0
- data/lib/aspera/secret/strength.rb +119 -0
- data/lib/aspera/servlet_helpers.rb +5 -2
- data/lib/aspera/ssh.rb +10 -7
- data/lib/aspera/ssl.rb +5 -2
- data/lib/aspera/string_ext.rb +3 -0
- data/lib/aspera/sync/args.schema.yaml +3 -0
- data/lib/aspera/sync/conf.schema.yaml +9 -4
- data/lib/aspera/sync/database.rb +3 -0
- data/lib/aspera/sync/operations.rb +17 -15
- data/lib/aspera/temp_file_manager.rb +8 -4
- data/lib/aspera/timer_limiter.rb +3 -0
- data/lib/aspera/transfer/error.rb +4 -1
- data/lib/aspera/transfer/faux_file.rb +4 -1
- data/lib/aspera/transfer/parameters.rb +17 -14
- data/lib/aspera/transfer/result.rb +23 -0
- data/lib/aspera/transfer/resumer.rb +6 -4
- data/lib/aspera/transfer/spec.rb +9 -22
- data/lib/aspera/units/decimal.rb +67 -0
- data/lib/aspera/units/duration.rb +109 -0
- data/lib/aspera/uri_reader.rb +64 -9
- data/lib/aspera/web_auth/index.html.erb +6 -2
- data/lib/aspera/web_auth/paste.rb +109 -0
- data/lib/aspera/web_auth/receiver.rb +50 -0
- data/lib/aspera/web_auth/relay.rb +76 -0
- data/lib/aspera/web_auth/reply.html.erb +150 -0
- data/lib/aspera/web_auth/server.rb +47 -10
- data/lib/aspera/web_page.rb +7 -2
- data/lib/aspera/web_server_simple.rb +6 -3
- data/lib/aspera/yaml.rb +5 -1
- data/llms-install.md +88 -0
- data/llms.txt +87 -0
- data/skills/ascli/SKILL.md +94 -0
- data.tar.gz.sig +0 -0
- metadata +77 -20
- metadata.gz.sig +0 -0
- data/lib/aspera/duration.rb +0 -42
- data/lib/aspera/keychain/base.rb +0 -30
- data/lib/aspera/keychain/encrypted_hash.rb +0 -192
- data/lib/aspera/keychain/factory.rb +0 -58
- data/lib/aspera/keychain/hashicorp_vault.rb +0 -77
- data/lib/aspera/keychain/macos_security.rb +0 -277
- data/lib/aspera/keychain/one_password_api.rb +0 -91
- data/lib/aspera/keychain/one_password_base.rb +0 -39
- data/lib/aspera/keychain/one_password_cli.rb +0 -100
- data/lib/aspera/secret_hider.rb +0 -196
- data/lib/aspera/secret_strength.rb +0 -114
|
@@ -1,24 +1,26 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
#
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
3
|
+
# SPDX-FileCopyrightText: IBM Corp.
|
|
4
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
5
|
+
|
|
6
|
+
require_relative 'basic_auth'
|
|
7
|
+
require_relative '../../preview/generator'
|
|
8
|
+
require_relative '../../preview/utils'
|
|
9
|
+
require_relative '../../preview/file_types'
|
|
10
|
+
require_relative '../../preview/terminal'
|
|
11
|
+
require_relative '../../transfer/spec'
|
|
12
|
+
require_relative '../../schema/registry'
|
|
13
|
+
require_relative '../../persistency_action_once'
|
|
14
|
+
require_relative '../../temp_file_manager'
|
|
15
|
+
require_relative '../../api/node'
|
|
16
|
+
require_relative '../../hash_ext'
|
|
17
|
+
require_relative '../../timer_limiter'
|
|
18
|
+
require_relative '../../id_generator'
|
|
19
|
+
require_relative '../../uri_reader'
|
|
20
|
+
require_relative '../../log'
|
|
21
|
+
require_relative '../../assert'
|
|
22
|
+
require_relative '../../rainbow'
|
|
20
23
|
require 'securerandom'
|
|
21
|
-
require 'aspera/rainbow'
|
|
22
24
|
using Rainbow
|
|
23
25
|
|
|
24
26
|
module Aspera
|
|
@@ -96,7 +98,7 @@ module Aspera
|
|
|
96
98
|
Aspera::Preview::Utils.office_tool = @gen_options.office_conversion
|
|
97
99
|
Api::Node.api_options[:cache] = !@option_folder_reset_cache.eql?(:header)
|
|
98
100
|
# Start from the full supported format list, then remove any skipped format.
|
|
99
|
-
@preview_formats_to_generate = Aspera::Preview::Generator::PREVIEW_FORMATS.
|
|
101
|
+
@preview_formats_to_generate = Aspera::Preview::Generator::PREVIEW_FORMATS.dup
|
|
100
102
|
skip = options.get_option(:skip_format)
|
|
101
103
|
@preview_formats_to_generate.delete(skip) if skip
|
|
102
104
|
# Created by `tools_cleanup`, only for commands that use it
|
|
@@ -165,13 +167,17 @@ module Aspera
|
|
|
165
167
|
return if events.empty?
|
|
166
168
|
events.each do |event|
|
|
167
169
|
# Ignore non-file events such as folder notifications.
|
|
168
|
-
if event.dig('data', 'type').eql?('file')
|
|
170
|
+
if !event.dig('data', 'type').eql?('file')
|
|
171
|
+
nil
|
|
172
|
+
elsif Array(event['types']).include?('file.deleted')
|
|
173
|
+
delete_preview(event['data'])
|
|
174
|
+
else
|
|
169
175
|
file_entry = @api_node.read("files/#{event['data']['id']}") rescue nil
|
|
170
176
|
if !file_entry.nil? &&
|
|
171
|
-
@option_skip_folders.none? { |d| file_entry['path'].start_with?(d) }
|
|
177
|
+
@option_skip_folders.none? { |d| file_entry['path'].start_with?(d) } &&
|
|
178
|
+
@filter_block.call(file_entry)
|
|
172
179
|
file_entry['parent_file_id'] = event['data']['parent_file_id']
|
|
173
|
-
|
|
174
|
-
generate_preview(file_entry) if event['types'].include?('file.deleted')
|
|
180
|
+
generate_preview(file_entry)
|
|
175
181
|
end
|
|
176
182
|
end
|
|
177
183
|
# Periodically log progress and persist the latest processed event.
|
|
@@ -186,6 +192,30 @@ module Aspera
|
|
|
186
192
|
nil
|
|
187
193
|
end
|
|
188
194
|
|
|
195
|
+
# Delete the preview folder of a deleted file, if it exists.
|
|
196
|
+
# Errors are logged: a failure shall not stop the processing of events.
|
|
197
|
+
# @param entry [Hash] file entry of the event, with at least `id`
|
|
198
|
+
# @return [nil]
|
|
199
|
+
def delete_preview(entry)
|
|
200
|
+
folder_name = entry_preview_folder_name(entry)
|
|
201
|
+
# The identifier comes from the API: it shall not designate another folder
|
|
202
|
+
Aspera.assert(!entry['id'].to_s.empty? && File.basename(folder_name).eql?(folder_name)) { "Invalid file id: #{entry['id']}" }
|
|
203
|
+
if @access_remote
|
|
204
|
+
preview_folder = @api_node.read_folder_content(@previews_folder_entry['id'], {name: folder_name}).find { |item| item['name'].eql?(folder_name) }
|
|
205
|
+
return if preview_folder.nil?
|
|
206
|
+
@api_node.delete("files/#{preview_folder['id']}")
|
|
207
|
+
else
|
|
208
|
+
local_entry_preview_dir = File.join(@local_preview_folder, folder_name)
|
|
209
|
+
return unless File.directory?(local_entry_preview_dir)
|
|
210
|
+
FileUtils.rm_rf(local_entry_preview_dir)
|
|
211
|
+
end
|
|
212
|
+
Log.log.info { "Deleted preview of #{entry['id']}" }
|
|
213
|
+
nil
|
|
214
|
+
rescue StandardError => e
|
|
215
|
+
Log.log.error { "Cannot delete preview of #{entry['id']}: #{e.message}" }
|
|
216
|
+
nil
|
|
217
|
+
end
|
|
218
|
+
|
|
189
219
|
# Transfer a file to or from the configured Node storage using a tagged transfer spec.
|
|
190
220
|
#
|
|
191
221
|
# @param direction [String] transfer direction, typically from [`Transfer::Spec`](lib/aspera/transfer/spec.rb)
|
|
@@ -195,6 +225,8 @@ module Aspera
|
|
|
195
225
|
# @return [Object] transfer result returned by [`Main.result_transfer`](lib/aspera/cli/main.rb)
|
|
196
226
|
def do_transfer(direction, folder_id, source_filename, destination = '/')
|
|
197
227
|
Aspera.assert(!(destination.nil? && direction.eql?(Transfer::Spec::DIRECTION_RECEIVE)), 'destination must be set for receive direction')
|
|
228
|
+
# Previews are generated from downloaded files, and uploaded
|
|
229
|
+
transfer.assert_no_dry_run
|
|
198
230
|
t_spec = @api_node.transfer_spec_gen4(folder_id, direction, {
|
|
199
231
|
'paths' => [{'source' => source_filename}],
|
|
200
232
|
'tags' => {Transfer::Spec::TAG_RESERVED => {PREV_GEN_TAG => true}}
|
|
@@ -240,16 +272,21 @@ module Aspera
|
|
|
240
272
|
# original_mtime=DateTime.parse(entry['modified_time'])
|
|
241
273
|
# Local directory where previews are generated before being uploaded back.
|
|
242
274
|
local_entry_preview_dir = File.join(@tmp_folder, entry_preview_folder_name(entry))
|
|
243
|
-
file_info = @api_node.read("files/#{entry['id']}")
|
|
244
|
-
#
|
|
245
|
-
|
|
246
|
-
|
|
275
|
+
file_info = @api_node.read("files/#{entry['id']}", headers: Api::Node.add_cache_control)
|
|
276
|
+
# Check if preview exists either from file_info or by checking preview endpoint
|
|
277
|
+
preview_exist = file_info.key?('preview')
|
|
278
|
+
unless preview_exist
|
|
279
|
+
begin
|
|
280
|
+
@api_node.read("files/#{entry['id']}/preview", headers: {'Accept' => 'image/png'})
|
|
281
|
+
preview_exist = true
|
|
282
|
+
rescue StandardError
|
|
283
|
+
preview_exist = false
|
|
284
|
+
end
|
|
285
|
+
end
|
|
247
286
|
gen_infos.each do |gen_info|
|
|
248
287
|
gen_info[:src] = local_original_filepath
|
|
249
288
|
gen_info[:dst] = File.join(local_entry_preview_dir, gen_info[:base_dest])
|
|
250
|
-
|
|
251
|
-
gen_info[:preview_exist] = file_info.key?('preview')
|
|
252
|
-
# TODO: Compare source and preview modification times when remote timestamps are available.
|
|
289
|
+
gen_info[:preview_exist] = preview_exist
|
|
253
290
|
gen_info[:preview_newer_than_original] = gen_info[:preview_exist]
|
|
254
291
|
end
|
|
255
292
|
return local_entry_preview_dir
|
|
@@ -352,25 +389,15 @@ module Aspera
|
|
|
352
389
|
|
|
353
390
|
# Scan all files in provided folder entry
|
|
354
391
|
# @param top_entry [Hash] the top entry to scan
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
unless top_path.nil?
|
|
358
|
-
# canonical path: start with / and ends with /
|
|
359
|
-
top_path = "/#{top_path.split('/').reject(&:empty?).join('/')}/"
|
|
360
|
-
end
|
|
361
|
-
Log.log.debug { "scan: #{top_entry} : #{top_path}".green }
|
|
392
|
+
def scan_folder_files(top_entry)
|
|
393
|
+
Log.log.debug { "scan: #{top_entry}".green }
|
|
362
394
|
# don't use recursive call, use list instead
|
|
363
395
|
entries_to_process = [top_entry]
|
|
364
396
|
until entries_to_process.empty?
|
|
365
397
|
entry = entries_to_process.shift
|
|
366
|
-
# process this entry only if it is within the top_path
|
|
367
398
|
entry_path_with_slash = entry['path']
|
|
368
399
|
Log.log.debug { "processing entry #{entry_path_with_slash}" } if @periodic.trigger?
|
|
369
400
|
entry_path_with_slash = "#{entry_path_with_slash}/" unless entry_path_with_slash.end_with?('/')
|
|
370
|
-
if !top_path.nil? && !top_path.start_with?(entry_path_with_slash) && !entry_path_with_slash.start_with?(top_path)
|
|
371
|
-
Log.log.debug { "#{entry['path']} folder (skip start)".bg(:red) }
|
|
372
|
-
next
|
|
373
|
-
end
|
|
374
401
|
Log.log.debug { "item:#{entry}" }
|
|
375
402
|
begin
|
|
376
403
|
case entry['type']
|
|
@@ -415,11 +442,11 @@ module Aspera
|
|
|
415
442
|
command :events, description: 'Process file events and generate previews', setup: :setup_node_api, action: ->(**) { run_event_loop(:events) }
|
|
416
443
|
command :trevents, description: 'Process transfer events and generate previews', setup: :setup_node_api, action: ->(**) { run_event_loop(:trevents) }
|
|
417
444
|
command :check, description: 'Check required tools are installed', action: ->(**) { tools_cleanup { Result::Status.new('Tools validated') } }
|
|
418
|
-
command :show, description: 'Generate and display preview of a source file',
|
|
419
|
-
arguments: [{name: :source_file, description: 'Local file to preview'}]
|
|
445
|
+
command :show, read_only: false, description: 'Generate and display preview of a source file',
|
|
446
|
+
arguments: [{name: :source_file, description: 'Local file to preview', complete: :file}]
|
|
420
447
|
command :test, description: 'Test preview generation for a source file',
|
|
421
448
|
arguments: [
|
|
422
|
-
{name: :source_file, description: 'Local file to preview'},
|
|
449
|
+
{name: :source_file, description: 'Local file to preview', complete: :file},
|
|
423
450
|
{name: :format, description: 'Preview output format', allowed: Aspera::Preview::Generator::PREVIEW_FORMATS, mandatory: false, default: :png}
|
|
424
451
|
]
|
|
425
452
|
|
|
@@ -1,22 +1,28 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
#
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
3
|
+
# SPDX-FileCopyrightText: IBM Corp.
|
|
4
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
5
|
+
|
|
6
|
+
require_relative 'basic_auth'
|
|
7
|
+
require_relative '../sync_actions'
|
|
8
|
+
require_relative '../ascp_logs_actions'
|
|
9
|
+
require_relative '../../transfer/result'
|
|
10
|
+
require_relative '../../transfer/spec'
|
|
11
|
+
require_relative '../../ascmd'
|
|
12
|
+
require_relative '../../ssh'
|
|
13
|
+
require_relative '../../nagios'
|
|
14
|
+
require_relative '../../log'
|
|
15
|
+
require_relative '../../assert'
|
|
16
|
+
require_relative '../../environment'
|
|
17
|
+
require 'shellwords'
|
|
18
|
+
require 'tmpdir'
|
|
14
19
|
module Aspera
|
|
15
20
|
module Cli
|
|
16
21
|
module Plugins
|
|
17
22
|
# Operations on HSTS with SSH/FASP (ascmd/ascp)
|
|
18
23
|
class Server < BasicAuth
|
|
19
24
|
include SyncActions
|
|
25
|
+
include AscpLogsActions
|
|
20
26
|
|
|
21
27
|
SSH_SCHEME = 'ssh'
|
|
22
28
|
LOCAL_SCHEME = 'local'
|
|
@@ -196,6 +202,9 @@ module Aspera
|
|
|
196
202
|
command :transfer, description: 'Check FASP transfer health'
|
|
197
203
|
end
|
|
198
204
|
|
|
205
|
+
# Analysis of ascp logs of the server: same commands as `config ascp logs`, on files read over SSH
|
|
206
|
+
ascp_logs_commands(:logs, arguments: [{name: :paths, description: 'Log files on the server (gzip supported)', multiple: true}])
|
|
207
|
+
|
|
199
208
|
# AsCmd operations (available only when SSH/local - not WSS)
|
|
200
209
|
# Arguments follow AsCmd::OPS_ARGS: one path per operation, or source and destination
|
|
201
210
|
PATHS_ARGS = [{name: :paths, multiple: true}].freeze
|
|
@@ -214,15 +223,15 @@ module Aspera
|
|
|
214
223
|
command :cp, description: 'Copy files: source destination pairs, or sources with --to-folder', condition: :ascmd_available?, arguments: PATHS_ARGS
|
|
215
224
|
command :mkdir, description: 'Create directory', condition: :ascmd_available?, arguments: PATHS_ARGS
|
|
216
225
|
command(
|
|
217
|
-
:df, description: 'Show disk usage', condition: :ascmd_available?,
|
|
226
|
+
:df, read_only: true, description: 'Show disk usage', condition: :ascmd_available?,
|
|
218
227
|
action: lambda do |**|
|
|
219
228
|
execute_ascmd(:df, []) do |result|
|
|
220
229
|
Result::ObjectList.new(result.map(&:stringify_keys))
|
|
221
230
|
end
|
|
222
231
|
end
|
|
223
232
|
)
|
|
224
|
-
command :du, description: 'Show file sizes', condition: :ascmd_available?, arguments: PATHS_ARGS
|
|
225
|
-
command :md5sum, description: 'Compute MD5 checksums', condition: :ascmd_available?, arguments: PATHS_ARGS
|
|
233
|
+
command :du, read_only: true, description: 'Show file sizes', condition: :ascmd_available?, arguments: PATHS_ARGS
|
|
234
|
+
command :md5sum, read_only: true, description: 'Compute MD5 checksums', condition: :ascmd_available?, arguments: PATHS_ARGS
|
|
226
235
|
command :info, description: 'Show server system information', condition: :ascmd_available?,
|
|
227
236
|
action: ->(**) { execute_ascmd(:info, []) { |r| Result::SingleObject.new(r.stringify_keys) } }
|
|
228
237
|
|
|
@@ -299,6 +308,49 @@ module Aspera
|
|
|
299
308
|
|
|
300
309
|
private
|
|
301
310
|
|
|
311
|
+
# Log files of the server, to analyze with the commands of `logs`.
|
|
312
|
+
# Read over SSH, in a temporary folder removed after the block.
|
|
313
|
+
# @param paths [Array<String>] files on the server
|
|
314
|
+
# @yieldparam files [Array<String>] local copies
|
|
315
|
+
def ascp_logs_files(paths)
|
|
316
|
+
executor = ascmd_executor
|
|
317
|
+
Aspera.assert(executor.is_a?(Ssh), type: Cli::BadArgument) { 'Reading the logs of the server requires an SSH connection (URL scheme ssh)' }
|
|
318
|
+
remaining = ascp_logs_params['remote_max_size']
|
|
319
|
+
Dir.mktmpdir('ascli_logs') do |folder|
|
|
320
|
+
files = paths.each_with_index.map do |remote, index|
|
|
321
|
+
local = File.join(folder, format('%02d_%s', index, File.basename(remote)))
|
|
322
|
+
remaining -= read_remote_log(executor, remote, local, remaining)
|
|
323
|
+
local
|
|
324
|
+
end
|
|
325
|
+
yield(files)
|
|
326
|
+
end
|
|
327
|
+
end
|
|
328
|
+
|
|
329
|
+
# Copy a file of the server over SSH. Shell characters of the path are escaped.
|
|
330
|
+
# The modification date is kept: the year of syslog lines is deduced from it.
|
|
331
|
+
# @param executor [Ssh] SSH connection
|
|
332
|
+
# @param remote [String] file on the server
|
|
333
|
+
# @param local [String] destination
|
|
334
|
+
# @param limit [Integer] maximum size in bytes
|
|
335
|
+
# @return [Integer] size in bytes
|
|
336
|
+
def read_remote_log(executor, remote, local, limit)
|
|
337
|
+
quoted = Shellwords.escape(remote)
|
|
338
|
+
# Force an integer: the value comes from a user option and is interpolated in a shell command
|
|
339
|
+
max_bytes = Integer(limit) + 1
|
|
340
|
+
# First line: modification time (0 if unknown), then the content, read up to one byte over the limit
|
|
341
|
+
output = executor.execute("(stat -c %Y -- #{quoted} 2>/dev/null || echo 0); head -c #{max_bytes} -- #{quoted}").b
|
|
342
|
+
mtime, _, content = output.partition("\n")
|
|
343
|
+
Aspera.assert(content.bytesize <= limit, type: Cli::BadArgument) do
|
|
344
|
+
"Log files from the server exceed the limit of #{ascp_logs_params['remote_max_size']} bytes (option ascp_logs.remote_max_size), at: #{remote}"
|
|
345
|
+
end
|
|
346
|
+
Log.log.debug { "Read #{remote}: #{content.bytesize} bytes" }
|
|
347
|
+
File.binwrite(local, content)
|
|
348
|
+
File.utime(Time.now, Time.at(mtime.to_i), local) if mtime.to_i.positive?
|
|
349
|
+
content.bytesize
|
|
350
|
+
rescue Ssh::Error => e
|
|
351
|
+
raise Cli::BadArgument, "Cannot read #{remote}: #{e.message}"
|
|
352
|
+
end
|
|
353
|
+
|
|
302
354
|
# AsCmd `cp` and `mv` take source and destination pairs.
|
|
303
355
|
# With option `to_folder`, paths are only sources, copied or moved into that folder.
|
|
304
356
|
# @param paths [Array<String>] command line arguments
|
|
@@ -1,9 +1,12 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
3
|
+
# SPDX-FileCopyrightText: IBM Corp.
|
|
4
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
5
|
+
|
|
6
|
+
require_relative 'basic_auth'
|
|
7
|
+
require_relative 'node'
|
|
8
|
+
require_relative '../../assert'
|
|
9
|
+
require_relative '../../rest/list'
|
|
7
10
|
|
|
8
11
|
module Aspera
|
|
9
12
|
module Cli
|
|
@@ -14,6 +17,13 @@ module Aspera
|
|
|
14
17
|
NODE_API_PATH = 'node_api'
|
|
15
18
|
# path for node admin after base url
|
|
16
19
|
ADMIN_API_PATH = 'api/v1'
|
|
20
|
+
# path for authentication (SAML login page, OAuth token) after base url
|
|
21
|
+
AUTH_API_PATH = 'auth'
|
|
22
|
+
# default URL that receives the tokens, with auth `web` and `ui.web_auth=port`
|
|
23
|
+
REDIRECT_LOCALHOST = 'http://localhost:12345'
|
|
24
|
+
AUTH_TYPES = %i[basic web].freeze
|
|
25
|
+
option :auth, description: 'Type of authentication: user and password, or Web browser (SAML)', allowed: AUTH_TYPES, default: :basic
|
|
26
|
+
option :redirect_uri, description: "Web authentication with ui.web_auth=port: loopback URL that receives the tokens (default: #{REDIRECT_LOCALHOST})"
|
|
17
27
|
class << self
|
|
18
28
|
# Check various endpoints on Shares
|
|
19
29
|
# @return [Hash] with version, ping, api
|
|
@@ -27,12 +37,13 @@ module Aspera
|
|
|
27
37
|
.new(base_url: url, redirect_max: 2)
|
|
28
38
|
.read('', headers: {'Accept'=>'text/html'})
|
|
29
39
|
Aspera.assert(login_page.include?('aspera-Shares')) { 'not Shares' }
|
|
30
|
-
if (m = login_page.match(/\(v([0-9a-f
|
|
40
|
+
if (m = login_page.match(/\(v([0-9a-f.]+)\)/))
|
|
31
41
|
version = m[1]
|
|
32
42
|
if (m = login_page.match(/Patch level ([0-9]+)/))
|
|
33
43
|
version = "#{version} #{m[0]}"
|
|
34
44
|
end
|
|
35
45
|
end
|
|
46
|
+
result[:saml] = login_page.include?('/auth/saml')
|
|
36
47
|
version.nil? ? 'no version' : version
|
|
37
48
|
rescue => e
|
|
38
49
|
e
|
|
@@ -79,7 +90,8 @@ module Aspera
|
|
|
79
90
|
return unless health[:api].is_a?(String)
|
|
80
91
|
return {
|
|
81
92
|
version: health[:version].is_a?(String) ? health[:version] : 'unknown',
|
|
82
|
-
url: address_or_url
|
|
93
|
+
url: address_or_url,
|
|
94
|
+
saml: health[:saml] || false
|
|
83
95
|
}
|
|
84
96
|
end
|
|
85
97
|
end
|
|
@@ -88,16 +100,53 @@ module Aspera
|
|
|
88
100
|
# @param app_url [String] Tested URL
|
|
89
101
|
# @return [Hash] :preset_value, :test_args
|
|
90
102
|
def wizard(wizard, app_url)
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
103
|
+
auth_type = if !options.option_def(:auth).source.eql?(:default)
|
|
104
|
+
options.get_option(:auth)
|
|
105
|
+
elsif options.get_option(:password)
|
|
106
|
+
:basic
|
|
107
|
+
else
|
|
108
|
+
saml_detected = begin
|
|
109
|
+
self.class.health_check(app_url)[:saml] || false
|
|
110
|
+
rescue StandardError
|
|
111
|
+
false
|
|
112
|
+
end
|
|
113
|
+
formatter.display_status('SAML authentication is available.') if saml_detected
|
|
114
|
+
if wizard.required
|
|
115
|
+
formatter.display_status('Select authentication method:')
|
|
116
|
+
options.prompt_user_input_in_list('auth', AUTH_TYPES)
|
|
117
|
+
else
|
|
118
|
+
saml_detected ? :web : :basic
|
|
119
|
+
end
|
|
120
|
+
end
|
|
121
|
+
|
|
122
|
+
case auth_type
|
|
123
|
+
when :web
|
|
124
|
+
username = options.get_option(:username)
|
|
125
|
+
preset_ids = username && !username.empty? ? [username] : ['web']
|
|
126
|
+
preset_value = {
|
|
127
|
+
url: app_url,
|
|
128
|
+
auth: :web.to_s
|
|
129
|
+
}
|
|
130
|
+
preset_value[:username] = username if username && !username.empty?
|
|
131
|
+
redirect_uri = options.get_option(:redirect_uri)
|
|
132
|
+
preset_value[:redirect_uri] = redirect_uri if redirect_uri
|
|
133
|
+
{
|
|
134
|
+
preset_ids: preset_ids,
|
|
135
|
+
preset_value: preset_value,
|
|
136
|
+
test_args: 'files browse /'
|
|
137
|
+
}
|
|
138
|
+
when :basic
|
|
139
|
+
username = options.get_option(:username, mandatory: true)
|
|
140
|
+
{
|
|
141
|
+
preset_ids: [username],
|
|
142
|
+
preset_value: {
|
|
143
|
+
url: app_url,
|
|
144
|
+
username: username,
|
|
145
|
+
password: options.get_option(:password, mandatory: true)
|
|
146
|
+
},
|
|
147
|
+
test_args: 'files browse /'
|
|
148
|
+
}
|
|
149
|
+
end
|
|
101
150
|
end
|
|
102
151
|
|
|
103
152
|
def initialize(**_)
|
|
@@ -105,6 +154,27 @@ module Aspera
|
|
|
105
154
|
@api_shares_admin = nil
|
|
106
155
|
end
|
|
107
156
|
|
|
157
|
+
# With `auth` `web`, the user authenticates with a browser (SAML), and the REST API uses the OAuth token.
|
|
158
|
+
# Otherwise: user and password.
|
|
159
|
+
# @param subpath [String, nil] Path of API after base URL
|
|
160
|
+
# @return [Hash] Parameters of `Rest::Client`
|
|
161
|
+
def basic_auth_params(subpath = nil)
|
|
162
|
+
return super unless options.get_option(:auth).eql?(:web)
|
|
163
|
+
url = options.get_option(:url, mandatory: true)
|
|
164
|
+
username = options.get_option(:username)
|
|
165
|
+
return {
|
|
166
|
+
base_url: subpath.nil? ? url : "#{url}/#{subpath}",
|
|
167
|
+
auth: {
|
|
168
|
+
type: :oauth2,
|
|
169
|
+
base_url: "#{url}/#{AUTH_API_PATH}",
|
|
170
|
+
grant_method: :shares_web,
|
|
171
|
+
redirect_uri: options.get_option(:redirect_uri) || REDIRECT_LOCALHOST,
|
|
172
|
+
# a token is cached for each user, if known
|
|
173
|
+
cache_ids: [username].compact
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
end
|
|
177
|
+
|
|
108
178
|
SAML_IMPORT_MANDATORY = %w[id name_id].freeze
|
|
109
179
|
SAML_IMPORT_ALLOWED = (%w[email given_name surname] + SAML_IMPORT_MANDATORY).freeze
|
|
110
180
|
|
|
@@ -287,8 +357,7 @@ module Aspera
|
|
|
287
357
|
# entity_plural captured at class-load time in the closure
|
|
288
358
|
entity_plural_search = entity_type.eql?(:user) ? 'users' : 'groups'
|
|
289
359
|
commands_under :search do
|
|
290
|
-
command :list,
|
|
291
|
-
description: "Search #{entity_type}s across all directories (use --query={q: '...'} to filter)",
|
|
360
|
+
command :list, description: "Search #{entity_type}s across all directories (use --query={q: '...'} to filter)",
|
|
292
361
|
action: ->(**) { entity_list(api: @api_shares_admin, entity: entity_plural_search, items_key: 'members') }
|
|
293
362
|
command :show,
|
|
294
363
|
description: "Get directory #{entity_type} details by URN",
|