aspera-cli 4.27.5 → 4.29.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/BUGS.md +5 -10
- data/CHANGELOG.md +301 -4
- data/CODE_OF_CONDUCT.md +18 -0
- data/CONTRIBUTING.md +139 -58
- data/GOVERNANCE.md +30 -0
- data/LICENSE +201 -0
- data/README.md +103 -26
- data/SECURITY.md +70 -0
- data/bin/ascli +8 -5
- data/bin/asession +10 -7
- data/docs/README.md +2551 -772
- data/docs/test-mcp-with-ai.md +13 -11
- data/lib/aspera/agent/base.rb +23 -2
- data/lib/aspera/agent/connect.rb +16 -12
- data/lib/aspera/agent/desktop.rb +10 -7
- data/lib/aspera/agent/direct.rb +34 -22
- data/lib/aspera/agent/factory.rb +8 -4
- data/lib/aspera/agent/httpgw.rb +7 -5
- data/lib/aspera/agent/node.rb +15 -7
- data/lib/aspera/agent/transferd.rb +167 -43
- data/lib/aspera/api/alee.rb +4 -1
- data/lib/aspera/api/aoc.rb +14 -11
- data/lib/aspera/api/ats.rb +4 -2
- data/lib/aspera/api/cos_node.rb +8 -5
- data/lib/aspera/api/faspex.rb +16 -9
- data/lib/aspera/api/httpgw.rb +9 -6
- data/lib/aspera/api/node.rb +23 -19
- data/lib/aspera/ascmd.rb +8 -5
- data/lib/aspera/ascp/installation.rb +84 -42
- data/lib/aspera/ascp/log.schema.yaml +1084 -0
- data/lib/aspera/ascp/log_analyzer/advice.rb +97 -0
- data/lib/aspera/ascp/log_analyzer/diagnostics.rb +344 -0
- data/lib/aspera/ascp/log_analyzer/lines.rb +75 -0
- data/lib/aspera/ascp/log_analyzer/log_file.rb +593 -0
- data/lib/aspera/ascp/log_analyzer/records.rb +379 -0
- data/lib/aspera/ascp/log_analyzer/rules.rb +172 -0
- data/lib/aspera/ascp/log_analyzer/session.rb +782 -0
- data/lib/aspera/ascp/log_analyzer.rb +354 -0
- data/lib/aspera/ascp/management.rb +102 -48
- data/lib/aspera/ascp/session_errors.rb +217 -0
- data/lib/aspera/assert.rb +4 -0
- data/lib/aspera/cli/ascp_actions.rb +16 -37
- data/lib/aspera/cli/ascp_logs_actions.rb +176 -0
- data/lib/aspera/cli/async_transfer_store.rb +7 -4
- data/lib/aspera/cli/bootstrapper.rb +37 -34
- data/lib/aspera/cli/command_line.rb +24 -6
- data/lib/aspera/cli/command_registry.rb +49 -4
- data/lib/aspera/cli/command_spec.rb +34 -4
- data/lib/aspera/cli/completion/ascli.bash +53 -5
- data/lib/aspera/cli/completion/ascli.fish +27 -8
- data/lib/aspera/cli/completion/ascli.ps1 +63 -0
- data/lib/aspera/cli/completion/ascli.zsh +36 -4
- data/lib/aspera/cli/completion_actions.rb +594 -0
- data/lib/aspera/cli/context.rb +9 -4
- data/lib/aspera/cli/deprecation.rb +5 -2
- data/lib/aspera/cli/error.rb +45 -0
- data/lib/aspera/cli/extended_value.rb +83 -24
- data/lib/aspera/cli/formatter.rb +142 -73
- data/lib/aspera/cli/gem_checker.rb +9 -6
- data/lib/aspera/cli/hints.rb +22 -11
- data/lib/aspera/cli/http.rb +17 -13
- data/lib/aspera/cli/info.rb +17 -6
- data/lib/aspera/cli/mailer.rb +7 -4
- data/lib/aspera/cli/mcp/http_servlet.rb +155 -0
- data/lib/aspera/cli/mcp/secrets.rb +146 -0
- data/lib/aspera/cli/mcp/tool.rb +232 -0
- data/lib/aspera/cli/option_declarator.rb +10 -4
- data/lib/aspera/cli/option_registry.rb +6 -3
- data/lib/aspera/cli/option_types.rb +17 -2
- data/lib/aspera/cli/option_value.rb +72 -11
- data/lib/aspera/cli/options.schema.yaml +856 -81
- data/lib/aspera/cli/parser.rb +64 -31
- data/lib/aspera/cli/plugins/alee.rb +7 -4
- data/lib/aspera/cli/plugins/aoc.rb +250 -44
- data/lib/aspera/cli/plugins/ats.rb +9 -8
- data/lib/aspera/cli/plugins/base.rb +51 -25
- data/lib/aspera/cli/plugins/basic_auth.rb +5 -2
- data/lib/aspera/cli/plugins/config.rb +207 -141
- data/lib/aspera/cli/plugins/console.rb +9 -4
- data/lib/aspera/cli/plugins/cos.rb +8 -4
- data/lib/aspera/cli/plugins/factory.rb +22 -8
- data/lib/aspera/cli/plugins/faspex5.rb +37 -44
- data/lib/aspera/cli/plugins/faspio.rb +41 -8
- data/lib/aspera/cli/plugins/httpgw.rb +8 -4
- data/lib/aspera/cli/plugins/mcp.rb +186 -121
- data/lib/aspera/cli/plugins/node.rb +41 -34
- data/lib/aspera/cli/plugins/oauth.rb +5 -1
- data/lib/aspera/cli/plugins/orchestrator.rb +572 -59
- data/lib/aspera/cli/plugins/preview.rb +69 -44
- data/lib/aspera/cli/plugins/server.rb +66 -12
- data/lib/aspera/cli/plugins/shares.rb +288 -48
- data/lib/aspera/cli/plugins/transferd.rb +223 -0
- data/lib/aspera/cli/preset_actions.rb +155 -33
- data/lib/aspera/cli/preset_manager.rb +72 -16
- data/lib/aspera/cli/prompt.rb +7 -2
- data/lib/aspera/cli/result.rb +28 -11
- data/lib/aspera/cli/runner.rb +193 -80
- data/lib/aspera/cli/secret_finder.rb +4 -1
- data/lib/aspera/cli/security.rb +204 -0
- data/lib/aspera/cli/special_values.rb +4 -1
- data/lib/aspera/cli/sync_actions.rb +13 -10
- data/lib/aspera/cli/table_fitter.rb +119 -0
- data/lib/aspera/cli/terminal_formatter.rb +32 -40
- data/lib/aspera/cli/transfer_actions.rb +11 -8
- data/lib/aspera/cli/transfer_agent.rb +31 -19
- data/lib/aspera/cli/transfer_progress.rb +13 -45
- data/lib/aspera/cli/vault_actions.rb +45 -0
- data/lib/aspera/cli/vault_manager.rb +197 -50
- data/lib/aspera/cli/version.rb +4 -1
- data/lib/aspera/cli/web_secret.html.erb +61 -0
- data/lib/aspera/cli/web_secret.rb +222 -0
- data/lib/aspera/cli/wizard.rb +58 -45
- data/lib/aspera/command_line_builder.rb +35 -5
- data/lib/aspera/command_line_converter.rb +4 -1
- data/lib/aspera/coverage.rb +103 -33
- data/lib/aspera/data_repository.rb +42 -25
- data/lib/aspera/dot_container.rb +4 -1
- data/lib/aspera/duration.rb +45 -0
- data/lib/aspera/environment.rb +106 -33
- data/lib/aspera/exec_spec.rb +9 -5
- data/lib/aspera/faspex_gw.rb +105 -35
- data/lib/aspera/faspex_postproc.rb +193 -47
- data/lib/aspera/formatter_interface.rb +4 -2
- data/lib/aspera/graphql.rb +5 -1
- data/lib/aspera/hash_ext.rb +3 -0
- data/lib/aspera/id_generator.rb +6 -3
- data/lib/aspera/json_rpc/client.rb +55 -35
- data/lib/aspera/json_rpc/version.rb +4 -0
- data/lib/aspera/line_logger.rb +4 -1
- data/lib/aspera/link_header.rb +4 -1
- data/lib/aspera/log.rb +20 -10
- data/lib/aspera/markdown.rb +155 -115
- data/lib/aspera/mime.rb +9 -0
- data/lib/aspera/nagios.rb +9 -5
- data/lib/aspera/node_emulator.rb +111 -112
- data/lib/aspera/oauth/client/base.rb +163 -0
- data/lib/aspera/oauth/{boot.rb → client/boot.rb} +8 -4
- data/lib/aspera/oauth/{factory.rb → client/factory.rb} +13 -3
- data/lib/aspera/oauth/{generic.rb → client/generic.rb} +5 -1
- data/lib/aspera/oauth/{json_credentials.rb → client/json_credentials.rb} +4 -1
- data/lib/aspera/oauth/{jwt.rb → client/jwt.rb} +6 -3
- data/lib/aspera/oauth/client/shares_web.rb +85 -0
- data/lib/aspera/oauth/{url_json.rb → client/url_json.rb} +4 -1
- data/lib/aspera/oauth/client/web.rb +84 -0
- data/lib/aspera/oauth/server/authorize.html.erb +34 -0
- data/lib/aspera/oauth/server/pkce.rb +427 -0
- data/lib/aspera/orchestrator_workflow_checker.rb +679 -0
- data/lib/aspera/persistency_action_once.rb +5 -3
- data/lib/aspera/persistency_folder.rb +6 -3
- data/lib/aspera/preview/file_types.rb +6 -3
- data/lib/aspera/preview/generator.rb +9 -5
- data/lib/aspera/preview/options.schema.yaml +3 -3
- data/lib/aspera/preview/terminal.rb +11 -6
- data/lib/aspera/preview/utils.rb +6 -4
- data/lib/aspera/products/connect.rb +5 -2
- data/lib/aspera/products/desktop.rb +4 -1
- data/lib/aspera/products/other.rb +4 -2
- data/lib/aspera/products/transferd.rb +6 -2
- data/lib/aspera/products/transferd_service.rb +610 -0
- data/lib/aspera/proxy_auto_config.js +2 -2
- data/lib/aspera/proxy_auto_config.rb +5 -2
- data/lib/aspera/rainbow.rb +30 -1
- data/lib/aspera/rest/aspera_errors.rb +55 -60
- data/lib/aspera/rest/call_error.rb +3 -0
- data/lib/aspera/rest/client.rb +42 -42
- data/lib/aspera/rest/error_analyzer.rb +14 -2
- data/lib/aspera/rest/list.rb +14 -10
- data/lib/aspera/rest/parameters.rb +3 -0
- data/lib/aspera/rest/util.rb +13 -4
- data/lib/aspera/rest.rb +9 -7
- data/lib/aspera/schema/IBM Aspera Faspex API-5.0-enhanced.yaml +9 -11
- data/lib/aspera/schema/IBM Aspera Orchestrator API-v1-enhanced.yaml +4089 -0
- data/lib/aspera/schema/IBM Aspera faspio Gateway API-1.0.0-enhanced.yaml +1097 -0
- data/lib/aspera/schema/IBM Aspera on Cloud API-0.2.6-enhanced.yaml +11 -11
- data/lib/aspera/schema/IBM Aspera on Cloud Automation API-1.0.5-enhanced.yaml +2437 -1437
- data/lib/aspera/schema/IBM_Aspera_Shares.yaml +819 -7
- data/lib/aspera/schema/async_tables.yaml +11 -10
- data/lib/aspera/schema/documentation.rb +56 -78
- data/lib/aspera/schema/reader.rb +46 -11
- data/lib/aspera/schema/registry.rb +45 -24
- data/lib/aspera/schema/transferd-config.schema.json +333 -0
- data/lib/aspera/schema/validator.rb +4 -1
- data/lib/aspera/sealed_hash.rb +166 -0
- data/lib/aspera/secret/hider.rb +201 -0
- data/lib/aspera/secret/reference.rb +150 -0
- data/lib/aspera/secret/resolver.rb +212 -0
- data/lib/aspera/secret/store/base.rb +153 -0
- data/lib/aspera/secret/store/encrypted_hash.rb +329 -0
- data/lib/aspera/secret/store/factory.rb +103 -0
- data/lib/aspera/secret/store/hashicorp_vault.rb +390 -0
- data/lib/aspera/secret/store/keeper.rb +141 -0
- data/lib/aspera/secret/store/macos_security.rb +284 -0
- data/lib/aspera/secret/store/one_password_api.rb +99 -0
- data/lib/aspera/secret/store/one_password_base.rb +42 -0
- data/lib/aspera/secret/store/one_password_cli.rb +110 -0
- data/lib/aspera/secret/store/secret_service.rb +125 -0
- data/lib/aspera/secret/strength.rb +119 -0
- data/lib/aspera/servlet_helpers.rb +114 -0
- data/lib/aspera/ssh.rb +10 -7
- data/lib/aspera/ssl.rb +5 -2
- data/lib/aspera/string_ext.rb +3 -0
- data/lib/aspera/sync/args.schema.yaml +6 -1
- data/lib/aspera/sync/conf.schema.yaml +25 -10
- data/lib/aspera/sync/database.rb +3 -0
- data/lib/aspera/sync/operations.rb +34 -12
- data/lib/aspera/temp_file_manager.rb +85 -22
- data/lib/aspera/timer_limiter.rb +3 -0
- data/lib/aspera/transfer/error.rb +4 -1
- data/lib/aspera/transfer/faux_file.rb +4 -1
- data/lib/aspera/transfer/parameters.rb +22 -14
- data/lib/aspera/transfer/result.rb +3 -0
- data/lib/aspera/transfer/resumer.rb +6 -4
- data/lib/aspera/transfer/spec.rb +45 -18
- data/lib/aspera/transfer/spec.schema.yaml +8 -5
- data/lib/aspera/transfer/units.rb +75 -0
- data/lib/aspera/uri_reader.rb +57 -7
- data/lib/aspera/web_auth/index.html.erb +20 -0
- data/lib/aspera/web_auth/paste.rb +109 -0
- data/lib/aspera/web_auth/receiver.rb +50 -0
- data/lib/aspera/web_auth/relay.rb +76 -0
- data/lib/aspera/web_auth/reply.html.erb +150 -0
- data/lib/aspera/web_auth/server.rb +59 -14
- data/lib/aspera/web_page.html.erb +262 -0
- data/lib/aspera/web_page.rb +66 -0
- data/lib/aspera/web_server_simple.rb +82 -13
- data/lib/aspera/yaml.rb +5 -1
- data/llms.txt +81 -0
- data/skills/ascli/SKILL.md +88 -0
- data.tar.gz.sig +0 -0
- metadata +116 -48
- metadata.gz.sig +0 -0
- data/lib/aspera/cli/mcp_tool.rb +0 -189
- data/lib/aspera/keychain/base.rb +0 -25
- data/lib/aspera/keychain/encrypted_hash.rb +0 -188
- data/lib/aspera/keychain/factory.rb +0 -59
- data/lib/aspera/keychain/hashicorp_vault.rb +0 -74
- data/lib/aspera/keychain/macos_security.rb +0 -171
- data/lib/aspera/keychain/one_password_api.rb +0 -86
- data/lib/aspera/keychain/one_password_base.rb +0 -34
- data/lib/aspera/keychain/one_password_cli.rb +0 -98
- data/lib/aspera/oauth/base.rb +0 -141
- data/lib/aspera/oauth/web.rb +0 -57
- data/lib/aspera/oauth.rb +0 -9
- data/lib/aspera/schema/IBM Aspera Orchestrator API-v1.yaml +0 -1784
- data/lib/aspera/schema/IBM Aspera faspio Gateway API-1.0.0.yaml +0 -249
- data/lib/aspera/secret_hider.rb +0 -117
- data/lib/aspera/web_auth/index.html +0 -147
|
@@ -1,7 +1,13 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
# SPDX-FileCopyrightText: IBM Corp.
|
|
4
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
5
|
+
|
|
6
|
+
require_relative 'assert'
|
|
7
|
+
require_relative 'log'
|
|
3
8
|
require 'singleton'
|
|
4
9
|
require 'fileutils'
|
|
10
|
+
require 'securerandom'
|
|
5
11
|
require 'etc'
|
|
6
12
|
require 'tmpdir'
|
|
7
13
|
|
|
@@ -15,21 +21,34 @@ module Aspera
|
|
|
15
21
|
class TempFileManager
|
|
16
22
|
include Singleton
|
|
17
23
|
|
|
18
|
-
|
|
19
|
-
#
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
private_constant :SEC_IN_DAY, :FILE_LIST_AGE_MAX_SEC
|
|
24
|
+
# Assume no transfer lasts longer than this
|
|
25
|
+
# (garbage collect file lists which were not deleted after transfer)
|
|
26
|
+
EXPIRY_DEFAULT_SEC = 5 * 86_400
|
|
27
|
+
private_constant :EXPIRY_DEFAULT_SEC
|
|
23
28
|
|
|
29
|
+
# @return [Boolean] `true`: files created by this execution are deleted by `cleanup`
|
|
24
30
|
attr_accessor :cleanup_on_exit
|
|
31
|
+
# @return [Numeric, nil] Age in seconds of files deleted by `cleanup_expired`, `nil`: never deleted
|
|
32
|
+
attr_reader :expiry_sec
|
|
25
33
|
attr_reader :global_temp
|
|
26
34
|
|
|
27
35
|
def initialize
|
|
28
36
|
@created_files = []
|
|
37
|
+
# Number of holders of a created file: not deleted by cleanup while held
|
|
38
|
+
@hold_counts = {}
|
|
39
|
+
# Files are created and released by transfer threads
|
|
40
|
+
@mutex = Mutex.new
|
|
29
41
|
@cleanup_on_exit = true
|
|
42
|
+
@expiry_sec = EXPIRY_DEFAULT_SEC
|
|
30
43
|
@global_temp = Etc.systmpdir
|
|
31
44
|
end
|
|
32
45
|
|
|
46
|
+
# @param value [Numeric, nil] Age in seconds, `0`: all files, `nil`: never delete
|
|
47
|
+
def expiry_sec=(value)
|
|
48
|
+
Aspera.assert(value.nil? || (value.is_a?(Numeric) && !value.negative?), type: ParameterError) { "Expiry must be a positive duration: #{value}" }
|
|
49
|
+
@expiry_sec = value
|
|
50
|
+
end
|
|
51
|
+
|
|
33
52
|
def global_temp=(value)
|
|
34
53
|
@global_temp = case value
|
|
35
54
|
when '@env' then Dir.tmpdir
|
|
@@ -39,17 +58,45 @@ module Aspera
|
|
|
39
58
|
end
|
|
40
59
|
|
|
41
60
|
def delete_file(filepath)
|
|
42
|
-
|
|
43
|
-
rescue => e
|
|
44
|
-
Log.log.warn { "Problem deleting file: #{filepath}: #{e.message}" }
|
|
61
|
+
remove_file(filepath) if @cleanup_on_exit
|
|
45
62
|
end
|
|
46
63
|
|
|
47
|
-
# Call this on process exit
|
|
64
|
+
# Call this on process exit, or at the end of a command.
|
|
65
|
+
# Held files are kept: they are deleted when released.
|
|
48
66
|
def cleanup
|
|
49
|
-
@
|
|
50
|
-
|
|
67
|
+
@mutex.synchronize do
|
|
68
|
+
@created_files.each do |filepath|
|
|
69
|
+
delete_file(filepath) if !@hold_counts.key?(filepath) && File.file?(filepath)
|
|
70
|
+
end
|
|
71
|
+
@created_files.select! { |filepath| @hold_counts.key?(filepath) }
|
|
72
|
+
end
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
# Keep created files after `cleanup`, until released, for example a file list used by a transfer
|
|
76
|
+
# that ends after the command (asynchronous transfer).
|
|
77
|
+
# A file can be held several times (multi-session): it is deleted when the last holder releases it.
|
|
78
|
+
# @param paths [Array<String>] Files created by this manager (others are ignored)
|
|
79
|
+
def hold(paths)
|
|
80
|
+
@mutex.synchronize do
|
|
81
|
+
paths.each do |filepath|
|
|
82
|
+
@hold_counts[filepath] = @hold_counts.fetch(filepath, 0) + 1 if @created_files.include?(filepath)
|
|
83
|
+
end
|
|
84
|
+
end
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
# Release files held with `hold`, and delete those that are no longer held.
|
|
88
|
+
# @param paths [Array<String>] Files given to `hold`
|
|
89
|
+
def release(paths)
|
|
90
|
+
@mutex.synchronize do
|
|
91
|
+
paths.each do |filepath|
|
|
92
|
+
next unless @hold_counts.key?(filepath)
|
|
93
|
+
@hold_counts[filepath] -= 1
|
|
94
|
+
next if @hold_counts[filepath].positive?
|
|
95
|
+
@hold_counts.delete(filepath)
|
|
96
|
+
@created_files.delete(filepath)
|
|
97
|
+
delete_file(filepath) if File.file?(filepath)
|
|
98
|
+
end
|
|
51
99
|
end
|
|
52
|
-
@created_files = []
|
|
53
100
|
end
|
|
54
101
|
|
|
55
102
|
# Ensure that provided folder exists, or create it, atomically create and reserve a unique file.
|
|
@@ -63,7 +110,7 @@ module Aspera
|
|
|
63
110
|
new_file = Dir::Tmpname.create([basename, suffix ? "-#{suffix}" : ''], temp_folder) do |path|
|
|
64
111
|
File.open(path, File::WRONLY | File::CREAT | File::EXCL, 0o600) { nil }
|
|
65
112
|
end
|
|
66
|
-
@created_files.push(new_file)
|
|
113
|
+
@mutex.synchronize { @created_files.push(new_file) }
|
|
67
114
|
new_file
|
|
68
115
|
end
|
|
69
116
|
|
|
@@ -71,7 +118,7 @@ module Aspera
|
|
|
71
118
|
def new_file_path_global(prefix = nil, suffix: nil)
|
|
72
119
|
username =
|
|
73
120
|
begin
|
|
74
|
-
Etc.getlogin || Etc.getpwuid(Process.uid).name || 'unknown_user'
|
|
121
|
+
Etc.getlogin || Etc.getpwuid(::Process.uid).name || 'unknown_user'
|
|
75
122
|
rescue StandardError
|
|
76
123
|
'unknown_user'
|
|
77
124
|
end
|
|
@@ -85,7 +132,7 @@ module Aspera
|
|
|
85
132
|
def new_dir_path_global(prefix = nil)
|
|
86
133
|
username =
|
|
87
134
|
begin
|
|
88
|
-
Etc.getlogin || Etc.getpwuid(Process.uid).name || 'unknown_user'
|
|
135
|
+
Etc.getlogin || Etc.getpwuid(::Process.uid).name || 'unknown_user'
|
|
89
136
|
rescue StandardError
|
|
90
137
|
'unknown_user'
|
|
91
138
|
end
|
|
@@ -93,17 +140,33 @@ module Aspera
|
|
|
93
140
|
File.join(@global_temp, dir_name)
|
|
94
141
|
end
|
|
95
142
|
|
|
96
|
-
# Garbage collect
|
|
143
|
+
# Garbage collect files not deleted by previous executions (e.g. killed), older than `expiry_sec`.
|
|
144
|
+
# Not disabled by `cleanup_on_exit`, which applies to files of this execution.
|
|
145
|
+
# @param temp_folder [String] Folder that contains only temporary files
|
|
97
146
|
def cleanup_expired(temp_folder)
|
|
98
|
-
|
|
147
|
+
return if @expiry_sec.nil?
|
|
148
|
+
now = Time.now
|
|
149
|
+
Dir.children(temp_folder).each do |name|
|
|
99
150
|
file_path = File.join(temp_folder, name)
|
|
100
|
-
age_sec = (Time.now - File.stat(file_path).mtime).to_i
|
|
101
151
|
# check age of file, delete too old
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
152
|
+
next unless File.file?(file_path) && (now - File.mtime(file_path)) >= @expiry_sec
|
|
153
|
+
Log.log.debug { "garbage collecting #{name}" }
|
|
154
|
+
remove_file(file_path)
|
|
155
|
+
rescue Errno::ENOENT
|
|
156
|
+
# deleted meanwhile by another execution
|
|
157
|
+
nil
|
|
106
158
|
end
|
|
107
159
|
end
|
|
160
|
+
|
|
161
|
+
private
|
|
162
|
+
|
|
163
|
+
# Delete a file, a failure is logged
|
|
164
|
+
def remove_file(filepath)
|
|
165
|
+
File.delete(filepath)
|
|
166
|
+
rescue Errno::ENOENT
|
|
167
|
+
nil
|
|
168
|
+
rescue => e
|
|
169
|
+
Log.log.warn { "Problem deleting file: #{filepath}: #{e.message}" }
|
|
170
|
+
end
|
|
108
171
|
end
|
|
109
172
|
end
|
data/lib/aspera/timer_limiter.rb
CHANGED
|
@@ -1,12 +1,15 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
# SPDX-FileCopyrightText: IBM Corp.
|
|
4
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
5
|
+
|
|
3
6
|
module Aspera
|
|
4
7
|
module Transfer
|
|
5
8
|
# Generates a pseudo file stream.
|
|
6
9
|
class FauxFile
|
|
7
10
|
SCHEME = 'faux'
|
|
8
11
|
# marker for faux file
|
|
9
|
-
PREFIX = "#{SCHEME}:///"
|
|
12
|
+
PREFIX = "#{SCHEME}:///".freeze
|
|
10
13
|
# size units, kilo, mega ...
|
|
11
14
|
SIZE_UNITS = %w[k m g t p e].freeze
|
|
12
15
|
private_constant :SCHEME, :PREFIX, :SIZE_UNITS
|
|
@@ -1,22 +1,25 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
3
|
+
# SPDX-FileCopyrightText: IBM Corp.
|
|
4
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
5
|
+
|
|
6
|
+
require_relative '../log'
|
|
7
|
+
require_relative '../assert'
|
|
8
|
+
require_relative '../exec_spec'
|
|
9
|
+
require_relative '../command_line_builder'
|
|
10
|
+
require_relative '../temp_file_manager'
|
|
11
|
+
require_relative 'error'
|
|
12
|
+
require_relative 'spec'
|
|
13
|
+
require_relative '../command_line_converter'
|
|
14
|
+
require_relative '../ascp/installation'
|
|
15
|
+
require_relative '../cli/formatter'
|
|
16
|
+
require_relative '../rest'
|
|
17
|
+
require_relative '../rainbow'
|
|
14
18
|
require 'securerandom'
|
|
15
19
|
require 'base64'
|
|
16
20
|
require 'json'
|
|
17
21
|
require 'fileutils'
|
|
18
22
|
require 'openssl'
|
|
19
|
-
require 'aspera/rainbow'
|
|
20
23
|
using Rainbow
|
|
21
24
|
|
|
22
25
|
module Aspera
|
|
@@ -80,6 +83,8 @@ module Aspera
|
|
|
80
83
|
Aspera.assert_values(@client_ssh_key, Ascp::Installation::CLIENT_SSH_KEY_OPTIONS)
|
|
81
84
|
@check_ignore_cb = check_ignore_cb
|
|
82
85
|
@builder = CommandLineBuilder.new(@job_spec, Spec::SCHEMA, CommandLineConverter)
|
|
86
|
+
# temporary files used by `ascp`: given to the agent in the ExecSpec
|
|
87
|
+
@temp_files = []
|
|
83
88
|
end
|
|
84
89
|
|
|
85
90
|
# Place source files on command line or in a temp file list, controlled by @file_list and file_list_folder
|
|
@@ -87,7 +92,7 @@ module Aspera
|
|
|
87
92
|
# is the file list provided through ascp parameters?
|
|
88
93
|
ascp_file_list_provided = self.class.ascp_args_file_list?(@ascp_args)
|
|
89
94
|
# set if paths is mandatory in ts
|
|
90
|
-
@builder.required('paths', !@job_spec.key?('keepalive') && !ascp_file_list_provided)
|
|
95
|
+
@builder.required('paths', !@job_spec.key?('keepalive') && !ascp_file_list_provided)
|
|
91
96
|
# get paths in transfer spec (after setting if it is mandatory)
|
|
92
97
|
ts_paths_array = @builder.read_param('paths')
|
|
93
98
|
file_list_option = nil
|
|
@@ -112,6 +117,7 @@ module Aspera
|
|
|
112
117
|
lines = ts_paths_array.map { |i| i['source'] }
|
|
113
118
|
end
|
|
114
119
|
file_list_file = TempFileManager.instance.new_file_path_in_folder(self.class.file_list_folder)
|
|
120
|
+
@temp_files.push(file_list_file)
|
|
115
121
|
Log.dump(:file_list, lines)
|
|
116
122
|
File.write(file_list_file, lines.join("\n"), encoding: 'UTF-8')
|
|
117
123
|
Log.log.debug { "#{file_list_option}=\n#{File.read(file_list_file)}".red }
|
|
@@ -137,6 +143,7 @@ module Aspera
|
|
|
137
143
|
# ignore cert for wss ?
|
|
138
144
|
if @check_ignore_cb&.call(@job_spec['remote_host'], @job_spec['wss_port'])
|
|
139
145
|
wss_cert_file = TempFileManager.instance.new_file_path_global('wss_cert')
|
|
146
|
+
@temp_files.push(wss_cert_file)
|
|
140
147
|
wss_url = "https://#{@job_spec['remote_host']}:#{@job_spec['wss_port']}"
|
|
141
148
|
File.write(wss_cert_file, Rest.remote_certificate_chain(wss_url))
|
|
142
149
|
# place in front, as more priority
|
|
@@ -213,10 +220,11 @@ module Aspera
|
|
|
213
220
|
end
|
|
214
221
|
# disable redis in client, only for ascp, this makes ascp4 fail
|
|
215
222
|
exec_spec.env['ASPERA_TEST_REDIS_DISABLE'] = 'true' if exec_spec.exec.eql?(:ascp)
|
|
223
|
+
exec_spec.temp_files.concat(@temp_files)
|
|
216
224
|
Log.log.debug { "ascp args: #{exec_spec}" }
|
|
217
225
|
return exec_spec
|
|
218
226
|
end
|
|
219
|
-
DELETE_EQUIV = %w[remove_after_transfer remove_empty_directories remove_empty_source_directory]
|
|
227
|
+
DELETE_EQUIV = %w[remove_after_transfer remove_empty_directories remove_empty_source_directory].freeze
|
|
220
228
|
# `ascp` options to provide a file list
|
|
221
229
|
FILE_LIST_OPTIONS = ['--file-list', '--file-pair-list'].freeze
|
|
222
230
|
private_constant :DELETE_EQUIV, :FILE_LIST_OPTIONS
|
|
@@ -1,9 +1,11 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
3
|
+
# SPDX-FileCopyrightText: IBM Corp.
|
|
4
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
5
|
+
|
|
6
|
+
require_relative '../log'
|
|
7
|
+
require_relative '../assert'
|
|
8
|
+
require_relative 'error'
|
|
7
9
|
|
|
8
10
|
module Aspera
|
|
9
11
|
module Transfer
|
data/lib/aspera/transfer/spec.rb
CHANGED
|
@@ -1,7 +1,12 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
|
|
3
|
+
# SPDX-FileCopyrightText: IBM Corp.
|
|
4
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
5
|
+
|
|
6
|
+
require_relative '../agent/factory'
|
|
7
|
+
require_relative '../command_line_builder'
|
|
8
|
+
require_relative '../assert'
|
|
9
|
+
require_relative 'units'
|
|
5
10
|
|
|
6
11
|
module Aspera
|
|
7
12
|
module Transfer
|
|
@@ -27,6 +32,8 @@ module Aspera
|
|
|
27
32
|
TAG_RESERVED = 'aspera'
|
|
28
33
|
# Specific fields to not display
|
|
29
34
|
SPECIFIC = %w{token paths direction source_root destination_root}.freeze
|
|
35
|
+
# Value of agent for documentation with one column per agent
|
|
36
|
+
DOC_OVERVIEW = :overview
|
|
30
37
|
class << self
|
|
31
38
|
# wrong def in transferd
|
|
32
39
|
POLICY_FIX = {
|
|
@@ -34,14 +41,8 @@ module Aspera
|
|
|
34
41
|
'attrs' => 'attributes',
|
|
35
42
|
'sparse_csum' => 'sparse_checksum',
|
|
36
43
|
'full_csum' => 'full_checksum'
|
|
37
|
-
}
|
|
38
|
-
# Multipliers for target_rate suffix (result is in kbps)
|
|
39
|
-
RATE_SUFFIX_KBPS = {
|
|
40
|
-
'k' => 1,
|
|
41
|
-
'm' => 1_000,
|
|
42
|
-
'g' => 1_000_000
|
|
43
44
|
}.freeze
|
|
44
|
-
private_constant :POLICY_FIX
|
|
45
|
+
private_constant :POLICY_FIX
|
|
45
46
|
# translate upload/download to send/receive
|
|
46
47
|
def transfer_type_to_direction(transfer_type)
|
|
47
48
|
XFER_TYPE_TO_DIR.fetch(transfer_type)
|
|
@@ -52,21 +53,47 @@ module Aspera
|
|
|
52
53
|
XFER_DIR_TO_TYPE.fetch(direction)
|
|
53
54
|
end
|
|
54
55
|
|
|
55
|
-
#
|
|
56
|
-
#
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
Aspera.assert(m) { "Invalid rate value: #{value.inspect}. Expected integer with optional suffix k/K, m/M or g/G." }
|
|
62
|
-
multiplier = m[2] ? RATE_SUFFIX_KBPS.fetch(m[2].downcase) : 1
|
|
63
|
-
return m[1].to_i * multiplier
|
|
56
|
+
# Resolve pseudo-parameter `target_rate` (bps) into `target_rate_kbps` (rounded down), overriding it if both are present.
|
|
57
|
+
# @param transfer_spec [Hash] Transfer spec, modified in place
|
|
58
|
+
def resolve_target_rate(transfer_spec)
|
|
59
|
+
return unless transfer_spec.key?('target_rate')
|
|
60
|
+
rate = transfer_spec.delete('target_rate')
|
|
61
|
+
transfer_spec['target_rate_kbps'] = Units.rate_string_to_bps(rate) / 1000 unless rate.nil?
|
|
64
62
|
end
|
|
65
63
|
|
|
66
64
|
def fix_transferd_resume_policy(transfer_spec)
|
|
67
65
|
# Fix discrepancy in transfer spec
|
|
68
66
|
transfer_spec['resume_policy'] = POLICY_FIX[transfer_spec['resume_policy']] if transfer_spec.key?('resume_policy')
|
|
69
67
|
end
|
|
68
|
+
|
|
69
|
+
# Options of `Schema::Documentation` for the transfer spec: supported agents and `ascp` options
|
|
70
|
+
# @param agent [Symbol, nil] `nil`: all parameters, name of agent: parameters it supports, `DOC_OVERVIEW`: one column per agent
|
|
71
|
+
# @return [Hash] Keyword arguments of `Schema::Documentation.new`
|
|
72
|
+
def doc_options(agent = nil)
|
|
73
|
+
return {annotate: ->(node, formatter) { doc_lines(node, formatter, agent: agent) }} unless agent.eql?(DOC_OVERVIEW)
|
|
74
|
+
{
|
|
75
|
+
annotate: ->(node, formatter) { doc_lines(node, formatter, agents_line: false) },
|
|
76
|
+
extra_columns: Agent::Factory::ALL.sort_by { |_, names| names[:short] }.to_h do |sym, names|
|
|
77
|
+
[names[:short].to_s, ->(node, formatter) { formatter.tick(CommandLineBuilder.supported_by_agent(sym, node)) }]
|
|
78
|
+
end
|
|
79
|
+
}
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
private
|
|
83
|
+
|
|
84
|
+
# Documentation of supported agents and command line option of a parameter: `annotate` of `Schema::Documentation`
|
|
85
|
+
# @param node [Hash] Schema of the parameter
|
|
86
|
+
# @param formatter [FormatterInterface] Formatter of the documentation
|
|
87
|
+
# @param agent [Symbol, nil] Show only parameters supported by this agent, or all if `nil`
|
|
88
|
+
# @param agents_line [Boolean] `true`: add a line with supported agents, if not all
|
|
89
|
+
# @return [Array<String>, nil] Lines added to the description, `nil` to hide the parameter
|
|
90
|
+
def doc_lines(node, formatter, agent: nil, agents_line: true)
|
|
91
|
+
agents = Agent::Factory::ALL.keys.select { |sym| CommandLineBuilder.supported_by_agent(sym, node) }
|
|
92
|
+
return if agents.empty? || (agent && !agents.include?(agent))
|
|
93
|
+
lines = []
|
|
94
|
+
lines.push("(#{agents.map { |i| Agent::Factory::ALL[i][:short].to_s.upcase }.sort.join(', ')})") if agents_line && !agents.length.eql?(Agent::Factory::ALL.length)
|
|
95
|
+
lines.concat(CommandLineBuilder.doc_lines(node, formatter))
|
|
96
|
+
end
|
|
70
97
|
end
|
|
71
98
|
SCHEMA = CommandLineBuilder.read_schema(Schema::Registry::TRANSFER_SPEC, ascp: true)
|
|
72
99
|
# Define constants for enums of parameters: <parameter>_<enum>, e.g. CIPHER_AES_128, DIRECTION_SEND, ...
|
|
@@ -757,11 +757,14 @@ properties:
|
|
|
757
757
|
x-cli-option: -l
|
|
758
758
|
target_rate:
|
|
759
759
|
description: >-
|
|
760
|
-
Pseudo-parameter: desired transfer rate
|
|
761
|
-
Converted to `target_rate_kbps` before the transfer is started, overriding it if both are present.
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
760
|
+
Pseudo-parameter: desired transfer rate in bps, with an optional unit suffix.
|
|
761
|
+
Converted to `target_rate_kbps` (rounded down) before the transfer is started, overriding it if both are present.
|
|
762
|
+
Also applies to `async` (sync).
|
|
763
|
+
Accepted formats: integer (bps), or integer followed by `k`/`K` (x1000), `m`/`M` (x1000000), `g`/`G` (x1000000000).
|
|
764
|
+
Examples: `100000000`, `100000k`, `100m`, `1g`.
|
|
765
|
+
type:
|
|
766
|
+
- string
|
|
767
|
+
- integer
|
|
765
768
|
x-cli-special: true
|
|
766
769
|
title:
|
|
767
770
|
description: Title of the transfer.
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# SPDX-FileCopyrightText: IBM Corp.
|
|
4
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
5
|
+
|
|
6
|
+
require_relative '../assert'
|
|
7
|
+
|
|
8
|
+
module Aspera
|
|
9
|
+
module Transfer
|
|
10
|
+
# Rates, sizes and durations of transfers: parsed from user input and formatted for display.
|
|
11
|
+
# Decimal units, like `ascp`. Depends only on `aspera/assert`: usable without the CLI.
|
|
12
|
+
module Units
|
|
13
|
+
# Decimal prefixes: each one is `KILO` times the previous one
|
|
14
|
+
PREFIX = ['', 'K', 'M', 'G', 'T', 'P'].freeze
|
|
15
|
+
KILO = 1000
|
|
16
|
+
# Multiplier of each prefix, e.g. `'M' => 1_000_000`
|
|
17
|
+
MULTIPLIER = PREFIX.each_with_index.to_h { |prefix, power| [prefix, KILO**power] }.freeze
|
|
18
|
+
# Units of sizes: `B`, `KB`, `MB`...
|
|
19
|
+
BYTE_UNITS = PREFIX.map { |prefix| "#{prefix}B" }.freeze
|
|
20
|
+
# Rate typed by a user: integer with an optional prefix, case ignored
|
|
21
|
+
RATE_INPUT = /\A(\d+)([#{PREFIX.join}])?\z/i
|
|
22
|
+
BITS_PER_BYTE = 8
|
|
23
|
+
private_constant :MULTIPLIER, :BYTE_UNITS, :RATE_INPUT, :BITS_PER_BYTE
|
|
24
|
+
|
|
25
|
+
class << self
|
|
26
|
+
# Parse a human-readable rate (as accepted by async -l) into an integer bps value.
|
|
27
|
+
# Accepted: plain integer (bps), or integer + prefix of `PREFIX`, case ignored, e.g. `k` (x1000), `M` (x1000000), `g` (x1000000000).
|
|
28
|
+
# @param value [String, Integer] e.g. "100m", "500000", "1g", 500000
|
|
29
|
+
# @return [Integer] value in bps
|
|
30
|
+
def rate_string_to_bps(value)
|
|
31
|
+
m = RATE_INPUT.match(value.to_s.strip)
|
|
32
|
+
Aspera.assert(m) { "Invalid rate value: #{value.inspect}. Expected integer with optional suffix #{PREFIX.drop(1).join(', ')} (case ignored)." }
|
|
33
|
+
return m[1].to_i * MULTIPLIER.fetch(m[2].to_s.upcase)
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
# Inverse of `rate_string_to_bps`: the shortest exact text, e.g. `100M`, `1G`, `1500K`, `999`
|
|
37
|
+
# @param bps [Integer] Rate in bits per second
|
|
38
|
+
# @return [String] Rate accepted by `rate_string_to_bps`
|
|
39
|
+
def rate_bps_to_string(bps)
|
|
40
|
+
prefix = PREFIX.reverse.find { |candidate| bps.positive? && (bps % MULTIPLIER[candidate]).zero? } || ''
|
|
41
|
+
"#{bps / MULTIPLIER[prefix]}#{prefix}"
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
# @param bytes [Integer] Size in bytes
|
|
45
|
+
# @return [String] Size with decimal unit, e.g. `12.3 MB`
|
|
46
|
+
def format_bytes(bytes)
|
|
47
|
+
value = bytes.to_f
|
|
48
|
+
index = 0
|
|
49
|
+
while value >= KILO && index < BYTE_UNITS.length - 1
|
|
50
|
+
value /= KILO
|
|
51
|
+
index += 1
|
|
52
|
+
end
|
|
53
|
+
return "#{bytes} B" if index.zero?
|
|
54
|
+
format('%.1f %s', value, BYTE_UNITS[index])
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
# @param bytes_per_second [Float, nil] Rate, `nil` if unknown
|
|
58
|
+
# @return [String] Rate in megabits per second (decimal, like `ascp`)
|
|
59
|
+
def format_rate(bytes_per_second)
|
|
60
|
+
return '-- Mbps' if bytes_per_second.nil?
|
|
61
|
+
mbps = bytes_per_second * BITS_PER_BYTE / MULTIPLIER.fetch('M')
|
|
62
|
+
mbps < 10 ? format('%.2f Mbps', mbps) : format('%.0f Mbps', mbps)
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
# @param seconds [Numeric, nil] Duration, `nil` if unknown
|
|
66
|
+
# @return [String] Duration as `hh:mm:ss`
|
|
67
|
+
def format_duration(seconds)
|
|
68
|
+
return '--:--:--' if seconds.nil?
|
|
69
|
+
seconds = seconds.round
|
|
70
|
+
format('%02d:%02d:%02d', seconds / 3600, (seconds / 60) % 60, seconds % 60)
|
|
71
|
+
end
|
|
72
|
+
end
|
|
73
|
+
end
|
|
74
|
+
end
|
|
75
|
+
end
|
data/lib/aspera/uri_reader.rb
CHANGED
|
@@ -1,10 +1,14 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
# SPDX-FileCopyrightText: IBM Corp.
|
|
4
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
5
|
+
|
|
6
|
+
require_relative 'assert'
|
|
7
|
+
require_relative 'log'
|
|
8
|
+
require_relative 'rest'
|
|
9
|
+
require_relative 'temp_file_manager'
|
|
3
10
|
require 'uri'
|
|
4
11
|
require 'base64'
|
|
5
|
-
require 'aspera/assert'
|
|
6
|
-
require 'aspera/rest'
|
|
7
|
-
require 'aspera/temp_file_manager'
|
|
8
12
|
|
|
9
13
|
module Aspera
|
|
10
14
|
# Read content from a URI; supported schemes: file:, http:, https:, data:
|
|
@@ -25,13 +29,15 @@ module Aspera
|
|
|
25
29
|
# The canonical form built by {file_url} uses the `file:///` prefix.
|
|
26
30
|
module UriReader
|
|
27
31
|
SCHEME_FILE = 'file'
|
|
28
|
-
SCHEME_FILE_PFX1 = "#{SCHEME_FILE}:"
|
|
32
|
+
SCHEME_FILE_PFX1 = "#{SCHEME_FILE}:".freeze
|
|
29
33
|
# Canonical prefix for file: URLs (no host, three slashes).
|
|
30
34
|
# What follows is the literal path: relative or absolute (starting with a second slash).
|
|
31
|
-
SCHEME_FILE_PFX2 = "#{SCHEME_FILE_PFX1}///"
|
|
32
|
-
|
|
35
|
+
SCHEME_FILE_PFX2 = "#{SCHEME_FILE_PFX1}///".freeze
|
|
36
|
+
# Timeout in seconds when another URL remains to be tried
|
|
37
|
+
FAIL_FAST_TIMEOUT = 10
|
|
38
|
+
private_constant :SCHEME_FILE, :SCHEME_FILE_PFX1, :SCHEME_FILE_PFX2, :FAIL_FAST_TIMEOUT
|
|
33
39
|
class << self
|
|
34
|
-
# @return [Boolean] true if `url` uses the file: scheme
|
|
40
|
+
# @return [Boolean] true if `url` uses the file: scheme recognized by this module
|
|
35
41
|
def file?(url)
|
|
36
42
|
url.start_with?(SCHEME_FILE_PFX1)
|
|
37
43
|
end
|
|
@@ -87,6 +93,50 @@ module Aspera
|
|
|
87
93
|
end
|
|
88
94
|
end
|
|
89
95
|
|
|
96
|
+
# Read URL with a short timeout and no retry (temporarily changes global HTTP parameters)
|
|
97
|
+
# @param url [String] URL to read
|
|
98
|
+
# @return [String] content
|
|
99
|
+
def read_fail_fast(url)
|
|
100
|
+
parameters = Rest::Parameters.instance
|
|
101
|
+
session_cb = parameters.session_cb
|
|
102
|
+
retry_max = parameters.retry_max
|
|
103
|
+
parameters.retry_max = 0
|
|
104
|
+
parameters.session_cb = lambda do |http_session|
|
|
105
|
+
session_cb&.call(http_session)
|
|
106
|
+
http_session.open_timeout = [http_session.open_timeout, FAIL_FAST_TIMEOUT].compact.min
|
|
107
|
+
http_session.read_timeout = [http_session.read_timeout, FAIL_FAST_TIMEOUT].compact.min
|
|
108
|
+
end
|
|
109
|
+
read(url)
|
|
110
|
+
ensure
|
|
111
|
+
parameters.session_cb = session_cb
|
|
112
|
+
parameters.retry_max = retry_max
|
|
113
|
+
end
|
|
114
|
+
|
|
115
|
+
# Read the first URL that provides valid content.
|
|
116
|
+
# All but the last URL are read with a short timeout and no retry, so that fallback is quick.
|
|
117
|
+
# On failure of a URL (read error, or validation error), a warning is logged and the next URL is tried.
|
|
118
|
+
# The failure of the last URL is raised.
|
|
119
|
+
# @param urls [Array<String>] URLs, in order of preference
|
|
120
|
+
# @param what [String] description of the content, for messages
|
|
121
|
+
# @yield [content] Optional: validates and converts content, shall raise on invalid content
|
|
122
|
+
# @yieldparam content [String] content read
|
|
123
|
+
# @yieldreturn [Object] value returned by this method
|
|
124
|
+
# @return [Object] result of block if given, else content
|
|
125
|
+
def read_first(urls, what: 'content')
|
|
126
|
+
Aspera.assert(!urls.empty?) { "no URL for #{what}" }
|
|
127
|
+
urls.each_with_index do |url, index|
|
|
128
|
+
last = index.eql?(urls.length - 1)
|
|
129
|
+
Log.log.debug { "Retrieving #{what} from #{url}" }
|
|
130
|
+
begin
|
|
131
|
+
content = last ? read(url) : read_fail_fast(url)
|
|
132
|
+
return block_given? ? yield(content) : content
|
|
133
|
+
rescue StandardError => e
|
|
134
|
+
raise if last
|
|
135
|
+
Log.log.warn { "Failed to retrieve #{what} from #{url}: #{e.message}, trying next" }
|
|
136
|
+
end
|
|
137
|
+
end
|
|
138
|
+
end
|
|
139
|
+
|
|
90
140
|
# Return the local file-system path for the content at `url`, downloading to a temp file if needed.
|
|
91
141
|
# For file: URLs the path is extracted directly (no download).
|
|
92
142
|
# For data: and http(s): URLs the content is written to a temporary file and its path is returned.
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
<%# Content of the page displayed after authentication, in layout of Aspera::WebPage %>
|
|
2
|
+
<div class="icon-circle ok">
|
|
3
|
+
<svg xmlns="http://www.w3.org/2000/svg" width="30" height="30" viewBox="0 0 32 32" fill="none" stroke="currentColor" stroke-width="3" stroke-linecap="round" stroke-linejoin="round">
|
|
4
|
+
<polyline points="6 16 13 23 26 9"/>
|
|
5
|
+
</svg>
|
|
6
|
+
</div>
|
|
7
|
+
|
|
8
|
+
<h1>Authentication Received</h1>
|
|
9
|
+
<p>Return to your terminal: it completes the login and shows the result. You can then close this window.</p>
|
|
10
|
+
|
|
11
|
+
<% unless additional_info.to_s.empty? %>
|
|
12
|
+
<div class="info-box">
|
|
13
|
+
<%= h additional_info %>
|
|
14
|
+
</div>
|
|
15
|
+
<% end %>
|
|
16
|
+
|
|
17
|
+
<%# The callback URL may contain a code or tokens: remove the query from the address bar and from the entry of the session history %>
|
|
18
|
+
<script>
|
|
19
|
+
if (window.history && history.replaceState) history.replaceState(null, '', location.pathname);
|
|
20
|
+
</script>
|