aspera-cli 4.26.1 → 4.27.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/CHANGELOG.md +300 -22
- data/CONTRIBUTING.md +100 -12
- data/bin/ascli +5 -5
- data/docs/README.md +10757 -0
- data/docs/test-mcp-with-ai.md +202 -0
- data/lib/aspera/agent/base.rb +20 -6
- data/lib/aspera/agent/connect.rb +53 -16
- data/lib/aspera/agent/desktop.rb +52 -17
- data/lib/aspera/agent/direct.rb +27 -25
- data/lib/aspera/agent/factory.rb +4 -0
- data/lib/aspera/agent/httpgw.rb +2 -2
- data/lib/aspera/agent/node.rb +34 -2
- data/lib/aspera/agent/transferd.rb +39 -3
- data/lib/aspera/api/alee.rb +1 -1
- data/lib/aspera/api/aoc.rb +56 -51
- data/lib/aspera/api/cos_node.rb +2 -2
- data/lib/aspera/api/faspex.rb +22 -20
- data/lib/aspera/api/httpgw.rb +4 -4
- data/lib/aspera/api/node.rb +33 -30
- data/lib/aspera/api/queries/bss_subscription_account.graphql +18 -0
- data/lib/aspera/api/queries/bss_subscription_usage.graphql +18 -0
- data/lib/aspera/ascmd.rb +25 -23
- data/lib/aspera/ascp/installation.rb +43 -40
- data/lib/aspera/ascp/management.rb +2 -3
- data/lib/aspera/cli/ascp_actions.rb +155 -0
- data/lib/aspera/cli/async_transfer_store.rb +81 -0
- data/lib/aspera/cli/bootstrapper.rb +197 -0
- data/lib/aspera/cli/command_registry.rb +145 -0
- data/lib/aspera/cli/command_spec.rb +138 -0
- data/lib/aspera/cli/context.rb +71 -0
- data/lib/aspera/cli/error.rb +16 -0
- data/lib/aspera/cli/extended_value.rb +5 -3
- data/lib/aspera/cli/formatter.rb +122 -209
- data/lib/aspera/cli/gem_checker.rb +65 -0
- data/lib/aspera/cli/hints.rb +2 -2
- data/lib/aspera/cli/http.rb +218 -0
- data/lib/aspera/cli/info.rb +2 -0
- data/lib/aspera/cli/mailer.rb +97 -0
- data/lib/aspera/cli/mcp_tool.rb +198 -0
- data/lib/aspera/cli/options.schema.yaml +364 -7
- data/lib/aspera/cli/{manager.rb → parser.rb} +337 -144
- data/lib/aspera/cli/plugins/alee.rb +20 -22
- data/lib/aspera/cli/plugins/aoc.rb +1173 -849
- data/lib/aspera/cli/plugins/ats.rb +200 -161
- data/lib/aspera/cli/plugins/base.rb +470 -107
- data/lib/aspera/cli/plugins/basic_auth.rb +14 -4
- data/lib/aspera/cli/plugins/config.rb +434 -1032
- data/lib/aspera/cli/plugins/console.rb +106 -64
- data/lib/aspera/cli/plugins/cos.rb +44 -32
- data/lib/aspera/cli/plugins/factory.rb +7 -4
- data/lib/aspera/cli/plugins/faspex.rb +296 -259
- data/lib/aspera/cli/plugins/faspex5.rb +592 -344
- data/lib/aspera/cli/plugins/faspio.rb +49 -51
- data/lib/aspera/cli/plugins/httpgw.rb +18 -25
- data/lib/aspera/cli/plugins/mcp.rb +279 -0
- data/lib/aspera/cli/plugins/node.rb +1001 -797
- data/lib/aspera/cli/plugins/oauth.rb +7 -10
- data/lib/aspera/cli/plugins/orchestrator.rb +111 -139
- data/lib/aspera/cli/plugins/preview.rb +183 -142
- data/lib/aspera/cli/plugins/server.rb +125 -80
- data/lib/aspera/cli/plugins/shares.rb +301 -107
- data/lib/aspera/cli/preset_actions.rb +139 -0
- data/lib/aspera/cli/preset_manager.rb +236 -0
- data/lib/aspera/cli/result.rb +360 -0
- data/lib/aspera/cli/runner.rb +473 -0
- data/lib/aspera/cli/secret_finder.rb +40 -0
- data/lib/aspera/cli/special_values.rb +1 -0
- data/lib/aspera/cli/sync_actions.rb +84 -56
- data/lib/aspera/cli/terminal_formatter.rb +65 -0
- data/lib/aspera/cli/transfer_actions.rb +83 -0
- data/lib/aspera/cli/transfer_agent.rb +116 -51
- data/lib/aspera/cli/transfer_progress.rb +9 -9
- data/lib/aspera/cli/vault_manager.rb +57 -0
- data/lib/aspera/cli/version.rb +1 -1
- data/lib/aspera/cli/wizard.rb +26 -26
- data/lib/aspera/command_line_builder.rb +23 -23
- data/lib/aspera/data_repository.rb +6 -1
- data/lib/aspera/dot_container.rb +5 -5
- data/lib/aspera/environment.rb +39 -13
- data/lib/aspera/exec_spec.rb +13 -0
- data/lib/aspera/faspex_gw.rb +2 -2
- data/lib/aspera/faspex_postproc.rb +4 -3
- data/lib/aspera/graphql.rb +35 -0
- data/lib/aspera/hash_ext.rb +6 -0
- data/lib/aspera/json_rpc/client.rb +62 -0
- data/lib/aspera/json_rpc/version.rb +7 -0
- data/lib/aspera/keychain/base.rb +1 -1
- data/lib/aspera/keychain/encrypted_hash.rb +2 -2
- data/lib/aspera/keychain/factory.rb +4 -4
- data/lib/aspera/keychain/macos_security.rb +4 -4
- data/lib/aspera/link_header.rb +82 -0
- data/lib/aspera/log.rb +23 -5
- data/lib/aspera/markdown.rb +85 -2
- data/lib/aspera/nagios.rb +2 -2
- data/lib/aspera/node_simulator.rb +3 -1
- data/lib/aspera/oauth/base.rb +10 -10
- data/lib/aspera/oauth/boot.rb +43 -0
- data/lib/aspera/oauth/factory.rb +41 -15
- data/lib/aspera/oauth/jwt.rb +4 -4
- data/lib/aspera/oauth/web.rb +4 -4
- data/lib/aspera/oauth.rb +1 -0
- data/lib/aspera/persistency_action_once.rb +10 -10
- data/lib/aspera/persistency_folder.rb +10 -3
- data/lib/aspera/preview/file_types.rb +7 -3
- data/lib/aspera/preview/generator.rb +3 -3
- data/lib/aspera/preview/utils.rb +1 -1
- data/lib/aspera/products/connect.rb +4 -1
- data/lib/aspera/products/other.rb +1 -1
- data/lib/aspera/products/transferd.rb +3 -3
- data/lib/aspera/proxy_auto_config.rb +10 -9
- data/lib/aspera/rest.rb +18 -40
- data/lib/aspera/rest_error_analyzer.rb +7 -3
- data/lib/aspera/rest_errors_aspera.rb +0 -10
- data/lib/aspera/rest_list.rb +13 -6
- data/lib/aspera/schema/IBM Aspera Faspex API-5.0-enhanced.yaml +14 -4
- data/lib/aspera/schema/IBM Aspera on Cloud API-0.2.6-enhanced.yaml +2730 -648
- data/lib/aspera/schema/async_tables.yaml +361 -0
- data/lib/aspera/schema/documentation.rb +57 -26
- data/lib/aspera/schema/reader.rb +60 -11
- data/lib/aspera/schema/registry.rb +50 -10
- data/lib/aspera/secret_hider.rb +11 -2
- data/lib/aspera/ssl.rb +4 -4
- data/lib/aspera/sync/conf.schema.yaml +2 -2
- data/lib/aspera/sync/database.rb +14 -10
- data/lib/aspera/sync/operations.rb +15 -18
- data/lib/aspera/temp_file_manager.rb +5 -1
- data/lib/aspera/timer_limiter.rb +1 -1
- data/lib/aspera/transfer/faux_file.rb +24 -11
- data/lib/aspera/transfer/parameters.rb +27 -25
- data/lib/aspera/transfer/result.rb +74 -0
- data/lib/aspera/transfer/resumer.rb +7 -7
- data/lib/aspera/transfer/spec.rb +18 -1
- data/lib/aspera/transfer/spec.schema.yaml +26 -8
- data/lib/aspera/transfer/uri.rb +1 -1
- data/lib/aspera/uri_reader.rb +2 -2
- data/lib/aspera/web_auth.rb +2 -2
- data/lib/aspera/web_server_simple.rb +8 -5
- data/lib/aspera/yaml.rb +2 -1
- data.tar.gz.sig +0 -0
- metadata +45 -16
- metadata.gz.sig +0 -0
- data/lib/aspera/cli/main.rb +0 -480
- data/lib/aspera/json_rpc.rb +0 -51
|
@@ -7,6 +7,9 @@ require 'aspera/schema/reader'
|
|
|
7
7
|
module Aspera
|
|
8
8
|
# base class for plugins modules
|
|
9
9
|
module Schema
|
|
10
|
+
# @!method self.instance
|
|
11
|
+
# Returns the singleton instance of Registry
|
|
12
|
+
# @return [Registry] the singleton instance
|
|
10
13
|
class Registry
|
|
11
14
|
include Singleton
|
|
12
15
|
|
|
@@ -15,18 +18,32 @@ module Aspera
|
|
|
15
18
|
LOCATIONS.key?(sym)
|
|
16
19
|
end
|
|
17
20
|
|
|
21
|
+
# Get path to request body schema, no check if it exists
|
|
22
|
+
# @param component [String] registry key (e.g. 'faspex', 'aoc')
|
|
23
|
+
# @param endpoint [String] endpoint path without leading slash (e.g. 'packages.post')
|
|
24
|
+
# @return [String] schema path usable in schema: keyword
|
|
18
25
|
def req_body(component, endpoint)
|
|
19
26
|
"#{component}:paths./#{endpoint}.requestBody.content.application/json.schema"
|
|
20
27
|
end
|
|
28
|
+
|
|
29
|
+
# Get path to query parameters for a GET endpoint
|
|
30
|
+
# @param component [String] registry key (e.g. 'faspex', 'aoc')
|
|
31
|
+
# @param endpoint [String] resource path without leading slash (e.g. 'packages')
|
|
32
|
+
# @param method [String] HTTP method (default: 'get')
|
|
33
|
+
# @return [String] schema path usable in query_schema: keyword
|
|
34
|
+
def query_params(component, endpoint, method: 'get')
|
|
35
|
+
"#{component}:paths./#{endpoint}.#{method}#{QUERY_PARAMS_SUFFIX}"
|
|
36
|
+
end
|
|
21
37
|
end
|
|
22
38
|
|
|
23
39
|
LOCATIONS = {
|
|
24
|
-
spec:
|
|
25
|
-
args:
|
|
26
|
-
conf:
|
|
27
|
-
opts:
|
|
28
|
-
aoc:
|
|
29
|
-
faspex:
|
|
40
|
+
spec: 'aspera/transfer/spec.schema.yaml',
|
|
41
|
+
args: 'aspera/sync/args.schema.yaml',
|
|
42
|
+
conf: 'aspera/sync/conf.schema.yaml',
|
|
43
|
+
opts: 'aspera/cli/options.schema.yaml',
|
|
44
|
+
aoc: 'aspera/schema/IBM Aspera on Cloud API-0.2.6-enhanced.yaml',
|
|
45
|
+
faspex: 'aspera/schema/IBM Aspera Faspex API-5.0-enhanced.yaml',
|
|
46
|
+
async_tables: 'aspera/schema/async_tables.yaml'
|
|
30
47
|
}
|
|
31
48
|
|
|
32
49
|
OPTIONS = 'opts'
|
|
@@ -35,18 +52,35 @@ module Aspera
|
|
|
35
52
|
SYNC_ARGS = 'args'
|
|
36
53
|
AOC = 'aoc'
|
|
37
54
|
FASPEX = 'faspex'
|
|
38
|
-
|
|
55
|
+
ASYNC_TABLES = 'async_tables'
|
|
56
|
+
LOG_OPTIONS = "#{OPTIONS}:components.schemas.LogOptions"
|
|
57
|
+
DIRECT_AGENT_OPTIONS = "#{OPTIONS}:components.schemas.DirectAgentOptions"
|
|
58
|
+
NODE_AGENT_OPTIONS = "#{OPTIONS}:components.schemas.NodeAgentOptions"
|
|
59
|
+
HTTPGW_AGENT_OPTIONS = "#{OPTIONS}:components.schemas.HttpgwAgentOptions"
|
|
60
|
+
TRANSFERD_AGENT_OPTIONS = "#{OPTIONS}:components.schemas.TransferdAgentOptions"
|
|
61
|
+
TRANSFER_AGENT_OPTIONS = "#{OPTIONS}:components.schemas.TransferAgentOptions"
|
|
62
|
+
SMTP_OPTIONS = "#{OPTIONS}:components.schemas.SmtpOptions"
|
|
63
|
+
HTTP_OPTIONS = "#{OPTIONS}:components.schemas.HttpOptions"
|
|
64
|
+
VAULT_OPTIONS = "#{OPTIONS}:components.schemas.VaultOptions"
|
|
65
|
+
IMAGE_OPTIONS = "#{OPTIONS}:components.schemas.ImageOptions"
|
|
66
|
+
PACKAGE_FOLDER_OPTIONS = "#{OPTIONS}:components.schemas.PackageFolderOptions"
|
|
39
67
|
|
|
40
68
|
REQ_BODY = '.requestBody.content.application/json.schema'
|
|
69
|
+
# Suffix appended to a dotted path to signal query-param extraction in reader()
|
|
70
|
+
QUERY_PARAMS_SUFFIX = '.parameters'
|
|
41
71
|
|
|
42
72
|
def initialize
|
|
43
73
|
@cache = {}
|
|
44
74
|
@main_folder = File.expand_path('../..', __dir__)
|
|
45
75
|
end
|
|
46
76
|
|
|
47
|
-
# Read schema from file or from cache
|
|
48
|
-
#
|
|
49
|
-
#
|
|
77
|
+
# Read schema from file or from cache.
|
|
78
|
+
# When name_path ends with QUERY_PARAMS_SUFFIX, the OAS `parameters` array at that path
|
|
79
|
+
# is synthesised into an object schema via Reader.from_query_params instead of navigating
|
|
80
|
+
# into the tree.
|
|
81
|
+
# @param name_path [String] registry key with optional colon-separated dotted path suffix,
|
|
82
|
+
# e.g. "faspex:paths./packages.get.parameters" or "faspex:paths./packages.post.requestBody..."
|
|
83
|
+
# @return [Reader] schema reader
|
|
50
84
|
def reader(name_path)
|
|
51
85
|
name, path = name_path.split(':', 2)
|
|
52
86
|
sym = name.to_sym
|
|
@@ -54,6 +88,12 @@ module Aspera
|
|
|
54
88
|
spec_file = File.join(@main_folder, LOCATIONS[sym])
|
|
55
89
|
@cache[sym] = Yaml.safe_load(File.read(spec_file)) if spec_file.end_with?('.yaml') && !@cache.key?(sym)
|
|
56
90
|
@cache[sym] = JSON.parse(File.read(spec_file)) if spec_file.end_with?('.json') && !@cache.key?(sym)
|
|
91
|
+
# Query-params path: strip the suffix, navigate to the operation node, extract parameters
|
|
92
|
+
if path&.end_with?(QUERY_PARAMS_SUFFIX)
|
|
93
|
+
parent_path = path.delete_suffix(QUERY_PARAMS_SUFFIX)
|
|
94
|
+
node = @cache[sym].dig(*parent_path.split('.'))
|
|
95
|
+
return Reader.from_query_params(node&.fetch('parameters', []) || [])
|
|
96
|
+
end
|
|
57
97
|
reader = Reader.new(@cache[sym])
|
|
58
98
|
return reader unless path
|
|
59
99
|
reader.dig(*path.split('.'))
|
data/lib/aspera/secret_hider.rb
CHANGED
|
@@ -6,6 +6,10 @@ require 'singleton'
|
|
|
6
6
|
|
|
7
7
|
module Aspera
|
|
8
8
|
# remove secret from logs and output
|
|
9
|
+
#
|
|
10
|
+
# @!method self.instance
|
|
11
|
+
# Returns the singleton instance of SecretHider
|
|
12
|
+
# @return [SecretHider] the singleton instance
|
|
9
13
|
class SecretHider
|
|
10
14
|
include Singleton
|
|
11
15
|
|
|
@@ -41,7 +45,7 @@ module Aspera
|
|
|
41
45
|
private_constant :HIDDEN_PASSWORD, :ASCP_ENV_SECRETS, :KEY_SECRETS, :HTTP_SECRETS, :ALL_SECRETS, :KEY_FALSE_POSITIVES, :REGEX_LOG_REPLACES
|
|
42
46
|
attr_accessor :log_secrets
|
|
43
47
|
|
|
44
|
-
# @return new log formatter that hides secrets
|
|
48
|
+
# @return [Proc] new log formatter that hides secrets
|
|
45
49
|
def log_formatter(original_formatter)
|
|
46
50
|
original_formatter ||= Logger::Formatter.new
|
|
47
51
|
# NOTE: that @log_secrets may be set AFTER this init is done, so it's done at runtime
|
|
@@ -55,11 +59,16 @@ module Aspera
|
|
|
55
59
|
end
|
|
56
60
|
end
|
|
57
61
|
|
|
62
|
+
# Replace private key content in a string with the hidden password placeholder
|
|
63
|
+
# @param value [String] Input string possibly containing a private key
|
|
64
|
+
# @return [String] String with private key replaced by placeholder
|
|
58
65
|
def hide_secrets_in_string(value)
|
|
59
66
|
return value.gsub(REGEX_LOG_REPLACES.first){"#{Regexp.last_match(:begin)}#{HIDDEN_PASSWORD}#{Regexp.last_match(:end)}"}
|
|
60
67
|
end
|
|
61
68
|
|
|
62
|
-
# @
|
|
69
|
+
# @param keyword [String, Symbol] Key name to check
|
|
70
|
+
# @param value [String] Associated value (must be a String to be a secret)
|
|
71
|
+
# @return [Boolean] true if the key denotes a secret
|
|
63
72
|
def secret?(keyword, value)
|
|
64
73
|
keyword = keyword.to_s if keyword.is_a?(Symbol)
|
|
65
74
|
# only Strings can be secrets, not booleans, or hash, arrays
|
data/lib/aspera/ssl.rb
CHANGED
|
@@ -12,16 +12,16 @@ module Aspera
|
|
|
12
12
|
@extra_options = OpenSSL::SSL::SSLContext::DEFAULT_PARAMS[:options]
|
|
13
13
|
attr_reader :extra_options
|
|
14
14
|
|
|
15
|
-
def option_list=(
|
|
16
|
-
Aspera.assert_type(
|
|
17
|
-
|
|
15
|
+
def option_list=(options)
|
|
16
|
+
Aspera.assert_type(options, Array){'ssl_options'}
|
|
17
|
+
options.each do |opt|
|
|
18
18
|
Aspera.assert_type(opt, String, Integer){'Expected String or Integer in ssl_options'}
|
|
19
19
|
case opt
|
|
20
20
|
when Integer
|
|
21
21
|
@extra_options = opt
|
|
22
22
|
when String
|
|
23
23
|
name = "OP_#{opt.start_with?('-') ? opt[1..] : opt}".upcase
|
|
24
|
-
|
|
24
|
+
Aspera.assert(OpenSSL::SSL.const_defined?(name), type: Cli::BadArgument){"Unknown ssl_option: #{name}, use one of: #{OpenSSL::SSL.constants.grep(/^OP_/).map{ |c| c.to_s.sub(/^OP_/, '')}.join(', ')}"}
|
|
25
25
|
if opt.start_with?('-')
|
|
26
26
|
@extra_options &= ~OpenSSL::SSL.const_get(name)
|
|
27
27
|
else
|
|
@@ -637,7 +637,7 @@ properties:
|
|
|
637
637
|
Partial transfers may exist if communication disruptions caused the underlying ascp processes to terminate early.
|
|
638
638
|
Note that transfer resumption can only happen if the `reset` option is disabled.
|
|
639
639
|
If an async session starts with `reset` enabled and resume enabled, transfers interrupted during that
|
|
640
|
-
session will be
|
|
640
|
+
session will be resumable, but only if async is then restarted with 'reset' disabled.
|
|
641
641
|
type: object
|
|
642
642
|
properties:
|
|
643
643
|
enabled:
|
|
@@ -654,7 +654,7 @@ properties:
|
|
|
654
654
|
Sets the age limit in days for temporary files that will be preserved
|
|
655
655
|
on cleanup (usually at async's start and stop) for potential transfer resume.
|
|
656
656
|
Temp files older than the given value will be removed
|
|
657
|
-
regardless of whether they might be
|
|
657
|
+
regardless of whether they might be resumable.
|
|
658
658
|
type: integer
|
|
659
659
|
default: "5"
|
|
660
660
|
x-cli-option: --resume-age-days
|
data/lib/aspera/sync/database.rb
CHANGED
|
@@ -35,12 +35,12 @@ class SqLite3Wrapper
|
|
|
35
35
|
end
|
|
36
36
|
|
|
37
37
|
# The table contains a single row
|
|
38
|
-
def single_table(table_name)
|
|
39
|
-
execute("SELECT * FROM #{table_name} LIMIT 1
|
|
38
|
+
def single_table(table_name, sql_suffix = nil)
|
|
39
|
+
execute(["SELECT * FROM #{table_name}", sql_suffix, 'LIMIT 1'].compact.join(' ')).first
|
|
40
40
|
end
|
|
41
41
|
|
|
42
|
-
def full_table(table_name)
|
|
43
|
-
execute("SELECT * FROM #{table_name}")
|
|
42
|
+
def full_table(table_name, sql_suffix = nil)
|
|
43
|
+
execute(["SELECT * FROM #{table_name}", sql_suffix].compact.join(' '))
|
|
44
44
|
end
|
|
45
45
|
|
|
46
46
|
private
|
|
@@ -68,16 +68,20 @@ module Aspera
|
|
|
68
68
|
end
|
|
69
69
|
end
|
|
70
70
|
|
|
71
|
-
def meta
|
|
72
|
-
@db.single_table('sync_snapmeta_table')
|
|
71
|
+
def meta(sql_suffix = nil)
|
|
72
|
+
@db.single_table('sync_snapmeta_table', sql_suffix)
|
|
73
73
|
end
|
|
74
74
|
|
|
75
|
-
def counters
|
|
76
|
-
@db.single_table('sync_snap_counters_table')
|
|
75
|
+
def counters(sql_suffix = nil)
|
|
76
|
+
@db.single_table('sync_snap_counters_table', sql_suffix)
|
|
77
77
|
end
|
|
78
78
|
|
|
79
|
-
def file_info
|
|
80
|
-
@db.full_table('sync_snapdb_table')
|
|
79
|
+
def file_info(sql_suffix = nil)
|
|
80
|
+
@db.full_table('sync_snapdb_table', sql_suffix)
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
def execute(sql)
|
|
84
|
+
@db.execute(sql)
|
|
81
85
|
end
|
|
82
86
|
end
|
|
83
87
|
end
|
|
@@ -4,6 +4,7 @@
|
|
|
4
4
|
|
|
5
5
|
require 'aspera/ascp/installation'
|
|
6
6
|
require 'aspera/agent/direct'
|
|
7
|
+
require 'aspera/exec_spec'
|
|
7
8
|
require 'aspera/command_line_converter'
|
|
8
9
|
require 'aspera/command_line_builder'
|
|
9
10
|
require 'aspera/log'
|
|
@@ -89,11 +90,8 @@ module Aspera
|
|
|
89
90
|
def start(sync_info, opt_ts = nil)
|
|
90
91
|
Log.dump(:sync_params_initial, sync_info)
|
|
91
92
|
Aspera.assert_type(sync_info, Hash)
|
|
92
|
-
Aspera.assert(PARAM_KEYS.any?{ |k| sync_info.key?(k)},
|
|
93
|
-
|
|
94
|
-
args: [],
|
|
95
|
-
env: {}
|
|
96
|
-
}
|
|
93
|
+
Aspera.assert(PARAM_KEYS.any?{ |k| sync_info.key?(k)}, 'At least one of `local` or `sessions` must be present in async parameters', type: Error)
|
|
94
|
+
exec_spec = ExecSpec.new(exec: :async)
|
|
97
95
|
if sync_info.key?('local')
|
|
98
96
|
# `conf` format
|
|
99
97
|
Aspera.assert_type(sync_info['local'], Hash){'local'}
|
|
@@ -115,10 +113,10 @@ module Aspera
|
|
|
115
113
|
remote['private_key_paths'].concat(add_certificates)
|
|
116
114
|
end
|
|
117
115
|
# '--exclusive-mgmt-port=12345', '--arg-err-path=-',
|
|
118
|
-
|
|
116
|
+
exec_spec.args = ["--conf64=#{Base64.strict_encode64(JSON.generate(sync_info))}"]
|
|
119
117
|
Log.dump(:sync_conf, sync_info)
|
|
120
118
|
agent = Agent::Direct.new
|
|
121
|
-
agent.start_and_monitor_process(session: {},
|
|
119
|
+
agent.start_and_monitor_process(session: {}, exec_spec: exec_spec)
|
|
122
120
|
else
|
|
123
121
|
# `args` format
|
|
124
122
|
raise StandardError, "Only 'sessions', and optionally 'instance' keys are allowed" unless
|
|
@@ -141,16 +139,16 @@ module Aspera
|
|
|
141
139
|
Aspera.assert_type(sync_info['instance'], Hash)
|
|
142
140
|
instance_builder = CommandLineBuilder.new(sync_info['instance'], ARGS_INSTANCE_SCHEMA, CommandLineConverter)
|
|
143
141
|
instance_builder.process_params
|
|
144
|
-
instance_builder.
|
|
142
|
+
instance_builder.add_to_exec_spec(exec_spec)
|
|
145
143
|
end
|
|
146
144
|
sync_info['sessions'].each do |session_params|
|
|
147
145
|
Aspera.assert_type(session_params, Hash)
|
|
148
|
-
Aspera.assert(session_params.key?('name')
|
|
146
|
+
Aspera.assert(session_params.key?('name'), 'session must contain at least: name')
|
|
149
147
|
session_builder = CommandLineBuilder.new(session_params, ARGS_SESSION_SCHEMA, CommandLineConverter)
|
|
150
148
|
session_builder.process_params
|
|
151
|
-
session_builder.
|
|
149
|
+
session_builder.add_to_exec_spec(exec_spec)
|
|
152
150
|
end
|
|
153
|
-
Environment.secure_execute(Ascp::Installation.instance.path(:async), *
|
|
151
|
+
Environment.secure_execute(Ascp::Installation.instance.path(:async), *exec_spec.args, env: exec_spec.env)
|
|
154
152
|
end
|
|
155
153
|
return
|
|
156
154
|
end
|
|
@@ -177,7 +175,7 @@ module Aspera
|
|
|
177
175
|
# @param sync_info [Hash] sync parameters in conf or args format
|
|
178
176
|
# @return [Hash] parsed output of asyncadmin
|
|
179
177
|
def admin_status(sync_info)
|
|
180
|
-
Aspera.assert(PARAM_KEYS.any?{ |k| sync_info.key?(k)},
|
|
178
|
+
Aspera.assert(PARAM_KEYS.any?{ |k| sync_info.key?(k)}, 'At least one of `local` or `sessions` must be present in async parameters', type: Error)
|
|
181
179
|
arguments = [ASYNC_ADMIN_EXECUTABLE, '--quiet']
|
|
182
180
|
if sync_info.key?('local')
|
|
183
181
|
# `conf` format
|
|
@@ -209,14 +207,14 @@ module Aspera
|
|
|
209
207
|
# @param sync_info [Hash] sync parameters in conf or args format
|
|
210
208
|
# @return [String, nil] Path to "local DB dir", i.e. folder that contains folders that contain `snap.db`
|
|
211
209
|
def local_db_folder(sync_info)
|
|
212
|
-
Aspera.assert(PARAM_KEYS.any?{ |k| sync_info.key?(k)},
|
|
210
|
+
Aspera.assert(PARAM_KEYS.any?{ |k| sync_info.key?(k)}, 'At least one of `local` or `sessions` must be present in async parameters', type: Error)
|
|
213
211
|
if sync_info.key?('local')
|
|
214
212
|
# `conf` format
|
|
215
213
|
if sync_info.key?('local_db_dir')
|
|
216
214
|
return sync_info['local_db_dir']
|
|
217
215
|
elsif (local_path = sync_info.dig('local', 'path'))
|
|
218
216
|
return local_path
|
|
219
|
-
|
|
217
|
+
else
|
|
220
218
|
raise Error, 'Missing either local_db_dir or local.path'
|
|
221
219
|
end
|
|
222
220
|
else
|
|
@@ -226,15 +224,14 @@ module Aspera
|
|
|
226
224
|
return session['local_db_dir']
|
|
227
225
|
elsif session.key?('local_dir')
|
|
228
226
|
return session['local_dir']
|
|
229
|
-
|
|
227
|
+
else
|
|
230
228
|
raise Error, 'Missing either local_db_dir or local_dir'
|
|
231
229
|
end
|
|
232
230
|
end
|
|
233
|
-
nil
|
|
234
231
|
end
|
|
235
232
|
|
|
236
233
|
def session_name(sync_info)
|
|
237
|
-
Aspera.assert(PARAM_KEYS.any?{ |k| sync_info.key?(k)},
|
|
234
|
+
Aspera.assert(PARAM_KEYS.any?{ |k| sync_info.key?(k)}, 'At least one of `local` or `sessions` must be present in async parameters', type: Error)
|
|
238
235
|
if sync_info.key?('local')
|
|
239
236
|
# `conf` format
|
|
240
237
|
return sync_info['name']
|
|
@@ -325,7 +322,7 @@ module Aspera
|
|
|
325
322
|
end
|
|
326
323
|
end
|
|
327
324
|
path, props = find_option(CONF_SCHEMA, [], option)
|
|
328
|
-
|
|
325
|
+
Aspera.assert(!path.nil?){"Option not found: #{option}"}
|
|
329
326
|
last_key = path.pop
|
|
330
327
|
# navigate in the current result to insert the value
|
|
331
328
|
current = result
|
|
@@ -7,6 +7,10 @@ require 'etc'
|
|
|
7
7
|
module Aspera
|
|
8
8
|
# create a temp file name for a given folder
|
|
9
9
|
# files can be deleted on process exit by calling cleanup
|
|
10
|
+
#
|
|
11
|
+
# @!method self.instance
|
|
12
|
+
# Returns the singleton instance of TempFileManager
|
|
13
|
+
# @return [TempFileManager] the singleton instance
|
|
10
14
|
class TempFileManager
|
|
11
15
|
include Singleton
|
|
12
16
|
|
|
@@ -48,7 +52,7 @@ module Aspera
|
|
|
48
52
|
end
|
|
49
53
|
|
|
50
54
|
# Ensure that provided folder exists, or create it, generate a unique filename
|
|
51
|
-
# @return path to that unique file
|
|
55
|
+
# @return [String] path to that unique file
|
|
52
56
|
def new_file_path_in_folder(temp_folder, prefix: nil, suffix: nil)
|
|
53
57
|
FileUtils.mkdir_p(temp_folder)
|
|
54
58
|
new_file = File.join(temp_folder, [prefix, SecureRandom.uuid, suffix].compact.join('-'))
|
data/lib/aspera/timer_limiter.rb
CHANGED
|
@@ -11,42 +11,55 @@ module Aspera
|
|
|
11
11
|
SIZE_UNITS = %w[k m g t p e].freeze
|
|
12
12
|
private_constant :SCHEME, :PREFIX, :SIZE_UNITS
|
|
13
13
|
class << self
|
|
14
|
-
#
|
|
14
|
+
# Parse a faux URI and return a `FauxFile` instance, or `nil` if the URI does not use the faux scheme.
|
|
15
|
+
# URI format: `faux:///<path>?<size>`, where `<size>` is a decimal integer with an optional
|
|
16
|
+
# case-insensitive unit suffix: `k`, `m`, `g`, `t`, `p`, `e` (powers of 1024).
|
|
17
|
+
# When no suffix is given, the size is interpreted as raw bytes.
|
|
18
|
+
# Examples: `faux:///file.bin?10` (10 bytes), `faux:///file.bin?10m` (10 MiB)
|
|
19
|
+
# @param name [String] source file name, possibly a faux URI
|
|
20
|
+
# @return [FauxFile, nil] `nil` if not a faux scheme, else a `FauxFile` instance
|
|
15
21
|
def create(name)
|
|
16
22
|
return unless name.start_with?(PREFIX)
|
|
17
23
|
name_params = name.delete_prefix(PREFIX).split('?', 2)
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
SIZE_UNITS.each do |s|
|
|
23
|
-
size *= 1024
|
|
24
|
-
break if s.eql?(suffix)
|
|
25
|
-
end
|
|
24
|
+
Aspera.assert(name_params.length.eql?(2), type: Error){"Format: #{PREFIX}<file path>?<size>"}
|
|
25
|
+
m = name_params[1].downcase.match(/^(\d+)([#{SIZE_UNITS.join('')}]?)$/)
|
|
26
|
+
Aspera.assert(m, type: Error){"Format: <integer>[#{SIZE_UNITS.join(',')}]"}
|
|
27
|
+
size = m[2].empty? ? m[1].to_i : m[1].to_i * (1024**(SIZE_UNITS.index(m[2]) + 1))
|
|
26
28
|
return FauxFile.new(name_params[0], size)
|
|
27
29
|
end
|
|
28
30
|
end
|
|
31
|
+
# @return [String] virtual file path
|
|
32
|
+
# @return [Integer] total size in bytes
|
|
29
33
|
attr_reader :path, :size
|
|
30
34
|
|
|
35
|
+
# @param path [String] virtual file path (from the faux URI)
|
|
36
|
+
# @param size [Integer] total number of bytes to produce
|
|
31
37
|
def initialize(path, size)
|
|
32
38
|
@path = path
|
|
33
39
|
@size = size
|
|
34
40
|
@offset = 0
|
|
35
|
-
#
|
|
41
|
+
# cache chunks by size so repeated reads of the same chunk length reuse the same buffer
|
|
36
42
|
@chunk_by_size = {}
|
|
37
43
|
end
|
|
38
44
|
|
|
45
|
+
# Read up to `chunk_size` bytes from the stream and advance the internal offset.
|
|
46
|
+
# Returns `nil` when the stream is exhausted.
|
|
47
|
+
# @param chunk_size [Integer] maximum number of bytes to read
|
|
48
|
+
# @return [String, nil] null-byte string of the bytes actually read, or `nil` at EOF
|
|
39
49
|
def read(chunk_size)
|
|
40
50
|
return if eof?
|
|
41
51
|
bytes_to_read = [chunk_size, @size - @offset].min
|
|
42
52
|
@offset += bytes_to_read
|
|
43
|
-
@chunk_by_size[bytes_to_read]
|
|
53
|
+
@chunk_by_size[bytes_to_read] ||= "\x00" * bytes_to_read
|
|
44
54
|
return @chunk_by_size[bytes_to_read]
|
|
45
55
|
end
|
|
46
56
|
|
|
57
|
+
# No-op: required to satisfy the IO-like interface used by transfer agents.
|
|
58
|
+
# @return [nil]
|
|
47
59
|
def close
|
|
48
60
|
end
|
|
49
61
|
|
|
62
|
+
# @return [Boolean] true when all bytes have been read
|
|
50
63
|
def eof?
|
|
51
64
|
return @offset >= @size
|
|
52
65
|
end
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
require 'aspera/log'
|
|
4
4
|
require 'aspera/assert'
|
|
5
|
+
require 'aspera/exec_spec'
|
|
5
6
|
require 'aspera/command_line_builder'
|
|
6
7
|
require 'aspera/temp_file_manager'
|
|
7
8
|
require 'aspera/transfer/error'
|
|
@@ -38,8 +39,9 @@ module Aspera
|
|
|
38
39
|
@file_list_folder ||= TempFileManager.instance.new_file_path_global('asession_filelists')
|
|
39
40
|
end
|
|
40
41
|
|
|
41
|
-
#
|
|
42
|
-
# @
|
|
42
|
+
# Check whether a file list option is already present in the `ascp` argument list
|
|
43
|
+
# @param ascp_args [Array, nil] `ascp` arguments
|
|
44
|
+
# @return [Boolean] true if a file list option is present
|
|
43
45
|
def ascp_args_file_list?(ascp_args)
|
|
44
46
|
ascp_args&.any?{ |i| FILE_LIST_OPTIONS.include?(i)}
|
|
45
47
|
end
|
|
@@ -47,7 +49,9 @@ module Aspera
|
|
|
47
49
|
|
|
48
50
|
# @param job_spec [Hash] Transfer specification
|
|
49
51
|
# @param ascp_args [Array] Other `ascp` args
|
|
52
|
+
# @param wss [Boolean] `true`: if both SSH and wss in ts: prefer wss
|
|
50
53
|
# @param quiet [Boolean] Remove `ascp` progress bar if `true`
|
|
54
|
+
# @param file_list [Boolean] `true`: provide source list via a temp file, `false`: place directly on command line
|
|
51
55
|
# @param client_ssh_key [:rsa,:dsa] Type of Aspera Client SSH key to use.
|
|
52
56
|
# @param trusted_certs [Array<String>] List of path to trusted certificates stores when using WSS
|
|
53
57
|
# @param check_ignore_cb [Proc] Callback to check if WSS connection shall ignore certificate validity
|
|
@@ -56,6 +60,7 @@ module Aspera
|
|
|
56
60
|
ascp_args: nil,
|
|
57
61
|
wss: true,
|
|
58
62
|
quiet: true,
|
|
63
|
+
file_list: true,
|
|
59
64
|
trusted_certs: nil,
|
|
60
65
|
client_ssh_key: nil,
|
|
61
66
|
check_ignore_cb: nil
|
|
@@ -66,6 +71,7 @@ module Aspera
|
|
|
66
71
|
Aspera.assert_array_all(@ascp_args, String){'ascp_args'}
|
|
67
72
|
@wss = wss
|
|
68
73
|
@quiet = quiet
|
|
74
|
+
@file_list = file_list
|
|
69
75
|
@trusted_certs = trusted_certs.nil? ? [] : trusted_certs
|
|
70
76
|
Aspera.assert_type(@trusted_certs, Array){'trusted_certs'}
|
|
71
77
|
@client_ssh_key = client_ssh_key.nil? ? :rsa : client_ssh_key.to_sym
|
|
@@ -74,7 +80,7 @@ module Aspera
|
|
|
74
80
|
@builder = CommandLineBuilder.new(@job_spec, Spec::SCHEMA, CommandLineConverter)
|
|
75
81
|
end
|
|
76
82
|
|
|
77
|
-
#
|
|
83
|
+
# Place source files on command line or in a temp file list, controlled by @file_list and file_list_folder
|
|
78
84
|
def process_file_list
|
|
79
85
|
# is the file list provided through ascp parameters?
|
|
80
86
|
ascp_file_list_provided = self.class.ascp_args_file_list?(@ascp_args)
|
|
@@ -85,12 +91,12 @@ module Aspera
|
|
|
85
91
|
file_list_option = nil
|
|
86
92
|
# transfer spec contains paths ?
|
|
87
93
|
if !ts_paths_array.nil?
|
|
88
|
-
Aspera.assert(!ascp_file_list_provided
|
|
89
|
-
Aspera.assert(ts_paths_array.all?{ |i| i.key?('source')}
|
|
94
|
+
Aspera.assert(!ascp_file_list_provided, 'file list provided both in transfer spec and ascp file list. Remove one of them.')
|
|
95
|
+
Aspera.assert(ts_paths_array.all?{ |i| i.key?('source')}, "All elements of paths must have a 'source' key")
|
|
90
96
|
is_pair_list = ts_paths_array.any?{ |i| i.key?('destination')}
|
|
91
|
-
|
|
92
|
-
if self.class.file_list_folder.nil?
|
|
93
|
-
Aspera.assert(!is_pair_list
|
|
97
|
+
Aspera.assert(!(is_pair_list && !ts_paths_array.all?{ |i| i.key?('destination')})){"All elements of paths must be consistent with 'destination' key"}
|
|
98
|
+
if !@file_list || self.class.file_list_folder.nil?
|
|
99
|
+
Aspera.assert(!is_pair_list, 'file pair list is not supported when file list folder is not set')
|
|
94
100
|
# not safe for special characters ? (maybe not, depends on OS)
|
|
95
101
|
Log.log.debug('placing source file list on command line (no file list file)')
|
|
96
102
|
@builder.add_command_line_options(ts_paths_array.map{ |i| i['source']})
|
|
@@ -112,7 +118,7 @@ module Aspera
|
|
|
112
118
|
@builder.add_command_line_options("#{file_list_option}=#{file_list_file}") unless file_list_option.nil?
|
|
113
119
|
end
|
|
114
120
|
|
|
115
|
-
# @return the list of certificates (option `-i`) to use when token/ssh or wss are used
|
|
121
|
+
# @return [Array<String>] the list of certificates (option `-i`) to use when token/ssh or wss are used
|
|
116
122
|
def remote_certificates
|
|
117
123
|
certificates_to_use = []
|
|
118
124
|
# use web socket secure for session ?
|
|
@@ -150,22 +156,18 @@ module Aspera
|
|
|
150
156
|
|
|
151
157
|
# Translate transfer spec to env vars and command line arguments for `ascp`
|
|
152
158
|
def ascp_args
|
|
153
|
-
|
|
154
|
-
args: [],
|
|
155
|
-
env: {},
|
|
156
|
-
name: :ascp
|
|
157
|
-
}
|
|
159
|
+
exec_spec = Aspera::ExecSpec.new(exec: :ascp)
|
|
158
160
|
|
|
159
161
|
# Special cases
|
|
160
162
|
@job_spec.delete('source_root') if @job_spec.key?('source_root') && @job_spec['source_root'].empty?
|
|
161
163
|
|
|
162
164
|
# Notify multi-session was already used, anyway it was deleted by agent direct
|
|
163
|
-
Aspera.assert(!@builder.read_param('multi_session'))
|
|
165
|
+
Aspera.assert(!@builder.read_param('multi_session'), 'multi_session already consumed, must not be set here')
|
|
164
166
|
|
|
165
167
|
# Add ssh or wss certificates
|
|
166
168
|
# (reverse, to keep order, as we unshift)
|
|
167
169
|
remote_certificates&.reverse_each do |cert|
|
|
168
|
-
|
|
170
|
+
exec_spec.args.unshift('-i', cert)
|
|
169
171
|
end
|
|
170
172
|
|
|
171
173
|
case (delete_source = @builder.read_param('delete_source'))
|
|
@@ -183,9 +185,9 @@ module Aspera
|
|
|
183
185
|
base64_destination = false
|
|
184
186
|
# symbol must be index of Ascp::Installation.paths
|
|
185
187
|
if @builder.read_param('use_ascp4')
|
|
186
|
-
|
|
188
|
+
exec_spec.exec = :ascp4
|
|
187
189
|
else
|
|
188
|
-
|
|
190
|
+
exec_spec.exec = :ascp
|
|
189
191
|
base64_destination = true
|
|
190
192
|
end
|
|
191
193
|
# destination will be base64 encoded, put this before source path arguments
|
|
@@ -200,17 +202,17 @@ module Aspera
|
|
|
200
202
|
destination_folder = Base64.strict_encode64(destination_folder) if base64_destination
|
|
201
203
|
# destination MUST be last command line argument to ascp
|
|
202
204
|
@builder.add_command_line_options(destination_folder)
|
|
203
|
-
@builder.
|
|
204
|
-
|
|
205
|
+
@builder.add_to_exec_spec(exec_spec)
|
|
206
|
+
exec_spec.args.unshift('-q') if @quiet
|
|
205
207
|
# add fallback cert and key as arguments if needed
|
|
206
208
|
if HTTP_FALLBACK_ACTIVATION_VALUES.include?(@job_spec['http_fallback'])
|
|
207
|
-
|
|
208
|
-
|
|
209
|
+
exec_spec.args.unshift('-Y', Ascp::Installation.instance.path(:fallback_private_key))
|
|
210
|
+
exec_spec.args.unshift('-I', Ascp::Installation.instance.path(:fallback_certificate))
|
|
209
211
|
end
|
|
210
212
|
# disable redis in client, only for ascp, this makes ascp4 fail
|
|
211
|
-
|
|
212
|
-
Log.log.debug{"ascp args: #{
|
|
213
|
-
return
|
|
213
|
+
exec_spec.env['ASPERA_TEST_REDIS_DISABLE'] = 'true' if exec_spec.exec.eql?(:ascp)
|
|
214
|
+
Log.log.debug{"ascp args: #{exec_spec}"}
|
|
215
|
+
return exec_spec
|
|
214
216
|
end
|
|
215
217
|
DELETE_EQUIV = %w[remove_after_transfer remove_empty_directories remove_empty_source_directory]
|
|
216
218
|
# `ascp` options to provide a file list
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Aspera
|
|
4
|
+
module Transfer
|
|
5
|
+
# Typed result of a transfer operation.
|
|
6
|
+
# Replaces the ad-hoc Array polymorphism previously returned by agents.
|
|
7
|
+
#
|
|
8
|
+
# Agents return one of the three concrete sub-classes:
|
|
9
|
+
# Transfer::Result::Success — all sessions completed successfully
|
|
10
|
+
# Transfer::Result::Error — at least one session failed
|
|
11
|
+
# Transfer::Result::Async — transfer submitted to a daemon; not yet complete
|
|
12
|
+
class Result
|
|
13
|
+
# All sessions completed successfully.
|
|
14
|
+
class Success < Result
|
|
15
|
+
def to_s
|
|
16
|
+
'success'
|
|
17
|
+
end
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
# At least one session failed.
|
|
21
|
+
class Error < Result
|
|
22
|
+
# @return [StandardError]
|
|
23
|
+
attr_reader :exception
|
|
24
|
+
|
|
25
|
+
def initialize(exception)
|
|
26
|
+
super()
|
|
27
|
+
@exception = exception
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
def to_s
|
|
31
|
+
"error: #{@exception.message}"
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
# Transfer submitted to an external daemon; status not yet known.
|
|
36
|
+
class Async < Result
|
|
37
|
+
# @return [String] UUID assigned to this background job
|
|
38
|
+
attr_reader :job_id
|
|
39
|
+
# @return [String] initial status string (e.g. 'running')
|
|
40
|
+
attr_reader :status
|
|
41
|
+
|
|
42
|
+
def initialize(job_id:, status: 'running')
|
|
43
|
+
super()
|
|
44
|
+
@job_id = job_id
|
|
45
|
+
@status = status
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
# Convenience: export as a plain Hash for serialization / display
|
|
49
|
+
def to_h
|
|
50
|
+
{'job_id' => @job_id, 'status' => @status}
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
def to_s
|
|
54
|
+
"async job_id=#{@job_id} status=#{@status}"
|
|
55
|
+
end
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
# Factory helpers
|
|
59
|
+
class << self
|
|
60
|
+
def success
|
|
61
|
+
Success.new
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
def error(exception)
|
|
65
|
+
Error.new(exception)
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
def async(job_id:, status: 'running')
|
|
69
|
+
Async.new(job_id: job_id, status: status)
|
|
70
|
+
end
|
|
71
|
+
end
|
|
72
|
+
end
|
|
73
|
+
end
|
|
74
|
+
end
|