aspera-cli 4.26.2 → 4.27.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/CHANGELOG.md +310 -21
- data/CONTRIBUTING.md +99 -14
- data/TODO.md +50 -0
- data/bin/ascli +3 -3
- data/docs/README.md +10977 -0
- data/docs/test-mcp-with-ai.md +202 -0
- data/lib/aspera/agent/base.rb +29 -7
- data/lib/aspera/agent/connect.rb +58 -21
- data/lib/aspera/agent/desktop.rb +53 -18
- data/lib/aspera/agent/direct.rb +71 -40
- data/lib/aspera/agent/factory.rb +3 -3
- data/lib/aspera/agent/httpgw.rb +55 -14
- data/lib/aspera/agent/node.rb +34 -2
- data/lib/aspera/agent/transferd.rb +48 -12
- data/lib/aspera/api/alee.rb +1 -1
- data/lib/aspera/api/aoc.rb +42 -44
- data/lib/aspera/api/cos_node.rb +10 -9
- data/lib/aspera/api/faspex.rb +16 -15
- data/lib/aspera/api/httpgw.rb +23 -23
- data/lib/aspera/api/node.rb +51 -70
- data/lib/aspera/api/queries/bss_subscription_account.graphql +18 -0
- data/lib/aspera/api/queries/bss_subscription_usage.graphql +18 -0
- data/lib/aspera/ascmd.rb +32 -30
- data/lib/aspera/ascp/installation.rb +75 -69
- data/lib/aspera/ascp/management.rb +7 -8
- data/lib/aspera/assert.rb +12 -13
- data/lib/aspera/cli/ascp_actions.rb +155 -0
- data/lib/aspera/cli/async_transfer_store.rb +125 -0
- data/lib/aspera/cli/bootstrapper.rb +197 -0
- data/lib/aspera/cli/command_registry.rb +145 -0
- data/lib/aspera/cli/command_spec.rb +141 -0
- data/lib/aspera/cli/context.rb +19 -6
- data/lib/aspera/cli/error.rb +16 -0
- data/lib/aspera/cli/extended_value.rb +27 -26
- data/lib/aspera/cli/formatter.rb +95 -59
- data/lib/aspera/cli/gem_checker.rb +65 -0
- data/lib/aspera/cli/hints.rb +3 -3
- data/lib/aspera/cli/http.rb +74 -30
- data/lib/aspera/cli/info.rb +2 -0
- data/lib/aspera/cli/mailer.rb +97 -0
- data/lib/aspera/cli/mcp_tool.rb +225 -0
- data/lib/aspera/cli/option_declarator.rb +77 -0
- data/lib/aspera/cli/options.schema.yaml +634 -7
- data/lib/aspera/cli/parser.rb +1217 -0
- data/lib/aspera/cli/plugins/alee.rb +20 -22
- data/lib/aspera/cli/plugins/aoc.rb +1178 -881
- data/lib/aspera/cli/plugins/ats.rb +205 -162
- data/lib/aspera/cli/plugins/base.rb +703 -175
- data/lib/aspera/cli/plugins/basic_auth.rb +7 -9
- data/lib/aspera/cli/plugins/config.rb +434 -757
- data/lib/aspera/cli/plugins/console.rb +107 -65
- data/lib/aspera/cli/plugins/cos.rb +46 -33
- data/lib/aspera/cli/plugins/factory.rb +6 -6
- data/lib/aspera/cli/plugins/faspex5.rb +627 -388
- data/lib/aspera/cli/plugins/faspio.rb +54 -51
- data/lib/aspera/cli/plugins/httpgw.rb +18 -25
- data/lib/aspera/cli/plugins/mcp.rb +279 -0
- data/lib/aspera/cli/plugins/node.rb +988 -840
- data/lib/aspera/cli/plugins/oauth.rb +7 -10
- data/lib/aspera/cli/plugins/orchestrator.rb +112 -135
- data/lib/aspera/cli/plugins/preview.rb +195 -157
- data/lib/aspera/cli/plugins/server.rb +141 -91
- data/lib/aspera/cli/plugins/shares.rb +343 -110
- data/lib/aspera/cli/preset_actions.rb +159 -0
- data/lib/aspera/cli/preset_manager.rb +82 -39
- data/lib/aspera/cli/result.rb +78 -26
- data/lib/aspera/cli/runner.rb +323 -153
- data/lib/aspera/cli/secret_finder.rb +40 -0
- data/lib/aspera/cli/special_values.rb +1 -0
- data/lib/aspera/cli/sync_actions.rb +89 -64
- data/lib/aspera/cli/terminal_formatter.rb +3 -3
- data/lib/aspera/cli/transfer_actions.rb +90 -0
- data/lib/aspera/cli/transfer_agent.rb +129 -64
- data/lib/aspera/cli/transfer_progress.rb +9 -9
- data/lib/aspera/cli/vault_manager.rb +79 -0
- data/lib/aspera/cli/version.rb +1 -1
- data/lib/aspera/cli/wizard.rb +28 -30
- data/lib/aspera/colors.rb +3 -3
- data/lib/aspera/command_line_builder.rb +34 -34
- data/lib/aspera/command_line_converter.rb +1 -1
- data/lib/aspera/coverage.rb +1 -2
- data/lib/aspera/data_repository.rb +2 -1
- data/lib/aspera/dot_container.rb +12 -12
- data/lib/aspera/environment.rb +30 -23
- data/lib/aspera/exec_spec.rb +13 -0
- data/lib/aspera/faspex_gw.rb +5 -5
- data/lib/aspera/faspex_postproc.rb +16 -10
- data/lib/aspera/graphql.rb +37 -0
- data/lib/aspera/hash_ext.rb +8 -2
- data/lib/aspera/json_rpc/client.rb +62 -0
- data/lib/aspera/json_rpc/version.rb +7 -0
- data/lib/aspera/keychain/base.rb +10 -3
- data/lib/aspera/keychain/encrypted_hash.rb +92 -17
- data/lib/aspera/keychain/factory.rb +20 -8
- data/lib/aspera/keychain/hashicorp_vault.rb +1 -1
- data/lib/aspera/keychain/macos_security.rb +23 -25
- data/lib/aspera/keychain/one_password_api.rb +86 -0
- data/lib/aspera/keychain/one_password_base.rb +34 -0
- data/lib/aspera/keychain/one_password_cli.rb +98 -0
- data/lib/aspera/link_header.rb +82 -0
- data/lib/aspera/log.rb +29 -15
- data/lib/aspera/markdown.rb +90 -7
- data/lib/aspera/nagios.rb +8 -8
- data/lib/aspera/node_simulator.rb +35 -27
- data/lib/aspera/oauth/base.rb +9 -10
- data/lib/aspera/oauth/boot.rb +5 -5
- data/lib/aspera/oauth/factory.rb +12 -10
- data/lib/aspera/oauth/jwt.rb +9 -11
- data/lib/aspera/oauth/web.rb +6 -6
- data/lib/aspera/persistency_action_once.rb +12 -12
- data/lib/aspera/persistency_folder.rb +18 -11
- data/lib/aspera/preview/file_types.rb +5 -5
- data/lib/aspera/preview/generator.rb +52 -49
- data/lib/aspera/preview/options.rb +3 -2
- data/lib/aspera/preview/terminal.rb +5 -5
- data/lib/aspera/preview/utils.rb +66 -19
- data/lib/aspera/products/connect.rb +2 -2
- data/lib/aspera/products/desktop.rb +1 -1
- data/lib/aspera/products/other.rb +3 -3
- data/lib/aspera/products/transferd.rb +4 -4
- data/lib/aspera/proxy_auto_config.rb +15 -14
- data/lib/aspera/rest.rb +159 -151
- data/lib/aspera/rest_error_analyzer.rb +6 -6
- data/lib/aspera/rest_errors_aspera.rb +0 -10
- data/lib/aspera/rest_list.rb +11 -10
- data/lib/aspera/schema/IBM Aspera Node API-4.4.6.yaml +6232 -0
- data/lib/aspera/schema/IBM Aspera faspio Gateway API-1.0.0.yaml +249 -0
- data/lib/aspera/schema/IBM Aspera on Cloud API-0.2.6-enhanced.yaml +1853 -510
- data/lib/aspera/schema/IBM_Aspera_Shares.yaml +4653 -0
- data/lib/aspera/schema/documentation.rb +65 -28
- data/lib/aspera/schema/reader.rb +136 -13
- data/lib/aspera/schema/registry.rb +60 -8
- data/lib/aspera/secret_hider.rb +12 -7
- data/lib/aspera/ssh.rb +64 -31
- data/lib/aspera/ssl.rb +5 -5
- data/lib/aspera/sync/conf.schema.yaml +2 -2
- data/lib/aspera/sync/database.rb +2 -2
- data/lib/aspera/sync/operations.rb +22 -25
- data/lib/aspera/temp_file_manager.rb +25 -5
- data/lib/aspera/timer_limiter.rb +1 -1
- data/lib/aspera/transfer/faux_file.rb +24 -11
- data/lib/aspera/transfer/parameters.rb +36 -34
- data/lib/aspera/transfer/result.rb +74 -0
- data/lib/aspera/transfer/resumer.rb +10 -10
- data/lib/aspera/transfer/spec.rb +18 -1
- data/lib/aspera/transfer/spec.schema.yaml +12 -3
- data/lib/aspera/uri_reader.rb +52 -15
- data/lib/aspera/web_auth.rb +8 -7
- data/lib/aspera/web_server_simple.rb +15 -12
- data/lib/aspera/yaml.rb +5 -4
- data.tar.gz.sig +0 -0
- metadata +34 -6
- metadata.gz.sig +0 -0
- data/lib/aspera/cli/manager.rb +0 -766
- data/lib/aspera/cli/plugins/faspex.rb +0 -539
- data/lib/aspera/json_rpc.rb +0 -52
- data/lib/aspera/transfer/uri.rb +0 -56
data/lib/aspera/ssh.rb
CHANGED
|
@@ -8,77 +8,110 @@ module Aspera
|
|
|
8
8
|
# A simple wrapper around Net::SSH
|
|
9
9
|
# executes one command and get its result from stdout
|
|
10
10
|
class Ssh
|
|
11
|
+
# Raised when the remote command fails or cannot be started
|
|
11
12
|
class Error < Aspera::Error
|
|
12
13
|
end
|
|
14
|
+
# Regexp matching ecdsa/ecdh-sha2 algorithm names (JRuby workaround)
|
|
15
|
+
EXCLUDE_ECDSHA2 = /^ecd(sa|h)-sha2/
|
|
16
|
+
|
|
13
17
|
class << self
|
|
14
|
-
#
|
|
18
|
+
# Removes ed25519 keys from the default identity list used by Net::SSH.
|
|
19
|
+
# Called when the `ed25519` gem is absent or explicitly disabled via
|
|
20
|
+
# `ASCLI_ENABLE_ED25519=false`.
|
|
21
|
+
# @return [void]
|
|
15
22
|
def disable_ed25519_keys
|
|
16
23
|
Log.log.debug('Disabling SSH ed25519 user keys')
|
|
17
24
|
old_verbose = $VERBOSE
|
|
18
25
|
$VERBOSE = nil
|
|
19
26
|
Net::SSH::Authentication::Session.class_eval do
|
|
20
27
|
define_method(:default_keys) do
|
|
21
|
-
%w[.ssh .ssh2].product(%w[rsa dsa ecdsa]).map{"~/#{_1}/id_#{_2}"}.freeze
|
|
28
|
+
%w[.ssh .ssh2].product(%w[rsa dsa ecdsa]).map { "~/#{_1}/id_#{_2}" }.freeze
|
|
22
29
|
end
|
|
23
30
|
private(:default_keys)
|
|
24
|
-
end
|
|
31
|
+
end
|
|
25
32
|
$VERBOSE = old_verbose
|
|
26
33
|
end
|
|
27
34
|
|
|
28
|
-
#
|
|
35
|
+
# Returns Net::SSH option overrides that exclude all ecdsa/ecdh-sha2 algorithms.
|
|
36
|
+
# Merge the result into `ssh_options` passed to {Ssh#initialize} to avoid
|
|
37
|
+
# mutating Net::SSH internal constants.
|
|
38
|
+
# @return [Hash{Symbol => Array<String>}] `:host_key` and `:kex` filtered lists
|
|
39
|
+
def no_ecd_sha2_options
|
|
40
|
+
Log.log.debug('Building SSH options without ecdsa/ecdh-sha2')
|
|
41
|
+
{
|
|
42
|
+
host_key: Net::SSH::Transport::Algorithms::ALGORITHMS[:host_key].reject { |a| a.match?(EXCLUDE_ECDSHA2) },
|
|
43
|
+
kex: Net::SSH::Transport::Algorithms::ALGORITHMS[:kex].reject { |a| a.match?(EXCLUDE_ECDSHA2) }
|
|
44
|
+
}
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
# Mutates Net::SSH internal algorithm lists to remove ecdsa/ecdh-sha2 entries globally.
|
|
48
|
+
# Kept for backwards compatibility and JRuby usage; prefer {no_ecd_sha2_options} for new code.
|
|
49
|
+
# @return [void]
|
|
29
50
|
def disable_ecd_sha2_algorithms
|
|
30
|
-
Log.log.debug('Disabling SSH ecdsa')
|
|
31
|
-
Net::SSH::Transport::Algorithms::ALGORITHMS.each_value{ |a| a.reject!{ |a| a
|
|
32
|
-
Net::SSH::KnownHosts::SUPPORTED_TYPE.reject!{ |t| t
|
|
51
|
+
Log.log.debug('Disabling SSH ecdsa (global)')
|
|
52
|
+
Net::SSH::Transport::Algorithms::ALGORITHMS.each_value { |a| a.reject! { |a| a.match?(EXCLUDE_ECDSHA2) } }
|
|
53
|
+
Net::SSH::KnownHosts::SUPPORTED_TYPE.reject! { |t| t.match?(EXCLUDE_ECDSHA2) }
|
|
33
54
|
end
|
|
34
55
|
end
|
|
35
|
-
#
|
|
36
|
-
#
|
|
56
|
+
# @param host [String] remote server address
|
|
57
|
+
# @param username [String] SSH user name
|
|
58
|
+
# @param ssh_options [Hash{Symbol => Object}] options forwarded to `Net::SSH.start`
|
|
59
|
+
# (see {https://net-ssh.github.io/net-ssh/classes/Net/SSH.html#method-c-start Net::SSH.start}).
|
|
60
|
+
# Defaults are injected for `:logger`, `:verbose`, and `:use_agent` if absent.
|
|
37
61
|
def initialize(host, username, ssh_options)
|
|
38
62
|
Aspera.assert_type(host, String)
|
|
39
63
|
Aspera.assert_type(username, String)
|
|
40
64
|
Aspera.assert_type(ssh_options, Hash)
|
|
65
|
+
Aspera.assert_hash_all(ssh_options, Symbol, nil)
|
|
41
66
|
@host = host
|
|
42
67
|
@username = username
|
|
43
68
|
@ssh_options = ssh_options.dup
|
|
44
69
|
@ssh_options[:logger] = Log.log unless @ssh_options.key?(:logger)
|
|
45
70
|
@ssh_options[:verbose] = :warn unless @ssh_options.key?(:verbose)
|
|
71
|
+
# @ssh_options[:verbose] = :debug
|
|
46
72
|
@ssh_options[:use_agent] = false unless @ssh_options.key?(:use_agent)
|
|
47
|
-
Log.log.debug{"ssh:#{@username}@#{@host}"}
|
|
73
|
+
Log.log.debug { "ssh:#{@username}@#{@host}" }
|
|
48
74
|
Log.dump(:ssh_options, @ssh_options)
|
|
49
75
|
end
|
|
50
76
|
|
|
51
|
-
#
|
|
52
|
-
|
|
77
|
+
# Executes a single command on the remote host over a new SSH session.
|
|
78
|
+
# @param cmd [String] shell command to execute remotely
|
|
79
|
+
# @param input [String, nil] data written to the command's stdin, or `nil`
|
|
80
|
+
# @return [String] concatenated stdout of the remote command
|
|
81
|
+
# @raise [Error] if the channel cannot be opened or the remote command exits with a non-zero status
|
|
82
|
+
def execute(cmd, input: nil)
|
|
53
83
|
Aspera.assert_type(cmd, String)
|
|
54
|
-
Log.log.debug{"cmd=#{cmd}"}
|
|
84
|
+
Log.log.debug { "cmd=#{cmd}" }
|
|
85
|
+
# @type response [Array<String>]
|
|
55
86
|
response = []
|
|
87
|
+
# @type error [Array<String>]
|
|
56
88
|
error = []
|
|
89
|
+
exit_code = nil
|
|
90
|
+
# @param session [Net::SSH::Connection::Session]
|
|
57
91
|
Net::SSH.start(@host, @username, @ssh_options) do |session|
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
channel.on_extended_data
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
92
|
+
# @param channel [Net::SSH::Connection::Channel]
|
|
93
|
+
session.open_channel do |channel|
|
|
94
|
+
# Register stdout/stderr before exec so no data is missed (e.g. ForceCommand errors)
|
|
95
|
+
channel.on_data { |_ch, data| response.push(data) }
|
|
96
|
+
channel.on_extended_data { |_ch, type, data| error.push(data) if type.eql?(1) }
|
|
97
|
+
# @param data [Net::SSH::Buffer]
|
|
98
|
+
channel.on_request('exit-status') do |_channel, data|
|
|
99
|
+
Log.dump(:data, data, level: :trace1)
|
|
66
100
|
exit_code = data.read_long
|
|
67
|
-
next if exit_code.zero?
|
|
68
|
-
error_message = "#{cmd}: exit #{exit_code}, #{error.join.chomp}"
|
|
69
|
-
raise Error, error_message if exception
|
|
70
|
-
# Happens when windows user hasn't logged in and created home account.
|
|
71
|
-
error_message += "\nHint: home not created in Windows?" if data.include?('Could not chdir to home directory')
|
|
72
|
-
Log.log.debug(error_message)
|
|
73
101
|
end
|
|
74
102
|
# send command to SSH channel (execute) cspell: disable-next-line
|
|
75
|
-
channel.send('cexe'.reverse, cmd)
|
|
103
|
+
channel.send('cexe'.reverse, cmd) do |_ch, success|
|
|
104
|
+
raise Error, "could not execute command: #{cmd}" unless success
|
|
105
|
+
channel.send_data(input) unless input.nil?
|
|
106
|
+
end
|
|
76
107
|
end
|
|
77
|
-
# wait for channel to finish
|
|
78
|
-
ssh_channel.wait
|
|
79
|
-
# main SSH session loop
|
|
108
|
+
# wait for channel to finish and flush the session
|
|
80
109
|
session.loop
|
|
81
110
|
end
|
|
111
|
+
error_text = error.join
|
|
112
|
+
hint = error_text.include?('Could not chdir to home directory') ? "\nHint: home not created in Windows?" : ''
|
|
113
|
+
raise Error, "#{cmd}: exit #{exit_code}, #{error_text.chomp}#{hint}" if exit_code&.nonzero?
|
|
114
|
+
Log.log.error { "#{error_text}#{hint}" } unless error_text.empty?
|
|
82
115
|
# response as single string
|
|
83
116
|
return response.join
|
|
84
117
|
end
|
data/lib/aspera/ssl.rb
CHANGED
|
@@ -12,16 +12,16 @@ module Aspera
|
|
|
12
12
|
@extra_options = OpenSSL::SSL::SSLContext::DEFAULT_PARAMS[:options]
|
|
13
13
|
attr_reader :extra_options
|
|
14
14
|
|
|
15
|
-
def option_list=(
|
|
16
|
-
Aspera.assert_type(
|
|
17
|
-
|
|
18
|
-
Aspera.assert_type(opt, String, Integer){'Expected String or Integer in ssl_options'}
|
|
15
|
+
def option_list=(options)
|
|
16
|
+
Aspera.assert_type(options, Array) { 'ssl_options' }
|
|
17
|
+
options.each do |opt|
|
|
18
|
+
Aspera.assert_type(opt, String, Integer) { 'Expected String or Integer in ssl_options' }
|
|
19
19
|
case opt
|
|
20
20
|
when Integer
|
|
21
21
|
@extra_options = opt
|
|
22
22
|
when String
|
|
23
23
|
name = "OP_#{opt.start_with?('-') ? opt[1..] : opt}".upcase
|
|
24
|
-
|
|
24
|
+
Aspera.assert(OpenSSL::SSL.const_defined?(name), type: Cli::BadArgument) { "Unknown ssl_option: #{name}, use one of: #{OpenSSL::SSL.constants.grep(/^OP_/).map { |c| c.to_s.sub(/^OP_/, '') }.join(', ')}" }
|
|
25
25
|
if opt.start_with?('-')
|
|
26
26
|
@extra_options &= ~OpenSSL::SSL.const_get(name)
|
|
27
27
|
else
|
|
@@ -637,7 +637,7 @@ properties:
|
|
|
637
637
|
Partial transfers may exist if communication disruptions caused the underlying ascp processes to terminate early.
|
|
638
638
|
Note that transfer resumption can only happen if the `reset` option is disabled.
|
|
639
639
|
If an async session starts with `reset` enabled and resume enabled, transfers interrupted during that
|
|
640
|
-
session will be
|
|
640
|
+
session will be resumable, but only if async is then restarted with 'reset' disabled.
|
|
641
641
|
type: object
|
|
642
642
|
properties:
|
|
643
643
|
enabled:
|
|
@@ -654,7 +654,7 @@ properties:
|
|
|
654
654
|
Sets the age limit in days for temporary files that will be preserved
|
|
655
655
|
on cleanup (usually at async's start and stop) for potential transfer resume.
|
|
656
656
|
Temp files older than the given value will be removed
|
|
657
|
-
regardless of whether they might be
|
|
657
|
+
regardless of whether they might be resumable.
|
|
658
658
|
type: integer
|
|
659
659
|
default: "5"
|
|
660
660
|
x-cli-option: --resume-age-days
|
data/lib/aspera/sync/database.rb
CHANGED
|
@@ -25,7 +25,7 @@ class SqLite3Wrapper
|
|
|
25
25
|
end
|
|
26
26
|
else
|
|
27
27
|
def execute(sql)
|
|
28
|
-
db = SQLite3::Database.new(@db_path).tap{ |d| d.results_as_hash = true}
|
|
28
|
+
db = SQLite3::Database.new(@db_path).tap { |d| d.results_as_hash = true }
|
|
29
29
|
begin
|
|
30
30
|
normalize_rows(db.execute(sql))
|
|
31
31
|
ensure
|
|
@@ -46,7 +46,7 @@ class SqLite3Wrapper
|
|
|
46
46
|
private
|
|
47
47
|
|
|
48
48
|
def normalize_rows(rows)
|
|
49
|
-
rows.map{ |r| r.transform_keys(&:to_s)}
|
|
49
|
+
rows.map { |r| r.transform_keys(&:to_s) }
|
|
50
50
|
end
|
|
51
51
|
end
|
|
52
52
|
|
|
@@ -4,6 +4,7 @@
|
|
|
4
4
|
|
|
5
5
|
require 'aspera/ascp/installation'
|
|
6
6
|
require 'aspera/agent/direct'
|
|
7
|
+
require 'aspera/exec_spec'
|
|
7
8
|
require 'aspera/command_line_converter'
|
|
8
9
|
require 'aspera/command_line_builder'
|
|
9
10
|
require 'aspera/log'
|
|
@@ -89,17 +90,14 @@ module Aspera
|
|
|
89
90
|
def start(sync_info, opt_ts = nil)
|
|
90
91
|
Log.dump(:sync_params_initial, sync_info)
|
|
91
92
|
Aspera.assert_type(sync_info, Hash)
|
|
92
|
-
Aspera.assert(PARAM_KEYS.any?{ |k| sync_info.key?(k)}, 'At least one of `local` or `sessions` must be present in async parameters', type: Error)
|
|
93
|
-
|
|
94
|
-
args: [],
|
|
95
|
-
env: {}
|
|
96
|
-
}
|
|
93
|
+
Aspera.assert(PARAM_KEYS.any? { |k| sync_info.key?(k) }, 'At least one of `local` or `sessions` must be present in async parameters', type: Error)
|
|
94
|
+
exec_spec = ExecSpec.new(exec: :async)
|
|
97
95
|
if sync_info.key?('local')
|
|
98
96
|
# `conf` format
|
|
99
|
-
Aspera.assert_type(sync_info['local'], Hash){'local'}
|
|
97
|
+
Aspera.assert_type(sync_info['local'], Hash) { 'local' }
|
|
100
98
|
remote = sync_info['remote']
|
|
101
|
-
Aspera.assert_type(remote, Hash){'remote'}
|
|
102
|
-
Aspera.assert_type(remote['path'], String){'remote path'}
|
|
99
|
+
Aspera.assert_type(remote, Hash) { 'remote' }
|
|
100
|
+
Aspera.assert_type(remote['path'], String) { 'remote path' }
|
|
103
101
|
# get transfer spec if possible, and feed back to new structure
|
|
104
102
|
if block_given?
|
|
105
103
|
transfer_spec = yield(direction_sym(sync_info), sync_info['local']['path'], remote['path'])
|
|
@@ -115,10 +113,10 @@ module Aspera
|
|
|
115
113
|
remote['private_key_paths'].concat(add_certificates)
|
|
116
114
|
end
|
|
117
115
|
# '--exclusive-mgmt-port=12345', '--arg-err-path=-',
|
|
118
|
-
|
|
116
|
+
exec_spec.args = ["--conf64=#{Base64.strict_encode64(JSON.generate(sync_info))}"]
|
|
119
117
|
Log.dump(:sync_conf, sync_info)
|
|
120
118
|
agent = Agent::Direct.new
|
|
121
|
-
agent.start_and_monitor_process(session: {},
|
|
119
|
+
agent.start_and_monitor_process(session: {}, exec_spec: exec_spec)
|
|
122
120
|
else
|
|
123
121
|
# `args` format
|
|
124
122
|
raise StandardError, "Only 'sessions', and optionally 'instance' keys are allowed" unless
|
|
@@ -127,8 +125,8 @@ module Aspera
|
|
|
127
125
|
Aspera.assert_type(sync_info['sessions'].first, Hash)
|
|
128
126
|
if block_given?
|
|
129
127
|
sync_info['sessions'].each do |session|
|
|
130
|
-
Aspera.assert_type(session['local_dir'], String){'local_dir'}
|
|
131
|
-
Aspera.assert_type(session['remote_dir'], String){'remote_dir'}
|
|
128
|
+
Aspera.assert_type(session['local_dir'], String) { 'local_dir' }
|
|
129
|
+
Aspera.assert_type(session['remote_dir'], String) { 'remote_dir' }
|
|
132
130
|
transfer_spec = yield(direction_sym(session), session['local_dir'], session['remote_dir'])
|
|
133
131
|
Log.dump(:auth_ts, transfer_spec)
|
|
134
132
|
transfer_spec.deep_merge!(opt_ts) unless opt_ts.nil?
|
|
@@ -141,23 +139,23 @@ module Aspera
|
|
|
141
139
|
Aspera.assert_type(sync_info['instance'], Hash)
|
|
142
140
|
instance_builder = CommandLineBuilder.new(sync_info['instance'], ARGS_INSTANCE_SCHEMA, CommandLineConverter)
|
|
143
141
|
instance_builder.process_params
|
|
144
|
-
instance_builder.
|
|
142
|
+
instance_builder.add_to_exec_spec(exec_spec)
|
|
145
143
|
end
|
|
146
144
|
sync_info['sessions'].each do |session_params|
|
|
147
145
|
Aspera.assert_type(session_params, Hash)
|
|
148
146
|
Aspera.assert(session_params.key?('name'), 'session must contain at least: name')
|
|
149
147
|
session_builder = CommandLineBuilder.new(session_params, ARGS_SESSION_SCHEMA, CommandLineConverter)
|
|
150
148
|
session_builder.process_params
|
|
151
|
-
session_builder.
|
|
149
|
+
session_builder.add_to_exec_spec(exec_spec)
|
|
152
150
|
end
|
|
153
|
-
Environment.secure_execute(Ascp::Installation.instance.path(:async), *
|
|
151
|
+
Environment.secure_execute(Ascp::Installation.instance.path(:async), *exec_spec.args, env: exec_spec.env)
|
|
154
152
|
end
|
|
155
153
|
return
|
|
156
154
|
end
|
|
157
155
|
|
|
158
156
|
# Parse output of asyncadmin
|
|
159
157
|
def parse_status(stdout)
|
|
160
|
-
Log.log.trace1{"stdout=#{stdout}"}
|
|
158
|
+
Log.log.trace1 { "stdout=#{stdout}" }
|
|
161
159
|
result = {}
|
|
162
160
|
ids = nil
|
|
163
161
|
stdout.split("\n").each do |line|
|
|
@@ -177,7 +175,7 @@ module Aspera
|
|
|
177
175
|
# @param sync_info [Hash] sync parameters in conf or args format
|
|
178
176
|
# @return [Hash] parsed output of asyncadmin
|
|
179
177
|
def admin_status(sync_info)
|
|
180
|
-
Aspera.assert(PARAM_KEYS.any?{ |k| sync_info.key?(k)}, 'At least one of `local` or `sessions` must be present in async parameters', type: Error)
|
|
178
|
+
Aspera.assert(PARAM_KEYS.any? { |k| sync_info.key?(k) }, 'At least one of `local` or `sessions` must be present in async parameters', type: Error)
|
|
181
179
|
arguments = [ASYNC_ADMIN_EXECUTABLE, '--quiet']
|
|
182
180
|
if sync_info.key?('local')
|
|
183
181
|
# `conf` format
|
|
@@ -209,14 +207,14 @@ module Aspera
|
|
|
209
207
|
# @param sync_info [Hash] sync parameters in conf or args format
|
|
210
208
|
# @return [String, nil] Path to "local DB dir", i.e. folder that contains folders that contain `snap.db`
|
|
211
209
|
def local_db_folder(sync_info)
|
|
212
|
-
Aspera.assert(PARAM_KEYS.any?{ |k| sync_info.key?(k)}, 'At least one of `local` or `sessions` must be present in async parameters', type: Error)
|
|
210
|
+
Aspera.assert(PARAM_KEYS.any? { |k| sync_info.key?(k) }, 'At least one of `local` or `sessions` must be present in async parameters', type: Error)
|
|
213
211
|
if sync_info.key?('local')
|
|
214
212
|
# `conf` format
|
|
215
213
|
if sync_info.key?('local_db_dir')
|
|
216
214
|
return sync_info['local_db_dir']
|
|
217
215
|
elsif (local_path = sync_info.dig('local', 'path'))
|
|
218
216
|
return local_path
|
|
219
|
-
|
|
217
|
+
else
|
|
220
218
|
raise Error, 'Missing either local_db_dir or local.path'
|
|
221
219
|
end
|
|
222
220
|
else
|
|
@@ -226,15 +224,14 @@ module Aspera
|
|
|
226
224
|
return session['local_db_dir']
|
|
227
225
|
elsif session.key?('local_dir')
|
|
228
226
|
return session['local_dir']
|
|
229
|
-
|
|
227
|
+
else
|
|
230
228
|
raise Error, 'Missing either local_db_dir or local_dir'
|
|
231
229
|
end
|
|
232
230
|
end
|
|
233
|
-
nil
|
|
234
231
|
end
|
|
235
232
|
|
|
236
233
|
def session_name(sync_info)
|
|
237
|
-
Aspera.assert(PARAM_KEYS.any?{ |k| sync_info.key?(k)}, 'At least one of `local` or `sessions` must be present in async parameters', type: Error)
|
|
234
|
+
Aspera.assert(PARAM_KEYS.any? { |k| sync_info.key?(k) }, 'At least one of `local` or `sessions` must be present in async parameters', type: Error)
|
|
238
235
|
if sync_info.key?('local')
|
|
239
236
|
# `conf` format
|
|
240
237
|
return sync_info['name']
|
|
@@ -246,7 +243,7 @@ module Aspera
|
|
|
246
243
|
|
|
247
244
|
def session_db_file(sync_info)
|
|
248
245
|
db_file = File.join(local_db_folder(sync_info), PRIVATE_FOLDER, session_name(sync_info), ASYNC_DB)
|
|
249
|
-
Aspera.assert(File.exist?(db_file)){"Database file #{db_file} does not exist"}
|
|
246
|
+
Aspera.assert(File.exist?(db_file)) { "Database file #{db_file} does not exist" }
|
|
250
247
|
db_file
|
|
251
248
|
end
|
|
252
249
|
|
|
@@ -287,7 +284,7 @@ module Aspera
|
|
|
287
284
|
# @param option [String] Option to search
|
|
288
285
|
# @return [Array,nil] with path/schema for that option
|
|
289
286
|
def find_option(schema, path, option)
|
|
290
|
-
if %w[x-cli-option x-cli-short].any?{ |i| schema[i].eql?(option)}
|
|
287
|
+
if %w[x-cli-option x-cli-short].any? { |i| schema[i].eql?(option) }
|
|
291
288
|
Log.log.debug('Special') if schema['x-cli-special']
|
|
292
289
|
return [path, schema]
|
|
293
290
|
end
|
|
@@ -325,7 +322,7 @@ module Aspera
|
|
|
325
322
|
end
|
|
326
323
|
end
|
|
327
324
|
path, props = find_option(CONF_SCHEMA, [], option)
|
|
328
|
-
|
|
325
|
+
Aspera.assert(!path.nil?) { "Option not found: #{option}" }
|
|
329
326
|
last_key = path.pop
|
|
330
327
|
# navigate in the current result to insert the value
|
|
331
328
|
current = result
|
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
require 'singleton'
|
|
4
4
|
require 'fileutils'
|
|
5
5
|
require 'etc'
|
|
6
|
+
require 'tempfile'
|
|
6
7
|
|
|
7
8
|
module Aspera
|
|
8
9
|
# create a temp file name for a given folder
|
|
@@ -40,7 +41,7 @@ module Aspera
|
|
|
40
41
|
def delete_file(filepath)
|
|
41
42
|
File.delete(filepath) if @cleanup_on_exit
|
|
42
43
|
rescue => e
|
|
43
|
-
Log.log.warn{"Problem deleting file: #{filepath}: #{e.message}"}
|
|
44
|
+
Log.log.warn { "Problem deleting file: #{filepath}: #{e.message}" }
|
|
44
45
|
end
|
|
45
46
|
|
|
46
47
|
# Call this on process exit
|
|
@@ -51,11 +52,16 @@ module Aspera
|
|
|
51
52
|
@created_files = []
|
|
52
53
|
end
|
|
53
54
|
|
|
54
|
-
# Ensure that provided folder exists, or create it,
|
|
55
|
-
#
|
|
55
|
+
# Ensure that provided folder exists, or create it, atomically create and reserve a unique file.
|
|
56
|
+
# The file is created immediately (O_EXCL) to prevent TOCTOU race conditions.
|
|
57
|
+
# @return [String] path to that unique file
|
|
56
58
|
def new_file_path_in_folder(temp_folder, prefix: nil, suffix: nil)
|
|
57
59
|
FileUtils.mkdir_p(temp_folder)
|
|
58
|
-
|
|
60
|
+
basename = [prefix || 'aspera', SecureRandom.uuid].join('-')
|
|
61
|
+
# Tempfile.new creates the file atomically with O_EXCL; close keeps it on disk
|
|
62
|
+
tmp = Tempfile.new([basename, suffix ? "-#{suffix}" : ''], temp_folder)
|
|
63
|
+
tmp.close
|
|
64
|
+
new_file = tmp.path
|
|
59
65
|
@created_files.push(new_file)
|
|
60
66
|
new_file
|
|
61
67
|
end
|
|
@@ -72,6 +78,20 @@ module Aspera
|
|
|
72
78
|
new_file_path_in_folder(@global_temp, prefix: prefix, suffix: suffix)
|
|
73
79
|
end
|
|
74
80
|
|
|
81
|
+
# Generate a unique directory path inside the global temp folder (does NOT create it).
|
|
82
|
+
# Unlike new_file_path_global, the caller is expected to use the path as a directory.
|
|
83
|
+
# @return [String] path to a unique (not yet existing) directory
|
|
84
|
+
def new_dir_path_global(prefix = nil)
|
|
85
|
+
username =
|
|
86
|
+
begin
|
|
87
|
+
Etc.getlogin || Etc.getpwuid(Process.uid).name || 'unknown_user'
|
|
88
|
+
rescue StandardError
|
|
89
|
+
'unknown_user'
|
|
90
|
+
end
|
|
91
|
+
dir_name = [prefix, username, SecureRandom.uuid].compact.join('-')
|
|
92
|
+
File.join(@global_temp, dir_name)
|
|
93
|
+
end
|
|
94
|
+
|
|
75
95
|
# Garbage collect undeleted files
|
|
76
96
|
def cleanup_expired(temp_folder)
|
|
77
97
|
Dir.entries(temp_folder).each do |name|
|
|
@@ -79,7 +99,7 @@ module Aspera
|
|
|
79
99
|
age_sec = (Time.now - File.stat(file_path).mtime).to_i
|
|
80
100
|
# check age of file, delete too old
|
|
81
101
|
if File.file?(file_path) && (age_sec > FILE_LIST_AGE_MAX_SEC)
|
|
82
|
-
Log.log.debug{"garbage collecting #{name}"}
|
|
102
|
+
Log.log.debug { "garbage collecting #{name}" }
|
|
83
103
|
delete_file(file_path)
|
|
84
104
|
end
|
|
85
105
|
end
|
data/lib/aspera/timer_limiter.rb
CHANGED
|
@@ -11,42 +11,55 @@ module Aspera
|
|
|
11
11
|
SIZE_UNITS = %w[k m g t p e].freeze
|
|
12
12
|
private_constant :SCHEME, :PREFIX, :SIZE_UNITS
|
|
13
13
|
class << self
|
|
14
|
-
#
|
|
14
|
+
# Parse a faux URI and return a `FauxFile` instance, or `nil` if the URI does not use the faux scheme.
|
|
15
|
+
# URI format: `faux:///<path>?<size>`, where `<size>` is a decimal integer with an optional
|
|
16
|
+
# case-insensitive unit suffix: `k`, `m`, `g`, `t`, `p`, `e` (powers of 1024).
|
|
17
|
+
# When no suffix is given, the size is interpreted as raw bytes.
|
|
18
|
+
# Examples: `faux:///file.bin?10` (10 bytes), `faux:///file.bin?10m` (10 MiB)
|
|
19
|
+
# @param name [String] source file name, possibly a faux URI
|
|
20
|
+
# @return [FauxFile, nil] `nil` if not a faux scheme, else a `FauxFile` instance
|
|
15
21
|
def create(name)
|
|
16
22
|
return unless name.start_with?(PREFIX)
|
|
17
23
|
name_params = name.delete_prefix(PREFIX).split('?', 2)
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
SIZE_UNITS.each do |s|
|
|
23
|
-
size *= 1024
|
|
24
|
-
break if s.eql?(suffix)
|
|
25
|
-
end
|
|
24
|
+
Aspera.assert(name_params.length.eql?(2), type: Error) { "Format: #{PREFIX}<file path>?<size>" }
|
|
25
|
+
m = name_params[1].downcase.match(/^(\d+)([#{SIZE_UNITS.join('')}]?)$/)
|
|
26
|
+
Aspera.assert(m, type: Error) { "Format: <integer>[#{SIZE_UNITS.join(',')}]" }
|
|
27
|
+
size = m[2].empty? ? m[1].to_i : m[1].to_i * (1024**(SIZE_UNITS.index(m[2]) + 1))
|
|
26
28
|
return FauxFile.new(name_params[0], size)
|
|
27
29
|
end
|
|
28
30
|
end
|
|
31
|
+
# @return [String] virtual file path
|
|
32
|
+
# @return [Integer] total size in bytes
|
|
29
33
|
attr_reader :path, :size
|
|
30
34
|
|
|
35
|
+
# @param path [String] virtual file path (from the faux URI)
|
|
36
|
+
# @param size [Integer] total number of bytes to produce
|
|
31
37
|
def initialize(path, size)
|
|
32
38
|
@path = path
|
|
33
39
|
@size = size
|
|
34
40
|
@offset = 0
|
|
35
|
-
#
|
|
41
|
+
# cache chunks by size so repeated reads of the same chunk length reuse the same buffer
|
|
36
42
|
@chunk_by_size = {}
|
|
37
43
|
end
|
|
38
44
|
|
|
45
|
+
# Read up to `chunk_size` bytes from the stream and advance the internal offset.
|
|
46
|
+
# Returns `nil` when the stream is exhausted.
|
|
47
|
+
# @param chunk_size [Integer] maximum number of bytes to read
|
|
48
|
+
# @return [String, nil] null-byte string of the bytes actually read, or `nil` at EOF
|
|
39
49
|
def read(chunk_size)
|
|
40
50
|
return if eof?
|
|
41
51
|
bytes_to_read = [chunk_size, @size - @offset].min
|
|
42
52
|
@offset += bytes_to_read
|
|
43
|
-
@chunk_by_size[bytes_to_read]
|
|
53
|
+
@chunk_by_size[bytes_to_read] ||= "\x00" * bytes_to_read
|
|
44
54
|
return @chunk_by_size[bytes_to_read]
|
|
45
55
|
end
|
|
46
56
|
|
|
57
|
+
# No-op: required to satisfy the IO-like interface used by transfer agents.
|
|
58
|
+
# @return [nil]
|
|
47
59
|
def close
|
|
48
60
|
end
|
|
49
61
|
|
|
62
|
+
# @return [Boolean] true when all bytes have been read
|
|
50
63
|
def eof?
|
|
51
64
|
return @offset >= @size
|
|
52
65
|
end
|