aspera-cli 4.22.0 → 4.24.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/CHANGELOG.md +405 -364
- data/CONTRIBUTING.md +86 -29
- data/README.md +1856 -961
- data/bin/ascli +2 -1
- data/bin/asession +4 -4
- data/lib/aspera/agent/base.rb +4 -0
- data/lib/aspera/agent/connect.rb +20 -18
- data/lib/aspera/agent/desktop.rb +14 -11
- data/lib/aspera/agent/direct.rb +39 -31
- data/lib/aspera/agent/httpgw.rb +2 -2
- data/lib/aspera/agent/node.rb +9 -11
- data/lib/aspera/agent/transferd.rb +18 -11
- data/lib/aspera/api/aoc.rb +53 -43
- data/lib/aspera/api/cos_node.rb +7 -5
- data/lib/aspera/api/httpgw.rb +23 -22
- data/lib/aspera/api/node.rb +104 -22
- data/lib/aspera/ascmd.rb +35 -21
- data/lib/aspera/ascp/installation.rb +43 -43
- data/lib/aspera/ascp/management.rb +5 -4
- data/lib/aspera/assert.rb +55 -24
- data/lib/aspera/cli/basic_auth_plugin.rb +8 -7
- data/lib/aspera/cli/error.rb +1 -1
- data/lib/aspera/cli/extended_value.rb +28 -29
- data/lib/aspera/cli/formatter.rb +191 -168
- data/lib/aspera/cli/hints.rb +38 -4
- data/lib/aspera/cli/main.rb +139 -108
- data/lib/aspera/cli/manager.rb +51 -31
- data/lib/aspera/cli/plugin.rb +149 -78
- data/lib/aspera/cli/plugin_factory.rb +2 -2
- data/lib/aspera/cli/plugins/aoc.rb +217 -88
- data/lib/aspera/cli/plugins/ats.rb +15 -13
- data/lib/aspera/cli/plugins/config.rb +105 -227
- data/lib/aspera/cli/plugins/console.rb +49 -18
- data/lib/aspera/cli/plugins/cos.rb +4 -4
- data/lib/aspera/cli/plugins/faspex.rb +45 -51
- data/lib/aspera/cli/plugins/faspex5.rb +162 -163
- data/lib/aspera/cli/plugins/faspio.rb +6 -5
- data/lib/aspera/cli/plugins/httpgw.rb +2 -2
- data/lib/aspera/cli/plugins/node.rb +233 -247
- data/lib/aspera/cli/plugins/orchestrator.rb +10 -14
- data/lib/aspera/cli/plugins/preview.rb +26 -29
- data/lib/aspera/cli/plugins/server.rb +29 -28
- data/lib/aspera/cli/plugins/shares.rb +40 -28
- data/lib/aspera/cli/sync_actions.rb +101 -80
- data/lib/aspera/cli/transfer_agent.rb +55 -58
- data/lib/aspera/cli/transfer_progress.rb +29 -20
- data/lib/aspera/cli/version.rb +1 -1
- data/lib/aspera/cli/wizard.rb +160 -0
- data/lib/aspera/colors.rb +13 -8
- data/lib/aspera/command_line_builder.rb +28 -22
- data/lib/aspera/command_line_converter.rb +31 -0
- data/lib/aspera/data_repository.rb +1 -0
- data/lib/aspera/environment.rb +144 -100
- data/lib/aspera/faspex_gw.rb +1 -1
- data/lib/aspera/faspex_postproc.rb +3 -2
- data/lib/aspera/hash_ext.rb +1 -1
- data/lib/aspera/id_generator.rb +10 -10
- data/lib/aspera/keychain/base.rb +18 -0
- data/lib/aspera/keychain/encrypted_hash.rb +6 -12
- data/lib/aspera/keychain/factory.rb +9 -3
- data/lib/aspera/keychain/hashicorp_vault.rb +9 -6
- data/lib/aspera/keychain/macos_security.rb +13 -13
- data/lib/aspera/log.rb +70 -20
- data/lib/aspera/nagios.rb +5 -6
- data/lib/aspera/node_simulator.rb +12 -7
- data/lib/aspera/oauth/base.rb +6 -2
- data/lib/aspera/oauth/factory.rb +25 -18
- data/lib/aspera/oauth/jwt.rb +13 -1
- data/lib/aspera/oauth/url_json.rb +3 -3
- data/lib/aspera/oauth/web.rb +5 -3
- data/lib/aspera/persistency_folder.rb +2 -2
- data/lib/aspera/preview/file_types.rb +43 -35
- data/lib/aspera/preview/generator.rb +26 -13
- data/lib/aspera/preview/terminal.rb +10 -7
- data/lib/aspera/preview/utils.rb +11 -9
- data/lib/aspera/products/connect.rb +2 -1
- data/lib/aspera/products/desktop.rb +1 -1
- data/lib/aspera/products/other.rb +2 -2
- data/lib/aspera/products/transferd.rb +8 -6
- data/lib/aspera/proxy_auto_config.rb +1 -1
- data/lib/aspera/rest.rb +46 -28
- data/lib/aspera/rest_call_error.rb +1 -1
- data/lib/aspera/rest_error_analyzer.rb +1 -0
- data/lib/aspera/resumer.rb +1 -1
- data/lib/aspera/secret_hider.rb +46 -40
- data/lib/aspera/ssh.rb +14 -4
- data/lib/aspera/sync/args.schema.yaml +102 -0
- data/lib/aspera/sync/conf.schema.yaml +701 -0
- data/lib/aspera/sync/database.rb +83 -0
- data/lib/aspera/{transfer/sync.rb → sync/operations.rb} +145 -68
- data/lib/aspera/temp_file_manager.rb +4 -2
- data/lib/aspera/timer_limiter.rb +7 -5
- data/lib/aspera/transfer/error.rb +1 -1
- data/lib/aspera/transfer/error_info.rb +1 -2
- data/lib/aspera/transfer/faux_file.rb +11 -10
- data/lib/aspera/transfer/parameters.rb +6 -5
- data/lib/aspera/transfer/spec.rb +15 -1
- data/lib/aspera/transfer/spec.schema.yaml +316 -293
- data/lib/aspera/transfer/spec_doc.rb +34 -16
- data/lib/aspera/transfer/uri.rb +5 -5
- data/lib/aspera/uri_reader.rb +14 -10
- data/lib/aspera/web_auth.rb +2 -2
- data/lib/aspera/web_server_simple.rb +2 -2
- data.tar.gz.sig +0 -0
- metadata +15 -15
- metadata.gz.sig +0 -0
- data/examples/dascli +0 -30
- data/examples/get_proto_file.rb +0 -8
- data/examples/proxy.pac +0 -60
- data/lib/aspera/transfer/convert.rb +0 -29
- data/lib/aspera/transfer/sync_instance.schema.yaml +0 -13
- data/lib/aspera/transfer/sync_session.schema.yaml +0 -79
data/bin/ascli
CHANGED
@@ -12,5 +12,6 @@ $VERBOSE = old_verbose
|
|
12
12
|
require 'aspera/coverage'
|
13
13
|
require 'aspera/environment'
|
14
14
|
require 'aspera/cli/main'
|
15
|
-
Aspera::
|
15
|
+
Aspera::Cli::Main.early_debug_setup(ARGV)
|
16
|
+
Aspera::Environment.instance.fix_home
|
16
17
|
Aspera::Cli::Main.new(ARGV).process_command_line
|
data/bin/asession
CHANGED
@@ -21,6 +21,7 @@ SAMPLE_DEMO = '"remote_host":"demo.asperasoft.com","remote_user":"asperaweb","ss
|
|
21
21
|
SAMPLE_DEMO2 = '"direction":"receive","destination_root":"./test.dir"'
|
22
22
|
def assert_usage(assertion, error_message)
|
23
23
|
return if assertion
|
24
|
+
# rubocop:disable Style/StderrPuts
|
24
25
|
$stderr.puts('ERROR: '.red.blink + error_message) if error_message
|
25
26
|
$stderr.puts('USAGE')
|
26
27
|
$stderr.puts(' asession')
|
@@ -47,6 +48,7 @@ def assert_usage(assertion, error_message)
|
|
47
48
|
$stderr.puts('EXAMPLES')
|
48
49
|
$stderr.puts(%Q( asession @json:'{"#{PARAM_SPEC}":{#{SAMPLE_DEMO},#{SAMPLE_DEMO2},"paths":[{"source":"/aspera-test-dir-tiny/200KB.1"}]}}'))
|
49
50
|
$stderr.puts(%Q( echo '{"#{PARAM_SPEC}":{"remote_host":...}}'|asession @json:@stdin))
|
51
|
+
# rubocop:enable Style/StderrPuts
|
50
52
|
Process.exit(0)
|
51
53
|
end
|
52
54
|
parameter_source_err_msg = ' (argument), did you specify: "@json:" ?'
|
@@ -71,9 +73,7 @@ assert_usage(session_spec[PARAM_SPEC].is_a?(Hash), "The value must contain key #
|
|
71
73
|
# additional debug capability
|
72
74
|
Aspera::Log.instance.level = session_spec[PARAM_LOG_LEVEL] if session_spec.key?(PARAM_LOG_LEVEL)
|
73
75
|
# possibly override temp folder
|
74
|
-
if session_spec.key?(PARAM_TMP_FILE_LIST_FOLDER)
|
75
|
-
Aspera::Transfer::Parameters.file_list_folder = session_spec[PARAM_TMP_FILE_LIST_FOLDER]
|
76
|
-
end
|
76
|
+
Aspera::Transfer::Parameters.file_list_folder = session_spec[PARAM_TMP_FILE_LIST_FOLDER] if session_spec.key?(PARAM_TMP_FILE_LIST_FOLDER)
|
77
77
|
session_spec[PARAM_SDK] = File.join(Dir.home, '.aspera', 'sdk') unless session_spec.key?(PARAM_SDK)
|
78
78
|
Aspera::Products::Transferd.sdk_directory = session_spec[PARAM_SDK]
|
79
79
|
session_spec[PARAM_AGENT] = {} unless session_spec.key?(PARAM_AGENT)
|
@@ -90,7 +90,7 @@ job_id = client.start_transfer(session_spec[PARAM_SPEC])
|
|
90
90
|
Thread.new do
|
91
91
|
# we assume here a single session
|
92
92
|
session_id = client.sessions_by_job(job_id).first
|
93
|
-
begin
|
93
|
+
begin
|
94
94
|
loop do
|
95
95
|
data = JSON.parse($stdin.gets)
|
96
96
|
client.send_command(session_id, data)
|
data/lib/aspera/agent/base.rb
CHANGED
@@ -5,6 +5,10 @@ require 'aspera/assert'
|
|
5
5
|
module Aspera
|
6
6
|
module Agent
|
7
7
|
# Base class for transfer agents
|
8
|
+
# Transfer agents provide methods:
|
9
|
+
# - `start_transfer` : take a transfer spec and start a transfer asynchronously
|
10
|
+
# - `wait_for_transfers_completion` : waits for all transfer sessions to finish
|
11
|
+
# - `notify_progress` : called back by transfer agent to notify transfer progress
|
8
12
|
class Base
|
9
13
|
RUBY_EXT = '.rb'
|
10
14
|
private_constant :RUBY_EXT
|
data/lib/aspera/agent/connect.rb
CHANGED
@@ -21,27 +21,25 @@ module Aspera
|
|
21
21
|
@connect_settings = {
|
22
22
|
'app_id' => SecureRandom.uuid
|
23
23
|
}
|
24
|
-
raise 'Using connect requires a graphical environment'
|
24
|
+
raise Error, 'Using connect requires a graphical environment' unless Environment.instance.graphical?
|
25
25
|
method_index = 0
|
26
26
|
begin
|
27
27
|
connect_url = connect_api_url
|
28
28
|
Log.log.debug{"found: #{connect_url}"}
|
29
29
|
@connect_api = Rest.new(
|
30
30
|
base_url: "#{connect_url}/v5/connect", # could use v6 also now
|
31
|
-
headers: {'Origin' => RestParameters.instance.user_agent}
|
31
|
+
headers: {'Origin' => RestParameters.instance.user_agent}
|
32
|
+
)
|
32
33
|
connect_info = @connect_api.read('info/version')
|
33
34
|
Log.log.info('Connect was reached') if method_index > 0
|
34
|
-
Log.
|
35
|
+
Log.dump(:connect_version, connect_info)
|
35
36
|
rescue StandardError => e # Errno::ECONNREFUSED
|
36
37
|
Log.log.debug{"Exception: #{e}"}
|
37
38
|
start_url = CONNECT_START_URIS[method_index]
|
38
39
|
method_index += 1
|
39
40
|
raise StandardError, "Unable to start connect #{method_index} times" if start_url.nil?
|
40
41
|
Log.log.warn{"Aspera Connect is not started (#{e}). Trying to start it ##{method_index}..."}
|
41
|
-
|
42
|
-
Environment.open_uri_graphical('https://www.ibm.com/aspera/connect/')
|
43
|
-
raise StandardError, 'Connect is not installed'
|
44
|
-
end
|
42
|
+
Environment.instance.open_uri_graphical(start_url)
|
45
43
|
sleep(SLEEP_SEC_BETWEEN_RETRY)
|
46
44
|
retry
|
47
45
|
end
|
@@ -57,7 +55,8 @@ module Aspera
|
|
57
55
|
'title' => 'Select Files',
|
58
56
|
'suggestedName' => '',
|
59
57
|
'allowMultipleSelection' => true,
|
60
|
-
'allowedFileTypes' => ''
|
58
|
+
'allowedFileTypes' => ''
|
59
|
+
})
|
61
60
|
transfer_spec['paths'] = selection['dataTransfer']['files'].map{ |i| {'source' => i['name']}}
|
62
61
|
end
|
63
62
|
# if there is a token, we ask connect client to use well known ssh private keys
|
@@ -70,7 +69,8 @@ module Aspera
|
|
70
69
|
}),
|
71
70
|
'transfer_specs' => [{
|
72
71
|
'transfer_spec' => transfer_spec
|
73
|
-
}]
|
72
|
+
}]
|
73
|
+
}
|
74
74
|
# asynchronous anyway
|
75
75
|
res = @connect_api.create('transfers/start', connect_transfer_args)
|
76
76
|
@transfer_id = res['transfer_specs'].first['transfer_spec']['tags'][Transfer::Spec::TAG_RESERVED]['xfer_id']
|
@@ -83,7 +83,7 @@ module Aspera
|
|
83
83
|
begin
|
84
84
|
loop do
|
85
85
|
tr_info = @connect_api.create("transfers/info/#{@transfer_id}", connect_activity_args)
|
86
|
-
Log.
|
86
|
+
Log.dump(:tr_info, tr_info, level: :trace1)
|
87
87
|
if tr_info['transfer_info'].is_a?(Hash)
|
88
88
|
transfer = tr_info['transfer_info']
|
89
89
|
if transfer.nil?
|
@@ -93,7 +93,7 @@ module Aspera
|
|
93
93
|
# TODO: get session id
|
94
94
|
case transfer['status']
|
95
95
|
when 'initiating', 'queued'
|
96
|
-
notify_progress(:
|
96
|
+
notify_progress(:sessions_init, info: transfer['status'])
|
97
97
|
when 'running'
|
98
98
|
if !started
|
99
99
|
notify_progress(:session_start, session_id: @transfer_id)
|
@@ -106,16 +106,20 @@ module Aspera
|
|
106
106
|
notify_progress(:transfer, session_id: @transfer_id, info: transfer['bytes_written'])
|
107
107
|
end
|
108
108
|
when 'completed'
|
109
|
-
notify_progress(:
|
109
|
+
notify_progress(:session_end, session_id: @transfer_id)
|
110
|
+
notify_progress(:end)
|
110
111
|
break
|
111
112
|
when 'failed'
|
112
|
-
notify_progress(:
|
113
|
+
notify_progress(:session_end, session_id: @transfer_id)
|
114
|
+
notify_progress(:end)
|
113
115
|
raise Transfer::Error, transfer['error_desc']
|
114
116
|
when 'cancelled'
|
115
|
-
notify_progress(:
|
117
|
+
notify_progress(:session_end, session_id: @transfer_id)
|
118
|
+
notify_progress(:end)
|
116
119
|
raise Transfer::Error, 'Transfer cancelled by user'
|
117
120
|
else
|
118
|
-
notify_progress(:
|
121
|
+
notify_progress(:session_end, session_id: @transfer_id)
|
122
|
+
notify_progress(:end)
|
119
123
|
raise Transfer::Error, "unknown status: #{transfer['status']}: #{transfer['error_desc']}"
|
120
124
|
end
|
121
125
|
end
|
@@ -137,9 +141,7 @@ module Aspera
|
|
137
141
|
['', 's'].each do |ext|
|
138
142
|
uri_file = File.join(folder, "http#{ext}.uri")
|
139
143
|
Log.log.debug{"checking connect port file: #{uri_file}"}
|
140
|
-
if File.exist?(uri_file)
|
141
|
-
return File.open(uri_file, &:gets).strip
|
142
|
-
end
|
144
|
+
return File.open(uri_file, &:gets).strip if File.exist?(uri_file)
|
143
145
|
end
|
144
146
|
raise "no connect uri file found in #{folder}"
|
145
147
|
end
|
data/lib/aspera/agent/desktop.rb
CHANGED
@@ -5,6 +5,7 @@ require 'aspera/rest'
|
|
5
5
|
require 'aspera/environment'
|
6
6
|
require 'aspera/json_rpc'
|
7
7
|
require 'aspera/products/desktop'
|
8
|
+
require 'aspera/transfer/spec'
|
8
9
|
require 'securerandom'
|
9
10
|
|
10
11
|
module Aspera
|
@@ -21,24 +22,21 @@ module Aspera
|
|
21
22
|
@application_id = SecureRandom.uuid
|
22
23
|
@transfer_id = nil
|
23
24
|
super
|
24
|
-
raise 'Using client requires a graphical environment'
|
25
|
+
raise Error, 'Using client requires a graphical environment' unless Environment.instance.graphical?
|
25
26
|
method_index = 0
|
26
27
|
begin
|
27
28
|
# curl 'http://127.0.0.1:33024/' -X POST -H 'content-type: application/json' --data-raw '{"jsonrpc":"2.0","params":[],"id":999999,"method":"rpc.discover"}'
|
28
29
|
# https://playground.open-rpc.org/?schemaUrl=http://127.0.0.1:33024
|
29
30
|
@client_app_api = Aspera::JsonRpcClient.new(Aspera::Rest.new(base_url: aspera_client_api_url))
|
30
31
|
client_info = @client_app_api.get_info
|
31
|
-
Log.
|
32
|
+
Log.dump(:client_version, client_info)
|
32
33
|
Log.log.info('Client was reached') if method_index > 0
|
33
34
|
rescue Errno::ECONNREFUSED => e
|
34
35
|
start_url = START_URIS[method_index]
|
35
36
|
method_index += 1
|
36
37
|
raise StandardError, "Unable to start #{Products::Desktop::APP_NAME} #{method_index} times" if start_url.nil?
|
37
38
|
Log.log.warn{"#{Products::Desktop::APP_NAME} is not started (#{e}). Trying to start it ##{method_index}..."}
|
38
|
-
|
39
|
-
Environment.open_uri_graphical('https://www.ibm.com/aspera/connect/')
|
40
|
-
raise StandardError, "#{Products::Desktop::APP_NAME} is not installed"
|
41
|
-
end
|
39
|
+
Environment.instance.open_uri_graphical(start_url)
|
42
40
|
sleep(SLEEP_SEC_BETWEEN_RETRY)
|
43
41
|
retry
|
44
42
|
end
|
@@ -46,6 +44,7 @@ module Aspera
|
|
46
44
|
|
47
45
|
# :reek:UnusedParameters token_regenerator
|
48
46
|
def start_transfer(transfer_spec, token_regenerator: nil)
|
47
|
+
Transfer::Spec.fix_transferd_resume_policy(transfer_spec)
|
49
48
|
@request_id = SecureRandom.uuid
|
50
49
|
# if there is a token, we ask the client app to use well known ssh private keys
|
51
50
|
# instead of asking password
|
@@ -62,7 +61,7 @@ module Aspera
|
|
62
61
|
transfer = @client_app_api.get_transfer(app_id: @application_id, transfer_id: @transfer_id)
|
63
62
|
case transfer['status']
|
64
63
|
when 'initiating', 'queued'
|
65
|
-
notify_progress(:
|
64
|
+
notify_progress(:sessions_init, info: transfer['status'])
|
66
65
|
when 'running'
|
67
66
|
if !started
|
68
67
|
notify_progress(:session_start, session_id: @transfer_id)
|
@@ -75,16 +74,20 @@ module Aspera
|
|
75
74
|
notify_progress(:transfer, session_id: @transfer_id, info: transfer['bytes_written'])
|
76
75
|
end
|
77
76
|
when 'completed'
|
78
|
-
notify_progress(:
|
77
|
+
notify_progress(:session_end, session_id: @transfer_id)
|
78
|
+
notify_progress(:end)
|
79
79
|
break
|
80
80
|
when 'failed'
|
81
|
-
notify_progress(:
|
81
|
+
notify_progress(:session_end, session_id: @transfer_id)
|
82
|
+
notify_progress(:end)
|
82
83
|
raise Transfer::Error, transfer['error_desc']
|
83
84
|
when 'cancelled'
|
84
|
-
notify_progress(:
|
85
|
+
notify_progress(:session_end, session_id: @transfer_id)
|
86
|
+
notify_progress(:end)
|
85
87
|
raise Transfer::Error, 'Transfer cancelled by user'
|
86
88
|
else
|
87
|
-
notify_progress(:
|
89
|
+
notify_progress(:session_end, session_id: @transfer_id)
|
90
|
+
notify_progress(:end)
|
88
91
|
raise Transfer::Error, "unknown status: #{transfer['status']}: #{transfer['error_desc']}"
|
89
92
|
end
|
90
93
|
sleep(1)
|
data/lib/aspera/agent/direct.rb
CHANGED
@@ -66,7 +66,7 @@ module Aspera
|
|
66
66
|
@spawn_timeout_sec = spawn_timeout_sec
|
67
67
|
@spawn_delay_sec = spawn_delay_sec
|
68
68
|
# default is true on Windows, false on other OSes
|
69
|
-
@multi_incr_udp = multi_incr_udp.nil? ? Environment.os.eql?(Environment::OS_WINDOWS) : multi_incr_udp
|
69
|
+
@multi_incr_udp = multi_incr_udp.nil? ? Environment.instance.os.eql?(Environment::OS_WINDOWS) : multi_incr_udp
|
70
70
|
@monitor = monitor
|
71
71
|
@management_cb = management_cb
|
72
72
|
@resume_policy = Resumer.new(resume.nil? ? {} : resume.symbolize_keys)
|
@@ -96,7 +96,7 @@ module Aspera
|
|
96
96
|
# TODO: useful ? node only ? seems to be a timeout for retry in node
|
97
97
|
transfer_spec['tags'][Transfer::Spec::TAG_RESERVED]['xfer_retry'] ||= 3600
|
98
98
|
end
|
99
|
-
Log.
|
99
|
+
Log.dump(:ts, transfer_spec)
|
100
100
|
# Compute this before using transfer spec because it potentially modifies the transfer spec
|
101
101
|
# (even if the var is not used in single session)
|
102
102
|
multi_session_info = nil
|
@@ -171,6 +171,7 @@ module Aspera
|
|
171
171
|
session[:thread].join
|
172
172
|
result.push(session[:error] || :success)
|
173
173
|
end
|
174
|
+
notify_progress(:end)
|
174
175
|
Log.log.debug('all transfers joined')
|
175
176
|
# since all are finished and we return the result, clear statuses
|
176
177
|
@sessions.clear
|
@@ -225,21 +226,24 @@ module Aspera
|
|
225
226
|
# This is the low level method to start the transfer process.
|
226
227
|
# Typically started in a thread.
|
227
228
|
# Start process with management port.
|
228
|
-
# @param session
|
229
|
-
# @param
|
230
|
-
# @param
|
231
|
-
# @param args
|
229
|
+
# @param session [Hash] This session information, keys :io and :token_regenerator
|
230
|
+
# @param name [Symbol] Name of executable: :ascp, :ascp4 or :async (comes from ascp_args)
|
231
|
+
# @param env [Hash] Environment variables (comes from ascp_args)
|
232
|
+
# @param args [Array] Command line arguments (comes from ascp_args)
|
232
233
|
# @return [nil] when process has exited
|
233
234
|
# @throw FaspError on error
|
234
235
|
def start_and_monitor_process(
|
235
236
|
session:,
|
236
|
-
env:,
|
237
237
|
name:,
|
238
|
+
env:,
|
238
239
|
args:
|
239
240
|
)
|
240
241
|
Aspera.assert_type(session, Hash)
|
241
|
-
notify_progress(:
|
242
|
+
notify_progress(:sessions_init, info: 'starting')
|
242
243
|
begin
|
244
|
+
capture_stderr = false
|
245
|
+
stderr_r, stderr_w = nil
|
246
|
+
spawn_args = {}
|
243
247
|
command_pid = nil
|
244
248
|
command_arguments = []
|
245
249
|
if @monitor
|
@@ -258,13 +262,17 @@ module Aspera
|
|
258
262
|
end
|
259
263
|
end
|
260
264
|
command_arguments.concat(args)
|
261
|
-
|
262
|
-
|
265
|
+
if capture_stderr
|
266
|
+
# capture process stderr
|
267
|
+
stderr_r, stderr_w = IO.pipe
|
268
|
+
spawn_args[err] = stderr_w
|
269
|
+
end
|
263
270
|
# get location of command executable (ascp, async)
|
264
271
|
command_path = Ascp::Installation.instance.path(name)
|
265
|
-
command_pid = Environment.secure_spawn(env: env, exec: command_path, args: command_arguments,
|
266
|
-
|
267
|
-
|
272
|
+
command_pid = Environment.secure_spawn(env: env, exec: command_path, args: command_arguments, **spawn_args)
|
273
|
+
# close here, but still used in other process (pipe)
|
274
|
+
stderr_w&.close
|
275
|
+
notify_progress(:sessions_init, info: "waiting for #{name} to start")
|
268
276
|
# "ensure" block will wait for process
|
269
277
|
return unless @monitor
|
270
278
|
# TODO: timeout does not work when Process.spawn is used... until process exits, then it works
|
@@ -272,7 +280,7 @@ module Aspera
|
|
272
280
|
Log.log.debug{"before select, timeout: #{@spawn_timeout_sec}"}
|
273
281
|
readable, _, _ = IO.select([mgt_server_socket], nil, nil, @spawn_timeout_sec)
|
274
282
|
Log.log.debug('after select, before accept')
|
275
|
-
Aspera.assert(readable,
|
283
|
+
Aspera.assert(readable, type: Transfer::Error){'timeout waiting mgt port connect (select not readable)'}
|
276
284
|
# There is a connection to accept
|
277
285
|
client_socket, _client_addrinfo = mgt_server_socket.accept
|
278
286
|
Log.log.debug('after accept')
|
@@ -288,12 +296,11 @@ module Aspera
|
|
288
296
|
event = processor.process_line(line.chomp)
|
289
297
|
next unless event
|
290
298
|
# event is ready
|
291
|
-
Log.
|
299
|
+
Log.dump(:management_port, event, level: :trace1)
|
292
300
|
# store session identifier
|
293
301
|
session[:id] = event['SessionId'] if event['Type'].eql?('INIT')
|
294
302
|
@management_cb&.call(event)
|
295
303
|
process_progress(event)
|
296
|
-
Log.log.error(event['Description'].to_s) if event['Type'].eql?('FILEERROR') # cspell:disable-line
|
297
304
|
end
|
298
305
|
Log.log.debug('management io closed')
|
299
306
|
# check that last status was received before process exit
|
@@ -311,9 +318,7 @@ module Aspera
|
|
311
318
|
env['ASPERA_SCP_TOKEN'] = session[:token_regenerator].refreshed_transfer_token
|
312
319
|
end
|
313
320
|
raise Transfer::Error.new(last_event['Description'], last_event['Code'].to_i)
|
314
|
-
else
|
315
|
-
Log.log.error{"unexpected last event type: #{last_event['Type']}"}
|
316
|
-
# raise Transfer::Error, "unexpected last event type: #{last_event['Type']}, #{last_event['Description']}"
|
321
|
+
else Aspera.error_unexpected_value(last_event['Type'], :error){'last event type'}
|
317
322
|
end
|
318
323
|
rescue SystemCallError => e
|
319
324
|
# Process.spawn failed, or socket error
|
@@ -325,14 +330,20 @@ module Aspera
|
|
325
330
|
session.delete(:io)
|
326
331
|
# if command was successfully started, check its status
|
327
332
|
unless command_pid.nil?
|
328
|
-
Process.kill(:INT, command_pid) if @monitor
|
333
|
+
Process.kill(:INT, command_pid) if @monitor && !Environment.instance.os.eql?(Environment::OS_WINDOWS)
|
329
334
|
# collect process exit status or wait for termination
|
330
335
|
_, status = Process.wait2(command_pid)
|
331
|
-
|
332
|
-
|
333
|
-
|
336
|
+
if stderr_r
|
337
|
+
# process stderr of ascp
|
338
|
+
stderr_flag = false
|
339
|
+
stderr_r.each_line do |line|
|
340
|
+
Log.log.error{"BEGIN stderr #{name}"} unless stderr_flag
|
341
|
+
Log.log.error{line.chomp}
|
342
|
+
stderr_flag = true
|
343
|
+
end
|
344
|
+
Log.log.error{"END stderr #{name}"} if stderr_flag
|
345
|
+
stderr_r.close
|
334
346
|
end
|
335
|
-
stderr_r.close
|
336
347
|
# status is nil if an exception occurred before starting command
|
337
348
|
if !status&.success?
|
338
349
|
message = "#{name} failed (#{status})"
|
@@ -370,17 +381,14 @@ module Aspera
|
|
370
381
|
notify_progress(:transfer, session_id: session_id, info: @pre_calc_last_size)
|
371
382
|
when 'DONE', 'ERROR' # end of session
|
372
383
|
total_size = event['TransferBytes'].to_i + event['StartByte'].to_i
|
373
|
-
if !@pre_calc_sent && !total_size.zero?
|
374
|
-
|
375
|
-
|
376
|
-
if @pre_calc_last_size != total_size
|
377
|
-
notify_progress(:transfer, session_id: session_id, info: total_size)
|
378
|
-
end
|
379
|
-
notify_progress(:end, session_id: session_id)
|
384
|
+
notify_progress(:session_size, session_id: session_id, info: total_size) if !@pre_calc_sent && !total_size.zero?
|
385
|
+
notify_progress(:transfer, session_id: session_id, info: total_size) if @pre_calc_last_size != total_size
|
386
|
+
notify_progress(:session_end, session_id: session_id)
|
380
387
|
# cspell:disable
|
381
388
|
when 'SESSION'
|
382
389
|
when 'ARGSTOP'
|
383
390
|
when 'FILEERROR'
|
391
|
+
Log.log.error{"#{event['Type']} #{event['Description']}"}
|
384
392
|
when 'STOP'
|
385
393
|
# cspell:enable
|
386
394
|
# stop event when one file is completed
|
data/lib/aspera/agent/httpgw.rb
CHANGED
@@ -32,10 +32,10 @@ module Aspera
|
|
32
32
|
# HTTP download only supports file list
|
33
33
|
# :reek:UnusedParameters token_regenerator
|
34
34
|
def start_transfer(transfer_spec, token_regenerator: nil)
|
35
|
-
|
35
|
+
Aspera.assert(!@gw_api.nil?){'GW URL must be set'}
|
36
36
|
Aspera.assert_type(transfer_spec['paths'], Array){'paths'}
|
37
37
|
Aspera.assert_type(transfer_spec['token'], String){'only token based transfer is supported in GW'}
|
38
|
-
Log.
|
38
|
+
Log.dump(:user_spec, transfer_spec)
|
39
39
|
transfer_spec['authentication'] ||= 'token'
|
40
40
|
case transfer_spec['direction']
|
41
41
|
when Transfer::Spec::DIRECTION_SEND
|
data/lib/aspera/agent/node.rb
CHANGED
@@ -29,7 +29,7 @@ module Aspera
|
|
29
29
|
# root id is required for access key
|
30
30
|
@root_id = root_id
|
31
31
|
rest_params = {base_url: url}
|
32
|
-
if OAuth::Factory.
|
32
|
+
if OAuth::Factory.bearer_auth?(password)
|
33
33
|
Aspera.assert(!@root_id.nil?){'root_id not allowed for access key'}
|
34
34
|
rest_params[:headers] = Api::Node.bearer_headers(password, access_key: username)
|
35
35
|
else
|
@@ -57,14 +57,10 @@ module Aspera
|
|
57
57
|
end
|
58
58
|
# add mandatory retry parameter for node api
|
59
59
|
ts_tags = transfer_spec['tags']
|
60
|
-
if ts_tags.is_a?(Hash) && ts_tags[Transfer::Spec::TAG_RESERVED].is_a?(Hash)
|
61
|
-
ts_tags[Transfer::Spec::TAG_RESERVED]['xfer_retry'] ||= 150
|
62
|
-
end
|
60
|
+
ts_tags[Transfer::Spec::TAG_RESERVED]['xfer_retry'] ||= 150 if ts_tags.is_a?(Hash) && ts_tags[Transfer::Spec::TAG_RESERVED].is_a?(Hash)
|
63
61
|
# Optimization in case of sending to the same node
|
64
62
|
# TODO: probably remove this, as /etc/hosts shall be used for that
|
65
|
-
if !transfer_spec['wss_enabled'] && transfer_spec['remote_host'].eql?(URI.parse(node_api_.base_url).host)
|
66
|
-
transfer_spec['remote_host'] = '127.0.0.1'
|
67
|
-
end
|
63
|
+
transfer_spec['remote_host'] = '127.0.0.1' if !transfer_spec['wss_enabled'] && transfer_spec['remote_host'].eql?(URI.parse(node_api_.base_url).host)
|
68
64
|
resp = node_api_.create('ops/transfers', transfer_spec)
|
69
65
|
@transfer_id = resp['id']
|
70
66
|
Log.log.debug{"tr_id=#{@transfer_id}"}
|
@@ -82,7 +78,7 @@ module Aspera
|
|
82
78
|
transfer_data = node_api_.read("ops/transfers/#{@transfer_id}") || {'status' => 'unknown'} rescue {'status' => 'waiting(api error)'}
|
83
79
|
case transfer_data['status']
|
84
80
|
when 'waiting', 'partially_completed', 'unknown', 'waiting(read error)'
|
85
|
-
notify_progress(:
|
81
|
+
notify_progress(:sessions_init, info: transfer_data['status'])
|
86
82
|
when 'running'
|
87
83
|
if !session_started
|
88
84
|
notify_progress(:session_start, session_id: @transfer_id)
|
@@ -90,7 +86,7 @@ module Aspera
|
|
90
86
|
end
|
91
87
|
message = transfer_data['status']
|
92
88
|
message = "#{message} (#{transfer_data['error_desc']})" if !transfer_data['error_desc']&.empty?
|
93
|
-
notify_progress(:
|
89
|
+
notify_progress(:sessions_init, info: message)
|
94
90
|
if bytes_expected.nil? &&
|
95
91
|
transfer_data['precalc'].is_a?(Hash) &&
|
96
92
|
transfer_data['precalc']['status'].eql?('ready')
|
@@ -100,10 +96,12 @@ module Aspera
|
|
100
96
|
notify_progress(:transfer, session_id: @transfer_id, info: transfer_data['bytes_transferred'])
|
101
97
|
when 'completed'
|
102
98
|
notify_progress(:transfer, session_id: @transfer_id, info: bytes_expected) if bytes_expected
|
103
|
-
notify_progress(:
|
99
|
+
notify_progress(:session_end, session_id: @transfer_id)
|
100
|
+
notify_progress(:end)
|
104
101
|
break
|
105
102
|
when 'failed'
|
106
|
-
notify_progress(:
|
103
|
+
notify_progress(:session_end, session_id: @transfer_id)
|
104
|
+
notify_progress(:end)
|
107
105
|
# Bug in HSTS ? transfer is marked failed, but there is no reason
|
108
106
|
break if transfer_data['error_code'].eql?(0) && transfer_data['error_desc'].empty?
|
109
107
|
raise Transfer::Error, "status: #{transfer_data['status']}. code: #{transfer_data['error_code']}. description: #{transfer_data['error_desc']}"
|
@@ -4,6 +4,7 @@ require 'aspera/environment'
|
|
4
4
|
require 'aspera/agent/base'
|
5
5
|
require 'aspera/products/transferd'
|
6
6
|
require 'aspera/temp_file_manager'
|
7
|
+
require 'aspera/transfer/spec'
|
7
8
|
require 'json'
|
8
9
|
require 'uri'
|
9
10
|
require 'transferd_services_pb'
|
@@ -34,11 +35,11 @@ module Aspera
|
|
34
35
|
@transfer_id = nil
|
35
36
|
@stop = stop
|
36
37
|
is_local_auto_port = url.eql?(AUTO_LOCAL_TCP_PORT)
|
37
|
-
raise 'Cannot set options `stop` or `start` to false with port zero' if is_local_auto_port && (!@stop || !start)
|
38
|
+
raise Error, 'Cannot set options `stop` or `start` to false with port zero' if is_local_auto_port && (!@stop || !start)
|
38
39
|
# keep PID for optional shutdown
|
39
40
|
@daemon_pid = nil
|
40
41
|
daemon_endpoint = url
|
41
|
-
Log.
|
42
|
+
Log.dump(:daemon_endpoint, daemon_endpoint)
|
42
43
|
# retry loop
|
43
44
|
begin
|
44
45
|
# no address: local bind
|
@@ -82,7 +83,8 @@ module Aspera
|
|
82
83
|
exec: Ascp::Installation.instance.path(:transferd),
|
83
84
|
args: ['--config', conf_file],
|
84
85
|
out: log_stdout,
|
85
|
-
err: log_stderr
|
86
|
+
err: log_stderr
|
87
|
+
)
|
86
88
|
begin
|
87
89
|
# wait for process to initialize, max 2 seconds
|
88
90
|
Timeout.timeout(2.0) do
|
@@ -105,15 +107,18 @@ module Aspera
|
|
105
107
|
|
106
108
|
# :reek:UnusedParameters token_regenerator
|
107
109
|
def start_transfer(transfer_spec, token_regenerator: nil)
|
110
|
+
Transfer::Spec.fix_transferd_resume_policy(transfer_spec)
|
108
111
|
# create a transfer request
|
109
112
|
transfer_request = ::Transferd::Api::TransferRequest.new(
|
110
113
|
transferType: ::Transferd::Api::TransferType::FILE_REGULAR, # transfer type (file/stream)
|
111
114
|
config: ::Transferd::Api::TransferConfig.new, # transfer configuration
|
112
|
-
transferSpec: transfer_spec.to_json
|
115
|
+
transferSpec: transfer_spec.to_json
|
116
|
+
) # transfer definition
|
113
117
|
# send start transfer request to the transfer manager daemon
|
114
|
-
|
115
|
-
|
116
|
-
|
118
|
+
start_response = @transfer_client.start_transfer(transfer_request)
|
119
|
+
raise Transfer::Error, start_response.error.description if start_response.status.eql?(:FAILED)
|
120
|
+
Log.log.debug{"start transfer response #{start_response}"}
|
121
|
+
@transfer_id = start_response.transferId
|
117
122
|
Log.log.debug{"transfer started with id #{@transfer_id}"}
|
118
123
|
end
|
119
124
|
|
@@ -123,7 +128,7 @@ module Aspera
|
|
123
128
|
bytes_expected = nil
|
124
129
|
# monitor transfer status
|
125
130
|
@transfer_client.monitor_transfers(::Transferd::Api::RegistrationRequest.new(transferId: [@transfer_id])) do |response|
|
126
|
-
Log.
|
131
|
+
Log.dump(:response, response.to_h)
|
127
132
|
# Log.log.debug{"#{response.sessionInfo.preTransferBytes} #{response.transferInfo.bytesTransferred}"}
|
128
133
|
case response.status
|
129
134
|
when :RUNNING
|
@@ -139,13 +144,15 @@ module Aspera
|
|
139
144
|
notify_progress(:transfer, session_id: @transfer_id, info: response.transferInfo.bytesTransferred)
|
140
145
|
when :COMPLETED
|
141
146
|
notify_progress(:transfer, session_id: @transfer_id, info: bytes_expected) if bytes_expected
|
142
|
-
notify_progress(:
|
147
|
+
notify_progress(:session_end, session_id: @transfer_id)
|
148
|
+
notify_progress(:end)
|
143
149
|
break
|
144
150
|
when :FAILED, :CANCELED
|
145
|
-
notify_progress(:
|
151
|
+
notify_progress(:session_end, session_id: @transfer_id)
|
152
|
+
notify_progress(:end)
|
146
153
|
raise Transfer::Error, JSON.parse(response.message)['Description']
|
147
154
|
when :QUEUED, :UNKNOWN_STATUS, :PAUSED, :ORPHANED
|
148
|
-
notify_progress(:
|
155
|
+
notify_progress(:sessions_init, info: response.status.to_s.downcase)
|
149
156
|
else
|
150
157
|
Log.log.error{"unknown status#{response.status}"}
|
151
158
|
end
|