aspera-cli 4.27.0 → 4.27.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/CHANGELOG.md +46 -1
- data/CONTRIBUTING.md +1 -7
- data/TODO.md +50 -0
- data/docs/README.md +709 -489
- data/lib/aspera/agent/base.rb +10 -2
- data/lib/aspera/agent/connect.rb +7 -7
- data/lib/aspera/agent/desktop.rb +3 -3
- data/lib/aspera/agent/direct.rb +48 -19
- data/lib/aspera/agent/factory.rb +3 -3
- data/lib/aspera/agent/httpgw.rb +55 -14
- data/lib/aspera/agent/node.rb +2 -2
- data/lib/aspera/agent/transferd.rb +12 -12
- data/lib/aspera/api/aoc.rb +28 -28
- data/lib/aspera/api/cos_node.rb +10 -9
- data/lib/aspera/api/faspex.rb +2 -2
- data/lib/aspera/api/httpgw.rb +20 -20
- data/lib/aspera/api/node.rb +31 -53
- data/lib/aspera/ascmd.rb +13 -13
- data/lib/aspera/ascp/installation.rb +43 -36
- data/lib/aspera/ascp/management.rb +6 -6
- data/lib/aspera/assert.rb +12 -13
- data/lib/aspera/cli/ascp_actions.rb +7 -7
- data/lib/aspera/cli/async_transfer_store.rb +49 -5
- data/lib/aspera/cli/bootstrapper.rb +9 -9
- data/lib/aspera/cli/command_registry.rb +1 -1
- data/lib/aspera/cli/command_spec.rb +6 -3
- data/lib/aspera/cli/context.rb +3 -3
- data/lib/aspera/cli/extended_value.rb +26 -26
- data/lib/aspera/cli/formatter.rb +76 -54
- data/lib/aspera/cli/gem_checker.rb +1 -1
- data/lib/aspera/cli/hints.rb +2 -2
- data/lib/aspera/cli/http.rb +22 -29
- data/lib/aspera/cli/mailer.rb +7 -7
- data/lib/aspera/cli/mcp_tool.rb +31 -4
- data/lib/aspera/cli/option_declarator.rb +77 -0
- data/lib/aspera/cli/options.schema.yaml +273 -3
- data/lib/aspera/cli/parser.rb +539 -280
- data/lib/aspera/cli/plugins/alee.rb +1 -1
- data/lib/aspera/cli/plugins/aoc.rb +303 -314
- data/lib/aspera/cli/plugins/ats.rb +25 -21
- data/lib/aspera/cli/plugins/base.rb +357 -186
- data/lib/aspera/cli/plugins/basic_auth.rb +0 -12
- data/lib/aspera/cli/plugins/config.rb +76 -73
- data/lib/aspera/cli/plugins/console.rb +8 -8
- data/lib/aspera/cli/plugins/cos.rb +4 -3
- data/lib/aspera/cli/plugins/factory.rb +2 -2
- data/lib/aspera/cli/plugins/faspex5.rb +259 -248
- data/lib/aspera/cli/plugins/faspio.rb +7 -2
- data/lib/aspera/cli/plugins/httpgw.rb +1 -1
- data/lib/aspera/cli/plugins/mcp.rb +11 -11
- data/lib/aspera/cli/plugins/node.rb +227 -283
- data/lib/aspera/cli/plugins/orchestrator.rb +23 -23
- data/lib/aspera/cli/plugins/preview.rb +61 -64
- data/lib/aspera/cli/plugins/server.rb +36 -31
- data/lib/aspera/cli/plugins/shares.rb +174 -135
- data/lib/aspera/cli/preset_actions.rb +39 -19
- data/lib/aspera/cli/preset_manager.rb +74 -32
- data/lib/aspera/cli/result.rb +18 -16
- data/lib/aspera/cli/runner.rb +75 -39
- data/lib/aspera/cli/secret_finder.rb +1 -1
- data/lib/aspera/cli/sync_actions.rb +36 -35
- data/lib/aspera/cli/terminal_formatter.rb +2 -2
- data/lib/aspera/cli/transfer_actions.rb +17 -10
- data/lib/aspera/cli/transfer_agent.rb +44 -44
- data/lib/aspera/cli/transfer_progress.rb +7 -7
- data/lib/aspera/cli/vault_manager.rb +34 -12
- data/lib/aspera/cli/version.rb +1 -1
- data/lib/aspera/cli/wizard.rb +17 -20
- data/lib/aspera/colors.rb +3 -3
- data/lib/aspera/command_line_builder.rb +17 -16
- data/lib/aspera/command_line_converter.rb +1 -1
- data/lib/aspera/coverage.rb +1 -2
- data/lib/aspera/data_repository.rb +1 -1
- data/lib/aspera/dot_container.rb +8 -8
- data/lib/aspera/environment.rb +13 -13
- data/lib/aspera/faspex_gw.rb +4 -4
- data/lib/aspera/faspex_postproc.rb +14 -9
- data/lib/aspera/graphql.rb +4 -2
- data/lib/aspera/hash_ext.rb +2 -2
- data/lib/aspera/json_rpc/client.rb +1 -1
- data/lib/aspera/keychain/base.rb +10 -3
- data/lib/aspera/keychain/encrypted_hash.rb +92 -17
- data/lib/aspera/keychain/factory.rb +17 -5
- data/lib/aspera/keychain/hashicorp_vault.rb +1 -1
- data/lib/aspera/keychain/macos_security.rb +23 -25
- data/lib/aspera/keychain/one_password_api.rb +86 -0
- data/lib/aspera/keychain/one_password_base.rb +34 -0
- data/lib/aspera/keychain/one_password_cli.rb +98 -0
- data/lib/aspera/link_header.rb +2 -2
- data/lib/aspera/log.rb +13 -13
- data/lib/aspera/markdown.rb +5 -5
- data/lib/aspera/nagios.rb +8 -8
- data/lib/aspera/node_simulator.rb +33 -27
- data/lib/aspera/oauth/base.rb +4 -5
- data/lib/aspera/oauth/boot.rb +3 -3
- data/lib/aspera/oauth/factory.rb +5 -5
- data/lib/aspera/oauth/jwt.rb +5 -7
- data/lib/aspera/oauth/web.rb +4 -4
- data/lib/aspera/persistency_action_once.rb +4 -4
- data/lib/aspera/persistency_folder.rb +8 -8
- data/lib/aspera/preview/file_types.rb +4 -4
- data/lib/aspera/preview/generator.rb +49 -46
- data/lib/aspera/preview/options.rb +3 -2
- data/lib/aspera/preview/terminal.rb +5 -5
- data/lib/aspera/preview/utils.rb +65 -18
- data/lib/aspera/products/connect.rb +2 -2
- data/lib/aspera/products/desktop.rb +1 -1
- data/lib/aspera/products/other.rb +2 -2
- data/lib/aspera/products/transferd.rb +3 -3
- data/lib/aspera/proxy_auto_config.rb +10 -10
- data/lib/aspera/rest.rb +149 -115
- data/lib/aspera/rest_error_analyzer.rb +3 -3
- data/lib/aspera/rest_list.rb +10 -9
- data/lib/aspera/schema/IBM Aspera Node API-4.4.6.yaml +6232 -0
- data/lib/aspera/schema/IBM Aspera faspio Gateway API-1.0.0.yaml +249 -0
- data/lib/aspera/schema/IBM Aspera on Cloud API-0.2.6-enhanced.yaml +39 -0
- data/lib/aspera/schema/IBM_Aspera_Shares.yaml +4653 -0
- data/lib/aspera/schema/documentation.rb +17 -11
- data/lib/aspera/schema/reader.rb +78 -4
- data/lib/aspera/schema/registry.rb +26 -7
- data/lib/aspera/secret_hider.rb +5 -5
- data/lib/aspera/ssh.rb +64 -31
- data/lib/aspera/ssl.rb +3 -3
- data/lib/aspera/sync/database.rb +2 -2
- data/lib/aspera/sync/operations.rb +13 -13
- data/lib/aspera/temp_file_manager.rb +24 -4
- data/lib/aspera/transfer/faux_file.rb +2 -2
- data/lib/aspera/transfer/parameters.rb +15 -15
- data/lib/aspera/transfer/resumer.rb +9 -9
- data/lib/aspera/transfer/spec.rb +1 -1
- data/lib/aspera/transfer/spec.schema.yaml +1 -0
- data/lib/aspera/uri_reader.rb +52 -15
- data/lib/aspera/web_auth.rb +6 -5
- data/lib/aspera/web_server_simple.rb +9 -9
- data/lib/aspera/yaml.rb +4 -4
- data.tar.gz.sig +0 -0
- metadata +10 -4
- metadata.gz.sig +0 -0
- data/lib/aspera/cli/plugins/faspex.rb +0 -576
- data/lib/aspera/transfer/uri.rb +0 -56
data/lib/aspera/agent/base.rb
CHANGED
|
@@ -28,11 +28,19 @@ module Aspera
|
|
|
28
28
|
statuses = wait_for_transfers_completion
|
|
29
29
|
@progress&.reset
|
|
30
30
|
Aspera.assert_type(statuses, Array)
|
|
31
|
-
Aspera.assert(statuses.none?{ |i| !i.eql?(:success) && !i.is_a?(StandardError)}){"bad statuses content: #{statuses}"}
|
|
32
|
-
errors = statuses.reject{ |i| i.eql?(:success)}
|
|
31
|
+
Aspera.assert(statuses.none? { |i| !i.eql?(:success) && !i.is_a?(StandardError) }) { "bad statuses content: #{statuses}" }
|
|
32
|
+
errors = statuses.reject { |i| i.eql?(:success) }
|
|
33
33
|
return errors.empty? ? Transfer::Result.success : Transfer::Result.error(errors.first)
|
|
34
34
|
end
|
|
35
35
|
|
|
36
|
+
# Return the job_id of the last transfer submitted by this agent.
|
|
37
|
+
# In-process agents (direct, httpgw) override this to expose their internal job id.
|
|
38
|
+
# Remote-daemon agents do not need to override it: TransferAgent generates a UUID itself.
|
|
39
|
+
# @return [String, nil]
|
|
40
|
+
def last_job_id
|
|
41
|
+
nil
|
|
42
|
+
end
|
|
43
|
+
|
|
36
44
|
private
|
|
37
45
|
|
|
38
46
|
Aspera.require_method!(:start_transfer)
|
data/lib/aspera/agent/connect.rb
CHANGED
|
@@ -66,12 +66,12 @@ module Aspera
|
|
|
66
66
|
@connect_settings = {
|
|
67
67
|
'app_id' => SecureRandom.uuid
|
|
68
68
|
}
|
|
69
|
-
Aspera.assert(Environment.instance.graphical?, type: Error){'Using connect requires a graphical environment'}
|
|
69
|
+
Aspera.assert(Environment.instance.graphical?, type: Error) { 'Using connect requires a graphical environment' }
|
|
70
70
|
method_index = 0
|
|
71
71
|
begin
|
|
72
72
|
# raise exception if connect not started and file does not exist
|
|
73
73
|
connect_url = self.class.connect_api_url
|
|
74
|
-
Log.log.debug{"found: #{connect_url}"}
|
|
74
|
+
Log.log.debug { "found: #{connect_url}" }
|
|
75
75
|
@connect_api = Rest.new(
|
|
76
76
|
base_url: "#{connect_url}/v5/connect", # could use v6 also now
|
|
77
77
|
headers: {'Origin' => RestParameters.instance.user_agent}
|
|
@@ -80,11 +80,11 @@ module Aspera
|
|
|
80
80
|
Log.log.debug('Connect was reached') if method_index > 0
|
|
81
81
|
Log.dump(:connect_version, connect_info)
|
|
82
82
|
rescue StandardError => e # Errno::ECONNREFUSED
|
|
83
|
-
Log.log.debug{"Exception: #{e}"}
|
|
83
|
+
Log.log.debug { "Exception: #{e}" }
|
|
84
84
|
start_url = CONNECT_START_URIS[method_index]
|
|
85
85
|
method_index += 1
|
|
86
|
-
Aspera.assert(!start_url.nil?){"Unable to start connect #{method_index} times"}
|
|
87
|
-
Log.log.warn{"Aspera Connect is not started (#{e}). Trying to start it ##{method_index}..."}
|
|
86
|
+
Aspera.assert(!start_url.nil?) { "Unable to start connect #{method_index} times" }
|
|
87
|
+
Log.log.warn { "Aspera Connect is not started (#{e}). Trying to start it ##{method_index}..." }
|
|
88
88
|
Environment.instance.open_uri_graphical(start_url)
|
|
89
89
|
sleep(SLEEP_SEC_BETWEEN_RETRY)
|
|
90
90
|
retry
|
|
@@ -99,7 +99,7 @@ module Aspera
|
|
|
99
99
|
# :reek:UnusedParameters token_regenerator
|
|
100
100
|
def start_transfer(transfer_spec, token_regenerator: nil)
|
|
101
101
|
if transfer_spec['direction'] == 'send'
|
|
102
|
-
Log.log.warn{"Connect requires upload selection using GUI, ignoring #{transfer_spec['paths']}".red}
|
|
102
|
+
Log.log.warn { "Connect requires upload selection using GUI, ignoring #{transfer_spec['paths']}".red }
|
|
103
103
|
transfer_spec.delete('paths')
|
|
104
104
|
selection = @connect_api.create('windows/select-open-file-dialog/', {
|
|
105
105
|
'aspera_connect_settings' => @connect_settings,
|
|
@@ -108,7 +108,7 @@ module Aspera
|
|
|
108
108
|
'allowMultipleSelection' => true,
|
|
109
109
|
'allowedFileTypes' => ''
|
|
110
110
|
})
|
|
111
|
-
transfer_spec['paths'] = selection['dataTransfer']['files'].map{ |i| {'source' => i['name']}}
|
|
111
|
+
transfer_spec['paths'] = selection['dataTransfer']['files'].map { |i| {'source' => i['name']} }
|
|
112
112
|
end
|
|
113
113
|
# if there is a token, we ask connect client to use well known ssh private keys
|
|
114
114
|
# instead of asking password
|
data/lib/aspera/agent/desktop.rb
CHANGED
|
@@ -65,7 +65,7 @@ module Aspera
|
|
|
65
65
|
@application_id = SecureRandom.uuid
|
|
66
66
|
@transfer_id = nil
|
|
67
67
|
super
|
|
68
|
-
Aspera.assert(Environment.instance.graphical?, type: Error){'Using client requires a graphical environment'}
|
|
68
|
+
Aspera.assert(Environment.instance.graphical?, type: Error) { 'Using client requires a graphical environment' }
|
|
69
69
|
method_index = 0
|
|
70
70
|
begin
|
|
71
71
|
# curl 'http://127.0.0.1:33024/' -X POST -H 'content-type: application/json' --data-raw '{"jsonrpc":"2.0","params":[],"id":999999,"method":"rpc.discover"}'
|
|
@@ -77,8 +77,8 @@ module Aspera
|
|
|
77
77
|
rescue Errno::ECONNREFUSED => e
|
|
78
78
|
start_url = START_URIS[method_index]
|
|
79
79
|
method_index += 1
|
|
80
|
-
Aspera.assert(!start_url.nil?){"Unable to start #{Products::Desktop::APP_NAME} #{method_index} times"}
|
|
81
|
-
Log.log.warn{"#{Products::Desktop::APP_NAME} is not started (#{e}). Trying to start it ##{method_index}..."}
|
|
80
|
+
Aspera.assert(!start_url.nil?) { "Unable to start #{Products::Desktop::APP_NAME} #{method_index} times" }
|
|
81
|
+
Log.log.warn { "#{Products::Desktop::APP_NAME} is not started (#{e}). Trying to start it ##{method_index}..." }
|
|
82
82
|
Environment.instance.open_uri_graphical(start_url)
|
|
83
83
|
sleep(SLEEP_SEC_BETWEEN_RETRY)
|
|
84
84
|
retry
|
data/lib/aspera/agent/direct.rb
CHANGED
|
@@ -19,6 +19,30 @@ module Aspera
|
|
|
19
19
|
module Agent
|
|
20
20
|
# Execute a local `ascp` and use its management port to monitor progress
|
|
21
21
|
class Direct < Base
|
|
22
|
+
class << self
|
|
23
|
+
# Re-query the status of a direct transfer that was started in-process.
|
|
24
|
+
# Only works if the same Ruby process is still alive (e.g. MCP mode).
|
|
25
|
+
# agent_params must contain '_agent_ref' pointing to the live Direct instance.
|
|
26
|
+
# @param transfer_id [String] job_id returned by start_transfer / last_job_id
|
|
27
|
+
# @param agent_params [Hash] connection parameters; '_agent_ref' is the live agent
|
|
28
|
+
# @return [Hash] status hash with at least 'status' key
|
|
29
|
+
def transfer_status(transfer_id, agent_params)
|
|
30
|
+
agent = agent_params['_agent_ref']
|
|
31
|
+
return {'status' => 'unknown', 'note' => 'direct agent not available (different process?)'} if agent.nil?
|
|
32
|
+
sessions = agent.sessions_by_job(transfer_id)
|
|
33
|
+
return {'status' => 'unknown', 'note' => "no sessions found for job #{transfer_id}"} if sessions.empty?
|
|
34
|
+
errors = sessions.filter_map { |s| s[:error] }
|
|
35
|
+
running = sessions.any? { |s| s[:thread]&.alive? }
|
|
36
|
+
if errors.any?
|
|
37
|
+
{'status' => 'failed', 'error' => errors.first.message}
|
|
38
|
+
elsif running
|
|
39
|
+
{'status' => 'running'}
|
|
40
|
+
else
|
|
41
|
+
{'status' => 'completed'}
|
|
42
|
+
end
|
|
43
|
+
end
|
|
44
|
+
end
|
|
45
|
+
|
|
22
46
|
# `ascp` started locally, so listen local
|
|
23
47
|
LISTEN_LOCAL_ADDRESS = '127.0.0.1'
|
|
24
48
|
# 0 means: use any available port
|
|
@@ -75,7 +99,7 @@ module Aspera
|
|
|
75
99
|
@multi_incr_udp = multi_incr_udp.nil? ? Environment.instance.os.eql?(Environment::OS_WINDOWS) : multi_incr_udp
|
|
76
100
|
@monitor = monitor
|
|
77
101
|
@management_cb = management_cb
|
|
78
|
-
Aspera.assert_type(resume, Hash){'resume'}
|
|
102
|
+
Aspera.assert_type(resume, Hash) { 'resume' }
|
|
79
103
|
@resume_policy = Transfer::Resumer.new(**resume.symbolize_keys)
|
|
80
104
|
# all transfer jobs, key = SecureRandom.uuid, protected by mutex, cond var on change
|
|
81
105
|
@sessions = []
|
|
@@ -101,7 +125,7 @@ module Aspera
|
|
|
101
125
|
# using a non unique id results in discard of tags in AoC, and a package is never finalized
|
|
102
126
|
# all sessions in a multi-session transfer must have the same xfer_id (see admin manual)
|
|
103
127
|
transfer_spec['tags'][Transfer::Spec::TAG_RESERVED]['xfer_id'] ||= SecureRandom.uuid
|
|
104
|
-
Log.log.debug{"xfer id=#{transfer_spec['xfer_id']}"}
|
|
128
|
+
Log.log.debug { "xfer id=#{transfer_spec['xfer_id']}" }
|
|
105
129
|
# TODO: useful ? node only ? seems to be a timeout for retry in node
|
|
106
130
|
transfer_spec['tags'][Transfer::Spec::TAG_RESERVED]['xfer_retry'] ||= 3600
|
|
107
131
|
end
|
|
@@ -115,7 +139,7 @@ module Aspera
|
|
|
115
139
|
count: transfer_spec.delete('multi_session').to_i
|
|
116
140
|
}
|
|
117
141
|
if multi_session_info[:count].negative?
|
|
118
|
-
Log.log.error{"multi_session(#{transfer_spec['multi_session']}) shall be integer >= 0"}
|
|
142
|
+
Log.log.error { "multi_session(#{transfer_spec['multi_session']}) shall be integer >= 0" }
|
|
119
143
|
multi_session_info = nil
|
|
120
144
|
elsif multi_session_info[:count].eql?(0)
|
|
121
145
|
Log.log.debug('multi_session count is zero: no multi session')
|
|
@@ -145,7 +169,7 @@ module Aspera
|
|
|
145
169
|
if multi_session_info.nil?
|
|
146
170
|
Log.log.debug('Starting single session thread')
|
|
147
171
|
# single session for transfer : simple
|
|
148
|
-
session[:thread] = Thread.new{transfer_thread_entry(session)}
|
|
172
|
+
session[:thread] = Thread.new { transfer_thread_entry(session) }
|
|
149
173
|
@sessions.push(session)
|
|
150
174
|
else
|
|
151
175
|
Log.log.debug('Starting multi session threads')
|
|
@@ -161,13 +185,18 @@ module Aspera
|
|
|
161
185
|
# option: increment (default as per ascp manual) or not (cluster on other side ?)
|
|
162
186
|
exec_spec.args.unshift('-O', (multi_session_info[:udp_base] + i - 1).to_s) if @multi_incr_udp
|
|
163
187
|
# finally start the thread
|
|
164
|
-
this_session[:thread] = Thread.new{transfer_thread_entry(this_session)}
|
|
188
|
+
this_session[:thread] = Thread.new { transfer_thread_entry(this_session) }
|
|
165
189
|
@sessions.push(this_session)
|
|
166
190
|
end
|
|
167
191
|
end
|
|
168
192
|
return session[:job_id]
|
|
169
193
|
end
|
|
170
194
|
|
|
195
|
+
# @return [String, nil] job_id of the last submitted transfer
|
|
196
|
+
def last_job_id
|
|
197
|
+
@sessions.last&.dig(:job_id)
|
|
198
|
+
end
|
|
199
|
+
|
|
171
200
|
# wait for completion of all jobs started
|
|
172
201
|
# @return [Array] list of :success or error message
|
|
173
202
|
def wait_for_transfers_completion
|
|
@@ -175,7 +204,7 @@ module Aspera
|
|
|
175
204
|
# set to non-nil to exit loop
|
|
176
205
|
result = []
|
|
177
206
|
@sessions.each do |session|
|
|
178
|
-
Log.log.debug{"join #{session[:thread]}"}
|
|
207
|
+
Log.log.debug { "join #{session[:thread]}" }
|
|
179
208
|
session[:thread].join
|
|
180
209
|
result.push(session[:error] || :success)
|
|
181
210
|
end
|
|
@@ -194,7 +223,7 @@ module Aspera
|
|
|
194
223
|
# @param id [String] Transfer session identifier
|
|
195
224
|
# @return [Array] list of sessions for a job
|
|
196
225
|
def sessions_by_job(id)
|
|
197
|
-
@sessions.select{ |session| session[:job_id].eql?(id)}
|
|
226
|
+
@sessions.select { |session| session[:job_id].eql?(id) }
|
|
198
227
|
end
|
|
199
228
|
|
|
200
229
|
# Send command to management port of command (used in `asession).
|
|
@@ -224,7 +253,7 @@ module Aspera
|
|
|
224
253
|
begin
|
|
225
254
|
# set name for logging
|
|
226
255
|
Thread.current[:name] = 'transfer'
|
|
227
|
-
Log.log.debug{"ENTER (#{Thread.current[:name]})"}
|
|
256
|
+
Log.log.debug { "ENTER (#{Thread.current[:name]})" }
|
|
228
257
|
# start transfer with selected resumer policy
|
|
229
258
|
@resume_policy.execute_with_resume do
|
|
230
259
|
start_and_monitor_process(session: session, exec_spec: session[:exec_spec])
|
|
@@ -234,7 +263,7 @@ module Aspera
|
|
|
234
263
|
session[:error] = e
|
|
235
264
|
raise if Log.log.debug? || !e.is_a?(Transfer::Error)
|
|
236
265
|
end
|
|
237
|
-
Log.log.debug{"EXIT (#{Thread.current[:name]})"}
|
|
266
|
+
Log.log.debug { "EXIT (#{Thread.current[:name]})" }
|
|
238
267
|
end
|
|
239
268
|
|
|
240
269
|
public
|
|
@@ -293,7 +322,7 @@ module Aspera
|
|
|
293
322
|
return unless @monitor
|
|
294
323
|
# TODO: timeout does not work when Process.spawn is used... until process exits, then it works
|
|
295
324
|
# So we use select to detect that anything happens on the socket (connection)
|
|
296
|
-
Log.log.debug{"before select, timeout: #{@spawn_timeout_sec}"}
|
|
325
|
+
Log.log.debug { "before select, timeout: #{@spawn_timeout_sec}" }
|
|
297
326
|
readable, _, _ = IO.select([mgt_server_socket], nil, nil, @spawn_timeout_sec)
|
|
298
327
|
Log.log.debug('after select, before accept')
|
|
299
328
|
Aspera.assert(readable, 'timeout waiting mgt port connect (select not readable)', type: Transfer::Error)
|
|
@@ -322,17 +351,17 @@ module Aspera
|
|
|
322
351
|
commands = JSON.parse(File.read(@command_file))
|
|
323
352
|
send_command(commands)
|
|
324
353
|
rescue => e
|
|
325
|
-
Log.log.error{e.to_s}
|
|
354
|
+
Log.log.error { e.to_s }
|
|
326
355
|
end
|
|
327
356
|
File.delete(@command_file)
|
|
328
357
|
end
|
|
329
358
|
Log.log.debug('management io closed')
|
|
330
359
|
# check that last status was received before process exit
|
|
331
360
|
last_event = processor.last_event
|
|
332
|
-
Aspera.assert_type(last_event, Hash, type: Transfer::Error){'management event'}
|
|
361
|
+
Aspera.assert_type(last_event, Hash, type: Transfer::Error) { 'management event' }
|
|
333
362
|
case last_event['Type']
|
|
334
363
|
when 'DONE'
|
|
335
|
-
Log.log.trace1{'Graceful shutdown, DONE message received'}
|
|
364
|
+
Log.log.trace1 { 'Graceful shutdown, DONE message received' }
|
|
336
365
|
when 'ERROR'
|
|
337
366
|
if /bearer token/i.match?(last_event['Description']) &&
|
|
338
367
|
session[:token_regenerator].respond_to?(:refreshed_transfer_token)
|
|
@@ -342,7 +371,7 @@ module Aspera
|
|
|
342
371
|
env['ASPERA_SCP_TOKEN'] = session[:token_regenerator].refreshed_transfer_token
|
|
343
372
|
end
|
|
344
373
|
raise Transfer::Error.new(last_event['Description'], code: last_event['Code'].to_i)
|
|
345
|
-
else Aspera.error_unexpected_value(last_event['Type'], :error){'last event type'}
|
|
374
|
+
else Aspera.error_unexpected_value(last_event['Type'], :error) { 'last event type' }
|
|
346
375
|
end
|
|
347
376
|
rescue SystemCallError => e
|
|
348
377
|
# Process.spawn failed, or socket error
|
|
@@ -361,11 +390,11 @@ module Aspera
|
|
|
361
390
|
# process stderr of ascp
|
|
362
391
|
stderr_flag = false
|
|
363
392
|
stderr_r.each_line do |line|
|
|
364
|
-
Log.log.error{"BEGIN stderr #{exec}"} unless stderr_flag
|
|
365
|
-
Log.log.error{line.chomp}
|
|
393
|
+
Log.log.error { "BEGIN stderr #{exec}" } unless stderr_flag
|
|
394
|
+
Log.log.error { line.chomp }
|
|
366
395
|
stderr_flag = true
|
|
367
396
|
end
|
|
368
|
-
Log.log.error{"END stderr #{exec}"} if stderr_flag
|
|
397
|
+
Log.log.error { "END stderr #{exec}" } if stderr_flag
|
|
369
398
|
stderr_r.close
|
|
370
399
|
end
|
|
371
400
|
# status is nil if an exception occurred before starting command
|
|
@@ -412,12 +441,12 @@ module Aspera
|
|
|
412
441
|
when 'SESSION'
|
|
413
442
|
when 'ARGSTOP'
|
|
414
443
|
when 'FILEERROR'
|
|
415
|
-
Log.log.error{"#{event['Type']} #{event['Description']}"}
|
|
444
|
+
Log.log.error { "#{event['Type']} #{event['Description']}" }
|
|
416
445
|
when 'STOP'
|
|
417
446
|
# cspell:enable
|
|
418
447
|
# stop event when one file is completed
|
|
419
448
|
else
|
|
420
|
-
Log.log.debug{"Unknown event type for progress: #{event['Type']}"}
|
|
449
|
+
Log.log.debug { "Unknown event type for progress: #{event['Type']}" }
|
|
421
450
|
end
|
|
422
451
|
end
|
|
423
452
|
end
|
data/lib/aspera/agent/factory.rb
CHANGED
|
@@ -24,9 +24,9 @@ module Aspera
|
|
|
24
24
|
# Available agents: :long : Capitalized name string, :short : single character symbol
|
|
25
25
|
ALL =
|
|
26
26
|
Dir.children(File.dirname(File.expand_path(__FILE__)))
|
|
27
|
-
.select{ |file| file.end_with?(Environment::RB_EXT)}
|
|
28
|
-
.map{ |file| File.basename(file, Environment::RB_EXT).to_sym}
|
|
29
|
-
.reject{ |item| IGNORED_ITEMS.include?(item)}.to_h do |agent_sym|
|
|
27
|
+
.select { |file| file.end_with?(Environment::RB_EXT) }
|
|
28
|
+
.map { |file| File.basename(file, Environment::RB_EXT).to_sym }
|
|
29
|
+
.reject { |item| IGNORED_ITEMS.include?(item) }.to_h do |agent_sym|
|
|
30
30
|
[agent_sym, {
|
|
31
31
|
long: agent_sym.to_s.capitalize,
|
|
32
32
|
short: agent_sym.eql?(:direct) ? :a : agent_sym.to_s[0].to_sym
|
data/lib/aspera/agent/httpgw.rb
CHANGED
|
@@ -5,10 +5,32 @@ require 'aspera/transfer/spec'
|
|
|
5
5
|
require 'aspera/api/httpgw'
|
|
6
6
|
require 'aspera/log'
|
|
7
7
|
require 'aspera/assert'
|
|
8
|
+
require 'securerandom'
|
|
8
9
|
|
|
9
10
|
module Aspera
|
|
10
11
|
module Agent
|
|
11
12
|
class Httpgw < Base
|
|
13
|
+
class << self
|
|
14
|
+
# Re-query the status of an httpgw transfer that was started in-process.
|
|
15
|
+
# Only works if the same Ruby process is still alive (e.g. MCP mode).
|
|
16
|
+
# agent_params must contain '_agent_ref' pointing to the live Httpgw instance.
|
|
17
|
+
# @param transfer_id [String] job_id returned by last_job_id
|
|
18
|
+
# @param agent_params [Hash] '_agent_ref' is the live agent instance
|
|
19
|
+
# @return [Hash] status hash with at least 'status' key
|
|
20
|
+
def transfer_status(transfer_id, agent_params)
|
|
21
|
+
agent = agent_params['_agent_ref']
|
|
22
|
+
return {'status' => 'unknown', 'note' => 'httpgw agent not available (different process?)'} if agent.nil?
|
|
23
|
+
return {'status' => 'unknown', 'note' => 'no transfer started'} if agent.instance_variable_get(:@transfer_thread).nil?
|
|
24
|
+
if agent.instance_variable_get(:@transfer_thread).alive?
|
|
25
|
+
{'status' => 'running'}
|
|
26
|
+
elsif (err = agent.instance_variable_get(:@transfer_error))
|
|
27
|
+
{'status' => 'failed', 'error' => err.message}
|
|
28
|
+
else
|
|
29
|
+
{'status' => 'completed'}
|
|
30
|
+
end
|
|
31
|
+
end
|
|
32
|
+
end
|
|
33
|
+
|
|
12
34
|
def initialize(
|
|
13
35
|
url:,
|
|
14
36
|
api_version: Api::Httpgw::API_V2,
|
|
@@ -23,34 +45,53 @@ module Aspera
|
|
|
23
45
|
api_version: api_version,
|
|
24
46
|
upload_chunk_size: upload_chunk_size,
|
|
25
47
|
synchronous: synchronous,
|
|
26
|
-
notify_cb: ->(*pa, **ka){notify_progress(*pa, **ka)}
|
|
48
|
+
notify_cb: ->(*pa, **ka) { notify_progress(*pa, **ka) }
|
|
27
49
|
)
|
|
50
|
+
@last_job_id = nil
|
|
51
|
+
@transfer_thread = nil
|
|
52
|
+
@transfer_error = nil
|
|
28
53
|
end
|
|
29
54
|
|
|
30
|
-
# Start FASP transfer based on transfer spec (hash table)
|
|
31
|
-
#
|
|
32
|
-
# HTTP download only supports file list
|
|
55
|
+
# Start FASP transfer based on transfer spec (hash table).
|
|
56
|
+
# Non-blocking: the actual upload/download runs in a dedicated thread.
|
|
57
|
+
# HTTP download only supports file list.
|
|
33
58
|
# :reek:UnusedParameters token_regenerator
|
|
34
59
|
def start_transfer(transfer_spec, token_regenerator: nil)
|
|
35
60
|
Aspera.assert(!@gw_api.nil?, 'GW URL must be set')
|
|
36
|
-
Aspera.assert_type(transfer_spec['paths'], Array){'paths'}
|
|
37
|
-
Aspera.assert_type(transfer_spec['token'], String){'only token based transfer is supported in GW'}
|
|
61
|
+
Aspera.assert_type(transfer_spec['paths'], Array) { 'paths' }
|
|
62
|
+
Aspera.assert_type(transfer_spec['token'], String) { 'only token based transfer is supported in GW' }
|
|
38
63
|
Log.dump(:user_spec, transfer_spec)
|
|
39
64
|
transfer_spec['authentication'] ||= 'token'
|
|
65
|
+
@last_job_id = SecureRandom.uuid
|
|
66
|
+
@transfer_error = nil
|
|
67
|
+
@transfer_thread = Thread.new { run_transfer(transfer_spec) }
|
|
68
|
+
@last_job_id
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
# Wait for the transfer thread to complete.
|
|
72
|
+
# @return [Array] list of :success or error
|
|
73
|
+
def wait_for_transfers_completion
|
|
74
|
+
@transfer_thread&.join
|
|
75
|
+
return [@transfer_error || :success]
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
# @return [String, nil] job_id of the last submitted transfer
|
|
79
|
+
attr_reader :last_job_id
|
|
80
|
+
|
|
81
|
+
private
|
|
82
|
+
|
|
83
|
+
# Execute the actual transfer (runs inside the transfer thread).
|
|
84
|
+
def run_transfer(transfer_spec)
|
|
40
85
|
case transfer_spec['direction']
|
|
41
86
|
when Transfer::Spec::DIRECTION_SEND
|
|
42
87
|
@gw_api.upload(transfer_spec)
|
|
43
88
|
when Transfer::Spec::DIRECTION_RECEIVE
|
|
44
89
|
@gw_api.download(transfer_spec)
|
|
45
|
-
else Aspera.error_unexpected_value(transfer_spec['direction']){'direction'}
|
|
90
|
+
else Aspera.error_unexpected_value(transfer_spec['direction']) { 'direction' }
|
|
46
91
|
end
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
# @return [Array] list of :success or error message
|
|
51
|
-
def wait_for_transfers_completion
|
|
52
|
-
# well ... transfer was done in "start"
|
|
53
|
-
return [:success]
|
|
92
|
+
rescue => e
|
|
93
|
+
@transfer_error = e
|
|
94
|
+
Log.log.error { "httpgw transfer thread error: #{e}" }
|
|
54
95
|
end
|
|
55
96
|
end
|
|
56
97
|
end
|
data/lib/aspera/agent/node.rb
CHANGED
|
@@ -95,7 +95,7 @@ module Aspera
|
|
|
95
95
|
transfer_spec['remote_host'] = '127.0.0.1' if !transfer_spec['wss_enabled'] && transfer_spec['remote_host'].eql?(URI.parse(node_api_.base_url).host)
|
|
96
96
|
resp = node_api_.create('ops/transfers', transfer_spec)
|
|
97
97
|
@transfer_id = resp['id']
|
|
98
|
-
Log.log.debug{"tr_id=#{@transfer_id}"}
|
|
98
|
+
Log.log.debug { "tr_id=#{@transfer_id}" }
|
|
99
99
|
return @transfer_id
|
|
100
100
|
end
|
|
101
101
|
|
|
@@ -137,7 +137,7 @@ module Aspera
|
|
|
137
137
|
# Bug in HSTS ? transfer is marked failed, but there is no reason
|
|
138
138
|
break if transfer_data['error_code'].eql?(0) && transfer_data['error_desc'].empty?
|
|
139
139
|
raise Transfer::Error, "status: #{transfer_data['status']}. code: #{transfer_data['error_code']}. description: #{transfer_data['error_desc']}"
|
|
140
|
-
else Aspera.error_unexpected_value(transfer_data['status']){"transfer_data -> #{transfer_data}"}
|
|
140
|
+
else Aspera.error_unexpected_value(transfer_data['status']) { "transfer_data -> #{transfer_data}" }
|
|
141
141
|
end
|
|
142
142
|
sleep(1.0)
|
|
143
143
|
end
|
|
@@ -66,7 +66,7 @@ module Aspera
|
|
|
66
66
|
@transfer_id = nil
|
|
67
67
|
@stop = stop
|
|
68
68
|
is_local_auto_port = url.eql?(AUTO_LOCAL_TCP_PORT)
|
|
69
|
-
Aspera.assert(!(is_local_auto_port && (!@stop || !start)), type: Error){'Cannot set options `stop` or `start` to false with port zero'}
|
|
69
|
+
Aspera.assert(!(is_local_auto_port && (!@stop || !start)), type: Error) { 'Cannot set options `stop` or `start` to false with port zero' }
|
|
70
70
|
# keep PID for optional shutdown
|
|
71
71
|
@daemon_pid = nil
|
|
72
72
|
@daemon_endpoint = nil
|
|
@@ -81,18 +81,18 @@ module Aspera
|
|
|
81
81
|
# Initiate actual connection
|
|
82
82
|
get_info_response = @transfer_client.get_info(::Transferd::Api::InstanceInfoRequest.new)
|
|
83
83
|
@daemon_endpoint = daemon_endpoint
|
|
84
|
-
Log.log.debug{"Daemon info: #{get_info_response}"}
|
|
84
|
+
Log.log.debug { "Daemon info: #{get_info_response}" }
|
|
85
85
|
Log.log.warn('Attached to existing daemon') unless @daemon_pid || !start || !@stop
|
|
86
|
-
at_exit{shutdown}
|
|
86
|
+
at_exit { shutdown }
|
|
87
87
|
rescue GRPC::Unavailable => e
|
|
88
88
|
# if transferd is external: do not start it, or other error
|
|
89
89
|
raise if !start || !e.message.include?('failed to connect')
|
|
90
90
|
# we already tried to start a daemon, but it failed
|
|
91
|
-
Aspera.assert(@daemon_pid.nil?){"Daemon started with PID #{@daemon_pid}, but connection failed to #{daemon_endpoint}"}
|
|
91
|
+
Aspera.assert(@daemon_pid.nil?) { "Daemon started with PID #{@daemon_pid}, but connection failed to #{daemon_endpoint}" }
|
|
92
92
|
Log.log.warn('no daemon present, starting daemon...') if !start
|
|
93
93
|
# transferd only supports local ip and port
|
|
94
94
|
daemon_uri = URI.parse("ipv4://#{daemon_endpoint}")
|
|
95
|
-
Aspera.assert(daemon_uri.scheme.eql?('ipv4')){"Invalid scheme daemon URI #{daemon_endpoint}"}
|
|
95
|
+
Aspera.assert(daemon_uri.scheme.eql?('ipv4')) { "Invalid scheme daemon URI #{daemon_endpoint}" }
|
|
96
96
|
# create a config file for daemon
|
|
97
97
|
config = {
|
|
98
98
|
address: daemon_uri.host,
|
|
@@ -107,7 +107,7 @@ module Aspera
|
|
|
107
107
|
}
|
|
108
108
|
# config file and logs are created in same folder
|
|
109
109
|
transferd_base_tmp = TempFileManager.instance.new_file_path_global('transferd')
|
|
110
|
-
Log.log.debug{"transferd base tmp #{transferd_base_tmp}"}
|
|
110
|
+
Log.log.debug { "transferd base tmp #{transferd_base_tmp}" }
|
|
111
111
|
conf_file = "#{transferd_base_tmp}.conf"
|
|
112
112
|
log_stdout = "#{transferd_base_tmp}.out"
|
|
113
113
|
log_stderr = "#{transferd_base_tmp}.err"
|
|
@@ -128,9 +128,9 @@ module Aspera
|
|
|
128
128
|
rescue Timeout::Error
|
|
129
129
|
nil
|
|
130
130
|
end
|
|
131
|
-
Log.log.debug{"Daemon started with pid #{@daemon_pid}"}
|
|
131
|
+
Log.log.debug { "Daemon started with pid #{@daemon_pid}" }
|
|
132
132
|
Process.detach(@daemon_pid) unless @stop
|
|
133
|
-
at_exit{shutdown}
|
|
133
|
+
at_exit { shutdown }
|
|
134
134
|
# update port for next connection attempt (if auto high port was requested)
|
|
135
135
|
daemon_endpoint = "#{LOCAL_SOCKET_ADDR}#{PORT_SEP}#{Products::Transferd.daemon_port_from_log(log_stdout)}" if is_local_auto_port
|
|
136
136
|
# local daemon started, try again
|
|
@@ -152,10 +152,10 @@ module Aspera
|
|
|
152
152
|
) # transfer definition
|
|
153
153
|
# send start transfer request to the transfer manager daemon
|
|
154
154
|
start_response = @transfer_client.start_transfer(transfer_request)
|
|
155
|
-
Aspera.assert(!start_response.status.eql?(:FAILED), type: Transfer::Error){start_response.error.description}
|
|
156
|
-
Log.log.debug{"start transfer response #{start_response}"}
|
|
155
|
+
Aspera.assert(!start_response.status.eql?(:FAILED), type: Transfer::Error) { start_response.error.description }
|
|
156
|
+
Log.log.debug { "start transfer response #{start_response}" }
|
|
157
157
|
@transfer_id = start_response.transferId
|
|
158
|
-
Log.log.debug{"transfer started with id #{@transfer_id}"}
|
|
158
|
+
Log.log.debug { "transfer started with id #{@transfer_id}" }
|
|
159
159
|
end
|
|
160
160
|
|
|
161
161
|
def wait_for_transfers_completion
|
|
@@ -190,7 +190,7 @@ module Aspera
|
|
|
190
190
|
when :QUEUED, :UNKNOWN_STATUS, :PAUSED, :ORPHANED
|
|
191
191
|
notify_progress(:sessions_init, info: response.status.to_s.downcase)
|
|
192
192
|
else
|
|
193
|
-
Log.log.error{"unknown status#{response.status}"}
|
|
193
|
+
Log.log.error { "unknown status#{response.status}" }
|
|
194
194
|
end
|
|
195
195
|
end
|
|
196
196
|
# TODO: return status
|