aspera-cli 4.21.1 → 4.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/BUGS.md +1 -1
- data/CHANGELOG.md +52 -22
- data/CONTRIBUTING.md +69 -148
- data/README.md +929 -668
- data/bin/ascli +5 -14
- data/bin/asession +1 -3
- data/examples/get_proto_file.rb +4 -3
- data/examples/proxy.pac +20 -20
- data/lib/aspera/agent/base.rb +11 -5
- data/lib/aspera/agent/connect.rb +30 -28
- data/lib/aspera/agent/{alpha.rb → desktop.rb} +35 -31
- data/lib/aspera/agent/direct.rb +141 -121
- data/lib/aspera/agent/httpgw.rb +22 -26
- data/lib/aspera/agent/node.rb +14 -11
- data/lib/aspera/agent/transferd.rb +30 -19
- data/lib/aspera/api/alee.rb +1 -1
- data/lib/aspera/api/aoc.rb +6 -6
- data/lib/aspera/api/cos_node.rb +2 -2
- data/lib/aspera/api/httpgw.rb +7 -3
- data/lib/aspera/api/node.rb +10 -8
- data/lib/aspera/ascmd.rb +3 -3
- data/lib/aspera/ascp/installation.rb +53 -72
- data/lib/aspera/ascp/management.rb +1 -1
- data/lib/aspera/assert.rb +11 -2
- data/lib/aspera/cli/error.rb +2 -2
- data/lib/aspera/cli/extended_value.rb +46 -21
- data/lib/aspera/cli/formatter.rb +55 -48
- data/lib/aspera/cli/hints.rb +1 -1
- data/lib/aspera/cli/info.rb +1 -0
- data/lib/aspera/cli/main.rb +192 -170
- data/lib/aspera/cli/manager.rb +18 -18
- data/lib/aspera/cli/plugin.rb +23 -20
- data/lib/aspera/cli/plugin_factory.rb +1 -1
- data/lib/aspera/cli/plugins/alee.rb +1 -1
- data/lib/aspera/cli/plugins/aoc.rb +247 -159
- data/lib/aspera/cli/plugins/ats.rb +19 -17
- data/lib/aspera/cli/plugins/config.rb +76 -113
- data/lib/aspera/cli/plugins/console.rb +5 -3
- data/lib/aspera/cli/plugins/faspex.rb +39 -35
- data/lib/aspera/cli/plugins/faspex5.rb +111 -84
- data/lib/aspera/cli/plugins/faspio.rb +13 -1
- data/lib/aspera/cli/plugins/httpgw.rb +13 -1
- data/lib/aspera/cli/plugins/node.rb +312 -182
- data/lib/aspera/cli/plugins/orchestrator.rb +34 -40
- data/lib/aspera/cli/plugins/preview.rb +3 -3
- data/lib/aspera/cli/plugins/server.rb +6 -6
- data/lib/aspera/cli/plugins/shares.rb +5 -5
- data/lib/aspera/cli/sync_actions.rb +19 -18
- data/lib/aspera/cli/transfer_agent.rb +5 -5
- data/lib/aspera/cli/transfer_progress.rb +2 -2
- data/lib/aspera/cli/version.rb +1 -1
- data/lib/aspera/command_line_builder.rb +116 -95
- data/lib/aspera/coverage.rb +8 -5
- data/lib/aspera/environment.rb +26 -17
- data/lib/aspera/faspex_gw.rb +14 -14
- data/lib/aspera/faspex_postproc.rb +10 -11
- data/lib/aspera/hash_ext.rb +4 -14
- data/lib/aspera/json_rpc.rb +1 -1
- data/lib/aspera/keychain/encrypted_hash.rb +47 -34
- data/lib/aspera/keychain/factory.rb +41 -0
- data/lib/aspera/keychain/hashicorp_vault.rb +71 -0
- data/lib/aspera/keychain/macos_security.rb +19 -11
- data/lib/aspera/log.rb +28 -34
- data/lib/aspera/nagios.rb +6 -6
- data/lib/aspera/node_simulator.rb +8 -8
- data/lib/aspera/oauth/base.rb +14 -7
- data/lib/aspera/oauth/factory.rb +5 -6
- data/lib/aspera/oauth/url_json.rb +6 -6
- data/lib/aspera/persistency_action_once.rb +6 -4
- data/lib/aspera/persistency_folder.rb +2 -2
- data/lib/aspera/preview/generator.rb +13 -10
- data/lib/aspera/preview/options.rb +16 -16
- data/lib/aspera/preview/terminal.rb +4 -4
- data/lib/aspera/preview/utils.rb +15 -17
- data/lib/aspera/products/connect.rb +35 -1
- data/lib/aspera/products/{alpha.rb → desktop.rb} +3 -3
- data/lib/aspera/products/transferd.rb +9 -2
- data/lib/aspera/proxy_auto_config.rb +2 -2
- data/lib/aspera/rest.rb +56 -47
- data/lib/aspera/rest_errors_aspera.rb +1 -1
- data/lib/aspera/secret_hider.rb +12 -5
- data/lib/aspera/ssh.rb +4 -4
- data/lib/aspera/temp_file_manager.rb +5 -4
- data/lib/aspera/transfer/convert.rb +29 -0
- data/lib/aspera/transfer/error_info.rb +66 -66
- data/lib/aspera/transfer/parameters.rb +13 -68
- data/lib/aspera/transfer/spec.rb +5 -6
- data/lib/aspera/transfer/spec.schema.yaml +753 -0
- data/lib/aspera/transfer/spec_doc.rb +62 -0
- data/lib/aspera/transfer/sync.rb +23 -72
- data/lib/aspera/transfer/sync_instance.schema.yaml +13 -0
- data/lib/aspera/transfer/sync_session.schema.yaml +79 -0
- data/lib/aspera/transfer/uri.rb +6 -6
- data/lib/aspera/uri_reader.rb +18 -1
- data/lib/aspera/web_auth.rb +1 -1
- data/lib/aspera/web_server_simple.rb +53 -44
- data.tar.gz.sig +0 -0
- metadata +28 -165
- metadata.gz.sig +0 -0
- data/examples/build_exec +0 -74
- data/examples/build_exec_rubyc +0 -40
- data/examples/build_package.sh +0 -28
- data/lib/aspera/transfer/spec.yaml +0 -718
data/lib/aspera/agent/direct.rb
CHANGED
@@ -16,15 +16,68 @@ require 'English'
|
|
16
16
|
|
17
17
|
module Aspera
|
18
18
|
module Agent
|
19
|
-
# executes a local "ascp",
|
19
|
+
# executes a local "ascp", create mgt port
|
20
20
|
class Direct < Base
|
21
|
+
# ascp started locally, so listen local
|
21
22
|
LISTEN_LOCAL_ADDRESS = '127.0.0.1'
|
22
|
-
# 0 means:
|
23
|
+
# 0 means: use any available port
|
23
24
|
SELECT_AVAILABLE_PORT = 0
|
24
|
-
# spellchecker: enable
|
25
25
|
private_constant :LISTEN_LOCAL_ADDRESS, :SELECT_AVAILABLE_PORT
|
26
26
|
|
27
|
-
#
|
27
|
+
# options for initialize (same as values in option transfer_info)
|
28
|
+
# @param ascp_args [Array] additional arguments to ascp
|
29
|
+
# @param wss [Boolean] true: if both SSH and wss in ts: prefer wss
|
30
|
+
# @param quiet [Boolean] by default no native ascp progress bar
|
31
|
+
# @param monitor [Boolean] set to false to eliminate management port
|
32
|
+
# @param trusted_certs [Array,NilClass] list of files with trusted certificates (stores)
|
33
|
+
# @param client_ssh_key [String] client ssh key option (from CLIENT_SSH_KEY_OPTIONS)
|
34
|
+
# @param check_ignore_cb [Proc] callback with host,port
|
35
|
+
# @param spawn_timeout_sec [Integer] timeout for ascp spawn
|
36
|
+
# @param spawn_delay_sec [Integer] optional delay to start between sessions
|
37
|
+
# @param multi_incr_udp [Boolean,NilClass] true: increment udp port for each session
|
38
|
+
# @param resume [Hash,NilClass] resume policy
|
39
|
+
# @param management_cb [Proc] callback for management events
|
40
|
+
# @param base_options [Hash] other options for base class
|
41
|
+
def initialize(
|
42
|
+
ascp_args: nil,
|
43
|
+
wss: true,
|
44
|
+
quiet: true,
|
45
|
+
trusted_certs: nil,
|
46
|
+
client_ssh_key: nil,
|
47
|
+
check_ignore_cb: nil,
|
48
|
+
spawn_timeout_sec: 2,
|
49
|
+
spawn_delay_sec: 2,
|
50
|
+
multi_incr_udp: nil,
|
51
|
+
resume: nil,
|
52
|
+
monitor: true,
|
53
|
+
management_cb: nil,
|
54
|
+
**base_options
|
55
|
+
)
|
56
|
+
super(**base_options)
|
57
|
+
# special transfer parameters
|
58
|
+
@tr_opts = {
|
59
|
+
ascp_args: ascp_args,
|
60
|
+
wss: wss,
|
61
|
+
quiet: quiet,
|
62
|
+
trusted_certs: trusted_certs,
|
63
|
+
client_ssh_key: client_ssh_key,
|
64
|
+
check_ignore_cb: check_ignore_cb
|
65
|
+
}
|
66
|
+
@spawn_timeout_sec = spawn_timeout_sec
|
67
|
+
@spawn_delay_sec = spawn_delay_sec
|
68
|
+
# default is true on Windows, false on other OSes
|
69
|
+
@multi_incr_udp = multi_incr_udp.nil? ? Environment.os.eql?(Environment::OS_WINDOWS) : multi_incr_udp
|
70
|
+
@monitor = monitor
|
71
|
+
@management_cb = management_cb
|
72
|
+
@resume_policy = Resumer.new(resume.nil? ? {} : resume.symbolize_keys)
|
73
|
+
# all transfer jobs, key = SecureRandom.uuid, protected by mutex, cond var on change
|
74
|
+
@sessions = []
|
75
|
+
# mutex protects global data accessed by threads
|
76
|
+
@mutex = Mutex.new
|
77
|
+
@pre_calc_sent = false
|
78
|
+
@pre_calc_last_size = nil
|
79
|
+
end
|
80
|
+
|
28
81
|
# start ascp transfer(s) (non blocking), single or multi-session
|
29
82
|
# session information added to @sessions
|
30
83
|
# @param transfer_spec [Hash] aspera transfer specification
|
@@ -48,11 +101,10 @@ module Aspera
|
|
48
101
|
# (even if the var is not used in single session)
|
49
102
|
multi_session_info = nil
|
50
103
|
if transfer_spec.key?('multi_session')
|
104
|
+
# Managed by multi-session, so delete from transfer spec
|
51
105
|
multi_session_info = {
|
52
|
-
count: transfer_spec
|
106
|
+
count: transfer_spec.delete('multi_session').to_i
|
53
107
|
}
|
54
|
-
# Managed by multi-session, so delete from transfer spec
|
55
|
-
transfer_spec.delete('multi_session')
|
56
108
|
if multi_session_info[:count].negative?
|
57
109
|
Log.log.error{"multi_session(#{transfer_spec['multi_session']}) shall be integer >= 0"}
|
58
110
|
multi_session_info = nil
|
@@ -77,14 +129,14 @@ module Aspera
|
|
77
129
|
error: nil, # exception if failed
|
78
130
|
io: nil, # management port server socket
|
79
131
|
token_regenerator: token_regenerator, # regenerate bearer token with oauth
|
80
|
-
# env vars and args
|
132
|
+
# env vars and args for ascp (from transfer spec)
|
81
133
|
env_args: Transfer::Parameters.new(transfer_spec, **@tr_opts).ascp_args
|
82
134
|
}
|
83
135
|
|
84
136
|
if multi_session_info.nil?
|
85
137
|
Log.log.debug('Starting single session thread')
|
86
138
|
# single session for transfer : simple
|
87
|
-
session[:thread] = Thread.new
|
139
|
+
session[:thread] = Thread.new{transfer_thread_entry(session)}
|
88
140
|
@sessions.push(session)
|
89
141
|
else
|
90
142
|
Log.log.debug('Starting multi session threads')
|
@@ -101,7 +153,7 @@ module Aspera
|
|
101
153
|
# option: increment (default as per ascp manual) or not (cluster on other side ?)
|
102
154
|
args.unshift('-O', (multi_session_info[:udp_base] + i - 1).to_s) if @multi_incr_udp
|
103
155
|
# finally start the thread
|
104
|
-
this_session[:thread] = Thread.new
|
156
|
+
this_session[:thread] = Thread.new{transfer_thread_entry(this_session)}
|
105
157
|
@sessions.push(this_session)
|
106
158
|
end
|
107
159
|
end
|
@@ -132,9 +184,44 @@ module Aspera
|
|
132
184
|
|
133
185
|
# @return [Array] list of sessions for a job
|
134
186
|
def sessions_by_job(job_id)
|
135
|
-
@sessions.select{|session| session[:job_id].eql?(job_id)}
|
187
|
+
@sessions.select{ |session| session[:job_id].eql?(job_id)}
|
136
188
|
end
|
137
189
|
|
190
|
+
# send command to management port of command (used in `asession)
|
191
|
+
# @param job_id identified transfer process
|
192
|
+
# @param session_index index of session (for multi session)
|
193
|
+
# @param data command on mgt port, examples:
|
194
|
+
# {'type'=>'START','source'=>_path_,'destination'=>_path_}
|
195
|
+
# {'type'=>'DONE'}
|
196
|
+
def send_command(job_id, data)
|
197
|
+
session = @sessions.find{ |session| session[:job_id].eql?(job_id)}
|
198
|
+
Log.log.debug{"command: #{data}"}
|
199
|
+
session[:io].puts(Ascp::Management.command_to_stream(data))
|
200
|
+
end
|
201
|
+
|
202
|
+
private
|
203
|
+
|
204
|
+
# transfer thread entry
|
205
|
+
# @param session information
|
206
|
+
def transfer_thread_entry(session)
|
207
|
+
begin
|
208
|
+
# set name for logging
|
209
|
+
Thread.current[:name] = 'transfer'
|
210
|
+
Log.log.debug{"ENTER (#{Thread.current[:name]})"}
|
211
|
+
# start transfer with selected resumer policy
|
212
|
+
@resume_policy.execute_with_resume do
|
213
|
+
start_and_monitor_process(session: session, **session[:env_args])
|
214
|
+
end
|
215
|
+
Log.log.debug('transfer ok'.bg_green)
|
216
|
+
rescue StandardError => e
|
217
|
+
session[:error] = e
|
218
|
+
Log.log.error{"Transfer thread error: #{e.class}:\n#{e.message}:\n#{e.backtrace.join("\n")}".red} if Log.instance.level.eql?(:debug)
|
219
|
+
end
|
220
|
+
Log.log.debug{"EXIT (#{Thread.current[:name]})"}
|
221
|
+
end
|
222
|
+
|
223
|
+
public
|
224
|
+
|
138
225
|
# This is the low level method to start the transfer process.
|
139
226
|
# Typically started in a thread.
|
140
227
|
# Start process with management port.
|
@@ -154,24 +241,32 @@ module Aspera
|
|
154
241
|
notify_progress(:pre_start, session_id: nil, info: 'starting')
|
155
242
|
begin
|
156
243
|
command_pid = nil
|
157
|
-
|
158
|
-
|
159
|
-
|
160
|
-
|
161
|
-
|
162
|
-
|
163
|
-
|
164
|
-
|
165
|
-
|
166
|
-
|
167
|
-
|
168
|
-
|
244
|
+
command_arguments = []
|
245
|
+
if @monitor
|
246
|
+
# we use Socket directly, instead of TCPServer, as it gives access to lower level options
|
247
|
+
socket_class = defined?(JRUBY_VERSION) ? ServerSocket : Socket
|
248
|
+
mgt_server_socket = socket_class.new(Socket::AF_INET, Socket::SOCK_STREAM, 0)
|
249
|
+
# open any available (0) local TCP port for use as management port
|
250
|
+
mgt_server_socket.bind(Addrinfo.tcp(LISTEN_LOCAL_ADDRESS, SELECT_AVAILABLE_PORT))
|
251
|
+
# make port ready to accept connections, before starting ascp
|
252
|
+
mgt_server_socket.listen(1)
|
253
|
+
# build arguments and add mgt port
|
254
|
+
command_arguments = if name.eql?(:async)
|
255
|
+
["--exclusive-mgmt-port=#{mgt_server_socket.local_address.ip_port}"]
|
256
|
+
else
|
257
|
+
['-M', mgt_server_socket.local_address.ip_port.to_s]
|
258
|
+
end
|
169
259
|
end
|
170
260
|
command_arguments.concat(args)
|
261
|
+
# capture process stderr
|
262
|
+
stderr_r, stderr_w = IO.pipe
|
171
263
|
# get location of command executable (ascp, async)
|
172
264
|
command_path = Ascp::Installation.instance.path(name)
|
173
|
-
command_pid = Environment.secure_spawn(env: env, exec: command_path, args: command_arguments)
|
265
|
+
command_pid = Environment.secure_spawn(env: env, exec: command_path, args: command_arguments, err: stderr_w)
|
266
|
+
stderr_w.close
|
174
267
|
notify_progress(:pre_start, session_id: nil, info: "waiting for #{name} to start")
|
268
|
+
# "ensure" block will wait for process
|
269
|
+
return unless @monitor
|
175
270
|
# TODO: timeout does not work when Process.spawn is used... until process exits, then it works
|
176
271
|
# So we use select to detect that anything happens on the socket (connection)
|
177
272
|
Log.log.debug{"before select, timeout: #{@spawn_timeout_sec}"}
|
@@ -182,9 +277,9 @@ module Aspera
|
|
182
277
|
client_socket, _client_addrinfo = mgt_server_socket.accept
|
183
278
|
Log.log.debug('after accept')
|
184
279
|
management_port_io = client_socket.to_io
|
185
|
-
# management messages include file names which may be utf8
|
186
280
|
# by default socket is US-ASCII
|
187
|
-
#
|
281
|
+
# management messages include file names which may be UTF-8
|
282
|
+
# TODO: use same value as Encoding.default_external ?
|
188
283
|
management_port_io.set_encoding(Encoding::UTF_8)
|
189
284
|
session[:io] = management_port_io
|
190
285
|
processor = Ascp::Management.new
|
@@ -194,7 +289,7 @@ module Aspera
|
|
194
289
|
next unless event
|
195
290
|
# event is ready
|
196
291
|
Log.log.trace1{Log.dump(:management_port, event)}
|
197
|
-
# store
|
292
|
+
# store session identifier
|
198
293
|
session[:id] = event['SessionId'] if event['Type'].eql?('INIT')
|
199
294
|
@management_cb&.call(event)
|
200
295
|
process_progress(event)
|
@@ -203,8 +298,10 @@ module Aspera
|
|
203
298
|
Log.log.debug('management io closed')
|
204
299
|
# check that last status was received before process exit
|
205
300
|
last_event = processor.last_event
|
206
|
-
raise Transfer::Error, "
|
301
|
+
raise Transfer::Error, "No management event (#{last_event.class})" unless last_event.is_a?(Hash)
|
207
302
|
case last_event['Type']
|
303
|
+
when 'DONE'
|
304
|
+
Log.log.trace1{'Graceful shutdown, DONE message received'}
|
208
305
|
when 'ERROR'
|
209
306
|
if /bearer token/i.match?(last_event['Description']) &&
|
210
307
|
session[:token_regenerator].respond_to?(:refreshed_transfer_token)
|
@@ -214,10 +311,9 @@ module Aspera
|
|
214
311
|
env['ASPERA_SCP_TOKEN'] = session[:token_regenerator].refreshed_transfer_token
|
215
312
|
end
|
216
313
|
raise Transfer::Error.new(last_event['Description'], last_event['Code'].to_i)
|
217
|
-
when 'DONE'
|
218
|
-
nil
|
219
314
|
else
|
220
|
-
|
315
|
+
Log.log.error{"unexpected last event type: #{last_event['Type']}"}
|
316
|
+
# raise Transfer::Error, "unexpected last event type: #{last_event['Type']}, #{last_event['Description']}"
|
221
317
|
end
|
222
318
|
rescue SystemCallError => e
|
223
319
|
# Process.spawn failed, or socket error
|
@@ -225,17 +321,21 @@ module Aspera
|
|
225
321
|
rescue Interrupt
|
226
322
|
raise Transfer::Error, 'transfer interrupted by user'
|
227
323
|
ensure
|
228
|
-
mgt_server_socket
|
324
|
+
mgt_server_socket&.close
|
325
|
+
session.delete(:io)
|
229
326
|
# if command was successfully started, check its status
|
230
327
|
unless command_pid.nil?
|
231
|
-
|
232
|
-
|
233
|
-
status =
|
234
|
-
#
|
235
|
-
|
328
|
+
Process.kill(:INT, command_pid) if @monitor
|
329
|
+
# collect process exit status or wait for termination
|
330
|
+
_, status = Process.wait2(command_pid)
|
331
|
+
# process stderr of ascp
|
332
|
+
stderr_r.each_line do |line|
|
333
|
+
Log.log.error(line.chomp)
|
334
|
+
end
|
335
|
+
stderr_r.close
|
236
336
|
# status is nil if an exception occurred before starting command
|
237
337
|
if !status&.success?
|
238
|
-
message =
|
338
|
+
message = "#{name} failed (#{status})"
|
239
339
|
# raise error only if there was not already an exception (ERROR_INFO)
|
240
340
|
raise Transfer::Error, message unless $ERROR_INFO
|
241
341
|
# else display this message also, as main exception is already here
|
@@ -246,10 +346,10 @@ module Aspera
|
|
246
346
|
nil
|
247
347
|
end
|
248
348
|
|
249
|
-
attr_reader :sessions
|
250
|
-
|
251
349
|
private
|
252
350
|
|
351
|
+
attr_reader :sessions
|
352
|
+
|
253
353
|
# notify progress to callback
|
254
354
|
# @param event management port event
|
255
355
|
# @param session sessin object
|
@@ -285,88 +385,8 @@ module Aspera
|
|
285
385
|
# cspell:enable
|
286
386
|
# stop event when one file is completed
|
287
387
|
else
|
288
|
-
Log.log.debug{"
|
289
|
-
end
|
290
|
-
end
|
291
|
-
|
292
|
-
# send command to management port of command (used in `asession)
|
293
|
-
# @param job_id identified transfer process
|
294
|
-
# @param session_index index of session (for multi session)
|
295
|
-
# @param data command on mgt port, examples:
|
296
|
-
# {'type'=>'START','source'=>_path_,'destination'=>_path_}
|
297
|
-
# {'type'=>'DONE'}
|
298
|
-
def send_command(job_id, data)
|
299
|
-
session = @sessions.find{|session| session[:job_id].eql?(job_id)}
|
300
|
-
Log.log.debug{"command: #{data}"}
|
301
|
-
session[:io].puts(Ascp::Management.command_to_stream(data))
|
302
|
-
end
|
303
|
-
|
304
|
-
# options for initialize (same as values in option transfer_info)
|
305
|
-
# @param ascp_args [Array] additional arguments to ascp
|
306
|
-
# @param wss [Boolean] true: if both SSH and wss in ts: prefer wss
|
307
|
-
# @param quiet [Boolean] by default no native ascp progress bar
|
308
|
-
# @param trusted_certs [Array,NilClass] list of files with trusted certificates (stores)
|
309
|
-
# @param client_ssh_key [String] client ssh key option (from CLIENT_SSH_KEY_OPTIONS)
|
310
|
-
# @param check_ignore_cb [Proc] callback with host,port
|
311
|
-
# @param spawn_timeout_sec [Integer] timeout for ascp spawn
|
312
|
-
# @param spawn_delay_sec [Integer] optional delay to start between sessions
|
313
|
-
# @param multi_incr_udp [Boolean,NilClass] true: increment udp port for each session
|
314
|
-
# @param resume [Hash,NilClass] resume policy
|
315
|
-
# @param management_cb [Proc] callback for management events
|
316
|
-
# @param base_options [Hash] other options for base class
|
317
|
-
def initialize(
|
318
|
-
ascp_args: nil,
|
319
|
-
wss: true,
|
320
|
-
quiet: true,
|
321
|
-
trusted_certs: nil,
|
322
|
-
client_ssh_key: nil,
|
323
|
-
check_ignore_cb: nil,
|
324
|
-
spawn_timeout_sec: 2,
|
325
|
-
spawn_delay_sec: 2,
|
326
|
-
multi_incr_udp: nil,
|
327
|
-
resume: nil,
|
328
|
-
management_cb: nil,
|
329
|
-
**base_options
|
330
|
-
)
|
331
|
-
super(**base_options)
|
332
|
-
# special transfer parameters
|
333
|
-
@tr_opts = {
|
334
|
-
ascp_args: ascp_args,
|
335
|
-
wss: wss,
|
336
|
-
quiet: quiet,
|
337
|
-
trusted_certs: trusted_certs,
|
338
|
-
client_ssh_key: client_ssh_key,
|
339
|
-
check_ignore_cb: check_ignore_cb
|
340
|
-
}
|
341
|
-
@spawn_timeout_sec = spawn_timeout_sec
|
342
|
-
@spawn_delay_sec = spawn_delay_sec
|
343
|
-
# default is true on windows, false on other platforms
|
344
|
-
@multi_incr_udp = multi_incr_udp.nil? ? Environment.os.eql?(Environment::OS_WINDOWS) : multi_incr_udp
|
345
|
-
@management_cb = management_cb
|
346
|
-
@resume_policy = Resumer.new(resume.nil? ? {} : resume.symbolize_keys)
|
347
|
-
# all transfer jobs, key = SecureRandom.uuid, protected by mutex, cond var on change
|
348
|
-
@sessions = []
|
349
|
-
# mutex protects global data accessed by threads
|
350
|
-
@mutex = Mutex.new
|
351
|
-
end
|
352
|
-
|
353
|
-
# transfer thread entry
|
354
|
-
# @param session information
|
355
|
-
def transfer_thread_entry(session)
|
356
|
-
begin
|
357
|
-
# set name for logging
|
358
|
-
Thread.current[:name] = 'transfer'
|
359
|
-
Log.log.debug{"ENTER (#{Thread.current[:name]})"}
|
360
|
-
# start transfer with selected resumer policy
|
361
|
-
@resume_policy.execute_with_resume do
|
362
|
-
start_and_monitor_process(session: session, **session[:env_args])
|
363
|
-
end
|
364
|
-
Log.log.debug('transfer ok'.bg_green)
|
365
|
-
rescue StandardError => e
|
366
|
-
session[:error] = e
|
367
|
-
Log.log.error{"Transfer thread error: #{e.class}:\n#{e.message}:\n#{e.backtrace.join("\n")}".red} if Log.instance.level.eql?(:debug)
|
388
|
+
Log.log.debug{"Unknown event type for progress: #{event['Type']}"}
|
368
389
|
end
|
369
|
-
Log.log.debug{"EXIT (#{Thread.current[:name]})"}
|
370
390
|
end
|
371
391
|
end
|
372
392
|
end
|
data/lib/aspera/agent/httpgw.rb
CHANGED
@@ -9,9 +9,28 @@ require 'aspera/assert'
|
|
9
9
|
module Aspera
|
10
10
|
module Agent
|
11
11
|
class Httpgw < Base
|
12
|
-
|
13
|
-
|
12
|
+
def initialize(
|
13
|
+
url:,
|
14
|
+
api_version: Api::Httpgw::API_V2,
|
15
|
+
upload_chunk_size: 64_000,
|
16
|
+
synchronous: false,
|
17
|
+
**base_options
|
18
|
+
)
|
19
|
+
super(**base_options)
|
20
|
+
@gw_api = Api::Httpgw.new(
|
21
|
+
# remove /v1 from end of user-provided GW url: we need the base url only
|
22
|
+
url: url,
|
23
|
+
api_version: api_version,
|
24
|
+
upload_chunk_size: upload_chunk_size,
|
25
|
+
synchronous: synchronous,
|
26
|
+
notify_cb: ->(*pa, **ka){notify_progress(*pa, **ka)}
|
27
|
+
)
|
28
|
+
end
|
29
|
+
|
30
|
+
# Start FASP transfer based on transfer spec (hash table)
|
31
|
+
# note that this should be asynchronous, but it is not
|
14
32
|
# HTTP download only supports file list
|
33
|
+
# :reek:UnusedParameters token_regenerator
|
15
34
|
def start_transfer(transfer_spec, token_regenerator: nil)
|
16
35
|
raise 'GW URL must be set' if @gw_api.nil?
|
17
36
|
Aspera.assert_type(transfer_spec['paths'], Array){'paths'}
|
@@ -27,35 +46,12 @@ module Aspera
|
|
27
46
|
end
|
28
47
|
end
|
29
48
|
|
30
|
-
#
|
49
|
+
# Wait for completion of all jobs started
|
31
50
|
# @return list of :success or error message
|
32
51
|
def wait_for_transfers_completion
|
33
52
|
# well ... transfer was done in "start"
|
34
53
|
return [:success]
|
35
54
|
end
|
36
|
-
|
37
|
-
# TODO: is that useful?
|
38
|
-
# def url=(api_url); end
|
39
|
-
|
40
|
-
private
|
41
|
-
|
42
|
-
def initialize(
|
43
|
-
url:,
|
44
|
-
api_version: Api::Httpgw::API_V2,
|
45
|
-
upload_chunk_size: 64_000,
|
46
|
-
synchronous: false,
|
47
|
-
**base_options
|
48
|
-
)
|
49
|
-
super(**base_options)
|
50
|
-
@gw_api = Api::Httpgw.new(
|
51
|
-
# remove /v1 from end of user-provided GW url: we need the base url only
|
52
|
-
url: url,
|
53
|
-
api_version: api_version,
|
54
|
-
upload_chunk_size: upload_chunk_size,
|
55
|
-
synchronous: synchronous,
|
56
|
-
notify_cb: ->(*pa, **ka) { notify_progress(*pa, **ka) }
|
57
|
-
)
|
58
|
-
end
|
59
55
|
end
|
60
56
|
end
|
61
57
|
end
|
data/lib/aspera/agent/node.rb
CHANGED
@@ -13,10 +13,10 @@ module Aspera
|
|
13
13
|
# this singleton class is used by the CLI to provide a common interface to start a transfer
|
14
14
|
# before using it, the use must set the `node_api` member.
|
15
15
|
class Node < Base
|
16
|
-
# @param url
|
17
|
-
# @param username
|
18
|
-
# @param password
|
19
|
-
# @param root_id
|
16
|
+
# @param url [String] the base url of the node api
|
17
|
+
# @param username [String] the username to use for the node api
|
18
|
+
# @param password [String] the password to use for the node api
|
19
|
+
# @param root_id [String] root file id if the node is an access key
|
20
20
|
# @param base_options [Hash] options for base class
|
21
21
|
def initialize(
|
22
22
|
url:,
|
@@ -28,7 +28,7 @@ module Aspera
|
|
28
28
|
super(**base_options)
|
29
29
|
# root id is required for access key
|
30
30
|
@root_id = root_id
|
31
|
-
rest_params = {
|
31
|
+
rest_params = {base_url: url}
|
32
32
|
if OAuth::Factory.bearer?(password)
|
33
33
|
Aspera.assert(!@root_id.nil?){'root_id not allowed for access key'}
|
34
34
|
rest_params[:headers] = Api::Node.bearer_headers(password, access_key: username)
|
@@ -44,13 +44,8 @@ module Aspera
|
|
44
44
|
@transfer_id = nil
|
45
45
|
end
|
46
46
|
|
47
|
-
# used internally to ensure node api is set before using.
|
48
|
-
def node_api_
|
49
|
-
Aspera.assert(!@node_api.nil?){'Before using this object, set the node_api attribute to a Aspera::Rest object'}
|
50
|
-
return @node_api
|
51
|
-
end
|
52
|
-
|
53
47
|
# generic method
|
48
|
+
# :reek:UnusedParameters token_regenerator
|
54
49
|
def start_transfer(transfer_spec, token_regenerator: nil)
|
55
50
|
# add root id if access key
|
56
51
|
if !@root_id.nil?
|
@@ -119,6 +114,14 @@ module Aspera
|
|
119
114
|
# TODO: get status of sessions
|
120
115
|
return []
|
121
116
|
end
|
117
|
+
|
118
|
+
private
|
119
|
+
|
120
|
+
# used internally to ensure node api is set before using.
|
121
|
+
def node_api_
|
122
|
+
Aspera.assert(!@node_api.nil?){'Before using this object, set the node_api attribute to a Aspera::Rest object'}
|
123
|
+
return @node_api
|
124
|
+
end
|
122
125
|
end
|
123
126
|
end
|
124
127
|
end
|
@@ -1,5 +1,6 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
+
require 'aspera/environment'
|
3
4
|
require 'aspera/agent/base'
|
4
5
|
require 'aspera/products/transferd'
|
5
6
|
require 'aspera/temp_file_manager'
|
@@ -17,20 +18,23 @@ module Aspera
|
|
17
18
|
# port zero means select a random available high port
|
18
19
|
AUTO_LOCAL_TCP_PORT = "#{PORT_SEP}0"
|
19
20
|
|
20
|
-
|
21
|
-
|
22
|
-
# @param
|
23
|
-
# @param
|
21
|
+
private_constant :LOCAL_SOCKET_ADDR, :PORT_SEP, :AUTO_LOCAL_TCP_PORT
|
22
|
+
|
23
|
+
# @param url [String] URL of the transfer manager daemon
|
24
|
+
# @param start [Bool] if false, expect that an external daemon is already running
|
25
|
+
# @param stop [Bool] if false, do not shutdown daemon on exit
|
26
|
+
# @param base [Hash] base class options
|
24
27
|
def initialize(
|
25
|
-
url:
|
26
|
-
|
27
|
-
|
28
|
-
**
|
28
|
+
url: AUTO_LOCAL_TCP_PORT,
|
29
|
+
start: true,
|
30
|
+
stop: true,
|
31
|
+
**base
|
29
32
|
)
|
30
|
-
super(**
|
31
|
-
@
|
33
|
+
super(**base)
|
34
|
+
@transfer_id = nil
|
35
|
+
@stop = stop
|
32
36
|
is_local_auto_port = url.eql?(AUTO_LOCAL_TCP_PORT)
|
33
|
-
raise 'Cannot
|
37
|
+
raise 'Cannot set options `stop` or `start` to false with port zero' if is_local_auto_port && (!@stop || !start)
|
34
38
|
# keep PID for optional shutdown
|
35
39
|
@daemon_pid = nil
|
36
40
|
daemon_endpoint = url
|
@@ -44,14 +48,14 @@ module Aspera
|
|
44
48
|
# Initiate actual connection
|
45
49
|
get_info_response = @transfer_client.get_info(::Transferd::Api::InstanceInfoRequest.new)
|
46
50
|
Log.log.debug{"Daemon info: #{get_info_response}"}
|
47
|
-
Log.log.warn
|
51
|
+
Log.log.warn('Attached to existing daemon') unless @daemon_pid || !start || !@stop
|
48
52
|
at_exit{shutdown}
|
49
53
|
rescue GRPC::Unavailable => e
|
50
54
|
# if transferd is external: do not start it, or other error
|
51
|
-
raise if
|
55
|
+
raise if !start || !e.message.include?('failed to connect')
|
52
56
|
# we already tried to start a daemon, but it failed
|
53
57
|
Aspera.assert(@daemon_pid.nil?){"Daemon started with PID #{@daemon_pid}, but connection failed to #{daemon_endpoint}}"}
|
54
|
-
Log.log.warn('no daemon present, starting daemon...') if
|
58
|
+
Log.log.warn('no daemon present, starting daemon...') if !start
|
55
59
|
# transferd only supports local ip and port
|
56
60
|
daemon_uri = URI.parse("ipv4://#{daemon_endpoint}")
|
57
61
|
Aspera.assert(daemon_uri.scheme.eql?('ipv4')){"Invalid scheme daemon URI #{daemon_endpoint}"}
|
@@ -74,7 +78,11 @@ module Aspera
|
|
74
78
|
log_stdout = "#{transferd_base_tmp}.out"
|
75
79
|
log_stderr = "#{transferd_base_tmp}.err"
|
76
80
|
File.write(conf_file, config.to_json)
|
77
|
-
@daemon_pid =
|
81
|
+
@daemon_pid = Environment.secure_spawn(
|
82
|
+
exec: Ascp::Installation.instance.path(:transferd),
|
83
|
+
args: ['--config', conf_file],
|
84
|
+
out: log_stdout,
|
85
|
+
err: log_stderr)
|
78
86
|
begin
|
79
87
|
# wait for process to initialize, max 2 seconds
|
80
88
|
Timeout.timeout(2.0) do
|
@@ -86,8 +94,8 @@ module Aspera
|
|
86
94
|
nil
|
87
95
|
end
|
88
96
|
Log.log.debug{"Daemon started with pid #{@daemon_pid}"}
|
89
|
-
Process.detach(@daemon_pid)
|
90
|
-
at_exit
|
97
|
+
Process.detach(@daemon_pid) unless @stop
|
98
|
+
at_exit{shutdown}
|
91
99
|
# update port for next connection attempt (if auto high port was requested)
|
92
100
|
daemon_endpoint = "#{LOCAL_SOCKET_ADDR}#{PORT_SEP}#{Products::Transferd.daemon_port_from_log(log_stdout)}" if is_local_auto_port
|
93
101
|
# local daemon started, try again
|
@@ -95,6 +103,7 @@ module Aspera
|
|
95
103
|
end
|
96
104
|
end
|
97
105
|
|
106
|
+
# :reek:UnusedParameters token_regenerator
|
98
107
|
def start_transfer(transfer_spec, token_regenerator: nil)
|
99
108
|
# create a transfer request
|
100
109
|
transfer_request = ::Transferd::Api::TransferRequest.new(
|
@@ -146,13 +155,15 @@ module Aspera
|
|
146
155
|
end
|
147
156
|
|
148
157
|
def shutdown
|
149
|
-
stop_daemon
|
158
|
+
stop_daemon if @stop
|
150
159
|
end
|
151
160
|
|
161
|
+
private
|
162
|
+
|
152
163
|
def stop_daemon
|
153
164
|
if !@daemon_pid.nil?
|
154
165
|
Log.log.debug("Stopping daemon #{@daemon_pid}")
|
155
|
-
Process.kill(
|
166
|
+
Process.kill(:INT, @daemon_pid)
|
156
167
|
_, status = Process.wait2(@daemon_pid)
|
157
168
|
Log.log.debug("daemon stopped #{status}")
|
158
169
|
@daemon_pid = nil
|
data/lib/aspera/api/alee.rb
CHANGED
@@ -7,7 +7,7 @@ module Aspera
|
|
7
7
|
def initialize(entitlement_id, customer_id, api_domain: AoC::SAAS_DOMAIN_PROD, version: 'v1')
|
8
8
|
super(
|
9
9
|
base_url: "https://api.#{api_domain}/metering/#{version}",
|
10
|
-
headers: {'X-Aspera-Entitlement-Authorization' => Rest.
|
10
|
+
headers: {'X-Aspera-Entitlement-Authorization' => Rest.basic_authorization(entitlement_id, customer_id)}
|
11
11
|
)
|
12
12
|
end
|
13
13
|
end
|
data/lib/aspera/api/aoc.rb
CHANGED
@@ -22,7 +22,7 @@ module Aspera
|
|
22
22
|
MAX_AOC_URL_REDIRECT = 10
|
23
23
|
CLIENT_ID_PREFIX = 'aspera.'
|
24
24
|
# Well-known AoC global client apps
|
25
|
-
GLOBAL_CLIENT_APPS = DataRepository::ELEMENTS.select{|i|i.to_s.start_with?(CLIENT_ID_PREFIX)}.freeze
|
25
|
+
GLOBAL_CLIENT_APPS = DataRepository::ELEMENTS.select{ |i| i.to_s.start_with?(CLIENT_ID_PREFIX)}.freeze
|
26
26
|
# cookie prefix so that console can decode identity
|
27
27
|
COOKIE_PREFIX_CONSOLE_AOC = 'aspera.aoc'
|
28
28
|
# path in URL of public links
|
@@ -237,7 +237,7 @@ module Aspera
|
|
237
237
|
Log.log.debug{"ignoring error: #{e}"}
|
238
238
|
{}
|
239
239
|
end
|
240
|
-
USER_INFO_FIELDS_MIN.each{|f
|
240
|
+
USER_INFO_FIELDS_MIN.each{ |f| @cache_user_info[f] = nil if @cache_user_info[f].nil?}
|
241
241
|
return @cache_user_info
|
242
242
|
end
|
243
243
|
|
@@ -283,7 +283,7 @@ module Aspera
|
|
283
283
|
if ws_info.nil?
|
284
284
|
{
|
285
285
|
id: nil,
|
286
|
-
name:
|
286
|
+
name: "Shared #{application}"
|
287
287
|
}
|
288
288
|
else
|
289
289
|
{
|
@@ -386,10 +386,10 @@ module Aspera
|
|
386
386
|
Aspera.assert(field.key?('name')){'metadata field must have name'}
|
387
387
|
Aspera.assert(field.key?('values')){'metadata field must have values'}
|
388
388
|
Aspera.assert_type(field['values'], Array){'metadata field values'}
|
389
|
-
Aspera.assert(!meta_schema.none?{|i|i['name'].eql?(field['name'])}){"unknown metadata field: #{field['name']}"}
|
389
|
+
Aspera.assert(!meta_schema.none?{ |i| i['name'].eql?(field['name'])}){"unknown metadata field: #{field['name']}"}
|
390
390
|
end
|
391
391
|
meta_schema.each do |field|
|
392
|
-
provided = pkg_meta.select{|i|i['name'].eql?(field['name'])}
|
392
|
+
provided = pkg_meta.select{ |i| i['name'].eql?(field['name'])}
|
393
393
|
raise "only one field with name #{field['name']} allowed" if provided.count > 1
|
394
394
|
raise "missing mandatory field: #{field['name']}" if field['required'] && provided.empty?
|
395
395
|
end
|
@@ -519,7 +519,7 @@ module Aspera
|
|
519
519
|
# Console cookie
|
520
520
|
################
|
521
521
|
# we are sure that fields are not nil
|
522
|
-
cookie_elements = [app_info[:app], current_user_info['name'] || 'public link', current_user_info['email'] || 'none'].map{|e|Base64.strict_encode64(e)}
|
522
|
+
cookie_elements = [app_info[:app], current_user_info['name'] || 'public link', current_user_info['email'] || 'none'].map{ |e| Base64.strict_encode64(e)}
|
523
523
|
cookie_elements.unshift(COOKIE_PREFIX_CONSOLE_AOC)
|
524
524
|
transfer_spec['cookie'] = cookie_elements.join(':')
|
525
525
|
# Application tags
|