aspera-cli 4.18.1 → 4.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/CHANGELOG.md +33 -0
- data/CONTRIBUTING.md +17 -12
- data/README.md +396 -185
- data/bin/asession +26 -19
- data/examples/build_exec +74 -0
- data/examples/{rubyc → build_exec_rubyc} +18 -2
- data/examples/get_proto_file.rb +7 -0
- data/lib/aspera/agent/alpha.rb +8 -8
- data/lib/aspera/agent/base.rb +4 -18
- data/lib/aspera/agent/connect.rb +14 -13
- data/lib/aspera/agent/direct.rb +123 -120
- data/lib/aspera/agent/httpgw.rb +2 -3
- data/lib/aspera/agent/node.rb +10 -10
- data/lib/aspera/agent/trsdk.rb +17 -20
- data/lib/aspera/api/alee.rb +15 -0
- data/lib/aspera/api/aoc.rb +128 -99
- data/lib/aspera/api/ats.rb +1 -1
- data/lib/aspera/api/cos_node.rb +1 -1
- data/lib/aspera/api/httpgw.rb +104 -64
- data/lib/aspera/api/node.rb +33 -12
- data/lib/aspera/ascmd.rb +56 -48
- data/lib/aspera/ascp/installation.rb +142 -70
- data/lib/aspera/ascp/management.rb +7 -3
- data/lib/aspera/ascp/products.rb +13 -7
- data/lib/aspera/assert.rb +10 -5
- data/lib/aspera/cli/formatter.rb +42 -26
- data/lib/aspera/cli/hints.rb +2 -1
- data/lib/aspera/cli/info.rb +12 -10
- data/lib/aspera/cli/main.rb +16 -13
- data/lib/aspera/cli/manager.rb +15 -10
- data/lib/aspera/cli/plugin.rb +17 -31
- data/lib/aspera/cli/plugin_factory.rb +10 -1
- data/lib/aspera/cli/plugins/alee.rb +3 -3
- data/lib/aspera/cli/plugins/aoc.rb +222 -194
- data/lib/aspera/cli/plugins/ats.rb +16 -14
- data/lib/aspera/cli/plugins/config.rb +66 -53
- data/lib/aspera/cli/plugins/console.rb +3 -3
- data/lib/aspera/cli/plugins/faspex.rb +11 -21
- data/lib/aspera/cli/plugins/faspex5.rb +44 -42
- data/lib/aspera/cli/plugins/faspio.rb +2 -2
- data/lib/aspera/cli/plugins/httpgw.rb +1 -1
- data/lib/aspera/cli/plugins/node.rb +155 -96
- data/lib/aspera/cli/plugins/orchestrator.rb +14 -13
- data/lib/aspera/cli/plugins/preview.rb +8 -9
- data/lib/aspera/cli/plugins/server.rb +6 -10
- data/lib/aspera/cli/plugins/shares.rb +13 -9
- data/lib/aspera/cli/sync_actions.rb +72 -31
- data/lib/aspera/cli/transfer_agent.rb +13 -14
- data/lib/aspera/cli/transfer_progress.rb +36 -18
- data/lib/aspera/cli/version.rb +1 -1
- data/lib/aspera/command_line_builder.rb +3 -4
- data/lib/aspera/coverage.rb +13 -1
- data/lib/aspera/environment.rb +59 -10
- data/lib/aspera/faspex_gw.rb +3 -3
- data/lib/aspera/json_rpc.rb +1 -1
- data/lib/aspera/keychain/encrypted_hash.rb +2 -0
- data/lib/aspera/keychain/macos_security.rb +7 -12
- data/lib/aspera/log.rb +4 -4
- data/lib/aspera/node_simulator.rb +1 -1
- data/lib/aspera/oauth/base.rb +39 -45
- data/lib/aspera/oauth/factory.rb +11 -4
- data/lib/aspera/oauth/generic.rb +4 -8
- data/lib/aspera/oauth/jwt.rb +4 -4
- data/lib/aspera/oauth/url_json.rb +3 -2
- data/lib/aspera/oauth/web.rb +10 -6
- data/lib/aspera/persistency_action_once.rb +16 -8
- data/lib/aspera/preview/utils.rb +5 -16
- data/lib/aspera/rest.rb +100 -76
- data/lib/aspera/secret_hider.rb +3 -2
- data/lib/aspera/ssh.rb +1 -1
- data/lib/aspera/transfer/faux_file.rb +7 -5
- data/lib/aspera/transfer/parameters.rb +41 -35
- data/lib/aspera/transfer/spec.rb +16 -18
- data/lib/aspera/transfer/sync.rb +51 -50
- data/lib/aspera/transfer/uri.rb +1 -1
- data/lib/aspera/uri_reader.rb +1 -1
- data/lib/aspera/web_auth.rb +166 -18
- data/lib/aspera/web_server_simple.rb +27 -15
- data/lib/transfer_pb.rb +84 -0
- data/lib/transfer_services_pb.rb +82 -0
- data.tar.gz.sig +0 -0
- metadata +25 -6
- metadata.gz.sig +0 -0
data/lib/aspera/agent/direct.rb
CHANGED
@@ -19,9 +19,10 @@ module Aspera
|
|
19
19
|
# executes a local "ascp", connects mgt port, equivalent of "Fasp Manager"
|
20
20
|
class Direct < Base
|
21
21
|
LISTEN_LOCAL_ADDRESS = '127.0.0.1'
|
22
|
-
|
22
|
+
# 0 means: select an available port
|
23
|
+
SELECT_AVAILABLE_PORT = 0
|
23
24
|
# spellchecker: enable
|
24
|
-
private_constant :LISTEN_LOCAL_ADDRESS, :
|
25
|
+
private_constant :LISTEN_LOCAL_ADDRESS, :SELECT_AVAILABLE_PORT
|
25
26
|
|
26
27
|
# method of Base
|
27
28
|
# start ascp transfer(s) (non blocking), single or multi-session
|
@@ -93,12 +94,12 @@ module Aspera
|
|
93
94
|
# do deep copy (each thread has its own copy because it is modified here below and in thread)
|
94
95
|
this_session = session.clone
|
95
96
|
this_session[:ts] = this_session[:ts].clone
|
96
|
-
this_session[:env_args] = this_session[:env_args].clone
|
97
|
-
|
97
|
+
env_args = this_session[:env_args] = this_session[:env_args].clone
|
98
|
+
args = env_args[:args] = env_args[:args].clone
|
98
99
|
# set multi session part
|
99
|
-
|
100
|
+
args.unshift("-C#{i}:#{multi_session_info[:count]}")
|
100
101
|
# option: increment (default as per ascp manual) or not (cluster on other side ?)
|
101
|
-
|
102
|
+
args.unshift('-O', (multi_session_info[:udp_base] + i - 1).to_s) if @multi_incr_udp
|
102
103
|
# finally start the thread
|
103
104
|
this_session[:thread] = Thread.new(this_session) {|session_info|transfer_thread_entry(session_info)}
|
104
105
|
@sessions.push(this_session)
|
@@ -129,81 +130,46 @@ module Aspera
|
|
129
130
|
Log.log.debug('fasp local shutdown')
|
130
131
|
end
|
131
132
|
|
132
|
-
# @
|
133
|
-
def
|
134
|
-
|
135
|
-
case event['Type']
|
136
|
-
when 'INIT'
|
137
|
-
@pre_calc_sent = false
|
138
|
-
@pre_calc_last_size = nil
|
139
|
-
notify_progress(session_id: session_id, type: :session_start)
|
140
|
-
when 'NOTIFICATION' # sent from remote
|
141
|
-
if event.key?('PreTransferBytes')
|
142
|
-
@pre_calc_sent = true
|
143
|
-
notify_progress(session_id: session_id, type: :session_size, info: event['PreTransferBytes'])
|
144
|
-
end
|
145
|
-
when 'STATS' # during transfer
|
146
|
-
@pre_calc_last_size = event['TransferBytes'].to_i + event['StartByte'].to_i
|
147
|
-
notify_progress(session_id: session_id, type: :transfer, info: @pre_calc_last_size)
|
148
|
-
when 'DONE', 'ERROR' # end of session
|
149
|
-
total_size = event['TransferBytes'].to_i + event['StartByte'].to_i
|
150
|
-
if !@pre_calc_sent && !total_size.zero?
|
151
|
-
notify_progress(session_id: session_id, type: :session_size, info: total_size)
|
152
|
-
end
|
153
|
-
if @pre_calc_last_size != total_size
|
154
|
-
notify_progress(session_id: session_id, type: :transfer, info: total_size)
|
155
|
-
end
|
156
|
-
notify_progress(session_id: session_id, type: :end)
|
157
|
-
# cspell:disable
|
158
|
-
when 'SESSION'
|
159
|
-
when 'ARGSTOP'
|
160
|
-
when 'FILEERROR'
|
161
|
-
when 'STOP'
|
162
|
-
# cspell:enable
|
163
|
-
# stop event when one file is completed
|
164
|
-
else
|
165
|
-
Log.log.debug{"unknown event type #{event['Type']}"}
|
166
|
-
end
|
133
|
+
# @return [Array] list of sessions for a job
|
134
|
+
def sessions_by_job(job_id)
|
135
|
+
@sessions.select{|session_info| session_info[:job_id].eql?(job_id)}
|
167
136
|
end
|
168
137
|
|
169
|
-
# This is the low level method to start the
|
170
|
-
#
|
171
|
-
#
|
138
|
+
# This is the low level method to start the transfer process
|
139
|
+
# Start process with management port.
|
140
|
+
# returns
|
172
141
|
# raises FaspError on error
|
173
|
-
#
|
174
|
-
#
|
175
|
-
# @param
|
176
|
-
# @param
|
177
|
-
#
|
178
|
-
def
|
179
|
-
|
142
|
+
# @param session this session information, keys :io and :token_regenerator
|
143
|
+
# @param env [Hash] environment variables
|
144
|
+
# @param name [Symbol] name of executable: :ascp, :ascp4 or :async
|
145
|
+
# @param args [Array] command line arguments
|
146
|
+
# @return [nil] when process has exited
|
147
|
+
def start_and_monitor_process(
|
148
|
+
session:,
|
149
|
+
env:,
|
150
|
+
name:,
|
151
|
+
args:
|
152
|
+
)
|
180
153
|
Aspera.assert_type(session, Hash)
|
181
|
-
|
182
|
-
notify_progress(session_id: nil, type: :pre_start, info: 'starting')
|
154
|
+
notify_progress(:pre_start, session_id: nil, info: 'starting')
|
183
155
|
begin
|
184
|
-
|
156
|
+
command_pid = nil
|
185
157
|
# we use Socket directly, instead of TCPServer, as it gives access to lower level options
|
186
|
-
socket_class =
|
158
|
+
socket_class = defined?(JRUBY_VERSION) ? ServerSocket : Socket
|
187
159
|
mgt_server_socket = socket_class.new(Socket::AF_INET, Socket::SOCK_STREAM, 0)
|
188
|
-
# open any available (0) local TCP port for use as
|
189
|
-
mgt_server_socket.bind(Addrinfo.tcp(LISTEN_LOCAL_ADDRESS,
|
160
|
+
# open any available (0) local TCP port for use as management port
|
161
|
+
mgt_server_socket.bind(Addrinfo.tcp(LISTEN_LOCAL_ADDRESS, SELECT_AVAILABLE_PORT))
|
190
162
|
# build arguments and add mgt port
|
191
|
-
|
192
|
-
|
193
|
-
|
194
|
-
|
195
|
-
Log.log.debug do
|
196
|
-
[
|
197
|
-
'execute:'.red,
|
198
|
-
env_args[:env].map{|k, v| "#{k}=#{Shellwords.shellescape(v)}"},
|
199
|
-
Shellwords.shellescape(ascp_path),
|
200
|
-
ascp_arguments.map{|a|Shellwords.shellescape(a)}
|
201
|
-
].flatten.join(' ')
|
163
|
+
command_arguments = if name.eql?(:async)
|
164
|
+
["--exclusive-mgmt-port=#{mgt_server_socket.local_address.ip_port}"]
|
165
|
+
else
|
166
|
+
['-M', mgt_server_socket.local_address.ip_port.to_s]
|
202
167
|
end
|
203
|
-
|
204
|
-
|
205
|
-
|
206
|
-
|
168
|
+
command_arguments.concat(args)
|
169
|
+
# get location of command executable (ascp, async)
|
170
|
+
command_path = Ascp::Installation.instance.path(name)
|
171
|
+
command_pid = Environment.secure_spawn(env: env, exec: command_path, args: command_arguments)
|
172
|
+
notify_progress(:pre_start, session_id: nil, info: "waiting for #{name} to start")
|
207
173
|
mgt_server_socket.listen(1)
|
208
174
|
# TODO: timeout does not work when Process.spawn is used... until process exits, then it works
|
209
175
|
Log.log.debug{"before select, timeout: #{@spawn_timeout_sec}"}
|
@@ -213,15 +179,15 @@ module Aspera
|
|
213
179
|
# There is a connection to accept
|
214
180
|
client_socket, _client_addrinfo = mgt_server_socket.accept
|
215
181
|
Log.log.debug('after accept')
|
216
|
-
|
182
|
+
management_port_io = client_socket.to_io
|
217
183
|
# management messages include file names which may be utf8
|
218
184
|
# by default socket is US-ASCII
|
219
185
|
# TODO: use same value as Encoding.default_external
|
220
|
-
|
221
|
-
session[:io] =
|
186
|
+
management_port_io.set_encoding(Encoding::UTF_8)
|
187
|
+
session[:io] = management_port_io
|
222
188
|
processor = Ascp::Management.new
|
223
189
|
# read management port, until socket is closed (gets returns nil)
|
224
|
-
while (line =
|
190
|
+
while (line = management_port_io.gets)
|
225
191
|
event = processor.process_line(line.chomp)
|
226
192
|
next unless event
|
227
193
|
# event is ready
|
@@ -231,24 +197,23 @@ module Aspera
|
|
231
197
|
Log.log.error((event['Description']).to_s) if event['Type'].eql?('FILEERROR') # cspell:disable-line
|
232
198
|
end
|
233
199
|
Log.log.debug('management io closed')
|
234
|
-
last_event = processor.last_event
|
235
200
|
# check that last status was received before process exit
|
236
|
-
|
237
|
-
|
238
|
-
|
239
|
-
|
240
|
-
|
241
|
-
|
242
|
-
|
243
|
-
|
244
|
-
|
245
|
-
|
246
|
-
raise Transfer::Error.new(last_event['Description'], last_event['Code'].to_i)
|
247
|
-
when 'DONE'
|
248
|
-
nil
|
249
|
-
else
|
250
|
-
raise "unexpected last event type: #{last_event['Type']}"
|
201
|
+
last_event = processor.last_event
|
202
|
+
raise Transfer::Error, "internal: no management event (#{last_event.class})" unless last_event.is_a?(Hash)
|
203
|
+
case last_event['Type']
|
204
|
+
when 'ERROR'
|
205
|
+
if /bearer token/i.match?(last_event['Description']) &&
|
206
|
+
session[:token_regenerator].respond_to?(:refreshed_transfer_token)
|
207
|
+
# regenerate token here, expired, or error on it
|
208
|
+
# Note: in multi-session, each session will have a different one.
|
209
|
+
Log.log.warn('Regenerating token for transfer')
|
210
|
+
env['ASPERA_SCP_TOKEN'] = session[:token_regenerator].refreshed_transfer_token
|
251
211
|
end
|
212
|
+
raise Transfer::Error.new(last_event['Description'], last_event['Code'].to_i)
|
213
|
+
when 'DONE'
|
214
|
+
nil
|
215
|
+
else
|
216
|
+
raise Transfer::Error, "unexpected last event type: #{last_event['Type']}, #{last_event['Description']}"
|
252
217
|
end
|
253
218
|
rescue SystemCallError => e
|
254
219
|
# Process.spawn failed, or socket error
|
@@ -257,16 +222,16 @@ module Aspera
|
|
257
222
|
raise Transfer::Error, 'transfer interrupted by user'
|
258
223
|
ensure
|
259
224
|
mgt_server_socket.close
|
260
|
-
# if
|
261
|
-
unless
|
225
|
+
# if command was successfully started, check its status
|
226
|
+
unless command_pid.nil?
|
262
227
|
# "wait" for process to avoid zombie
|
263
|
-
Process.wait(
|
228
|
+
Process.wait(command_pid)
|
264
229
|
status = $CHILD_STATUS
|
265
|
-
|
230
|
+
# command_pid = nil
|
266
231
|
session.delete(:io)
|
267
|
-
# status is nil if an exception occurred before starting
|
232
|
+
# status is nil if an exception occurred before starting command
|
268
233
|
if !status&.success?
|
269
|
-
message = status.nil? ?
|
234
|
+
message = status.nil? ? "#{name} not started" : "#{name} failed (#{status})"
|
270
235
|
# raise error only if there was not already an exception (ERROR_INFO)
|
271
236
|
raise Transfer::Error, message unless $ERROR_INFO
|
272
237
|
# else display this message also, as main exception is already here
|
@@ -274,11 +239,47 @@ module Aspera
|
|
274
239
|
end
|
275
240
|
end
|
276
241
|
end
|
242
|
+
nil
|
277
243
|
end
|
278
244
|
|
279
|
-
|
280
|
-
|
281
|
-
|
245
|
+
private
|
246
|
+
|
247
|
+
# notify progress to callback
|
248
|
+
# @param event management port event
|
249
|
+
def process_progress(event)
|
250
|
+
session_id = event['SessionId']
|
251
|
+
case event['Type']
|
252
|
+
when 'INIT'
|
253
|
+
@pre_calc_sent = false
|
254
|
+
@pre_calc_last_size = nil
|
255
|
+
notify_progress(:session_start, session_id: session_id)
|
256
|
+
when 'NOTIFICATION' # sent from remote
|
257
|
+
if event.key?('PreTransferBytes')
|
258
|
+
@pre_calc_sent = true
|
259
|
+
notify_progress(:session_size, session_id: session_id, info: event['PreTransferBytes'])
|
260
|
+
end
|
261
|
+
when 'STATS' # during transfer
|
262
|
+
@pre_calc_last_size = event['TransferBytes'].to_i + event['StartByte'].to_i
|
263
|
+
notify_progress(:transfer, session_id: session_id, info: @pre_calc_last_size)
|
264
|
+
when 'DONE', 'ERROR' # end of session
|
265
|
+
total_size = event['TransferBytes'].to_i + event['StartByte'].to_i
|
266
|
+
if !@pre_calc_sent && !total_size.zero?
|
267
|
+
notify_progress(:session_size, session_id: session_id, info: total_size)
|
268
|
+
end
|
269
|
+
if @pre_calc_last_size != total_size
|
270
|
+
notify_progress(:transfer, session_id: session_id, info: total_size)
|
271
|
+
end
|
272
|
+
notify_progress(:end, session_id: session_id)
|
273
|
+
# cspell:disable
|
274
|
+
when 'SESSION'
|
275
|
+
when 'ARGSTOP'
|
276
|
+
when 'FILEERROR'
|
277
|
+
when 'STOP'
|
278
|
+
# cspell:enable
|
279
|
+
# stop event when one file is completed
|
280
|
+
else
|
281
|
+
Log.log.debug{"unknown event type #{event['Type']}"}
|
282
|
+
end
|
282
283
|
end
|
283
284
|
|
284
285
|
# @return [Hash] session information
|
@@ -289,7 +290,7 @@ module Aspera
|
|
289
290
|
return matches.first
|
290
291
|
end
|
291
292
|
|
292
|
-
# send command
|
293
|
+
# send command to management port of command (used in `asession)
|
293
294
|
# @param job_id identified transfer process
|
294
295
|
# @param session_index index of session (for multi session)
|
295
296
|
# @param data command on mgt port, examples:
|
@@ -309,47 +310,49 @@ module Aspera
|
|
309
310
|
end
|
310
311
|
attr_reader :sessions
|
311
312
|
|
312
|
-
private
|
313
|
-
|
314
313
|
# options for initialize (same as values in option transfer_info)
|
315
|
-
# @param wss [Boolean] true: if both SSH and wss in ts: prefer wss
|
316
314
|
# @param ascp_args [Array] additional arguments to ascp
|
317
|
-
# @param
|
318
|
-
# @param spawn_delay_sec [Integer] optional delay to start between sessions
|
319
|
-
# @param multi_incr_udp [Boolean] true: increment udp port for each session
|
320
|
-
# @param trusted_certs [Array] list of files with trusted certificates (stores)
|
321
|
-
# @param resume [Hash] resume policy
|
315
|
+
# @param wss [Boolean] true: if both SSH and wss in ts: prefer wss
|
322
316
|
# @param quiet [Boolean] by default no native ascp progress bar
|
317
|
+
# @param trusted_certs [Array,NilClass] list of files with trusted certificates (stores)
|
318
|
+
# @param client_ssh_key [String] client ssh key option (from CLIENT_SSH_KEY_OPTIONS)
|
323
319
|
# @param check_ignore_cb [Proc] callback with host,port
|
320
|
+
# @param spawn_timeout_sec [Integer] timeout for ascp spawn
|
321
|
+
# @param spawn_delay_sec [Integer] optional delay to start between sessions
|
322
|
+
# @param multi_incr_udp [Boolean,NilClass] true: increment udp port for each session
|
323
|
+
# @param resume [Hash,NilClass] resume policy
|
324
324
|
# @param management_cb [Proc] callback for management events
|
325
325
|
# @param base_options [Hash] other options for base class
|
326
326
|
def initialize(
|
327
|
+
ascp_args: nil,
|
327
328
|
wss: true,
|
328
|
-
ascp_args: [],
|
329
|
-
spawn_timeout_sec: 2,
|
330
|
-
spawn_delay_sec: 2,
|
331
|
-
multi_incr_udp: true,
|
332
|
-
trusted_certs: [],
|
333
|
-
resume: {},
|
334
329
|
quiet: true,
|
330
|
+
trusted_certs: nil,
|
331
|
+
client_ssh_key: nil,
|
335
332
|
check_ignore_cb: nil,
|
333
|
+
spawn_timeout_sec: 2,
|
334
|
+
spawn_delay_sec: 2,
|
335
|
+
multi_incr_udp: nil,
|
336
|
+
resume: nil,
|
336
337
|
management_cb: nil,
|
337
338
|
**base_options
|
338
339
|
)
|
339
340
|
super(**base_options)
|
341
|
+
# special transfer parameters
|
340
342
|
@tr_opts = {
|
341
343
|
ascp_args: ascp_args,
|
342
344
|
wss: wss,
|
343
345
|
quiet: quiet,
|
344
346
|
trusted_certs: trusted_certs,
|
347
|
+
client_ssh_key: client_ssh_key,
|
345
348
|
check_ignore_cb: check_ignore_cb
|
346
349
|
}
|
347
350
|
@spawn_timeout_sec = spawn_timeout_sec
|
348
351
|
@spawn_delay_sec = spawn_delay_sec
|
349
|
-
|
350
|
-
@
|
352
|
+
# default is true on windows, false on other platforms
|
353
|
+
@multi_incr_udp = multi_incr_udp.nil? ? Environment.os.eql?(Environment::OS_WINDOWS) : multi_incr_udp
|
351
354
|
@management_cb = management_cb
|
352
|
-
@resume_policy = Resumer.new(
|
355
|
+
@resume_policy = Resumer.new(resume.nil? ? {} : resume.symbolize_keys)
|
353
356
|
# all transfer jobs, key = SecureRandom.uuid, protected by mutex, cond var on change
|
354
357
|
@sessions = []
|
355
358
|
# mutex protects global data accessed by threads
|
@@ -365,7 +368,7 @@ module Aspera
|
|
365
368
|
Log.log.debug{"ENTER (#{Thread.current[:name]})"}
|
366
369
|
# start transfer with selected resumer policy
|
367
370
|
@resume_policy.execute_with_resume do
|
368
|
-
|
371
|
+
start_and_monitor_process(session: session, **session[:env_args])
|
369
372
|
end
|
370
373
|
Log.log.debug('transfer ok'.bg_green)
|
371
374
|
rescue StandardError => e
|
data/lib/aspera/agent/httpgw.rb
CHANGED
@@ -23,8 +23,7 @@ module Aspera
|
|
23
23
|
@gw_api.upload(transfer_spec)
|
24
24
|
when Transfer::Spec::DIRECTION_RECEIVE
|
25
25
|
@gw_api.download(transfer_spec)
|
26
|
-
else
|
27
|
-
raise "unexpected direction: [#{transfer_spec['direction']}]"
|
26
|
+
else Aspera.error_unexpected_value(transfer_spec['direction']){'direction'}
|
28
27
|
end
|
29
28
|
end
|
30
29
|
|
@@ -54,7 +53,7 @@ module Aspera
|
|
54
53
|
api_version: api_version,
|
55
54
|
upload_chunk_size: upload_chunk_size,
|
56
55
|
synchronous: synchronous,
|
57
|
-
notify_cb: ->(**
|
56
|
+
notify_cb: ->(*pa, **ka) { notify_progress(*pa, **ka) }
|
58
57
|
)
|
59
58
|
end
|
60
59
|
end
|
data/lib/aspera/agent/node.rb
CHANGED
@@ -78,7 +78,7 @@ module Aspera
|
|
78
78
|
if !transfer_spec['wss_enabled'] && transfer_spec['remote_host'].eql?(URI.parse(node_api_.base_url).host)
|
79
79
|
transfer_spec['remote_host'] = '127.0.0.1'
|
80
80
|
end
|
81
|
-
resp = node_api_.create('ops/transfers', transfer_spec)
|
81
|
+
resp = node_api_.create('ops/transfers', transfer_spec)
|
82
82
|
@transfer_id = resp['id']
|
83
83
|
Log.log.debug{"tr_id=#{@transfer_id}"}
|
84
84
|
return @transfer_id
|
@@ -92,31 +92,31 @@ module Aspera
|
|
92
92
|
# lets emulate management events to display progress bar
|
93
93
|
loop do
|
94
94
|
# status is empty sometimes with status 200...
|
95
|
-
transfer_data = node_api_.read("ops/transfers/#{@transfer_id}")
|
95
|
+
transfer_data = node_api_.read("ops/transfers/#{@transfer_id}") || {'status' => 'unknown'} rescue {'status' => 'waiting(api error)'}
|
96
96
|
case transfer_data['status']
|
97
97
|
when 'waiting', 'partially_completed', 'unknown', 'waiting(read error)'
|
98
|
-
notify_progress(
|
98
|
+
notify_progress(:pre_start, session_id: nil, info: transfer_data['status'])
|
99
99
|
when 'running'
|
100
100
|
if !session_started
|
101
|
-
notify_progress(session_id: @transfer_id
|
101
|
+
notify_progress(:session_start, session_id: @transfer_id)
|
102
102
|
session_started = true
|
103
103
|
end
|
104
104
|
message = transfer_data['status']
|
105
105
|
message = "#{message} (#{transfer_data['error_desc']})" if !transfer_data['error_desc']&.empty?
|
106
|
-
notify_progress(
|
106
|
+
notify_progress(:pre_start, session_id: nil, info: message)
|
107
107
|
if bytes_expected.nil? &&
|
108
108
|
transfer_data['precalc'].is_a?(Hash) &&
|
109
109
|
transfer_data['precalc']['status'].eql?('ready')
|
110
110
|
bytes_expected = transfer_data['precalc']['bytes_expected']
|
111
|
-
notify_progress(
|
111
|
+
notify_progress(:session_size, session_id: @transfer_id, info: bytes_expected)
|
112
112
|
end
|
113
|
-
notify_progress(
|
113
|
+
notify_progress(:transfer, session_id: @transfer_id, info: transfer_data['bytes_transferred'])
|
114
114
|
when 'completed'
|
115
|
-
notify_progress(
|
116
|
-
notify_progress(
|
115
|
+
notify_progress(:transfer, session_id: @transfer_id, info: bytes_expected) if bytes_expected
|
116
|
+
notify_progress(:end, session_id: @transfer_id)
|
117
117
|
break
|
118
118
|
when 'failed'
|
119
|
-
notify_progress(
|
119
|
+
notify_progress(:end, session_id: @transfer_id)
|
120
120
|
# Bug in HSTS ? transfer is marked failed, but there is no reason
|
121
121
|
break if transfer_data['error_code'].eql?(0) && transfer_data['error_desc'].empty?
|
122
122
|
raise Transfer::Error, "status: #{transfer_data['status']}. code: #{transfer_data['error_code']}. description: #{transfer_data['error_desc']}"
|
data/lib/aspera/agent/trsdk.rb
CHANGED
@@ -7,6 +7,7 @@ require 'aspera/log'
|
|
7
7
|
require 'aspera/assert'
|
8
8
|
require 'json'
|
9
9
|
require 'uri'
|
10
|
+
require 'transfer_services_pb'
|
10
11
|
|
11
12
|
module Aspera
|
12
13
|
module Agent
|
@@ -48,14 +49,12 @@ module Aspera
|
|
48
49
|
**base_options
|
49
50
|
)
|
50
51
|
super(**base_options)
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
$LOAD_PATH.unshift(Ascp::Installation.instance.sdk_ruby_folder)
|
55
|
-
require 'transfer_services_pb'
|
52
|
+
@keep = keep
|
53
|
+
is_local_auto_port = url.eql?(AUTO_LOCAL_TCP_PORT)
|
54
|
+
raise 'Cannot use options `keep` or `external` with port zero' if is_local_auto_port && (@keep || external)
|
56
55
|
# keep PID for optional shutdown
|
57
56
|
@daemon_pid = nil
|
58
|
-
daemon_endpoint =
|
57
|
+
daemon_endpoint = url
|
59
58
|
Log.log.debug{Log.dump(:daemon_endpoint, daemon_endpoint)}
|
60
59
|
# retry loop
|
61
60
|
begin
|
@@ -66,16 +65,14 @@ module Aspera
|
|
66
65
|
# Initiate actual connection
|
67
66
|
get_info_response = @transfer_client.get_info(Transfersdk::InstanceInfoRequest.new)
|
68
67
|
Log.log.debug{"Daemon info: #{get_info_response}"}
|
69
|
-
Log.log.warn{'Attached to existing daemon'} unless @daemon_pid ||
|
68
|
+
Log.log.warn{'Attached to existing daemon'} unless @daemon_pid || external || @keep
|
70
69
|
at_exit{shutdown}
|
71
70
|
rescue GRPC::Unavailable => e
|
72
71
|
# if transferd is external: do not start it, or other error
|
73
|
-
raise if
|
72
|
+
raise if external || !e.message.include?('failed to connect')
|
74
73
|
# we already tried to start a daemon, but it failed
|
75
74
|
Aspera.assert(@daemon_pid.nil?){"Daemon started with PID #{@daemon_pid}, but connection failed to #{daemon_endpoint}}"}
|
76
|
-
Log.log.warn('no daemon present, starting daemon...') if
|
77
|
-
# location of daemon binary
|
78
|
-
sdk_folder = File.realpath(File.join(Ascp::Installation.instance.sdk_ruby_folder, '..'))
|
75
|
+
Log.log.warn('no daemon present, starting daemon...') if external
|
79
76
|
# transferd only supports local ip and port
|
80
77
|
daemon_uri = URI.parse("ipv4://#{daemon_endpoint}")
|
81
78
|
Aspera.assert(daemon_uri.scheme.eql?('ipv4')){"Invalid scheme daemon URI #{daemon_endpoint}"}
|
@@ -86,8 +83,8 @@ module Aspera
|
|
86
83
|
fasp_runtime: {
|
87
84
|
use_embedded: false,
|
88
85
|
user_defined: {
|
89
|
-
bin: sdk_folder,
|
90
|
-
etc: sdk_folder
|
86
|
+
bin: Ascp::Installation.instance.sdk_folder,
|
87
|
+
etc: Ascp::Installation.instance.sdk_folder
|
91
88
|
}
|
92
89
|
}
|
93
90
|
}
|
@@ -143,24 +140,24 @@ module Aspera
|
|
143
140
|
case response.status
|
144
141
|
when :RUNNING
|
145
142
|
if !session_started
|
146
|
-
notify_progress(session_id: @transfer_id
|
143
|
+
notify_progress(:session_start, session_id: @transfer_id)
|
147
144
|
session_started = true
|
148
145
|
end
|
149
146
|
if bytes_expected.nil? &&
|
150
147
|
!response.sessionInfo.preTransferBytes.eql?(0)
|
151
148
|
bytes_expected = response.sessionInfo.preTransferBytes
|
152
|
-
notify_progress(
|
149
|
+
notify_progress(:session_size, session_id: @transfer_id, info: bytes_expected)
|
153
150
|
end
|
154
|
-
notify_progress(
|
151
|
+
notify_progress(:transfer, session_id: @transfer_id, info: response.transferInfo.bytesTransferred)
|
155
152
|
when :COMPLETED
|
156
|
-
notify_progress(
|
157
|
-
notify_progress(
|
153
|
+
notify_progress(:transfer, session_id: @transfer_id, info: bytes_expected) if bytes_expected
|
154
|
+
notify_progress(:end, session_id: @transfer_id)
|
158
155
|
break
|
159
156
|
when :FAILED, :CANCELED
|
160
|
-
notify_progress(
|
157
|
+
notify_progress(:end, session_id: @transfer_id)
|
161
158
|
raise Transfer::Error, JSON.parse(response.message)['Description']
|
162
159
|
when :QUEUED, :UNKNOWN_STATUS, :PAUSED, :ORPHANED
|
163
|
-
notify_progress(
|
160
|
+
notify_progress(:pre_start, session_id: nil, info: response.status.to_s.downcase)
|
164
161
|
else
|
165
162
|
Log.log.error{"unknown status#{response.status}"}
|
166
163
|
end
|
@@ -0,0 +1,15 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'aspera/api/aoc.rb'
|
4
|
+
module Aspera
|
5
|
+
module Api
|
6
|
+
class Alee < Aspera::Rest
|
7
|
+
def initialize(entitlement_id, customer_id, api_domain: AoC::SAAS_DOMAIN_PROD, version: 'v1')
|
8
|
+
super(
|
9
|
+
base_url: "https://api.#{api_domain}/metering/#{version}",
|
10
|
+
headers: {'X-Aspera-Entitlement-Authorization' => Rest.basic_token(entitlement_id, customer_id)}
|
11
|
+
)
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
15
|
+
end
|