aspera-cli 4.4.0 → 4.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +2095 -1503
- data/bin/ascli +2 -1
- data/bin/asession +4 -5
- data/docs/test_env.conf +3 -0
- data/examples/aoc.rb +4 -3
- data/examples/faspex4.rb +25 -25
- data/examples/proxy.pac +1 -1
- data/examples/transfer.rb +17 -17
- data/lib/aspera/aoc.rb +238 -185
- data/lib/aspera/ascmd.rb +93 -83
- data/lib/aspera/ats_api.rb +11 -10
- data/lib/aspera/cli/basic_auth_plugin.rb +13 -14
- data/lib/aspera/cli/extended_value.rb +42 -33
- data/lib/aspera/cli/formater.rb +142 -108
- data/lib/aspera/cli/info.rb +17 -0
- data/lib/aspera/cli/listener/line_dump.rb +3 -2
- data/lib/aspera/cli/listener/logger.rb +2 -1
- data/lib/aspera/cli/listener/progress.rb +16 -18
- data/lib/aspera/cli/listener/progress_multi.rb +18 -21
- data/lib/aspera/cli/main.rb +173 -149
- data/lib/aspera/cli/manager.rb +163 -168
- data/lib/aspera/cli/plugin.rb +43 -31
- data/lib/aspera/cli/plugins/alee.rb +6 -6
- data/lib/aspera/cli/plugins/aoc.rb +405 -370
- data/lib/aspera/cli/plugins/ats.rb +86 -79
- data/lib/aspera/cli/plugins/bss.rb +14 -16
- data/lib/aspera/cli/plugins/config.rb +580 -362
- data/lib/aspera/cli/plugins/console.rb +23 -19
- data/lib/aspera/cli/plugins/cos.rb +18 -18
- data/lib/aspera/cli/plugins/faspex.rb +201 -158
- data/lib/aspera/cli/plugins/faspex5.rb +80 -57
- data/lib/aspera/cli/plugins/node.rb +183 -166
- data/lib/aspera/cli/plugins/orchestrator.rb +71 -67
- data/lib/aspera/cli/plugins/preview.rb +92 -96
- data/lib/aspera/cli/plugins/server.rb +79 -75
- data/lib/aspera/cli/plugins/shares.rb +35 -19
- data/lib/aspera/cli/plugins/sync.rb +20 -22
- data/lib/aspera/cli/transfer_agent.rb +76 -113
- data/lib/aspera/cli/version.rb +2 -1
- data/lib/aspera/colors.rb +35 -27
- data/lib/aspera/command_line_builder.rb +48 -34
- data/lib/aspera/cos_node.rb +29 -21
- data/lib/aspera/data_repository.rb +3 -2
- data/lib/aspera/environment.rb +50 -45
- data/lib/aspera/fasp/{manager.rb → agent_base.rb} +28 -25
- data/lib/aspera/fasp/{connect.rb → agent_connect.rb} +52 -43
- data/lib/aspera/fasp/{local.rb → agent_direct.rb} +58 -72
- data/lib/aspera/fasp/{http_gw.rb → agent_httpgw.rb} +37 -43
- data/lib/aspera/fasp/{node.rb → agent_node.rb} +35 -16
- data/lib/aspera/fasp/agent_trsdk.rb +104 -0
- data/lib/aspera/fasp/error.rb +2 -1
- data/lib/aspera/fasp/error_info.rb +68 -52
- data/lib/aspera/fasp/installation.rb +152 -124
- data/lib/aspera/fasp/listener.rb +1 -0
- data/lib/aspera/fasp/parameters.rb +87 -92
- data/lib/aspera/fasp/parameters.yaml +305 -249
- data/lib/aspera/fasp/resume_policy.rb +11 -14
- data/lib/aspera/fasp/transfer_spec.rb +26 -0
- data/lib/aspera/fasp/uri.rb +22 -21
- data/lib/aspera/faspex_gw.rb +55 -89
- data/lib/aspera/hash_ext.rb +4 -3
- data/lib/aspera/id_generator.rb +8 -7
- data/lib/aspera/keychain/encrypted_hash.rb +121 -0
- data/lib/aspera/keychain/macos_security.rb +90 -0
- data/lib/aspera/log.rb +55 -37
- data/lib/aspera/nagios.rb +13 -12
- data/lib/aspera/node.rb +30 -25
- data/lib/aspera/oauth.rb +175 -226
- data/lib/aspera/open_application.rb +4 -3
- data/lib/aspera/persistency_action_once.rb +6 -6
- data/lib/aspera/persistency_folder.rb +5 -9
- data/lib/aspera/preview/file_types.rb +6 -5
- data/lib/aspera/preview/generator.rb +25 -24
- data/lib/aspera/preview/options.rb +16 -14
- data/lib/aspera/preview/utils.rb +98 -98
- data/lib/aspera/{proxy_auto_config.erb.js → proxy_auto_config.js} +23 -31
- data/lib/aspera/proxy_auto_config.rb +111 -20
- data/lib/aspera/rest.rb +154 -135
- data/lib/aspera/rest_call_error.rb +2 -2
- data/lib/aspera/rest_error_analyzer.rb +23 -25
- data/lib/aspera/rest_errors_aspera.rb +15 -14
- data/lib/aspera/ssh.rb +12 -10
- data/lib/aspera/sync.rb +42 -41
- data/lib/aspera/temp_file_manager.rb +18 -14
- data/lib/aspera/timer_limiter.rb +2 -1
- data/lib/aspera/uri_reader.rb +7 -5
- data/lib/aspera/web_auth.rb +79 -76
- metadata +116 -29
- data/docs/Makefile +0 -66
- data/docs/README.erb.md +0 -3973
- data/docs/README.md +0 -13
- data/docs/diagrams.txt +0 -49
- data/docs/doc_tools.rb +0 -58
- data/lib/aspera/api_detector.rb +0 -60
- data/lib/aspera/cli/plugins/shares2.rb +0 -114
- data/lib/aspera/secrets.rb +0 -20
|
@@ -1,36 +1,31 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
# Aspera 2016
|
|
5
|
-
# Laurent Martin
|
|
6
|
-
#
|
|
7
|
-
##############################################################################
|
|
8
|
-
require 'aspera/fasp/manager'
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
require 'English'
|
|
3
|
+
require 'aspera/fasp/agent_base'
|
|
9
4
|
require 'aspera/fasp/error'
|
|
10
5
|
require 'aspera/fasp/parameters'
|
|
11
6
|
require 'aspera/fasp/installation'
|
|
12
7
|
require 'aspera/fasp/resume_policy'
|
|
8
|
+
require 'aspera/fasp/transfer_spec'
|
|
13
9
|
require 'aspera/log'
|
|
14
10
|
require 'socket'
|
|
15
11
|
require 'timeout'
|
|
16
12
|
require 'securerandom'
|
|
13
|
+
require 'shellwords'
|
|
17
14
|
|
|
18
15
|
module Aspera
|
|
19
16
|
module Fasp
|
|
20
17
|
# executes a local "ascp", connects mgt port, equivalent of "Fasp Manager"
|
|
21
|
-
class
|
|
18
|
+
class AgentDirect < AgentBase
|
|
22
19
|
# options for initialize (same as values in option transfer_info)
|
|
23
20
|
DEFAULT_OPTIONS = {
|
|
24
|
-
:
|
|
25
|
-
:
|
|
26
|
-
:
|
|
27
|
-
:
|
|
28
|
-
:
|
|
21
|
+
spawn_timeout_sec: 3,
|
|
22
|
+
spawn_delay_sec: 2,
|
|
23
|
+
wss: false,
|
|
24
|
+
multi_incr_udp: true,
|
|
25
|
+
resume: {},
|
|
26
|
+
quiet: true # by default no interactive progress bar
|
|
29
27
|
}
|
|
30
|
-
DEFAULT_UDP_PORT=33001
|
|
31
28
|
private_constant :DEFAULT_OPTIONS
|
|
32
|
-
# set to false to keep ascp progress bar display ("true" adds ascp's option -q)
|
|
33
|
-
attr_accessor :quiet
|
|
34
29
|
|
|
35
30
|
# start ascp transfer (non blocking), single or multi-session
|
|
36
31
|
# job information added to @jobs
|
|
@@ -44,10 +39,11 @@ module Aspera
|
|
|
44
39
|
# clone transfer spec because we modify it (first level keys)
|
|
45
40
|
transfer_spec=transfer_spec.clone
|
|
46
41
|
# if there is aspera tags
|
|
47
|
-
if transfer_spec['tags'].is_a?(Hash)
|
|
42
|
+
if transfer_spec['tags'].is_a?(Hash) && transfer_spec['tags']['aspera'].is_a?(Hash)
|
|
48
43
|
# TODO: what is this for ? only on local ascp ?
|
|
49
44
|
# NOTE: important: transfer id must be unique: generate random id
|
|
50
45
|
# using a non unique id results in discard of tags in AoC, and a package is never finalized
|
|
46
|
+
# all sessions in a multi-session transfer must have the same xfer_id (see admin manual)
|
|
51
47
|
transfer_spec['tags']['aspera']['xfer_id']||=SecureRandom.uuid
|
|
52
48
|
Log.log.debug("xfer id=#{transfer_spec['xfer_id']}")
|
|
53
49
|
# TODO: useful ? node only ?
|
|
@@ -56,8 +52,8 @@ module Aspera
|
|
|
56
52
|
Log.dump('ts',transfer_spec)
|
|
57
53
|
|
|
58
54
|
# add bypass keys when authentication is token and no auth is provided
|
|
59
|
-
if transfer_spec.has_key?('token')
|
|
60
|
-
!transfer_spec.has_key?('remote_password')
|
|
55
|
+
if transfer_spec.has_key?('token') &&
|
|
56
|
+
!transfer_spec.has_key?('remote_password') &&
|
|
61
57
|
!transfer_spec.has_key?('EX_ssh_key_paths')
|
|
62
58
|
# transfer_spec['remote_password'] = Installation.instance.bypass_pass # not used
|
|
63
59
|
transfer_spec['EX_ssh_key_paths'] = Installation.instance.bypass_keys
|
|
@@ -68,24 +64,22 @@ module Aspera
|
|
|
68
64
|
multi_session_info=nil
|
|
69
65
|
if transfer_spec.has_key?('multi_session')
|
|
70
66
|
multi_session_info={
|
|
71
|
-
count: transfer_spec['multi_session'].to_i
|
|
67
|
+
count: transfer_spec['multi_session'].to_i
|
|
72
68
|
}
|
|
73
69
|
# Managed by multi-session, so delete from transfer spec
|
|
74
70
|
transfer_spec.delete('multi_session')
|
|
75
|
-
if multi_session_info[:count]
|
|
71
|
+
if multi_session_info[:count].negative?
|
|
76
72
|
Log.log.error("multi_session(#{transfer_spec['multi_session']}) shall be integer >= 0")
|
|
77
73
|
multi_session_info = nil
|
|
78
74
|
elsif multi_session_info[:count].eql?(0)
|
|
79
|
-
Log.log.debug(
|
|
75
|
+
Log.log.debug('multi_session count is zero: no multisession')
|
|
80
76
|
multi_session_info = nil
|
|
81
|
-
|
|
77
|
+
elsif @options[:multi_incr_udp] # multi_session_info[:count] > 0
|
|
82
78
|
# if option not true: keep default udp port for all sessions
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
transfer_spec.delete('fasp_port')
|
|
88
|
-
end
|
|
79
|
+
multi_session_info[:udp_base]=transfer_spec.has_key?('fasp_port') ? transfer_spec['fasp_port'] : TransferSpec::UDP_PORT
|
|
80
|
+
# delete from original transfer spec, as we will increment values
|
|
81
|
+
transfer_spec.delete('fasp_port')
|
|
82
|
+
# override if specified, else use default value
|
|
89
83
|
end
|
|
90
84
|
end
|
|
91
85
|
|
|
@@ -98,22 +92,22 @@ module Aspera
|
|
|
98
92
|
env_args[:args].unshift('-I',Installation.instance.path(:fallback_cert))
|
|
99
93
|
end
|
|
100
94
|
|
|
101
|
-
env_args[:args].unshift('-q') if @quiet
|
|
95
|
+
env_args[:args].unshift('-q') if @options[:quiet]
|
|
102
96
|
|
|
103
97
|
# transfer job can be multi session
|
|
104
98
|
xfer_job={
|
|
105
|
-
:
|
|
106
|
-
:
|
|
99
|
+
id: job_options[:job_id],
|
|
100
|
+
sessions: [] # all sessions as below
|
|
107
101
|
}
|
|
108
102
|
|
|
109
103
|
# generic session information
|
|
110
104
|
session={
|
|
111
|
-
:
|
|
112
|
-
:
|
|
113
|
-
:
|
|
114
|
-
:
|
|
115
|
-
:env_args
|
|
116
|
-
:
|
|
105
|
+
thread: nil, # Thread object monitoring management port, not nil when pushed to :sessions
|
|
106
|
+
error: nil, # exception if failed
|
|
107
|
+
io: nil, # management port server socket
|
|
108
|
+
id: nil, # SessionId from INIT message in mgt port
|
|
109
|
+
env_args: env_args, # env vars and args to ascp (from transfer spec)
|
|
110
|
+
options: job_options # [Hash]
|
|
117
111
|
}
|
|
118
112
|
|
|
119
113
|
if multi_session_info.nil?
|
|
@@ -132,7 +126,7 @@ module Aspera
|
|
|
132
126
|
this_session[:env_args][:args]=this_session[:env_args][:args].clone()
|
|
133
127
|
this_session[:env_args][:args].unshift("-C#{i}:#{multi_session_info[:count]}")
|
|
134
128
|
# option: increment (default as per ascp manual) or not (cluster on other side ?)
|
|
135
|
-
this_session[:env_args][:args].unshift('-O',
|
|
129
|
+
this_session[:env_args][:args].unshift('-O',(multi_session_info[:udp_base]+i-1).to_s) if @options[:multi_incr_udp]
|
|
136
130
|
this_session[:thread] = Thread.new(this_session) {|s|transfer_thread_entry(s)}
|
|
137
131
|
xfer_job[:sessions].push(this_session)
|
|
138
132
|
end
|
|
@@ -152,11 +146,11 @@ module Aspera
|
|
|
152
146
|
Log.log.debug('wait_for_transfers_completion')
|
|
153
147
|
# set to non-nil to exit loop
|
|
154
148
|
result=[]
|
|
155
|
-
@jobs.each do |
|
|
149
|
+
@jobs.each do |_id,job|
|
|
156
150
|
job[:sessions].each do |session|
|
|
157
151
|
Log.log.debug("join #{session[:thread]}")
|
|
158
152
|
session[:thread].join
|
|
159
|
-
result.push(session[:error]
|
|
153
|
+
result.push(session[:error] || :success)
|
|
160
154
|
end
|
|
161
155
|
end
|
|
162
156
|
Log.log.debug('all transfers joined')
|
|
@@ -190,7 +184,7 @@ module Aspera
|
|
|
190
184
|
Fasp::Installation.instance.path(env_args[:ascp_version])
|
|
191
185
|
end
|
|
192
186
|
# (optional) check it exists
|
|
193
|
-
raise Fasp::Error
|
|
187
|
+
raise Fasp::Error, "no such file: #{ascp_path}" unless File.exist?(ascp_path)
|
|
194
188
|
# open random local TCP port for listening for ascp management
|
|
195
189
|
mgt_sock = TCPServer.new('127.0.0.1',0)
|
|
196
190
|
# clone arguments as we eed to modify with mgt port
|
|
@@ -198,7 +192,7 @@ module Aspera
|
|
|
198
192
|
# add management port
|
|
199
193
|
ascp_arguments.unshift('-M', mgt_sock.addr[1].to_s)
|
|
200
194
|
# start ascp in sub process
|
|
201
|
-
Log.log.debug("execute: #{env_args[:env].map{|k,v| "#{k}
|
|
195
|
+
Log.log.debug("execute: #{env_args[:env].map{|k,v| "#{k}=#{Shellwords.shellescape(v)}"}.join(' ')} #{Shellwords.shellescape(ascp_path)} #{ascp_arguments.map{|a|Shellwords.shellescape(a)}.join(' ')}")
|
|
202
196
|
# start process
|
|
203
197
|
ascp_pid = Process.spawn(env_args[:env],[ascp_path,ascp_path],*ascp_arguments)
|
|
204
198
|
# in parent, wait for connection to socket max 3 seconds
|
|
@@ -226,21 +220,21 @@ module Aspera
|
|
|
226
220
|
line = ascp_mgt_io.gets
|
|
227
221
|
# nil when ascp process exits
|
|
228
222
|
break if line.nil?
|
|
229
|
-
current_event_text
|
|
223
|
+
current_event_text+=line
|
|
230
224
|
line.chomp!
|
|
231
225
|
Log.log.debug("line=[#{line}]")
|
|
232
226
|
case line
|
|
233
227
|
when 'FASPMGR 2'
|
|
234
228
|
# begin event
|
|
235
|
-
current_event_data =
|
|
229
|
+
current_event_data = {}
|
|
236
230
|
current_event_text = ''
|
|
237
231
|
when /^([^:]+): (.*)$/
|
|
238
232
|
# event field
|
|
239
|
-
current_event_data[
|
|
233
|
+
current_event_data[Regexp.last_match(1)] = Regexp.last_match(2)
|
|
240
234
|
when ''
|
|
241
235
|
# empty line is separator to end event information
|
|
242
236
|
raise 'unexpected empty line' if current_event_data.nil?
|
|
243
|
-
current_event_data[
|
|
237
|
+
current_event_data[AgentBase::LISTENER_SESSION_ID_B]=ascp_pid
|
|
244
238
|
notify_listeners(current_event_text,current_event_data)
|
|
245
239
|
case current_event_data['Type']
|
|
246
240
|
when 'INIT'
|
|
@@ -262,9 +256,9 @@ module Aspera
|
|
|
262
256
|
exception_raised=false
|
|
263
257
|
when 'ERROR'
|
|
264
258
|
Log.log.error("code: #{last_status_event['Code']}")
|
|
265
|
-
if last_status_event['Description']
|
|
259
|
+
if last_status_event['Description'] =~ /bearer token/i
|
|
266
260
|
Log.log.error('need to regenerate token'.red)
|
|
267
|
-
if session[:options].is_a?(Hash)
|
|
261
|
+
if session[:options].is_a?(Hash) && session[:options].has_key?(:regenerate_token)
|
|
268
262
|
# regenerate token here, expired, or error on it
|
|
269
263
|
# Note: in multi-session, each session will have a different one.
|
|
270
264
|
env_args[:env]['ASPERA_SCP_TOKEN']=session[:options][:regenerate_token].call(true)
|
|
@@ -280,27 +274,25 @@ module Aspera
|
|
|
280
274
|
end
|
|
281
275
|
rescue SystemCallError => e
|
|
282
276
|
# Process.spawn
|
|
283
|
-
raise Fasp::Error
|
|
284
|
-
rescue Timeout::Error
|
|
285
|
-
raise Fasp::Error
|
|
286
|
-
rescue Interrupt
|
|
287
|
-
raise Fasp::Error
|
|
277
|
+
raise Fasp::Error, e.message
|
|
278
|
+
rescue Timeout::Error
|
|
279
|
+
raise Fasp::Error, 'timeout waiting mgt port connect'
|
|
280
|
+
rescue Interrupt
|
|
281
|
+
raise Fasp::Error, 'transfer interrupted by user'
|
|
288
282
|
ensure
|
|
289
283
|
# if ascp was successfully started
|
|
290
284
|
unless ascp_pid.nil?
|
|
291
285
|
# "wait" for process to avoid zombie
|
|
292
286
|
Process.wait(ascp_pid)
|
|
293
|
-
status
|
|
287
|
+
status=$CHILD_STATUS
|
|
294
288
|
ascp_pid=nil
|
|
295
289
|
session.delete(:io)
|
|
296
290
|
if !status.success?
|
|
297
291
|
message="ascp failed with code #{status.exitstatus}"
|
|
298
|
-
if
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
raise Fasp::Error.new(message)
|
|
303
|
-
end
|
|
292
|
+
# raise error only if there was not already an exception
|
|
293
|
+
raise Fasp::Error, message unless exception_raised
|
|
294
|
+
# else just debug, as main exception is already here
|
|
295
|
+
Log.log.debug(message)
|
|
304
296
|
end
|
|
305
297
|
end
|
|
306
298
|
end # begin-ensure
|
|
@@ -333,8 +325,6 @@ module Aspera
|
|
|
333
325
|
# @param options : keys(symbol): see DEFAULT_OPTIONS
|
|
334
326
|
def initialize(options=nil)
|
|
335
327
|
super()
|
|
336
|
-
# by default no interactive progress bar
|
|
337
|
-
@quiet=true
|
|
338
328
|
# all transfer jobs, key = SecureRandom.uuid, protected by mutex, condvar on change
|
|
339
329
|
@jobs={}
|
|
340
330
|
# mutex protects global data accessed by threads
|
|
@@ -344,11 +334,8 @@ module Aspera
|
|
|
344
334
|
if !options.nil?
|
|
345
335
|
raise "expecting Hash (or nil), but have #{options.class}" unless options.is_a?(Hash)
|
|
346
336
|
options.each do |k,v|
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
else
|
|
350
|
-
raise "Unknown local agent parameter: #{k}, expect one of #{DEFAULT_OPTIONS.keys.map{|i|i.to_s}.join(",")}"
|
|
351
|
-
end
|
|
337
|
+
raise "Unknown local agent parameter: #{k}, expect one of #{DEFAULT_OPTIONS.keys.map(&:to_s).join(',')}" unless DEFAULT_OPTIONS.has_key?(k)
|
|
338
|
+
@options[k]=v
|
|
352
339
|
end
|
|
353
340
|
end
|
|
354
341
|
Log.log.debug("local options= #{options}")
|
|
@@ -367,13 +354,12 @@ module Aspera
|
|
|
367
354
|
start_transfer_with_args_env(session[:env_args],session)
|
|
368
355
|
end
|
|
369
356
|
Log.log.debug('transfer ok'.bg_green)
|
|
370
|
-
rescue => e
|
|
357
|
+
rescue StandardError => e
|
|
371
358
|
session[:error]=e
|
|
372
359
|
Log.log.error("Transfer thread error: #{e.class}:\n#{e.message}:\n#{e.backtrace.join("\n")}".red) if Log.instance.level.eql?(:debug)
|
|
373
360
|
end
|
|
374
361
|
Log.log.debug("EXIT (#{Thread.current[:name]})")
|
|
375
362
|
end
|
|
376
|
-
|
|
377
|
-
end # Local
|
|
363
|
+
end # AgentDirect
|
|
378
364
|
end
|
|
379
365
|
end
|
|
@@ -1,5 +1,6 @@
|
|
|
1
|
-
|
|
2
|
-
require 'aspera/fasp/
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
require 'aspera/fasp/agent_base'
|
|
3
|
+
require 'aspera/fasp/transfer_spec'
|
|
3
4
|
require 'aspera/log'
|
|
4
5
|
require 'aspera/rest'
|
|
5
6
|
require 'websocket-client-simple'
|
|
@@ -12,7 +13,7 @@ require 'json'
|
|
|
12
13
|
module Aspera
|
|
13
14
|
module Fasp
|
|
14
15
|
# start a transfer using Aspera HTTP Gateway, using web socket session
|
|
15
|
-
class
|
|
16
|
+
class AgentHttpgw < AgentBase
|
|
16
17
|
# message returned by HTTP GW in case of success
|
|
17
18
|
OK_MESSAGE='end upload'
|
|
18
19
|
# refresh rate for progress
|
|
@@ -29,7 +30,7 @@ module Aspera
|
|
|
29
30
|
# we need to keep track of actual file path because transfer spec is modified to be sent in web socket
|
|
30
31
|
source_paths=[]
|
|
31
32
|
# get source root or nil
|
|
32
|
-
source_root =
|
|
33
|
+
source_root = transfer_spec.has_key?('source_root') && !transfer_spec['source_root'].empty? ? transfer_spec['source_root'] : nil
|
|
33
34
|
# source root is ignored by GW, used only here
|
|
34
35
|
transfer_spec.delete('source_root')
|
|
35
36
|
# compute total size of files to upload (for progress)
|
|
@@ -61,27 +62,23 @@ module Aspera
|
|
|
61
62
|
received+=1
|
|
62
63
|
else
|
|
63
64
|
message.chomp!
|
|
64
|
-
|
|
65
|
-
error=JSON.parse(Base64.strict_decode64(message.chomp[1..-2]))['message']
|
|
66
|
-
else
|
|
67
|
-
error="expecting quotes in [#{message}]"
|
|
68
|
-
end
|
|
65
|
+
error=message.start_with?('"') && message.end_with?('"') ? JSON.parse(Base64.strict_decode64(message.chomp[1..-2]))['message'] : "expecting quotes in [#{message}]"
|
|
69
66
|
end
|
|
70
67
|
end
|
|
71
68
|
ws.on :error do |e|
|
|
72
69
|
error=e
|
|
73
70
|
end
|
|
74
71
|
ws.on :open do
|
|
75
|
-
Log.log.info(
|
|
72
|
+
Log.log.info('ws: open')
|
|
76
73
|
end
|
|
77
74
|
ws.on :close do
|
|
78
|
-
Log.log.info(
|
|
75
|
+
Log.log.info('ws: close')
|
|
79
76
|
end
|
|
80
77
|
# open web socket to end point
|
|
81
78
|
ws.connect("#{@gw_api.params[:base_url]}/upload")
|
|
82
79
|
# async wait ready
|
|
83
|
-
while !ws.open?
|
|
84
|
-
Log.log.info(
|
|
80
|
+
while !ws.open? && error.nil?
|
|
81
|
+
Log.log.info('ws: wait')
|
|
85
82
|
sleep(0.2)
|
|
86
83
|
end
|
|
87
84
|
# notify progress bar
|
|
@@ -101,11 +98,11 @@ module Aspera
|
|
|
101
98
|
file_size=item['file_size']
|
|
102
99
|
file_name=File.basename(item[item['destination'].nil? ? 'source' : 'destination'])
|
|
103
100
|
# compute total number of slices
|
|
104
|
-
numslices=1+(file_size-1)/@upload_chunksize
|
|
101
|
+
numslices=1+((file_size-1)/@upload_chunksize)
|
|
105
102
|
File.open(source_paths[file_index]) do |file|
|
|
106
103
|
# current slice index
|
|
107
104
|
slicenum=0
|
|
108
|
-
while !file.eof?
|
|
105
|
+
while !file.eof?
|
|
109
106
|
data=file.read(@upload_chunksize)
|
|
110
107
|
slice_data={
|
|
111
108
|
name: file_name,
|
|
@@ -117,11 +114,11 @@ module Aspera
|
|
|
117
114
|
fileIndex: file_index
|
|
118
115
|
}
|
|
119
116
|
# log without data
|
|
120
|
-
Log.dump(:slide_data,slice_data.keys.
|
|
117
|
+
Log.dump(:slide_data,slice_data.keys.each_with_object({}){|i,m|m[i]=i.eql?(:data)?'base64 data':slice_data[i];}) if slicenum.eql?(0)
|
|
121
118
|
ws_send(ws,:slice_upload, slice_data)
|
|
122
119
|
sent_bytes+=data.length
|
|
123
120
|
currenttime=Time.now
|
|
124
|
-
if lastevent.nil?
|
|
121
|
+
if lastevent.nil? || ((currenttime-lastevent)>UPLOAD_REFRESH_SEC)
|
|
125
122
|
notify_progress(session_id,sent_bytes)
|
|
126
123
|
lastevent=currenttime
|
|
127
124
|
end
|
|
@@ -146,40 +143,40 @@ module Aspera
|
|
|
146
143
|
dname=dname.gsub(/\.@gw_api.*$/,'')
|
|
147
144
|
# ands add indication of number of files if there is more than one
|
|
148
145
|
if transfer_spec['paths'].length > 1
|
|
149
|
-
dname
|
|
146
|
+
dname+=" #{transfer_spec['paths'].length} Files"
|
|
150
147
|
end
|
|
151
148
|
transfer_spec['download_name']=dname
|
|
152
149
|
end
|
|
153
150
|
creation=@gw_api.create('download',{'transfer_spec'=>transfer_spec})[:data]
|
|
154
151
|
transfer_uuid=creation['url'].split('/').last
|
|
155
|
-
if transfer_spec['zip_required']
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
152
|
+
file_dest=if transfer_spec['zip_required'] || transfer_spec['paths'].length > 1
|
|
153
|
+
# it is a zip file if zip is required or there is more than 1 file
|
|
154
|
+
transfer_spec['download_name']+'.zip'
|
|
155
|
+
else
|
|
156
|
+
# it is a plain file if we don't require zip and there is only one file
|
|
157
|
+
File.basename(transfer_spec['paths'].first['source'])
|
|
161
158
|
end
|
|
162
159
|
file_dest=File.join(transfer_spec['destination_root'],file_dest)
|
|
163
|
-
@gw_api.call({:
|
|
160
|
+
@gw_api.call({operation: 'GET',subpath: "download/#{transfer_uuid}",save_to_file: file_dest})
|
|
164
161
|
end
|
|
165
162
|
|
|
166
163
|
# start FASP transfer based on transfer spec (hash table)
|
|
167
164
|
# note that it is asynchronous
|
|
168
165
|
# HTTP download only supports file list
|
|
169
166
|
def start_transfer(transfer_spec,options={})
|
|
170
|
-
raise
|
|
171
|
-
raise
|
|
172
|
-
raise
|
|
173
|
-
raise
|
|
167
|
+
raise 'GW URL must be set' if @gw_api.nil?
|
|
168
|
+
raise 'option: must be hash (or nil)' unless options.is_a?(Hash)
|
|
169
|
+
raise 'paths: must be Array' unless transfer_spec['paths'].is_a?(Array)
|
|
170
|
+
raise 'only token based transfer is supported in GW' unless transfer_spec['token'].is_a?(String)
|
|
174
171
|
Log.dump(:user_spec,transfer_spec)
|
|
175
172
|
transfer_spec['authentication']||='token'
|
|
176
173
|
case transfer_spec['direction']
|
|
177
|
-
when
|
|
174
|
+
when Fasp::TransferSpec::DIRECTION_SEND
|
|
178
175
|
upload(transfer_spec)
|
|
179
|
-
when
|
|
176
|
+
when Fasp::TransferSpec::DIRECTION_RECEIVE
|
|
180
177
|
download(transfer_spec)
|
|
181
178
|
else
|
|
182
|
-
raise "
|
|
179
|
+
raise "unexpected direction: [#{transfer_spec['direction']}]"
|
|
183
180
|
end
|
|
184
181
|
end # start_transfer
|
|
185
182
|
|
|
@@ -190,25 +187,22 @@ module Aspera
|
|
|
190
187
|
end
|
|
191
188
|
|
|
192
189
|
# terminates monitor thread
|
|
193
|
-
def shutdown
|
|
194
|
-
end
|
|
190
|
+
def shutdown; end
|
|
195
191
|
|
|
196
|
-
def url=(api_url)
|
|
197
|
-
end
|
|
192
|
+
def url=(api_url); end
|
|
198
193
|
|
|
199
194
|
private
|
|
200
195
|
|
|
201
196
|
def initialize(params)
|
|
202
|
-
raise
|
|
197
|
+
raise 'params must be Hash' unless params.is_a?(Hash)
|
|
203
198
|
params=params.symbolize_keys
|
|
204
|
-
raise
|
|
199
|
+
raise 'must have only one param: url' unless params.keys.eql?([:url])
|
|
205
200
|
super()
|
|
206
|
-
@gw_api=Rest.new({:
|
|
201
|
+
@gw_api=Rest.new({base_url: params[:url]})
|
|
207
202
|
api_info = @gw_api.read('info')[:data]
|
|
208
|
-
Log.log.info(
|
|
209
|
-
@upload_chunksize=
|
|
203
|
+
Log.log.info(api_info.to_s)
|
|
204
|
+
@upload_chunksize=128_000 # TODO: configurable ?
|
|
210
205
|
end
|
|
211
|
-
|
|
212
|
-
end # HttpGW
|
|
206
|
+
end # AgentHttpgw
|
|
213
207
|
end
|
|
214
208
|
end
|
|
@@ -1,4 +1,6 @@
|
|
|
1
|
-
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
require 'aspera/fasp/agent_base'
|
|
3
|
+
require 'aspera/fasp/transfer_spec'
|
|
2
4
|
require 'aspera/log'
|
|
3
5
|
require 'tty-spinner'
|
|
4
6
|
|
|
@@ -6,13 +8,30 @@ module Aspera
|
|
|
6
8
|
module Fasp
|
|
7
9
|
# this singleton class is used by the CLI to provide a common interface to start a transfer
|
|
8
10
|
# before using it, the use must set the `node_api` member.
|
|
9
|
-
class
|
|
11
|
+
class AgentNode < AgentBase
|
|
10
12
|
# option include: root_id if the node is an access key
|
|
11
13
|
attr_writer :options
|
|
12
|
-
def initialize(
|
|
14
|
+
def initialize(options)
|
|
15
|
+
raise 'node specification must be Hash' unless options.is_a?(Hash)
|
|
16
|
+
[:url,:username,:password].each { |k| raise "missing parameter [#{k}] in node specification: #{options}" unless options.has_key?(k) }
|
|
13
17
|
super()
|
|
14
|
-
|
|
15
|
-
@
|
|
18
|
+
# root id is required for access key
|
|
19
|
+
@root_id=options[:root_id]
|
|
20
|
+
rest_params={ base_url: options[:url]}
|
|
21
|
+
if options[:password].match(/^Bearer /)
|
|
22
|
+
rest_params[:headers]={
|
|
23
|
+
'X-Aspera-AccessKey'=>options[:username],
|
|
24
|
+
'Authorization' =>options[:password]
|
|
25
|
+
}
|
|
26
|
+
raise 'root_id is required for access key' if @root_id.nil?
|
|
27
|
+
else
|
|
28
|
+
rest_params[:auth]={
|
|
29
|
+
type: :basic,
|
|
30
|
+
username: options[:username],
|
|
31
|
+
password: options[:password]
|
|
32
|
+
}
|
|
33
|
+
end
|
|
34
|
+
@node_api=Rest.new(rest_params)
|
|
16
35
|
# TODO: currently only supports one transfer. This is bad shortcut. but ok for CLI.
|
|
17
36
|
@transfer_id=nil
|
|
18
37
|
end
|
|
@@ -27,24 +46,24 @@ module Aspera
|
|
|
27
46
|
|
|
28
47
|
# use this to set the node_api end point before using the class.
|
|
29
48
|
def node_api=(new_value)
|
|
30
|
-
if !@node_api.nil?
|
|
49
|
+
if !@node_api.nil? && !new_value.nil?
|
|
31
50
|
Log.log.warn('overriding existing node api value')
|
|
32
51
|
end
|
|
33
52
|
@node_api=new_value
|
|
34
53
|
end
|
|
35
54
|
|
|
36
55
|
# generic method
|
|
37
|
-
def start_transfer(transfer_spec,
|
|
56
|
+
def start_transfer(transfer_spec,_options=nil)
|
|
38
57
|
# add root id if access key
|
|
39
|
-
if
|
|
58
|
+
if !@root_id.nil?
|
|
40
59
|
case transfer_spec['direction']
|
|
41
|
-
when
|
|
42
|
-
when
|
|
60
|
+
when Fasp::TransferSpec::DIRECTION_SEND then transfer_spec['source_root_id']=@root_id
|
|
61
|
+
when Fasp::TransferSpec::DIRECTION_RECEIVE then transfer_spec['destination_root_id']=@root_id
|
|
43
62
|
else raise "unexpected direction in ts: #{transfer_spec['direction']}"
|
|
44
63
|
end
|
|
45
64
|
end
|
|
46
65
|
# manage special additional parameter
|
|
47
|
-
if transfer_spec.has_key?('EX_ssh_key_paths')
|
|
66
|
+
if transfer_spec.has_key?('EX_ssh_key_paths') && transfer_spec['EX_ssh_key_paths'].is_a?(Array) && !transfer_spec['EX_ssh_key_paths'].empty?
|
|
48
67
|
# not standard, so place standard field
|
|
49
68
|
if transfer_spec.has_key?('ssh_private_key')
|
|
50
69
|
Log.log.warn('Both ssh_private_key and EX_ssh_key_paths are present, using ssh_private_key')
|
|
@@ -54,7 +73,7 @@ module Aspera
|
|
|
54
73
|
transfer_spec.delete('EX_ssh_key_paths')
|
|
55
74
|
end
|
|
56
75
|
end
|
|
57
|
-
if transfer_spec['tags'].is_a?(Hash)
|
|
76
|
+
if transfer_spec['tags'].is_a?(Hash) && transfer_spec['tags']['aspera'].is_a?(Hash)
|
|
58
77
|
transfer_spec['tags']['aspera']['xfer_retry']||=150
|
|
59
78
|
end
|
|
60
79
|
# optimisation in case of sending to the same node
|
|
@@ -74,14 +93,14 @@ module Aspera
|
|
|
74
93
|
# lets emulate management events to display progress bar
|
|
75
94
|
loop do
|
|
76
95
|
# status is empty sometimes with status 200...
|
|
77
|
-
trdata=node_api_.read("ops/transfers/#{@transfer_id}")[:data] || {
|
|
96
|
+
trdata=node_api_.read("ops/transfers/#{@transfer_id}")[:data] || {'status'=>'unknown'} rescue {'status'=>'waiting(read error)'}
|
|
78
97
|
case trdata['status']
|
|
79
98
|
when 'completed'
|
|
80
99
|
notify_end(@transfer_id)
|
|
81
100
|
break
|
|
82
101
|
when 'waiting','partially_completed','unknown','waiting(read error)'
|
|
83
102
|
if spinner.nil?
|
|
84
|
-
spinner = TTY::Spinner.new(
|
|
103
|
+
spinner = TTY::Spinner.new('[:spinner] :title', format: :classic)
|
|
85
104
|
spinner.start
|
|
86
105
|
end
|
|
87
106
|
spinner.update(title: trdata['status'])
|
|
@@ -89,7 +108,7 @@ module Aspera
|
|
|
89
108
|
#puts trdata
|
|
90
109
|
when 'running'
|
|
91
110
|
#puts "running: sessions:#{trdata["sessions"].length}, #{trdata["sessions"].map{|i| i['bytes_transferred']}.join(',')}"
|
|
92
|
-
if !started
|
|
111
|
+
if !started && trdata['precalc'].is_a?(Hash) &&
|
|
93
112
|
trdata['precalc']['status'].eql?('ready')
|
|
94
113
|
notify_begin(@transfer_id,trdata['precalc']['bytes_expected'])
|
|
95
114
|
started=true
|
|
@@ -98,7 +117,7 @@ module Aspera
|
|
|
98
117
|
end
|
|
99
118
|
else
|
|
100
119
|
Log.log.warn("trdata -> #{trdata}")
|
|
101
|
-
raise Fasp::Error
|
|
120
|
+
raise Fasp::Error, "#{trdata['status']}: #{trdata['error_desc']}"
|
|
102
121
|
end
|
|
103
122
|
sleep 1
|
|
104
123
|
end
|