aspera-cli 4.4.0 → 4.7.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +2095 -1503
- data/bin/ascli +2 -1
- data/bin/asession +4 -5
- data/docs/test_env.conf +3 -0
- data/examples/aoc.rb +4 -3
- data/examples/faspex4.rb +25 -25
- data/examples/proxy.pac +1 -1
- data/examples/transfer.rb +17 -17
- data/lib/aspera/aoc.rb +238 -185
- data/lib/aspera/ascmd.rb +93 -83
- data/lib/aspera/ats_api.rb +11 -10
- data/lib/aspera/cli/basic_auth_plugin.rb +13 -14
- data/lib/aspera/cli/extended_value.rb +42 -33
- data/lib/aspera/cli/formater.rb +142 -108
- data/lib/aspera/cli/info.rb +17 -0
- data/lib/aspera/cli/listener/line_dump.rb +3 -2
- data/lib/aspera/cli/listener/logger.rb +2 -1
- data/lib/aspera/cli/listener/progress.rb +16 -18
- data/lib/aspera/cli/listener/progress_multi.rb +18 -21
- data/lib/aspera/cli/main.rb +173 -149
- data/lib/aspera/cli/manager.rb +163 -168
- data/lib/aspera/cli/plugin.rb +43 -31
- data/lib/aspera/cli/plugins/alee.rb +6 -6
- data/lib/aspera/cli/plugins/aoc.rb +405 -370
- data/lib/aspera/cli/plugins/ats.rb +86 -79
- data/lib/aspera/cli/plugins/bss.rb +14 -16
- data/lib/aspera/cli/plugins/config.rb +580 -362
- data/lib/aspera/cli/plugins/console.rb +23 -19
- data/lib/aspera/cli/plugins/cos.rb +18 -18
- data/lib/aspera/cli/plugins/faspex.rb +201 -158
- data/lib/aspera/cli/plugins/faspex5.rb +80 -57
- data/lib/aspera/cli/plugins/node.rb +183 -166
- data/lib/aspera/cli/plugins/orchestrator.rb +71 -67
- data/lib/aspera/cli/plugins/preview.rb +92 -96
- data/lib/aspera/cli/plugins/server.rb +79 -75
- data/lib/aspera/cli/plugins/shares.rb +35 -19
- data/lib/aspera/cli/plugins/sync.rb +20 -22
- data/lib/aspera/cli/transfer_agent.rb +76 -113
- data/lib/aspera/cli/version.rb +2 -1
- data/lib/aspera/colors.rb +35 -27
- data/lib/aspera/command_line_builder.rb +48 -34
- data/lib/aspera/cos_node.rb +29 -21
- data/lib/aspera/data_repository.rb +3 -2
- data/lib/aspera/environment.rb +50 -45
- data/lib/aspera/fasp/{manager.rb → agent_base.rb} +28 -25
- data/lib/aspera/fasp/{connect.rb → agent_connect.rb} +52 -43
- data/lib/aspera/fasp/{local.rb → agent_direct.rb} +58 -72
- data/lib/aspera/fasp/{http_gw.rb → agent_httpgw.rb} +37 -43
- data/lib/aspera/fasp/{node.rb → agent_node.rb} +35 -16
- data/lib/aspera/fasp/agent_trsdk.rb +104 -0
- data/lib/aspera/fasp/error.rb +2 -1
- data/lib/aspera/fasp/error_info.rb +68 -52
- data/lib/aspera/fasp/installation.rb +152 -124
- data/lib/aspera/fasp/listener.rb +1 -0
- data/lib/aspera/fasp/parameters.rb +87 -92
- data/lib/aspera/fasp/parameters.yaml +305 -249
- data/lib/aspera/fasp/resume_policy.rb +11 -14
- data/lib/aspera/fasp/transfer_spec.rb +26 -0
- data/lib/aspera/fasp/uri.rb +22 -21
- data/lib/aspera/faspex_gw.rb +55 -89
- data/lib/aspera/hash_ext.rb +4 -3
- data/lib/aspera/id_generator.rb +8 -7
- data/lib/aspera/keychain/encrypted_hash.rb +121 -0
- data/lib/aspera/keychain/macos_security.rb +90 -0
- data/lib/aspera/log.rb +55 -37
- data/lib/aspera/nagios.rb +13 -12
- data/lib/aspera/node.rb +30 -25
- data/lib/aspera/oauth.rb +175 -226
- data/lib/aspera/open_application.rb +4 -3
- data/lib/aspera/persistency_action_once.rb +6 -6
- data/lib/aspera/persistency_folder.rb +5 -9
- data/lib/aspera/preview/file_types.rb +6 -5
- data/lib/aspera/preview/generator.rb +25 -24
- data/lib/aspera/preview/options.rb +16 -14
- data/lib/aspera/preview/utils.rb +98 -98
- data/lib/aspera/{proxy_auto_config.erb.js → proxy_auto_config.js} +23 -31
- data/lib/aspera/proxy_auto_config.rb +111 -20
- data/lib/aspera/rest.rb +154 -135
- data/lib/aspera/rest_call_error.rb +2 -2
- data/lib/aspera/rest_error_analyzer.rb +23 -25
- data/lib/aspera/rest_errors_aspera.rb +15 -14
- data/lib/aspera/ssh.rb +12 -10
- data/lib/aspera/sync.rb +42 -41
- data/lib/aspera/temp_file_manager.rb +18 -14
- data/lib/aspera/timer_limiter.rb +2 -1
- data/lib/aspera/uri_reader.rb +7 -5
- data/lib/aspera/web_auth.rb +79 -76
- metadata +116 -29
- data/docs/Makefile +0 -66
- data/docs/README.erb.md +0 -3973
- data/docs/README.md +0 -13
- data/docs/diagrams.txt +0 -49
- data/docs/doc_tools.rb +0 -58
- data/lib/aspera/api_detector.rb +0 -60
- data/lib/aspera/cli/plugins/shares2.rb +0 -114
- data/lib/aspera/secrets.rb +0 -20
@@ -1,36 +1,31 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
|
4
|
-
# Aspera 2016
|
5
|
-
# Laurent Martin
|
6
|
-
#
|
7
|
-
##############################################################################
|
8
|
-
require 'aspera/fasp/manager'
|
1
|
+
# frozen_string_literal: true
|
2
|
+
require 'English'
|
3
|
+
require 'aspera/fasp/agent_base'
|
9
4
|
require 'aspera/fasp/error'
|
10
5
|
require 'aspera/fasp/parameters'
|
11
6
|
require 'aspera/fasp/installation'
|
12
7
|
require 'aspera/fasp/resume_policy'
|
8
|
+
require 'aspera/fasp/transfer_spec'
|
13
9
|
require 'aspera/log'
|
14
10
|
require 'socket'
|
15
11
|
require 'timeout'
|
16
12
|
require 'securerandom'
|
13
|
+
require 'shellwords'
|
17
14
|
|
18
15
|
module Aspera
|
19
16
|
module Fasp
|
20
17
|
# executes a local "ascp", connects mgt port, equivalent of "Fasp Manager"
|
21
|
-
class
|
18
|
+
class AgentDirect < AgentBase
|
22
19
|
# options for initialize (same as values in option transfer_info)
|
23
20
|
DEFAULT_OPTIONS = {
|
24
|
-
:
|
25
|
-
:
|
26
|
-
:
|
27
|
-
:
|
28
|
-
:
|
21
|
+
spawn_timeout_sec: 3,
|
22
|
+
spawn_delay_sec: 2,
|
23
|
+
wss: false,
|
24
|
+
multi_incr_udp: true,
|
25
|
+
resume: {},
|
26
|
+
quiet: true # by default no interactive progress bar
|
29
27
|
}
|
30
|
-
DEFAULT_UDP_PORT=33001
|
31
28
|
private_constant :DEFAULT_OPTIONS
|
32
|
-
# set to false to keep ascp progress bar display ("true" adds ascp's option -q)
|
33
|
-
attr_accessor :quiet
|
34
29
|
|
35
30
|
# start ascp transfer (non blocking), single or multi-session
|
36
31
|
# job information added to @jobs
|
@@ -44,10 +39,11 @@ module Aspera
|
|
44
39
|
# clone transfer spec because we modify it (first level keys)
|
45
40
|
transfer_spec=transfer_spec.clone
|
46
41
|
# if there is aspera tags
|
47
|
-
if transfer_spec['tags'].is_a?(Hash)
|
42
|
+
if transfer_spec['tags'].is_a?(Hash) && transfer_spec['tags']['aspera'].is_a?(Hash)
|
48
43
|
# TODO: what is this for ? only on local ascp ?
|
49
44
|
# NOTE: important: transfer id must be unique: generate random id
|
50
45
|
# using a non unique id results in discard of tags in AoC, and a package is never finalized
|
46
|
+
# all sessions in a multi-session transfer must have the same xfer_id (see admin manual)
|
51
47
|
transfer_spec['tags']['aspera']['xfer_id']||=SecureRandom.uuid
|
52
48
|
Log.log.debug("xfer id=#{transfer_spec['xfer_id']}")
|
53
49
|
# TODO: useful ? node only ?
|
@@ -56,8 +52,8 @@ module Aspera
|
|
56
52
|
Log.dump('ts',transfer_spec)
|
57
53
|
|
58
54
|
# add bypass keys when authentication is token and no auth is provided
|
59
|
-
if transfer_spec.has_key?('token')
|
60
|
-
!transfer_spec.has_key?('remote_password')
|
55
|
+
if transfer_spec.has_key?('token') &&
|
56
|
+
!transfer_spec.has_key?('remote_password') &&
|
61
57
|
!transfer_spec.has_key?('EX_ssh_key_paths')
|
62
58
|
# transfer_spec['remote_password'] = Installation.instance.bypass_pass # not used
|
63
59
|
transfer_spec['EX_ssh_key_paths'] = Installation.instance.bypass_keys
|
@@ -68,24 +64,22 @@ module Aspera
|
|
68
64
|
multi_session_info=nil
|
69
65
|
if transfer_spec.has_key?('multi_session')
|
70
66
|
multi_session_info={
|
71
|
-
count: transfer_spec['multi_session'].to_i
|
67
|
+
count: transfer_spec['multi_session'].to_i
|
72
68
|
}
|
73
69
|
# Managed by multi-session, so delete from transfer spec
|
74
70
|
transfer_spec.delete('multi_session')
|
75
|
-
if multi_session_info[:count]
|
71
|
+
if multi_session_info[:count].negative?
|
76
72
|
Log.log.error("multi_session(#{transfer_spec['multi_session']}) shall be integer >= 0")
|
77
73
|
multi_session_info = nil
|
78
74
|
elsif multi_session_info[:count].eql?(0)
|
79
|
-
Log.log.debug(
|
75
|
+
Log.log.debug('multi_session count is zero: no multisession')
|
80
76
|
multi_session_info = nil
|
81
|
-
|
77
|
+
elsif @options[:multi_incr_udp] # multi_session_info[:count] > 0
|
82
78
|
# if option not true: keep default udp port for all sessions
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
transfer_spec.delete('fasp_port')
|
88
|
-
end
|
79
|
+
multi_session_info[:udp_base]=transfer_spec.has_key?('fasp_port') ? transfer_spec['fasp_port'] : TransferSpec::UDP_PORT
|
80
|
+
# delete from original transfer spec, as we will increment values
|
81
|
+
transfer_spec.delete('fasp_port')
|
82
|
+
# override if specified, else use default value
|
89
83
|
end
|
90
84
|
end
|
91
85
|
|
@@ -98,22 +92,22 @@ module Aspera
|
|
98
92
|
env_args[:args].unshift('-I',Installation.instance.path(:fallback_cert))
|
99
93
|
end
|
100
94
|
|
101
|
-
env_args[:args].unshift('-q') if @quiet
|
95
|
+
env_args[:args].unshift('-q') if @options[:quiet]
|
102
96
|
|
103
97
|
# transfer job can be multi session
|
104
98
|
xfer_job={
|
105
|
-
:
|
106
|
-
:
|
99
|
+
id: job_options[:job_id],
|
100
|
+
sessions: [] # all sessions as below
|
107
101
|
}
|
108
102
|
|
109
103
|
# generic session information
|
110
104
|
session={
|
111
|
-
:
|
112
|
-
:
|
113
|
-
:
|
114
|
-
:
|
115
|
-
:env_args
|
116
|
-
:
|
105
|
+
thread: nil, # Thread object monitoring management port, not nil when pushed to :sessions
|
106
|
+
error: nil, # exception if failed
|
107
|
+
io: nil, # management port server socket
|
108
|
+
id: nil, # SessionId from INIT message in mgt port
|
109
|
+
env_args: env_args, # env vars and args to ascp (from transfer spec)
|
110
|
+
options: job_options # [Hash]
|
117
111
|
}
|
118
112
|
|
119
113
|
if multi_session_info.nil?
|
@@ -132,7 +126,7 @@ module Aspera
|
|
132
126
|
this_session[:env_args][:args]=this_session[:env_args][:args].clone()
|
133
127
|
this_session[:env_args][:args].unshift("-C#{i}:#{multi_session_info[:count]}")
|
134
128
|
# option: increment (default as per ascp manual) or not (cluster on other side ?)
|
135
|
-
this_session[:env_args][:args].unshift('-O',
|
129
|
+
this_session[:env_args][:args].unshift('-O',(multi_session_info[:udp_base]+i-1).to_s) if @options[:multi_incr_udp]
|
136
130
|
this_session[:thread] = Thread.new(this_session) {|s|transfer_thread_entry(s)}
|
137
131
|
xfer_job[:sessions].push(this_session)
|
138
132
|
end
|
@@ -152,11 +146,11 @@ module Aspera
|
|
152
146
|
Log.log.debug('wait_for_transfers_completion')
|
153
147
|
# set to non-nil to exit loop
|
154
148
|
result=[]
|
155
|
-
@jobs.each do |
|
149
|
+
@jobs.each do |_id,job|
|
156
150
|
job[:sessions].each do |session|
|
157
151
|
Log.log.debug("join #{session[:thread]}")
|
158
152
|
session[:thread].join
|
159
|
-
result.push(session[:error]
|
153
|
+
result.push(session[:error] || :success)
|
160
154
|
end
|
161
155
|
end
|
162
156
|
Log.log.debug('all transfers joined')
|
@@ -190,7 +184,7 @@ module Aspera
|
|
190
184
|
Fasp::Installation.instance.path(env_args[:ascp_version])
|
191
185
|
end
|
192
186
|
# (optional) check it exists
|
193
|
-
raise Fasp::Error
|
187
|
+
raise Fasp::Error, "no such file: #{ascp_path}" unless File.exist?(ascp_path)
|
194
188
|
# open random local TCP port for listening for ascp management
|
195
189
|
mgt_sock = TCPServer.new('127.0.0.1',0)
|
196
190
|
# clone arguments as we eed to modify with mgt port
|
@@ -198,7 +192,7 @@ module Aspera
|
|
198
192
|
# add management port
|
199
193
|
ascp_arguments.unshift('-M', mgt_sock.addr[1].to_s)
|
200
194
|
# start ascp in sub process
|
201
|
-
Log.log.debug("execute: #{env_args[:env].map{|k,v| "#{k}
|
195
|
+
Log.log.debug("execute: #{env_args[:env].map{|k,v| "#{k}=#{Shellwords.shellescape(v)}"}.join(' ')} #{Shellwords.shellescape(ascp_path)} #{ascp_arguments.map{|a|Shellwords.shellescape(a)}.join(' ')}")
|
202
196
|
# start process
|
203
197
|
ascp_pid = Process.spawn(env_args[:env],[ascp_path,ascp_path],*ascp_arguments)
|
204
198
|
# in parent, wait for connection to socket max 3 seconds
|
@@ -226,21 +220,21 @@ module Aspera
|
|
226
220
|
line = ascp_mgt_io.gets
|
227
221
|
# nil when ascp process exits
|
228
222
|
break if line.nil?
|
229
|
-
current_event_text
|
223
|
+
current_event_text+=line
|
230
224
|
line.chomp!
|
231
225
|
Log.log.debug("line=[#{line}]")
|
232
226
|
case line
|
233
227
|
when 'FASPMGR 2'
|
234
228
|
# begin event
|
235
|
-
current_event_data =
|
229
|
+
current_event_data = {}
|
236
230
|
current_event_text = ''
|
237
231
|
when /^([^:]+): (.*)$/
|
238
232
|
# event field
|
239
|
-
current_event_data[
|
233
|
+
current_event_data[Regexp.last_match(1)] = Regexp.last_match(2)
|
240
234
|
when ''
|
241
235
|
# empty line is separator to end event information
|
242
236
|
raise 'unexpected empty line' if current_event_data.nil?
|
243
|
-
current_event_data[
|
237
|
+
current_event_data[AgentBase::LISTENER_SESSION_ID_B]=ascp_pid
|
244
238
|
notify_listeners(current_event_text,current_event_data)
|
245
239
|
case current_event_data['Type']
|
246
240
|
when 'INIT'
|
@@ -262,9 +256,9 @@ module Aspera
|
|
262
256
|
exception_raised=false
|
263
257
|
when 'ERROR'
|
264
258
|
Log.log.error("code: #{last_status_event['Code']}")
|
265
|
-
if last_status_event['Description']
|
259
|
+
if last_status_event['Description'] =~ /bearer token/i
|
266
260
|
Log.log.error('need to regenerate token'.red)
|
267
|
-
if session[:options].is_a?(Hash)
|
261
|
+
if session[:options].is_a?(Hash) && session[:options].has_key?(:regenerate_token)
|
268
262
|
# regenerate token here, expired, or error on it
|
269
263
|
# Note: in multi-session, each session will have a different one.
|
270
264
|
env_args[:env]['ASPERA_SCP_TOKEN']=session[:options][:regenerate_token].call(true)
|
@@ -280,27 +274,25 @@ module Aspera
|
|
280
274
|
end
|
281
275
|
rescue SystemCallError => e
|
282
276
|
# Process.spawn
|
283
|
-
raise Fasp::Error
|
284
|
-
rescue Timeout::Error
|
285
|
-
raise Fasp::Error
|
286
|
-
rescue Interrupt
|
287
|
-
raise Fasp::Error
|
277
|
+
raise Fasp::Error, e.message
|
278
|
+
rescue Timeout::Error
|
279
|
+
raise Fasp::Error, 'timeout waiting mgt port connect'
|
280
|
+
rescue Interrupt
|
281
|
+
raise Fasp::Error, 'transfer interrupted by user'
|
288
282
|
ensure
|
289
283
|
# if ascp was successfully started
|
290
284
|
unless ascp_pid.nil?
|
291
285
|
# "wait" for process to avoid zombie
|
292
286
|
Process.wait(ascp_pid)
|
293
|
-
status
|
287
|
+
status=$CHILD_STATUS
|
294
288
|
ascp_pid=nil
|
295
289
|
session.delete(:io)
|
296
290
|
if !status.success?
|
297
291
|
message="ascp failed with code #{status.exitstatus}"
|
298
|
-
if
|
299
|
-
|
300
|
-
|
301
|
-
|
302
|
-
raise Fasp::Error.new(message)
|
303
|
-
end
|
292
|
+
# raise error only if there was not already an exception
|
293
|
+
raise Fasp::Error, message unless exception_raised
|
294
|
+
# else just debug, as main exception is already here
|
295
|
+
Log.log.debug(message)
|
304
296
|
end
|
305
297
|
end
|
306
298
|
end # begin-ensure
|
@@ -333,8 +325,6 @@ module Aspera
|
|
333
325
|
# @param options : keys(symbol): see DEFAULT_OPTIONS
|
334
326
|
def initialize(options=nil)
|
335
327
|
super()
|
336
|
-
# by default no interactive progress bar
|
337
|
-
@quiet=true
|
338
328
|
# all transfer jobs, key = SecureRandom.uuid, protected by mutex, condvar on change
|
339
329
|
@jobs={}
|
340
330
|
# mutex protects global data accessed by threads
|
@@ -344,11 +334,8 @@ module Aspera
|
|
344
334
|
if !options.nil?
|
345
335
|
raise "expecting Hash (or nil), but have #{options.class}" unless options.is_a?(Hash)
|
346
336
|
options.each do |k,v|
|
347
|
-
|
348
|
-
|
349
|
-
else
|
350
|
-
raise "Unknown local agent parameter: #{k}, expect one of #{DEFAULT_OPTIONS.keys.map{|i|i.to_s}.join(",")}"
|
351
|
-
end
|
337
|
+
raise "Unknown local agent parameter: #{k}, expect one of #{DEFAULT_OPTIONS.keys.map(&:to_s).join(',')}" unless DEFAULT_OPTIONS.has_key?(k)
|
338
|
+
@options[k]=v
|
352
339
|
end
|
353
340
|
end
|
354
341
|
Log.log.debug("local options= #{options}")
|
@@ -367,13 +354,12 @@ module Aspera
|
|
367
354
|
start_transfer_with_args_env(session[:env_args],session)
|
368
355
|
end
|
369
356
|
Log.log.debug('transfer ok'.bg_green)
|
370
|
-
rescue => e
|
357
|
+
rescue StandardError => e
|
371
358
|
session[:error]=e
|
372
359
|
Log.log.error("Transfer thread error: #{e.class}:\n#{e.message}:\n#{e.backtrace.join("\n")}".red) if Log.instance.level.eql?(:debug)
|
373
360
|
end
|
374
361
|
Log.log.debug("EXIT (#{Thread.current[:name]})")
|
375
362
|
end
|
376
|
-
|
377
|
-
end # Local
|
363
|
+
end # AgentDirect
|
378
364
|
end
|
379
365
|
end
|
@@ -1,5 +1,6 @@
|
|
1
|
-
|
2
|
-
require 'aspera/fasp/
|
1
|
+
# frozen_string_literal: true
|
2
|
+
require 'aspera/fasp/agent_base'
|
3
|
+
require 'aspera/fasp/transfer_spec'
|
3
4
|
require 'aspera/log'
|
4
5
|
require 'aspera/rest'
|
5
6
|
require 'websocket-client-simple'
|
@@ -12,7 +13,7 @@ require 'json'
|
|
12
13
|
module Aspera
|
13
14
|
module Fasp
|
14
15
|
# start a transfer using Aspera HTTP Gateway, using web socket session
|
15
|
-
class
|
16
|
+
class AgentHttpgw < AgentBase
|
16
17
|
# message returned by HTTP GW in case of success
|
17
18
|
OK_MESSAGE='end upload'
|
18
19
|
# refresh rate for progress
|
@@ -29,7 +30,7 @@ module Aspera
|
|
29
30
|
# we need to keep track of actual file path because transfer spec is modified to be sent in web socket
|
30
31
|
source_paths=[]
|
31
32
|
# get source root or nil
|
32
|
-
source_root =
|
33
|
+
source_root = transfer_spec.has_key?('source_root') && !transfer_spec['source_root'].empty? ? transfer_spec['source_root'] : nil
|
33
34
|
# source root is ignored by GW, used only here
|
34
35
|
transfer_spec.delete('source_root')
|
35
36
|
# compute total size of files to upload (for progress)
|
@@ -61,27 +62,23 @@ module Aspera
|
|
61
62
|
received+=1
|
62
63
|
else
|
63
64
|
message.chomp!
|
64
|
-
|
65
|
-
error=JSON.parse(Base64.strict_decode64(message.chomp[1..-2]))['message']
|
66
|
-
else
|
67
|
-
error="expecting quotes in [#{message}]"
|
68
|
-
end
|
65
|
+
error=message.start_with?('"') && message.end_with?('"') ? JSON.parse(Base64.strict_decode64(message.chomp[1..-2]))['message'] : "expecting quotes in [#{message}]"
|
69
66
|
end
|
70
67
|
end
|
71
68
|
ws.on :error do |e|
|
72
69
|
error=e
|
73
70
|
end
|
74
71
|
ws.on :open do
|
75
|
-
Log.log.info(
|
72
|
+
Log.log.info('ws: open')
|
76
73
|
end
|
77
74
|
ws.on :close do
|
78
|
-
Log.log.info(
|
75
|
+
Log.log.info('ws: close')
|
79
76
|
end
|
80
77
|
# open web socket to end point
|
81
78
|
ws.connect("#{@gw_api.params[:base_url]}/upload")
|
82
79
|
# async wait ready
|
83
|
-
while !ws.open?
|
84
|
-
Log.log.info(
|
80
|
+
while !ws.open? && error.nil?
|
81
|
+
Log.log.info('ws: wait')
|
85
82
|
sleep(0.2)
|
86
83
|
end
|
87
84
|
# notify progress bar
|
@@ -101,11 +98,11 @@ module Aspera
|
|
101
98
|
file_size=item['file_size']
|
102
99
|
file_name=File.basename(item[item['destination'].nil? ? 'source' : 'destination'])
|
103
100
|
# compute total number of slices
|
104
|
-
numslices=1+(file_size-1)/@upload_chunksize
|
101
|
+
numslices=1+((file_size-1)/@upload_chunksize)
|
105
102
|
File.open(source_paths[file_index]) do |file|
|
106
103
|
# current slice index
|
107
104
|
slicenum=0
|
108
|
-
while !file.eof?
|
105
|
+
while !file.eof?
|
109
106
|
data=file.read(@upload_chunksize)
|
110
107
|
slice_data={
|
111
108
|
name: file_name,
|
@@ -117,11 +114,11 @@ module Aspera
|
|
117
114
|
fileIndex: file_index
|
118
115
|
}
|
119
116
|
# log without data
|
120
|
-
Log.dump(:slide_data,slice_data.keys.
|
117
|
+
Log.dump(:slide_data,slice_data.keys.each_with_object({}){|i,m|m[i]=i.eql?(:data)?'base64 data':slice_data[i];}) if slicenum.eql?(0)
|
121
118
|
ws_send(ws,:slice_upload, slice_data)
|
122
119
|
sent_bytes+=data.length
|
123
120
|
currenttime=Time.now
|
124
|
-
if lastevent.nil?
|
121
|
+
if lastevent.nil? || ((currenttime-lastevent)>UPLOAD_REFRESH_SEC)
|
125
122
|
notify_progress(session_id,sent_bytes)
|
126
123
|
lastevent=currenttime
|
127
124
|
end
|
@@ -146,40 +143,40 @@ module Aspera
|
|
146
143
|
dname=dname.gsub(/\.@gw_api.*$/,'')
|
147
144
|
# ands add indication of number of files if there is more than one
|
148
145
|
if transfer_spec['paths'].length > 1
|
149
|
-
dname
|
146
|
+
dname+=" #{transfer_spec['paths'].length} Files"
|
150
147
|
end
|
151
148
|
transfer_spec['download_name']=dname
|
152
149
|
end
|
153
150
|
creation=@gw_api.create('download',{'transfer_spec'=>transfer_spec})[:data]
|
154
151
|
transfer_uuid=creation['url'].split('/').last
|
155
|
-
if transfer_spec['zip_required']
|
156
|
-
|
157
|
-
|
158
|
-
|
159
|
-
|
160
|
-
|
152
|
+
file_dest=if transfer_spec['zip_required'] || transfer_spec['paths'].length > 1
|
153
|
+
# it is a zip file if zip is required or there is more than 1 file
|
154
|
+
transfer_spec['download_name']+'.zip'
|
155
|
+
else
|
156
|
+
# it is a plain file if we don't require zip and there is only one file
|
157
|
+
File.basename(transfer_spec['paths'].first['source'])
|
161
158
|
end
|
162
159
|
file_dest=File.join(transfer_spec['destination_root'],file_dest)
|
163
|
-
@gw_api.call({:
|
160
|
+
@gw_api.call({operation: 'GET',subpath: "download/#{transfer_uuid}",save_to_file: file_dest})
|
164
161
|
end
|
165
162
|
|
166
163
|
# start FASP transfer based on transfer spec (hash table)
|
167
164
|
# note that it is asynchronous
|
168
165
|
# HTTP download only supports file list
|
169
166
|
def start_transfer(transfer_spec,options={})
|
170
|
-
raise
|
171
|
-
raise
|
172
|
-
raise
|
173
|
-
raise
|
167
|
+
raise 'GW URL must be set' if @gw_api.nil?
|
168
|
+
raise 'option: must be hash (or nil)' unless options.is_a?(Hash)
|
169
|
+
raise 'paths: must be Array' unless transfer_spec['paths'].is_a?(Array)
|
170
|
+
raise 'only token based transfer is supported in GW' unless transfer_spec['token'].is_a?(String)
|
174
171
|
Log.dump(:user_spec,transfer_spec)
|
175
172
|
transfer_spec['authentication']||='token'
|
176
173
|
case transfer_spec['direction']
|
177
|
-
when
|
174
|
+
when Fasp::TransferSpec::DIRECTION_SEND
|
178
175
|
upload(transfer_spec)
|
179
|
-
when
|
176
|
+
when Fasp::TransferSpec::DIRECTION_RECEIVE
|
180
177
|
download(transfer_spec)
|
181
178
|
else
|
182
|
-
raise "
|
179
|
+
raise "unexpected direction: [#{transfer_spec['direction']}]"
|
183
180
|
end
|
184
181
|
end # start_transfer
|
185
182
|
|
@@ -190,25 +187,22 @@ module Aspera
|
|
190
187
|
end
|
191
188
|
|
192
189
|
# terminates monitor thread
|
193
|
-
def shutdown
|
194
|
-
end
|
190
|
+
def shutdown; end
|
195
191
|
|
196
|
-
def url=(api_url)
|
197
|
-
end
|
192
|
+
def url=(api_url); end
|
198
193
|
|
199
194
|
private
|
200
195
|
|
201
196
|
def initialize(params)
|
202
|
-
raise
|
197
|
+
raise 'params must be Hash' unless params.is_a?(Hash)
|
203
198
|
params=params.symbolize_keys
|
204
|
-
raise
|
199
|
+
raise 'must have only one param: url' unless params.keys.eql?([:url])
|
205
200
|
super()
|
206
|
-
@gw_api=Rest.new({:
|
201
|
+
@gw_api=Rest.new({base_url: params[:url]})
|
207
202
|
api_info = @gw_api.read('info')[:data]
|
208
|
-
Log.log.info(
|
209
|
-
@upload_chunksize=
|
203
|
+
Log.log.info(api_info.to_s)
|
204
|
+
@upload_chunksize=128_000 # TODO: configurable ?
|
210
205
|
end
|
211
|
-
|
212
|
-
end # HttpGW
|
206
|
+
end # AgentHttpgw
|
213
207
|
end
|
214
208
|
end
|
@@ -1,4 +1,6 @@
|
|
1
|
-
|
1
|
+
# frozen_string_literal: true
|
2
|
+
require 'aspera/fasp/agent_base'
|
3
|
+
require 'aspera/fasp/transfer_spec'
|
2
4
|
require 'aspera/log'
|
3
5
|
require 'tty-spinner'
|
4
6
|
|
@@ -6,13 +8,30 @@ module Aspera
|
|
6
8
|
module Fasp
|
7
9
|
# this singleton class is used by the CLI to provide a common interface to start a transfer
|
8
10
|
# before using it, the use must set the `node_api` member.
|
9
|
-
class
|
11
|
+
class AgentNode < AgentBase
|
10
12
|
# option include: root_id if the node is an access key
|
11
13
|
attr_writer :options
|
12
|
-
def initialize(
|
14
|
+
def initialize(options)
|
15
|
+
raise 'node specification must be Hash' unless options.is_a?(Hash)
|
16
|
+
[:url,:username,:password].each { |k| raise "missing parameter [#{k}] in node specification: #{options}" unless options.has_key?(k) }
|
13
17
|
super()
|
14
|
-
|
15
|
-
@
|
18
|
+
# root id is required for access key
|
19
|
+
@root_id=options[:root_id]
|
20
|
+
rest_params={ base_url: options[:url]}
|
21
|
+
if options[:password].match(/^Bearer /)
|
22
|
+
rest_params[:headers]={
|
23
|
+
'X-Aspera-AccessKey'=>options[:username],
|
24
|
+
'Authorization' =>options[:password]
|
25
|
+
}
|
26
|
+
raise 'root_id is required for access key' if @root_id.nil?
|
27
|
+
else
|
28
|
+
rest_params[:auth]={
|
29
|
+
type: :basic,
|
30
|
+
username: options[:username],
|
31
|
+
password: options[:password]
|
32
|
+
}
|
33
|
+
end
|
34
|
+
@node_api=Rest.new(rest_params)
|
16
35
|
# TODO: currently only supports one transfer. This is bad shortcut. but ok for CLI.
|
17
36
|
@transfer_id=nil
|
18
37
|
end
|
@@ -27,24 +46,24 @@ module Aspera
|
|
27
46
|
|
28
47
|
# use this to set the node_api end point before using the class.
|
29
48
|
def node_api=(new_value)
|
30
|
-
if !@node_api.nil?
|
49
|
+
if !@node_api.nil? && !new_value.nil?
|
31
50
|
Log.log.warn('overriding existing node api value')
|
32
51
|
end
|
33
52
|
@node_api=new_value
|
34
53
|
end
|
35
54
|
|
36
55
|
# generic method
|
37
|
-
def start_transfer(transfer_spec,
|
56
|
+
def start_transfer(transfer_spec,_options=nil)
|
38
57
|
# add root id if access key
|
39
|
-
if
|
58
|
+
if !@root_id.nil?
|
40
59
|
case transfer_spec['direction']
|
41
|
-
when
|
42
|
-
when
|
60
|
+
when Fasp::TransferSpec::DIRECTION_SEND then transfer_spec['source_root_id']=@root_id
|
61
|
+
when Fasp::TransferSpec::DIRECTION_RECEIVE then transfer_spec['destination_root_id']=@root_id
|
43
62
|
else raise "unexpected direction in ts: #{transfer_spec['direction']}"
|
44
63
|
end
|
45
64
|
end
|
46
65
|
# manage special additional parameter
|
47
|
-
if transfer_spec.has_key?('EX_ssh_key_paths')
|
66
|
+
if transfer_spec.has_key?('EX_ssh_key_paths') && transfer_spec['EX_ssh_key_paths'].is_a?(Array) && !transfer_spec['EX_ssh_key_paths'].empty?
|
48
67
|
# not standard, so place standard field
|
49
68
|
if transfer_spec.has_key?('ssh_private_key')
|
50
69
|
Log.log.warn('Both ssh_private_key and EX_ssh_key_paths are present, using ssh_private_key')
|
@@ -54,7 +73,7 @@ module Aspera
|
|
54
73
|
transfer_spec.delete('EX_ssh_key_paths')
|
55
74
|
end
|
56
75
|
end
|
57
|
-
if transfer_spec['tags'].is_a?(Hash)
|
76
|
+
if transfer_spec['tags'].is_a?(Hash) && transfer_spec['tags']['aspera'].is_a?(Hash)
|
58
77
|
transfer_spec['tags']['aspera']['xfer_retry']||=150
|
59
78
|
end
|
60
79
|
# optimisation in case of sending to the same node
|
@@ -74,14 +93,14 @@ module Aspera
|
|
74
93
|
# lets emulate management events to display progress bar
|
75
94
|
loop do
|
76
95
|
# status is empty sometimes with status 200...
|
77
|
-
trdata=node_api_.read("ops/transfers/#{@transfer_id}")[:data] || {
|
96
|
+
trdata=node_api_.read("ops/transfers/#{@transfer_id}")[:data] || {'status'=>'unknown'} rescue {'status'=>'waiting(read error)'}
|
78
97
|
case trdata['status']
|
79
98
|
when 'completed'
|
80
99
|
notify_end(@transfer_id)
|
81
100
|
break
|
82
101
|
when 'waiting','partially_completed','unknown','waiting(read error)'
|
83
102
|
if spinner.nil?
|
84
|
-
spinner = TTY::Spinner.new(
|
103
|
+
spinner = TTY::Spinner.new('[:spinner] :title', format: :classic)
|
85
104
|
spinner.start
|
86
105
|
end
|
87
106
|
spinner.update(title: trdata['status'])
|
@@ -89,7 +108,7 @@ module Aspera
|
|
89
108
|
#puts trdata
|
90
109
|
when 'running'
|
91
110
|
#puts "running: sessions:#{trdata["sessions"].length}, #{trdata["sessions"].map{|i| i['bytes_transferred']}.join(',')}"
|
92
|
-
if !started
|
111
|
+
if !started && trdata['precalc'].is_a?(Hash) &&
|
93
112
|
trdata['precalc']['status'].eql?('ready')
|
94
113
|
notify_begin(@transfer_id,trdata['precalc']['bytes_expected'])
|
95
114
|
started=true
|
@@ -98,7 +117,7 @@ module Aspera
|
|
98
117
|
end
|
99
118
|
else
|
100
119
|
Log.log.warn("trdata -> #{trdata}")
|
101
|
-
raise Fasp::Error
|
120
|
+
raise Fasp::Error, "#{trdata['status']}: #{trdata['error_desc']}"
|
102
121
|
end
|
103
122
|
sleep 1
|
104
123
|
end
|