aspera-cli 4.0.0 → 4.2.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +843 -304
- data/bin/dascli +13 -0
- data/docs/Makefile +4 -4
- data/docs/README.erb.md +805 -172
- data/docs/test_env.conf +22 -3
- data/examples/aoc.rb +14 -3
- data/examples/faspex4.rb +89 -0
- data/lib/aspera/aoc.rb +87 -108
- data/lib/aspera/cli/formater.rb +2 -0
- data/lib/aspera/cli/main.rb +89 -49
- data/lib/aspera/cli/plugin.rb +9 -4
- data/lib/aspera/cli/plugins/alee.rb +1 -1
- data/lib/aspera/cli/plugins/aoc.rb +188 -173
- data/lib/aspera/cli/plugins/ats.rb +2 -2
- data/lib/aspera/cli/plugins/config.rb +218 -145
- data/lib/aspera/cli/plugins/console.rb +2 -2
- data/lib/aspera/cli/plugins/faspex.rb +114 -61
- data/lib/aspera/cli/plugins/faspex5.rb +85 -43
- data/lib/aspera/cli/plugins/node.rb +3 -3
- data/lib/aspera/cli/plugins/preview.rb +59 -45
- data/lib/aspera/cli/plugins/server.rb +23 -8
- data/lib/aspera/cli/transfer_agent.rb +77 -49
- data/lib/aspera/cli/version.rb +1 -1
- data/lib/aspera/command_line_builder.rb +49 -31
- data/lib/aspera/cos_node.rb +33 -28
- data/lib/aspera/environment.rb +2 -2
- data/lib/aspera/fasp/connect.rb +28 -21
- data/lib/aspera/fasp/http_gw.rb +140 -28
- data/lib/aspera/fasp/installation.rb +93 -46
- data/lib/aspera/fasp/local.rb +88 -45
- data/lib/aspera/fasp/manager.rb +15 -0
- data/lib/aspera/fasp/node.rb +4 -4
- data/lib/aspera/fasp/parameters.rb +59 -101
- data/lib/aspera/fasp/parameters.yaml +531 -0
- data/lib/aspera/fasp/resume_policy.rb +13 -12
- data/lib/aspera/fasp/uri.rb +1 -1
- data/lib/aspera/log.rb +1 -1
- data/lib/aspera/node.rb +61 -1
- data/lib/aspera/oauth.rb +49 -46
- data/lib/aspera/persistency_folder.rb +9 -4
- data/lib/aspera/preview/file_types.rb +53 -21
- data/lib/aspera/preview/generator.rb +3 -3
- data/lib/aspera/rest.rb +29 -18
- data/lib/aspera/secrets.rb +20 -0
- data/lib/aspera/sync.rb +40 -35
- data/lib/aspera/temp_file_manager.rb +19 -0
- data/lib/aspera/web_auth.rb +105 -0
- metadata +54 -20
- data/docs/transfer_spec.html +0 -99
@@ -22,11 +22,18 @@ module Aspera
|
|
22
22
|
PRODUCT_CLI_V1='Aspera CLI'
|
23
23
|
PRODUCT_DRIVE='Aspera Drive'
|
24
24
|
PRODUCT_ENTSRV='Enterprise Server'
|
25
|
-
|
25
|
+
MAX_REDIRECT_SDK=2
|
26
|
+
private_constant :MAX_REDIRECT_SDK
|
27
|
+
# set ascp executable path
|
26
28
|
def ascp_path=(v)
|
27
29
|
@path_to_ascp=v
|
28
30
|
end
|
29
31
|
|
32
|
+
# filename for ascp with optional extension (Windows)
|
33
|
+
def ascp_filename
|
34
|
+
return 'ascp'+Environment.exe_extension
|
35
|
+
end
|
36
|
+
|
30
37
|
# location of SDK files
|
31
38
|
def folder=(v)
|
32
39
|
@sdk_folder=v
|
@@ -50,26 +57,30 @@ module Aspera
|
|
50
57
|
# @return the list of installed products in format of product_locations
|
51
58
|
def installed_products
|
52
59
|
if @found_products.nil?
|
53
|
-
|
54
|
-
# add
|
55
|
-
|
60
|
+
scan_locations=product_locations.clone
|
61
|
+
# add SDK as first search path
|
62
|
+
scan_locations.unshift({
|
56
63
|
:expected =>'SDK',
|
57
64
|
:app_root =>folder_path,
|
58
65
|
:sub_bin =>''
|
59
66
|
})
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
+
# search installed products: with ascp
|
68
|
+
@found_products=scan_locations.select! do |item|
|
69
|
+
# skip if not main folder
|
70
|
+
next false unless Dir.exist?(item[:app_root])
|
71
|
+
Log.log.debug("Found #{item[:app_root]}")
|
72
|
+
sub_bin = item[:sub_bin] || BIN_SUBFOLDER
|
73
|
+
item[:ascp_path]=File.join(item[:app_root],sub_bin,ascp_filename)
|
74
|
+
# skip if no ascp
|
75
|
+
next false unless File.exist?(item[:ascp_path])
|
76
|
+
# read info from product info file if present
|
77
|
+
product_info_file="#{item[:app_root]}/#{PRODUCT_INFO}"
|
67
78
|
if File.exist?(product_info_file)
|
68
|
-
res_s=XmlSimple.xml_in(File.read(product_info_file),{
|
69
|
-
|
70
|
-
|
79
|
+
res_s=XmlSimple.xml_in(File.read(product_info_file),{'ForceArray'=>false})
|
80
|
+
item[:name]=res_s['name']
|
81
|
+
item[:version]=res_s['version']
|
71
82
|
else
|
72
|
-
|
83
|
+
item[:name]=item[:expected]
|
73
84
|
end
|
74
85
|
true # select this version
|
75
86
|
end
|
@@ -77,6 +88,7 @@ module Aspera
|
|
77
88
|
return @found_products
|
78
89
|
end
|
79
90
|
|
91
|
+
# all ascp files (in SDK)
|
80
92
|
FILES=[:ascp,:ascp4,:ssh_bypass_key_dsa,:ssh_bypass_key_rsa,:aspera_license,:aspera_conf,:fallback_cert,:fallback_key]
|
81
93
|
|
82
94
|
# get path of one resource file of currently activated product
|
@@ -106,14 +118,8 @@ module Aspera
|
|
106
118
|
<CONF version="2">
|
107
119
|
<default>
|
108
120
|
<file_system>
|
109
|
-
<storage_rc>
|
110
|
-
<adaptive>
|
111
|
-
true
|
112
|
-
</adaptive>
|
113
|
-
</storage_rc>
|
114
121
|
<resume_suffix>.aspera-ckpt</resume_suffix>
|
115
122
|
<partial_file_suffix>.partial</partial_file_suffix>
|
116
|
-
<replace_illegal_chars>_</replace_illegal_chars>
|
117
123
|
</file_system>
|
118
124
|
</default>
|
119
125
|
</CONF>
|
@@ -147,7 +153,7 @@ module Aspera
|
|
147
153
|
return file
|
148
154
|
end
|
149
155
|
|
150
|
-
# @
|
156
|
+
# @return the file path of local connect where API's URI can be read
|
151
157
|
def connect_uri
|
152
158
|
connect=get_product_folders(PRODUCT_CONNECT)
|
153
159
|
folder=File.join(connect[:run_root],VARRUN_SUBFOLDER)
|
@@ -176,23 +182,67 @@ module Aspera
|
|
176
182
|
return [:ssh_bypass_key_dsa,:ssh_bypass_key_rsa].map{|i|Installation.instance.path(i)}
|
177
183
|
end
|
178
184
|
|
179
|
-
|
185
|
+
# Check that specified path is ascp and get version
|
186
|
+
def get_ascp_version(ascp_path)
|
187
|
+
raise "File basename of #{ascp_path} must be #{ascp_filename}" unless File.basename(ascp_path).eql?(ascp_filename)
|
188
|
+
ascp_version='n/a'
|
189
|
+
raise "error in sdk: no ascp included" if ascp_path.nil?
|
190
|
+
cmd_out=%x{"#{ascp_path}" -A}
|
191
|
+
raise "An error occured when testing #{ascp_filename}: #{cmd_out}" unless $? == 0
|
192
|
+
# get version from ascp, only after full extract, as windows requires DLLs (SSL/TLS/etc...)
|
193
|
+
m=cmd_out.match(/ascp version (.*)/)
|
194
|
+
ascp_version=m[1] unless m.nil?
|
195
|
+
end
|
196
|
+
|
197
|
+
# download aspera SDK or use local file
|
198
|
+
# extracts ascp binary for current system architecture
|
199
|
+
# @return ascp version (from execution)
|
200
|
+
def install_sdk(sdk_url)
|
180
201
|
require 'zip'
|
181
202
|
sdk_zip_path=File.join(Dir.tmpdir,'sdk.zip')
|
182
|
-
|
203
|
+
if sdk_url.start_with?('file:')
|
204
|
+
# require specific file scheme: the path part is "relative", or absolute if there are 4 slash
|
205
|
+
raise 'use format: file:///<path>' unless sdk_url.start_with?('file:///')
|
206
|
+
sdk_zip_path=sdk_url.gsub(%r{^file:///},'')
|
207
|
+
else
|
208
|
+
redirect_remain=MAX_REDIRECT_SDK
|
209
|
+
begin
|
210
|
+
Aspera::Rest.new(base_url: sdk_url).call(operation: 'GET',save_to_file: sdk_zip_path)
|
211
|
+
rescue Aspera::RestCallError => e
|
212
|
+
if e.response.is_a?(Net::HTTPRedirection)
|
213
|
+
if redirect_remain > 0
|
214
|
+
redirect_remain-=1
|
215
|
+
sdk_url=e.response['location']
|
216
|
+
retry
|
217
|
+
else
|
218
|
+
raise "Too many redirect"
|
219
|
+
end
|
220
|
+
else
|
221
|
+
raise e
|
222
|
+
end
|
223
|
+
end
|
224
|
+
end
|
225
|
+
# SDK is organized by architecture
|
183
226
|
filter="/#{Environment.architecture}/"
|
184
227
|
ascp_path=nil
|
228
|
+
sdk_path=folder_path
|
229
|
+
# rename old install
|
230
|
+
if File.exist?(File.join(sdk_path,ascp_filename))
|
231
|
+
Log.log.warn("Previous install exists, renaming.")
|
232
|
+
File.rename(sdk_path,"#{sdk_path}.#{Time.now.strftime("%Y%m%d%H%M%S")}")
|
233
|
+
end
|
185
234
|
# first ensure license file is here so that ascp invokation for version works
|
186
235
|
self.path(:aspera_license)
|
187
236
|
self.path(:aspera_conf)
|
188
237
|
Zip::File.open(sdk_zip_path) do |zip_file|
|
189
238
|
zip_file.each do |entry|
|
239
|
+
# get only specified arch, but not folder, only files
|
190
240
|
if entry.name.include?(filter) and !entry.name.end_with?('/')
|
191
|
-
archive_file=File.join(
|
241
|
+
archive_file=File.join(sdk_path,File.basename(entry.name))
|
192
242
|
File.open(archive_file, 'wb') do |output_stream|
|
193
243
|
IO.copy_stream(entry.get_input_stream, output_stream)
|
194
244
|
end
|
195
|
-
if entry.name.
|
245
|
+
if File.basename(entry.name).eql?(ascp_filename)
|
196
246
|
FileUtils.chmod(0755,archive_file)
|
197
247
|
ascp_path=archive_file
|
198
248
|
end
|
@@ -200,13 +250,7 @@ module Aspera
|
|
200
250
|
end
|
201
251
|
end
|
202
252
|
File.unlink(sdk_zip_path) rescue nil # Windows may give error
|
203
|
-
ascp_version=
|
204
|
-
raise "error in sdk: no ascp included" if ascp_path.nil?
|
205
|
-
cmd_out=%x{#{ascp_path} -A}
|
206
|
-
raise "An error occured when testing ascp: #{cmd_out}" unless $? == 0
|
207
|
-
# get version from ascp, only after full extract, as windows requires DLLs (SSL/TLS/etc...)
|
208
|
-
m=cmd_out.match(/ascp version (.*)/)
|
209
|
-
ascp_version=m[1] unless m.nil?
|
253
|
+
ascp_version=get_ascp_version(ascp_path)
|
210
254
|
File.write(File.join(folder_path,PRODUCT_INFO),"<product><name>IBM Aspera SDK</name><version>#{ascp_version}</version></product>")
|
211
255
|
return ascp_version
|
212
256
|
end
|
@@ -220,16 +264,8 @@ module Aspera
|
|
220
264
|
PRODUCT_INFO='product-info.mf'
|
221
265
|
# policy for product selection
|
222
266
|
FIRST_FOUND='FIRST'
|
223
|
-
SDK_URL='https://eudemo.asperademo.com/aspera/faspex/sdk.zip'
|
224
267
|
|
225
|
-
private_constant :BIN_SUBFOLDER,:ETC_SUBFOLDER,:VARRUN_SUBFOLDER,:PRODUCT_INFO
|
226
|
-
|
227
|
-
# get some specific folder from specific applications: Connect or CLI
|
228
|
-
def get_product_folders(name)
|
229
|
-
found=installed_products.select{|i|i[:expected].eql?(name) or i[:name].eql?(name)}
|
230
|
-
raise "Product: #{name} not found, please install." if found.empty?
|
231
|
-
return found.first
|
232
|
-
end
|
268
|
+
private_constant :BIN_SUBFOLDER,:ETC_SUBFOLDER,:VARRUN_SUBFOLDER,:PRODUCT_INFO
|
233
269
|
|
234
270
|
def initialize
|
235
271
|
@path_to_ascp=nil
|
@@ -237,14 +273,22 @@ module Aspera
|
|
237
273
|
@found_products=nil
|
238
274
|
end
|
239
275
|
|
276
|
+
# @return folder paths for specified applications
|
277
|
+
# @param name Connect or CLI
|
278
|
+
def get_product_folders(name)
|
279
|
+
found=installed_products.select{|i|i[:expected].eql?(name) or i[:name].eql?(name)}
|
280
|
+
raise "Product: #{name} not found, please install." if found.empty?
|
281
|
+
return found.first
|
282
|
+
end
|
283
|
+
|
284
|
+
# @return the path to folder where SDK is installed
|
240
285
|
def folder_path
|
241
|
-
raise "
|
286
|
+
raise "Undefined path to SDK" if @sdk_folder.nil?
|
242
287
|
FileUtils.mkdir_p(@sdk_folder) unless Dir.exist?(@sdk_folder)
|
243
288
|
@sdk_folder
|
244
289
|
end
|
245
290
|
|
246
|
-
#
|
247
|
-
# fields
|
291
|
+
# @return product folders depending on OS fields
|
248
292
|
# :expected M app name is taken from the manifest if present, else defaults to this value
|
249
293
|
# :app_root M main folder for the application
|
250
294
|
# :log_root O location of log files (Linux uses syslog)
|
@@ -286,7 +330,7 @@ module Aspera
|
|
286
330
|
:log_root =>File.join(Dir.home,'Library','Logs','Aspera_Drive'),
|
287
331
|
:sub_bin =>File.join('Contents','Resources'),
|
288
332
|
}]
|
289
|
-
else; return [{ # other: Linux and
|
333
|
+
else; return [{ # other: Linux and Unix family
|
290
334
|
:expected =>PRODUCT_CONNECT,
|
291
335
|
:app_root =>File.join(Dir.home,'.aspera','connect'),
|
292
336
|
:run_root =>File.join(Dir.home,'.aspera','connect')
|
@@ -300,6 +344,9 @@ module Aspera
|
|
300
344
|
end
|
301
345
|
end
|
302
346
|
|
347
|
+
# @return a standard bypass key
|
348
|
+
# @param type rsa or dsa
|
349
|
+
# @param id in repository 1 for dsa, 2 for rsa
|
303
350
|
def get_key(type,id)
|
304
351
|
hf=['begin','end'].map{|t|"-----#{t} #{type} private key-----".upcase}
|
305
352
|
bin=Base64.strict_encode64(DataRepository.instance.get_bin(id))
|
data/lib/aspera/fasp/local.rb
CHANGED
@@ -21,16 +21,24 @@ module Aspera
|
|
21
21
|
ACCESS_KEY_TRANSFER_USER='xfer'
|
22
22
|
# executes a local "ascp", connects mgt port, equivalent of "Fasp Manager"
|
23
23
|
class Local < Manager
|
24
|
-
|
25
|
-
|
26
|
-
|
24
|
+
# options for initialize
|
25
|
+
DEFAULT_OPTIONS = {
|
26
|
+
:spawn_timeout_sec => 3,
|
27
|
+
:spawn_delay_sec => 2,
|
28
|
+
:wss => false,
|
29
|
+
:resume => {}
|
30
|
+
}
|
31
|
+
DEFAULT_UDP_PORT=33001
|
32
|
+
private_constant :DEFAULT_OPTIONS
|
33
|
+
# set to false to keep ascp progress bar display ("true" adds ascp's option -q)
|
27
34
|
attr_accessor :quiet
|
35
|
+
|
28
36
|
# start ascp transfer (non blocking), single or multi-session
|
29
37
|
# job information added to @jobs
|
30
38
|
# @param transfer_spec [Hash] aspera transfer specification
|
31
39
|
# @param options [Hash] :resumer, :regenerate_token
|
32
40
|
def start_transfer(transfer_spec,options={})
|
33
|
-
raise
|
41
|
+
raise 'option: must be hash (or nil)' unless options.is_a?(Hash)
|
34
42
|
job_options = options.clone
|
35
43
|
job_options[:resumer] ||= @resume_policy
|
36
44
|
job_options[:job_id] ||= SecureRandom.uuid
|
@@ -58,21 +66,26 @@ module Aspera
|
|
58
66
|
|
59
67
|
# TODO: check if changing fasp(UDP) port is really necessary, not clear from doc
|
60
68
|
# compute this before using transfer spec, even if the var is not used in single session
|
61
|
-
multi_session_udp_port_base=
|
62
|
-
multi_session_number=
|
69
|
+
multi_session_udp_port_base=DEFAULT_UDP_PORT
|
70
|
+
multi_session_number=0
|
63
71
|
if transfer_spec.has_key?('multi_session')
|
64
72
|
multi_session_number=transfer_spec['multi_session'].to_i
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
73
|
+
if multi_session_number < 0
|
74
|
+
Log.log.error("multi_session(#{transfer_spec['multi_session']}) shall be integer >= 0")
|
75
|
+
multi_session_number = 0
|
76
|
+
end
|
77
|
+
if multi_session_number > 0
|
78
|
+
# managed here, so delete from transfer spec
|
79
|
+
transfer_spec.delete('multi_session')
|
80
|
+
if transfer_spec.has_key?('fasp_port')
|
81
|
+
multi_session_udp_port_base=transfer_spec['fasp_port']
|
82
|
+
transfer_spec.delete('fasp_port')
|
83
|
+
end
|
71
84
|
end
|
72
85
|
end
|
73
86
|
|
74
87
|
# compute known args
|
75
|
-
env_args=Parameters.ts_to_env_args(transfer_spec,wss: @
|
88
|
+
env_args=Parameters.ts_to_env_args(transfer_spec,wss: @options[:wss])
|
76
89
|
|
77
90
|
# add fallback cert and key as arguments if needed
|
78
91
|
if ['1','force'].include?(transfer_spec['http_fallback'])
|
@@ -98,25 +111,27 @@ module Aspera
|
|
98
111
|
:options => job_options # [Hash]
|
99
112
|
}
|
100
113
|
|
101
|
-
|
102
|
-
|
114
|
+
if multi_session_number <= 1
|
115
|
+
Log.log.debug('Starting single session thread')
|
103
116
|
# single session for transfer : simple
|
104
117
|
session[:thread] = Thread.new(session) {|s|transfer_thread_entry(s)}
|
105
118
|
xfer_job[:sessions].push(session)
|
106
119
|
else
|
120
|
+
Log.log.debug('Starting multi session threads')
|
107
121
|
1.upto(multi_session_number) do |i|
|
122
|
+
sleep(@options[:spawn_delay_sec]) unless i.eql?(1)
|
108
123
|
# do deep copy (each thread has its own copy because it is modified here below and in thread)
|
109
124
|
this_session=session.clone()
|
110
125
|
this_session[:env_args]=this_session[:env_args].clone()
|
111
126
|
this_session[:env_args][:args]=this_session[:env_args][:args].clone()
|
112
127
|
this_session[:env_args][:args].unshift("-C#{i}:#{multi_session_number}")
|
113
128
|
# necessary only if server is not linux, i.e. server does not support port re-use
|
114
|
-
this_session[:env_args][:args].unshift(
|
129
|
+
this_session[:env_args][:args].unshift('-O',"#{multi_session_udp_port_base+i-1}")
|
115
130
|
this_session[:thread] = Thread.new(this_session) {|s|transfer_thread_entry(s)}
|
116
131
|
xfer_job[:sessions].push(this_session)
|
117
132
|
end
|
118
133
|
end
|
119
|
-
Log.log.debug(
|
134
|
+
Log.log.debug('started session thread(s)')
|
120
135
|
|
121
136
|
# add job to list of jobs
|
122
137
|
@jobs[job_options[:job_id]]=xfer_job
|
@@ -128,7 +143,7 @@ module Aspera
|
|
128
143
|
# wait for completion of all jobs started
|
129
144
|
# @return list of :success or error message
|
130
145
|
def wait_for_transfers_completion
|
131
|
-
Log.log.debug(
|
146
|
+
Log.log.debug('wait_for_transfers_completion')
|
132
147
|
# set to non-nil to exit loop
|
133
148
|
result=[]
|
134
149
|
@jobs.each do |id,job|
|
@@ -138,7 +153,7 @@ module Aspera
|
|
138
153
|
result.push(session[:error] ? session[:error] : :success)
|
139
154
|
end
|
140
155
|
end
|
141
|
-
Log.log.debug(
|
156
|
+
Log.log.debug('all transfers joined')
|
142
157
|
# since all are finished and we return the result, clear statuses
|
143
158
|
@jobs.clear
|
144
159
|
return result
|
@@ -146,7 +161,7 @@ module Aspera
|
|
146
161
|
|
147
162
|
# used by asession (to be removed ?)
|
148
163
|
def shutdown
|
149
|
-
Log.log.debug(
|
164
|
+
Log.log.debug('fasp local shutdown')
|
150
165
|
end
|
151
166
|
|
152
167
|
# This is the low level method to start the "ascp" process
|
@@ -158,8 +173,10 @@ module Aspera
|
|
158
173
|
# @param session this session information
|
159
174
|
# could be private method
|
160
175
|
def start_transfer_with_args_env(env_args,session)
|
161
|
-
raise
|
162
|
-
raise
|
176
|
+
raise 'env_args must be Hash' unless env_args.is_a?(Hash)
|
177
|
+
raise 'session must be Hash' unless session.is_a?(Hash)
|
178
|
+
# by default we assume an exception will be raised (for ensure block)
|
179
|
+
exception_raised=true
|
163
180
|
begin
|
164
181
|
Log.log.debug("env_args=#{env_args.inspect}")
|
165
182
|
# get location of ascp executable
|
@@ -182,7 +199,7 @@ module Aspera
|
|
182
199
|
Log.log.debug("before accept for pid (#{ascp_pid})")
|
183
200
|
# init management socket
|
184
201
|
ascp_mgt_io=nil
|
185
|
-
Timeout.timeout(
|
202
|
+
Timeout.timeout(@options[:spawn_timeout_sec]) do
|
186
203
|
ascp_mgt_io = mgt_sock.accept
|
187
204
|
# management messages include file names which may be utf8
|
188
205
|
# by default socket is US-ASCII
|
@@ -216,7 +233,7 @@ module Aspera
|
|
216
233
|
current_event_data[$1] = $2
|
217
234
|
when ''
|
218
235
|
# empty line is separator to end event information
|
219
|
-
raise
|
236
|
+
raise 'unexpected empty line' if current_event_data.nil?
|
220
237
|
current_event_data[Manager::LISTENER_SESSION_ID_B]=ascp_pid
|
221
238
|
notify_listeners(current_event_text,current_event_data)
|
222
239
|
case current_event_data['Type']
|
@@ -232,23 +249,27 @@ module Aspera
|
|
232
249
|
end # case
|
233
250
|
end # loop (process mgt port lines)
|
234
251
|
# check that last status was received before process exit
|
235
|
-
|
236
|
-
|
237
|
-
|
238
|
-
|
239
|
-
|
240
|
-
|
241
|
-
|
242
|
-
|
243
|
-
|
244
|
-
|
245
|
-
|
246
|
-
|
252
|
+
if last_status_event.is_a?(Hash)
|
253
|
+
case last_status_event['Type']
|
254
|
+
when 'DONE'
|
255
|
+
# all went well
|
256
|
+
exception_raised=false
|
257
|
+
when 'ERROR'
|
258
|
+
Log.log.error("code: #{last_status_event['Code']}")
|
259
|
+
if last_status_event['Description'] =~ /bearer token/i
|
260
|
+
Log.log.error('need to regenerate token'.red)
|
261
|
+
if session[:options].is_a?(Hash) and session[:options].has_key?(:regenerate_token)
|
262
|
+
# regenerate token here, expired, or error on it
|
263
|
+
env_args[:env]['ASPERA_SCP_TOKEN']=session[:options][:regenerate_token].call(true)
|
264
|
+
end
|
247
265
|
end
|
266
|
+
raise Fasp::Error.new(last_status_event['Description'],last_status_event['Code'].to_i)
|
267
|
+
else # case
|
268
|
+
raise "unexpected last event type: #{last_status_event['Type']}"
|
248
269
|
end
|
249
|
-
raise Fasp::Error.new(last_status_event['Description'],last_status_event['Code'].to_i)
|
250
270
|
else
|
251
|
-
|
271
|
+
exception_raised=false
|
272
|
+
Log.log.debug('no status read from ascp mgt port')
|
252
273
|
end
|
253
274
|
rescue SystemCallError => e
|
254
275
|
# Process.spawn
|
@@ -262,8 +283,18 @@ module Aspera
|
|
262
283
|
unless ascp_pid.nil?
|
263
284
|
# "wait" for process to avoid zombie
|
264
285
|
Process.wait(ascp_pid)
|
286
|
+
status=$?
|
265
287
|
ascp_pid=nil
|
266
288
|
session.delete(:io)
|
289
|
+
if !status.success?
|
290
|
+
message="ascp failed with code #{status.exitstatus}"
|
291
|
+
if exception_raised
|
292
|
+
# just debug, as main exception is already here
|
293
|
+
Log.log.debug(message)
|
294
|
+
else
|
295
|
+
raise Fasp::Error.new(message)
|
296
|
+
end
|
297
|
+
end
|
267
298
|
end
|
268
299
|
end # begin-ensure
|
269
300
|
end # start_transfer_with_args_env
|
@@ -276,9 +307,9 @@ module Aspera
|
|
276
307
|
# {'type'=>'DONE'}
|
277
308
|
def send_command(job_id,session_index,data)
|
278
309
|
job=@jobs[job_id]
|
279
|
-
raise
|
310
|
+
raise 'no such job' if job.nil?
|
280
311
|
session=job[:sessions][session_index]
|
281
|
-
raise
|
312
|
+
raise 'no such session' if session.nil?
|
282
313
|
Log.log.debug("command: #{data}")
|
283
314
|
# build command
|
284
315
|
command=data.
|
@@ -292,8 +323,8 @@ module Aspera
|
|
292
323
|
|
293
324
|
private
|
294
325
|
|
295
|
-
|
296
|
-
|
326
|
+
# @param options : keys(symbol): wss, resume
|
327
|
+
def initialize(options=nil)
|
297
328
|
super()
|
298
329
|
# by default no interactive progress bar
|
299
330
|
@quiet=true
|
@@ -301,8 +332,20 @@ module Aspera
|
|
301
332
|
@jobs={}
|
302
333
|
# mutex protects global data accessed by threads
|
303
334
|
@mutex=Mutex.new
|
304
|
-
|
305
|
-
@
|
335
|
+
# manage options
|
336
|
+
@options=DEFAULT_OPTIONS.clone
|
337
|
+
if !options.nil?
|
338
|
+
raise "expecting Hash (or nil), but have #{options.class}" unless options.is_a?(Hash)
|
339
|
+
options.each do |k,v|
|
340
|
+
if DEFAULT_OPTIONS.has_key?(k)
|
341
|
+
@options[k]=v
|
342
|
+
else
|
343
|
+
raise "unknown local agent parameter: #{k}, expect one of #{DEFAULT_OPTIONS.keys.map{|i|i.to_s}.join(",")}"
|
344
|
+
end
|
345
|
+
end
|
346
|
+
end
|
347
|
+
Log.log.debug("local options= #{options}")
|
348
|
+
@resume_policy=ResumePolicy.new(@options[:resume].symbolize_keys)
|
306
349
|
end
|
307
350
|
|
308
351
|
# transfer thread entry
|
@@ -310,7 +353,7 @@ module Aspera
|
|
310
353
|
def transfer_thread_entry(session)
|
311
354
|
begin
|
312
355
|
# set name for logging
|
313
|
-
Thread.current[:name]=
|
356
|
+
Thread.current[:name]='transfer'
|
314
357
|
Log.log.debug("ENTER (#{Thread.current[:name]})")
|
315
358
|
# start transfer with selected resumer policy
|
316
359
|
session[:options][:resumer].process do
|
data/lib/aspera/fasp/manager.rb
CHANGED
@@ -45,6 +45,18 @@ module Aspera
|
|
45
45
|
end
|
46
46
|
end # notify_listeners
|
47
47
|
|
48
|
+
def notify_begin(id,size)
|
49
|
+
notify_listeners('emulated',{LISTENER_SESSION_ID_B=>id,'Type'=>'NOTIFICATION','PreTransferBytes'=>size})
|
50
|
+
end
|
51
|
+
|
52
|
+
def notify_progress(id,size)
|
53
|
+
notify_listeners('emulated',{LISTENER_SESSION_ID_B=>id,'Type'=>'STATS','Bytescont'=>size})
|
54
|
+
end
|
55
|
+
|
56
|
+
def notify_end(id)
|
57
|
+
notify_listeners('emulated',{LISTENER_SESSION_ID_B=>id,'Type'=>'DONE'})
|
58
|
+
end
|
59
|
+
|
48
60
|
public
|
49
61
|
LISTENER_SESSION_ID_B='ListenerSessionId'
|
50
62
|
LISTENER_SESSION_ID_S='listener_session_id'
|
@@ -60,6 +72,9 @@ module Aspera
|
|
60
72
|
# start_transfer(transfer_spec,options) : start and wait for completion
|
61
73
|
# wait_for_transfers_completion : wait for termination of all transfers, @return list of : :success or error message
|
62
74
|
# optional: shutdown
|
75
|
+
|
76
|
+
# This checks the validity of the value returned by wait_for_transfers_completion
|
77
|
+
# it must be a list of :success or exception
|
63
78
|
def self.validate_status_list(statuses)
|
64
79
|
raise "internal error: bad statuses type: #{statuses.class}" unless statuses.is_a?(Array)
|
65
80
|
raise "internal error: bad statuses content: #{statuses}" unless statuses.select{|i|!i.eql?(:success) and !i.is_a?(StandardError)}.empty?
|
data/lib/aspera/fasp/node.rb
CHANGED
@@ -55,7 +55,7 @@ module Aspera
|
|
55
55
|
trdata=node_api_.read("ops/transfers/#{@transfer_id}")[:data] || {"status"=>"unknown"} rescue {"status"=>"waiting(read error)"}
|
56
56
|
case trdata['status']
|
57
57
|
when 'completed'
|
58
|
-
|
58
|
+
notify_end(@transfer_id)
|
59
59
|
break
|
60
60
|
when 'waiting','partially_completed','unknown','waiting(read error)'
|
61
61
|
if spinner.nil?
|
@@ -69,10 +69,10 @@ module Aspera
|
|
69
69
|
#puts "running: sessions:#{trdata["sessions"].length}, #{trdata["sessions"].map{|i| i['bytes_transferred']}.join(',')}"
|
70
70
|
if !started and trdata['precalc'].is_a?(Hash) and
|
71
71
|
trdata['precalc']['status'].eql?('ready')
|
72
|
-
|
72
|
+
notify_begin(@transfer_id,trdata['precalc']['bytes_expected'])
|
73
73
|
started=true
|
74
74
|
else
|
75
|
-
|
75
|
+
notify_progress(@transfer_id,trdata['bytes_transferred'])
|
76
76
|
end
|
77
77
|
else
|
78
78
|
Log.log.warn("trdata -> #{trdata}")
|
@@ -81,7 +81,7 @@ module Aspera
|
|
81
81
|
sleep 1
|
82
82
|
end
|
83
83
|
#TODO get status of sessions
|
84
|
-
return []
|
84
|
+
return []
|
85
85
|
end
|
86
86
|
end
|
87
87
|
end
|