aspera-cli 4.0.0 → 4.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +843 -304
- data/bin/dascli +13 -0
- data/docs/Makefile +4 -4
- data/docs/README.erb.md +805 -172
- data/docs/test_env.conf +22 -3
- data/examples/aoc.rb +14 -3
- data/examples/faspex4.rb +89 -0
- data/lib/aspera/aoc.rb +87 -108
- data/lib/aspera/cli/formater.rb +2 -0
- data/lib/aspera/cli/main.rb +89 -49
- data/lib/aspera/cli/plugin.rb +9 -4
- data/lib/aspera/cli/plugins/alee.rb +1 -1
- data/lib/aspera/cli/plugins/aoc.rb +188 -173
- data/lib/aspera/cli/plugins/ats.rb +2 -2
- data/lib/aspera/cli/plugins/config.rb +218 -145
- data/lib/aspera/cli/plugins/console.rb +2 -2
- data/lib/aspera/cli/plugins/faspex.rb +114 -61
- data/lib/aspera/cli/plugins/faspex5.rb +85 -43
- data/lib/aspera/cli/plugins/node.rb +3 -3
- data/lib/aspera/cli/plugins/preview.rb +59 -45
- data/lib/aspera/cli/plugins/server.rb +23 -8
- data/lib/aspera/cli/transfer_agent.rb +77 -49
- data/lib/aspera/cli/version.rb +1 -1
- data/lib/aspera/command_line_builder.rb +49 -31
- data/lib/aspera/cos_node.rb +33 -28
- data/lib/aspera/environment.rb +2 -2
- data/lib/aspera/fasp/connect.rb +28 -21
- data/lib/aspera/fasp/http_gw.rb +140 -28
- data/lib/aspera/fasp/installation.rb +93 -46
- data/lib/aspera/fasp/local.rb +88 -45
- data/lib/aspera/fasp/manager.rb +15 -0
- data/lib/aspera/fasp/node.rb +4 -4
- data/lib/aspera/fasp/parameters.rb +59 -101
- data/lib/aspera/fasp/parameters.yaml +531 -0
- data/lib/aspera/fasp/resume_policy.rb +13 -12
- data/lib/aspera/fasp/uri.rb +1 -1
- data/lib/aspera/log.rb +1 -1
- data/lib/aspera/node.rb +61 -1
- data/lib/aspera/oauth.rb +49 -46
- data/lib/aspera/persistency_folder.rb +9 -4
- data/lib/aspera/preview/file_types.rb +53 -21
- data/lib/aspera/preview/generator.rb +3 -3
- data/lib/aspera/rest.rb +29 -18
- data/lib/aspera/secrets.rb +20 -0
- data/lib/aspera/sync.rb +40 -35
- data/lib/aspera/temp_file_manager.rb +19 -0
- data/lib/aspera/web_auth.rb +105 -0
- metadata +54 -20
- data/docs/transfer_spec.html +0 -99
@@ -22,11 +22,18 @@ module Aspera
|
|
22
22
|
PRODUCT_CLI_V1='Aspera CLI'
|
23
23
|
PRODUCT_DRIVE='Aspera Drive'
|
24
24
|
PRODUCT_ENTSRV='Enterprise Server'
|
25
|
-
|
25
|
+
MAX_REDIRECT_SDK=2
|
26
|
+
private_constant :MAX_REDIRECT_SDK
|
27
|
+
# set ascp executable path
|
26
28
|
def ascp_path=(v)
|
27
29
|
@path_to_ascp=v
|
28
30
|
end
|
29
31
|
|
32
|
+
# filename for ascp with optional extension (Windows)
|
33
|
+
def ascp_filename
|
34
|
+
return 'ascp'+Environment.exe_extension
|
35
|
+
end
|
36
|
+
|
30
37
|
# location of SDK files
|
31
38
|
def folder=(v)
|
32
39
|
@sdk_folder=v
|
@@ -50,26 +57,30 @@ module Aspera
|
|
50
57
|
# @return the list of installed products in format of product_locations
|
51
58
|
def installed_products
|
52
59
|
if @found_products.nil?
|
53
|
-
|
54
|
-
# add
|
55
|
-
|
60
|
+
scan_locations=product_locations.clone
|
61
|
+
# add SDK as first search path
|
62
|
+
scan_locations.unshift({
|
56
63
|
:expected =>'SDK',
|
57
64
|
:app_root =>folder_path,
|
58
65
|
:sub_bin =>''
|
59
66
|
})
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
+
# search installed products: with ascp
|
68
|
+
@found_products=scan_locations.select! do |item|
|
69
|
+
# skip if not main folder
|
70
|
+
next false unless Dir.exist?(item[:app_root])
|
71
|
+
Log.log.debug("Found #{item[:app_root]}")
|
72
|
+
sub_bin = item[:sub_bin] || BIN_SUBFOLDER
|
73
|
+
item[:ascp_path]=File.join(item[:app_root],sub_bin,ascp_filename)
|
74
|
+
# skip if no ascp
|
75
|
+
next false unless File.exist?(item[:ascp_path])
|
76
|
+
# read info from product info file if present
|
77
|
+
product_info_file="#{item[:app_root]}/#{PRODUCT_INFO}"
|
67
78
|
if File.exist?(product_info_file)
|
68
|
-
res_s=XmlSimple.xml_in(File.read(product_info_file),{
|
69
|
-
|
70
|
-
|
79
|
+
res_s=XmlSimple.xml_in(File.read(product_info_file),{'ForceArray'=>false})
|
80
|
+
item[:name]=res_s['name']
|
81
|
+
item[:version]=res_s['version']
|
71
82
|
else
|
72
|
-
|
83
|
+
item[:name]=item[:expected]
|
73
84
|
end
|
74
85
|
true # select this version
|
75
86
|
end
|
@@ -77,6 +88,7 @@ module Aspera
|
|
77
88
|
return @found_products
|
78
89
|
end
|
79
90
|
|
91
|
+
# all ascp files (in SDK)
|
80
92
|
FILES=[:ascp,:ascp4,:ssh_bypass_key_dsa,:ssh_bypass_key_rsa,:aspera_license,:aspera_conf,:fallback_cert,:fallback_key]
|
81
93
|
|
82
94
|
# get path of one resource file of currently activated product
|
@@ -106,14 +118,8 @@ module Aspera
|
|
106
118
|
<CONF version="2">
|
107
119
|
<default>
|
108
120
|
<file_system>
|
109
|
-
<storage_rc>
|
110
|
-
<adaptive>
|
111
|
-
true
|
112
|
-
</adaptive>
|
113
|
-
</storage_rc>
|
114
121
|
<resume_suffix>.aspera-ckpt</resume_suffix>
|
115
122
|
<partial_file_suffix>.partial</partial_file_suffix>
|
116
|
-
<replace_illegal_chars>_</replace_illegal_chars>
|
117
123
|
</file_system>
|
118
124
|
</default>
|
119
125
|
</CONF>
|
@@ -147,7 +153,7 @@ module Aspera
|
|
147
153
|
return file
|
148
154
|
end
|
149
155
|
|
150
|
-
# @
|
156
|
+
# @return the file path of local connect where API's URI can be read
|
151
157
|
def connect_uri
|
152
158
|
connect=get_product_folders(PRODUCT_CONNECT)
|
153
159
|
folder=File.join(connect[:run_root],VARRUN_SUBFOLDER)
|
@@ -176,23 +182,67 @@ module Aspera
|
|
176
182
|
return [:ssh_bypass_key_dsa,:ssh_bypass_key_rsa].map{|i|Installation.instance.path(i)}
|
177
183
|
end
|
178
184
|
|
179
|
-
|
185
|
+
# Check that specified path is ascp and get version
|
186
|
+
def get_ascp_version(ascp_path)
|
187
|
+
raise "File basename of #{ascp_path} must be #{ascp_filename}" unless File.basename(ascp_path).eql?(ascp_filename)
|
188
|
+
ascp_version='n/a'
|
189
|
+
raise "error in sdk: no ascp included" if ascp_path.nil?
|
190
|
+
cmd_out=%x{"#{ascp_path}" -A}
|
191
|
+
raise "An error occured when testing #{ascp_filename}: #{cmd_out}" unless $? == 0
|
192
|
+
# get version from ascp, only after full extract, as windows requires DLLs (SSL/TLS/etc...)
|
193
|
+
m=cmd_out.match(/ascp version (.*)/)
|
194
|
+
ascp_version=m[1] unless m.nil?
|
195
|
+
end
|
196
|
+
|
197
|
+
# download aspera SDK or use local file
|
198
|
+
# extracts ascp binary for current system architecture
|
199
|
+
# @return ascp version (from execution)
|
200
|
+
def install_sdk(sdk_url)
|
180
201
|
require 'zip'
|
181
202
|
sdk_zip_path=File.join(Dir.tmpdir,'sdk.zip')
|
182
|
-
|
203
|
+
if sdk_url.start_with?('file:')
|
204
|
+
# require specific file scheme: the path part is "relative", or absolute if there are 4 slash
|
205
|
+
raise 'use format: file:///<path>' unless sdk_url.start_with?('file:///')
|
206
|
+
sdk_zip_path=sdk_url.gsub(%r{^file:///},'')
|
207
|
+
else
|
208
|
+
redirect_remain=MAX_REDIRECT_SDK
|
209
|
+
begin
|
210
|
+
Aspera::Rest.new(base_url: sdk_url).call(operation: 'GET',save_to_file: sdk_zip_path)
|
211
|
+
rescue Aspera::RestCallError => e
|
212
|
+
if e.response.is_a?(Net::HTTPRedirection)
|
213
|
+
if redirect_remain > 0
|
214
|
+
redirect_remain-=1
|
215
|
+
sdk_url=e.response['location']
|
216
|
+
retry
|
217
|
+
else
|
218
|
+
raise "Too many redirect"
|
219
|
+
end
|
220
|
+
else
|
221
|
+
raise e
|
222
|
+
end
|
223
|
+
end
|
224
|
+
end
|
225
|
+
# SDK is organized by architecture
|
183
226
|
filter="/#{Environment.architecture}/"
|
184
227
|
ascp_path=nil
|
228
|
+
sdk_path=folder_path
|
229
|
+
# rename old install
|
230
|
+
if File.exist?(File.join(sdk_path,ascp_filename))
|
231
|
+
Log.log.warn("Previous install exists, renaming.")
|
232
|
+
File.rename(sdk_path,"#{sdk_path}.#{Time.now.strftime("%Y%m%d%H%M%S")}")
|
233
|
+
end
|
185
234
|
# first ensure license file is here so that ascp invokation for version works
|
186
235
|
self.path(:aspera_license)
|
187
236
|
self.path(:aspera_conf)
|
188
237
|
Zip::File.open(sdk_zip_path) do |zip_file|
|
189
238
|
zip_file.each do |entry|
|
239
|
+
# get only specified arch, but not folder, only files
|
190
240
|
if entry.name.include?(filter) and !entry.name.end_with?('/')
|
191
|
-
archive_file=File.join(
|
241
|
+
archive_file=File.join(sdk_path,File.basename(entry.name))
|
192
242
|
File.open(archive_file, 'wb') do |output_stream|
|
193
243
|
IO.copy_stream(entry.get_input_stream, output_stream)
|
194
244
|
end
|
195
|
-
if entry.name.
|
245
|
+
if File.basename(entry.name).eql?(ascp_filename)
|
196
246
|
FileUtils.chmod(0755,archive_file)
|
197
247
|
ascp_path=archive_file
|
198
248
|
end
|
@@ -200,13 +250,7 @@ module Aspera
|
|
200
250
|
end
|
201
251
|
end
|
202
252
|
File.unlink(sdk_zip_path) rescue nil # Windows may give error
|
203
|
-
ascp_version=
|
204
|
-
raise "error in sdk: no ascp included" if ascp_path.nil?
|
205
|
-
cmd_out=%x{#{ascp_path} -A}
|
206
|
-
raise "An error occured when testing ascp: #{cmd_out}" unless $? == 0
|
207
|
-
# get version from ascp, only after full extract, as windows requires DLLs (SSL/TLS/etc...)
|
208
|
-
m=cmd_out.match(/ascp version (.*)/)
|
209
|
-
ascp_version=m[1] unless m.nil?
|
253
|
+
ascp_version=get_ascp_version(ascp_path)
|
210
254
|
File.write(File.join(folder_path,PRODUCT_INFO),"<product><name>IBM Aspera SDK</name><version>#{ascp_version}</version></product>")
|
211
255
|
return ascp_version
|
212
256
|
end
|
@@ -220,16 +264,8 @@ module Aspera
|
|
220
264
|
PRODUCT_INFO='product-info.mf'
|
221
265
|
# policy for product selection
|
222
266
|
FIRST_FOUND='FIRST'
|
223
|
-
SDK_URL='https://eudemo.asperademo.com/aspera/faspex/sdk.zip'
|
224
267
|
|
225
|
-
private_constant :BIN_SUBFOLDER,:ETC_SUBFOLDER,:VARRUN_SUBFOLDER,:PRODUCT_INFO
|
226
|
-
|
227
|
-
# get some specific folder from specific applications: Connect or CLI
|
228
|
-
def get_product_folders(name)
|
229
|
-
found=installed_products.select{|i|i[:expected].eql?(name) or i[:name].eql?(name)}
|
230
|
-
raise "Product: #{name} not found, please install." if found.empty?
|
231
|
-
return found.first
|
232
|
-
end
|
268
|
+
private_constant :BIN_SUBFOLDER,:ETC_SUBFOLDER,:VARRUN_SUBFOLDER,:PRODUCT_INFO
|
233
269
|
|
234
270
|
def initialize
|
235
271
|
@path_to_ascp=nil
|
@@ -237,14 +273,22 @@ module Aspera
|
|
237
273
|
@found_products=nil
|
238
274
|
end
|
239
275
|
|
276
|
+
# @return folder paths for specified applications
|
277
|
+
# @param name Connect or CLI
|
278
|
+
def get_product_folders(name)
|
279
|
+
found=installed_products.select{|i|i[:expected].eql?(name) or i[:name].eql?(name)}
|
280
|
+
raise "Product: #{name} not found, please install." if found.empty?
|
281
|
+
return found.first
|
282
|
+
end
|
283
|
+
|
284
|
+
# @return the path to folder where SDK is installed
|
240
285
|
def folder_path
|
241
|
-
raise "
|
286
|
+
raise "Undefined path to SDK" if @sdk_folder.nil?
|
242
287
|
FileUtils.mkdir_p(@sdk_folder) unless Dir.exist?(@sdk_folder)
|
243
288
|
@sdk_folder
|
244
289
|
end
|
245
290
|
|
246
|
-
#
|
247
|
-
# fields
|
291
|
+
# @return product folders depending on OS fields
|
248
292
|
# :expected M app name is taken from the manifest if present, else defaults to this value
|
249
293
|
# :app_root M main folder for the application
|
250
294
|
# :log_root O location of log files (Linux uses syslog)
|
@@ -286,7 +330,7 @@ module Aspera
|
|
286
330
|
:log_root =>File.join(Dir.home,'Library','Logs','Aspera_Drive'),
|
287
331
|
:sub_bin =>File.join('Contents','Resources'),
|
288
332
|
}]
|
289
|
-
else; return [{ # other: Linux and
|
333
|
+
else; return [{ # other: Linux and Unix family
|
290
334
|
:expected =>PRODUCT_CONNECT,
|
291
335
|
:app_root =>File.join(Dir.home,'.aspera','connect'),
|
292
336
|
:run_root =>File.join(Dir.home,'.aspera','connect')
|
@@ -300,6 +344,9 @@ module Aspera
|
|
300
344
|
end
|
301
345
|
end
|
302
346
|
|
347
|
+
# @return a standard bypass key
|
348
|
+
# @param type rsa or dsa
|
349
|
+
# @param id in repository 1 for dsa, 2 for rsa
|
303
350
|
def get_key(type,id)
|
304
351
|
hf=['begin','end'].map{|t|"-----#{t} #{type} private key-----".upcase}
|
305
352
|
bin=Base64.strict_encode64(DataRepository.instance.get_bin(id))
|
data/lib/aspera/fasp/local.rb
CHANGED
@@ -21,16 +21,24 @@ module Aspera
|
|
21
21
|
ACCESS_KEY_TRANSFER_USER='xfer'
|
22
22
|
# executes a local "ascp", connects mgt port, equivalent of "Fasp Manager"
|
23
23
|
class Local < Manager
|
24
|
-
|
25
|
-
|
26
|
-
|
24
|
+
# options for initialize
|
25
|
+
DEFAULT_OPTIONS = {
|
26
|
+
:spawn_timeout_sec => 3,
|
27
|
+
:spawn_delay_sec => 2,
|
28
|
+
:wss => false,
|
29
|
+
:resume => {}
|
30
|
+
}
|
31
|
+
DEFAULT_UDP_PORT=33001
|
32
|
+
private_constant :DEFAULT_OPTIONS
|
33
|
+
# set to false to keep ascp progress bar display ("true" adds ascp's option -q)
|
27
34
|
attr_accessor :quiet
|
35
|
+
|
28
36
|
# start ascp transfer (non blocking), single or multi-session
|
29
37
|
# job information added to @jobs
|
30
38
|
# @param transfer_spec [Hash] aspera transfer specification
|
31
39
|
# @param options [Hash] :resumer, :regenerate_token
|
32
40
|
def start_transfer(transfer_spec,options={})
|
33
|
-
raise
|
41
|
+
raise 'option: must be hash (or nil)' unless options.is_a?(Hash)
|
34
42
|
job_options = options.clone
|
35
43
|
job_options[:resumer] ||= @resume_policy
|
36
44
|
job_options[:job_id] ||= SecureRandom.uuid
|
@@ -58,21 +66,26 @@ module Aspera
|
|
58
66
|
|
59
67
|
# TODO: check if changing fasp(UDP) port is really necessary, not clear from doc
|
60
68
|
# compute this before using transfer spec, even if the var is not used in single session
|
61
|
-
multi_session_udp_port_base=
|
62
|
-
multi_session_number=
|
69
|
+
multi_session_udp_port_base=DEFAULT_UDP_PORT
|
70
|
+
multi_session_number=0
|
63
71
|
if transfer_spec.has_key?('multi_session')
|
64
72
|
multi_session_number=transfer_spec['multi_session'].to_i
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
73
|
+
if multi_session_number < 0
|
74
|
+
Log.log.error("multi_session(#{transfer_spec['multi_session']}) shall be integer >= 0")
|
75
|
+
multi_session_number = 0
|
76
|
+
end
|
77
|
+
if multi_session_number > 0
|
78
|
+
# managed here, so delete from transfer spec
|
79
|
+
transfer_spec.delete('multi_session')
|
80
|
+
if transfer_spec.has_key?('fasp_port')
|
81
|
+
multi_session_udp_port_base=transfer_spec['fasp_port']
|
82
|
+
transfer_spec.delete('fasp_port')
|
83
|
+
end
|
71
84
|
end
|
72
85
|
end
|
73
86
|
|
74
87
|
# compute known args
|
75
|
-
env_args=Parameters.ts_to_env_args(transfer_spec,wss: @
|
88
|
+
env_args=Parameters.ts_to_env_args(transfer_spec,wss: @options[:wss])
|
76
89
|
|
77
90
|
# add fallback cert and key as arguments if needed
|
78
91
|
if ['1','force'].include?(transfer_spec['http_fallback'])
|
@@ -98,25 +111,27 @@ module Aspera
|
|
98
111
|
:options => job_options # [Hash]
|
99
112
|
}
|
100
113
|
|
101
|
-
|
102
|
-
|
114
|
+
if multi_session_number <= 1
|
115
|
+
Log.log.debug('Starting single session thread')
|
103
116
|
# single session for transfer : simple
|
104
117
|
session[:thread] = Thread.new(session) {|s|transfer_thread_entry(s)}
|
105
118
|
xfer_job[:sessions].push(session)
|
106
119
|
else
|
120
|
+
Log.log.debug('Starting multi session threads')
|
107
121
|
1.upto(multi_session_number) do |i|
|
122
|
+
sleep(@options[:spawn_delay_sec]) unless i.eql?(1)
|
108
123
|
# do deep copy (each thread has its own copy because it is modified here below and in thread)
|
109
124
|
this_session=session.clone()
|
110
125
|
this_session[:env_args]=this_session[:env_args].clone()
|
111
126
|
this_session[:env_args][:args]=this_session[:env_args][:args].clone()
|
112
127
|
this_session[:env_args][:args].unshift("-C#{i}:#{multi_session_number}")
|
113
128
|
# necessary only if server is not linux, i.e. server does not support port re-use
|
114
|
-
this_session[:env_args][:args].unshift(
|
129
|
+
this_session[:env_args][:args].unshift('-O',"#{multi_session_udp_port_base+i-1}")
|
115
130
|
this_session[:thread] = Thread.new(this_session) {|s|transfer_thread_entry(s)}
|
116
131
|
xfer_job[:sessions].push(this_session)
|
117
132
|
end
|
118
133
|
end
|
119
|
-
Log.log.debug(
|
134
|
+
Log.log.debug('started session thread(s)')
|
120
135
|
|
121
136
|
# add job to list of jobs
|
122
137
|
@jobs[job_options[:job_id]]=xfer_job
|
@@ -128,7 +143,7 @@ module Aspera
|
|
128
143
|
# wait for completion of all jobs started
|
129
144
|
# @return list of :success or error message
|
130
145
|
def wait_for_transfers_completion
|
131
|
-
Log.log.debug(
|
146
|
+
Log.log.debug('wait_for_transfers_completion')
|
132
147
|
# set to non-nil to exit loop
|
133
148
|
result=[]
|
134
149
|
@jobs.each do |id,job|
|
@@ -138,7 +153,7 @@ module Aspera
|
|
138
153
|
result.push(session[:error] ? session[:error] : :success)
|
139
154
|
end
|
140
155
|
end
|
141
|
-
Log.log.debug(
|
156
|
+
Log.log.debug('all transfers joined')
|
142
157
|
# since all are finished and we return the result, clear statuses
|
143
158
|
@jobs.clear
|
144
159
|
return result
|
@@ -146,7 +161,7 @@ module Aspera
|
|
146
161
|
|
147
162
|
# used by asession (to be removed ?)
|
148
163
|
def shutdown
|
149
|
-
Log.log.debug(
|
164
|
+
Log.log.debug('fasp local shutdown')
|
150
165
|
end
|
151
166
|
|
152
167
|
# This is the low level method to start the "ascp" process
|
@@ -158,8 +173,10 @@ module Aspera
|
|
158
173
|
# @param session this session information
|
159
174
|
# could be private method
|
160
175
|
def start_transfer_with_args_env(env_args,session)
|
161
|
-
raise
|
162
|
-
raise
|
176
|
+
raise 'env_args must be Hash' unless env_args.is_a?(Hash)
|
177
|
+
raise 'session must be Hash' unless session.is_a?(Hash)
|
178
|
+
# by default we assume an exception will be raised (for ensure block)
|
179
|
+
exception_raised=true
|
163
180
|
begin
|
164
181
|
Log.log.debug("env_args=#{env_args.inspect}")
|
165
182
|
# get location of ascp executable
|
@@ -182,7 +199,7 @@ module Aspera
|
|
182
199
|
Log.log.debug("before accept for pid (#{ascp_pid})")
|
183
200
|
# init management socket
|
184
201
|
ascp_mgt_io=nil
|
185
|
-
Timeout.timeout(
|
202
|
+
Timeout.timeout(@options[:spawn_timeout_sec]) do
|
186
203
|
ascp_mgt_io = mgt_sock.accept
|
187
204
|
# management messages include file names which may be utf8
|
188
205
|
# by default socket is US-ASCII
|
@@ -216,7 +233,7 @@ module Aspera
|
|
216
233
|
current_event_data[$1] = $2
|
217
234
|
when ''
|
218
235
|
# empty line is separator to end event information
|
219
|
-
raise
|
236
|
+
raise 'unexpected empty line' if current_event_data.nil?
|
220
237
|
current_event_data[Manager::LISTENER_SESSION_ID_B]=ascp_pid
|
221
238
|
notify_listeners(current_event_text,current_event_data)
|
222
239
|
case current_event_data['Type']
|
@@ -232,23 +249,27 @@ module Aspera
|
|
232
249
|
end # case
|
233
250
|
end # loop (process mgt port lines)
|
234
251
|
# check that last status was received before process exit
|
235
|
-
|
236
|
-
|
237
|
-
|
238
|
-
|
239
|
-
|
240
|
-
|
241
|
-
|
242
|
-
|
243
|
-
|
244
|
-
|
245
|
-
|
246
|
-
|
252
|
+
if last_status_event.is_a?(Hash)
|
253
|
+
case last_status_event['Type']
|
254
|
+
when 'DONE'
|
255
|
+
# all went well
|
256
|
+
exception_raised=false
|
257
|
+
when 'ERROR'
|
258
|
+
Log.log.error("code: #{last_status_event['Code']}")
|
259
|
+
if last_status_event['Description'] =~ /bearer token/i
|
260
|
+
Log.log.error('need to regenerate token'.red)
|
261
|
+
if session[:options].is_a?(Hash) and session[:options].has_key?(:regenerate_token)
|
262
|
+
# regenerate token here, expired, or error on it
|
263
|
+
env_args[:env]['ASPERA_SCP_TOKEN']=session[:options][:regenerate_token].call(true)
|
264
|
+
end
|
247
265
|
end
|
266
|
+
raise Fasp::Error.new(last_status_event['Description'],last_status_event['Code'].to_i)
|
267
|
+
else # case
|
268
|
+
raise "unexpected last event type: #{last_status_event['Type']}"
|
248
269
|
end
|
249
|
-
raise Fasp::Error.new(last_status_event['Description'],last_status_event['Code'].to_i)
|
250
270
|
else
|
251
|
-
|
271
|
+
exception_raised=false
|
272
|
+
Log.log.debug('no status read from ascp mgt port')
|
252
273
|
end
|
253
274
|
rescue SystemCallError => e
|
254
275
|
# Process.spawn
|
@@ -262,8 +283,18 @@ module Aspera
|
|
262
283
|
unless ascp_pid.nil?
|
263
284
|
# "wait" for process to avoid zombie
|
264
285
|
Process.wait(ascp_pid)
|
286
|
+
status=$?
|
265
287
|
ascp_pid=nil
|
266
288
|
session.delete(:io)
|
289
|
+
if !status.success?
|
290
|
+
message="ascp failed with code #{status.exitstatus}"
|
291
|
+
if exception_raised
|
292
|
+
# just debug, as main exception is already here
|
293
|
+
Log.log.debug(message)
|
294
|
+
else
|
295
|
+
raise Fasp::Error.new(message)
|
296
|
+
end
|
297
|
+
end
|
267
298
|
end
|
268
299
|
end # begin-ensure
|
269
300
|
end # start_transfer_with_args_env
|
@@ -276,9 +307,9 @@ module Aspera
|
|
276
307
|
# {'type'=>'DONE'}
|
277
308
|
def send_command(job_id,session_index,data)
|
278
309
|
job=@jobs[job_id]
|
279
|
-
raise
|
310
|
+
raise 'no such job' if job.nil?
|
280
311
|
session=job[:sessions][session_index]
|
281
|
-
raise
|
312
|
+
raise 'no such session' if session.nil?
|
282
313
|
Log.log.debug("command: #{data}")
|
283
314
|
# build command
|
284
315
|
command=data.
|
@@ -292,8 +323,8 @@ module Aspera
|
|
292
323
|
|
293
324
|
private
|
294
325
|
|
295
|
-
|
296
|
-
|
326
|
+
# @param options : keys(symbol): wss, resume
|
327
|
+
def initialize(options=nil)
|
297
328
|
super()
|
298
329
|
# by default no interactive progress bar
|
299
330
|
@quiet=true
|
@@ -301,8 +332,20 @@ module Aspera
|
|
301
332
|
@jobs={}
|
302
333
|
# mutex protects global data accessed by threads
|
303
334
|
@mutex=Mutex.new
|
304
|
-
|
305
|
-
@
|
335
|
+
# manage options
|
336
|
+
@options=DEFAULT_OPTIONS.clone
|
337
|
+
if !options.nil?
|
338
|
+
raise "expecting Hash (or nil), but have #{options.class}" unless options.is_a?(Hash)
|
339
|
+
options.each do |k,v|
|
340
|
+
if DEFAULT_OPTIONS.has_key?(k)
|
341
|
+
@options[k]=v
|
342
|
+
else
|
343
|
+
raise "unknown local agent parameter: #{k}, expect one of #{DEFAULT_OPTIONS.keys.map{|i|i.to_s}.join(",")}"
|
344
|
+
end
|
345
|
+
end
|
346
|
+
end
|
347
|
+
Log.log.debug("local options= #{options}")
|
348
|
+
@resume_policy=ResumePolicy.new(@options[:resume].symbolize_keys)
|
306
349
|
end
|
307
350
|
|
308
351
|
# transfer thread entry
|
@@ -310,7 +353,7 @@ module Aspera
|
|
310
353
|
def transfer_thread_entry(session)
|
311
354
|
begin
|
312
355
|
# set name for logging
|
313
|
-
Thread.current[:name]=
|
356
|
+
Thread.current[:name]='transfer'
|
314
357
|
Log.log.debug("ENTER (#{Thread.current[:name]})")
|
315
358
|
# start transfer with selected resumer policy
|
316
359
|
session[:options][:resumer].process do
|
data/lib/aspera/fasp/manager.rb
CHANGED
@@ -45,6 +45,18 @@ module Aspera
|
|
45
45
|
end
|
46
46
|
end # notify_listeners
|
47
47
|
|
48
|
+
def notify_begin(id,size)
|
49
|
+
notify_listeners('emulated',{LISTENER_SESSION_ID_B=>id,'Type'=>'NOTIFICATION','PreTransferBytes'=>size})
|
50
|
+
end
|
51
|
+
|
52
|
+
def notify_progress(id,size)
|
53
|
+
notify_listeners('emulated',{LISTENER_SESSION_ID_B=>id,'Type'=>'STATS','Bytescont'=>size})
|
54
|
+
end
|
55
|
+
|
56
|
+
def notify_end(id)
|
57
|
+
notify_listeners('emulated',{LISTENER_SESSION_ID_B=>id,'Type'=>'DONE'})
|
58
|
+
end
|
59
|
+
|
48
60
|
public
|
49
61
|
LISTENER_SESSION_ID_B='ListenerSessionId'
|
50
62
|
LISTENER_SESSION_ID_S='listener_session_id'
|
@@ -60,6 +72,9 @@ module Aspera
|
|
60
72
|
# start_transfer(transfer_spec,options) : start and wait for completion
|
61
73
|
# wait_for_transfers_completion : wait for termination of all transfers, @return list of : :success or error message
|
62
74
|
# optional: shutdown
|
75
|
+
|
76
|
+
# This checks the validity of the value returned by wait_for_transfers_completion
|
77
|
+
# it must be a list of :success or exception
|
63
78
|
def self.validate_status_list(statuses)
|
64
79
|
raise "internal error: bad statuses type: #{statuses.class}" unless statuses.is_a?(Array)
|
65
80
|
raise "internal error: bad statuses content: #{statuses}" unless statuses.select{|i|!i.eql?(:success) and !i.is_a?(StandardError)}.empty?
|
data/lib/aspera/fasp/node.rb
CHANGED
@@ -55,7 +55,7 @@ module Aspera
|
|
55
55
|
trdata=node_api_.read("ops/transfers/#{@transfer_id}")[:data] || {"status"=>"unknown"} rescue {"status"=>"waiting(read error)"}
|
56
56
|
case trdata['status']
|
57
57
|
when 'completed'
|
58
|
-
|
58
|
+
notify_end(@transfer_id)
|
59
59
|
break
|
60
60
|
when 'waiting','partially_completed','unknown','waiting(read error)'
|
61
61
|
if spinner.nil?
|
@@ -69,10 +69,10 @@ module Aspera
|
|
69
69
|
#puts "running: sessions:#{trdata["sessions"].length}, #{trdata["sessions"].map{|i| i['bytes_transferred']}.join(',')}"
|
70
70
|
if !started and trdata['precalc'].is_a?(Hash) and
|
71
71
|
trdata['precalc']['status'].eql?('ready')
|
72
|
-
|
72
|
+
notify_begin(@transfer_id,trdata['precalc']['bytes_expected'])
|
73
73
|
started=true
|
74
74
|
else
|
75
|
-
|
75
|
+
notify_progress(@transfer_id,trdata['bytes_transferred'])
|
76
76
|
end
|
77
77
|
else
|
78
78
|
Log.log.warn("trdata -> #{trdata}")
|
@@ -81,7 +81,7 @@ module Aspera
|
|
81
81
|
sleep 1
|
82
82
|
end
|
83
83
|
#TODO get status of sessions
|
84
|
-
return []
|
84
|
+
return []
|
85
85
|
end
|
86
86
|
end
|
87
87
|
end
|