aspera-cli 4.0.0 → 4.2.2

Sign up to get free protection for your applications and to get access to all the features.
Files changed (50) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +843 -304
  3. data/bin/dascli +13 -0
  4. data/docs/Makefile +4 -4
  5. data/docs/README.erb.md +805 -172
  6. data/docs/test_env.conf +22 -3
  7. data/examples/aoc.rb +14 -3
  8. data/examples/faspex4.rb +89 -0
  9. data/lib/aspera/aoc.rb +87 -108
  10. data/lib/aspera/cli/formater.rb +2 -0
  11. data/lib/aspera/cli/main.rb +89 -49
  12. data/lib/aspera/cli/plugin.rb +9 -4
  13. data/lib/aspera/cli/plugins/alee.rb +1 -1
  14. data/lib/aspera/cli/plugins/aoc.rb +188 -173
  15. data/lib/aspera/cli/plugins/ats.rb +2 -2
  16. data/lib/aspera/cli/plugins/config.rb +218 -145
  17. data/lib/aspera/cli/plugins/console.rb +2 -2
  18. data/lib/aspera/cli/plugins/faspex.rb +114 -61
  19. data/lib/aspera/cli/plugins/faspex5.rb +85 -43
  20. data/lib/aspera/cli/plugins/node.rb +3 -3
  21. data/lib/aspera/cli/plugins/preview.rb +59 -45
  22. data/lib/aspera/cli/plugins/server.rb +23 -8
  23. data/lib/aspera/cli/transfer_agent.rb +77 -49
  24. data/lib/aspera/cli/version.rb +1 -1
  25. data/lib/aspera/command_line_builder.rb +49 -31
  26. data/lib/aspera/cos_node.rb +33 -28
  27. data/lib/aspera/environment.rb +2 -2
  28. data/lib/aspera/fasp/connect.rb +28 -21
  29. data/lib/aspera/fasp/http_gw.rb +140 -28
  30. data/lib/aspera/fasp/installation.rb +93 -46
  31. data/lib/aspera/fasp/local.rb +88 -45
  32. data/lib/aspera/fasp/manager.rb +15 -0
  33. data/lib/aspera/fasp/node.rb +4 -4
  34. data/lib/aspera/fasp/parameters.rb +59 -101
  35. data/lib/aspera/fasp/parameters.yaml +531 -0
  36. data/lib/aspera/fasp/resume_policy.rb +13 -12
  37. data/lib/aspera/fasp/uri.rb +1 -1
  38. data/lib/aspera/log.rb +1 -1
  39. data/lib/aspera/node.rb +61 -1
  40. data/lib/aspera/oauth.rb +49 -46
  41. data/lib/aspera/persistency_folder.rb +9 -4
  42. data/lib/aspera/preview/file_types.rb +53 -21
  43. data/lib/aspera/preview/generator.rb +3 -3
  44. data/lib/aspera/rest.rb +29 -18
  45. data/lib/aspera/secrets.rb +20 -0
  46. data/lib/aspera/sync.rb +40 -35
  47. data/lib/aspera/temp_file_manager.rb +19 -0
  48. data/lib/aspera/web_auth.rb +105 -0
  49. metadata +54 -20
  50. data/docs/transfer_spec.html +0 -99
@@ -22,11 +22,18 @@ module Aspera
22
22
  PRODUCT_CLI_V1='Aspera CLI'
23
23
  PRODUCT_DRIVE='Aspera Drive'
24
24
  PRODUCT_ENTSRV='Enterprise Server'
25
- # currently used ascp executable
25
+ MAX_REDIRECT_SDK=2
26
+ private_constant :MAX_REDIRECT_SDK
27
+ # set ascp executable path
26
28
  def ascp_path=(v)
27
29
  @path_to_ascp=v
28
30
  end
29
31
 
32
+ # filename for ascp with optional extension (Windows)
33
+ def ascp_filename
34
+ return 'ascp'+Environment.exe_extension
35
+ end
36
+
30
37
  # location of SDK files
31
38
  def folder=(v)
32
39
  @sdk_folder=v
@@ -50,26 +57,30 @@ module Aspera
50
57
  # @return the list of installed products in format of product_locations
51
58
  def installed_products
52
59
  if @found_products.nil?
53
- @found_products=product_locations
54
- # add sdk as first search path
55
- @found_products.unshift({# SDK
60
+ scan_locations=product_locations.clone
61
+ # add SDK as first search path
62
+ scan_locations.unshift({
56
63
  :expected =>'SDK',
57
64
  :app_root =>folder_path,
58
65
  :sub_bin =>''
59
66
  })
60
- @found_products.select! do |pl|
61
- next false unless Dir.exist?(pl[:app_root])
62
- Log.log.debug("found #{pl[:app_root]}")
63
- sub_bin = pl[:sub_bin] || BIN_SUBFOLDER
64
- pl[:ascp_path]=File.join(pl[:app_root],sub_bin,'ascp'+Environment.exe_extension)
65
- next false unless File.exist?(pl[:ascp_path])
66
- product_info_file="#{pl[:app_root]}/#{PRODUCT_INFO}"
67
+ # search installed products: with ascp
68
+ @found_products=scan_locations.select! do |item|
69
+ # skip if not main folder
70
+ next false unless Dir.exist?(item[:app_root])
71
+ Log.log.debug("Found #{item[:app_root]}")
72
+ sub_bin = item[:sub_bin] || BIN_SUBFOLDER
73
+ item[:ascp_path]=File.join(item[:app_root],sub_bin,ascp_filename)
74
+ # skip if no ascp
75
+ next false unless File.exist?(item[:ascp_path])
76
+ # read info from product info file if present
77
+ product_info_file="#{item[:app_root]}/#{PRODUCT_INFO}"
67
78
  if File.exist?(product_info_file)
68
- res_s=XmlSimple.xml_in(File.read(product_info_file),{"ForceArray"=>false})
69
- pl[:name]=res_s['name']
70
- pl[:version]=res_s['version']
79
+ res_s=XmlSimple.xml_in(File.read(product_info_file),{'ForceArray'=>false})
80
+ item[:name]=res_s['name']
81
+ item[:version]=res_s['version']
71
82
  else
72
- pl[:name]=pl[:expected]
83
+ item[:name]=item[:expected]
73
84
  end
74
85
  true # select this version
75
86
  end
@@ -77,6 +88,7 @@ module Aspera
77
88
  return @found_products
78
89
  end
79
90
 
91
+ # all ascp files (in SDK)
80
92
  FILES=[:ascp,:ascp4,:ssh_bypass_key_dsa,:ssh_bypass_key_rsa,:aspera_license,:aspera_conf,:fallback_cert,:fallback_key]
81
93
 
82
94
  # get path of one resource file of currently activated product
@@ -106,14 +118,8 @@ module Aspera
106
118
  <CONF version="2">
107
119
  <default>
108
120
  <file_system>
109
- <storage_rc>
110
- <adaptive>
111
- true
112
- </adaptive>
113
- </storage_rc>
114
121
  <resume_suffix>.aspera-ckpt</resume_suffix>
115
122
  <partial_file_suffix>.partial</partial_file_suffix>
116
- <replace_illegal_chars>_</replace_illegal_chars>
117
123
  </file_system>
118
124
  </default>
119
125
  </CONF>
@@ -147,7 +153,7 @@ module Aspera
147
153
  return file
148
154
  end
149
155
 
150
- # @returns the file path of local connect where API's URI can be read
156
+ # @return the file path of local connect where API's URI can be read
151
157
  def connect_uri
152
158
  connect=get_product_folders(PRODUCT_CONNECT)
153
159
  folder=File.join(connect[:run_root],VARRUN_SUBFOLDER)
@@ -176,23 +182,67 @@ module Aspera
176
182
  return [:ssh_bypass_key_dsa,:ssh_bypass_key_rsa].map{|i|Installation.instance.path(i)}
177
183
  end
178
184
 
179
- def install_sdk
185
+ # Check that specified path is ascp and get version
186
+ def get_ascp_version(ascp_path)
187
+ raise "File basename of #{ascp_path} must be #{ascp_filename}" unless File.basename(ascp_path).eql?(ascp_filename)
188
+ ascp_version='n/a'
189
+ raise "error in sdk: no ascp included" if ascp_path.nil?
190
+ cmd_out=%x{"#{ascp_path}" -A}
191
+ raise "An error occured when testing #{ascp_filename}: #{cmd_out}" unless $? == 0
192
+ # get version from ascp, only after full extract, as windows requires DLLs (SSL/TLS/etc...)
193
+ m=cmd_out.match(/ascp version (.*)/)
194
+ ascp_version=m[1] unless m.nil?
195
+ end
196
+
197
+ # download aspera SDK or use local file
198
+ # extracts ascp binary for current system architecture
199
+ # @return ascp version (from execution)
200
+ def install_sdk(sdk_url)
180
201
  require 'zip'
181
202
  sdk_zip_path=File.join(Dir.tmpdir,'sdk.zip')
182
- Aspera::Rest.new(base_url: SDK_URL).call(operation: 'GET',save_to_file: sdk_zip_path)
203
+ if sdk_url.start_with?('file:')
204
+ # require specific file scheme: the path part is "relative", or absolute if there are 4 slash
205
+ raise 'use format: file:///<path>' unless sdk_url.start_with?('file:///')
206
+ sdk_zip_path=sdk_url.gsub(%r{^file:///},'')
207
+ else
208
+ redirect_remain=MAX_REDIRECT_SDK
209
+ begin
210
+ Aspera::Rest.new(base_url: sdk_url).call(operation: 'GET',save_to_file: sdk_zip_path)
211
+ rescue Aspera::RestCallError => e
212
+ if e.response.is_a?(Net::HTTPRedirection)
213
+ if redirect_remain > 0
214
+ redirect_remain-=1
215
+ sdk_url=e.response['location']
216
+ retry
217
+ else
218
+ raise "Too many redirect"
219
+ end
220
+ else
221
+ raise e
222
+ end
223
+ end
224
+ end
225
+ # SDK is organized by architecture
183
226
  filter="/#{Environment.architecture}/"
184
227
  ascp_path=nil
228
+ sdk_path=folder_path
229
+ # rename old install
230
+ if File.exist?(File.join(sdk_path,ascp_filename))
231
+ Log.log.warn("Previous install exists, renaming.")
232
+ File.rename(sdk_path,"#{sdk_path}.#{Time.now.strftime("%Y%m%d%H%M%S")}")
233
+ end
185
234
  # first ensure license file is here so that ascp invokation for version works
186
235
  self.path(:aspera_license)
187
236
  self.path(:aspera_conf)
188
237
  Zip::File.open(sdk_zip_path) do |zip_file|
189
238
  zip_file.each do |entry|
239
+ # get only specified arch, but not folder, only files
190
240
  if entry.name.include?(filter) and !entry.name.end_with?('/')
191
- archive_file=File.join(folder_path,File.basename(entry.name))
241
+ archive_file=File.join(sdk_path,File.basename(entry.name))
192
242
  File.open(archive_file, 'wb') do |output_stream|
193
243
  IO.copy_stream(entry.get_input_stream, output_stream)
194
244
  end
195
- if entry.name.include?('ascp')
245
+ if File.basename(entry.name).eql?(ascp_filename)
196
246
  FileUtils.chmod(0755,archive_file)
197
247
  ascp_path=archive_file
198
248
  end
@@ -200,13 +250,7 @@ module Aspera
200
250
  end
201
251
  end
202
252
  File.unlink(sdk_zip_path) rescue nil # Windows may give error
203
- ascp_version='n/a'
204
- raise "error in sdk: no ascp included" if ascp_path.nil?
205
- cmd_out=%x{#{ascp_path} -A}
206
- raise "An error occured when testing ascp: #{cmd_out}" unless $? == 0
207
- # get version from ascp, only after full extract, as windows requires DLLs (SSL/TLS/etc...)
208
- m=cmd_out.match(/ascp version (.*)/)
209
- ascp_version=m[1] unless m.nil?
253
+ ascp_version=get_ascp_version(ascp_path)
210
254
  File.write(File.join(folder_path,PRODUCT_INFO),"<product><name>IBM Aspera SDK</name><version>#{ascp_version}</version></product>")
211
255
  return ascp_version
212
256
  end
@@ -220,16 +264,8 @@ module Aspera
220
264
  PRODUCT_INFO='product-info.mf'
221
265
  # policy for product selection
222
266
  FIRST_FOUND='FIRST'
223
- SDK_URL='https://eudemo.asperademo.com/aspera/faspex/sdk.zip'
224
267
 
225
- private_constant :BIN_SUBFOLDER,:ETC_SUBFOLDER,:VARRUN_SUBFOLDER,:PRODUCT_INFO,:SDK_URL
226
-
227
- # get some specific folder from specific applications: Connect or CLI
228
- def get_product_folders(name)
229
- found=installed_products.select{|i|i[:expected].eql?(name) or i[:name].eql?(name)}
230
- raise "Product: #{name} not found, please install." if found.empty?
231
- return found.first
232
- end
268
+ private_constant :BIN_SUBFOLDER,:ETC_SUBFOLDER,:VARRUN_SUBFOLDER,:PRODUCT_INFO
233
269
 
234
270
  def initialize
235
271
  @path_to_ascp=nil
@@ -237,14 +273,22 @@ module Aspera
237
273
  @found_products=nil
238
274
  end
239
275
 
276
+ # @return folder paths for specified applications
277
+ # @param name Connect or CLI
278
+ def get_product_folders(name)
279
+ found=installed_products.select{|i|i[:expected].eql?(name) or i[:name].eql?(name)}
280
+ raise "Product: #{name} not found, please install." if found.empty?
281
+ return found.first
282
+ end
283
+
284
+ # @return the path to folder where SDK is installed
240
285
  def folder_path
241
- raise "undefined path to SDK" if @sdk_folder.nil?
286
+ raise "Undefined path to SDK" if @sdk_folder.nil?
242
287
  FileUtils.mkdir_p(@sdk_folder) unless Dir.exist?(@sdk_folder)
243
288
  @sdk_folder
244
289
  end
245
290
 
246
- # returns product folders depending on OS
247
- # fields
291
+ # @return product folders depending on OS fields
248
292
  # :expected M app name is taken from the manifest if present, else defaults to this value
249
293
  # :app_root M main folder for the application
250
294
  # :log_root O location of log files (Linux uses syslog)
@@ -286,7 +330,7 @@ module Aspera
286
330
  :log_root =>File.join(Dir.home,'Library','Logs','Aspera_Drive'),
287
331
  :sub_bin =>File.join('Contents','Resources'),
288
332
  }]
289
- else; return [{ # other: Linux and unix family
333
+ else; return [{ # other: Linux and Unix family
290
334
  :expected =>PRODUCT_CONNECT,
291
335
  :app_root =>File.join(Dir.home,'.aspera','connect'),
292
336
  :run_root =>File.join(Dir.home,'.aspera','connect')
@@ -300,6 +344,9 @@ module Aspera
300
344
  end
301
345
  end
302
346
 
347
+ # @return a standard bypass key
348
+ # @param type rsa or dsa
349
+ # @param id in repository 1 for dsa, 2 for rsa
303
350
  def get_key(type,id)
304
351
  hf=['begin','end'].map{|t|"-----#{t} #{type} private key-----".upcase}
305
352
  bin=Base64.strict_encode64(DataRepository.instance.get_bin(id))
@@ -21,16 +21,24 @@ module Aspera
21
21
  ACCESS_KEY_TRANSFER_USER='xfer'
22
22
  # executes a local "ascp", connects mgt port, equivalent of "Fasp Manager"
23
23
  class Local < Manager
24
- ASCP_SPAWN_TIMEOUT_SEC = 3
25
- private_constant :ASCP_SPAWN_TIMEOUT_SEC
26
- # set to false to keep ascp progress bar display (basically: removes ascp's option -q)
24
+ # options for initialize
25
+ DEFAULT_OPTIONS = {
26
+ :spawn_timeout_sec => 3,
27
+ :spawn_delay_sec => 2,
28
+ :wss => false,
29
+ :resume => {}
30
+ }
31
+ DEFAULT_UDP_PORT=33001
32
+ private_constant :DEFAULT_OPTIONS
33
+ # set to false to keep ascp progress bar display ("true" adds ascp's option -q)
27
34
  attr_accessor :quiet
35
+
28
36
  # start ascp transfer (non blocking), single or multi-session
29
37
  # job information added to @jobs
30
38
  # @param transfer_spec [Hash] aspera transfer specification
31
39
  # @param options [Hash] :resumer, :regenerate_token
32
40
  def start_transfer(transfer_spec,options={})
33
- raise "option: must be hash (or nil)" unless options.is_a?(Hash)
41
+ raise 'option: must be hash (or nil)' unless options.is_a?(Hash)
34
42
  job_options = options.clone
35
43
  job_options[:resumer] ||= @resume_policy
36
44
  job_options[:job_id] ||= SecureRandom.uuid
@@ -58,21 +66,26 @@ module Aspera
58
66
 
59
67
  # TODO: check if changing fasp(UDP) port is really necessary, not clear from doc
60
68
  # compute this before using transfer spec, even if the var is not used in single session
61
- multi_session_udp_port_base=33001
62
- multi_session_number=nil
69
+ multi_session_udp_port_base=DEFAULT_UDP_PORT
70
+ multi_session_number=0
63
71
  if transfer_spec.has_key?('multi_session')
64
72
  multi_session_number=transfer_spec['multi_session'].to_i
65
- raise "multi_session(#{transfer_spec['multi_session']}) shall be integer > 1" unless multi_session_number >= 1
66
- # managed here, so delete from transfer spec
67
- transfer_spec.delete('multi_session')
68
- if transfer_spec.has_key?('fasp_port')
69
- multi_session_udp_port_base=transfer_spec['fasp_port']
70
- transfer_spec.delete('fasp_port')
73
+ if multi_session_number < 0
74
+ Log.log.error("multi_session(#{transfer_spec['multi_session']}) shall be integer >= 0")
75
+ multi_session_number = 0
76
+ end
77
+ if multi_session_number > 0
78
+ # managed here, so delete from transfer spec
79
+ transfer_spec.delete('multi_session')
80
+ if transfer_spec.has_key?('fasp_port')
81
+ multi_session_udp_port_base=transfer_spec['fasp_port']
82
+ transfer_spec.delete('fasp_port')
83
+ end
71
84
  end
72
85
  end
73
86
 
74
87
  # compute known args
75
- env_args=Parameters.ts_to_env_args(transfer_spec,wss: @enable_wss)
88
+ env_args=Parameters.ts_to_env_args(transfer_spec,wss: @options[:wss])
76
89
 
77
90
  # add fallback cert and key as arguments if needed
78
91
  if ['1','force'].include?(transfer_spec['http_fallback'])
@@ -98,25 +111,27 @@ module Aspera
98
111
  :options => job_options # [Hash]
99
112
  }
100
113
 
101
- Log.log.debug("starting session thread(s)")
102
- if !multi_session_number
114
+ if multi_session_number <= 1
115
+ Log.log.debug('Starting single session thread')
103
116
  # single session for transfer : simple
104
117
  session[:thread] = Thread.new(session) {|s|transfer_thread_entry(s)}
105
118
  xfer_job[:sessions].push(session)
106
119
  else
120
+ Log.log.debug('Starting multi session threads')
107
121
  1.upto(multi_session_number) do |i|
122
+ sleep(@options[:spawn_delay_sec]) unless i.eql?(1)
108
123
  # do deep copy (each thread has its own copy because it is modified here below and in thread)
109
124
  this_session=session.clone()
110
125
  this_session[:env_args]=this_session[:env_args].clone()
111
126
  this_session[:env_args][:args]=this_session[:env_args][:args].clone()
112
127
  this_session[:env_args][:args].unshift("-C#{i}:#{multi_session_number}")
113
128
  # necessary only if server is not linux, i.e. server does not support port re-use
114
- this_session[:env_args][:args].unshift("-O","#{multi_session_udp_port_base+i-1}")
129
+ this_session[:env_args][:args].unshift('-O',"#{multi_session_udp_port_base+i-1}")
115
130
  this_session[:thread] = Thread.new(this_session) {|s|transfer_thread_entry(s)}
116
131
  xfer_job[:sessions].push(this_session)
117
132
  end
118
133
  end
119
- Log.log.debug("started session thread(s)")
134
+ Log.log.debug('started session thread(s)')
120
135
 
121
136
  # add job to list of jobs
122
137
  @jobs[job_options[:job_id]]=xfer_job
@@ -128,7 +143,7 @@ module Aspera
128
143
  # wait for completion of all jobs started
129
144
  # @return list of :success or error message
130
145
  def wait_for_transfers_completion
131
- Log.log.debug("wait_for_transfers_completion")
146
+ Log.log.debug('wait_for_transfers_completion')
132
147
  # set to non-nil to exit loop
133
148
  result=[]
134
149
  @jobs.each do |id,job|
@@ -138,7 +153,7 @@ module Aspera
138
153
  result.push(session[:error] ? session[:error] : :success)
139
154
  end
140
155
  end
141
- Log.log.debug("all transfers joined")
156
+ Log.log.debug('all transfers joined')
142
157
  # since all are finished and we return the result, clear statuses
143
158
  @jobs.clear
144
159
  return result
@@ -146,7 +161,7 @@ module Aspera
146
161
 
147
162
  # used by asession (to be removed ?)
148
163
  def shutdown
149
- Log.log.debug("fasp local shutdown")
164
+ Log.log.debug('fasp local shutdown')
150
165
  end
151
166
 
152
167
  # This is the low level method to start the "ascp" process
@@ -158,8 +173,10 @@ module Aspera
158
173
  # @param session this session information
159
174
  # could be private method
160
175
  def start_transfer_with_args_env(env_args,session)
161
- raise "env_args must be Hash" unless env_args.is_a?(Hash)
162
- raise "session must be Hash" unless session.is_a?(Hash)
176
+ raise 'env_args must be Hash' unless env_args.is_a?(Hash)
177
+ raise 'session must be Hash' unless session.is_a?(Hash)
178
+ # by default we assume an exception will be raised (for ensure block)
179
+ exception_raised=true
163
180
  begin
164
181
  Log.log.debug("env_args=#{env_args.inspect}")
165
182
  # get location of ascp executable
@@ -182,7 +199,7 @@ module Aspera
182
199
  Log.log.debug("before accept for pid (#{ascp_pid})")
183
200
  # init management socket
184
201
  ascp_mgt_io=nil
185
- Timeout.timeout(ASCP_SPAWN_TIMEOUT_SEC) do
202
+ Timeout.timeout(@options[:spawn_timeout_sec]) do
186
203
  ascp_mgt_io = mgt_sock.accept
187
204
  # management messages include file names which may be utf8
188
205
  # by default socket is US-ASCII
@@ -216,7 +233,7 @@ module Aspera
216
233
  current_event_data[$1] = $2
217
234
  when ''
218
235
  # empty line is separator to end event information
219
- raise "unexpected empty line" if current_event_data.nil?
236
+ raise 'unexpected empty line' if current_event_data.nil?
220
237
  current_event_data[Manager::LISTENER_SESSION_ID_B]=ascp_pid
221
238
  notify_listeners(current_event_text,current_event_data)
222
239
  case current_event_data['Type']
@@ -232,23 +249,27 @@ module Aspera
232
249
  end # case
233
250
  end # loop (process mgt port lines)
234
251
  # check that last status was received before process exit
235
- raise "INTERNAL: nil last status" if last_status_event.nil?
236
- case last_status_event['Type']
237
- when 'DONE'
238
- # return method (or just don't do anything)
239
- return
240
- when 'ERROR'
241
- Log.log.error("code: #{last_status_event['Code']}")
242
- if last_status_event['Description'] =~ /bearer token/i
243
- Log.log.error("need to regenerate token".red)
244
- if session[:options].is_a?(Hash) and session[:options].has_key?(:regenerate_token)
245
- # regenerate token here, expired, or error on it
246
- env_args[:env]['ASPERA_SCP_TOKEN']=session[:options][:regenerate_token].call(true)
252
+ if last_status_event.is_a?(Hash)
253
+ case last_status_event['Type']
254
+ when 'DONE'
255
+ # all went well
256
+ exception_raised=false
257
+ when 'ERROR'
258
+ Log.log.error("code: #{last_status_event['Code']}")
259
+ if last_status_event['Description'] =~ /bearer token/i
260
+ Log.log.error('need to regenerate token'.red)
261
+ if session[:options].is_a?(Hash) and session[:options].has_key?(:regenerate_token)
262
+ # regenerate token here, expired, or error on it
263
+ env_args[:env]['ASPERA_SCP_TOKEN']=session[:options][:regenerate_token].call(true)
264
+ end
247
265
  end
266
+ raise Fasp::Error.new(last_status_event['Description'],last_status_event['Code'].to_i)
267
+ else # case
268
+ raise "unexpected last event type: #{last_status_event['Type']}"
248
269
  end
249
- raise Fasp::Error.new(last_status_event['Description'],last_status_event['Code'].to_i)
250
270
  else
251
- raise "unexpected last event type: #{last_status_event['Type']}"
271
+ exception_raised=false
272
+ Log.log.debug('no status read from ascp mgt port')
252
273
  end
253
274
  rescue SystemCallError => e
254
275
  # Process.spawn
@@ -262,8 +283,18 @@ module Aspera
262
283
  unless ascp_pid.nil?
263
284
  # "wait" for process to avoid zombie
264
285
  Process.wait(ascp_pid)
286
+ status=$?
265
287
  ascp_pid=nil
266
288
  session.delete(:io)
289
+ if !status.success?
290
+ message="ascp failed with code #{status.exitstatus}"
291
+ if exception_raised
292
+ # just debug, as main exception is already here
293
+ Log.log.debug(message)
294
+ else
295
+ raise Fasp::Error.new(message)
296
+ end
297
+ end
267
298
  end
268
299
  end # begin-ensure
269
300
  end # start_transfer_with_args_env
@@ -276,9 +307,9 @@ module Aspera
276
307
  # {'type'=>'DONE'}
277
308
  def send_command(job_id,session_index,data)
278
309
  job=@jobs[job_id]
279
- raise "no such job" if job.nil?
310
+ raise 'no such job' if job.nil?
280
311
  session=job[:sessions][session_index]
281
- raise "no such session" if session.nil?
312
+ raise 'no such session' if session.nil?
282
313
  Log.log.debug("command: #{data}")
283
314
  # build command
284
315
  command=data.
@@ -292,8 +323,8 @@ module Aspera
292
323
 
293
324
  private
294
325
 
295
- def initialize(agent_options=nil)
296
- agent_options||={}
326
+ # @param options : keys(symbol): wss, resume
327
+ def initialize(options=nil)
297
328
  super()
298
329
  # by default no interactive progress bar
299
330
  @quiet=true
@@ -301,8 +332,20 @@ module Aspera
301
332
  @jobs={}
302
333
  # mutex protects global data accessed by threads
303
334
  @mutex=Mutex.new
304
- @resume_policy=ResumePolicy.new(agent_options)
305
- @enable_wss = agent_options[:wss] || false
335
+ # manage options
336
+ @options=DEFAULT_OPTIONS.clone
337
+ if !options.nil?
338
+ raise "expecting Hash (or nil), but have #{options.class}" unless options.is_a?(Hash)
339
+ options.each do |k,v|
340
+ if DEFAULT_OPTIONS.has_key?(k)
341
+ @options[k]=v
342
+ else
343
+ raise "unknown local agent parameter: #{k}, expect one of #{DEFAULT_OPTIONS.keys.map{|i|i.to_s}.join(",")}"
344
+ end
345
+ end
346
+ end
347
+ Log.log.debug("local options= #{options}")
348
+ @resume_policy=ResumePolicy.new(@options[:resume].symbolize_keys)
306
349
  end
307
350
 
308
351
  # transfer thread entry
@@ -310,7 +353,7 @@ module Aspera
310
353
  def transfer_thread_entry(session)
311
354
  begin
312
355
  # set name for logging
313
- Thread.current[:name]="transfer"
356
+ Thread.current[:name]='transfer'
314
357
  Log.log.debug("ENTER (#{Thread.current[:name]})")
315
358
  # start transfer with selected resumer policy
316
359
  session[:options][:resumer].process do
@@ -45,6 +45,18 @@ module Aspera
45
45
  end
46
46
  end # notify_listeners
47
47
 
48
+ def notify_begin(id,size)
49
+ notify_listeners('emulated',{LISTENER_SESSION_ID_B=>id,'Type'=>'NOTIFICATION','PreTransferBytes'=>size})
50
+ end
51
+
52
+ def notify_progress(id,size)
53
+ notify_listeners('emulated',{LISTENER_SESSION_ID_B=>id,'Type'=>'STATS','Bytescont'=>size})
54
+ end
55
+
56
+ def notify_end(id)
57
+ notify_listeners('emulated',{LISTENER_SESSION_ID_B=>id,'Type'=>'DONE'})
58
+ end
59
+
48
60
  public
49
61
  LISTENER_SESSION_ID_B='ListenerSessionId'
50
62
  LISTENER_SESSION_ID_S='listener_session_id'
@@ -60,6 +72,9 @@ module Aspera
60
72
  # start_transfer(transfer_spec,options) : start and wait for completion
61
73
  # wait_for_transfers_completion : wait for termination of all transfers, @return list of : :success or error message
62
74
  # optional: shutdown
75
+
76
+ # This checks the validity of the value returned by wait_for_transfers_completion
77
+ # it must be a list of :success or exception
63
78
  def self.validate_status_list(statuses)
64
79
  raise "internal error: bad statuses type: #{statuses.class}" unless statuses.is_a?(Array)
65
80
  raise "internal error: bad statuses content: #{statuses}" unless statuses.select{|i|!i.eql?(:success) and !i.is_a?(StandardError)}.empty?
@@ -55,7 +55,7 @@ module Aspera
55
55
  trdata=node_api_.read("ops/transfers/#{@transfer_id}")[:data] || {"status"=>"unknown"} rescue {"status"=>"waiting(read error)"}
56
56
  case trdata['status']
57
57
  when 'completed'
58
- notify_listeners('emulated',{Manager::LISTENER_SESSION_ID_B=>@transfer_id,'Type'=>'DONE'})
58
+ notify_end(@transfer_id)
59
59
  break
60
60
  when 'waiting','partially_completed','unknown','waiting(read error)'
61
61
  if spinner.nil?
@@ -69,10 +69,10 @@ module Aspera
69
69
  #puts "running: sessions:#{trdata["sessions"].length}, #{trdata["sessions"].map{|i| i['bytes_transferred']}.join(',')}"
70
70
  if !started and trdata['precalc'].is_a?(Hash) and
71
71
  trdata['precalc']['status'].eql?('ready')
72
- notify_listeners('emulated',{Manager::LISTENER_SESSION_ID_B=>@transfer_id,'Type'=>'NOTIFICATION','PreTransferBytes'=>trdata['precalc']['bytes_expected']})
72
+ notify_begin(@transfer_id,trdata['precalc']['bytes_expected'])
73
73
  started=true
74
74
  else
75
- notify_listeners('emulated',{Manager::LISTENER_SESSION_ID_B=>@transfer_id,'Type'=>'STATS','Bytescont'=>trdata['bytes_transferred']})
75
+ notify_progress(@transfer_id,trdata['bytes_transferred'])
76
76
  end
77
77
  else
78
78
  Log.log.warn("trdata -> #{trdata}")
@@ -81,7 +81,7 @@ module Aspera
81
81
  sleep 1
82
82
  end
83
83
  #TODO get status of sessions
84
- return []
84
+ return []
85
85
  end
86
86
  end
87
87
  end