aspera-cli 4.0.0 → 4.2.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +843 -304
  3. data/bin/dascli +13 -0
  4. data/docs/Makefile +4 -4
  5. data/docs/README.erb.md +805 -172
  6. data/docs/test_env.conf +22 -3
  7. data/examples/aoc.rb +14 -3
  8. data/examples/faspex4.rb +89 -0
  9. data/lib/aspera/aoc.rb +87 -108
  10. data/lib/aspera/cli/formater.rb +2 -0
  11. data/lib/aspera/cli/main.rb +89 -49
  12. data/lib/aspera/cli/plugin.rb +9 -4
  13. data/lib/aspera/cli/plugins/alee.rb +1 -1
  14. data/lib/aspera/cli/plugins/aoc.rb +188 -173
  15. data/lib/aspera/cli/plugins/ats.rb +2 -2
  16. data/lib/aspera/cli/plugins/config.rb +218 -145
  17. data/lib/aspera/cli/plugins/console.rb +2 -2
  18. data/lib/aspera/cli/plugins/faspex.rb +114 -61
  19. data/lib/aspera/cli/plugins/faspex5.rb +85 -43
  20. data/lib/aspera/cli/plugins/node.rb +3 -3
  21. data/lib/aspera/cli/plugins/preview.rb +59 -45
  22. data/lib/aspera/cli/plugins/server.rb +23 -8
  23. data/lib/aspera/cli/transfer_agent.rb +77 -49
  24. data/lib/aspera/cli/version.rb +1 -1
  25. data/lib/aspera/command_line_builder.rb +49 -31
  26. data/lib/aspera/cos_node.rb +33 -28
  27. data/lib/aspera/environment.rb +2 -2
  28. data/lib/aspera/fasp/connect.rb +28 -21
  29. data/lib/aspera/fasp/http_gw.rb +140 -28
  30. data/lib/aspera/fasp/installation.rb +93 -46
  31. data/lib/aspera/fasp/local.rb +88 -45
  32. data/lib/aspera/fasp/manager.rb +15 -0
  33. data/lib/aspera/fasp/node.rb +4 -4
  34. data/lib/aspera/fasp/parameters.rb +59 -101
  35. data/lib/aspera/fasp/parameters.yaml +531 -0
  36. data/lib/aspera/fasp/resume_policy.rb +13 -12
  37. data/lib/aspera/fasp/uri.rb +1 -1
  38. data/lib/aspera/log.rb +1 -1
  39. data/lib/aspera/node.rb +61 -1
  40. data/lib/aspera/oauth.rb +49 -46
  41. data/lib/aspera/persistency_folder.rb +9 -4
  42. data/lib/aspera/preview/file_types.rb +53 -21
  43. data/lib/aspera/preview/generator.rb +3 -3
  44. data/lib/aspera/rest.rb +29 -18
  45. data/lib/aspera/secrets.rb +20 -0
  46. data/lib/aspera/sync.rb +40 -35
  47. data/lib/aspera/temp_file_manager.rb +19 -0
  48. data/lib/aspera/web_auth.rb +105 -0
  49. metadata +54 -20
  50. data/docs/transfer_spec.html +0 -99
@@ -22,11 +22,18 @@ module Aspera
22
22
  PRODUCT_CLI_V1='Aspera CLI'
23
23
  PRODUCT_DRIVE='Aspera Drive'
24
24
  PRODUCT_ENTSRV='Enterprise Server'
25
- # currently used ascp executable
25
+ MAX_REDIRECT_SDK=2
26
+ private_constant :MAX_REDIRECT_SDK
27
+ # set ascp executable path
26
28
  def ascp_path=(v)
27
29
  @path_to_ascp=v
28
30
  end
29
31
 
32
+ # filename for ascp with optional extension (Windows)
33
+ def ascp_filename
34
+ return 'ascp'+Environment.exe_extension
35
+ end
36
+
30
37
  # location of SDK files
31
38
  def folder=(v)
32
39
  @sdk_folder=v
@@ -50,26 +57,30 @@ module Aspera
50
57
  # @return the list of installed products in format of product_locations
51
58
  def installed_products
52
59
  if @found_products.nil?
53
- @found_products=product_locations
54
- # add sdk as first search path
55
- @found_products.unshift({# SDK
60
+ scan_locations=product_locations.clone
61
+ # add SDK as first search path
62
+ scan_locations.unshift({
56
63
  :expected =>'SDK',
57
64
  :app_root =>folder_path,
58
65
  :sub_bin =>''
59
66
  })
60
- @found_products.select! do |pl|
61
- next false unless Dir.exist?(pl[:app_root])
62
- Log.log.debug("found #{pl[:app_root]}")
63
- sub_bin = pl[:sub_bin] || BIN_SUBFOLDER
64
- pl[:ascp_path]=File.join(pl[:app_root],sub_bin,'ascp'+Environment.exe_extension)
65
- next false unless File.exist?(pl[:ascp_path])
66
- product_info_file="#{pl[:app_root]}/#{PRODUCT_INFO}"
67
+ # search installed products: with ascp
68
+ @found_products=scan_locations.select! do |item|
69
+ # skip if not main folder
70
+ next false unless Dir.exist?(item[:app_root])
71
+ Log.log.debug("Found #{item[:app_root]}")
72
+ sub_bin = item[:sub_bin] || BIN_SUBFOLDER
73
+ item[:ascp_path]=File.join(item[:app_root],sub_bin,ascp_filename)
74
+ # skip if no ascp
75
+ next false unless File.exist?(item[:ascp_path])
76
+ # read info from product info file if present
77
+ product_info_file="#{item[:app_root]}/#{PRODUCT_INFO}"
67
78
  if File.exist?(product_info_file)
68
- res_s=XmlSimple.xml_in(File.read(product_info_file),{"ForceArray"=>false})
69
- pl[:name]=res_s['name']
70
- pl[:version]=res_s['version']
79
+ res_s=XmlSimple.xml_in(File.read(product_info_file),{'ForceArray'=>false})
80
+ item[:name]=res_s['name']
81
+ item[:version]=res_s['version']
71
82
  else
72
- pl[:name]=pl[:expected]
83
+ item[:name]=item[:expected]
73
84
  end
74
85
  true # select this version
75
86
  end
@@ -77,6 +88,7 @@ module Aspera
77
88
  return @found_products
78
89
  end
79
90
 
91
+ # all ascp files (in SDK)
80
92
  FILES=[:ascp,:ascp4,:ssh_bypass_key_dsa,:ssh_bypass_key_rsa,:aspera_license,:aspera_conf,:fallback_cert,:fallback_key]
81
93
 
82
94
  # get path of one resource file of currently activated product
@@ -106,14 +118,8 @@ module Aspera
106
118
  <CONF version="2">
107
119
  <default>
108
120
  <file_system>
109
- <storage_rc>
110
- <adaptive>
111
- true
112
- </adaptive>
113
- </storage_rc>
114
121
  <resume_suffix>.aspera-ckpt</resume_suffix>
115
122
  <partial_file_suffix>.partial</partial_file_suffix>
116
- <replace_illegal_chars>_</replace_illegal_chars>
117
123
  </file_system>
118
124
  </default>
119
125
  </CONF>
@@ -147,7 +153,7 @@ module Aspera
147
153
  return file
148
154
  end
149
155
 
150
- # @returns the file path of local connect where API's URI can be read
156
+ # @return the file path of local connect where API's URI can be read
151
157
  def connect_uri
152
158
  connect=get_product_folders(PRODUCT_CONNECT)
153
159
  folder=File.join(connect[:run_root],VARRUN_SUBFOLDER)
@@ -176,23 +182,67 @@ module Aspera
176
182
  return [:ssh_bypass_key_dsa,:ssh_bypass_key_rsa].map{|i|Installation.instance.path(i)}
177
183
  end
178
184
 
179
- def install_sdk
185
+ # Check that specified path is ascp and get version
186
+ def get_ascp_version(ascp_path)
187
+ raise "File basename of #{ascp_path} must be #{ascp_filename}" unless File.basename(ascp_path).eql?(ascp_filename)
188
+ ascp_version='n/a'
189
+ raise "error in sdk: no ascp included" if ascp_path.nil?
190
+ cmd_out=%x{"#{ascp_path}" -A}
191
+ raise "An error occured when testing #{ascp_filename}: #{cmd_out}" unless $? == 0
192
+ # get version from ascp, only after full extract, as windows requires DLLs (SSL/TLS/etc...)
193
+ m=cmd_out.match(/ascp version (.*)/)
194
+ ascp_version=m[1] unless m.nil?
195
+ end
196
+
197
+ # download aspera SDK or use local file
198
+ # extracts ascp binary for current system architecture
199
+ # @return ascp version (from execution)
200
+ def install_sdk(sdk_url)
180
201
  require 'zip'
181
202
  sdk_zip_path=File.join(Dir.tmpdir,'sdk.zip')
182
- Aspera::Rest.new(base_url: SDK_URL).call(operation: 'GET',save_to_file: sdk_zip_path)
203
+ if sdk_url.start_with?('file:')
204
+ # require specific file scheme: the path part is "relative", or absolute if there are 4 slash
205
+ raise 'use format: file:///<path>' unless sdk_url.start_with?('file:///')
206
+ sdk_zip_path=sdk_url.gsub(%r{^file:///},'')
207
+ else
208
+ redirect_remain=MAX_REDIRECT_SDK
209
+ begin
210
+ Aspera::Rest.new(base_url: sdk_url).call(operation: 'GET',save_to_file: sdk_zip_path)
211
+ rescue Aspera::RestCallError => e
212
+ if e.response.is_a?(Net::HTTPRedirection)
213
+ if redirect_remain > 0
214
+ redirect_remain-=1
215
+ sdk_url=e.response['location']
216
+ retry
217
+ else
218
+ raise "Too many redirect"
219
+ end
220
+ else
221
+ raise e
222
+ end
223
+ end
224
+ end
225
+ # SDK is organized by architecture
183
226
  filter="/#{Environment.architecture}/"
184
227
  ascp_path=nil
228
+ sdk_path=folder_path
229
+ # rename old install
230
+ if File.exist?(File.join(sdk_path,ascp_filename))
231
+ Log.log.warn("Previous install exists, renaming.")
232
+ File.rename(sdk_path,"#{sdk_path}.#{Time.now.strftime("%Y%m%d%H%M%S")}")
233
+ end
185
234
  # first ensure license file is here so that ascp invokation for version works
186
235
  self.path(:aspera_license)
187
236
  self.path(:aspera_conf)
188
237
  Zip::File.open(sdk_zip_path) do |zip_file|
189
238
  zip_file.each do |entry|
239
+ # get only specified arch, but not folder, only files
190
240
  if entry.name.include?(filter) and !entry.name.end_with?('/')
191
- archive_file=File.join(folder_path,File.basename(entry.name))
241
+ archive_file=File.join(sdk_path,File.basename(entry.name))
192
242
  File.open(archive_file, 'wb') do |output_stream|
193
243
  IO.copy_stream(entry.get_input_stream, output_stream)
194
244
  end
195
- if entry.name.include?('ascp')
245
+ if File.basename(entry.name).eql?(ascp_filename)
196
246
  FileUtils.chmod(0755,archive_file)
197
247
  ascp_path=archive_file
198
248
  end
@@ -200,13 +250,7 @@ module Aspera
200
250
  end
201
251
  end
202
252
  File.unlink(sdk_zip_path) rescue nil # Windows may give error
203
- ascp_version='n/a'
204
- raise "error in sdk: no ascp included" if ascp_path.nil?
205
- cmd_out=%x{#{ascp_path} -A}
206
- raise "An error occured when testing ascp: #{cmd_out}" unless $? == 0
207
- # get version from ascp, only after full extract, as windows requires DLLs (SSL/TLS/etc...)
208
- m=cmd_out.match(/ascp version (.*)/)
209
- ascp_version=m[1] unless m.nil?
253
+ ascp_version=get_ascp_version(ascp_path)
210
254
  File.write(File.join(folder_path,PRODUCT_INFO),"<product><name>IBM Aspera SDK</name><version>#{ascp_version}</version></product>")
211
255
  return ascp_version
212
256
  end
@@ -220,16 +264,8 @@ module Aspera
220
264
  PRODUCT_INFO='product-info.mf'
221
265
  # policy for product selection
222
266
  FIRST_FOUND='FIRST'
223
- SDK_URL='https://eudemo.asperademo.com/aspera/faspex/sdk.zip'
224
267
 
225
- private_constant :BIN_SUBFOLDER,:ETC_SUBFOLDER,:VARRUN_SUBFOLDER,:PRODUCT_INFO,:SDK_URL
226
-
227
- # get some specific folder from specific applications: Connect or CLI
228
- def get_product_folders(name)
229
- found=installed_products.select{|i|i[:expected].eql?(name) or i[:name].eql?(name)}
230
- raise "Product: #{name} not found, please install." if found.empty?
231
- return found.first
232
- end
268
+ private_constant :BIN_SUBFOLDER,:ETC_SUBFOLDER,:VARRUN_SUBFOLDER,:PRODUCT_INFO
233
269
 
234
270
  def initialize
235
271
  @path_to_ascp=nil
@@ -237,14 +273,22 @@ module Aspera
237
273
  @found_products=nil
238
274
  end
239
275
 
276
+ # @return folder paths for specified applications
277
+ # @param name Connect or CLI
278
+ def get_product_folders(name)
279
+ found=installed_products.select{|i|i[:expected].eql?(name) or i[:name].eql?(name)}
280
+ raise "Product: #{name} not found, please install." if found.empty?
281
+ return found.first
282
+ end
283
+
284
+ # @return the path to folder where SDK is installed
240
285
  def folder_path
241
- raise "undefined path to SDK" if @sdk_folder.nil?
286
+ raise "Undefined path to SDK" if @sdk_folder.nil?
242
287
  FileUtils.mkdir_p(@sdk_folder) unless Dir.exist?(@sdk_folder)
243
288
  @sdk_folder
244
289
  end
245
290
 
246
- # returns product folders depending on OS
247
- # fields
291
+ # @return product folders depending on OS fields
248
292
  # :expected M app name is taken from the manifest if present, else defaults to this value
249
293
  # :app_root M main folder for the application
250
294
  # :log_root O location of log files (Linux uses syslog)
@@ -286,7 +330,7 @@ module Aspera
286
330
  :log_root =>File.join(Dir.home,'Library','Logs','Aspera_Drive'),
287
331
  :sub_bin =>File.join('Contents','Resources'),
288
332
  }]
289
- else; return [{ # other: Linux and unix family
333
+ else; return [{ # other: Linux and Unix family
290
334
  :expected =>PRODUCT_CONNECT,
291
335
  :app_root =>File.join(Dir.home,'.aspera','connect'),
292
336
  :run_root =>File.join(Dir.home,'.aspera','connect')
@@ -300,6 +344,9 @@ module Aspera
300
344
  end
301
345
  end
302
346
 
347
+ # @return a standard bypass key
348
+ # @param type rsa or dsa
349
+ # @param id in repository 1 for dsa, 2 for rsa
303
350
  def get_key(type,id)
304
351
  hf=['begin','end'].map{|t|"-----#{t} #{type} private key-----".upcase}
305
352
  bin=Base64.strict_encode64(DataRepository.instance.get_bin(id))
@@ -21,16 +21,24 @@ module Aspera
21
21
  ACCESS_KEY_TRANSFER_USER='xfer'
22
22
  # executes a local "ascp", connects mgt port, equivalent of "Fasp Manager"
23
23
  class Local < Manager
24
- ASCP_SPAWN_TIMEOUT_SEC = 3
25
- private_constant :ASCP_SPAWN_TIMEOUT_SEC
26
- # set to false to keep ascp progress bar display (basically: removes ascp's option -q)
24
+ # options for initialize
25
+ DEFAULT_OPTIONS = {
26
+ :spawn_timeout_sec => 3,
27
+ :spawn_delay_sec => 2,
28
+ :wss => false,
29
+ :resume => {}
30
+ }
31
+ DEFAULT_UDP_PORT=33001
32
+ private_constant :DEFAULT_OPTIONS
33
+ # set to false to keep ascp progress bar display ("true" adds ascp's option -q)
27
34
  attr_accessor :quiet
35
+
28
36
  # start ascp transfer (non blocking), single or multi-session
29
37
  # job information added to @jobs
30
38
  # @param transfer_spec [Hash] aspera transfer specification
31
39
  # @param options [Hash] :resumer, :regenerate_token
32
40
  def start_transfer(transfer_spec,options={})
33
- raise "option: must be hash (or nil)" unless options.is_a?(Hash)
41
+ raise 'option: must be hash (or nil)' unless options.is_a?(Hash)
34
42
  job_options = options.clone
35
43
  job_options[:resumer] ||= @resume_policy
36
44
  job_options[:job_id] ||= SecureRandom.uuid
@@ -58,21 +66,26 @@ module Aspera
58
66
 
59
67
  # TODO: check if changing fasp(UDP) port is really necessary, not clear from doc
60
68
  # compute this before using transfer spec, even if the var is not used in single session
61
- multi_session_udp_port_base=33001
62
- multi_session_number=nil
69
+ multi_session_udp_port_base=DEFAULT_UDP_PORT
70
+ multi_session_number=0
63
71
  if transfer_spec.has_key?('multi_session')
64
72
  multi_session_number=transfer_spec['multi_session'].to_i
65
- raise "multi_session(#{transfer_spec['multi_session']}) shall be integer > 1" unless multi_session_number >= 1
66
- # managed here, so delete from transfer spec
67
- transfer_spec.delete('multi_session')
68
- if transfer_spec.has_key?('fasp_port')
69
- multi_session_udp_port_base=transfer_spec['fasp_port']
70
- transfer_spec.delete('fasp_port')
73
+ if multi_session_number < 0
74
+ Log.log.error("multi_session(#{transfer_spec['multi_session']}) shall be integer >= 0")
75
+ multi_session_number = 0
76
+ end
77
+ if multi_session_number > 0
78
+ # managed here, so delete from transfer spec
79
+ transfer_spec.delete('multi_session')
80
+ if transfer_spec.has_key?('fasp_port')
81
+ multi_session_udp_port_base=transfer_spec['fasp_port']
82
+ transfer_spec.delete('fasp_port')
83
+ end
71
84
  end
72
85
  end
73
86
 
74
87
  # compute known args
75
- env_args=Parameters.ts_to_env_args(transfer_spec,wss: @enable_wss)
88
+ env_args=Parameters.ts_to_env_args(transfer_spec,wss: @options[:wss])
76
89
 
77
90
  # add fallback cert and key as arguments if needed
78
91
  if ['1','force'].include?(transfer_spec['http_fallback'])
@@ -98,25 +111,27 @@ module Aspera
98
111
  :options => job_options # [Hash]
99
112
  }
100
113
 
101
- Log.log.debug("starting session thread(s)")
102
- if !multi_session_number
114
+ if multi_session_number <= 1
115
+ Log.log.debug('Starting single session thread')
103
116
  # single session for transfer : simple
104
117
  session[:thread] = Thread.new(session) {|s|transfer_thread_entry(s)}
105
118
  xfer_job[:sessions].push(session)
106
119
  else
120
+ Log.log.debug('Starting multi session threads')
107
121
  1.upto(multi_session_number) do |i|
122
+ sleep(@options[:spawn_delay_sec]) unless i.eql?(1)
108
123
  # do deep copy (each thread has its own copy because it is modified here below and in thread)
109
124
  this_session=session.clone()
110
125
  this_session[:env_args]=this_session[:env_args].clone()
111
126
  this_session[:env_args][:args]=this_session[:env_args][:args].clone()
112
127
  this_session[:env_args][:args].unshift("-C#{i}:#{multi_session_number}")
113
128
  # necessary only if server is not linux, i.e. server does not support port re-use
114
- this_session[:env_args][:args].unshift("-O","#{multi_session_udp_port_base+i-1}")
129
+ this_session[:env_args][:args].unshift('-O',"#{multi_session_udp_port_base+i-1}")
115
130
  this_session[:thread] = Thread.new(this_session) {|s|transfer_thread_entry(s)}
116
131
  xfer_job[:sessions].push(this_session)
117
132
  end
118
133
  end
119
- Log.log.debug("started session thread(s)")
134
+ Log.log.debug('started session thread(s)')
120
135
 
121
136
  # add job to list of jobs
122
137
  @jobs[job_options[:job_id]]=xfer_job
@@ -128,7 +143,7 @@ module Aspera
128
143
  # wait for completion of all jobs started
129
144
  # @return list of :success or error message
130
145
  def wait_for_transfers_completion
131
- Log.log.debug("wait_for_transfers_completion")
146
+ Log.log.debug('wait_for_transfers_completion')
132
147
  # set to non-nil to exit loop
133
148
  result=[]
134
149
  @jobs.each do |id,job|
@@ -138,7 +153,7 @@ module Aspera
138
153
  result.push(session[:error] ? session[:error] : :success)
139
154
  end
140
155
  end
141
- Log.log.debug("all transfers joined")
156
+ Log.log.debug('all transfers joined')
142
157
  # since all are finished and we return the result, clear statuses
143
158
  @jobs.clear
144
159
  return result
@@ -146,7 +161,7 @@ module Aspera
146
161
 
147
162
  # used by asession (to be removed ?)
148
163
  def shutdown
149
- Log.log.debug("fasp local shutdown")
164
+ Log.log.debug('fasp local shutdown')
150
165
  end
151
166
 
152
167
  # This is the low level method to start the "ascp" process
@@ -158,8 +173,10 @@ module Aspera
158
173
  # @param session this session information
159
174
  # could be private method
160
175
  def start_transfer_with_args_env(env_args,session)
161
- raise "env_args must be Hash" unless env_args.is_a?(Hash)
162
- raise "session must be Hash" unless session.is_a?(Hash)
176
+ raise 'env_args must be Hash' unless env_args.is_a?(Hash)
177
+ raise 'session must be Hash' unless session.is_a?(Hash)
178
+ # by default we assume an exception will be raised (for ensure block)
179
+ exception_raised=true
163
180
  begin
164
181
  Log.log.debug("env_args=#{env_args.inspect}")
165
182
  # get location of ascp executable
@@ -182,7 +199,7 @@ module Aspera
182
199
  Log.log.debug("before accept for pid (#{ascp_pid})")
183
200
  # init management socket
184
201
  ascp_mgt_io=nil
185
- Timeout.timeout(ASCP_SPAWN_TIMEOUT_SEC) do
202
+ Timeout.timeout(@options[:spawn_timeout_sec]) do
186
203
  ascp_mgt_io = mgt_sock.accept
187
204
  # management messages include file names which may be utf8
188
205
  # by default socket is US-ASCII
@@ -216,7 +233,7 @@ module Aspera
216
233
  current_event_data[$1] = $2
217
234
  when ''
218
235
  # empty line is separator to end event information
219
- raise "unexpected empty line" if current_event_data.nil?
236
+ raise 'unexpected empty line' if current_event_data.nil?
220
237
  current_event_data[Manager::LISTENER_SESSION_ID_B]=ascp_pid
221
238
  notify_listeners(current_event_text,current_event_data)
222
239
  case current_event_data['Type']
@@ -232,23 +249,27 @@ module Aspera
232
249
  end # case
233
250
  end # loop (process mgt port lines)
234
251
  # check that last status was received before process exit
235
- raise "INTERNAL: nil last status" if last_status_event.nil?
236
- case last_status_event['Type']
237
- when 'DONE'
238
- # return method (or just don't do anything)
239
- return
240
- when 'ERROR'
241
- Log.log.error("code: #{last_status_event['Code']}")
242
- if last_status_event['Description'] =~ /bearer token/i
243
- Log.log.error("need to regenerate token".red)
244
- if session[:options].is_a?(Hash) and session[:options].has_key?(:regenerate_token)
245
- # regenerate token here, expired, or error on it
246
- env_args[:env]['ASPERA_SCP_TOKEN']=session[:options][:regenerate_token].call(true)
252
+ if last_status_event.is_a?(Hash)
253
+ case last_status_event['Type']
254
+ when 'DONE'
255
+ # all went well
256
+ exception_raised=false
257
+ when 'ERROR'
258
+ Log.log.error("code: #{last_status_event['Code']}")
259
+ if last_status_event['Description'] =~ /bearer token/i
260
+ Log.log.error('need to regenerate token'.red)
261
+ if session[:options].is_a?(Hash) and session[:options].has_key?(:regenerate_token)
262
+ # regenerate token here, expired, or error on it
263
+ env_args[:env]['ASPERA_SCP_TOKEN']=session[:options][:regenerate_token].call(true)
264
+ end
247
265
  end
266
+ raise Fasp::Error.new(last_status_event['Description'],last_status_event['Code'].to_i)
267
+ else # case
268
+ raise "unexpected last event type: #{last_status_event['Type']}"
248
269
  end
249
- raise Fasp::Error.new(last_status_event['Description'],last_status_event['Code'].to_i)
250
270
  else
251
- raise "unexpected last event type: #{last_status_event['Type']}"
271
+ exception_raised=false
272
+ Log.log.debug('no status read from ascp mgt port')
252
273
  end
253
274
  rescue SystemCallError => e
254
275
  # Process.spawn
@@ -262,8 +283,18 @@ module Aspera
262
283
  unless ascp_pid.nil?
263
284
  # "wait" for process to avoid zombie
264
285
  Process.wait(ascp_pid)
286
+ status=$?
265
287
  ascp_pid=nil
266
288
  session.delete(:io)
289
+ if !status.success?
290
+ message="ascp failed with code #{status.exitstatus}"
291
+ if exception_raised
292
+ # just debug, as main exception is already here
293
+ Log.log.debug(message)
294
+ else
295
+ raise Fasp::Error.new(message)
296
+ end
297
+ end
267
298
  end
268
299
  end # begin-ensure
269
300
  end # start_transfer_with_args_env
@@ -276,9 +307,9 @@ module Aspera
276
307
  # {'type'=>'DONE'}
277
308
  def send_command(job_id,session_index,data)
278
309
  job=@jobs[job_id]
279
- raise "no such job" if job.nil?
310
+ raise 'no such job' if job.nil?
280
311
  session=job[:sessions][session_index]
281
- raise "no such session" if session.nil?
312
+ raise 'no such session' if session.nil?
282
313
  Log.log.debug("command: #{data}")
283
314
  # build command
284
315
  command=data.
@@ -292,8 +323,8 @@ module Aspera
292
323
 
293
324
  private
294
325
 
295
- def initialize(agent_options=nil)
296
- agent_options||={}
326
+ # @param options : keys(symbol): wss, resume
327
+ def initialize(options=nil)
297
328
  super()
298
329
  # by default no interactive progress bar
299
330
  @quiet=true
@@ -301,8 +332,20 @@ module Aspera
301
332
  @jobs={}
302
333
  # mutex protects global data accessed by threads
303
334
  @mutex=Mutex.new
304
- @resume_policy=ResumePolicy.new(agent_options)
305
- @enable_wss = agent_options[:wss] || false
335
+ # manage options
336
+ @options=DEFAULT_OPTIONS.clone
337
+ if !options.nil?
338
+ raise "expecting Hash (or nil), but have #{options.class}" unless options.is_a?(Hash)
339
+ options.each do |k,v|
340
+ if DEFAULT_OPTIONS.has_key?(k)
341
+ @options[k]=v
342
+ else
343
+ raise "unknown local agent parameter: #{k}, expect one of #{DEFAULT_OPTIONS.keys.map{|i|i.to_s}.join(",")}"
344
+ end
345
+ end
346
+ end
347
+ Log.log.debug("local options= #{options}")
348
+ @resume_policy=ResumePolicy.new(@options[:resume].symbolize_keys)
306
349
  end
307
350
 
308
351
  # transfer thread entry
@@ -310,7 +353,7 @@ module Aspera
310
353
  def transfer_thread_entry(session)
311
354
  begin
312
355
  # set name for logging
313
- Thread.current[:name]="transfer"
356
+ Thread.current[:name]='transfer'
314
357
  Log.log.debug("ENTER (#{Thread.current[:name]})")
315
358
  # start transfer with selected resumer policy
316
359
  session[:options][:resumer].process do
@@ -45,6 +45,18 @@ module Aspera
45
45
  end
46
46
  end # notify_listeners
47
47
 
48
+ def notify_begin(id,size)
49
+ notify_listeners('emulated',{LISTENER_SESSION_ID_B=>id,'Type'=>'NOTIFICATION','PreTransferBytes'=>size})
50
+ end
51
+
52
+ def notify_progress(id,size)
53
+ notify_listeners('emulated',{LISTENER_SESSION_ID_B=>id,'Type'=>'STATS','Bytescont'=>size})
54
+ end
55
+
56
+ def notify_end(id)
57
+ notify_listeners('emulated',{LISTENER_SESSION_ID_B=>id,'Type'=>'DONE'})
58
+ end
59
+
48
60
  public
49
61
  LISTENER_SESSION_ID_B='ListenerSessionId'
50
62
  LISTENER_SESSION_ID_S='listener_session_id'
@@ -60,6 +72,9 @@ module Aspera
60
72
  # start_transfer(transfer_spec,options) : start and wait for completion
61
73
  # wait_for_transfers_completion : wait for termination of all transfers, @return list of : :success or error message
62
74
  # optional: shutdown
75
+
76
+ # This checks the validity of the value returned by wait_for_transfers_completion
77
+ # it must be a list of :success or exception
63
78
  def self.validate_status_list(statuses)
64
79
  raise "internal error: bad statuses type: #{statuses.class}" unless statuses.is_a?(Array)
65
80
  raise "internal error: bad statuses content: #{statuses}" unless statuses.select{|i|!i.eql?(:success) and !i.is_a?(StandardError)}.empty?
@@ -55,7 +55,7 @@ module Aspera
55
55
  trdata=node_api_.read("ops/transfers/#{@transfer_id}")[:data] || {"status"=>"unknown"} rescue {"status"=>"waiting(read error)"}
56
56
  case trdata['status']
57
57
  when 'completed'
58
- notify_listeners('emulated',{Manager::LISTENER_SESSION_ID_B=>@transfer_id,'Type'=>'DONE'})
58
+ notify_end(@transfer_id)
59
59
  break
60
60
  when 'waiting','partially_completed','unknown','waiting(read error)'
61
61
  if spinner.nil?
@@ -69,10 +69,10 @@ module Aspera
69
69
  #puts "running: sessions:#{trdata["sessions"].length}, #{trdata["sessions"].map{|i| i['bytes_transferred']}.join(',')}"
70
70
  if !started and trdata['precalc'].is_a?(Hash) and
71
71
  trdata['precalc']['status'].eql?('ready')
72
- notify_listeners('emulated',{Manager::LISTENER_SESSION_ID_B=>@transfer_id,'Type'=>'NOTIFICATION','PreTransferBytes'=>trdata['precalc']['bytes_expected']})
72
+ notify_begin(@transfer_id,trdata['precalc']['bytes_expected'])
73
73
  started=true
74
74
  else
75
- notify_listeners('emulated',{Manager::LISTENER_SESSION_ID_B=>@transfer_id,'Type'=>'STATS','Bytescont'=>trdata['bytes_transferred']})
75
+ notify_progress(@transfer_id,trdata['bytes_transferred'])
76
76
  end
77
77
  else
78
78
  Log.log.warn("trdata -> #{trdata}")
@@ -81,7 +81,7 @@ module Aspera
81
81
  sleep 1
82
82
  end
83
83
  #TODO get status of sessions
84
- return []
84
+ return []
85
85
  end
86
86
  end
87
87
  end