aspera-cli 4.2.1 → 4.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (59) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +1580 -946
  3. data/bin/ascli +1 -1
  4. data/bin/asession +3 -5
  5. data/docs/Makefile +8 -11
  6. data/docs/README.erb.md +1521 -829
  7. data/docs/doc_tools.rb +58 -0
  8. data/docs/test_env.conf +3 -1
  9. data/examples/faspex4.rb +28 -19
  10. data/examples/transfer.rb +2 -2
  11. data/lib/aspera/aoc.rb +157 -134
  12. data/lib/aspera/cli/listener/progress_multi.rb +5 -5
  13. data/lib/aspera/cli/main.rb +106 -48
  14. data/lib/aspera/cli/manager.rb +19 -20
  15. data/lib/aspera/cli/plugin.rb +22 -7
  16. data/lib/aspera/cli/plugins/aoc.rb +260 -208
  17. data/lib/aspera/cli/plugins/ats.rb +11 -10
  18. data/lib/aspera/cli/plugins/bss.rb +2 -2
  19. data/lib/aspera/cli/plugins/config.rb +360 -189
  20. data/lib/aspera/cli/plugins/faspex.rb +119 -56
  21. data/lib/aspera/cli/plugins/faspex5.rb +32 -17
  22. data/lib/aspera/cli/plugins/node.rb +72 -31
  23. data/lib/aspera/cli/plugins/orchestrator.rb +5 -3
  24. data/lib/aspera/cli/plugins/preview.rb +94 -68
  25. data/lib/aspera/cli/plugins/server.rb +16 -5
  26. data/lib/aspera/cli/plugins/shares.rb +17 -0
  27. data/lib/aspera/cli/transfer_agent.rb +64 -82
  28. data/lib/aspera/cli/version.rb +1 -1
  29. data/lib/aspera/command_line_builder.rb +48 -31
  30. data/lib/aspera/cos_node.rb +4 -3
  31. data/lib/aspera/environment.rb +4 -4
  32. data/lib/aspera/fasp/{manager.rb → agent_base.rb} +7 -6
  33. data/lib/aspera/fasp/{connect.rb → agent_connect.rb} +46 -39
  34. data/lib/aspera/fasp/{local.rb → agent_direct.rb} +42 -38
  35. data/lib/aspera/fasp/{http_gw.rb → agent_httpgw.rb} +50 -29
  36. data/lib/aspera/fasp/{node.rb → agent_node.rb} +43 -4
  37. data/lib/aspera/fasp/agent_trsdk.rb +106 -0
  38. data/lib/aspera/fasp/default.rb +17 -0
  39. data/lib/aspera/fasp/installation.rb +64 -48
  40. data/lib/aspera/fasp/parameters.rb +78 -91
  41. data/lib/aspera/fasp/parameters.yaml +531 -0
  42. data/lib/aspera/fasp/uri.rb +1 -1
  43. data/lib/aspera/faspex_gw.rb +12 -11
  44. data/lib/aspera/id_generator.rb +22 -0
  45. data/lib/aspera/keychain/encrypted_hash.rb +120 -0
  46. data/lib/aspera/keychain/macos_security.rb +94 -0
  47. data/lib/aspera/log.rb +45 -32
  48. data/lib/aspera/node.rb +9 -4
  49. data/lib/aspera/oauth.rb +116 -100
  50. data/lib/aspera/persistency_action_once.rb +11 -7
  51. data/lib/aspera/persistency_folder.rb +6 -26
  52. data/lib/aspera/rest.rb +66 -50
  53. data/lib/aspera/sync.rb +40 -35
  54. data/lib/aspera/timer_limiter.rb +22 -0
  55. metadata +86 -29
  56. data/docs/transfer_spec.html +0 -99
  57. data/lib/aspera/api_detector.rb +0 -60
  58. data/lib/aspera/fasp/aoc.rb +0 -24
  59. data/lib/aspera/secrets.rb +0 -20
@@ -6,6 +6,8 @@ require 'aspera/preview/file_types'
6
6
  require 'aspera/persistency_action_once'
7
7
  require 'aspera/node'
8
8
  require 'aspera/hash_ext'
9
+ require 'aspera/timer_limiter'
10
+ require 'aspera/id_generator'
9
11
  require 'date'
10
12
  require 'securerandom'
11
13
 
@@ -24,7 +26,8 @@ module Aspera
24
26
  DEFAULT_PREVIEWS_FOLDER='previews'
25
27
  AK_MARKER_FILE='.aspera_access_key'
26
28
  LOCAL_STORAGE_PCVL='file:///'
27
- private_constant :PREV_GEN_TAG, :PREVIEW_FOLDER_SUFFIX, :PREVIEW_BASENAME, :TMP_DIR_PREFIX, :DEFAULT_PREVIEWS_FOLDER, :LOCAL_STORAGE_PCVL, :AK_MARKER_FILE
29
+ LOG_LIMITER_SEC=30.0
30
+ private_constant :PREV_GEN_TAG, :PREVIEW_FOLDER_SUFFIX, :PREVIEW_BASENAME, :TMP_DIR_PREFIX, :DEFAULT_PREVIEWS_FOLDER, :LOCAL_STORAGE_PCVL, :AK_MARKER_FILE, :LOG_LIMITER_SEC
28
31
 
29
32
  # option_skip_format has special accessors
30
33
  attr_accessor :option_previews_folder
@@ -40,6 +43,8 @@ module Aspera
40
43
  @preview_formats_to_generate=Aspera::Preview::Generator::PREVIEW_FORMATS.clone
41
44
  # options for generation
42
45
  @gen_options=Aspera::Preview::Options.new
46
+ # used to trigger periodic processing
47
+ @periodic=TimerLimiter.new(LOG_LIMITER_SEC)
43
48
  # link CLI options to gen_info attributes
44
49
  self.options.set_obj_attr(:skip_format,self,:option_skip_format,[]) # no skip
45
50
  self.options.set_obj_attr(:folder_reset_cache,self,:option_folder_reset_cache,:no)
@@ -113,13 +118,14 @@ module Aspera
113
118
  end
114
119
 
115
120
  # old version based on folders
116
- def process_trevents(iteration_token)
121
+ # @param iteration_persistency can be nil
122
+ def process_trevents(iteration_persistency)
117
123
  events_filter={
118
124
  'access_key'=>@access_key_self['id'],
119
125
  'type'=>'download.ended'
120
126
  }
121
- # optionally by iteration token
122
- events_filter['iteration_token']=iteration_token unless iteration_token.nil?
127
+ # optionally add iteration token from persistency
128
+ events_filter['iteration_token']=iteration_persistency.data.first unless iteration_persistency.nil?
123
129
  begin
124
130
  events=@api_node.read('events',events_filter)[:data]
125
131
  rescue RestCallError => e
@@ -132,47 +138,63 @@ module Aspera
132
138
  end
133
139
  return if events.empty?
134
140
  events.each do |event|
135
- next unless event['data']['direction'].eql?('receive')
136
- next unless event['data']['status'].eql?('completed')
137
- next unless event['data']['error_code'].eql?(0)
138
- next unless event['data'].dig('tags','aspera',PREV_GEN_TAG).nil?
139
- folder_id=event.dig('data','tags','aspera','node','file_id')
140
- folder_id||=event.dig('data','file_id')
141
- next if folder_id.nil?
142
- folder_entry=@api_node.read("files/#{folder_id}")[:data] rescue nil
143
- next if folder_entry.nil?
144
- scan_folder_files(folder_entry)
141
+ if event['data']['direction'].eql?('receive') and
142
+ event['data']['status'].eql?('completed') and
143
+ event['data']['error_code'].eql?(0) and
144
+ event['data'].dig('tags','aspera',PREV_GEN_TAG).nil?
145
+ folder_id=event.dig('data','tags','aspera','node','file_id')
146
+ folder_id||=event.dig('data','file_id')
147
+ if !folder_id.nil?
148
+ folder_entry=@api_node.read("files/#{folder_id}")[:data] rescue nil
149
+ scan_folder_files(folder_entry) unless folder_entry.nil?
150
+ end
151
+ end
152
+ if @periodic.trigger? or event.equal?(events.last)
153
+ Log.log.info("Processed event #{event['id']}")
154
+ # save checkpoint to avoid losing processing in case of error
155
+ if !iteration_persistency.nil?
156
+ iteration_persistency.data[0]=event['id'].to_s
157
+ iteration_persistency.save
158
+ end
159
+ end
145
160
  end
146
- return events.last['id'].to_s
147
161
  end
148
162
 
149
163
  # requests recent events on node api and process newly modified folders
150
- def process_events(iteration_token)
164
+ def process_events(iteration_persistency)
151
165
  # get new file creation by access key (TODO: what if file already existed?)
152
166
  events_filter={
153
167
  'access_key'=>@access_key_self['id'],
154
168
  'type'=>'file.*'
155
169
  }
156
- # and optionally by iteration token
157
- events_filter['iteration_token']=iteration_token unless iteration_token.nil?
170
+ # optionally add iteration token from persistency
171
+ events_filter['iteration_token']=iteration_persistency.data.first unless iteration_persistency.nil?
158
172
  events=@api_node.read('events',events_filter)[:data]
159
173
  return if events.empty?
160
174
  events.each do |event|
161
175
  # process only files
162
- next unless event.dig('data','type').eql?('file')
163
- file_entry=@api_node.read("files/#{event['data']['id']}")[:data] rescue nil
164
- next if file_entry.nil?
165
- next unless @option_skip_folders.select{|d|file_entry['path'].start_with?(d)}.empty?
166
- file_entry['parent_file_id']=event['data']['parent_file_id']
167
- if event['types'].include?('file.deleted')
168
- Log.log.error('TODO'.red)
176
+ if event.dig('data','type').eql?('file')
177
+ file_entry=@api_node.read("files/#{event['data']['id']}")[:data] rescue nil
178
+ if !file_entry.nil? and
179
+ @option_skip_folders.select{|d|file_entry['path'].start_with?(d)}.empty?
180
+ file_entry['parent_file_id']=event['data']['parent_file_id']
181
+ if event['types'].include?('file.deleted')
182
+ Log.log.error('TODO'.red)
183
+ end
184
+ if event['types'].include?('file.deleted')
185
+ generate_preview(file_entry)
186
+ end
187
+ end
169
188
  end
170
- if event['types'].include?('file.deleted')
171
- generate_preview(file_entry)
189
+ if @periodic.trigger? or event.equal?(events.last)
190
+ Log.log.info("Processing event #{event['id']}")
191
+ # save checkpoint to avoid losing processing in case of error
192
+ if !iteration_persistency.nil?
193
+ iteration_persistency.data[0]=event['id'].to_s
194
+ iteration_persistency.save
195
+ end
172
196
  end
173
197
  end
174
- # write new iteration file
175
- return events.last['id'].to_s
176
198
  end
177
199
 
178
200
  def do_transfer(direction,folder_id,source_filename,destination='/')
@@ -180,15 +202,17 @@ module Aspera
180
202
  if @default_transfer_spec.nil?
181
203
  # make a dummy call to get some default transfer parameters
182
204
  res=@api_node.create('files/upload_setup',{'transfer_requests'=>[{'transfer_request'=>{'paths'=>[{}],'destination_root'=>'/'}}]})
183
- sample_transfer_spec=res[:data]['transfer_specs'].first['transfer_spec']
205
+ template_ts=res[:data]['transfer_specs'].first['transfer_spec']
184
206
  # get ports, anyway that should be 33001 for both. add remote_user ?
185
- @default_transfer_spec=['ssh_port','fasp_port'].inject({}){|h,e|h[e]=sample_transfer_spec[e];h}
207
+ @default_transfer_spec=['ssh_port','fasp_port'].inject({}){|h,e|h[e]=template_ts[e];h}
208
+ if ! @default_transfer_spec['remote_user'].eql?(Aspera::Fasp::Default::ACCESS_KEY_TRANSFER_USER)
209
+ Log.log.warn("remote_user shall be xfer")
210
+ @default_transfer_spec['remote_user']=Aspera::Fasp::Default::ACCESS_KEY_TRANSFER_USER
211
+ end
212
+ Aspera::Node::set_ak_basic_token(@default_transfer_spec,@access_key_self['id'],self.options.get_option(:password,:mandatory))
186
213
  # note: we use the same address for ascp than for node api instead of the one from upload_setup
187
- @default_transfer_spec.merge!({
188
- 'token' => "Basic #{Base64.strict_encode64("#{@access_key_self['id']}:#{self.options.get_option(:password,:mandatory)}")}",
189
- 'remote_host' => @transfer_server_address,
190
- 'remote_user' => Fasp::ACCESS_KEY_TRANSFER_USER
191
- })
214
+ # TODO: configurable ? useful ?
215
+ @default_transfer_spec['remote_host']=@transfer_server_address
192
216
  end
193
217
  tspec=@default_transfer_spec.merge({
194
218
  'direction' => direction,
@@ -338,40 +362,45 @@ module Aspera
338
362
  entry=entries_to_process.shift
339
363
  # process this entry only if it is within the scan_start
340
364
  entry_path_with_slash=entry['path']
365
+ Log.log.info("processing entry #{entry_path_with_slash}") if @periodic.trigger?
341
366
  entry_path_with_slash="#{entry_path_with_slash}/" unless entry_path_with_slash.end_with?('/')
342
367
  if !scan_start.nil? and !scan_start.start_with?(entry_path_with_slash) and !entry_path_with_slash.start_with?(scan_start)
343
368
  Log.log.debug("#{entry['path']} folder (skip start)".bg_red)
344
369
  next
345
370
  end
346
371
  Log.log.debug("item:#{entry}")
347
- case entry['type']
348
- when 'file'
349
- if filter_block.call(entry)
350
- generate_preview(entry)
351
- else
352
- Log.log.debug('skip by filter')
353
- end
354
- when 'link'
355
- Log.log.debug('Ignoring link.')
356
- when 'folder'
357
- if @option_skip_folders.include?(entry['path'])
358
- Log.log.debug("#{entry['path']} folder (skip list)".bg_red)
359
- else
360
- Log.log.debug("#{entry['path']} folder".green)
361
- # get folder content
362
- folder_entries=get_folder_entries(entry['id'])
363
- # process all items in current folder
364
- folder_entries.each do |folder_entry|
365
- # add path for older versions of ES
366
- if !folder_entry.has_key?('path')
367
- folder_entry['path']=entry_path_with_slash+folder_entry['name']
372
+ begin
373
+ case entry['type']
374
+ when 'file'
375
+ if filter_block.call(entry)
376
+ generate_preview(entry)
377
+ else
378
+ Log.log.debug('skip by filter')
379
+ end
380
+ when 'link'
381
+ Log.log.debug('Ignoring link.')
382
+ when 'folder'
383
+ if @option_skip_folders.include?(entry['path'])
384
+ Log.log.debug("#{entry['path']} folder (skip list)".bg_red)
385
+ else
386
+ Log.log.debug("#{entry['path']} folder".green)
387
+ # get folder content
388
+ folder_entries=get_folder_entries(entry['id'])
389
+ # process all items in current folder
390
+ folder_entries.each do |folder_entry|
391
+ # add path for older versions of ES
392
+ if !folder_entry.has_key?('path')
393
+ folder_entry['path']=entry_path_with_slash+folder_entry['name']
394
+ end
395
+ folder_entry['parent_file_id']=entry['id']
396
+ entries_to_process.push(folder_entry)
368
397
  end
369
- folder_entry['parent_file_id']=entry['id']
370
- entries_to_process.push(folder_entry)
371
398
  end
399
+ else
400
+ Log.log.warn("unknown entry type: #{entry['type']}")
372
401
  end
373
- else
374
- Log.log.warn("unknown entry type: #{entry['type']}")
402
+ rescue => e
403
+ Log.log.warn("An error occured: #{e}, ignoring")
375
404
  end
376
405
  end
377
406
  end
@@ -437,18 +466,15 @@ module Aspera
437
466
  scan_folder_files(folder_info,scan_path)
438
467
  return Main.result_status('scan finished')
439
468
  when :events,:trevents
440
- iteration_data=[]
441
469
  iteration_persistency=nil
442
470
  if self.options.get_option(:once_only,:mandatory)
443
471
  iteration_persistency=PersistencyActionOnce.new(
444
472
  manager: @agents[:persistency],
445
- data: iteration_data,
446
- ids: ["preview_iteration_#{command}",self.options.get_option(:url,:mandatory),self.options.get_option(:username,:mandatory)])
473
+ data: [],
474
+ id: IdGenerator.from_list(['preview_iteration',command.to_s,self.options.get_option(:url,:mandatory),self.options.get_option(:username,:mandatory)]))
447
475
  end
448
-
449
- # call method specified
450
- iteration_data[0]=send("process_#{command}",iteration_data[0])
451
- iteration_persistency.save unless iteration_persistency.nil?
476
+ # call processing method specified by command line command
477
+ send("process_#{command}",iteration_persistency)
452
478
  return Main.result_status("#{command} finished")
453
479
  when :check
454
480
  Aspera::Preview::Utils.check_tools(@skip_types)
@@ -1,5 +1,6 @@
1
1
  require 'aspera/cli/basic_auth_plugin'
2
2
  require 'aspera/ascmd'
3
+ require 'aspera/fasp/default'
3
4
  require 'aspera/ssh'
4
5
  require 'aspera/nagios'
5
6
  require 'tempfile'
@@ -67,13 +68,15 @@ module Aspera
67
68
  def execute_action
68
69
  server_uri=URI.parse(self.options.get_option(:url,:mandatory))
69
70
  Log.log.debug("URI : #{server_uri}, port=#{server_uri.port}, scheme:#{server_uri.scheme}")
71
+ server_transfer_spec={'remote_host'=>server_uri.hostname}
70
72
  shell_executor=nil
71
73
  case server_uri.scheme
72
74
  when 'ssh'
73
- server_transfer_spec={
74
- 'remote_host'=>server_uri.hostname,
75
- 'remote_user'=>self.options.get_option(:username,:mandatory),
76
- }
75
+ if self.options.get_option(:username,:optional).nil?
76
+ self.options.set_option(:username,Aspera::Fasp::Default::ACCESS_KEY_TRANSFER_USER)
77
+ Log.log.info("Using default transfer user: #{Aspera::Fasp::Default::ACCESS_KEY_TRANSFER_USER}")
78
+ end
79
+ server_transfer_spec['remote_user']=self.options.get_option(:username,:mandatory)
77
80
  ssh_options=self.options.get_option(:ssh_options,:optional)
78
81
  raise 'expecting a Hash for ssh_options' unless ssh_options.is_a?(Hash)
79
82
  if !server_uri.port.nil?
@@ -102,8 +105,16 @@ module Aspera
102
105
  cred_set=true
103
106
  end
104
107
  end
105
- raise 'either password or key must be provided' if !cred_set
108
+ # if user provided transfer spec has a token, we will use by pass keys
109
+ cred_set=true if self.transfer.option_transfer_spec['token'].is_a?(String)
110
+ raise 'either password, key , or transfer spec token must be provided' if !cred_set
106
111
  shell_executor=Ssh.new(server_transfer_spec['remote_host'],server_transfer_spec['remote_user'],ssh_options)
112
+ when 'https'
113
+ raise "ERROR: transfer spec with token required" unless self.transfer.option_transfer_spec['token'].is_a?(String)
114
+ server_transfer_spec.merge!({
115
+ 'wss_enabled'=>true,
116
+ 'wss_port' =>server_uri.port
117
+ })
107
118
  when 'local'
108
119
  shell_executor=LocalExecutor.new
109
120
  else
@@ -4,6 +4,23 @@ module Aspera
4
4
  module Cli
5
5
  module Plugins
6
6
  class Shares < BasicAuthPlugin
7
+ class << self
8
+ def detect(base_url)
9
+ api=Rest.new({:base_url=>base_url})
10
+ # Shares
11
+ begin
12
+ # shall fail: shares requires auth, but we check error message
13
+ api.read('node_api/app')
14
+ rescue RestCallError => e
15
+ if e.response.code.to_s.eql?('401') and e.response.body.eql?('{"error":{"user_message":"API user authentication failed"}}')
16
+ return {:version=>'unknown'}
17
+ end
18
+ rescue
19
+ end
20
+ nil
21
+ end
22
+ end
23
+
7
24
  def initialize(env)
8
25
  super(env)
9
26
  #self.options.parse_options!
@@ -1,8 +1,4 @@
1
- require 'aspera/fasp/local'
2
- require 'aspera/fasp/connect'
3
- require 'aspera/fasp/node'
4
- require 'aspera/fasp/aoc'
5
- require 'aspera/fasp/http_gw'
1
+ require 'aspera/fasp/parameters'
6
2
  require 'aspera/cli/listener/logger'
7
3
  require 'aspera/cli/listener/progress_multi'
8
4
 
@@ -16,13 +12,25 @@ module Aspera
16
12
  FILE_LIST_FROM_ARGS='@args'
17
13
  # special value for --sources : read file list from transfer spec (--ts)
18
14
  FILE_LIST_FROM_TRANSFER_SPEC='@ts'
19
- private_constant :FILE_LIST_FROM_ARGS,:FILE_LIST_FROM_TRANSFER_SPEC
20
- # @param cli_objects external objects: option manager, config file manager
21
- def initialize(cli_objects)
22
- @opt_mgr=cli_objects[:options]
23
- @config=cli_objects[:config]
24
- # transfer spec overrides provided on command line
25
- @transfer_spec_cmdline={"create_dir"=>true}
15
+ DEFAULT_TRANSFER_NOTIF_TMPL=<<END_OF_TEMPLATE
16
+ From: <%=from_name%> <<%=from_email%>>
17
+ To: <<%=to%>>
18
+ Subject: <%=subject%>
19
+
20
+ Transfer is: <%=global_transfer_status%>
21
+
22
+ <%=ts.to_yaml%>
23
+ END_OF_TEMPLATE
24
+ #% (formating bug in eclipse)
25
+ private_constant :FILE_LIST_FROM_ARGS,:FILE_LIST_FROM_TRANSFER_SPEC,:DEFAULT_TRANSFER_NOTIF_TMPL
26
+ TRANSFER_AGENTS=[:direct,:node,:connect,:httpgw,:trsdk]
27
+
28
+ # @param env external objects: option manager, config file manager
29
+ def initialize(opt_mgr,config)
30
+ @opt_mgr=opt_mgr
31
+ @config=config
32
+ # command line can override transfer spec
33
+ @transfer_spec_cmdline={'create_dir'=>true}
26
34
  # the currently selected transfer agent
27
35
  @agent=nil
28
36
  @progress_listener=Listener::ProgressMulti.new
@@ -33,9 +41,9 @@ module Aspera
33
41
  @opt_mgr.add_opt_simple(:local_resume,"set resume policy (Hash, use @json: prefix), current=#{@opt_mgr.get_option(:local_resume,:optional)}")
34
42
  @opt_mgr.add_opt_simple(:to_folder,"destination folder for downloaded files")
35
43
  @opt_mgr.add_opt_simple(:sources,"list of source files (see doc)")
36
- @opt_mgr.add_opt_simple(:transfer_info,"additional information for transfer client")
44
+ @opt_mgr.add_opt_simple(:transfer_info,"parameters for transfer agent")
37
45
  @opt_mgr.add_opt_list(:src_type,[:list,:pair],"type of file list")
38
- @opt_mgr.add_opt_list(:transfer,[:direct,:httpgw,:connect,:node,:aoc],"type of transfer")
46
+ @opt_mgr.add_opt_list(:transfer,TRANSFER_AGENTS,"type of transfer agent")
39
47
  @opt_mgr.add_opt_list(:progress,[:none,:native,:multi],"type of progress bar")
40
48
  @opt_mgr.set_option(:transfer,:direct)
41
49
  @opt_mgr.set_option(:src_type,:list)
@@ -55,7 +63,7 @@ module Aspera
55
63
  @agent.add_listener(Listener::Logger.new)
56
64
  # use local progress bar if asked so, or if native and non local ascp (because only local ascp has native progress bar)
57
65
  if @opt_mgr.get_option(:progress,:mandatory).eql?(:multi) or
58
- (@opt_mgr.get_option(:progress,:mandatory).eql?(:native) and !@opt_mgr.get_option(:transfer,:mandatory).eql?(:direct))
66
+ (@opt_mgr.get_option(:progress,:mandatory).eql?(:native) and ! instance.class.to_s.eql?('Aspera::Fasp::AgentDirect'))
59
67
  @agent.add_listener(@progress_listener)
60
68
  end
61
69
  end
@@ -64,63 +72,23 @@ module Aspera
64
72
  def set_agent_by_options
65
73
  return nil unless @agent.nil?
66
74
  agent_type=@opt_mgr.get_option(:transfer,:mandatory)
67
- case agent_type
68
- when :direct
69
- agent_options=@opt_mgr.get_option(:transfer_info,:optional)
70
- agent_options=agent_options.symbolize_keys if agent_options.is_a?(Hash)
71
- new_agent=Fasp::Local.new(agent_options)
72
- new_agent.quiet=false if @opt_mgr.get_option(:progress,:mandatory).eql?(:native)
73
- when :httpgw
74
- httpgw_config=@opt_mgr.get_option(:transfer_info,:mandatory)
75
- new_agent=Fasp::HttpGW.new(httpgw_config)
76
- when :connect
77
- new_agent=Fasp::Connect.new
78
- when :node
79
- # way for code to setup alternate node api in advance
80
- # support: @preset:<name>
81
- # support extended values
82
- node_config=@opt_mgr.get_option(:transfer_info,:optional)
83
- # if not specified: use default node
84
- if node_config.nil?
85
- param_set_name=@config.get_plugin_default_config_name(:node)
86
- raise CliBadArgument,"No default node configured, Please specify --#{:transfer_info.to_s.gsub('_','-')}" if param_set_name.nil?
87
- node_config=@config.preset_by_name(param_set_name)
88
- end
89
- Log.log.debug("node=#{node_config}")
90
- raise CliBadArgument,"the node configuration shall be Hash, not #{node_config.class} (#{node_config}), use either @json:<json> or @preset:<parameter set name>" if !node_config.is_a?(Hash)
91
- # now check there are required parameters
92
- sym_config=[:url,:username,:password].inject({}) do |h,param|
93
- raise CliBadArgument,"missing parameter [#{param}] in node specification: #{node_config}" if !node_config.has_key?(param.to_s)
94
- h[param]=node_config[param.to_s]
95
- h
96
- end
97
- node_api=Rest.new({
98
- :base_url => sym_config[:url],
99
- :auth => {
100
- :type =>:basic,
101
- :username => sym_config[:username],
102
- :password => sym_config[:password]
103
- }})
104
- new_agent=Fasp::Node.new(node_api)
105
- when :aoc
106
- aoc_config=@opt_mgr.get_option(:transfer_info,:optional)
107
- if aoc_config.nil?
108
- param_set_name=@config.get_plugin_default_config_name(:aspera)
109
- raise CliBadArgument,"No default AoC configured, Please specify --#{:transfer_info.to_s.gsub('_','-')}" if param_set_name.nil?
110
- aoc_config=@config.preset_by_name(param_set_name)
111
- end
112
- Log.log.debug("aoc=#{aoc_config}")
113
- raise CliBadArgument,"the aoc configuration shall be Hash, not #{aoc_config.class} (#{aoc_config}), refer to manual" if !aoc_config.is_a?(Hash)
114
- # convert keys from string (config) to symbol (agent)
115
- aoc_config=aoc_config.symbolize_keys
116
- # convert auth value from string (config) to symbol (agent)
117
- aoc_config[:auth]=aoc_config[:auth].to_sym if aoc_config[:auth].is_a?(String)
118
- # private key could be @file:... in config
119
- aoc_config[:private_key]=ExtendedValue.instance.evaluate(aoc_config[:private_key])
120
- new_agent=Fasp::Aoc.new(aoc_config)
121
- else
122
- raise "Unexpected transfer agent type: #{agent_type}"
75
+ require "aspera/fasp/agent_#{agent_type}"
76
+ agent_options=@opt_mgr.get_option(:transfer_info,:optional)
77
+ raise CliBadArgument,"the transfer agent configuration shall be Hash, not #{agent_options.class} (#{agent_options}), use either @json:<json> or @preset:<parameter set name>" unless [Hash,NilClass].include?(agent_options.class)
78
+ # special case
79
+ if agent_type.eql?(:node) and agent_options.nil?
80
+ param_set_name=@config.get_plugin_default_config_name(:node)
81
+ raise CliBadArgument,"No default node configured, Please specify --#{:transfer_info.to_s.gsub('_','-')}" if param_set_name.nil?
82
+ agent_options=@config.preset_by_name(param_set_name)
83
+ end
84
+ # special case
85
+ if agent_type.eql?(:direct) and @opt_mgr.get_option(:progress,:mandatory).eql?(:native)
86
+ agent_options={} if agent_options.nil?
87
+ agent_options[:quiet]=false
123
88
  end
89
+ agent_options=agent_options.symbolize_keys if agent_options.is_a?(Hash)
90
+ # get agent instance
91
+ new_agent=Kernel.const_get("Aspera::Fasp::Agent#{agent_type.capitalize}").new(agent_options)
124
92
  set_agent_instance(new_agent)
125
93
  return nil
126
94
  end
@@ -130,7 +98,7 @@ module Aspera
130
98
  # param: 'send' or 'receive'
131
99
  def destination_folder(direction)
132
100
  dest_folder=@opt_mgr.get_option(:to_folder,:optional)
133
- return dest_folder unless dest_folder.nil?
101
+ return File.expand_path(dest_folder) unless dest_folder.nil?
134
102
  dest_folder=@transfer_spec_cmdline['destination_root']
135
103
  return dest_folder unless dest_folder.nil?
136
104
  # default: / on remote, . on local
@@ -161,7 +129,8 @@ module Aspera
161
129
  raise CliBadArgument,"specify at least one file on command line or use --sources=#{FILE_LIST_FROM_TRANSFER_SPEC} to use transfer spec" if !file_list.is_a?(Array) or file_list.empty?
162
130
  when FILE_LIST_FROM_TRANSFER_SPEC
163
131
  Log.log.debug("assume list provided in transfer spec")
164
- raise CliBadArgument,"transfer spec on command line must have sources" if @transfer_paths.nil?
132
+ special_case_direct_with_list=@opt_mgr.get_option(:transfer,:mandatory).eql?(:direct) and Fasp::Parameters.ts_has_file_list(@transfer_spec_cmdline)
133
+ raise CliBadArgument,"transfer spec on command line must have sources" if @transfer_paths.nil? and !special_case_direct_with_list
165
134
  # here we assume check of sources is made in transfer agent
166
135
  return @transfer_paths
167
136
  when Array
@@ -189,20 +158,20 @@ module Aspera
189
158
 
190
159
  # start a transfer and wait for completion, plugins shall use this method
191
160
  # @param transfer_spec
192
- # @param options specific options for the transfer_agent
193
- # options[:src] specifies how destination_root is set (how transfer spec was generated)
161
+ # @param tr_opts specific options for the transfer_agent
162
+ # tr_opts[:src] specifies how destination_root is set (how transfer spec was generated)
194
163
  # other options are carried to specific agent
195
- def start(transfer_spec,options)
164
+ def start(transfer_spec,tr_opts)
196
165
  # check parameters
197
166
  raise "transfer_spec must be hash" unless transfer_spec.is_a?(Hash)
198
- raise "options must be hash" unless options.is_a?(Hash)
167
+ raise "tr_opts must be hash" unless tr_opts.is_a?(Hash)
199
168
  # process :src option
200
169
  case transfer_spec['direction']
201
170
  when 'receive'
202
171
  # init default if required in any case
203
172
  @transfer_spec_cmdline['destination_root']||=destination_folder(transfer_spec['direction'])
204
173
  when 'send'
205
- case options[:src]
174
+ case tr_opts[:src]
206
175
  when :direct
207
176
  # init default if required
208
177
  @transfer_spec_cmdline['destination_root']||=destination_folder(transfer_spec['direction'])
@@ -213,12 +182,12 @@ module Aspera
213
182
  when :node_gen4
214
183
  @transfer_spec_cmdline.delete('destination_root') if @transfer_spec_cmdline.has_key?('destination_root_id')
215
184
  else
216
- raise StandardError,"InternalError: unsupported value: #{options[:src]}"
185
+ raise StandardError,"InternalError: unsupported value: #{tr_opts[:src]}"
217
186
  end
218
187
  end
219
188
 
220
189
  # only used here
221
- options.delete(:src)
190
+ tr_opts.delete(:src)
222
191
 
223
192
  # update command line paths, unless destination already has one
224
193
  @transfer_spec_cmdline['paths']=transfer_spec['paths'] || ts_source_paths
@@ -227,13 +196,26 @@ module Aspera
227
196
  # create transfer agent
228
197
  self.set_agent_by_options
229
198
  Log.log.debug("transfer agent is a #{@agent.class}")
230
- @agent.start_transfer(transfer_spec,options)
199
+ @agent.start_transfer(transfer_spec,tr_opts)
231
200
  result=@agent.wait_for_transfers_completion
232
201
  @progress_listener.reset
233
- Fasp::Manager.validate_status_list(result)
202
+ Fasp::AgentBase.validate_status_list(result)
203
+ send_email_transfer_notification(transfer_spec,result)
234
204
  return result
235
205
  end
236
206
 
207
+ def send_email_transfer_notification(transfer_spec,statuses)
208
+ return if @opt_mgr.get_option(:notif_to,:optional).nil?
209
+ global_status=self.class.session_status(statuses)
210
+ email_vars={
211
+ global_transfer_status: global_status,
212
+ subject: "ascli transfer: #{global_status}",
213
+ body: "Transfer is: #{global_status}",
214
+ ts: transfer_spec
215
+ }
216
+ @config.send_email_template(email_vars,DEFAULT_TRANSFER_NOTIF_TMPL)
217
+ end
218
+
237
219
  # @return :success if all sessions statuses returned by "start" are success
238
220
  # else return the first error exception object
239
221
  def self.session_status(statuses)
@@ -1,5 +1,5 @@
1
1
  module Aspera
2
2
  module Cli
3
- VERSION = "4.2.1"
3
+ VERSION = "4.5.0"
4
4
  end
5
5
  end