aspera-cli 4.2.1 → 4.5.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (59) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +1580 -946
  3. data/bin/ascli +1 -1
  4. data/bin/asession +3 -5
  5. data/docs/Makefile +8 -11
  6. data/docs/README.erb.md +1521 -829
  7. data/docs/doc_tools.rb +58 -0
  8. data/docs/test_env.conf +3 -1
  9. data/examples/faspex4.rb +28 -19
  10. data/examples/transfer.rb +2 -2
  11. data/lib/aspera/aoc.rb +157 -134
  12. data/lib/aspera/cli/listener/progress_multi.rb +5 -5
  13. data/lib/aspera/cli/main.rb +106 -48
  14. data/lib/aspera/cli/manager.rb +19 -20
  15. data/lib/aspera/cli/plugin.rb +22 -7
  16. data/lib/aspera/cli/plugins/aoc.rb +260 -208
  17. data/lib/aspera/cli/plugins/ats.rb +11 -10
  18. data/lib/aspera/cli/plugins/bss.rb +2 -2
  19. data/lib/aspera/cli/plugins/config.rb +360 -189
  20. data/lib/aspera/cli/plugins/faspex.rb +119 -56
  21. data/lib/aspera/cli/plugins/faspex5.rb +32 -17
  22. data/lib/aspera/cli/plugins/node.rb +72 -31
  23. data/lib/aspera/cli/plugins/orchestrator.rb +5 -3
  24. data/lib/aspera/cli/plugins/preview.rb +94 -68
  25. data/lib/aspera/cli/plugins/server.rb +16 -5
  26. data/lib/aspera/cli/plugins/shares.rb +17 -0
  27. data/lib/aspera/cli/transfer_agent.rb +64 -82
  28. data/lib/aspera/cli/version.rb +1 -1
  29. data/lib/aspera/command_line_builder.rb +48 -31
  30. data/lib/aspera/cos_node.rb +4 -3
  31. data/lib/aspera/environment.rb +4 -4
  32. data/lib/aspera/fasp/{manager.rb → agent_base.rb} +7 -6
  33. data/lib/aspera/fasp/{connect.rb → agent_connect.rb} +46 -39
  34. data/lib/aspera/fasp/{local.rb → agent_direct.rb} +42 -38
  35. data/lib/aspera/fasp/{http_gw.rb → agent_httpgw.rb} +50 -29
  36. data/lib/aspera/fasp/{node.rb → agent_node.rb} +43 -4
  37. data/lib/aspera/fasp/agent_trsdk.rb +106 -0
  38. data/lib/aspera/fasp/default.rb +17 -0
  39. data/lib/aspera/fasp/installation.rb +64 -48
  40. data/lib/aspera/fasp/parameters.rb +78 -91
  41. data/lib/aspera/fasp/parameters.yaml +531 -0
  42. data/lib/aspera/fasp/uri.rb +1 -1
  43. data/lib/aspera/faspex_gw.rb +12 -11
  44. data/lib/aspera/id_generator.rb +22 -0
  45. data/lib/aspera/keychain/encrypted_hash.rb +120 -0
  46. data/lib/aspera/keychain/macos_security.rb +94 -0
  47. data/lib/aspera/log.rb +45 -32
  48. data/lib/aspera/node.rb +9 -4
  49. data/lib/aspera/oauth.rb +116 -100
  50. data/lib/aspera/persistency_action_once.rb +11 -7
  51. data/lib/aspera/persistency_folder.rb +6 -26
  52. data/lib/aspera/rest.rb +66 -50
  53. data/lib/aspera/sync.rb +40 -35
  54. data/lib/aspera/timer_limiter.rb +22 -0
  55. metadata +86 -29
  56. data/docs/transfer_spec.html +0 -99
  57. data/lib/aspera/api_detector.rb +0 -60
  58. data/lib/aspera/fasp/aoc.rb +0 -24
  59. data/lib/aspera/secrets.rb +0 -20
@@ -6,6 +6,8 @@ require 'aspera/preview/file_types'
6
6
  require 'aspera/persistency_action_once'
7
7
  require 'aspera/node'
8
8
  require 'aspera/hash_ext'
9
+ require 'aspera/timer_limiter'
10
+ require 'aspera/id_generator'
9
11
  require 'date'
10
12
  require 'securerandom'
11
13
 
@@ -24,7 +26,8 @@ module Aspera
24
26
  DEFAULT_PREVIEWS_FOLDER='previews'
25
27
  AK_MARKER_FILE='.aspera_access_key'
26
28
  LOCAL_STORAGE_PCVL='file:///'
27
- private_constant :PREV_GEN_TAG, :PREVIEW_FOLDER_SUFFIX, :PREVIEW_BASENAME, :TMP_DIR_PREFIX, :DEFAULT_PREVIEWS_FOLDER, :LOCAL_STORAGE_PCVL, :AK_MARKER_FILE
29
+ LOG_LIMITER_SEC=30.0
30
+ private_constant :PREV_GEN_TAG, :PREVIEW_FOLDER_SUFFIX, :PREVIEW_BASENAME, :TMP_DIR_PREFIX, :DEFAULT_PREVIEWS_FOLDER, :LOCAL_STORAGE_PCVL, :AK_MARKER_FILE, :LOG_LIMITER_SEC
28
31
 
29
32
  # option_skip_format has special accessors
30
33
  attr_accessor :option_previews_folder
@@ -40,6 +43,8 @@ module Aspera
40
43
  @preview_formats_to_generate=Aspera::Preview::Generator::PREVIEW_FORMATS.clone
41
44
  # options for generation
42
45
  @gen_options=Aspera::Preview::Options.new
46
+ # used to trigger periodic processing
47
+ @periodic=TimerLimiter.new(LOG_LIMITER_SEC)
43
48
  # link CLI options to gen_info attributes
44
49
  self.options.set_obj_attr(:skip_format,self,:option_skip_format,[]) # no skip
45
50
  self.options.set_obj_attr(:folder_reset_cache,self,:option_folder_reset_cache,:no)
@@ -113,13 +118,14 @@ module Aspera
113
118
  end
114
119
 
115
120
  # old version based on folders
116
- def process_trevents(iteration_token)
121
+ # @param iteration_persistency can be nil
122
+ def process_trevents(iteration_persistency)
117
123
  events_filter={
118
124
  'access_key'=>@access_key_self['id'],
119
125
  'type'=>'download.ended'
120
126
  }
121
- # optionally by iteration token
122
- events_filter['iteration_token']=iteration_token unless iteration_token.nil?
127
+ # optionally add iteration token from persistency
128
+ events_filter['iteration_token']=iteration_persistency.data.first unless iteration_persistency.nil?
123
129
  begin
124
130
  events=@api_node.read('events',events_filter)[:data]
125
131
  rescue RestCallError => e
@@ -132,47 +138,63 @@ module Aspera
132
138
  end
133
139
  return if events.empty?
134
140
  events.each do |event|
135
- next unless event['data']['direction'].eql?('receive')
136
- next unless event['data']['status'].eql?('completed')
137
- next unless event['data']['error_code'].eql?(0)
138
- next unless event['data'].dig('tags','aspera',PREV_GEN_TAG).nil?
139
- folder_id=event.dig('data','tags','aspera','node','file_id')
140
- folder_id||=event.dig('data','file_id')
141
- next if folder_id.nil?
142
- folder_entry=@api_node.read("files/#{folder_id}")[:data] rescue nil
143
- next if folder_entry.nil?
144
- scan_folder_files(folder_entry)
141
+ if event['data']['direction'].eql?('receive') and
142
+ event['data']['status'].eql?('completed') and
143
+ event['data']['error_code'].eql?(0) and
144
+ event['data'].dig('tags','aspera',PREV_GEN_TAG).nil?
145
+ folder_id=event.dig('data','tags','aspera','node','file_id')
146
+ folder_id||=event.dig('data','file_id')
147
+ if !folder_id.nil?
148
+ folder_entry=@api_node.read("files/#{folder_id}")[:data] rescue nil
149
+ scan_folder_files(folder_entry) unless folder_entry.nil?
150
+ end
151
+ end
152
+ if @periodic.trigger? or event.equal?(events.last)
153
+ Log.log.info("Processed event #{event['id']}")
154
+ # save checkpoint to avoid losing processing in case of error
155
+ if !iteration_persistency.nil?
156
+ iteration_persistency.data[0]=event['id'].to_s
157
+ iteration_persistency.save
158
+ end
159
+ end
145
160
  end
146
- return events.last['id'].to_s
147
161
  end
148
162
 
149
163
  # requests recent events on node api and process newly modified folders
150
- def process_events(iteration_token)
164
+ def process_events(iteration_persistency)
151
165
  # get new file creation by access key (TODO: what if file already existed?)
152
166
  events_filter={
153
167
  'access_key'=>@access_key_self['id'],
154
168
  'type'=>'file.*'
155
169
  }
156
- # and optionally by iteration token
157
- events_filter['iteration_token']=iteration_token unless iteration_token.nil?
170
+ # optionally add iteration token from persistency
171
+ events_filter['iteration_token']=iteration_persistency.data.first unless iteration_persistency.nil?
158
172
  events=@api_node.read('events',events_filter)[:data]
159
173
  return if events.empty?
160
174
  events.each do |event|
161
175
  # process only files
162
- next unless event.dig('data','type').eql?('file')
163
- file_entry=@api_node.read("files/#{event['data']['id']}")[:data] rescue nil
164
- next if file_entry.nil?
165
- next unless @option_skip_folders.select{|d|file_entry['path'].start_with?(d)}.empty?
166
- file_entry['parent_file_id']=event['data']['parent_file_id']
167
- if event['types'].include?('file.deleted')
168
- Log.log.error('TODO'.red)
176
+ if event.dig('data','type').eql?('file')
177
+ file_entry=@api_node.read("files/#{event['data']['id']}")[:data] rescue nil
178
+ if !file_entry.nil? and
179
+ @option_skip_folders.select{|d|file_entry['path'].start_with?(d)}.empty?
180
+ file_entry['parent_file_id']=event['data']['parent_file_id']
181
+ if event['types'].include?('file.deleted')
182
+ Log.log.error('TODO'.red)
183
+ end
184
+ if event['types'].include?('file.deleted')
185
+ generate_preview(file_entry)
186
+ end
187
+ end
169
188
  end
170
- if event['types'].include?('file.deleted')
171
- generate_preview(file_entry)
189
+ if @periodic.trigger? or event.equal?(events.last)
190
+ Log.log.info("Processing event #{event['id']}")
191
+ # save checkpoint to avoid losing processing in case of error
192
+ if !iteration_persistency.nil?
193
+ iteration_persistency.data[0]=event['id'].to_s
194
+ iteration_persistency.save
195
+ end
172
196
  end
173
197
  end
174
- # write new iteration file
175
- return events.last['id'].to_s
176
198
  end
177
199
 
178
200
  def do_transfer(direction,folder_id,source_filename,destination='/')
@@ -180,15 +202,17 @@ module Aspera
180
202
  if @default_transfer_spec.nil?
181
203
  # make a dummy call to get some default transfer parameters
182
204
  res=@api_node.create('files/upload_setup',{'transfer_requests'=>[{'transfer_request'=>{'paths'=>[{}],'destination_root'=>'/'}}]})
183
- sample_transfer_spec=res[:data]['transfer_specs'].first['transfer_spec']
205
+ template_ts=res[:data]['transfer_specs'].first['transfer_spec']
184
206
  # get ports, anyway that should be 33001 for both. add remote_user ?
185
- @default_transfer_spec=['ssh_port','fasp_port'].inject({}){|h,e|h[e]=sample_transfer_spec[e];h}
207
+ @default_transfer_spec=['ssh_port','fasp_port'].inject({}){|h,e|h[e]=template_ts[e];h}
208
+ if ! @default_transfer_spec['remote_user'].eql?(Aspera::Fasp::Default::ACCESS_KEY_TRANSFER_USER)
209
+ Log.log.warn("remote_user shall be xfer")
210
+ @default_transfer_spec['remote_user']=Aspera::Fasp::Default::ACCESS_KEY_TRANSFER_USER
211
+ end
212
+ Aspera::Node::set_ak_basic_token(@default_transfer_spec,@access_key_self['id'],self.options.get_option(:password,:mandatory))
186
213
  # note: we use the same address for ascp than for node api instead of the one from upload_setup
187
- @default_transfer_spec.merge!({
188
- 'token' => "Basic #{Base64.strict_encode64("#{@access_key_self['id']}:#{self.options.get_option(:password,:mandatory)}")}",
189
- 'remote_host' => @transfer_server_address,
190
- 'remote_user' => Fasp::ACCESS_KEY_TRANSFER_USER
191
- })
214
+ # TODO: configurable ? useful ?
215
+ @default_transfer_spec['remote_host']=@transfer_server_address
192
216
  end
193
217
  tspec=@default_transfer_spec.merge({
194
218
  'direction' => direction,
@@ -338,40 +362,45 @@ module Aspera
338
362
  entry=entries_to_process.shift
339
363
  # process this entry only if it is within the scan_start
340
364
  entry_path_with_slash=entry['path']
365
+ Log.log.info("processing entry #{entry_path_with_slash}") if @periodic.trigger?
341
366
  entry_path_with_slash="#{entry_path_with_slash}/" unless entry_path_with_slash.end_with?('/')
342
367
  if !scan_start.nil? and !scan_start.start_with?(entry_path_with_slash) and !entry_path_with_slash.start_with?(scan_start)
343
368
  Log.log.debug("#{entry['path']} folder (skip start)".bg_red)
344
369
  next
345
370
  end
346
371
  Log.log.debug("item:#{entry}")
347
- case entry['type']
348
- when 'file'
349
- if filter_block.call(entry)
350
- generate_preview(entry)
351
- else
352
- Log.log.debug('skip by filter')
353
- end
354
- when 'link'
355
- Log.log.debug('Ignoring link.')
356
- when 'folder'
357
- if @option_skip_folders.include?(entry['path'])
358
- Log.log.debug("#{entry['path']} folder (skip list)".bg_red)
359
- else
360
- Log.log.debug("#{entry['path']} folder".green)
361
- # get folder content
362
- folder_entries=get_folder_entries(entry['id'])
363
- # process all items in current folder
364
- folder_entries.each do |folder_entry|
365
- # add path for older versions of ES
366
- if !folder_entry.has_key?('path')
367
- folder_entry['path']=entry_path_with_slash+folder_entry['name']
372
+ begin
373
+ case entry['type']
374
+ when 'file'
375
+ if filter_block.call(entry)
376
+ generate_preview(entry)
377
+ else
378
+ Log.log.debug('skip by filter')
379
+ end
380
+ when 'link'
381
+ Log.log.debug('Ignoring link.')
382
+ when 'folder'
383
+ if @option_skip_folders.include?(entry['path'])
384
+ Log.log.debug("#{entry['path']} folder (skip list)".bg_red)
385
+ else
386
+ Log.log.debug("#{entry['path']} folder".green)
387
+ # get folder content
388
+ folder_entries=get_folder_entries(entry['id'])
389
+ # process all items in current folder
390
+ folder_entries.each do |folder_entry|
391
+ # add path for older versions of ES
392
+ if !folder_entry.has_key?('path')
393
+ folder_entry['path']=entry_path_with_slash+folder_entry['name']
394
+ end
395
+ folder_entry['parent_file_id']=entry['id']
396
+ entries_to_process.push(folder_entry)
368
397
  end
369
- folder_entry['parent_file_id']=entry['id']
370
- entries_to_process.push(folder_entry)
371
398
  end
399
+ else
400
+ Log.log.warn("unknown entry type: #{entry['type']}")
372
401
  end
373
- else
374
- Log.log.warn("unknown entry type: #{entry['type']}")
402
+ rescue => e
403
+ Log.log.warn("An error occured: #{e}, ignoring")
375
404
  end
376
405
  end
377
406
  end
@@ -437,18 +466,15 @@ module Aspera
437
466
  scan_folder_files(folder_info,scan_path)
438
467
  return Main.result_status('scan finished')
439
468
  when :events,:trevents
440
- iteration_data=[]
441
469
  iteration_persistency=nil
442
470
  if self.options.get_option(:once_only,:mandatory)
443
471
  iteration_persistency=PersistencyActionOnce.new(
444
472
  manager: @agents[:persistency],
445
- data: iteration_data,
446
- ids: ["preview_iteration_#{command}",self.options.get_option(:url,:mandatory),self.options.get_option(:username,:mandatory)])
473
+ data: [],
474
+ id: IdGenerator.from_list(['preview_iteration',command.to_s,self.options.get_option(:url,:mandatory),self.options.get_option(:username,:mandatory)]))
447
475
  end
448
-
449
- # call method specified
450
- iteration_data[0]=send("process_#{command}",iteration_data[0])
451
- iteration_persistency.save unless iteration_persistency.nil?
476
+ # call processing method specified by command line command
477
+ send("process_#{command}",iteration_persistency)
452
478
  return Main.result_status("#{command} finished")
453
479
  when :check
454
480
  Aspera::Preview::Utils.check_tools(@skip_types)
@@ -1,5 +1,6 @@
1
1
  require 'aspera/cli/basic_auth_plugin'
2
2
  require 'aspera/ascmd'
3
+ require 'aspera/fasp/default'
3
4
  require 'aspera/ssh'
4
5
  require 'aspera/nagios'
5
6
  require 'tempfile'
@@ -67,13 +68,15 @@ module Aspera
67
68
  def execute_action
68
69
  server_uri=URI.parse(self.options.get_option(:url,:mandatory))
69
70
  Log.log.debug("URI : #{server_uri}, port=#{server_uri.port}, scheme:#{server_uri.scheme}")
71
+ server_transfer_spec={'remote_host'=>server_uri.hostname}
70
72
  shell_executor=nil
71
73
  case server_uri.scheme
72
74
  when 'ssh'
73
- server_transfer_spec={
74
- 'remote_host'=>server_uri.hostname,
75
- 'remote_user'=>self.options.get_option(:username,:mandatory),
76
- }
75
+ if self.options.get_option(:username,:optional).nil?
76
+ self.options.set_option(:username,Aspera::Fasp::Default::ACCESS_KEY_TRANSFER_USER)
77
+ Log.log.info("Using default transfer user: #{Aspera::Fasp::Default::ACCESS_KEY_TRANSFER_USER}")
78
+ end
79
+ server_transfer_spec['remote_user']=self.options.get_option(:username,:mandatory)
77
80
  ssh_options=self.options.get_option(:ssh_options,:optional)
78
81
  raise 'expecting a Hash for ssh_options' unless ssh_options.is_a?(Hash)
79
82
  if !server_uri.port.nil?
@@ -102,8 +105,16 @@ module Aspera
102
105
  cred_set=true
103
106
  end
104
107
  end
105
- raise 'either password or key must be provided' if !cred_set
108
+ # if user provided transfer spec has a token, we will use by pass keys
109
+ cred_set=true if self.transfer.option_transfer_spec['token'].is_a?(String)
110
+ raise 'either password, key , or transfer spec token must be provided' if !cred_set
106
111
  shell_executor=Ssh.new(server_transfer_spec['remote_host'],server_transfer_spec['remote_user'],ssh_options)
112
+ when 'https'
113
+ raise "ERROR: transfer spec with token required" unless self.transfer.option_transfer_spec['token'].is_a?(String)
114
+ server_transfer_spec.merge!({
115
+ 'wss_enabled'=>true,
116
+ 'wss_port' =>server_uri.port
117
+ })
107
118
  when 'local'
108
119
  shell_executor=LocalExecutor.new
109
120
  else
@@ -4,6 +4,23 @@ module Aspera
4
4
  module Cli
5
5
  module Plugins
6
6
  class Shares < BasicAuthPlugin
7
+ class << self
8
+ def detect(base_url)
9
+ api=Rest.new({:base_url=>base_url})
10
+ # Shares
11
+ begin
12
+ # shall fail: shares requires auth, but we check error message
13
+ api.read('node_api/app')
14
+ rescue RestCallError => e
15
+ if e.response.code.to_s.eql?('401') and e.response.body.eql?('{"error":{"user_message":"API user authentication failed"}}')
16
+ return {:version=>'unknown'}
17
+ end
18
+ rescue
19
+ end
20
+ nil
21
+ end
22
+ end
23
+
7
24
  def initialize(env)
8
25
  super(env)
9
26
  #self.options.parse_options!
@@ -1,8 +1,4 @@
1
- require 'aspera/fasp/local'
2
- require 'aspera/fasp/connect'
3
- require 'aspera/fasp/node'
4
- require 'aspera/fasp/aoc'
5
- require 'aspera/fasp/http_gw'
1
+ require 'aspera/fasp/parameters'
6
2
  require 'aspera/cli/listener/logger'
7
3
  require 'aspera/cli/listener/progress_multi'
8
4
 
@@ -16,13 +12,25 @@ module Aspera
16
12
  FILE_LIST_FROM_ARGS='@args'
17
13
  # special value for --sources : read file list from transfer spec (--ts)
18
14
  FILE_LIST_FROM_TRANSFER_SPEC='@ts'
19
- private_constant :FILE_LIST_FROM_ARGS,:FILE_LIST_FROM_TRANSFER_SPEC
20
- # @param cli_objects external objects: option manager, config file manager
21
- def initialize(cli_objects)
22
- @opt_mgr=cli_objects[:options]
23
- @config=cli_objects[:config]
24
- # transfer spec overrides provided on command line
25
- @transfer_spec_cmdline={"create_dir"=>true}
15
+ DEFAULT_TRANSFER_NOTIF_TMPL=<<END_OF_TEMPLATE
16
+ From: <%=from_name%> <<%=from_email%>>
17
+ To: <<%=to%>>
18
+ Subject: <%=subject%>
19
+
20
+ Transfer is: <%=global_transfer_status%>
21
+
22
+ <%=ts.to_yaml%>
23
+ END_OF_TEMPLATE
24
+ #% (formating bug in eclipse)
25
+ private_constant :FILE_LIST_FROM_ARGS,:FILE_LIST_FROM_TRANSFER_SPEC,:DEFAULT_TRANSFER_NOTIF_TMPL
26
+ TRANSFER_AGENTS=[:direct,:node,:connect,:httpgw,:trsdk]
27
+
28
+ # @param env external objects: option manager, config file manager
29
+ def initialize(opt_mgr,config)
30
+ @opt_mgr=opt_mgr
31
+ @config=config
32
+ # command line can override transfer spec
33
+ @transfer_spec_cmdline={'create_dir'=>true}
26
34
  # the currently selected transfer agent
27
35
  @agent=nil
28
36
  @progress_listener=Listener::ProgressMulti.new
@@ -33,9 +41,9 @@ module Aspera
33
41
  @opt_mgr.add_opt_simple(:local_resume,"set resume policy (Hash, use @json: prefix), current=#{@opt_mgr.get_option(:local_resume,:optional)}")
34
42
  @opt_mgr.add_opt_simple(:to_folder,"destination folder for downloaded files")
35
43
  @opt_mgr.add_opt_simple(:sources,"list of source files (see doc)")
36
- @opt_mgr.add_opt_simple(:transfer_info,"additional information for transfer client")
44
+ @opt_mgr.add_opt_simple(:transfer_info,"parameters for transfer agent")
37
45
  @opt_mgr.add_opt_list(:src_type,[:list,:pair],"type of file list")
38
- @opt_mgr.add_opt_list(:transfer,[:direct,:httpgw,:connect,:node,:aoc],"type of transfer")
46
+ @opt_mgr.add_opt_list(:transfer,TRANSFER_AGENTS,"type of transfer agent")
39
47
  @opt_mgr.add_opt_list(:progress,[:none,:native,:multi],"type of progress bar")
40
48
  @opt_mgr.set_option(:transfer,:direct)
41
49
  @opt_mgr.set_option(:src_type,:list)
@@ -55,7 +63,7 @@ module Aspera
55
63
  @agent.add_listener(Listener::Logger.new)
56
64
  # use local progress bar if asked so, or if native and non local ascp (because only local ascp has native progress bar)
57
65
  if @opt_mgr.get_option(:progress,:mandatory).eql?(:multi) or
58
- (@opt_mgr.get_option(:progress,:mandatory).eql?(:native) and !@opt_mgr.get_option(:transfer,:mandatory).eql?(:direct))
66
+ (@opt_mgr.get_option(:progress,:mandatory).eql?(:native) and ! instance.class.to_s.eql?('Aspera::Fasp::AgentDirect'))
59
67
  @agent.add_listener(@progress_listener)
60
68
  end
61
69
  end
@@ -64,63 +72,23 @@ module Aspera
64
72
  def set_agent_by_options
65
73
  return nil unless @agent.nil?
66
74
  agent_type=@opt_mgr.get_option(:transfer,:mandatory)
67
- case agent_type
68
- when :direct
69
- agent_options=@opt_mgr.get_option(:transfer_info,:optional)
70
- agent_options=agent_options.symbolize_keys if agent_options.is_a?(Hash)
71
- new_agent=Fasp::Local.new(agent_options)
72
- new_agent.quiet=false if @opt_mgr.get_option(:progress,:mandatory).eql?(:native)
73
- when :httpgw
74
- httpgw_config=@opt_mgr.get_option(:transfer_info,:mandatory)
75
- new_agent=Fasp::HttpGW.new(httpgw_config)
76
- when :connect
77
- new_agent=Fasp::Connect.new
78
- when :node
79
- # way for code to setup alternate node api in advance
80
- # support: @preset:<name>
81
- # support extended values
82
- node_config=@opt_mgr.get_option(:transfer_info,:optional)
83
- # if not specified: use default node
84
- if node_config.nil?
85
- param_set_name=@config.get_plugin_default_config_name(:node)
86
- raise CliBadArgument,"No default node configured, Please specify --#{:transfer_info.to_s.gsub('_','-')}" if param_set_name.nil?
87
- node_config=@config.preset_by_name(param_set_name)
88
- end
89
- Log.log.debug("node=#{node_config}")
90
- raise CliBadArgument,"the node configuration shall be Hash, not #{node_config.class} (#{node_config}), use either @json:<json> or @preset:<parameter set name>" if !node_config.is_a?(Hash)
91
- # now check there are required parameters
92
- sym_config=[:url,:username,:password].inject({}) do |h,param|
93
- raise CliBadArgument,"missing parameter [#{param}] in node specification: #{node_config}" if !node_config.has_key?(param.to_s)
94
- h[param]=node_config[param.to_s]
95
- h
96
- end
97
- node_api=Rest.new({
98
- :base_url => sym_config[:url],
99
- :auth => {
100
- :type =>:basic,
101
- :username => sym_config[:username],
102
- :password => sym_config[:password]
103
- }})
104
- new_agent=Fasp::Node.new(node_api)
105
- when :aoc
106
- aoc_config=@opt_mgr.get_option(:transfer_info,:optional)
107
- if aoc_config.nil?
108
- param_set_name=@config.get_plugin_default_config_name(:aspera)
109
- raise CliBadArgument,"No default AoC configured, Please specify --#{:transfer_info.to_s.gsub('_','-')}" if param_set_name.nil?
110
- aoc_config=@config.preset_by_name(param_set_name)
111
- end
112
- Log.log.debug("aoc=#{aoc_config}")
113
- raise CliBadArgument,"the aoc configuration shall be Hash, not #{aoc_config.class} (#{aoc_config}), refer to manual" if !aoc_config.is_a?(Hash)
114
- # convert keys from string (config) to symbol (agent)
115
- aoc_config=aoc_config.symbolize_keys
116
- # convert auth value from string (config) to symbol (agent)
117
- aoc_config[:auth]=aoc_config[:auth].to_sym if aoc_config[:auth].is_a?(String)
118
- # private key could be @file:... in config
119
- aoc_config[:private_key]=ExtendedValue.instance.evaluate(aoc_config[:private_key])
120
- new_agent=Fasp::Aoc.new(aoc_config)
121
- else
122
- raise "Unexpected transfer agent type: #{agent_type}"
75
+ require "aspera/fasp/agent_#{agent_type}"
76
+ agent_options=@opt_mgr.get_option(:transfer_info,:optional)
77
+ raise CliBadArgument,"the transfer agent configuration shall be Hash, not #{agent_options.class} (#{agent_options}), use either @json:<json> or @preset:<parameter set name>" unless [Hash,NilClass].include?(agent_options.class)
78
+ # special case
79
+ if agent_type.eql?(:node) and agent_options.nil?
80
+ param_set_name=@config.get_plugin_default_config_name(:node)
81
+ raise CliBadArgument,"No default node configured, Please specify --#{:transfer_info.to_s.gsub('_','-')}" if param_set_name.nil?
82
+ agent_options=@config.preset_by_name(param_set_name)
83
+ end
84
+ # special case
85
+ if agent_type.eql?(:direct) and @opt_mgr.get_option(:progress,:mandatory).eql?(:native)
86
+ agent_options={} if agent_options.nil?
87
+ agent_options[:quiet]=false
123
88
  end
89
+ agent_options=agent_options.symbolize_keys if agent_options.is_a?(Hash)
90
+ # get agent instance
91
+ new_agent=Kernel.const_get("Aspera::Fasp::Agent#{agent_type.capitalize}").new(agent_options)
124
92
  set_agent_instance(new_agent)
125
93
  return nil
126
94
  end
@@ -130,7 +98,7 @@ module Aspera
130
98
  # param: 'send' or 'receive'
131
99
  def destination_folder(direction)
132
100
  dest_folder=@opt_mgr.get_option(:to_folder,:optional)
133
- return dest_folder unless dest_folder.nil?
101
+ return File.expand_path(dest_folder) unless dest_folder.nil?
134
102
  dest_folder=@transfer_spec_cmdline['destination_root']
135
103
  return dest_folder unless dest_folder.nil?
136
104
  # default: / on remote, . on local
@@ -161,7 +129,8 @@ module Aspera
161
129
  raise CliBadArgument,"specify at least one file on command line or use --sources=#{FILE_LIST_FROM_TRANSFER_SPEC} to use transfer spec" if !file_list.is_a?(Array) or file_list.empty?
162
130
  when FILE_LIST_FROM_TRANSFER_SPEC
163
131
  Log.log.debug("assume list provided in transfer spec")
164
- raise CliBadArgument,"transfer spec on command line must have sources" if @transfer_paths.nil?
132
+ special_case_direct_with_list=@opt_mgr.get_option(:transfer,:mandatory).eql?(:direct) and Fasp::Parameters.ts_has_file_list(@transfer_spec_cmdline)
133
+ raise CliBadArgument,"transfer spec on command line must have sources" if @transfer_paths.nil? and !special_case_direct_with_list
165
134
  # here we assume check of sources is made in transfer agent
166
135
  return @transfer_paths
167
136
  when Array
@@ -189,20 +158,20 @@ module Aspera
189
158
 
190
159
  # start a transfer and wait for completion, plugins shall use this method
191
160
  # @param transfer_spec
192
- # @param options specific options for the transfer_agent
193
- # options[:src] specifies how destination_root is set (how transfer spec was generated)
161
+ # @param tr_opts specific options for the transfer_agent
162
+ # tr_opts[:src] specifies how destination_root is set (how transfer spec was generated)
194
163
  # other options are carried to specific agent
195
- def start(transfer_spec,options)
164
+ def start(transfer_spec,tr_opts)
196
165
  # check parameters
197
166
  raise "transfer_spec must be hash" unless transfer_spec.is_a?(Hash)
198
- raise "options must be hash" unless options.is_a?(Hash)
167
+ raise "tr_opts must be hash" unless tr_opts.is_a?(Hash)
199
168
  # process :src option
200
169
  case transfer_spec['direction']
201
170
  when 'receive'
202
171
  # init default if required in any case
203
172
  @transfer_spec_cmdline['destination_root']||=destination_folder(transfer_spec['direction'])
204
173
  when 'send'
205
- case options[:src]
174
+ case tr_opts[:src]
206
175
  when :direct
207
176
  # init default if required
208
177
  @transfer_spec_cmdline['destination_root']||=destination_folder(transfer_spec['direction'])
@@ -213,12 +182,12 @@ module Aspera
213
182
  when :node_gen4
214
183
  @transfer_spec_cmdline.delete('destination_root') if @transfer_spec_cmdline.has_key?('destination_root_id')
215
184
  else
216
- raise StandardError,"InternalError: unsupported value: #{options[:src]}"
185
+ raise StandardError,"InternalError: unsupported value: #{tr_opts[:src]}"
217
186
  end
218
187
  end
219
188
 
220
189
  # only used here
221
- options.delete(:src)
190
+ tr_opts.delete(:src)
222
191
 
223
192
  # update command line paths, unless destination already has one
224
193
  @transfer_spec_cmdline['paths']=transfer_spec['paths'] || ts_source_paths
@@ -227,13 +196,26 @@ module Aspera
227
196
  # create transfer agent
228
197
  self.set_agent_by_options
229
198
  Log.log.debug("transfer agent is a #{@agent.class}")
230
- @agent.start_transfer(transfer_spec,options)
199
+ @agent.start_transfer(transfer_spec,tr_opts)
231
200
  result=@agent.wait_for_transfers_completion
232
201
  @progress_listener.reset
233
- Fasp::Manager.validate_status_list(result)
202
+ Fasp::AgentBase.validate_status_list(result)
203
+ send_email_transfer_notification(transfer_spec,result)
234
204
  return result
235
205
  end
236
206
 
207
+ def send_email_transfer_notification(transfer_spec,statuses)
208
+ return if @opt_mgr.get_option(:notif_to,:optional).nil?
209
+ global_status=self.class.session_status(statuses)
210
+ email_vars={
211
+ global_transfer_status: global_status,
212
+ subject: "ascli transfer: #{global_status}",
213
+ body: "Transfer is: #{global_status}",
214
+ ts: transfer_spec
215
+ }
216
+ @config.send_email_template(email_vars,DEFAULT_TRANSFER_NOTIF_TMPL)
217
+ end
218
+
237
219
  # @return :success if all sessions statuses returned by "start" are success
238
220
  # else return the first error exception object
239
221
  def self.session_status(statuses)
@@ -1,5 +1,5 @@
1
1
  module Aspera
2
2
  module Cli
3
- VERSION = "4.2.1"
3
+ VERSION = "4.5.0"
4
4
  end
5
5
  end