aspera-cli 4.2.1 → 4.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +1580 -946
- data/bin/ascli +1 -1
- data/bin/asession +3 -5
- data/docs/Makefile +8 -11
- data/docs/README.erb.md +1521 -829
- data/docs/doc_tools.rb +58 -0
- data/docs/test_env.conf +3 -1
- data/examples/faspex4.rb +28 -19
- data/examples/transfer.rb +2 -2
- data/lib/aspera/aoc.rb +157 -134
- data/lib/aspera/cli/listener/progress_multi.rb +5 -5
- data/lib/aspera/cli/main.rb +106 -48
- data/lib/aspera/cli/manager.rb +19 -20
- data/lib/aspera/cli/plugin.rb +22 -7
- data/lib/aspera/cli/plugins/aoc.rb +260 -208
- data/lib/aspera/cli/plugins/ats.rb +11 -10
- data/lib/aspera/cli/plugins/bss.rb +2 -2
- data/lib/aspera/cli/plugins/config.rb +360 -189
- data/lib/aspera/cli/plugins/faspex.rb +119 -56
- data/lib/aspera/cli/plugins/faspex5.rb +32 -17
- data/lib/aspera/cli/plugins/node.rb +72 -31
- data/lib/aspera/cli/plugins/orchestrator.rb +5 -3
- data/lib/aspera/cli/plugins/preview.rb +94 -68
- data/lib/aspera/cli/plugins/server.rb +16 -5
- data/lib/aspera/cli/plugins/shares.rb +17 -0
- data/lib/aspera/cli/transfer_agent.rb +64 -82
- data/lib/aspera/cli/version.rb +1 -1
- data/lib/aspera/command_line_builder.rb +48 -31
- data/lib/aspera/cos_node.rb +4 -3
- data/lib/aspera/environment.rb +4 -4
- data/lib/aspera/fasp/{manager.rb → agent_base.rb} +7 -6
- data/lib/aspera/fasp/{connect.rb → agent_connect.rb} +46 -39
- data/lib/aspera/fasp/{local.rb → agent_direct.rb} +42 -38
- data/lib/aspera/fasp/{http_gw.rb → agent_httpgw.rb} +50 -29
- data/lib/aspera/fasp/{node.rb → agent_node.rb} +43 -4
- data/lib/aspera/fasp/agent_trsdk.rb +106 -0
- data/lib/aspera/fasp/default.rb +17 -0
- data/lib/aspera/fasp/installation.rb +64 -48
- data/lib/aspera/fasp/parameters.rb +78 -91
- data/lib/aspera/fasp/parameters.yaml +531 -0
- data/lib/aspera/fasp/uri.rb +1 -1
- data/lib/aspera/faspex_gw.rb +12 -11
- data/lib/aspera/id_generator.rb +22 -0
- data/lib/aspera/keychain/encrypted_hash.rb +120 -0
- data/lib/aspera/keychain/macos_security.rb +94 -0
- data/lib/aspera/log.rb +45 -32
- data/lib/aspera/node.rb +9 -4
- data/lib/aspera/oauth.rb +116 -100
- data/lib/aspera/persistency_action_once.rb +11 -7
- data/lib/aspera/persistency_folder.rb +6 -26
- data/lib/aspera/rest.rb +66 -50
- data/lib/aspera/sync.rb +40 -35
- data/lib/aspera/timer_limiter.rb +22 -0
- metadata +86 -29
- data/docs/transfer_spec.html +0 -99
- data/lib/aspera/api_detector.rb +0 -60
- data/lib/aspera/fasp/aoc.rb +0 -24
- data/lib/aspera/secrets.rb +0 -20
@@ -6,6 +6,8 @@ require 'aspera/preview/file_types'
|
|
6
6
|
require 'aspera/persistency_action_once'
|
7
7
|
require 'aspera/node'
|
8
8
|
require 'aspera/hash_ext'
|
9
|
+
require 'aspera/timer_limiter'
|
10
|
+
require 'aspera/id_generator'
|
9
11
|
require 'date'
|
10
12
|
require 'securerandom'
|
11
13
|
|
@@ -24,7 +26,8 @@ module Aspera
|
|
24
26
|
DEFAULT_PREVIEWS_FOLDER='previews'
|
25
27
|
AK_MARKER_FILE='.aspera_access_key'
|
26
28
|
LOCAL_STORAGE_PCVL='file:///'
|
27
|
-
|
29
|
+
LOG_LIMITER_SEC=30.0
|
30
|
+
private_constant :PREV_GEN_TAG, :PREVIEW_FOLDER_SUFFIX, :PREVIEW_BASENAME, :TMP_DIR_PREFIX, :DEFAULT_PREVIEWS_FOLDER, :LOCAL_STORAGE_PCVL, :AK_MARKER_FILE, :LOG_LIMITER_SEC
|
28
31
|
|
29
32
|
# option_skip_format has special accessors
|
30
33
|
attr_accessor :option_previews_folder
|
@@ -40,6 +43,8 @@ module Aspera
|
|
40
43
|
@preview_formats_to_generate=Aspera::Preview::Generator::PREVIEW_FORMATS.clone
|
41
44
|
# options for generation
|
42
45
|
@gen_options=Aspera::Preview::Options.new
|
46
|
+
# used to trigger periodic processing
|
47
|
+
@periodic=TimerLimiter.new(LOG_LIMITER_SEC)
|
43
48
|
# link CLI options to gen_info attributes
|
44
49
|
self.options.set_obj_attr(:skip_format,self,:option_skip_format,[]) # no skip
|
45
50
|
self.options.set_obj_attr(:folder_reset_cache,self,:option_folder_reset_cache,:no)
|
@@ -113,13 +118,14 @@ module Aspera
|
|
113
118
|
end
|
114
119
|
|
115
120
|
# old version based on folders
|
116
|
-
|
121
|
+
# @param iteration_persistency can be nil
|
122
|
+
def process_trevents(iteration_persistency)
|
117
123
|
events_filter={
|
118
124
|
'access_key'=>@access_key_self['id'],
|
119
125
|
'type'=>'download.ended'
|
120
126
|
}
|
121
|
-
# optionally
|
122
|
-
events_filter['iteration_token']=
|
127
|
+
# optionally add iteration token from persistency
|
128
|
+
events_filter['iteration_token']=iteration_persistency.data.first unless iteration_persistency.nil?
|
123
129
|
begin
|
124
130
|
events=@api_node.read('events',events_filter)[:data]
|
125
131
|
rescue RestCallError => e
|
@@ -132,47 +138,63 @@ module Aspera
|
|
132
138
|
end
|
133
139
|
return if events.empty?
|
134
140
|
events.each do |event|
|
135
|
-
|
136
|
-
|
137
|
-
|
138
|
-
|
139
|
-
|
140
|
-
|
141
|
-
|
142
|
-
|
143
|
-
|
144
|
-
|
141
|
+
if event['data']['direction'].eql?('receive') and
|
142
|
+
event['data']['status'].eql?('completed') and
|
143
|
+
event['data']['error_code'].eql?(0) and
|
144
|
+
event['data'].dig('tags','aspera',PREV_GEN_TAG).nil?
|
145
|
+
folder_id=event.dig('data','tags','aspera','node','file_id')
|
146
|
+
folder_id||=event.dig('data','file_id')
|
147
|
+
if !folder_id.nil?
|
148
|
+
folder_entry=@api_node.read("files/#{folder_id}")[:data] rescue nil
|
149
|
+
scan_folder_files(folder_entry) unless folder_entry.nil?
|
150
|
+
end
|
151
|
+
end
|
152
|
+
if @periodic.trigger? or event.equal?(events.last)
|
153
|
+
Log.log.info("Processed event #{event['id']}")
|
154
|
+
# save checkpoint to avoid losing processing in case of error
|
155
|
+
if !iteration_persistency.nil?
|
156
|
+
iteration_persistency.data[0]=event['id'].to_s
|
157
|
+
iteration_persistency.save
|
158
|
+
end
|
159
|
+
end
|
145
160
|
end
|
146
|
-
return events.last['id'].to_s
|
147
161
|
end
|
148
162
|
|
149
163
|
# requests recent events on node api and process newly modified folders
|
150
|
-
def process_events(
|
164
|
+
def process_events(iteration_persistency)
|
151
165
|
# get new file creation by access key (TODO: what if file already existed?)
|
152
166
|
events_filter={
|
153
167
|
'access_key'=>@access_key_self['id'],
|
154
168
|
'type'=>'file.*'
|
155
169
|
}
|
156
|
-
#
|
157
|
-
events_filter['iteration_token']=
|
170
|
+
# optionally add iteration token from persistency
|
171
|
+
events_filter['iteration_token']=iteration_persistency.data.first unless iteration_persistency.nil?
|
158
172
|
events=@api_node.read('events',events_filter)[:data]
|
159
173
|
return if events.empty?
|
160
174
|
events.each do |event|
|
161
175
|
# process only files
|
162
|
-
|
163
|
-
|
164
|
-
|
165
|
-
|
166
|
-
|
167
|
-
|
168
|
-
|
176
|
+
if event.dig('data','type').eql?('file')
|
177
|
+
file_entry=@api_node.read("files/#{event['data']['id']}")[:data] rescue nil
|
178
|
+
if !file_entry.nil? and
|
179
|
+
@option_skip_folders.select{|d|file_entry['path'].start_with?(d)}.empty?
|
180
|
+
file_entry['parent_file_id']=event['data']['parent_file_id']
|
181
|
+
if event['types'].include?('file.deleted')
|
182
|
+
Log.log.error('TODO'.red)
|
183
|
+
end
|
184
|
+
if event['types'].include?('file.deleted')
|
185
|
+
generate_preview(file_entry)
|
186
|
+
end
|
187
|
+
end
|
169
188
|
end
|
170
|
-
if event
|
171
|
-
|
189
|
+
if @periodic.trigger? or event.equal?(events.last)
|
190
|
+
Log.log.info("Processing event #{event['id']}")
|
191
|
+
# save checkpoint to avoid losing processing in case of error
|
192
|
+
if !iteration_persistency.nil?
|
193
|
+
iteration_persistency.data[0]=event['id'].to_s
|
194
|
+
iteration_persistency.save
|
195
|
+
end
|
172
196
|
end
|
173
197
|
end
|
174
|
-
# write new iteration file
|
175
|
-
return events.last['id'].to_s
|
176
198
|
end
|
177
199
|
|
178
200
|
def do_transfer(direction,folder_id,source_filename,destination='/')
|
@@ -180,15 +202,17 @@ module Aspera
|
|
180
202
|
if @default_transfer_spec.nil?
|
181
203
|
# make a dummy call to get some default transfer parameters
|
182
204
|
res=@api_node.create('files/upload_setup',{'transfer_requests'=>[{'transfer_request'=>{'paths'=>[{}],'destination_root'=>'/'}}]})
|
183
|
-
|
205
|
+
template_ts=res[:data]['transfer_specs'].first['transfer_spec']
|
184
206
|
# get ports, anyway that should be 33001 for both. add remote_user ?
|
185
|
-
@default_transfer_spec=['ssh_port','fasp_port'].inject({}){|h,e|h[e]=
|
207
|
+
@default_transfer_spec=['ssh_port','fasp_port'].inject({}){|h,e|h[e]=template_ts[e];h}
|
208
|
+
if ! @default_transfer_spec['remote_user'].eql?(Aspera::Fasp::Default::ACCESS_KEY_TRANSFER_USER)
|
209
|
+
Log.log.warn("remote_user shall be xfer")
|
210
|
+
@default_transfer_spec['remote_user']=Aspera::Fasp::Default::ACCESS_KEY_TRANSFER_USER
|
211
|
+
end
|
212
|
+
Aspera::Node::set_ak_basic_token(@default_transfer_spec,@access_key_self['id'],self.options.get_option(:password,:mandatory))
|
186
213
|
# note: we use the same address for ascp than for node api instead of the one from upload_setup
|
187
|
-
|
188
|
-
|
189
|
-
'remote_host' => @transfer_server_address,
|
190
|
-
'remote_user' => Fasp::ACCESS_KEY_TRANSFER_USER
|
191
|
-
})
|
214
|
+
# TODO: configurable ? useful ?
|
215
|
+
@default_transfer_spec['remote_host']=@transfer_server_address
|
192
216
|
end
|
193
217
|
tspec=@default_transfer_spec.merge({
|
194
218
|
'direction' => direction,
|
@@ -338,40 +362,45 @@ module Aspera
|
|
338
362
|
entry=entries_to_process.shift
|
339
363
|
# process this entry only if it is within the scan_start
|
340
364
|
entry_path_with_slash=entry['path']
|
365
|
+
Log.log.info("processing entry #{entry_path_with_slash}") if @periodic.trigger?
|
341
366
|
entry_path_with_slash="#{entry_path_with_slash}/" unless entry_path_with_slash.end_with?('/')
|
342
367
|
if !scan_start.nil? and !scan_start.start_with?(entry_path_with_slash) and !entry_path_with_slash.start_with?(scan_start)
|
343
368
|
Log.log.debug("#{entry['path']} folder (skip start)".bg_red)
|
344
369
|
next
|
345
370
|
end
|
346
371
|
Log.log.debug("item:#{entry}")
|
347
|
-
|
348
|
-
|
349
|
-
|
350
|
-
|
351
|
-
|
352
|
-
|
353
|
-
|
354
|
-
|
355
|
-
|
356
|
-
|
357
|
-
|
358
|
-
|
359
|
-
|
360
|
-
|
361
|
-
|
362
|
-
|
363
|
-
|
364
|
-
|
365
|
-
|
366
|
-
|
367
|
-
folder_entry
|
372
|
+
begin
|
373
|
+
case entry['type']
|
374
|
+
when 'file'
|
375
|
+
if filter_block.call(entry)
|
376
|
+
generate_preview(entry)
|
377
|
+
else
|
378
|
+
Log.log.debug('skip by filter')
|
379
|
+
end
|
380
|
+
when 'link'
|
381
|
+
Log.log.debug('Ignoring link.')
|
382
|
+
when 'folder'
|
383
|
+
if @option_skip_folders.include?(entry['path'])
|
384
|
+
Log.log.debug("#{entry['path']} folder (skip list)".bg_red)
|
385
|
+
else
|
386
|
+
Log.log.debug("#{entry['path']} folder".green)
|
387
|
+
# get folder content
|
388
|
+
folder_entries=get_folder_entries(entry['id'])
|
389
|
+
# process all items in current folder
|
390
|
+
folder_entries.each do |folder_entry|
|
391
|
+
# add path for older versions of ES
|
392
|
+
if !folder_entry.has_key?('path')
|
393
|
+
folder_entry['path']=entry_path_with_slash+folder_entry['name']
|
394
|
+
end
|
395
|
+
folder_entry['parent_file_id']=entry['id']
|
396
|
+
entries_to_process.push(folder_entry)
|
368
397
|
end
|
369
|
-
folder_entry['parent_file_id']=entry['id']
|
370
|
-
entries_to_process.push(folder_entry)
|
371
398
|
end
|
399
|
+
else
|
400
|
+
Log.log.warn("unknown entry type: #{entry['type']}")
|
372
401
|
end
|
373
|
-
|
374
|
-
Log.log.warn("
|
402
|
+
rescue => e
|
403
|
+
Log.log.warn("An error occured: #{e}, ignoring")
|
375
404
|
end
|
376
405
|
end
|
377
406
|
end
|
@@ -437,18 +466,15 @@ module Aspera
|
|
437
466
|
scan_folder_files(folder_info,scan_path)
|
438
467
|
return Main.result_status('scan finished')
|
439
468
|
when :events,:trevents
|
440
|
-
iteration_data=[]
|
441
469
|
iteration_persistency=nil
|
442
470
|
if self.options.get_option(:once_only,:mandatory)
|
443
471
|
iteration_persistency=PersistencyActionOnce.new(
|
444
472
|
manager: @agents[:persistency],
|
445
|
-
data:
|
446
|
-
|
473
|
+
data: [],
|
474
|
+
id: IdGenerator.from_list(['preview_iteration',command.to_s,self.options.get_option(:url,:mandatory),self.options.get_option(:username,:mandatory)]))
|
447
475
|
end
|
448
|
-
|
449
|
-
#
|
450
|
-
iteration_data[0]=send("process_#{command}",iteration_data[0])
|
451
|
-
iteration_persistency.save unless iteration_persistency.nil?
|
476
|
+
# call processing method specified by command line command
|
477
|
+
send("process_#{command}",iteration_persistency)
|
452
478
|
return Main.result_status("#{command} finished")
|
453
479
|
when :check
|
454
480
|
Aspera::Preview::Utils.check_tools(@skip_types)
|
@@ -1,5 +1,6 @@
|
|
1
1
|
require 'aspera/cli/basic_auth_plugin'
|
2
2
|
require 'aspera/ascmd'
|
3
|
+
require 'aspera/fasp/default'
|
3
4
|
require 'aspera/ssh'
|
4
5
|
require 'aspera/nagios'
|
5
6
|
require 'tempfile'
|
@@ -67,13 +68,15 @@ module Aspera
|
|
67
68
|
def execute_action
|
68
69
|
server_uri=URI.parse(self.options.get_option(:url,:mandatory))
|
69
70
|
Log.log.debug("URI : #{server_uri}, port=#{server_uri.port}, scheme:#{server_uri.scheme}")
|
71
|
+
server_transfer_spec={'remote_host'=>server_uri.hostname}
|
70
72
|
shell_executor=nil
|
71
73
|
case server_uri.scheme
|
72
74
|
when 'ssh'
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
75
|
+
if self.options.get_option(:username,:optional).nil?
|
76
|
+
self.options.set_option(:username,Aspera::Fasp::Default::ACCESS_KEY_TRANSFER_USER)
|
77
|
+
Log.log.info("Using default transfer user: #{Aspera::Fasp::Default::ACCESS_KEY_TRANSFER_USER}")
|
78
|
+
end
|
79
|
+
server_transfer_spec['remote_user']=self.options.get_option(:username,:mandatory)
|
77
80
|
ssh_options=self.options.get_option(:ssh_options,:optional)
|
78
81
|
raise 'expecting a Hash for ssh_options' unless ssh_options.is_a?(Hash)
|
79
82
|
if !server_uri.port.nil?
|
@@ -102,8 +105,16 @@ module Aspera
|
|
102
105
|
cred_set=true
|
103
106
|
end
|
104
107
|
end
|
105
|
-
|
108
|
+
# if user provided transfer spec has a token, we will use by pass keys
|
109
|
+
cred_set=true if self.transfer.option_transfer_spec['token'].is_a?(String)
|
110
|
+
raise 'either password, key , or transfer spec token must be provided' if !cred_set
|
106
111
|
shell_executor=Ssh.new(server_transfer_spec['remote_host'],server_transfer_spec['remote_user'],ssh_options)
|
112
|
+
when 'https'
|
113
|
+
raise "ERROR: transfer spec with token required" unless self.transfer.option_transfer_spec['token'].is_a?(String)
|
114
|
+
server_transfer_spec.merge!({
|
115
|
+
'wss_enabled'=>true,
|
116
|
+
'wss_port' =>server_uri.port
|
117
|
+
})
|
107
118
|
when 'local'
|
108
119
|
shell_executor=LocalExecutor.new
|
109
120
|
else
|
@@ -4,6 +4,23 @@ module Aspera
|
|
4
4
|
module Cli
|
5
5
|
module Plugins
|
6
6
|
class Shares < BasicAuthPlugin
|
7
|
+
class << self
|
8
|
+
def detect(base_url)
|
9
|
+
api=Rest.new({:base_url=>base_url})
|
10
|
+
# Shares
|
11
|
+
begin
|
12
|
+
# shall fail: shares requires auth, but we check error message
|
13
|
+
api.read('node_api/app')
|
14
|
+
rescue RestCallError => e
|
15
|
+
if e.response.code.to_s.eql?('401') and e.response.body.eql?('{"error":{"user_message":"API user authentication failed"}}')
|
16
|
+
return {:version=>'unknown'}
|
17
|
+
end
|
18
|
+
rescue
|
19
|
+
end
|
20
|
+
nil
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
7
24
|
def initialize(env)
|
8
25
|
super(env)
|
9
26
|
#self.options.parse_options!
|
@@ -1,8 +1,4 @@
|
|
1
|
-
require 'aspera/fasp/
|
2
|
-
require 'aspera/fasp/connect'
|
3
|
-
require 'aspera/fasp/node'
|
4
|
-
require 'aspera/fasp/aoc'
|
5
|
-
require 'aspera/fasp/http_gw'
|
1
|
+
require 'aspera/fasp/parameters'
|
6
2
|
require 'aspera/cli/listener/logger'
|
7
3
|
require 'aspera/cli/listener/progress_multi'
|
8
4
|
|
@@ -16,13 +12,25 @@ module Aspera
|
|
16
12
|
FILE_LIST_FROM_ARGS='@args'
|
17
13
|
# special value for --sources : read file list from transfer spec (--ts)
|
18
14
|
FILE_LIST_FROM_TRANSFER_SPEC='@ts'
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
15
|
+
DEFAULT_TRANSFER_NOTIF_TMPL=<<END_OF_TEMPLATE
|
16
|
+
From: <%=from_name%> <<%=from_email%>>
|
17
|
+
To: <<%=to%>>
|
18
|
+
Subject: <%=subject%>
|
19
|
+
|
20
|
+
Transfer is: <%=global_transfer_status%>
|
21
|
+
|
22
|
+
<%=ts.to_yaml%>
|
23
|
+
END_OF_TEMPLATE
|
24
|
+
#% (formating bug in eclipse)
|
25
|
+
private_constant :FILE_LIST_FROM_ARGS,:FILE_LIST_FROM_TRANSFER_SPEC,:DEFAULT_TRANSFER_NOTIF_TMPL
|
26
|
+
TRANSFER_AGENTS=[:direct,:node,:connect,:httpgw,:trsdk]
|
27
|
+
|
28
|
+
# @param env external objects: option manager, config file manager
|
29
|
+
def initialize(opt_mgr,config)
|
30
|
+
@opt_mgr=opt_mgr
|
31
|
+
@config=config
|
32
|
+
# command line can override transfer spec
|
33
|
+
@transfer_spec_cmdline={'create_dir'=>true}
|
26
34
|
# the currently selected transfer agent
|
27
35
|
@agent=nil
|
28
36
|
@progress_listener=Listener::ProgressMulti.new
|
@@ -33,9 +41,9 @@ module Aspera
|
|
33
41
|
@opt_mgr.add_opt_simple(:local_resume,"set resume policy (Hash, use @json: prefix), current=#{@opt_mgr.get_option(:local_resume,:optional)}")
|
34
42
|
@opt_mgr.add_opt_simple(:to_folder,"destination folder for downloaded files")
|
35
43
|
@opt_mgr.add_opt_simple(:sources,"list of source files (see doc)")
|
36
|
-
@opt_mgr.add_opt_simple(:transfer_info,"
|
44
|
+
@opt_mgr.add_opt_simple(:transfer_info,"parameters for transfer agent")
|
37
45
|
@opt_mgr.add_opt_list(:src_type,[:list,:pair],"type of file list")
|
38
|
-
@opt_mgr.add_opt_list(:transfer,
|
46
|
+
@opt_mgr.add_opt_list(:transfer,TRANSFER_AGENTS,"type of transfer agent")
|
39
47
|
@opt_mgr.add_opt_list(:progress,[:none,:native,:multi],"type of progress bar")
|
40
48
|
@opt_mgr.set_option(:transfer,:direct)
|
41
49
|
@opt_mgr.set_option(:src_type,:list)
|
@@ -55,7 +63,7 @@ module Aspera
|
|
55
63
|
@agent.add_listener(Listener::Logger.new)
|
56
64
|
# use local progress bar if asked so, or if native and non local ascp (because only local ascp has native progress bar)
|
57
65
|
if @opt_mgr.get_option(:progress,:mandatory).eql?(:multi) or
|
58
|
-
(@opt_mgr.get_option(:progress,:mandatory).eql?(:native) and
|
66
|
+
(@opt_mgr.get_option(:progress,:mandatory).eql?(:native) and ! instance.class.to_s.eql?('Aspera::Fasp::AgentDirect'))
|
59
67
|
@agent.add_listener(@progress_listener)
|
60
68
|
end
|
61
69
|
end
|
@@ -64,63 +72,23 @@ module Aspera
|
|
64
72
|
def set_agent_by_options
|
65
73
|
return nil unless @agent.nil?
|
66
74
|
agent_type=@opt_mgr.get_option(:transfer,:mandatory)
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
# support: @preset:<name>
|
81
|
-
# support extended values
|
82
|
-
node_config=@opt_mgr.get_option(:transfer_info,:optional)
|
83
|
-
# if not specified: use default node
|
84
|
-
if node_config.nil?
|
85
|
-
param_set_name=@config.get_plugin_default_config_name(:node)
|
86
|
-
raise CliBadArgument,"No default node configured, Please specify --#{:transfer_info.to_s.gsub('_','-')}" if param_set_name.nil?
|
87
|
-
node_config=@config.preset_by_name(param_set_name)
|
88
|
-
end
|
89
|
-
Log.log.debug("node=#{node_config}")
|
90
|
-
raise CliBadArgument,"the node configuration shall be Hash, not #{node_config.class} (#{node_config}), use either @json:<json> or @preset:<parameter set name>" if !node_config.is_a?(Hash)
|
91
|
-
# now check there are required parameters
|
92
|
-
sym_config=[:url,:username,:password].inject({}) do |h,param|
|
93
|
-
raise CliBadArgument,"missing parameter [#{param}] in node specification: #{node_config}" if !node_config.has_key?(param.to_s)
|
94
|
-
h[param]=node_config[param.to_s]
|
95
|
-
h
|
96
|
-
end
|
97
|
-
node_api=Rest.new({
|
98
|
-
:base_url => sym_config[:url],
|
99
|
-
:auth => {
|
100
|
-
:type =>:basic,
|
101
|
-
:username => sym_config[:username],
|
102
|
-
:password => sym_config[:password]
|
103
|
-
}})
|
104
|
-
new_agent=Fasp::Node.new(node_api)
|
105
|
-
when :aoc
|
106
|
-
aoc_config=@opt_mgr.get_option(:transfer_info,:optional)
|
107
|
-
if aoc_config.nil?
|
108
|
-
param_set_name=@config.get_plugin_default_config_name(:aspera)
|
109
|
-
raise CliBadArgument,"No default AoC configured, Please specify --#{:transfer_info.to_s.gsub('_','-')}" if param_set_name.nil?
|
110
|
-
aoc_config=@config.preset_by_name(param_set_name)
|
111
|
-
end
|
112
|
-
Log.log.debug("aoc=#{aoc_config}")
|
113
|
-
raise CliBadArgument,"the aoc configuration shall be Hash, not #{aoc_config.class} (#{aoc_config}), refer to manual" if !aoc_config.is_a?(Hash)
|
114
|
-
# convert keys from string (config) to symbol (agent)
|
115
|
-
aoc_config=aoc_config.symbolize_keys
|
116
|
-
# convert auth value from string (config) to symbol (agent)
|
117
|
-
aoc_config[:auth]=aoc_config[:auth].to_sym if aoc_config[:auth].is_a?(String)
|
118
|
-
# private key could be @file:... in config
|
119
|
-
aoc_config[:private_key]=ExtendedValue.instance.evaluate(aoc_config[:private_key])
|
120
|
-
new_agent=Fasp::Aoc.new(aoc_config)
|
121
|
-
else
|
122
|
-
raise "Unexpected transfer agent type: #{agent_type}"
|
75
|
+
require "aspera/fasp/agent_#{agent_type}"
|
76
|
+
agent_options=@opt_mgr.get_option(:transfer_info,:optional)
|
77
|
+
raise CliBadArgument,"the transfer agent configuration shall be Hash, not #{agent_options.class} (#{agent_options}), use either @json:<json> or @preset:<parameter set name>" unless [Hash,NilClass].include?(agent_options.class)
|
78
|
+
# special case
|
79
|
+
if agent_type.eql?(:node) and agent_options.nil?
|
80
|
+
param_set_name=@config.get_plugin_default_config_name(:node)
|
81
|
+
raise CliBadArgument,"No default node configured, Please specify --#{:transfer_info.to_s.gsub('_','-')}" if param_set_name.nil?
|
82
|
+
agent_options=@config.preset_by_name(param_set_name)
|
83
|
+
end
|
84
|
+
# special case
|
85
|
+
if agent_type.eql?(:direct) and @opt_mgr.get_option(:progress,:mandatory).eql?(:native)
|
86
|
+
agent_options={} if agent_options.nil?
|
87
|
+
agent_options[:quiet]=false
|
123
88
|
end
|
89
|
+
agent_options=agent_options.symbolize_keys if agent_options.is_a?(Hash)
|
90
|
+
# get agent instance
|
91
|
+
new_agent=Kernel.const_get("Aspera::Fasp::Agent#{agent_type.capitalize}").new(agent_options)
|
124
92
|
set_agent_instance(new_agent)
|
125
93
|
return nil
|
126
94
|
end
|
@@ -130,7 +98,7 @@ module Aspera
|
|
130
98
|
# param: 'send' or 'receive'
|
131
99
|
def destination_folder(direction)
|
132
100
|
dest_folder=@opt_mgr.get_option(:to_folder,:optional)
|
133
|
-
return dest_folder unless dest_folder.nil?
|
101
|
+
return File.expand_path(dest_folder) unless dest_folder.nil?
|
134
102
|
dest_folder=@transfer_spec_cmdline['destination_root']
|
135
103
|
return dest_folder unless dest_folder.nil?
|
136
104
|
# default: / on remote, . on local
|
@@ -161,7 +129,8 @@ module Aspera
|
|
161
129
|
raise CliBadArgument,"specify at least one file on command line or use --sources=#{FILE_LIST_FROM_TRANSFER_SPEC} to use transfer spec" if !file_list.is_a?(Array) or file_list.empty?
|
162
130
|
when FILE_LIST_FROM_TRANSFER_SPEC
|
163
131
|
Log.log.debug("assume list provided in transfer spec")
|
164
|
-
|
132
|
+
special_case_direct_with_list=@opt_mgr.get_option(:transfer,:mandatory).eql?(:direct) and Fasp::Parameters.ts_has_file_list(@transfer_spec_cmdline)
|
133
|
+
raise CliBadArgument,"transfer spec on command line must have sources" if @transfer_paths.nil? and !special_case_direct_with_list
|
165
134
|
# here we assume check of sources is made in transfer agent
|
166
135
|
return @transfer_paths
|
167
136
|
when Array
|
@@ -189,20 +158,20 @@ module Aspera
|
|
189
158
|
|
190
159
|
# start a transfer and wait for completion, plugins shall use this method
|
191
160
|
# @param transfer_spec
|
192
|
-
# @param
|
193
|
-
#
|
161
|
+
# @param tr_opts specific options for the transfer_agent
|
162
|
+
# tr_opts[:src] specifies how destination_root is set (how transfer spec was generated)
|
194
163
|
# other options are carried to specific agent
|
195
|
-
def start(transfer_spec,
|
164
|
+
def start(transfer_spec,tr_opts)
|
196
165
|
# check parameters
|
197
166
|
raise "transfer_spec must be hash" unless transfer_spec.is_a?(Hash)
|
198
|
-
raise "
|
167
|
+
raise "tr_opts must be hash" unless tr_opts.is_a?(Hash)
|
199
168
|
# process :src option
|
200
169
|
case transfer_spec['direction']
|
201
170
|
when 'receive'
|
202
171
|
# init default if required in any case
|
203
172
|
@transfer_spec_cmdline['destination_root']||=destination_folder(transfer_spec['direction'])
|
204
173
|
when 'send'
|
205
|
-
case
|
174
|
+
case tr_opts[:src]
|
206
175
|
when :direct
|
207
176
|
# init default if required
|
208
177
|
@transfer_spec_cmdline['destination_root']||=destination_folder(transfer_spec['direction'])
|
@@ -213,12 +182,12 @@ module Aspera
|
|
213
182
|
when :node_gen4
|
214
183
|
@transfer_spec_cmdline.delete('destination_root') if @transfer_spec_cmdline.has_key?('destination_root_id')
|
215
184
|
else
|
216
|
-
raise StandardError,"InternalError: unsupported value: #{
|
185
|
+
raise StandardError,"InternalError: unsupported value: #{tr_opts[:src]}"
|
217
186
|
end
|
218
187
|
end
|
219
188
|
|
220
189
|
# only used here
|
221
|
-
|
190
|
+
tr_opts.delete(:src)
|
222
191
|
|
223
192
|
# update command line paths, unless destination already has one
|
224
193
|
@transfer_spec_cmdline['paths']=transfer_spec['paths'] || ts_source_paths
|
@@ -227,13 +196,26 @@ module Aspera
|
|
227
196
|
# create transfer agent
|
228
197
|
self.set_agent_by_options
|
229
198
|
Log.log.debug("transfer agent is a #{@agent.class}")
|
230
|
-
@agent.start_transfer(transfer_spec,
|
199
|
+
@agent.start_transfer(transfer_spec,tr_opts)
|
231
200
|
result=@agent.wait_for_transfers_completion
|
232
201
|
@progress_listener.reset
|
233
|
-
Fasp::
|
202
|
+
Fasp::AgentBase.validate_status_list(result)
|
203
|
+
send_email_transfer_notification(transfer_spec,result)
|
234
204
|
return result
|
235
205
|
end
|
236
206
|
|
207
|
+
def send_email_transfer_notification(transfer_spec,statuses)
|
208
|
+
return if @opt_mgr.get_option(:notif_to,:optional).nil?
|
209
|
+
global_status=self.class.session_status(statuses)
|
210
|
+
email_vars={
|
211
|
+
global_transfer_status: global_status,
|
212
|
+
subject: "ascli transfer: #{global_status}",
|
213
|
+
body: "Transfer is: #{global_status}",
|
214
|
+
ts: transfer_spec
|
215
|
+
}
|
216
|
+
@config.send_email_template(email_vars,DEFAULT_TRANSFER_NOTIF_TMPL)
|
217
|
+
end
|
218
|
+
|
237
219
|
# @return :success if all sessions statuses returned by "start" are success
|
238
220
|
# else return the first error exception object
|
239
221
|
def self.session_status(statuses)
|
data/lib/aspera/cli/version.rb
CHANGED