aspera-cli 4.2.1 → 4.5.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +1580 -946
- data/bin/ascli +1 -1
- data/bin/asession +3 -5
- data/docs/Makefile +8 -11
- data/docs/README.erb.md +1521 -829
- data/docs/doc_tools.rb +58 -0
- data/docs/test_env.conf +3 -1
- data/examples/faspex4.rb +28 -19
- data/examples/transfer.rb +2 -2
- data/lib/aspera/aoc.rb +157 -134
- data/lib/aspera/cli/listener/progress_multi.rb +5 -5
- data/lib/aspera/cli/main.rb +106 -48
- data/lib/aspera/cli/manager.rb +19 -20
- data/lib/aspera/cli/plugin.rb +22 -7
- data/lib/aspera/cli/plugins/aoc.rb +260 -208
- data/lib/aspera/cli/plugins/ats.rb +11 -10
- data/lib/aspera/cli/plugins/bss.rb +2 -2
- data/lib/aspera/cli/plugins/config.rb +360 -189
- data/lib/aspera/cli/plugins/faspex.rb +119 -56
- data/lib/aspera/cli/plugins/faspex5.rb +32 -17
- data/lib/aspera/cli/plugins/node.rb +72 -31
- data/lib/aspera/cli/plugins/orchestrator.rb +5 -3
- data/lib/aspera/cli/plugins/preview.rb +94 -68
- data/lib/aspera/cli/plugins/server.rb +16 -5
- data/lib/aspera/cli/plugins/shares.rb +17 -0
- data/lib/aspera/cli/transfer_agent.rb +64 -82
- data/lib/aspera/cli/version.rb +1 -1
- data/lib/aspera/command_line_builder.rb +48 -31
- data/lib/aspera/cos_node.rb +4 -3
- data/lib/aspera/environment.rb +4 -4
- data/lib/aspera/fasp/{manager.rb → agent_base.rb} +7 -6
- data/lib/aspera/fasp/{connect.rb → agent_connect.rb} +46 -39
- data/lib/aspera/fasp/{local.rb → agent_direct.rb} +42 -38
- data/lib/aspera/fasp/{http_gw.rb → agent_httpgw.rb} +50 -29
- data/lib/aspera/fasp/{node.rb → agent_node.rb} +43 -4
- data/lib/aspera/fasp/agent_trsdk.rb +106 -0
- data/lib/aspera/fasp/default.rb +17 -0
- data/lib/aspera/fasp/installation.rb +64 -48
- data/lib/aspera/fasp/parameters.rb +78 -91
- data/lib/aspera/fasp/parameters.yaml +531 -0
- data/lib/aspera/fasp/uri.rb +1 -1
- data/lib/aspera/faspex_gw.rb +12 -11
- data/lib/aspera/id_generator.rb +22 -0
- data/lib/aspera/keychain/encrypted_hash.rb +120 -0
- data/lib/aspera/keychain/macos_security.rb +94 -0
- data/lib/aspera/log.rb +45 -32
- data/lib/aspera/node.rb +9 -4
- data/lib/aspera/oauth.rb +116 -100
- data/lib/aspera/persistency_action_once.rb +11 -7
- data/lib/aspera/persistency_folder.rb +6 -26
- data/lib/aspera/rest.rb +66 -50
- data/lib/aspera/sync.rb +40 -35
- data/lib/aspera/timer_limiter.rb +22 -0
- metadata +86 -29
- data/docs/transfer_spec.html +0 -99
- data/lib/aspera/api_detector.rb +0 -60
- data/lib/aspera/fasp/aoc.rb +0 -24
- data/lib/aspera/secrets.rb +0 -20
@@ -6,6 +6,8 @@ require 'aspera/preview/file_types'
|
|
6
6
|
require 'aspera/persistency_action_once'
|
7
7
|
require 'aspera/node'
|
8
8
|
require 'aspera/hash_ext'
|
9
|
+
require 'aspera/timer_limiter'
|
10
|
+
require 'aspera/id_generator'
|
9
11
|
require 'date'
|
10
12
|
require 'securerandom'
|
11
13
|
|
@@ -24,7 +26,8 @@ module Aspera
|
|
24
26
|
DEFAULT_PREVIEWS_FOLDER='previews'
|
25
27
|
AK_MARKER_FILE='.aspera_access_key'
|
26
28
|
LOCAL_STORAGE_PCVL='file:///'
|
27
|
-
|
29
|
+
LOG_LIMITER_SEC=30.0
|
30
|
+
private_constant :PREV_GEN_TAG, :PREVIEW_FOLDER_SUFFIX, :PREVIEW_BASENAME, :TMP_DIR_PREFIX, :DEFAULT_PREVIEWS_FOLDER, :LOCAL_STORAGE_PCVL, :AK_MARKER_FILE, :LOG_LIMITER_SEC
|
28
31
|
|
29
32
|
# option_skip_format has special accessors
|
30
33
|
attr_accessor :option_previews_folder
|
@@ -40,6 +43,8 @@ module Aspera
|
|
40
43
|
@preview_formats_to_generate=Aspera::Preview::Generator::PREVIEW_FORMATS.clone
|
41
44
|
# options for generation
|
42
45
|
@gen_options=Aspera::Preview::Options.new
|
46
|
+
# used to trigger periodic processing
|
47
|
+
@periodic=TimerLimiter.new(LOG_LIMITER_SEC)
|
43
48
|
# link CLI options to gen_info attributes
|
44
49
|
self.options.set_obj_attr(:skip_format,self,:option_skip_format,[]) # no skip
|
45
50
|
self.options.set_obj_attr(:folder_reset_cache,self,:option_folder_reset_cache,:no)
|
@@ -113,13 +118,14 @@ module Aspera
|
|
113
118
|
end
|
114
119
|
|
115
120
|
# old version based on folders
|
116
|
-
|
121
|
+
# @param iteration_persistency can be nil
|
122
|
+
def process_trevents(iteration_persistency)
|
117
123
|
events_filter={
|
118
124
|
'access_key'=>@access_key_self['id'],
|
119
125
|
'type'=>'download.ended'
|
120
126
|
}
|
121
|
-
# optionally
|
122
|
-
events_filter['iteration_token']=
|
127
|
+
# optionally add iteration token from persistency
|
128
|
+
events_filter['iteration_token']=iteration_persistency.data.first unless iteration_persistency.nil?
|
123
129
|
begin
|
124
130
|
events=@api_node.read('events',events_filter)[:data]
|
125
131
|
rescue RestCallError => e
|
@@ -132,47 +138,63 @@ module Aspera
|
|
132
138
|
end
|
133
139
|
return if events.empty?
|
134
140
|
events.each do |event|
|
135
|
-
|
136
|
-
|
137
|
-
|
138
|
-
|
139
|
-
|
140
|
-
|
141
|
-
|
142
|
-
|
143
|
-
|
144
|
-
|
141
|
+
if event['data']['direction'].eql?('receive') and
|
142
|
+
event['data']['status'].eql?('completed') and
|
143
|
+
event['data']['error_code'].eql?(0) and
|
144
|
+
event['data'].dig('tags','aspera',PREV_GEN_TAG).nil?
|
145
|
+
folder_id=event.dig('data','tags','aspera','node','file_id')
|
146
|
+
folder_id||=event.dig('data','file_id')
|
147
|
+
if !folder_id.nil?
|
148
|
+
folder_entry=@api_node.read("files/#{folder_id}")[:data] rescue nil
|
149
|
+
scan_folder_files(folder_entry) unless folder_entry.nil?
|
150
|
+
end
|
151
|
+
end
|
152
|
+
if @periodic.trigger? or event.equal?(events.last)
|
153
|
+
Log.log.info("Processed event #{event['id']}")
|
154
|
+
# save checkpoint to avoid losing processing in case of error
|
155
|
+
if !iteration_persistency.nil?
|
156
|
+
iteration_persistency.data[0]=event['id'].to_s
|
157
|
+
iteration_persistency.save
|
158
|
+
end
|
159
|
+
end
|
145
160
|
end
|
146
|
-
return events.last['id'].to_s
|
147
161
|
end
|
148
162
|
|
149
163
|
# requests recent events on node api and process newly modified folders
|
150
|
-
def process_events(
|
164
|
+
def process_events(iteration_persistency)
|
151
165
|
# get new file creation by access key (TODO: what if file already existed?)
|
152
166
|
events_filter={
|
153
167
|
'access_key'=>@access_key_self['id'],
|
154
168
|
'type'=>'file.*'
|
155
169
|
}
|
156
|
-
#
|
157
|
-
events_filter['iteration_token']=
|
170
|
+
# optionally add iteration token from persistency
|
171
|
+
events_filter['iteration_token']=iteration_persistency.data.first unless iteration_persistency.nil?
|
158
172
|
events=@api_node.read('events',events_filter)[:data]
|
159
173
|
return if events.empty?
|
160
174
|
events.each do |event|
|
161
175
|
# process only files
|
162
|
-
|
163
|
-
|
164
|
-
|
165
|
-
|
166
|
-
|
167
|
-
|
168
|
-
|
176
|
+
if event.dig('data','type').eql?('file')
|
177
|
+
file_entry=@api_node.read("files/#{event['data']['id']}")[:data] rescue nil
|
178
|
+
if !file_entry.nil? and
|
179
|
+
@option_skip_folders.select{|d|file_entry['path'].start_with?(d)}.empty?
|
180
|
+
file_entry['parent_file_id']=event['data']['parent_file_id']
|
181
|
+
if event['types'].include?('file.deleted')
|
182
|
+
Log.log.error('TODO'.red)
|
183
|
+
end
|
184
|
+
if event['types'].include?('file.deleted')
|
185
|
+
generate_preview(file_entry)
|
186
|
+
end
|
187
|
+
end
|
169
188
|
end
|
170
|
-
if event
|
171
|
-
|
189
|
+
if @periodic.trigger? or event.equal?(events.last)
|
190
|
+
Log.log.info("Processing event #{event['id']}")
|
191
|
+
# save checkpoint to avoid losing processing in case of error
|
192
|
+
if !iteration_persistency.nil?
|
193
|
+
iteration_persistency.data[0]=event['id'].to_s
|
194
|
+
iteration_persistency.save
|
195
|
+
end
|
172
196
|
end
|
173
197
|
end
|
174
|
-
# write new iteration file
|
175
|
-
return events.last['id'].to_s
|
176
198
|
end
|
177
199
|
|
178
200
|
def do_transfer(direction,folder_id,source_filename,destination='/')
|
@@ -180,15 +202,17 @@ module Aspera
|
|
180
202
|
if @default_transfer_spec.nil?
|
181
203
|
# make a dummy call to get some default transfer parameters
|
182
204
|
res=@api_node.create('files/upload_setup',{'transfer_requests'=>[{'transfer_request'=>{'paths'=>[{}],'destination_root'=>'/'}}]})
|
183
|
-
|
205
|
+
template_ts=res[:data]['transfer_specs'].first['transfer_spec']
|
184
206
|
# get ports, anyway that should be 33001 for both. add remote_user ?
|
185
|
-
@default_transfer_spec=['ssh_port','fasp_port'].inject({}){|h,e|h[e]=
|
207
|
+
@default_transfer_spec=['ssh_port','fasp_port'].inject({}){|h,e|h[e]=template_ts[e];h}
|
208
|
+
if ! @default_transfer_spec['remote_user'].eql?(Aspera::Fasp::Default::ACCESS_KEY_TRANSFER_USER)
|
209
|
+
Log.log.warn("remote_user shall be xfer")
|
210
|
+
@default_transfer_spec['remote_user']=Aspera::Fasp::Default::ACCESS_KEY_TRANSFER_USER
|
211
|
+
end
|
212
|
+
Aspera::Node::set_ak_basic_token(@default_transfer_spec,@access_key_self['id'],self.options.get_option(:password,:mandatory))
|
186
213
|
# note: we use the same address for ascp than for node api instead of the one from upload_setup
|
187
|
-
|
188
|
-
|
189
|
-
'remote_host' => @transfer_server_address,
|
190
|
-
'remote_user' => Fasp::ACCESS_KEY_TRANSFER_USER
|
191
|
-
})
|
214
|
+
# TODO: configurable ? useful ?
|
215
|
+
@default_transfer_spec['remote_host']=@transfer_server_address
|
192
216
|
end
|
193
217
|
tspec=@default_transfer_spec.merge({
|
194
218
|
'direction' => direction,
|
@@ -338,40 +362,45 @@ module Aspera
|
|
338
362
|
entry=entries_to_process.shift
|
339
363
|
# process this entry only if it is within the scan_start
|
340
364
|
entry_path_with_slash=entry['path']
|
365
|
+
Log.log.info("processing entry #{entry_path_with_slash}") if @periodic.trigger?
|
341
366
|
entry_path_with_slash="#{entry_path_with_slash}/" unless entry_path_with_slash.end_with?('/')
|
342
367
|
if !scan_start.nil? and !scan_start.start_with?(entry_path_with_slash) and !entry_path_with_slash.start_with?(scan_start)
|
343
368
|
Log.log.debug("#{entry['path']} folder (skip start)".bg_red)
|
344
369
|
next
|
345
370
|
end
|
346
371
|
Log.log.debug("item:#{entry}")
|
347
|
-
|
348
|
-
|
349
|
-
|
350
|
-
|
351
|
-
|
352
|
-
|
353
|
-
|
354
|
-
|
355
|
-
|
356
|
-
|
357
|
-
|
358
|
-
|
359
|
-
|
360
|
-
|
361
|
-
|
362
|
-
|
363
|
-
|
364
|
-
|
365
|
-
|
366
|
-
|
367
|
-
folder_entry
|
372
|
+
begin
|
373
|
+
case entry['type']
|
374
|
+
when 'file'
|
375
|
+
if filter_block.call(entry)
|
376
|
+
generate_preview(entry)
|
377
|
+
else
|
378
|
+
Log.log.debug('skip by filter')
|
379
|
+
end
|
380
|
+
when 'link'
|
381
|
+
Log.log.debug('Ignoring link.')
|
382
|
+
when 'folder'
|
383
|
+
if @option_skip_folders.include?(entry['path'])
|
384
|
+
Log.log.debug("#{entry['path']} folder (skip list)".bg_red)
|
385
|
+
else
|
386
|
+
Log.log.debug("#{entry['path']} folder".green)
|
387
|
+
# get folder content
|
388
|
+
folder_entries=get_folder_entries(entry['id'])
|
389
|
+
# process all items in current folder
|
390
|
+
folder_entries.each do |folder_entry|
|
391
|
+
# add path for older versions of ES
|
392
|
+
if !folder_entry.has_key?('path')
|
393
|
+
folder_entry['path']=entry_path_with_slash+folder_entry['name']
|
394
|
+
end
|
395
|
+
folder_entry['parent_file_id']=entry['id']
|
396
|
+
entries_to_process.push(folder_entry)
|
368
397
|
end
|
369
|
-
folder_entry['parent_file_id']=entry['id']
|
370
|
-
entries_to_process.push(folder_entry)
|
371
398
|
end
|
399
|
+
else
|
400
|
+
Log.log.warn("unknown entry type: #{entry['type']}")
|
372
401
|
end
|
373
|
-
|
374
|
-
Log.log.warn("
|
402
|
+
rescue => e
|
403
|
+
Log.log.warn("An error occured: #{e}, ignoring")
|
375
404
|
end
|
376
405
|
end
|
377
406
|
end
|
@@ -437,18 +466,15 @@ module Aspera
|
|
437
466
|
scan_folder_files(folder_info,scan_path)
|
438
467
|
return Main.result_status('scan finished')
|
439
468
|
when :events,:trevents
|
440
|
-
iteration_data=[]
|
441
469
|
iteration_persistency=nil
|
442
470
|
if self.options.get_option(:once_only,:mandatory)
|
443
471
|
iteration_persistency=PersistencyActionOnce.new(
|
444
472
|
manager: @agents[:persistency],
|
445
|
-
data:
|
446
|
-
|
473
|
+
data: [],
|
474
|
+
id: IdGenerator.from_list(['preview_iteration',command.to_s,self.options.get_option(:url,:mandatory),self.options.get_option(:username,:mandatory)]))
|
447
475
|
end
|
448
|
-
|
449
|
-
#
|
450
|
-
iteration_data[0]=send("process_#{command}",iteration_data[0])
|
451
|
-
iteration_persistency.save unless iteration_persistency.nil?
|
476
|
+
# call processing method specified by command line command
|
477
|
+
send("process_#{command}",iteration_persistency)
|
452
478
|
return Main.result_status("#{command} finished")
|
453
479
|
when :check
|
454
480
|
Aspera::Preview::Utils.check_tools(@skip_types)
|
@@ -1,5 +1,6 @@
|
|
1
1
|
require 'aspera/cli/basic_auth_plugin'
|
2
2
|
require 'aspera/ascmd'
|
3
|
+
require 'aspera/fasp/default'
|
3
4
|
require 'aspera/ssh'
|
4
5
|
require 'aspera/nagios'
|
5
6
|
require 'tempfile'
|
@@ -67,13 +68,15 @@ module Aspera
|
|
67
68
|
def execute_action
|
68
69
|
server_uri=URI.parse(self.options.get_option(:url,:mandatory))
|
69
70
|
Log.log.debug("URI : #{server_uri}, port=#{server_uri.port}, scheme:#{server_uri.scheme}")
|
71
|
+
server_transfer_spec={'remote_host'=>server_uri.hostname}
|
70
72
|
shell_executor=nil
|
71
73
|
case server_uri.scheme
|
72
74
|
when 'ssh'
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
75
|
+
if self.options.get_option(:username,:optional).nil?
|
76
|
+
self.options.set_option(:username,Aspera::Fasp::Default::ACCESS_KEY_TRANSFER_USER)
|
77
|
+
Log.log.info("Using default transfer user: #{Aspera::Fasp::Default::ACCESS_KEY_TRANSFER_USER}")
|
78
|
+
end
|
79
|
+
server_transfer_spec['remote_user']=self.options.get_option(:username,:mandatory)
|
77
80
|
ssh_options=self.options.get_option(:ssh_options,:optional)
|
78
81
|
raise 'expecting a Hash for ssh_options' unless ssh_options.is_a?(Hash)
|
79
82
|
if !server_uri.port.nil?
|
@@ -102,8 +105,16 @@ module Aspera
|
|
102
105
|
cred_set=true
|
103
106
|
end
|
104
107
|
end
|
105
|
-
|
108
|
+
# if user provided transfer spec has a token, we will use by pass keys
|
109
|
+
cred_set=true if self.transfer.option_transfer_spec['token'].is_a?(String)
|
110
|
+
raise 'either password, key , or transfer spec token must be provided' if !cred_set
|
106
111
|
shell_executor=Ssh.new(server_transfer_spec['remote_host'],server_transfer_spec['remote_user'],ssh_options)
|
112
|
+
when 'https'
|
113
|
+
raise "ERROR: transfer spec with token required" unless self.transfer.option_transfer_spec['token'].is_a?(String)
|
114
|
+
server_transfer_spec.merge!({
|
115
|
+
'wss_enabled'=>true,
|
116
|
+
'wss_port' =>server_uri.port
|
117
|
+
})
|
107
118
|
when 'local'
|
108
119
|
shell_executor=LocalExecutor.new
|
109
120
|
else
|
@@ -4,6 +4,23 @@ module Aspera
|
|
4
4
|
module Cli
|
5
5
|
module Plugins
|
6
6
|
class Shares < BasicAuthPlugin
|
7
|
+
class << self
|
8
|
+
def detect(base_url)
|
9
|
+
api=Rest.new({:base_url=>base_url})
|
10
|
+
# Shares
|
11
|
+
begin
|
12
|
+
# shall fail: shares requires auth, but we check error message
|
13
|
+
api.read('node_api/app')
|
14
|
+
rescue RestCallError => e
|
15
|
+
if e.response.code.to_s.eql?('401') and e.response.body.eql?('{"error":{"user_message":"API user authentication failed"}}')
|
16
|
+
return {:version=>'unknown'}
|
17
|
+
end
|
18
|
+
rescue
|
19
|
+
end
|
20
|
+
nil
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
7
24
|
def initialize(env)
|
8
25
|
super(env)
|
9
26
|
#self.options.parse_options!
|
@@ -1,8 +1,4 @@
|
|
1
|
-
require 'aspera/fasp/
|
2
|
-
require 'aspera/fasp/connect'
|
3
|
-
require 'aspera/fasp/node'
|
4
|
-
require 'aspera/fasp/aoc'
|
5
|
-
require 'aspera/fasp/http_gw'
|
1
|
+
require 'aspera/fasp/parameters'
|
6
2
|
require 'aspera/cli/listener/logger'
|
7
3
|
require 'aspera/cli/listener/progress_multi'
|
8
4
|
|
@@ -16,13 +12,25 @@ module Aspera
|
|
16
12
|
FILE_LIST_FROM_ARGS='@args'
|
17
13
|
# special value for --sources : read file list from transfer spec (--ts)
|
18
14
|
FILE_LIST_FROM_TRANSFER_SPEC='@ts'
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
15
|
+
DEFAULT_TRANSFER_NOTIF_TMPL=<<END_OF_TEMPLATE
|
16
|
+
From: <%=from_name%> <<%=from_email%>>
|
17
|
+
To: <<%=to%>>
|
18
|
+
Subject: <%=subject%>
|
19
|
+
|
20
|
+
Transfer is: <%=global_transfer_status%>
|
21
|
+
|
22
|
+
<%=ts.to_yaml%>
|
23
|
+
END_OF_TEMPLATE
|
24
|
+
#% (formating bug in eclipse)
|
25
|
+
private_constant :FILE_LIST_FROM_ARGS,:FILE_LIST_FROM_TRANSFER_SPEC,:DEFAULT_TRANSFER_NOTIF_TMPL
|
26
|
+
TRANSFER_AGENTS=[:direct,:node,:connect,:httpgw,:trsdk]
|
27
|
+
|
28
|
+
# @param env external objects: option manager, config file manager
|
29
|
+
def initialize(opt_mgr,config)
|
30
|
+
@opt_mgr=opt_mgr
|
31
|
+
@config=config
|
32
|
+
# command line can override transfer spec
|
33
|
+
@transfer_spec_cmdline={'create_dir'=>true}
|
26
34
|
# the currently selected transfer agent
|
27
35
|
@agent=nil
|
28
36
|
@progress_listener=Listener::ProgressMulti.new
|
@@ -33,9 +41,9 @@ module Aspera
|
|
33
41
|
@opt_mgr.add_opt_simple(:local_resume,"set resume policy (Hash, use @json: prefix), current=#{@opt_mgr.get_option(:local_resume,:optional)}")
|
34
42
|
@opt_mgr.add_opt_simple(:to_folder,"destination folder for downloaded files")
|
35
43
|
@opt_mgr.add_opt_simple(:sources,"list of source files (see doc)")
|
36
|
-
@opt_mgr.add_opt_simple(:transfer_info,"
|
44
|
+
@opt_mgr.add_opt_simple(:transfer_info,"parameters for transfer agent")
|
37
45
|
@opt_mgr.add_opt_list(:src_type,[:list,:pair],"type of file list")
|
38
|
-
@opt_mgr.add_opt_list(:transfer,
|
46
|
+
@opt_mgr.add_opt_list(:transfer,TRANSFER_AGENTS,"type of transfer agent")
|
39
47
|
@opt_mgr.add_opt_list(:progress,[:none,:native,:multi],"type of progress bar")
|
40
48
|
@opt_mgr.set_option(:transfer,:direct)
|
41
49
|
@opt_mgr.set_option(:src_type,:list)
|
@@ -55,7 +63,7 @@ module Aspera
|
|
55
63
|
@agent.add_listener(Listener::Logger.new)
|
56
64
|
# use local progress bar if asked so, or if native and non local ascp (because only local ascp has native progress bar)
|
57
65
|
if @opt_mgr.get_option(:progress,:mandatory).eql?(:multi) or
|
58
|
-
(@opt_mgr.get_option(:progress,:mandatory).eql?(:native) and
|
66
|
+
(@opt_mgr.get_option(:progress,:mandatory).eql?(:native) and ! instance.class.to_s.eql?('Aspera::Fasp::AgentDirect'))
|
59
67
|
@agent.add_listener(@progress_listener)
|
60
68
|
end
|
61
69
|
end
|
@@ -64,63 +72,23 @@ module Aspera
|
|
64
72
|
def set_agent_by_options
|
65
73
|
return nil unless @agent.nil?
|
66
74
|
agent_type=@opt_mgr.get_option(:transfer,:mandatory)
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
# support: @preset:<name>
|
81
|
-
# support extended values
|
82
|
-
node_config=@opt_mgr.get_option(:transfer_info,:optional)
|
83
|
-
# if not specified: use default node
|
84
|
-
if node_config.nil?
|
85
|
-
param_set_name=@config.get_plugin_default_config_name(:node)
|
86
|
-
raise CliBadArgument,"No default node configured, Please specify --#{:transfer_info.to_s.gsub('_','-')}" if param_set_name.nil?
|
87
|
-
node_config=@config.preset_by_name(param_set_name)
|
88
|
-
end
|
89
|
-
Log.log.debug("node=#{node_config}")
|
90
|
-
raise CliBadArgument,"the node configuration shall be Hash, not #{node_config.class} (#{node_config}), use either @json:<json> or @preset:<parameter set name>" if !node_config.is_a?(Hash)
|
91
|
-
# now check there are required parameters
|
92
|
-
sym_config=[:url,:username,:password].inject({}) do |h,param|
|
93
|
-
raise CliBadArgument,"missing parameter [#{param}] in node specification: #{node_config}" if !node_config.has_key?(param.to_s)
|
94
|
-
h[param]=node_config[param.to_s]
|
95
|
-
h
|
96
|
-
end
|
97
|
-
node_api=Rest.new({
|
98
|
-
:base_url => sym_config[:url],
|
99
|
-
:auth => {
|
100
|
-
:type =>:basic,
|
101
|
-
:username => sym_config[:username],
|
102
|
-
:password => sym_config[:password]
|
103
|
-
}})
|
104
|
-
new_agent=Fasp::Node.new(node_api)
|
105
|
-
when :aoc
|
106
|
-
aoc_config=@opt_mgr.get_option(:transfer_info,:optional)
|
107
|
-
if aoc_config.nil?
|
108
|
-
param_set_name=@config.get_plugin_default_config_name(:aspera)
|
109
|
-
raise CliBadArgument,"No default AoC configured, Please specify --#{:transfer_info.to_s.gsub('_','-')}" if param_set_name.nil?
|
110
|
-
aoc_config=@config.preset_by_name(param_set_name)
|
111
|
-
end
|
112
|
-
Log.log.debug("aoc=#{aoc_config}")
|
113
|
-
raise CliBadArgument,"the aoc configuration shall be Hash, not #{aoc_config.class} (#{aoc_config}), refer to manual" if !aoc_config.is_a?(Hash)
|
114
|
-
# convert keys from string (config) to symbol (agent)
|
115
|
-
aoc_config=aoc_config.symbolize_keys
|
116
|
-
# convert auth value from string (config) to symbol (agent)
|
117
|
-
aoc_config[:auth]=aoc_config[:auth].to_sym if aoc_config[:auth].is_a?(String)
|
118
|
-
# private key could be @file:... in config
|
119
|
-
aoc_config[:private_key]=ExtendedValue.instance.evaluate(aoc_config[:private_key])
|
120
|
-
new_agent=Fasp::Aoc.new(aoc_config)
|
121
|
-
else
|
122
|
-
raise "Unexpected transfer agent type: #{agent_type}"
|
75
|
+
require "aspera/fasp/agent_#{agent_type}"
|
76
|
+
agent_options=@opt_mgr.get_option(:transfer_info,:optional)
|
77
|
+
raise CliBadArgument,"the transfer agent configuration shall be Hash, not #{agent_options.class} (#{agent_options}), use either @json:<json> or @preset:<parameter set name>" unless [Hash,NilClass].include?(agent_options.class)
|
78
|
+
# special case
|
79
|
+
if agent_type.eql?(:node) and agent_options.nil?
|
80
|
+
param_set_name=@config.get_plugin_default_config_name(:node)
|
81
|
+
raise CliBadArgument,"No default node configured, Please specify --#{:transfer_info.to_s.gsub('_','-')}" if param_set_name.nil?
|
82
|
+
agent_options=@config.preset_by_name(param_set_name)
|
83
|
+
end
|
84
|
+
# special case
|
85
|
+
if agent_type.eql?(:direct) and @opt_mgr.get_option(:progress,:mandatory).eql?(:native)
|
86
|
+
agent_options={} if agent_options.nil?
|
87
|
+
agent_options[:quiet]=false
|
123
88
|
end
|
89
|
+
agent_options=agent_options.symbolize_keys if agent_options.is_a?(Hash)
|
90
|
+
# get agent instance
|
91
|
+
new_agent=Kernel.const_get("Aspera::Fasp::Agent#{agent_type.capitalize}").new(agent_options)
|
124
92
|
set_agent_instance(new_agent)
|
125
93
|
return nil
|
126
94
|
end
|
@@ -130,7 +98,7 @@ module Aspera
|
|
130
98
|
# param: 'send' or 'receive'
|
131
99
|
def destination_folder(direction)
|
132
100
|
dest_folder=@opt_mgr.get_option(:to_folder,:optional)
|
133
|
-
return dest_folder unless dest_folder.nil?
|
101
|
+
return File.expand_path(dest_folder) unless dest_folder.nil?
|
134
102
|
dest_folder=@transfer_spec_cmdline['destination_root']
|
135
103
|
return dest_folder unless dest_folder.nil?
|
136
104
|
# default: / on remote, . on local
|
@@ -161,7 +129,8 @@ module Aspera
|
|
161
129
|
raise CliBadArgument,"specify at least one file on command line or use --sources=#{FILE_LIST_FROM_TRANSFER_SPEC} to use transfer spec" if !file_list.is_a?(Array) or file_list.empty?
|
162
130
|
when FILE_LIST_FROM_TRANSFER_SPEC
|
163
131
|
Log.log.debug("assume list provided in transfer spec")
|
164
|
-
|
132
|
+
special_case_direct_with_list=@opt_mgr.get_option(:transfer,:mandatory).eql?(:direct) and Fasp::Parameters.ts_has_file_list(@transfer_spec_cmdline)
|
133
|
+
raise CliBadArgument,"transfer spec on command line must have sources" if @transfer_paths.nil? and !special_case_direct_with_list
|
165
134
|
# here we assume check of sources is made in transfer agent
|
166
135
|
return @transfer_paths
|
167
136
|
when Array
|
@@ -189,20 +158,20 @@ module Aspera
|
|
189
158
|
|
190
159
|
# start a transfer and wait for completion, plugins shall use this method
|
191
160
|
# @param transfer_spec
|
192
|
-
# @param
|
193
|
-
#
|
161
|
+
# @param tr_opts specific options for the transfer_agent
|
162
|
+
# tr_opts[:src] specifies how destination_root is set (how transfer spec was generated)
|
194
163
|
# other options are carried to specific agent
|
195
|
-
def start(transfer_spec,
|
164
|
+
def start(transfer_spec,tr_opts)
|
196
165
|
# check parameters
|
197
166
|
raise "transfer_spec must be hash" unless transfer_spec.is_a?(Hash)
|
198
|
-
raise "
|
167
|
+
raise "tr_opts must be hash" unless tr_opts.is_a?(Hash)
|
199
168
|
# process :src option
|
200
169
|
case transfer_spec['direction']
|
201
170
|
when 'receive'
|
202
171
|
# init default if required in any case
|
203
172
|
@transfer_spec_cmdline['destination_root']||=destination_folder(transfer_spec['direction'])
|
204
173
|
when 'send'
|
205
|
-
case
|
174
|
+
case tr_opts[:src]
|
206
175
|
when :direct
|
207
176
|
# init default if required
|
208
177
|
@transfer_spec_cmdline['destination_root']||=destination_folder(transfer_spec['direction'])
|
@@ -213,12 +182,12 @@ module Aspera
|
|
213
182
|
when :node_gen4
|
214
183
|
@transfer_spec_cmdline.delete('destination_root') if @transfer_spec_cmdline.has_key?('destination_root_id')
|
215
184
|
else
|
216
|
-
raise StandardError,"InternalError: unsupported value: #{
|
185
|
+
raise StandardError,"InternalError: unsupported value: #{tr_opts[:src]}"
|
217
186
|
end
|
218
187
|
end
|
219
188
|
|
220
189
|
# only used here
|
221
|
-
|
190
|
+
tr_opts.delete(:src)
|
222
191
|
|
223
192
|
# update command line paths, unless destination already has one
|
224
193
|
@transfer_spec_cmdline['paths']=transfer_spec['paths'] || ts_source_paths
|
@@ -227,13 +196,26 @@ module Aspera
|
|
227
196
|
# create transfer agent
|
228
197
|
self.set_agent_by_options
|
229
198
|
Log.log.debug("transfer agent is a #{@agent.class}")
|
230
|
-
@agent.start_transfer(transfer_spec,
|
199
|
+
@agent.start_transfer(transfer_spec,tr_opts)
|
231
200
|
result=@agent.wait_for_transfers_completion
|
232
201
|
@progress_listener.reset
|
233
|
-
Fasp::
|
202
|
+
Fasp::AgentBase.validate_status_list(result)
|
203
|
+
send_email_transfer_notification(transfer_spec,result)
|
234
204
|
return result
|
235
205
|
end
|
236
206
|
|
207
|
+
def send_email_transfer_notification(transfer_spec,statuses)
|
208
|
+
return if @opt_mgr.get_option(:notif_to,:optional).nil?
|
209
|
+
global_status=self.class.session_status(statuses)
|
210
|
+
email_vars={
|
211
|
+
global_transfer_status: global_status,
|
212
|
+
subject: "ascli transfer: #{global_status}",
|
213
|
+
body: "Transfer is: #{global_status}",
|
214
|
+
ts: transfer_spec
|
215
|
+
}
|
216
|
+
@config.send_email_template(email_vars,DEFAULT_TRANSFER_NOTIF_TMPL)
|
217
|
+
end
|
218
|
+
|
237
219
|
# @return :success if all sessions statuses returned by "start" are success
|
238
220
|
# else return the first error exception object
|
239
221
|
def self.session_status(statuses)
|
data/lib/aspera/cli/version.rb
CHANGED