aspera-cli 4.0.0 → 4.2.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +843 -304
- data/bin/dascli +13 -0
- data/docs/Makefile +4 -4
- data/docs/README.erb.md +805 -172
- data/docs/test_env.conf +22 -3
- data/examples/aoc.rb +14 -3
- data/examples/faspex4.rb +89 -0
- data/lib/aspera/aoc.rb +87 -108
- data/lib/aspera/cli/formater.rb +2 -0
- data/lib/aspera/cli/main.rb +89 -49
- data/lib/aspera/cli/plugin.rb +9 -4
- data/lib/aspera/cli/plugins/alee.rb +1 -1
- data/lib/aspera/cli/plugins/aoc.rb +188 -173
- data/lib/aspera/cli/plugins/ats.rb +2 -2
- data/lib/aspera/cli/plugins/config.rb +218 -145
- data/lib/aspera/cli/plugins/console.rb +2 -2
- data/lib/aspera/cli/plugins/faspex.rb +114 -61
- data/lib/aspera/cli/plugins/faspex5.rb +85 -43
- data/lib/aspera/cli/plugins/node.rb +3 -3
- data/lib/aspera/cli/plugins/preview.rb +59 -45
- data/lib/aspera/cli/plugins/server.rb +23 -8
- data/lib/aspera/cli/transfer_agent.rb +77 -49
- data/lib/aspera/cli/version.rb +1 -1
- data/lib/aspera/command_line_builder.rb +49 -31
- data/lib/aspera/cos_node.rb +33 -28
- data/lib/aspera/environment.rb +2 -2
- data/lib/aspera/fasp/connect.rb +28 -21
- data/lib/aspera/fasp/http_gw.rb +140 -28
- data/lib/aspera/fasp/installation.rb +93 -46
- data/lib/aspera/fasp/local.rb +88 -45
- data/lib/aspera/fasp/manager.rb +15 -0
- data/lib/aspera/fasp/node.rb +4 -4
- data/lib/aspera/fasp/parameters.rb +59 -101
- data/lib/aspera/fasp/parameters.yaml +531 -0
- data/lib/aspera/fasp/resume_policy.rb +13 -12
- data/lib/aspera/fasp/uri.rb +1 -1
- data/lib/aspera/log.rb +1 -1
- data/lib/aspera/node.rb +61 -1
- data/lib/aspera/oauth.rb +49 -46
- data/lib/aspera/persistency_folder.rb +9 -4
- data/lib/aspera/preview/file_types.rb +53 -21
- data/lib/aspera/preview/generator.rb +3 -3
- data/lib/aspera/rest.rb +29 -18
- data/lib/aspera/secrets.rb +20 -0
- data/lib/aspera/sync.rb +40 -35
- data/lib/aspera/temp_file_manager.rb +19 -0
- data/lib/aspera/web_auth.rb +105 -0
- metadata +54 -20
- data/docs/transfer_spec.html +0 -99
@@ -4,6 +4,7 @@ require 'aspera/preview/options'
|
|
4
4
|
require 'aspera/preview/utils'
|
5
5
|
require 'aspera/preview/file_types'
|
6
6
|
require 'aspera/persistency_action_once'
|
7
|
+
require 'aspera/node'
|
7
8
|
require 'aspera/hash_ext'
|
8
9
|
require 'date'
|
9
10
|
require 'securerandom'
|
@@ -56,9 +57,11 @@ module Aspera
|
|
56
57
|
self.options.add_opt_simple(:case,'basename of output for for test')
|
57
58
|
self.options.add_opt_simple(:scan_path,'subpath in folder id to start scan in (default=/)')
|
58
59
|
self.options.add_opt_simple(:scan_id,'forder id in storage to start scan in, default is access key main folder id')
|
60
|
+
self.options.add_opt_boolean(:mimemagic,'use Mime type detection of gem mimemagic')
|
59
61
|
self.options.add_opt_list(:overwrite,[:always,:never,:mtime],'when to overwrite result file')
|
60
62
|
self.options.add_opt_list(:file_access,[:local,:remote],'how to read and write files in repository')
|
61
63
|
self.options.set_option(:temp_folder,Dir.tmpdir)
|
64
|
+
self.options.set_option(:mimemagic,:false)
|
62
65
|
|
63
66
|
# add other options for generator (and set default values)
|
64
67
|
Aspera::Preview::Options::DESCRIPTIONS.each do |opt|
|
@@ -100,8 +103,6 @@ module Aspera
|
|
100
103
|
return @preview_formats_to_generate.map{|i|i.to_s}.join(',')
|
101
104
|
end
|
102
105
|
|
103
|
-
ACTIONS=[:scan,:events,:trevents,:check,:test]
|
104
|
-
|
105
106
|
# /files/id/files is normally cached in redis, but we can discard the cache
|
106
107
|
# but /files/id is not cached
|
107
108
|
def get_folder_entries(file_id,request_args=nil)
|
@@ -112,14 +113,23 @@ module Aspera
|
|
112
113
|
end
|
113
114
|
|
114
115
|
# old version based on folders
|
115
|
-
def
|
116
|
+
def process_trevents(iteration_token)
|
116
117
|
events_filter={
|
117
118
|
'access_key'=>@access_key_self['id'],
|
118
119
|
'type'=>'download.ended'
|
119
120
|
}
|
120
121
|
# optionally by iteration token
|
121
122
|
events_filter['iteration_token']=iteration_token unless iteration_token.nil?
|
122
|
-
|
123
|
+
begin
|
124
|
+
events=@api_node.read('events',events_filter)[:data]
|
125
|
+
rescue RestCallError => e
|
126
|
+
if e.message.include?('Invalid iteration_token')
|
127
|
+
Log.log.warn("Retrying without iteration token: #{e}")
|
128
|
+
events_filter.delete('iteration_token')
|
129
|
+
retry
|
130
|
+
end
|
131
|
+
raise e
|
132
|
+
end
|
123
133
|
return if events.empty?
|
124
134
|
events.each do |event|
|
125
135
|
next unless event['data']['direction'].eql?('receive')
|
@@ -137,7 +147,7 @@ module Aspera
|
|
137
147
|
end
|
138
148
|
|
139
149
|
# requests recent events on node api and process newly modified folders
|
140
|
-
def
|
150
|
+
def process_events(iteration_token)
|
141
151
|
# get new file creation by access key (TODO: what if file already existed?)
|
142
152
|
events_filter={
|
143
153
|
'access_key'=>@access_key_self['id'],
|
@@ -276,7 +286,7 @@ module Aspera
|
|
276
286
|
end
|
277
287
|
end
|
278
288
|
# need generator for further checks
|
279
|
-
gen_info[:generator]=Aspera::Preview::Generator.new(@gen_options,gen_info[:src],gen_info[:dst],@tmp_folder,entry['content_type']
|
289
|
+
gen_info[:generator]=Aspera::Preview::Generator.new(@gen_options,gen_info[:src],gen_info[:dst],@tmp_folder,entry['content_type'])
|
280
290
|
# get conversion_type (if known) and check if supported
|
281
291
|
next false unless gen_info[:generator].supported?
|
282
292
|
# shall we skip it ?
|
@@ -313,12 +323,14 @@ module Aspera
|
|
313
323
|
end # generate_preview
|
314
324
|
|
315
325
|
# scan all files in provided folder entry
|
326
|
+
# @param scan_start subpath to start folder scan inside
|
316
327
|
def scan_folder_files(top_entry,scan_start=nil)
|
317
328
|
if !scan_start.nil?
|
318
329
|
# canonical path: start with / and ends with /
|
319
330
|
scan_start='/'+scan_start.split('/').select{|i|!i.empty?}.join('/')
|
320
331
|
scan_start="#{scan_start}/" #unless scan_start.end_with?('/')
|
321
332
|
end
|
333
|
+
filter_block=Aspera::Node.file_matcher(options.get_option(:value,:optional))
|
322
334
|
Log.log.debug("scan: #{top_entry} : #{scan_start}".green)
|
323
335
|
# don't use recursive call, use list instead
|
324
336
|
entries_to_process=[top_entry]
|
@@ -332,34 +344,44 @@ module Aspera
|
|
332
344
|
next
|
333
345
|
end
|
334
346
|
Log.log.debug("item:#{entry}")
|
335
|
-
|
336
|
-
|
337
|
-
|
338
|
-
|
339
|
-
|
340
|
-
|
341
|
-
|
342
|
-
|
343
|
-
|
344
|
-
Log.log.debug(
|
345
|
-
|
346
|
-
|
347
|
-
|
348
|
-
|
349
|
-
#
|
350
|
-
|
351
|
-
|
347
|
+
begin
|
348
|
+
case entry['type']
|
349
|
+
when 'file'
|
350
|
+
if filter_block.call(entry)
|
351
|
+
generate_preview(entry)
|
352
|
+
else
|
353
|
+
Log.log.debug('skip by filter')
|
354
|
+
end
|
355
|
+
when 'link'
|
356
|
+
Log.log.debug('Ignoring link.')
|
357
|
+
when 'folder'
|
358
|
+
if @option_skip_folders.include?(entry['path'])
|
359
|
+
Log.log.debug("#{entry['path']} folder (skip list)".bg_red)
|
360
|
+
else
|
361
|
+
Log.log.debug("#{entry['path']} folder".green)
|
362
|
+
# get folder content
|
363
|
+
folder_entries=get_folder_entries(entry['id'])
|
364
|
+
# process all items in current folder
|
365
|
+
folder_entries.each do |folder_entry|
|
366
|
+
# add path for older versions of ES
|
367
|
+
if !folder_entry.has_key?('path')
|
368
|
+
folder_entry['path']=entry_path_with_slash+folder_entry['name']
|
369
|
+
end
|
370
|
+
folder_entry['parent_file_id']=entry['id']
|
371
|
+
entries_to_process.push(folder_entry)
|
352
372
|
end
|
353
|
-
folder_entry['parent_file_id']=entry['id']
|
354
|
-
entries_to_process.push(folder_entry)
|
355
373
|
end
|
374
|
+
else
|
375
|
+
Log.log.warn("unknown entry type: #{entry['type']}")
|
356
376
|
end
|
357
|
-
|
358
|
-
Log.log.warn("
|
377
|
+
rescue => e
|
378
|
+
Log.log.warn("An error occured: #{e}, ignoring")
|
359
379
|
end
|
360
380
|
end
|
361
381
|
end
|
362
382
|
|
383
|
+
ACTIONS=[:scan,:events,:trevents,:check,:test]
|
384
|
+
|
363
385
|
def execute_action
|
364
386
|
command=self.options.get_next_command(ACTIONS)
|
365
387
|
unless [:check,:test].include?(command)
|
@@ -402,6 +424,7 @@ module Aspera
|
|
402
424
|
end
|
403
425
|
end
|
404
426
|
end
|
427
|
+
Aspera::Preview::FileTypes.instance.use_mimemagic = self.options.get_option(:mimemagic,:mandatory)
|
405
428
|
case command
|
406
429
|
when :scan
|
407
430
|
scan_path=self.options.get_option(:scan_path,:optional)
|
@@ -417,30 +440,20 @@ module Aspera
|
|
417
440
|
end
|
418
441
|
scan_folder_files(folder_info,scan_path)
|
419
442
|
return Main.result_status('scan finished')
|
420
|
-
when :events
|
421
|
-
iteration_data=[]
|
422
|
-
iteration_persistency=nil
|
423
|
-
if self.options.get_option(:once_only,:mandatory)
|
424
|
-
iteration_persistency=PersistencyActionOnce.new(
|
425
|
-
manager: @agents[:persistency],
|
426
|
-
data: iteration_data,
|
427
|
-
ids: ['preview_iteration_events',self.options.get_option(:url,:mandatory),self.options.get_option(:username,:mandatory)])
|
428
|
-
end
|
429
|
-
iteration_data[0]=process_file_events(iteration_data[0])
|
430
|
-
iteration_persistency.save unless iteration_persistency.nil?
|
431
|
-
return Main.result_status('events finished')
|
432
|
-
when :trevents
|
443
|
+
when :events,:trevents
|
433
444
|
iteration_data=[]
|
434
445
|
iteration_persistency=nil
|
435
446
|
if self.options.get_option(:once_only,:mandatory)
|
436
447
|
iteration_persistency=PersistencyActionOnce.new(
|
437
448
|
manager: @agents[:persistency],
|
438
449
|
data: iteration_data,
|
439
|
-
ids:
|
450
|
+
ids: ["preview_iteration_#{command}",self.options.get_option(:url,:mandatory),self.options.get_option(:username,:mandatory)])
|
440
451
|
end
|
441
|
-
|
452
|
+
|
453
|
+
# call method specified
|
454
|
+
iteration_data[0]=send("process_#{command}",iteration_data[0])
|
442
455
|
iteration_persistency.save unless iteration_persistency.nil?
|
443
|
-
return Main.result_status(
|
456
|
+
return Main.result_status("#{command} finished")
|
444
457
|
when :check
|
445
458
|
Aspera::Preview::Utils.check_tools(@skip_types)
|
446
459
|
return Main.result_status('tools validated')
|
@@ -448,8 +461,9 @@ module Aspera
|
|
448
461
|
format = self.options.get_next_argument('format',Aspera::Preview::Generator::PREVIEW_FORMATS)
|
449
462
|
source = self.options.get_next_argument('source file')
|
450
463
|
dest=preview_filename(format,self.options.get_option(:case,:optional))
|
451
|
-
g=Aspera::Preview::Generator.new(@gen_options,source,dest,@tmp_folder)
|
452
|
-
raise "
|
464
|
+
g=Aspera::Preview::Generator.new(@gen_options,source,dest,@tmp_folder,nil)
|
465
|
+
raise "cannot find file type for #{source}" if g.conversion_type.nil?
|
466
|
+
raise "out format #{format} not supported" unless g.supported?
|
453
467
|
g.generate
|
454
468
|
return Main.result_status("generated: #{dest}")
|
455
469
|
else
|
@@ -62,7 +62,7 @@ module Aspera
|
|
62
62
|
end.select{|i|!i.nil?}
|
63
63
|
end
|
64
64
|
|
65
|
-
ACTIONS=[:
|
65
|
+
ACTIONS=[:health,:nodeadmin,:userdata,:configurator,:ctl,:download,:upload,:browse,:delete,:rename].concat(Aspera::AsCmd::OPERATIONS)
|
66
66
|
|
67
67
|
def execute_action
|
68
68
|
server_uri=URI.parse(self.options.get_option(:url,:mandatory))
|
@@ -93,12 +93,14 @@ module Aspera
|
|
93
93
|
ssh_keys=[ssh_keys] if ssh_keys.is_a?(String)
|
94
94
|
ssh_keys.map!{|p|File.expand_path(p)}
|
95
95
|
Log.log.debug("ssh keys=#{ssh_keys}")
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
96
|
+
if !ssh_keys.empty?
|
97
|
+
ssh_options[:keys]=ssh_keys
|
98
|
+
server_transfer_spec['EX_ssh_key_paths']=ssh_keys
|
99
|
+
ssh_keys.each do |k|
|
100
|
+
Log.log.warn("no such key file: #{k}") unless File.exist?(k)
|
101
|
+
end
|
102
|
+
cred_set=true
|
100
103
|
end
|
101
|
-
cred_set=true
|
102
104
|
end
|
103
105
|
raise 'either password or key must be provided' if !cred_set
|
104
106
|
shell_executor=Ssh.new(server_transfer_spec['remote_host'],server_transfer_spec['remote_user'],ssh_options)
|
@@ -114,9 +116,9 @@ module Aspera
|
|
114
116
|
command=:rm if command.eql?(:delete)
|
115
117
|
command=:mv if command.eql?(:rename)
|
116
118
|
case command
|
117
|
-
when :
|
119
|
+
when :health
|
118
120
|
nagios=Nagios.new
|
119
|
-
command_nagios=self.options.get_next_command([ :app_services, :transfer ])
|
121
|
+
command_nagios=self.options.get_next_command([ :app_services, :transfer, :asctlstatus ])
|
120
122
|
case command_nagios
|
121
123
|
when :app_services
|
122
124
|
# will not work with aspshell, requires Linux/bash
|
@@ -145,6 +147,19 @@ module Aspera
|
|
145
147
|
else
|
146
148
|
nagios.add_critical('transfer',statuses.select{|i|!i.eql?(:success)}.first.to_s)
|
147
149
|
end
|
150
|
+
when :asctlstatus
|
151
|
+
realcmd='asctl'
|
152
|
+
prefix=self.options.get_option(:cmd_prefix,:optional)
|
153
|
+
realcmd="#{prefix}#{realcmd} all:status" unless prefix.nil?
|
154
|
+
result=shell_executor.execute(realcmd.split(' '))
|
155
|
+
data=asctl_parse(result)
|
156
|
+
data.each do |i|
|
157
|
+
if i['state'].eql?('running')
|
158
|
+
nagios.add_ok(i['process'],i['state'])
|
159
|
+
else
|
160
|
+
nagios.add_critical(i['process'],i['state'])
|
161
|
+
end
|
162
|
+
end
|
148
163
|
else raise "ERROR"
|
149
164
|
end
|
150
165
|
return nagios.result
|
@@ -10,41 +10,56 @@ module Aspera
|
|
10
10
|
module Cli
|
11
11
|
# The Transfer agent is a common interface to start a transfer using
|
12
12
|
# one of the supported transfer agents
|
13
|
-
# provides CLI options to select one of the transfer agents (
|
13
|
+
# provides CLI options to select one of the transfer agents (FASP/ascp client)
|
14
14
|
class TransferAgent
|
15
15
|
# special value for --sources : read file list from arguments
|
16
16
|
FILE_LIST_FROM_ARGS='@args'
|
17
17
|
# special value for --sources : read file list from transfer spec (--ts)
|
18
18
|
FILE_LIST_FROM_TRANSFER_SPEC='@ts'
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
19
|
+
DEFAULT_TRANSFER_NOTIF_TMPL=<<END_OF_TEMPLATE
|
20
|
+
From: <%=from_name%> <<%=from_email%>>
|
21
|
+
To: <<%=to%>>
|
22
|
+
Subject: <%=subject%>
|
23
|
+
|
24
|
+
Transfer is: <%=global_transfer_status%>
|
25
|
+
|
26
|
+
<%=ts.to_yaml%>
|
27
|
+
END_OF_TEMPLATE
|
28
|
+
#%
|
29
|
+
private_constant :FILE_LIST_FROM_ARGS,:FILE_LIST_FROM_TRANSFER_SPEC,:DEFAULT_TRANSFER_NOTIF_TMPL
|
30
|
+
# @param env external objects: option manager, config file manager
|
31
|
+
def initialize(env)
|
32
|
+
# same as plugin environment
|
33
|
+
@env=env
|
34
|
+
# command line can override transfer spec
|
35
|
+
@transfer_spec_cmdline={'create_dir'=>true}
|
26
36
|
# the currently selected transfer agent
|
27
37
|
@agent=nil
|
28
38
|
@progress_listener=Listener::ProgressMulti.new
|
29
39
|
# source/destination pair, like "paths" of transfer spec
|
30
40
|
@transfer_paths=nil
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
41
|
+
options.set_obj_attr(:ts,self,:option_transfer_spec)
|
42
|
+
options.add_opt_simple(:ts,"override transfer spec values (Hash, use @json: prefix), current=#{options.get_option(:ts,:optional)}")
|
43
|
+
options.add_opt_simple(:local_resume,"set resume policy (Hash, use @json: prefix), current=#{options.get_option(:local_resume,:optional)}")
|
44
|
+
options.add_opt_simple(:to_folder,"destination folder for downloaded files")
|
45
|
+
options.add_opt_simple(:sources,"list of source files (see doc)")
|
46
|
+
options.add_opt_simple(:transfer_info,"additional information for transfer client")
|
47
|
+
options.add_opt_list(:src_type,[:list,:pair],"type of file list")
|
48
|
+
options.add_opt_list(:transfer,[:direct,:httpgw,:connect,:node,:aoc],"type of transfer")
|
49
|
+
options.add_opt_list(:progress,[:none,:native,:multi],"type of progress bar")
|
50
|
+
options.set_option(:transfer,:direct)
|
51
|
+
options.set_option(:src_type,:list)
|
52
|
+
options.set_option(:progress,:native) # use native ascp progress bar as it is more reliable
|
53
|
+
options.parse_options!
|
44
54
|
end
|
45
55
|
|
56
|
+
def options; @env[:options];end
|
57
|
+
|
58
|
+
def config; @env[:config];end
|
59
|
+
|
46
60
|
def option_transfer_spec; @transfer_spec_cmdline; end
|
47
61
|
|
62
|
+
# multiple option are merged
|
48
63
|
def option_transfer_spec=(value); @transfer_spec_cmdline.merge!(value); end
|
49
64
|
|
50
65
|
def option_transfer_spec_deep_merge(ts); @transfer_spec_cmdline.deep_merge!(ts); end
|
@@ -53,8 +68,8 @@ module Aspera
|
|
53
68
|
@agent=instance
|
54
69
|
@agent.add_listener(Listener::Logger.new)
|
55
70
|
# use local progress bar if asked so, or if native and non local ascp (because only local ascp has native progress bar)
|
56
|
-
if
|
57
|
-
(
|
71
|
+
if options.get_option(:progress,:mandatory).eql?(:multi) or
|
72
|
+
(options.get_option(:progress,:mandatory).eql?(:native) and !options.get_option(:transfer,:mandatory).eql?(:direct))
|
58
73
|
@agent.add_listener(@progress_listener)
|
59
74
|
end
|
60
75
|
end
|
@@ -62,28 +77,28 @@ module Aspera
|
|
62
77
|
# analyze options and create new agent if not already created or set
|
63
78
|
def set_agent_by_options
|
64
79
|
return nil unless @agent.nil?
|
65
|
-
agent_type
|
80
|
+
agent_type=options.get_option(:transfer,:mandatory)
|
66
81
|
case agent_type
|
67
82
|
when :direct
|
68
|
-
agent_options
|
83
|
+
agent_options=options.get_option(:transfer_info,:optional)
|
69
84
|
agent_options=agent_options.symbolize_keys if agent_options.is_a?(Hash)
|
70
85
|
new_agent=Fasp::Local.new(agent_options)
|
71
|
-
new_agent.quiet=false if
|
86
|
+
new_agent.quiet=false if options.get_option(:progress,:mandatory).eql?(:native)
|
72
87
|
when :httpgw
|
73
|
-
httpgw_config
|
88
|
+
httpgw_config=options.get_option(:transfer_info,:mandatory)
|
74
89
|
new_agent=Fasp::HttpGW.new(httpgw_config)
|
75
90
|
when :connect
|
76
91
|
new_agent=Fasp::Connect.new
|
77
92
|
when :node
|
78
|
-
# way for code to setup alternate node api in
|
93
|
+
# way for code to setup alternate node api in advance
|
79
94
|
# support: @preset:<name>
|
80
95
|
# support extended values
|
81
|
-
node_config
|
96
|
+
node_config=options.get_option(:transfer_info,:optional)
|
82
97
|
# if not specified: use default node
|
83
98
|
if node_config.nil?
|
84
|
-
param_set_name
|
99
|
+
param_set_name=config.get_plugin_default_config_name(:node)
|
85
100
|
raise CliBadArgument,"No default node configured, Please specify --#{:transfer_info.to_s.gsub('_','-')}" if param_set_name.nil?
|
86
|
-
node_config
|
101
|
+
node_config=config.preset_by_name(param_set_name)
|
87
102
|
end
|
88
103
|
Log.log.debug("node=#{node_config}")
|
89
104
|
raise CliBadArgument,"the node configuration shall be Hash, not #{node_config.class} (#{node_config}), use either @json:<json> or @preset:<parameter set name>" if !node_config.is_a?(Hash)
|
@@ -102,11 +117,11 @@ module Aspera
|
|
102
117
|
}})
|
103
118
|
new_agent=Fasp::Node.new(node_api)
|
104
119
|
when :aoc
|
105
|
-
aoc_config
|
120
|
+
aoc_config=options.get_option(:transfer_info,:optional)
|
106
121
|
if aoc_config.nil?
|
107
|
-
param_set_name
|
122
|
+
param_set_name=config.get_plugin_default_config_name(:aspera)
|
108
123
|
raise CliBadArgument,"No default AoC configured, Please specify --#{:transfer_info.to_s.gsub('_','-')}" if param_set_name.nil?
|
109
|
-
aoc_config
|
124
|
+
aoc_config=config.preset_by_name(param_set_name)
|
110
125
|
end
|
111
126
|
Log.log.debug("aoc=#{aoc_config}")
|
112
127
|
raise CliBadArgument,"the aoc configuration shall be Hash, not #{aoc_config.class} (#{aoc_config}), refer to manual" if !aoc_config.is_a?(Hash)
|
@@ -118,7 +133,7 @@ module Aspera
|
|
118
133
|
aoc_config[:private_key]=ExtendedValue.instance.evaluate(aoc_config[:private_key])
|
119
134
|
new_agent=Fasp::Aoc.new(aoc_config)
|
120
135
|
else
|
121
|
-
raise "
|
136
|
+
raise "Unexpected transfer agent type: #{agent_type}"
|
122
137
|
end
|
123
138
|
set_agent_instance(new_agent)
|
124
139
|
return nil
|
@@ -128,7 +143,7 @@ module Aspera
|
|
128
143
|
# sets default if needed
|
129
144
|
# param: 'send' or 'receive'
|
130
145
|
def destination_folder(direction)
|
131
|
-
dest_folder
|
146
|
+
dest_folder=options.get_option(:to_folder,:optional)
|
132
147
|
return dest_folder unless dest_folder.nil?
|
133
148
|
dest_folder=@transfer_spec_cmdline['destination_root']
|
134
149
|
return dest_folder unless dest_folder.nil?
|
@@ -151,12 +166,12 @@ module Aspera
|
|
151
166
|
# start with lower priority : get paths from transfer spec on command line
|
152
167
|
@transfer_paths=@transfer_spec_cmdline['paths'] if @transfer_spec_cmdline.has_key?('paths')
|
153
168
|
# is there a source list option ?
|
154
|
-
file_list
|
169
|
+
file_list=options.get_option(:sources,:optional)
|
155
170
|
case file_list
|
156
171
|
when nil,FILE_LIST_FROM_ARGS
|
157
172
|
Log.log.debug("getting file list as parameters")
|
158
173
|
# get remaining arguments
|
159
|
-
file_list
|
174
|
+
file_list=options.get_next_argument("source file list",:multiple)
|
160
175
|
raise CliBadArgument,"specify at least one file on command line or use --sources=#{FILE_LIST_FROM_TRANSFER_SPEC} to use transfer spec" if !file_list.is_a?(Array) or file_list.empty?
|
161
176
|
when FILE_LIST_FROM_TRANSFER_SPEC
|
162
177
|
Log.log.debug("assume list provided in transfer spec")
|
@@ -173,14 +188,14 @@ module Aspera
|
|
173
188
|
if !@transfer_paths.nil?
|
174
189
|
Log.log.warn("--sources overrides paths from --ts")
|
175
190
|
end
|
176
|
-
case
|
191
|
+
case options.get_option(:src_type,:mandatory)
|
177
192
|
when :list
|
178
193
|
# when providing a list, just specify source
|
179
194
|
@transfer_paths=file_list.map{|i|{'source'=>i}}
|
180
195
|
when :pair
|
181
|
-
raise CliBadArgument,"
|
196
|
+
raise CliBadArgument,"When using pair, provide an even number of paths: #{file_list.length}" unless file_list.length.even?
|
182
197
|
@transfer_paths=file_list.each_slice(2).to_a.map{|s,d|{'source'=>s,'destination'=>d}}
|
183
|
-
else raise "
|
198
|
+
else raise "Unsupported src_type"
|
184
199
|
end
|
185
200
|
Log.log.debug("paths=#{@transfer_paths}")
|
186
201
|
return @transfer_paths
|
@@ -188,20 +203,20 @@ module Aspera
|
|
188
203
|
|
189
204
|
# start a transfer and wait for completion, plugins shall use this method
|
190
205
|
# @param transfer_spec
|
191
|
-
# @param
|
192
|
-
#
|
206
|
+
# @param tr_opts specific options for the transfer_agent
|
207
|
+
# tr_opts[:src] specifies how destination_root is set (how transfer spec was generated)
|
193
208
|
# other options are carried to specific agent
|
194
|
-
def start(transfer_spec,
|
209
|
+
def start(transfer_spec,tr_opts)
|
195
210
|
# check parameters
|
196
211
|
raise "transfer_spec must be hash" unless transfer_spec.is_a?(Hash)
|
197
|
-
raise "
|
212
|
+
raise "tr_opts must be hash" unless tr_opts.is_a?(Hash)
|
198
213
|
# process :src option
|
199
214
|
case transfer_spec['direction']
|
200
215
|
when 'receive'
|
201
216
|
# init default if required in any case
|
202
217
|
@transfer_spec_cmdline['destination_root']||=destination_folder(transfer_spec['direction'])
|
203
218
|
when 'send'
|
204
|
-
case
|
219
|
+
case tr_opts[:src]
|
205
220
|
when :direct
|
206
221
|
# init default if required
|
207
222
|
@transfer_spec_cmdline['destination_root']||=destination_folder(transfer_spec['direction'])
|
@@ -212,12 +227,12 @@ module Aspera
|
|
212
227
|
when :node_gen4
|
213
228
|
@transfer_spec_cmdline.delete('destination_root') if @transfer_spec_cmdline.has_key?('destination_root_id')
|
214
229
|
else
|
215
|
-
raise StandardError,"InternalError: unsupported value: #{
|
230
|
+
raise StandardError,"InternalError: unsupported value: #{tr_opts[:src]}"
|
216
231
|
end
|
217
232
|
end
|
218
233
|
|
219
234
|
# only used here
|
220
|
-
|
235
|
+
tr_opts.delete(:src)
|
221
236
|
|
222
237
|
# update command line paths, unless destination already has one
|
223
238
|
@transfer_spec_cmdline['paths']=transfer_spec['paths'] || ts_source_paths
|
@@ -226,13 +241,26 @@ module Aspera
|
|
226
241
|
# create transfer agent
|
227
242
|
self.set_agent_by_options
|
228
243
|
Log.log.debug("transfer agent is a #{@agent.class}")
|
229
|
-
@agent.start_transfer(transfer_spec,
|
244
|
+
@agent.start_transfer(transfer_spec,tr_opts)
|
230
245
|
result=@agent.wait_for_transfers_completion
|
231
246
|
@progress_listener.reset
|
232
247
|
Fasp::Manager.validate_status_list(result)
|
248
|
+
send_email_transfer_notification(transfer_spec,result)
|
233
249
|
return result
|
234
250
|
end
|
235
251
|
|
252
|
+
def send_email_transfer_notification(transfer_spec,statuses)
|
253
|
+
return if options.get_option(:notif_to,:optional).nil?
|
254
|
+
global_status=self.class.session_status(statuses)
|
255
|
+
email_vars={
|
256
|
+
global_transfer_status: global_status,
|
257
|
+
subject: "ascli transfer: #{global_status}",
|
258
|
+
body: "Transfer is: #{global_status}",
|
259
|
+
ts: transfer_spec
|
260
|
+
}
|
261
|
+
@env[:config].send_email_template(email_vars,DEFAULT_TRANSFER_NOTIF_TMPL)
|
262
|
+
end
|
263
|
+
|
236
264
|
# @return :success if all sessions statuses returned by "start" are success
|
237
265
|
# else return the first error exception object
|
238
266
|
def self.session_status(statuses)
|
data/lib/aspera/cli/version.rb
CHANGED