aspera-cli 4.0.0 → 4.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +843 -304
- data/bin/dascli +13 -0
- data/docs/Makefile +4 -4
- data/docs/README.erb.md +805 -172
- data/docs/test_env.conf +22 -3
- data/examples/aoc.rb +14 -3
- data/examples/faspex4.rb +89 -0
- data/lib/aspera/aoc.rb +87 -108
- data/lib/aspera/cli/formater.rb +2 -0
- data/lib/aspera/cli/main.rb +89 -49
- data/lib/aspera/cli/plugin.rb +9 -4
- data/lib/aspera/cli/plugins/alee.rb +1 -1
- data/lib/aspera/cli/plugins/aoc.rb +188 -173
- data/lib/aspera/cli/plugins/ats.rb +2 -2
- data/lib/aspera/cli/plugins/config.rb +218 -145
- data/lib/aspera/cli/plugins/console.rb +2 -2
- data/lib/aspera/cli/plugins/faspex.rb +114 -61
- data/lib/aspera/cli/plugins/faspex5.rb +85 -43
- data/lib/aspera/cli/plugins/node.rb +3 -3
- data/lib/aspera/cli/plugins/preview.rb +59 -45
- data/lib/aspera/cli/plugins/server.rb +23 -8
- data/lib/aspera/cli/transfer_agent.rb +77 -49
- data/lib/aspera/cli/version.rb +1 -1
- data/lib/aspera/command_line_builder.rb +49 -31
- data/lib/aspera/cos_node.rb +33 -28
- data/lib/aspera/environment.rb +2 -2
- data/lib/aspera/fasp/connect.rb +28 -21
- data/lib/aspera/fasp/http_gw.rb +140 -28
- data/lib/aspera/fasp/installation.rb +93 -46
- data/lib/aspera/fasp/local.rb +88 -45
- data/lib/aspera/fasp/manager.rb +15 -0
- data/lib/aspera/fasp/node.rb +4 -4
- data/lib/aspera/fasp/parameters.rb +59 -101
- data/lib/aspera/fasp/parameters.yaml +531 -0
- data/lib/aspera/fasp/resume_policy.rb +13 -12
- data/lib/aspera/fasp/uri.rb +1 -1
- data/lib/aspera/log.rb +1 -1
- data/lib/aspera/node.rb +61 -1
- data/lib/aspera/oauth.rb +49 -46
- data/lib/aspera/persistency_folder.rb +9 -4
- data/lib/aspera/preview/file_types.rb +53 -21
- data/lib/aspera/preview/generator.rb +3 -3
- data/lib/aspera/rest.rb +29 -18
- data/lib/aspera/secrets.rb +20 -0
- data/lib/aspera/sync.rb +40 -35
- data/lib/aspera/temp_file_manager.rb +19 -0
- data/lib/aspera/web_auth.rb +105 -0
- metadata +54 -20
- data/docs/transfer_spec.html +0 -99
@@ -4,6 +4,7 @@ require 'aspera/preview/options'
|
|
4
4
|
require 'aspera/preview/utils'
|
5
5
|
require 'aspera/preview/file_types'
|
6
6
|
require 'aspera/persistency_action_once'
|
7
|
+
require 'aspera/node'
|
7
8
|
require 'aspera/hash_ext'
|
8
9
|
require 'date'
|
9
10
|
require 'securerandom'
|
@@ -56,9 +57,11 @@ module Aspera
|
|
56
57
|
self.options.add_opt_simple(:case,'basename of output for for test')
|
57
58
|
self.options.add_opt_simple(:scan_path,'subpath in folder id to start scan in (default=/)')
|
58
59
|
self.options.add_opt_simple(:scan_id,'forder id in storage to start scan in, default is access key main folder id')
|
60
|
+
self.options.add_opt_boolean(:mimemagic,'use Mime type detection of gem mimemagic')
|
59
61
|
self.options.add_opt_list(:overwrite,[:always,:never,:mtime],'when to overwrite result file')
|
60
62
|
self.options.add_opt_list(:file_access,[:local,:remote],'how to read and write files in repository')
|
61
63
|
self.options.set_option(:temp_folder,Dir.tmpdir)
|
64
|
+
self.options.set_option(:mimemagic,:false)
|
62
65
|
|
63
66
|
# add other options for generator (and set default values)
|
64
67
|
Aspera::Preview::Options::DESCRIPTIONS.each do |opt|
|
@@ -100,8 +103,6 @@ module Aspera
|
|
100
103
|
return @preview_formats_to_generate.map{|i|i.to_s}.join(',')
|
101
104
|
end
|
102
105
|
|
103
|
-
ACTIONS=[:scan,:events,:trevents,:check,:test]
|
104
|
-
|
105
106
|
# /files/id/files is normally cached in redis, but we can discard the cache
|
106
107
|
# but /files/id is not cached
|
107
108
|
def get_folder_entries(file_id,request_args=nil)
|
@@ -112,14 +113,23 @@ module Aspera
|
|
112
113
|
end
|
113
114
|
|
114
115
|
# old version based on folders
|
115
|
-
def
|
116
|
+
def process_trevents(iteration_token)
|
116
117
|
events_filter={
|
117
118
|
'access_key'=>@access_key_self['id'],
|
118
119
|
'type'=>'download.ended'
|
119
120
|
}
|
120
121
|
# optionally by iteration token
|
121
122
|
events_filter['iteration_token']=iteration_token unless iteration_token.nil?
|
122
|
-
|
123
|
+
begin
|
124
|
+
events=@api_node.read('events',events_filter)[:data]
|
125
|
+
rescue RestCallError => e
|
126
|
+
if e.message.include?('Invalid iteration_token')
|
127
|
+
Log.log.warn("Retrying without iteration token: #{e}")
|
128
|
+
events_filter.delete('iteration_token')
|
129
|
+
retry
|
130
|
+
end
|
131
|
+
raise e
|
132
|
+
end
|
123
133
|
return if events.empty?
|
124
134
|
events.each do |event|
|
125
135
|
next unless event['data']['direction'].eql?('receive')
|
@@ -137,7 +147,7 @@ module Aspera
|
|
137
147
|
end
|
138
148
|
|
139
149
|
# requests recent events on node api and process newly modified folders
|
140
|
-
def
|
150
|
+
def process_events(iteration_token)
|
141
151
|
# get new file creation by access key (TODO: what if file already existed?)
|
142
152
|
events_filter={
|
143
153
|
'access_key'=>@access_key_self['id'],
|
@@ -276,7 +286,7 @@ module Aspera
|
|
276
286
|
end
|
277
287
|
end
|
278
288
|
# need generator for further checks
|
279
|
-
gen_info[:generator]=Aspera::Preview::Generator.new(@gen_options,gen_info[:src],gen_info[:dst],@tmp_folder,entry['content_type']
|
289
|
+
gen_info[:generator]=Aspera::Preview::Generator.new(@gen_options,gen_info[:src],gen_info[:dst],@tmp_folder,entry['content_type'])
|
280
290
|
# get conversion_type (if known) and check if supported
|
281
291
|
next false unless gen_info[:generator].supported?
|
282
292
|
# shall we skip it ?
|
@@ -313,12 +323,14 @@ module Aspera
|
|
313
323
|
end # generate_preview
|
314
324
|
|
315
325
|
# scan all files in provided folder entry
|
326
|
+
# @param scan_start subpath to start folder scan inside
|
316
327
|
def scan_folder_files(top_entry,scan_start=nil)
|
317
328
|
if !scan_start.nil?
|
318
329
|
# canonical path: start with / and ends with /
|
319
330
|
scan_start='/'+scan_start.split('/').select{|i|!i.empty?}.join('/')
|
320
331
|
scan_start="#{scan_start}/" #unless scan_start.end_with?('/')
|
321
332
|
end
|
333
|
+
filter_block=Aspera::Node.file_matcher(options.get_option(:value,:optional))
|
322
334
|
Log.log.debug("scan: #{top_entry} : #{scan_start}".green)
|
323
335
|
# don't use recursive call, use list instead
|
324
336
|
entries_to_process=[top_entry]
|
@@ -332,34 +344,44 @@ module Aspera
|
|
332
344
|
next
|
333
345
|
end
|
334
346
|
Log.log.debug("item:#{entry}")
|
335
|
-
|
336
|
-
|
337
|
-
|
338
|
-
|
339
|
-
|
340
|
-
|
341
|
-
|
342
|
-
|
343
|
-
|
344
|
-
Log.log.debug(
|
345
|
-
|
346
|
-
|
347
|
-
|
348
|
-
|
349
|
-
#
|
350
|
-
|
351
|
-
|
347
|
+
begin
|
348
|
+
case entry['type']
|
349
|
+
when 'file'
|
350
|
+
if filter_block.call(entry)
|
351
|
+
generate_preview(entry)
|
352
|
+
else
|
353
|
+
Log.log.debug('skip by filter')
|
354
|
+
end
|
355
|
+
when 'link'
|
356
|
+
Log.log.debug('Ignoring link.')
|
357
|
+
when 'folder'
|
358
|
+
if @option_skip_folders.include?(entry['path'])
|
359
|
+
Log.log.debug("#{entry['path']} folder (skip list)".bg_red)
|
360
|
+
else
|
361
|
+
Log.log.debug("#{entry['path']} folder".green)
|
362
|
+
# get folder content
|
363
|
+
folder_entries=get_folder_entries(entry['id'])
|
364
|
+
# process all items in current folder
|
365
|
+
folder_entries.each do |folder_entry|
|
366
|
+
# add path for older versions of ES
|
367
|
+
if !folder_entry.has_key?('path')
|
368
|
+
folder_entry['path']=entry_path_with_slash+folder_entry['name']
|
369
|
+
end
|
370
|
+
folder_entry['parent_file_id']=entry['id']
|
371
|
+
entries_to_process.push(folder_entry)
|
352
372
|
end
|
353
|
-
folder_entry['parent_file_id']=entry['id']
|
354
|
-
entries_to_process.push(folder_entry)
|
355
373
|
end
|
374
|
+
else
|
375
|
+
Log.log.warn("unknown entry type: #{entry['type']}")
|
356
376
|
end
|
357
|
-
|
358
|
-
Log.log.warn("
|
377
|
+
rescue => e
|
378
|
+
Log.log.warn("An error occured: #{e}, ignoring")
|
359
379
|
end
|
360
380
|
end
|
361
381
|
end
|
362
382
|
|
383
|
+
ACTIONS=[:scan,:events,:trevents,:check,:test]
|
384
|
+
|
363
385
|
def execute_action
|
364
386
|
command=self.options.get_next_command(ACTIONS)
|
365
387
|
unless [:check,:test].include?(command)
|
@@ -402,6 +424,7 @@ module Aspera
|
|
402
424
|
end
|
403
425
|
end
|
404
426
|
end
|
427
|
+
Aspera::Preview::FileTypes.instance.use_mimemagic = self.options.get_option(:mimemagic,:mandatory)
|
405
428
|
case command
|
406
429
|
when :scan
|
407
430
|
scan_path=self.options.get_option(:scan_path,:optional)
|
@@ -417,30 +440,20 @@ module Aspera
|
|
417
440
|
end
|
418
441
|
scan_folder_files(folder_info,scan_path)
|
419
442
|
return Main.result_status('scan finished')
|
420
|
-
when :events
|
421
|
-
iteration_data=[]
|
422
|
-
iteration_persistency=nil
|
423
|
-
if self.options.get_option(:once_only,:mandatory)
|
424
|
-
iteration_persistency=PersistencyActionOnce.new(
|
425
|
-
manager: @agents[:persistency],
|
426
|
-
data: iteration_data,
|
427
|
-
ids: ['preview_iteration_events',self.options.get_option(:url,:mandatory),self.options.get_option(:username,:mandatory)])
|
428
|
-
end
|
429
|
-
iteration_data[0]=process_file_events(iteration_data[0])
|
430
|
-
iteration_persistency.save unless iteration_persistency.nil?
|
431
|
-
return Main.result_status('events finished')
|
432
|
-
when :trevents
|
443
|
+
when :events,:trevents
|
433
444
|
iteration_data=[]
|
434
445
|
iteration_persistency=nil
|
435
446
|
if self.options.get_option(:once_only,:mandatory)
|
436
447
|
iteration_persistency=PersistencyActionOnce.new(
|
437
448
|
manager: @agents[:persistency],
|
438
449
|
data: iteration_data,
|
439
|
-
ids:
|
450
|
+
ids: ["preview_iteration_#{command}",self.options.get_option(:url,:mandatory),self.options.get_option(:username,:mandatory)])
|
440
451
|
end
|
441
|
-
|
452
|
+
|
453
|
+
# call method specified
|
454
|
+
iteration_data[0]=send("process_#{command}",iteration_data[0])
|
442
455
|
iteration_persistency.save unless iteration_persistency.nil?
|
443
|
-
return Main.result_status(
|
456
|
+
return Main.result_status("#{command} finished")
|
444
457
|
when :check
|
445
458
|
Aspera::Preview::Utils.check_tools(@skip_types)
|
446
459
|
return Main.result_status('tools validated')
|
@@ -448,8 +461,9 @@ module Aspera
|
|
448
461
|
format = self.options.get_next_argument('format',Aspera::Preview::Generator::PREVIEW_FORMATS)
|
449
462
|
source = self.options.get_next_argument('source file')
|
450
463
|
dest=preview_filename(format,self.options.get_option(:case,:optional))
|
451
|
-
g=Aspera::Preview::Generator.new(@gen_options,source,dest,@tmp_folder)
|
452
|
-
raise "
|
464
|
+
g=Aspera::Preview::Generator.new(@gen_options,source,dest,@tmp_folder,nil)
|
465
|
+
raise "cannot find file type for #{source}" if g.conversion_type.nil?
|
466
|
+
raise "out format #{format} not supported" unless g.supported?
|
453
467
|
g.generate
|
454
468
|
return Main.result_status("generated: #{dest}")
|
455
469
|
else
|
@@ -62,7 +62,7 @@ module Aspera
|
|
62
62
|
end.select{|i|!i.nil?}
|
63
63
|
end
|
64
64
|
|
65
|
-
ACTIONS=[:
|
65
|
+
ACTIONS=[:health,:nodeadmin,:userdata,:configurator,:ctl,:download,:upload,:browse,:delete,:rename].concat(Aspera::AsCmd::OPERATIONS)
|
66
66
|
|
67
67
|
def execute_action
|
68
68
|
server_uri=URI.parse(self.options.get_option(:url,:mandatory))
|
@@ -93,12 +93,14 @@ module Aspera
|
|
93
93
|
ssh_keys=[ssh_keys] if ssh_keys.is_a?(String)
|
94
94
|
ssh_keys.map!{|p|File.expand_path(p)}
|
95
95
|
Log.log.debug("ssh keys=#{ssh_keys}")
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
96
|
+
if !ssh_keys.empty?
|
97
|
+
ssh_options[:keys]=ssh_keys
|
98
|
+
server_transfer_spec['EX_ssh_key_paths']=ssh_keys
|
99
|
+
ssh_keys.each do |k|
|
100
|
+
Log.log.warn("no such key file: #{k}") unless File.exist?(k)
|
101
|
+
end
|
102
|
+
cred_set=true
|
100
103
|
end
|
101
|
-
cred_set=true
|
102
104
|
end
|
103
105
|
raise 'either password or key must be provided' if !cred_set
|
104
106
|
shell_executor=Ssh.new(server_transfer_spec['remote_host'],server_transfer_spec['remote_user'],ssh_options)
|
@@ -114,9 +116,9 @@ module Aspera
|
|
114
116
|
command=:rm if command.eql?(:delete)
|
115
117
|
command=:mv if command.eql?(:rename)
|
116
118
|
case command
|
117
|
-
when :
|
119
|
+
when :health
|
118
120
|
nagios=Nagios.new
|
119
|
-
command_nagios=self.options.get_next_command([ :app_services, :transfer ])
|
121
|
+
command_nagios=self.options.get_next_command([ :app_services, :transfer, :asctlstatus ])
|
120
122
|
case command_nagios
|
121
123
|
when :app_services
|
122
124
|
# will not work with aspshell, requires Linux/bash
|
@@ -145,6 +147,19 @@ module Aspera
|
|
145
147
|
else
|
146
148
|
nagios.add_critical('transfer',statuses.select{|i|!i.eql?(:success)}.first.to_s)
|
147
149
|
end
|
150
|
+
when :asctlstatus
|
151
|
+
realcmd='asctl'
|
152
|
+
prefix=self.options.get_option(:cmd_prefix,:optional)
|
153
|
+
realcmd="#{prefix}#{realcmd} all:status" unless prefix.nil?
|
154
|
+
result=shell_executor.execute(realcmd.split(' '))
|
155
|
+
data=asctl_parse(result)
|
156
|
+
data.each do |i|
|
157
|
+
if i['state'].eql?('running')
|
158
|
+
nagios.add_ok(i['process'],i['state'])
|
159
|
+
else
|
160
|
+
nagios.add_critical(i['process'],i['state'])
|
161
|
+
end
|
162
|
+
end
|
148
163
|
else raise "ERROR"
|
149
164
|
end
|
150
165
|
return nagios.result
|
@@ -10,41 +10,56 @@ module Aspera
|
|
10
10
|
module Cli
|
11
11
|
# The Transfer agent is a common interface to start a transfer using
|
12
12
|
# one of the supported transfer agents
|
13
|
-
# provides CLI options to select one of the transfer agents (
|
13
|
+
# provides CLI options to select one of the transfer agents (FASP/ascp client)
|
14
14
|
class TransferAgent
|
15
15
|
# special value for --sources : read file list from arguments
|
16
16
|
FILE_LIST_FROM_ARGS='@args'
|
17
17
|
# special value for --sources : read file list from transfer spec (--ts)
|
18
18
|
FILE_LIST_FROM_TRANSFER_SPEC='@ts'
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
19
|
+
DEFAULT_TRANSFER_NOTIF_TMPL=<<END_OF_TEMPLATE
|
20
|
+
From: <%=from_name%> <<%=from_email%>>
|
21
|
+
To: <<%=to%>>
|
22
|
+
Subject: <%=subject%>
|
23
|
+
|
24
|
+
Transfer is: <%=global_transfer_status%>
|
25
|
+
|
26
|
+
<%=ts.to_yaml%>
|
27
|
+
END_OF_TEMPLATE
|
28
|
+
#%
|
29
|
+
private_constant :FILE_LIST_FROM_ARGS,:FILE_LIST_FROM_TRANSFER_SPEC,:DEFAULT_TRANSFER_NOTIF_TMPL
|
30
|
+
# @param env external objects: option manager, config file manager
|
31
|
+
def initialize(env)
|
32
|
+
# same as plugin environment
|
33
|
+
@env=env
|
34
|
+
# command line can override transfer spec
|
35
|
+
@transfer_spec_cmdline={'create_dir'=>true}
|
26
36
|
# the currently selected transfer agent
|
27
37
|
@agent=nil
|
28
38
|
@progress_listener=Listener::ProgressMulti.new
|
29
39
|
# source/destination pair, like "paths" of transfer spec
|
30
40
|
@transfer_paths=nil
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
41
|
+
options.set_obj_attr(:ts,self,:option_transfer_spec)
|
42
|
+
options.add_opt_simple(:ts,"override transfer spec values (Hash, use @json: prefix), current=#{options.get_option(:ts,:optional)}")
|
43
|
+
options.add_opt_simple(:local_resume,"set resume policy (Hash, use @json: prefix), current=#{options.get_option(:local_resume,:optional)}")
|
44
|
+
options.add_opt_simple(:to_folder,"destination folder for downloaded files")
|
45
|
+
options.add_opt_simple(:sources,"list of source files (see doc)")
|
46
|
+
options.add_opt_simple(:transfer_info,"additional information for transfer client")
|
47
|
+
options.add_opt_list(:src_type,[:list,:pair],"type of file list")
|
48
|
+
options.add_opt_list(:transfer,[:direct,:httpgw,:connect,:node,:aoc],"type of transfer")
|
49
|
+
options.add_opt_list(:progress,[:none,:native,:multi],"type of progress bar")
|
50
|
+
options.set_option(:transfer,:direct)
|
51
|
+
options.set_option(:src_type,:list)
|
52
|
+
options.set_option(:progress,:native) # use native ascp progress bar as it is more reliable
|
53
|
+
options.parse_options!
|
44
54
|
end
|
45
55
|
|
56
|
+
def options; @env[:options];end
|
57
|
+
|
58
|
+
def config; @env[:config];end
|
59
|
+
|
46
60
|
def option_transfer_spec; @transfer_spec_cmdline; end
|
47
61
|
|
62
|
+
# multiple option are merged
|
48
63
|
def option_transfer_spec=(value); @transfer_spec_cmdline.merge!(value); end
|
49
64
|
|
50
65
|
def option_transfer_spec_deep_merge(ts); @transfer_spec_cmdline.deep_merge!(ts); end
|
@@ -53,8 +68,8 @@ module Aspera
|
|
53
68
|
@agent=instance
|
54
69
|
@agent.add_listener(Listener::Logger.new)
|
55
70
|
# use local progress bar if asked so, or if native and non local ascp (because only local ascp has native progress bar)
|
56
|
-
if
|
57
|
-
(
|
71
|
+
if options.get_option(:progress,:mandatory).eql?(:multi) or
|
72
|
+
(options.get_option(:progress,:mandatory).eql?(:native) and !options.get_option(:transfer,:mandatory).eql?(:direct))
|
58
73
|
@agent.add_listener(@progress_listener)
|
59
74
|
end
|
60
75
|
end
|
@@ -62,28 +77,28 @@ module Aspera
|
|
62
77
|
# analyze options and create new agent if not already created or set
|
63
78
|
def set_agent_by_options
|
64
79
|
return nil unless @agent.nil?
|
65
|
-
agent_type
|
80
|
+
agent_type=options.get_option(:transfer,:mandatory)
|
66
81
|
case agent_type
|
67
82
|
when :direct
|
68
|
-
agent_options
|
83
|
+
agent_options=options.get_option(:transfer_info,:optional)
|
69
84
|
agent_options=agent_options.symbolize_keys if agent_options.is_a?(Hash)
|
70
85
|
new_agent=Fasp::Local.new(agent_options)
|
71
|
-
new_agent.quiet=false if
|
86
|
+
new_agent.quiet=false if options.get_option(:progress,:mandatory).eql?(:native)
|
72
87
|
when :httpgw
|
73
|
-
httpgw_config
|
88
|
+
httpgw_config=options.get_option(:transfer_info,:mandatory)
|
74
89
|
new_agent=Fasp::HttpGW.new(httpgw_config)
|
75
90
|
when :connect
|
76
91
|
new_agent=Fasp::Connect.new
|
77
92
|
when :node
|
78
|
-
# way for code to setup alternate node api in
|
93
|
+
# way for code to setup alternate node api in advance
|
79
94
|
# support: @preset:<name>
|
80
95
|
# support extended values
|
81
|
-
node_config
|
96
|
+
node_config=options.get_option(:transfer_info,:optional)
|
82
97
|
# if not specified: use default node
|
83
98
|
if node_config.nil?
|
84
|
-
param_set_name
|
99
|
+
param_set_name=config.get_plugin_default_config_name(:node)
|
85
100
|
raise CliBadArgument,"No default node configured, Please specify --#{:transfer_info.to_s.gsub('_','-')}" if param_set_name.nil?
|
86
|
-
node_config
|
101
|
+
node_config=config.preset_by_name(param_set_name)
|
87
102
|
end
|
88
103
|
Log.log.debug("node=#{node_config}")
|
89
104
|
raise CliBadArgument,"the node configuration shall be Hash, not #{node_config.class} (#{node_config}), use either @json:<json> or @preset:<parameter set name>" if !node_config.is_a?(Hash)
|
@@ -102,11 +117,11 @@ module Aspera
|
|
102
117
|
}})
|
103
118
|
new_agent=Fasp::Node.new(node_api)
|
104
119
|
when :aoc
|
105
|
-
aoc_config
|
120
|
+
aoc_config=options.get_option(:transfer_info,:optional)
|
106
121
|
if aoc_config.nil?
|
107
|
-
param_set_name
|
122
|
+
param_set_name=config.get_plugin_default_config_name(:aspera)
|
108
123
|
raise CliBadArgument,"No default AoC configured, Please specify --#{:transfer_info.to_s.gsub('_','-')}" if param_set_name.nil?
|
109
|
-
aoc_config
|
124
|
+
aoc_config=config.preset_by_name(param_set_name)
|
110
125
|
end
|
111
126
|
Log.log.debug("aoc=#{aoc_config}")
|
112
127
|
raise CliBadArgument,"the aoc configuration shall be Hash, not #{aoc_config.class} (#{aoc_config}), refer to manual" if !aoc_config.is_a?(Hash)
|
@@ -118,7 +133,7 @@ module Aspera
|
|
118
133
|
aoc_config[:private_key]=ExtendedValue.instance.evaluate(aoc_config[:private_key])
|
119
134
|
new_agent=Fasp::Aoc.new(aoc_config)
|
120
135
|
else
|
121
|
-
raise "
|
136
|
+
raise "Unexpected transfer agent type: #{agent_type}"
|
122
137
|
end
|
123
138
|
set_agent_instance(new_agent)
|
124
139
|
return nil
|
@@ -128,7 +143,7 @@ module Aspera
|
|
128
143
|
# sets default if needed
|
129
144
|
# param: 'send' or 'receive'
|
130
145
|
def destination_folder(direction)
|
131
|
-
dest_folder
|
146
|
+
dest_folder=options.get_option(:to_folder,:optional)
|
132
147
|
return dest_folder unless dest_folder.nil?
|
133
148
|
dest_folder=@transfer_spec_cmdline['destination_root']
|
134
149
|
return dest_folder unless dest_folder.nil?
|
@@ -151,12 +166,12 @@ module Aspera
|
|
151
166
|
# start with lower priority : get paths from transfer spec on command line
|
152
167
|
@transfer_paths=@transfer_spec_cmdline['paths'] if @transfer_spec_cmdline.has_key?('paths')
|
153
168
|
# is there a source list option ?
|
154
|
-
file_list
|
169
|
+
file_list=options.get_option(:sources,:optional)
|
155
170
|
case file_list
|
156
171
|
when nil,FILE_LIST_FROM_ARGS
|
157
172
|
Log.log.debug("getting file list as parameters")
|
158
173
|
# get remaining arguments
|
159
|
-
file_list
|
174
|
+
file_list=options.get_next_argument("source file list",:multiple)
|
160
175
|
raise CliBadArgument,"specify at least one file on command line or use --sources=#{FILE_LIST_FROM_TRANSFER_SPEC} to use transfer spec" if !file_list.is_a?(Array) or file_list.empty?
|
161
176
|
when FILE_LIST_FROM_TRANSFER_SPEC
|
162
177
|
Log.log.debug("assume list provided in transfer spec")
|
@@ -173,14 +188,14 @@ module Aspera
|
|
173
188
|
if !@transfer_paths.nil?
|
174
189
|
Log.log.warn("--sources overrides paths from --ts")
|
175
190
|
end
|
176
|
-
case
|
191
|
+
case options.get_option(:src_type,:mandatory)
|
177
192
|
when :list
|
178
193
|
# when providing a list, just specify source
|
179
194
|
@transfer_paths=file_list.map{|i|{'source'=>i}}
|
180
195
|
when :pair
|
181
|
-
raise CliBadArgument,"
|
196
|
+
raise CliBadArgument,"When using pair, provide an even number of paths: #{file_list.length}" unless file_list.length.even?
|
182
197
|
@transfer_paths=file_list.each_slice(2).to_a.map{|s,d|{'source'=>s,'destination'=>d}}
|
183
|
-
else raise "
|
198
|
+
else raise "Unsupported src_type"
|
184
199
|
end
|
185
200
|
Log.log.debug("paths=#{@transfer_paths}")
|
186
201
|
return @transfer_paths
|
@@ -188,20 +203,20 @@ module Aspera
|
|
188
203
|
|
189
204
|
# start a transfer and wait for completion, plugins shall use this method
|
190
205
|
# @param transfer_spec
|
191
|
-
# @param
|
192
|
-
#
|
206
|
+
# @param tr_opts specific options for the transfer_agent
|
207
|
+
# tr_opts[:src] specifies how destination_root is set (how transfer spec was generated)
|
193
208
|
# other options are carried to specific agent
|
194
|
-
def start(transfer_spec,
|
209
|
+
def start(transfer_spec,tr_opts)
|
195
210
|
# check parameters
|
196
211
|
raise "transfer_spec must be hash" unless transfer_spec.is_a?(Hash)
|
197
|
-
raise "
|
212
|
+
raise "tr_opts must be hash" unless tr_opts.is_a?(Hash)
|
198
213
|
# process :src option
|
199
214
|
case transfer_spec['direction']
|
200
215
|
when 'receive'
|
201
216
|
# init default if required in any case
|
202
217
|
@transfer_spec_cmdline['destination_root']||=destination_folder(transfer_spec['direction'])
|
203
218
|
when 'send'
|
204
|
-
case
|
219
|
+
case tr_opts[:src]
|
205
220
|
when :direct
|
206
221
|
# init default if required
|
207
222
|
@transfer_spec_cmdline['destination_root']||=destination_folder(transfer_spec['direction'])
|
@@ -212,12 +227,12 @@ module Aspera
|
|
212
227
|
when :node_gen4
|
213
228
|
@transfer_spec_cmdline.delete('destination_root') if @transfer_spec_cmdline.has_key?('destination_root_id')
|
214
229
|
else
|
215
|
-
raise StandardError,"InternalError: unsupported value: #{
|
230
|
+
raise StandardError,"InternalError: unsupported value: #{tr_opts[:src]}"
|
216
231
|
end
|
217
232
|
end
|
218
233
|
|
219
234
|
# only used here
|
220
|
-
|
235
|
+
tr_opts.delete(:src)
|
221
236
|
|
222
237
|
# update command line paths, unless destination already has one
|
223
238
|
@transfer_spec_cmdline['paths']=transfer_spec['paths'] || ts_source_paths
|
@@ -226,13 +241,26 @@ module Aspera
|
|
226
241
|
# create transfer agent
|
227
242
|
self.set_agent_by_options
|
228
243
|
Log.log.debug("transfer agent is a #{@agent.class}")
|
229
|
-
@agent.start_transfer(transfer_spec,
|
244
|
+
@agent.start_transfer(transfer_spec,tr_opts)
|
230
245
|
result=@agent.wait_for_transfers_completion
|
231
246
|
@progress_listener.reset
|
232
247
|
Fasp::Manager.validate_status_list(result)
|
248
|
+
send_email_transfer_notification(transfer_spec,result)
|
233
249
|
return result
|
234
250
|
end
|
235
251
|
|
252
|
+
def send_email_transfer_notification(transfer_spec,statuses)
|
253
|
+
return if options.get_option(:notif_to,:optional).nil?
|
254
|
+
global_status=self.class.session_status(statuses)
|
255
|
+
email_vars={
|
256
|
+
global_transfer_status: global_status,
|
257
|
+
subject: "ascli transfer: #{global_status}",
|
258
|
+
body: "Transfer is: #{global_status}",
|
259
|
+
ts: transfer_spec
|
260
|
+
}
|
261
|
+
@env[:config].send_email_template(email_vars,DEFAULT_TRANSFER_NOTIF_TMPL)
|
262
|
+
end
|
263
|
+
|
236
264
|
# @return :success if all sessions statuses returned by "start" are success
|
237
265
|
# else return the first error exception object
|
238
266
|
def self.session_status(statuses)
|
data/lib/aspera/cli/version.rb
CHANGED