aspera-cli 4.1.0 → 4.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +455 -229
- data/docs/Makefile +4 -4
- data/docs/README.erb.md +457 -126
- data/docs/test_env.conf +19 -2
- data/examples/aoc.rb +14 -3
- data/examples/faspex4.rb +89 -0
- data/lib/aspera/aoc.rb +38 -40
- data/lib/aspera/cli/main.rb +65 -33
- data/lib/aspera/cli/plugins/aoc.rb +54 -65
- data/lib/aspera/cli/plugins/ats.rb +2 -2
- data/lib/aspera/cli/plugins/config.rb +158 -137
- data/lib/aspera/cli/plugins/faspex.rb +111 -64
- data/lib/aspera/cli/plugins/faspex5.rb +35 -48
- data/lib/aspera/cli/plugins/node.rb +3 -2
- data/lib/aspera/cli/plugins/preview.rb +88 -55
- data/lib/aspera/cli/transfer_agent.rb +98 -62
- data/lib/aspera/cli/version.rb +1 -1
- data/lib/aspera/command_line_builder.rb +48 -31
- data/lib/aspera/cos_node.rb +34 -28
- data/lib/aspera/environment.rb +2 -2
- data/lib/aspera/fasp/aoc.rb +1 -1
- data/lib/aspera/fasp/installation.rb +68 -45
- data/lib/aspera/fasp/local.rb +89 -45
- data/lib/aspera/fasp/manager.rb +3 -0
- data/lib/aspera/fasp/node.rb +23 -1
- data/lib/aspera/fasp/parameters.rb +57 -86
- data/lib/aspera/fasp/parameters.yaml +531 -0
- data/lib/aspera/fasp/resume_policy.rb +13 -12
- data/lib/aspera/fasp/uri.rb +1 -1
- data/lib/aspera/id_generator.rb +22 -0
- data/lib/aspera/node.rb +14 -3
- data/lib/aspera/oauth.rb +135 -129
- data/lib/aspera/persistency_action_once.rb +11 -7
- data/lib/aspera/persistency_folder.rb +6 -26
- data/lib/aspera/rest.rb +3 -12
- data/lib/aspera/secrets.rb +20 -0
- data/lib/aspera/sync.rb +40 -35
- data/lib/aspera/timer_limiter.rb +22 -0
- data/lib/aspera/web_auth.rb +105 -0
- metadata +22 -3
- data/docs/transfer_spec.html +0 -99
data/docs/test_env.conf
CHANGED
|
@@ -6,7 +6,7 @@ default:
|
|
|
6
6
|
aoc: tst_aoc1
|
|
7
7
|
faspex: tst_faspex
|
|
8
8
|
faspex5: tst_faspex5
|
|
9
|
-
shares:
|
|
9
|
+
shares: tst_shares_1
|
|
10
10
|
shares2: tst_shares2
|
|
11
11
|
node: tst_node
|
|
12
12
|
server: tst_server
|
|
@@ -47,9 +47,23 @@ tst_node_faspex:
|
|
|
47
47
|
url: your value here
|
|
48
48
|
username: your value here
|
|
49
49
|
password: your value here
|
|
50
|
+
tst_faspex5_boot:
|
|
51
|
+
url: your value here
|
|
52
|
+
auth: your value here
|
|
53
|
+
username: your value here
|
|
54
|
+
password: your value here
|
|
55
|
+
tst_faspex5_web:
|
|
56
|
+
url: your value here
|
|
57
|
+
auth: your value here
|
|
58
|
+
redirect_uri: your value here
|
|
59
|
+
client_id: your value here
|
|
60
|
+
client_secret: your value here
|
|
50
61
|
tst_faspex5:
|
|
51
62
|
url: your value here
|
|
52
63
|
auth: your value here
|
|
64
|
+
client_id: your value here
|
|
65
|
+
client_secret: your value here
|
|
66
|
+
private_key: your value here
|
|
53
67
|
username: your value here
|
|
54
68
|
password: your value here
|
|
55
69
|
tst_shares:
|
|
@@ -111,6 +125,8 @@ misc:
|
|
|
111
125
|
faspex_publink_send_to_fxuser: your value here
|
|
112
126
|
faspex_publink_send_to_dropbox: your value here
|
|
113
127
|
shares_upload: your value here
|
|
128
|
+
console_smart_id: your value here
|
|
129
|
+
console_smart_file: your value here
|
|
114
130
|
orch_workflow_id: your value here
|
|
115
131
|
file_dcm: your value here
|
|
116
132
|
file_pdf: your value here
|
|
@@ -139,5 +155,6 @@ misc:
|
|
|
139
155
|
email_internal: your value here
|
|
140
156
|
email_external: your value here
|
|
141
157
|
aoc_org: your value here
|
|
142
|
-
|
|
158
|
+
aoc_user_email: your value here
|
|
159
|
+
aoc_workspace2: your value here
|
|
143
160
|
http_gw_fqdn_port: your value here
|
data/examples/aoc.rb
CHANGED
|
@@ -4,11 +4,22 @@ require 'aspera/log'
|
|
|
4
4
|
|
|
5
5
|
Aspera::Log.instance.level=:debug
|
|
6
6
|
|
|
7
|
+
if ! ARGV.length.eql?(3)
|
|
8
|
+
Aspera::Log.log.error("wrong number of args: #{ARGV.length}")
|
|
9
|
+
Aspera::Log.log.error("Usage: #{$0} <aoc URL> <aoc username> <aoc private key content>")
|
|
10
|
+
Aspera::Log.log.error("Example: #{$0} https://myorg.ibmaspera.com john@example.com $(cat /home/john/my_key.pem)")
|
|
11
|
+
Process.exit(1)
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
aoc_url=ARGV[0]
|
|
15
|
+
aoc_user=ARGV[1]
|
|
16
|
+
aoc_key_value=ARGV[2]
|
|
17
|
+
|
|
7
18
|
aocapi=Aspera::AoC.new(
|
|
8
|
-
url:
|
|
19
|
+
url: aoc_url,
|
|
9
20
|
auth: :jwt,
|
|
10
|
-
private_key:
|
|
11
|
-
username:
|
|
21
|
+
private_key: aoc_key_value,
|
|
22
|
+
username: aoc_user,
|
|
12
23
|
scope: 'user:all',
|
|
13
24
|
subpath: 'api/v1')
|
|
14
25
|
|
data/examples/faspex4.rb
ADDED
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
#!/usr/bin/env ruby
|
|
2
|
+
# find Faspex API here: https://developer.ibm.com/apis/catalog/?search=faspex
|
|
3
|
+
# this example makes use of class Aspera::Rest for REST calls, alternatively class RestClient of gem rest-client could be used
|
|
4
|
+
# this example makes use of class Aspera::Fasp::Local for transfers, alternatively the official "Transfer SDK" could be used
|
|
5
|
+
# Aspera SDK can be downloaded with: `ascli conf ascp install` , it installs in $HOME/.aspera/ascli/sdk
|
|
6
|
+
require 'aspera/rest'
|
|
7
|
+
require 'aspera/log'
|
|
8
|
+
require 'aspera/fasp/local'
|
|
9
|
+
|
|
10
|
+
tmpdir=ENV['tmp']||Dir.tmpdir || '.'
|
|
11
|
+
|
|
12
|
+
# Set high log level for the example, decrease to :warn usually
|
|
13
|
+
Aspera::Log.instance.level=:debug
|
|
14
|
+
|
|
15
|
+
# Set folder where SDK is installed (mandatory)
|
|
16
|
+
# (if ascp is not there, the lib will try to find in usual locations)
|
|
17
|
+
# (if data files are not there, they will be created)
|
|
18
|
+
Aspera::Fasp::Installation.instance.folder = tmpdir
|
|
19
|
+
|
|
20
|
+
if ! ARGV.length.eql?(3)
|
|
21
|
+
Aspera::Log.log.error("Wrong number of args: #{ARGV.length}")
|
|
22
|
+
Aspera::Log.log.error("Usage: #{$0} <faspex URL> <faspex username> <faspex password>")
|
|
23
|
+
Aspera::Log.log.error("Example: #{$0} https://faspex.com/aspera/faspex john p@sSw0rd")
|
|
24
|
+
Process.exit(1)
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
faspex_url=ARGV[0] # typically: https://faspex.example.com/aspera/faspex
|
|
28
|
+
faspex_user=ARGV[1]
|
|
29
|
+
faspex_pass=ARGV[2]
|
|
30
|
+
|
|
31
|
+
# comment out this if certificate is valid, keep line to ignore certificate
|
|
32
|
+
Aspera::Rest.insecure=true
|
|
33
|
+
|
|
34
|
+
# 1: Faspex 4 API v3
|
|
35
|
+
#---------------
|
|
36
|
+
|
|
37
|
+
# create REST API object
|
|
38
|
+
api_v3=Aspera::Rest.new({
|
|
39
|
+
:base_url => faspex_url,
|
|
40
|
+
:auth => {
|
|
41
|
+
:type => :basic,
|
|
42
|
+
:username => faspex_user,
|
|
43
|
+
:password => faspex_pass
|
|
44
|
+
}})
|
|
45
|
+
|
|
46
|
+
# very simple api call
|
|
47
|
+
api_v3.read('me')
|
|
48
|
+
|
|
49
|
+
# 2: send a package
|
|
50
|
+
#---------------
|
|
51
|
+
|
|
52
|
+
# create a sample file to send
|
|
53
|
+
file_to_send=File.join(tmpdir,'myfile.bin')
|
|
54
|
+
File.open(file_to_send, "w") {|f| f.write("sample data") }
|
|
55
|
+
# package creation parameters
|
|
56
|
+
package_create_params={'delivery'=>{'title'=>'test package','recipients'=>['aspera.user1@gmail.com'],'sources'=>[{'paths'=>[file_to_send]}]}}
|
|
57
|
+
pkg_created=api_v3.create('send',package_create_params)[:data]
|
|
58
|
+
# get transfer specification (normally: only one)
|
|
59
|
+
transfer_spec=pkg_created['xfer_sessions'].first
|
|
60
|
+
# set paths of files to send
|
|
61
|
+
transfer_spec['paths']=[{'source'=>file_to_send}]
|
|
62
|
+
# get the local agent (i.e. ascp)
|
|
63
|
+
transfer_client=Aspera::Fasp::Local.new
|
|
64
|
+
# disable ascp output on stdout (optional)
|
|
65
|
+
transfer_client.quiet=true
|
|
66
|
+
# start transfer (asynchronous)
|
|
67
|
+
job_id=transfer_client.start_transfer(transfer_spec)
|
|
68
|
+
# wait for all transfer completion (for the example)
|
|
69
|
+
result=transfer_client.wait_for_transfers_completion
|
|
70
|
+
# notify of any transfer error
|
|
71
|
+
result.select{|i|!i.eql?(:success)}.each do |e|
|
|
72
|
+
Aspera::Log.log.error("A transfer error occured: #{e.message}")
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
# 3: Faspex 4 API v4
|
|
76
|
+
#---------------
|
|
77
|
+
api_v4=Aspera::Rest.new({
|
|
78
|
+
:base_url => faspex_url+'/api',
|
|
79
|
+
:auth => {
|
|
80
|
+
:type => :oauth2,
|
|
81
|
+
:base_url => faspex_url+'/auth/oauth2',
|
|
82
|
+
:grant => :header_userpass,
|
|
83
|
+
:user_name => faspex_user,
|
|
84
|
+
:user_pass => faspex_pass,
|
|
85
|
+
:scope => 'admin'
|
|
86
|
+
}})
|
|
87
|
+
|
|
88
|
+
# Use it. Note that Faspex 4 API v4 is totally different from Faspex 4 v3 APIs, see ref on line 2
|
|
89
|
+
Aspera::Log.dump('users',api_v4.read('users')[:data])
|
data/lib/aspera/aoc.rb
CHANGED
|
@@ -9,6 +9,8 @@ module Aspera
|
|
|
9
9
|
private
|
|
10
10
|
@@use_standard_ports = true
|
|
11
11
|
|
|
12
|
+
API_V1='api/v1'
|
|
13
|
+
|
|
12
14
|
PRODUCT_NAME='Aspera on Cloud'
|
|
13
15
|
# Production domain of AoC
|
|
14
16
|
PROD_DOMAIN='ibmaspera.com'
|
|
@@ -42,11 +44,6 @@ module Aspera
|
|
|
42
44
|
FILES_APP='files'
|
|
43
45
|
PACKAGES_APP='packages'
|
|
44
46
|
|
|
45
|
-
CLIENT_RANDOM={
|
|
46
|
-
'aspera.global-cli-client' => 'frpmsRsG4mjZ0PlxCgdJlvONqBg4Vlpz_IX7gXmBMAfsgMLy2FO6CXLodKfKAuhqnCqSptLbe_wdmnm9JRuEPO-PpFqpq_Kb',
|
|
47
|
-
'aspera.drive' => 'UegzQ3LcbLht5dLYAXaR-7ZMnJ6-kwPEXWEXaqLSOMGmtzNA9r6kPFLElqBfq66BfgMabdO96k5sPXV-H8M3vsx9LbGlewF1'
|
|
48
|
-
}
|
|
49
|
-
|
|
50
47
|
def self.get_client_info(client_name=CLIENT_APPS.first)
|
|
51
48
|
client_index=CLIENT_APPS.index(client_name)
|
|
52
49
|
raise "no such pre-defined client: #{client_name}" if client_index.nil?
|
|
@@ -68,9 +65,14 @@ module Aspera
|
|
|
68
65
|
return organization,instance_domain
|
|
69
66
|
end
|
|
70
67
|
|
|
68
|
+
# base API url depends on domain, which could be "qa.xxx"
|
|
69
|
+
def self.api_base_url(api_domain=PROD_DOMAIN)
|
|
70
|
+
return "https://api.#{api_domain}"
|
|
71
|
+
end
|
|
72
|
+
|
|
71
73
|
def self.metering_api(entitlement_id,customer_id,api_domain=PROD_DOMAIN)
|
|
72
74
|
return Rest.new({
|
|
73
|
-
:base_url => "
|
|
75
|
+
:base_url => "#{api_base_url(api_domain)}/metering/v1",
|
|
74
76
|
:headers => {'X-Aspera-Entitlement-Authorization' => Rest.basic_creds(entitlement_id,customer_id)}
|
|
75
77
|
})
|
|
76
78
|
end
|
|
@@ -126,7 +128,7 @@ module Aspera
|
|
|
126
128
|
# access key secrets are provided out of band to get node api access
|
|
127
129
|
# key: access key
|
|
128
130
|
# value: associated secret
|
|
129
|
-
@
|
|
131
|
+
@key_chain=nil
|
|
130
132
|
|
|
131
133
|
# init rest params
|
|
132
134
|
aoc_rest_p={:auth=>{:type =>:oauth2}}
|
|
@@ -147,12 +149,12 @@ module Aspera
|
|
|
147
149
|
|
|
148
150
|
# get org name and domain from url
|
|
149
151
|
organization,instance_domain=self.class.parse_url(opt[:url])
|
|
150
|
-
# this is the base API url
|
|
151
|
-
api_base_url
|
|
152
|
-
#
|
|
153
|
-
aoc_rest_p[:base_url]="#{
|
|
152
|
+
# this is the base API url
|
|
153
|
+
api_url_base=self.class.api_base_url(instance_domain)
|
|
154
|
+
# API URL, including subpath (version ...)
|
|
155
|
+
aoc_rest_p[:base_url]="#{api_url_base}/#{opt[:subpath]}"
|
|
154
156
|
# base auth URL
|
|
155
|
-
aoc_auth_p[:base_url] = "#{
|
|
157
|
+
aoc_auth_p[:base_url] = "#{api_url_base}/#{OAUTH_API_SUBPATH}/#{organization}"
|
|
156
158
|
aoc_auth_p[:client_id]=opt[:client_id]
|
|
157
159
|
aoc_auth_p[:client_secret] = opt[:client_secret]
|
|
158
160
|
|
|
@@ -192,17 +194,12 @@ module Aspera
|
|
|
192
194
|
super(aoc_rest_p)
|
|
193
195
|
end
|
|
194
196
|
|
|
195
|
-
def
|
|
196
|
-
@
|
|
197
|
-
|
|
197
|
+
def key_chain=(keychain)
|
|
198
|
+
raise "keychain already set" unless @key_chain.nil?
|
|
199
|
+
@key_chain=keychain
|
|
198
200
|
nil
|
|
199
201
|
end
|
|
200
202
|
|
|
201
|
-
def has_secret(ak)
|
|
202
|
-
Log.log.debug("has key:#{ak} -> #{@secrets.has_key?(ak)}")
|
|
203
|
-
return @secrets.has_key?(ak)
|
|
204
|
-
end
|
|
205
|
-
|
|
206
203
|
# additional transfer spec (tags) for package information
|
|
207
204
|
def self.package_tags(package_info,operation)
|
|
208
205
|
return {'tags'=>{'aspera'=>{'files'=>{
|
|
@@ -276,7 +273,7 @@ module Aspera
|
|
|
276
273
|
transfer_spec['remote_host']=node_file[:node_info]['host']
|
|
277
274
|
else
|
|
278
275
|
# retrieve values from API
|
|
279
|
-
std_t_spec=get_node_api(node_file[:node_info],SCOPE_NODE_USER).create('files/download_setup',{:transfer_requests => [ { :transfer_request => {:paths => [ {"source"=>'/'} ] } } ] } )[:data]['transfer_specs'].first['transfer_spec']
|
|
276
|
+
std_t_spec=get_node_api(node_file[:node_info],scope: SCOPE_NODE_USER).create('files/download_setup',{:transfer_requests => [ { :transfer_request => {:paths => [ {"source"=>'/'} ] } } ] } )[:data]['transfer_specs'].first['transfer_spec']
|
|
280
277
|
['remote_host','remote_user','ssh_port','fasp_port'].each {|i| transfer_spec[i]=std_t_spec[i]}
|
|
281
278
|
end
|
|
282
279
|
# add caller provided transfer spec
|
|
@@ -293,26 +290,27 @@ module Aspera
|
|
|
293
290
|
# @param scope e.g. SCOPE_NODE_USER
|
|
294
291
|
# no scope: requires secret
|
|
295
292
|
# if secret provided beforehand: use it
|
|
296
|
-
def get_node_api(node_info,
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
if ak_secret.nil? and node_scope.nil?
|
|
304
|
-
raise 'There must be at least one of: secret, node scope'
|
|
293
|
+
def get_node_api(node_info,options={})
|
|
294
|
+
raise "INTERNAL ERROR: method parameters: options must ne hash" unless options.is_a?(Hash)
|
|
295
|
+
options.keys.each {|k| raise "INTERNAL ERROR: not valid option: #{k}" unless [:scope,:use_secret].include?(k)}
|
|
296
|
+
# get optional secret unless :use_secret is false (default is true)
|
|
297
|
+
ak_secret=@key_chain.get_secret(node_info['access_key'],false) if !options.has_key?(:use_secret) or options[:use_secret]
|
|
298
|
+
if ak_secret.nil? and !options.has_key?(:scope)
|
|
299
|
+
raise "There must be at least one of: 'secret' or 'scope' for access key #{node_info['access_key']}"
|
|
305
300
|
end
|
|
306
|
-
|
|
307
|
-
if
|
|
301
|
+
node_rest_params={base_url: node_info['url']}
|
|
302
|
+
# if secret is available
|
|
303
|
+
if !ak_secret.nil?
|
|
308
304
|
node_rest_params[:auth]={
|
|
309
|
-
:
|
|
310
|
-
:
|
|
311
|
-
:
|
|
305
|
+
type: :basic,
|
|
306
|
+
username: node_info['access_key'],
|
|
307
|
+
password: ak_secret
|
|
312
308
|
}
|
|
313
309
|
else
|
|
310
|
+
# X-Aspera-AccessKey required for bearer token only
|
|
311
|
+
node_rest_params[:headers]= {'X-Aspera-AccessKey'=>node_info['access_key']}
|
|
314
312
|
node_rest_params[:auth]=self.params[:auth].clone
|
|
315
|
-
node_rest_params[:auth][:scope]=self.class.node_scope(node_info['access_key'],
|
|
313
|
+
node_rest_params[:auth][:scope]=self.class.node_scope(node_info['access_key'],options[:scope])
|
|
316
314
|
end
|
|
317
315
|
return Node.new(node_rest_params)
|
|
318
316
|
end
|
|
@@ -339,7 +337,7 @@ module Aspera
|
|
|
339
337
|
if entry[:type].eql?('link')
|
|
340
338
|
sub_node_info=self.read("nodes/#{entry['target_node_id']}")[:data]
|
|
341
339
|
sub_opt={method: process_find_files, top_file_id: entry['target_id'], top_file_path: path}
|
|
342
|
-
get_node_api(sub_node_info,SCOPE_NODE_USER).crawl(self,sub_opt)
|
|
340
|
+
get_node_api(sub_node_info,scope: SCOPE_NODE_USER).crawl(self,sub_opt)
|
|
343
341
|
end
|
|
344
342
|
rescue => e
|
|
345
343
|
Log.log.error("#{path}: #{e.message}")
|
|
@@ -352,7 +350,7 @@ module Aspera
|
|
|
352
350
|
top_node_info,top_file_id=check_get_node_file(top_node_file)
|
|
353
351
|
Log.log.debug("find_files: node_info=#{top_node_info}, fileid=#{top_file_id}")
|
|
354
352
|
@find_state={found: [], test_block: test_block}
|
|
355
|
-
get_node_api(top_node_info,SCOPE_NODE_USER).crawl(self,{method: :process_find_files, top_file_id: top_file_id})
|
|
353
|
+
get_node_api(top_node_info,scope: SCOPE_NODE_USER).crawl(self,{method: :process_find_files, top_file_id: top_file_id})
|
|
356
354
|
result=@find_state[:found]
|
|
357
355
|
@find_state=nil
|
|
358
356
|
return result
|
|
@@ -373,7 +371,7 @@ module Aspera
|
|
|
373
371
|
if @resolve_state[:path].empty?
|
|
374
372
|
@resolve_state[:result][:file_id]=entry['target_id']
|
|
375
373
|
else
|
|
376
|
-
get_node_api(@resolve_state[:result][:node_info],SCOPE_NODE_USER).crawl(self,{method: :process_resolve_node_file, top_file_id: entry['target_id']})
|
|
374
|
+
get_node_api(@resolve_state[:result][:node_info],scope: SCOPE_NODE_USER).crawl(self,{method: :process_resolve_node_file, top_file_id: entry['target_id']})
|
|
377
375
|
end
|
|
378
376
|
when 'folder'
|
|
379
377
|
if @resolve_state[:path].empty?
|
|
@@ -400,7 +398,7 @@ module Aspera
|
|
|
400
398
|
result[:file_id]=top_file_id
|
|
401
399
|
else
|
|
402
400
|
@resolve_state={path: path_elements, result: result}
|
|
403
|
-
get_node_api(top_node_info,SCOPE_NODE_USER).crawl(self,{method: :process_resolve_node_file, top_file_id: top_file_id})
|
|
401
|
+
get_node_api(top_node_info,scope: SCOPE_NODE_USER).crawl(self,{method: :process_resolve_node_file, top_file_id: top_file_id})
|
|
404
402
|
not_found=@resolve_state[:path]
|
|
405
403
|
@resolve_state=nil
|
|
406
404
|
raise "entry not found: #{not_found}" if result[:file_id].nil?
|
data/lib/aspera/cli/main.rb
CHANGED
|
@@ -10,6 +10,7 @@ require 'aspera/persistency_folder'
|
|
|
10
10
|
require 'aspera/log'
|
|
11
11
|
require 'aspera/rest'
|
|
12
12
|
require 'aspera/nagios'
|
|
13
|
+
require 'aspera/secrets'
|
|
13
14
|
|
|
14
15
|
module Aspera
|
|
15
16
|
module Cli
|
|
@@ -20,15 +21,17 @@ module Aspera
|
|
|
20
21
|
private
|
|
21
22
|
# name of application, also foldername where config is stored
|
|
22
23
|
PROGRAM_NAME = 'ascli'
|
|
24
|
+
# name of the containing gem, same as in <gem name>.gemspec
|
|
23
25
|
GEM_NAME = 'aspera-cli'
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
26
|
+
HELP_URL = "http://www.rubydoc.info/gems/#{GEM_NAME}"
|
|
27
|
+
GEM_URL = "https://rubygems.org/gems/#{GEM_NAME}"
|
|
28
|
+
SRC_URL = "https://github.com/IBM/aspera-cli"
|
|
29
|
+
STATUS_FIELD = 'status'
|
|
27
30
|
|
|
31
|
+
private_constant :PROGRAM_NAME,:GEM_NAME,:HELP_URL,:GEM_URL
|
|
28
32
|
# =============================================================
|
|
29
33
|
# Parameter handlers
|
|
30
34
|
#
|
|
31
|
-
|
|
32
35
|
def option_insecure; Rest.insecure ; end
|
|
33
36
|
|
|
34
37
|
def option_insecure=(value); Rest.insecure = value; end
|
|
@@ -39,20 +42,19 @@ module Aspera
|
|
|
39
42
|
|
|
40
43
|
# minimum initialization
|
|
41
44
|
def initialize(argv)
|
|
42
|
-
# first thing : manage debug level (allows debugging
|
|
45
|
+
# first thing : manage debug level (allows debugging of option parser)
|
|
43
46
|
early_debug_setup(argv)
|
|
47
|
+
# compare $0 with expected name
|
|
44
48
|
current_prog_name=File.basename($PROGRAM_NAME)
|
|
45
49
|
unless current_prog_name.eql?(PROGRAM_NAME)
|
|
46
50
|
@plugin_env[:formater].display_message(:error,"#{"WARNING".bg_red.blink.gray} Please use '#{PROGRAM_NAME}' instead of '#{current_prog_name}', '#{current_prog_name}' will be removed in a future version")
|
|
47
51
|
end
|
|
48
|
-
# overriding parameters on transfer spec
|
|
49
52
|
@option_help=false
|
|
50
53
|
@bash_completion=false
|
|
51
54
|
@option_show_config=false
|
|
55
|
+
# environment provided to plugin for various capabilities
|
|
52
56
|
@plugin_env={}
|
|
53
|
-
|
|
54
|
-
@gem_url='https://rubygems.org/gems/'+GEM_NAME
|
|
55
|
-
# give command line arguments to option manager (no parsing)
|
|
57
|
+
# find out application main folder
|
|
56
58
|
app_main_folder=ENV[conf_dir_env_var]
|
|
57
59
|
# if env var undefined or empty
|
|
58
60
|
if app_main_folder.nil? or app_main_folder.empty?
|
|
@@ -60,18 +62,21 @@ module Aspera
|
|
|
60
62
|
raise CliError,"Home folder does not exist: #{user_home_folder}. Check your user environment or use #{conf_dir_env_var}." unless Dir.exist?(user_home_folder)
|
|
61
63
|
app_main_folder=File.join(user_home_folder,Plugins::Config::ASPERA_HOME_FOLDER_NAME,PROGRAM_NAME)
|
|
62
64
|
end
|
|
65
|
+
# give command line arguments to option manager (no parsing)
|
|
63
66
|
@plugin_env[:options]=@opt_mgr=Manager.new(PROGRAM_NAME,argv,app_banner())
|
|
64
67
|
@plugin_env[:formater]=Formater.new(@plugin_env[:options])
|
|
65
68
|
Rest.user_agent=PROGRAM_NAME
|
|
66
|
-
#
|
|
69
|
+
# declare and parse global options
|
|
67
70
|
init_global_options()
|
|
71
|
+
# secret manager
|
|
72
|
+
@plugin_env[:secret]=Aspera::Secrets.new
|
|
68
73
|
# the Config plugin adds the @preset parser
|
|
69
|
-
@plugin_env[:config]=Plugins::Config.new(@plugin_env,PROGRAM_NAME
|
|
74
|
+
@plugin_env[:config]=Plugins::Config.new(@plugin_env,PROGRAM_NAME,HELP_URL,Aspera::Cli::VERSION,app_main_folder)
|
|
70
75
|
# the TransferAgent plugin may use the @preset parser
|
|
71
76
|
@plugin_env[:transfer]=TransferAgent.new(@plugin_env)
|
|
72
|
-
Log.log.debug('created plugin env'.red)
|
|
73
77
|
# set application folder for modules
|
|
74
78
|
@plugin_env[:persistency]=PersistencyFolder.new(File.join(@plugin_env[:config].main_folder,'persist_store'))
|
|
79
|
+
Log.log.debug('plugin env created'.red)
|
|
75
80
|
Oauth.persist_mgr=@plugin_env[:persistency]
|
|
76
81
|
Fasp::Parameters.file_list_folder=File.join(@plugin_env[:config].main_folder,'filelists')
|
|
77
82
|
Aspera::RestErrorAnalyzer.instance.log_file=File.join(@plugin_env[:config].main_folder,'rest_exceptions.log')
|
|
@@ -85,9 +90,9 @@ module Aspera
|
|
|
85
90
|
banner << "\t#{PROGRAM_NAME} COMMANDS [OPTIONS] [ARGS]\n"
|
|
86
91
|
banner << "\nDESCRIPTION\n"
|
|
87
92
|
banner << "\tUse Aspera application to perform operations on command line.\n"
|
|
88
|
-
banner << "\tDocumentation and examples: #{
|
|
93
|
+
banner << "\tDocumentation and examples: #{GEM_URL}\n"
|
|
89
94
|
banner << "\texecute: #{PROGRAM_NAME} conf doc\n"
|
|
90
|
-
banner << "\tor visit: #{
|
|
95
|
+
banner << "\tor visit: #{HELP_URL}\n"
|
|
91
96
|
banner << "\nENVIRONMENT VARIABLES\n"
|
|
92
97
|
banner << "\t#{conf_dir_env_var} config folder, default: $HOME/#{Plugins::Config::ASPERA_HOME_FOLDER_NAME}/#{PROGRAM_NAME}\n"
|
|
93
98
|
banner << "\t#any option can be set as an environment variable, refer to the manual\n"
|
|
@@ -185,6 +190,8 @@ module Aspera
|
|
|
185
190
|
|
|
186
191
|
protected
|
|
187
192
|
|
|
193
|
+
# env var name to override the app's main folder
|
|
194
|
+
# default main folder is $HOME/<vendor main app folder>/<program name>
|
|
188
195
|
def conf_dir_env_var
|
|
189
196
|
return "#{PROGRAM_NAME}_home".upcase
|
|
190
197
|
end
|
|
@@ -216,10 +223,28 @@ module Aspera
|
|
|
216
223
|
return Main.result_nothing
|
|
217
224
|
end
|
|
218
225
|
|
|
226
|
+
# used when one command executes several transfer jobs (each job being possibly multi session)
|
|
227
|
+
# @param status_table [Array] [{STATUS_FIELD=>[status array],...},...]
|
|
228
|
+
# each element has a key STATUS_FIELD which contains the result of possibly mulmtiple sessions
|
|
229
|
+
def self.result_transfer_multiple(status_table)
|
|
230
|
+
global_status=:success
|
|
231
|
+
# transform status into string and find if there was problem
|
|
232
|
+
status_table.each do |item|
|
|
233
|
+
worst=TransferAgent.session_status(item[STATUS_FIELD])
|
|
234
|
+
global_status=worst unless worst.eql?(:success)
|
|
235
|
+
item[STATUS_FIELD]=item[STATUS_FIELD].map{|i|i.to_s}.join(',')
|
|
236
|
+
end
|
|
237
|
+
raise global_status unless global_status.eql?(:success)
|
|
238
|
+
return {:type=>:object_list,:data=>status_table}
|
|
239
|
+
end
|
|
240
|
+
|
|
219
241
|
# this is the main function called by initial script just after constructor
|
|
220
242
|
def process_command_line
|
|
221
243
|
Log.log.debug('process_command_line')
|
|
244
|
+
# catch exception information , if any
|
|
222
245
|
exception_info=nil
|
|
246
|
+
# false if command shall not be executed ("once_only")
|
|
247
|
+
execute_command=true
|
|
223
248
|
begin
|
|
224
249
|
# find plugins, shall be after parse! ?
|
|
225
250
|
@plugin_env[:config].add_plugins_from_lookup_folders
|
|
@@ -229,16 +254,6 @@ module Aspera
|
|
|
229
254
|
# load global default options and process
|
|
230
255
|
@plugin_env[:config].add_plugin_default_preset(Plugins::Config::CONF_GLOBAL_SYM)
|
|
231
256
|
@opt_mgr.parse_options!
|
|
232
|
-
# dual execution locking
|
|
233
|
-
lock_port=@opt_mgr.get_option(:lock_port,:optional)
|
|
234
|
-
if !lock_port.nil?
|
|
235
|
-
begin
|
|
236
|
-
# no need to close later, will be freed on process exit. must save in member else it is garbage collected
|
|
237
|
-
@tcp_server=TCPServer.new('127.0.0.1',lock_port.to_i)
|
|
238
|
-
rescue => e
|
|
239
|
-
raise CliError,"Another instance is already running (lock port=#{lock_port})."
|
|
240
|
-
end
|
|
241
|
-
end
|
|
242
257
|
@plugin_env[:config].periodic_check_newer_gem_version
|
|
243
258
|
if @option_show_config and @opt_mgr.command_or_arg_empty?
|
|
244
259
|
command_sym=Plugins::Config::CONF_PLUGIN_SYM
|
|
@@ -263,18 +278,30 @@ module Aspera
|
|
|
263
278
|
@plugin_env[:formater].display_results({:type=>:single_object,:data=>@opt_mgr.declared_options(false)})
|
|
264
279
|
Process.exit(0)
|
|
265
280
|
end
|
|
266
|
-
#
|
|
267
|
-
|
|
281
|
+
# locking for single execution (only after "per plugin" option, in case lock port is there)
|
|
282
|
+
lock_port=@opt_mgr.get_option(:lock_port,:optional)
|
|
283
|
+
if !lock_port.nil?
|
|
284
|
+
begin
|
|
285
|
+
# no need to close later, will be freed on process exit. must save in member else it is garbage collected
|
|
286
|
+
Log.log.debug("Opening lock port #{lock_port.to_i}")
|
|
287
|
+
@tcp_server=TCPServer.new('127.0.0.1',lock_port.to_i)
|
|
288
|
+
rescue => e
|
|
289
|
+
execute_command=false
|
|
290
|
+
Log.log.warn("Another instance is already running (#{e.message}).")
|
|
291
|
+
end
|
|
292
|
+
end
|
|
293
|
+
# execute and display (if not exclusive execution)
|
|
294
|
+
@plugin_env[:formater].display_results(command_plugin.execute_action) if execute_command
|
|
268
295
|
# finish
|
|
269
296
|
@plugin_env[:transfer].shutdown
|
|
270
297
|
rescue CliBadArgument => e; exception_info=[e,'Argument',:usage]
|
|
271
298
|
rescue CliNoSuchId => e; exception_info=[e,'Identifier']
|
|
272
299
|
rescue CliError => e; exception_info=[e,'Tool',:usage]
|
|
273
|
-
rescue Fasp::Error => e; exception_info=[e,
|
|
274
|
-
rescue Aspera::RestCallError => e;
|
|
275
|
-
rescue SocketError => e; exception_info=[e,
|
|
276
|
-
rescue StandardError => e; exception_info=[e,
|
|
277
|
-
rescue Interrupt => e; exception_info=[e,
|
|
300
|
+
rescue Fasp::Error => e; exception_info=[e,'FASP(ascp)']
|
|
301
|
+
rescue Aspera::RestCallError => e; exception_info=[e,'Rest']
|
|
302
|
+
rescue SocketError => e; exception_info=[e,'Network']
|
|
303
|
+
rescue StandardError => e; exception_info=[e,'Other',:debug]
|
|
304
|
+
rescue Interrupt => e; exception_info=[e,'Interruption',:debug]
|
|
278
305
|
end
|
|
279
306
|
# cleanup file list files
|
|
280
307
|
TempFileManager.instance.cleanup
|
|
@@ -282,10 +309,15 @@ module Aspera
|
|
|
282
309
|
unless exception_info.nil?
|
|
283
310
|
@plugin_env[:formater].display_message(:error,"ERROR:".bg_red.gray.blink+" "+exception_info[1]+": "+exception_info[0].message)
|
|
284
311
|
@plugin_env[:formater].display_message(:error,"Use '-h' option to get help.") if exception_info[2].eql?(:usage)
|
|
312
|
+
if exception_info.first.is_a?(Fasp::Error) and exception_info.first.message.eql?('Remote host is not who we expected')
|
|
313
|
+
@plugin_env[:formater].display_message(:error,"For this specific error, refer to:\n#{SRC_URL}#error-remote-host-is-not-who-we-expected\nAdd this to arguments:\n--ts=@json:'{\"sshfp\":null}'")
|
|
314
|
+
end
|
|
285
315
|
end
|
|
286
316
|
# 2- processing of command not processed (due to exception or bad command line)
|
|
287
|
-
|
|
288
|
-
@
|
|
317
|
+
if execute_command
|
|
318
|
+
@opt_mgr.final_errors.each do |msg|
|
|
319
|
+
@plugin_env[:formater].display_message(:error,"ERROR:".bg_red.gray.blink+" Argument: "+msg)
|
|
320
|
+
end
|
|
289
321
|
end
|
|
290
322
|
# 3- in case of error, fail the process status
|
|
291
323
|
unless exception_info.nil?
|