aspera-cli 4.0.0.pre2 → 4.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +761 -210
- data/bin/ascli +2 -0
- data/bin/dascli +13 -0
- data/docs/Makefile +2 -1
- data/docs/README.erb.md +628 -160
- data/docs/test_env.conf +22 -10
- data/docs/transfer_spec.html +1 -1
- data/lib/aspera/aoc.rb +87 -108
- data/lib/aspera/cli/formater.rb +2 -0
- data/lib/aspera/cli/main.rb +48 -45
- data/lib/aspera/cli/manager.rb +19 -6
- data/lib/aspera/cli/plugin.rb +9 -4
- data/lib/aspera/cli/plugins/alee.rb +1 -1
- data/lib/aspera/cli/plugins/aoc.rb +208 -183
- data/lib/aspera/cli/plugins/ats.rb +2 -2
- data/lib/aspera/cli/plugins/config.rb +205 -125
- data/lib/aspera/cli/plugins/console.rb +2 -2
- data/lib/aspera/cli/plugins/faspex.rb +15 -8
- data/lib/aspera/cli/plugins/faspex5.rb +76 -37
- data/lib/aspera/cli/plugins/node.rb +3 -3
- data/lib/aspera/cli/plugins/preview.rb +35 -25
- data/lib/aspera/cli/plugins/server.rb +23 -8
- data/lib/aspera/cli/transfer_agent.rb +7 -6
- data/lib/aspera/cli/version.rb +1 -1
- data/lib/aspera/colors.rb +5 -1
- data/lib/aspera/cos_node.rb +33 -28
- data/lib/aspera/environment.rb +15 -4
- data/lib/aspera/fasp/connect.rb +28 -21
- data/lib/aspera/fasp/http_gw.rb +140 -28
- data/lib/aspera/fasp/installation.rb +119 -57
- data/lib/aspera/fasp/local.rb +174 -178
- data/lib/aspera/fasp/manager.rb +12 -0
- data/lib/aspera/fasp/node.rb +4 -4
- data/lib/aspera/fasp/parameters.rb +6 -18
- data/lib/aspera/fasp/resume_policy.rb +13 -12
- data/lib/aspera/log.rb +10 -2
- data/lib/aspera/node.rb +61 -1
- data/lib/aspera/oauth.rb +36 -13
- data/lib/aspera/persistency_folder.rb +9 -4
- data/lib/aspera/preview/file_types.rb +53 -21
- data/lib/aspera/preview/generator.rb +3 -3
- data/lib/aspera/rest.rb +29 -18
- data/lib/aspera/secrets.rb +20 -0
- data/lib/aspera/temp_file_manager.rb +19 -0
- metadata +40 -22
data/docs/test_env.conf
CHANGED
|
@@ -1,16 +1,16 @@
|
|
|
1
1
|
---
|
|
2
2
|
config:
|
|
3
|
-
version: 4.0.0
|
|
3
|
+
version: 4.0.0
|
|
4
4
|
default:
|
|
5
5
|
config: cli_default
|
|
6
|
-
aoc:
|
|
6
|
+
aoc: tst_aoc1
|
|
7
7
|
faspex: tst_faspex
|
|
8
|
-
faspex5:
|
|
9
|
-
shares:
|
|
8
|
+
faspex5: tst_faspex5_boot
|
|
9
|
+
shares: tst_shares_1
|
|
10
10
|
shares2: tst_shares2
|
|
11
11
|
node: tst_node
|
|
12
12
|
server: tst_server
|
|
13
|
-
orchestrator:
|
|
13
|
+
orchestrator: tst_orch
|
|
14
14
|
console: tst_console
|
|
15
15
|
preview: tst_ak_preview
|
|
16
16
|
ats: tst_ats
|
|
@@ -19,7 +19,6 @@ default:
|
|
|
19
19
|
cli_default:
|
|
20
20
|
interactive: your value here
|
|
21
21
|
smtp: your value here
|
|
22
|
-
ascp_path: your value here
|
|
23
22
|
local_user:
|
|
24
23
|
ssh_keys: your value here
|
|
25
24
|
smtp_config:
|
|
@@ -31,7 +30,7 @@ smtp_config:
|
|
|
31
30
|
from_name: your value here
|
|
32
31
|
username: your value here
|
|
33
32
|
password: your value here
|
|
34
|
-
|
|
33
|
+
tst_aoc1:
|
|
35
34
|
url: your value here
|
|
36
35
|
username: your value here
|
|
37
36
|
auth: your value here
|
|
@@ -48,10 +47,18 @@ tst_node_faspex:
|
|
|
48
47
|
url: your value here
|
|
49
48
|
username: your value here
|
|
50
49
|
password: your value here
|
|
51
|
-
|
|
50
|
+
tst_faspex5_boot:
|
|
52
51
|
url: your value here
|
|
52
|
+
auth: your value here
|
|
53
53
|
username: your value here
|
|
54
54
|
password: your value here
|
|
55
|
+
tst_faspex5:
|
|
56
|
+
url: your value here
|
|
57
|
+
auth: your value here
|
|
58
|
+
client_id: your value here
|
|
59
|
+
client_secret: your value here
|
|
60
|
+
private_key: your value here
|
|
61
|
+
username: your value here
|
|
55
62
|
tst_shares:
|
|
56
63
|
url: your value here
|
|
57
64
|
username: your value here
|
|
@@ -79,7 +86,7 @@ tst_server:
|
|
|
79
86
|
tst_server_bykey:
|
|
80
87
|
url: your value here
|
|
81
88
|
username: your value here
|
|
82
|
-
|
|
89
|
+
tst_orch:
|
|
83
90
|
url: your value here
|
|
84
91
|
username: your value here
|
|
85
92
|
password: your value here
|
|
@@ -94,6 +101,7 @@ tst_ak_preview:
|
|
|
94
101
|
url: your value here
|
|
95
102
|
username: your value here
|
|
96
103
|
password: your value here
|
|
104
|
+
mimemagic: your value here
|
|
97
105
|
tst_node_preview:
|
|
98
106
|
url: your value here
|
|
99
107
|
username: your value here
|
|
@@ -110,6 +118,8 @@ misc:
|
|
|
110
118
|
faspex_publink_send_to_fxuser: your value here
|
|
111
119
|
faspex_publink_send_to_dropbox: your value here
|
|
112
120
|
shares_upload: your value here
|
|
121
|
+
console_smart_id: your value here
|
|
122
|
+
console_smart_file: your value here
|
|
113
123
|
orch_workflow_id: your value here
|
|
114
124
|
file_dcm: your value here
|
|
115
125
|
file_pdf: your value here
|
|
@@ -122,6 +132,7 @@ misc:
|
|
|
122
132
|
aoc_publink_recv_from_aocuser: your value here
|
|
123
133
|
aoc_publink_send_shd_inbox: your value here
|
|
124
134
|
aoc_publink_send_aoc_user: your value here
|
|
135
|
+
aoc_publink_send_use_pass: your value here
|
|
125
136
|
aoc_publink_folder: your value here
|
|
126
137
|
aoc_shbx_ws: your value here
|
|
127
138
|
aoc_shbx_name: your value here
|
|
@@ -137,4 +148,5 @@ misc:
|
|
|
137
148
|
email_internal: your value here
|
|
138
149
|
email_external: your value here
|
|
139
150
|
aoc_org: your value here
|
|
140
|
-
|
|
151
|
+
aoc_user_email: your value here
|
|
152
|
+
http_gw_fqdn_port: your value here
|
data/docs/transfer_spec.html
CHANGED
|
@@ -28,7 +28,7 @@ arg: related ascp argument or env var suffix (PASS for ASPERA_SCP_PASS)
|
|
|
28
28
|
</p>
|
|
29
29
|
<p>
|
|
30
30
|
UNDER CONSTRUCTION<br/>
|
|
31
|
-
<a href="https://developer.ibm.com/
|
|
31
|
+
<a href="https://developer.ibm.com/apis/catalog/?search=aspera">Aspera API Documentation</a>→Node API→/opt/transfers<br/>
|
|
32
32
|
</p>
|
|
33
33
|
|
|
34
34
|
<table>
|
data/lib/aspera/aoc.rb
CHANGED
|
@@ -9,6 +9,8 @@ module Aspera
|
|
|
9
9
|
private
|
|
10
10
|
@@use_standard_ports = true
|
|
11
11
|
|
|
12
|
+
API_V1='api/v1'
|
|
13
|
+
|
|
12
14
|
PRODUCT_NAME='Aspera on Cloud'
|
|
13
15
|
# Production domain of AoC
|
|
14
16
|
PROD_DOMAIN='ibmaspera.com'
|
|
@@ -42,11 +44,6 @@ module Aspera
|
|
|
42
44
|
FILES_APP='files'
|
|
43
45
|
PACKAGES_APP='packages'
|
|
44
46
|
|
|
45
|
-
CLIENT_RANDOM={
|
|
46
|
-
'aspera.global-cli-client' => 'frpmsRsG4mjZ0PlxCgdJlvONqBg4Vlpz_IX7gXmBMAfsgMLy2FO6CXLodKfKAuhqnCqSptLbe_wdmnm9JRuEPO-PpFqpq_Kb',
|
|
47
|
-
'aspera.drive' => 'UegzQ3LcbLht5dLYAXaR-7ZMnJ6-kwPEXWEXaqLSOMGmtzNA9r6kPFLElqBfq66BfgMabdO96k5sPXV-H8M3vsx9LbGlewF1'
|
|
48
|
-
}
|
|
49
|
-
|
|
50
47
|
def self.get_client_info(client_name=CLIENT_APPS.first)
|
|
51
48
|
client_index=CLIENT_APPS.index(client_name)
|
|
52
49
|
raise "no such pre-defined client: #{client_name}" if client_index.nil?
|
|
@@ -68,9 +65,14 @@ module Aspera
|
|
|
68
65
|
return organization,instance_domain
|
|
69
66
|
end
|
|
70
67
|
|
|
68
|
+
# base API url depends on domain, which could be "qa.xxx"
|
|
69
|
+
def self.api_base_url(api_domain=PROD_DOMAIN)
|
|
70
|
+
return "https://api.#{api_domain}"
|
|
71
|
+
end
|
|
72
|
+
|
|
71
73
|
def self.metering_api(entitlement_id,customer_id,api_domain=PROD_DOMAIN)
|
|
72
74
|
return Rest.new({
|
|
73
|
-
:base_url => "
|
|
75
|
+
:base_url => "#{api_base_url(api_domain)}/metering/v1",
|
|
74
76
|
:headers => {'X-Aspera-Entitlement-Authorization' => Rest.basic_creds(entitlement_id,customer_id)}
|
|
75
77
|
})
|
|
76
78
|
end
|
|
@@ -121,12 +123,12 @@ module Aspera
|
|
|
121
123
|
raise RuntimeError,'too many redirections'
|
|
122
124
|
end
|
|
123
125
|
|
|
124
|
-
# @param :link,:url,:auth,:client_id,:client_secret,:scope,:redirect_uri,:private_key,:username,:subpath
|
|
126
|
+
# @param :link,:url,:auth,:client_id,:client_secret,:scope,:redirect_uri,:private_key,:username,:subpath,:password (for pub link)
|
|
125
127
|
def initialize(opt)
|
|
126
128
|
# access key secrets are provided out of band to get node api access
|
|
127
129
|
# key: access key
|
|
128
130
|
# value: associated secret
|
|
129
|
-
@
|
|
131
|
+
@key_chain=nil
|
|
130
132
|
|
|
131
133
|
# init rest params
|
|
132
134
|
aoc_rest_p={:auth=>{:type =>:oauth2}}
|
|
@@ -147,12 +149,12 @@ module Aspera
|
|
|
147
149
|
|
|
148
150
|
# get org name and domain from url
|
|
149
151
|
organization,instance_domain=self.class.parse_url(opt[:url])
|
|
150
|
-
# this is the base API url
|
|
151
|
-
api_base_url
|
|
152
|
-
#
|
|
153
|
-
aoc_rest_p[:base_url]="#{
|
|
152
|
+
# this is the base API url
|
|
153
|
+
api_url_base=self.class.api_base_url(instance_domain)
|
|
154
|
+
# API URL, including subpath (version ...)
|
|
155
|
+
aoc_rest_p[:base_url]="#{api_url_base}/#{opt[:subpath]}"
|
|
154
156
|
# base auth URL
|
|
155
|
-
aoc_auth_p[:base_url] = "#{
|
|
157
|
+
aoc_auth_p[:base_url] = "#{api_url_base}/#{OAUTH_API_SUBPATH}/#{organization}"
|
|
156
158
|
aoc_auth_p[:client_id]=opt[:client_id]
|
|
157
159
|
aoc_auth_p[:client_secret] = opt[:client_secret]
|
|
158
160
|
|
|
@@ -185,23 +187,19 @@ module Aspera
|
|
|
185
187
|
aoc_auth_p[:jwt_subject] = opt[:username]
|
|
186
188
|
aoc_auth_p[:jwt_private_key_obj] = OpenSSL::PKey::RSA.new(private_key_PEM_string)
|
|
187
189
|
when :url_token
|
|
190
|
+
aoc_auth_p[:password]=opt[:password] unless opt[:password].nil?
|
|
188
191
|
# nothing more
|
|
189
192
|
else raise "ERROR: unsupported auth method: #{aoc_auth_p[:grant]}"
|
|
190
193
|
end
|
|
191
194
|
super(aoc_rest_p)
|
|
192
195
|
end
|
|
193
196
|
|
|
194
|
-
def
|
|
195
|
-
@
|
|
196
|
-
|
|
197
|
+
def key_chain=(keychain)
|
|
198
|
+
raise "keychain already set" unless @key_chain.nil?
|
|
199
|
+
@key_chain=keychain
|
|
197
200
|
nil
|
|
198
201
|
end
|
|
199
202
|
|
|
200
|
-
def has_secret(ak)
|
|
201
|
-
Log.log.debug("has key:#{ak} -> #{@secrets.has_key?(ak)}")
|
|
202
|
-
return @secrets.has_key?(ak)
|
|
203
|
-
end
|
|
204
|
-
|
|
205
203
|
# additional transfer spec (tags) for package information
|
|
206
204
|
def self.package_tags(package_info,operation)
|
|
207
205
|
return {'tags'=>{'aspera'=>{'files'=>{
|
|
@@ -293,13 +291,14 @@ module Aspera
|
|
|
293
291
|
# no scope: requires secret
|
|
294
292
|
# if secret provided beforehand: use it
|
|
295
293
|
def get_node_api(node_info,node_scope=nil)
|
|
294
|
+
# X-Aspera-AccessKey required for bearer token only
|
|
296
295
|
node_rest_params={
|
|
297
296
|
:base_url => node_info['url'],
|
|
298
297
|
:headers => {'X-Aspera-AccessKey'=>node_info['access_key']},
|
|
299
298
|
}
|
|
300
|
-
ak_secret=@
|
|
299
|
+
ak_secret=@key_chain.get_secret(node_info['access_key'],false)
|
|
301
300
|
if ak_secret.nil? and node_scope.nil?
|
|
302
|
-
raise
|
|
301
|
+
raise "There must be at least one of: 'secret' or 'scope' for access key #{node_info['access_key']}"
|
|
303
302
|
end
|
|
304
303
|
# if secret provided on command line or if there is no scope
|
|
305
304
|
if !ak_secret.nil? or node_scope.nil?
|
|
@@ -312,7 +311,7 @@ module Aspera
|
|
|
312
311
|
node_rest_params[:auth]=self.params[:auth].clone
|
|
313
312
|
node_rest_params[:auth][:scope]=self.class.node_scope(node_info['access_key'],node_scope)
|
|
314
313
|
end
|
|
315
|
-
return
|
|
314
|
+
return Node.new(node_rest_params)
|
|
316
315
|
end
|
|
317
316
|
|
|
318
317
|
# check that parameter has necessary types
|
|
@@ -328,102 +327,82 @@ module Aspera
|
|
|
328
327
|
return node_info,file_id
|
|
329
328
|
end
|
|
330
329
|
|
|
331
|
-
#
|
|
332
|
-
def
|
|
333
|
-
|
|
334
|
-
|
|
330
|
+
# add entry to list if test block is success
|
|
331
|
+
def process_find_files(entry,path)
|
|
332
|
+
begin
|
|
333
|
+
# add to result if match filter
|
|
334
|
+
@find_state[:found].push(entry.merge({'path'=>path})) if @find_state[:test_block].call(entry)
|
|
335
|
+
# process link
|
|
336
|
+
if entry[:type].eql?('link')
|
|
337
|
+
sub_node_info=self.read("nodes/#{entry['target_node_id']}")[:data]
|
|
338
|
+
sub_opt={method: process_find_files, top_file_id: entry['target_id'], top_file_path: path}
|
|
339
|
+
get_node_api(sub_node_info,SCOPE_NODE_USER).crawl(self,sub_opt)
|
|
340
|
+
end
|
|
341
|
+
rescue => e
|
|
342
|
+
Log.log.error("#{path}: #{e.message}")
|
|
343
|
+
end
|
|
344
|
+
# process all folders
|
|
345
|
+
return true
|
|
335
346
|
end
|
|
336
347
|
|
|
337
|
-
# @returns list of file paths that match given regex
|
|
338
348
|
def find_files( top_node_file, test_block )
|
|
339
349
|
top_node_info,top_file_id=check_get_node_file(top_node_file)
|
|
340
350
|
Log.log.debug("find_files: node_info=#{top_node_info}, fileid=#{top_file_id}")
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
351
|
+
@find_state={found: [], test_block: test_block}
|
|
352
|
+
get_node_api(top_node_info,SCOPE_NODE_USER).crawl(self,{method: :process_find_files, top_file_id: top_file_id})
|
|
353
|
+
result=@find_state[:found]
|
|
354
|
+
@find_state=nil
|
|
355
|
+
return result
|
|
356
|
+
end
|
|
357
|
+
|
|
358
|
+
def process_resolve_node_file(entry,path)
|
|
359
|
+
# stop digging here if not in right path
|
|
360
|
+
return false unless entry['name'].eql?(@resolve_state[:path].first)
|
|
361
|
+
# ok it matches, so we remove the match
|
|
362
|
+
@resolve_state[:path].shift
|
|
363
|
+
case entry['type']
|
|
364
|
+
when 'file'
|
|
365
|
+
# file must be terminal
|
|
366
|
+
raise "#{entry['name']} is a file, expecting folder to find: #{@resolve_state[:path]}" unless @resolve_state[:path].empty?
|
|
367
|
+
@resolve_state[:result][:file_id]=entry['id']
|
|
368
|
+
when 'link'
|
|
369
|
+
@resolve_state[:result][:node_info]=self.read("nodes/#{entry['target_node_id']}")[:data]
|
|
370
|
+
if @resolve_state[:path].empty?
|
|
371
|
+
@resolve_state[:result][:file_id]=entry['target_id']
|
|
372
|
+
else
|
|
373
|
+
get_node_api(@resolve_state[:result][:node_info],SCOPE_NODE_USER).crawl(self,{method: :process_resolve_node_file, top_file_id: entry['target_id']})
|
|
356
374
|
end
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
begin
|
|
363
|
-
# does item match ?
|
|
364
|
-
result.push(current_file_info.merge({'path'=>item_path})) if test_block.call(current_file_info)
|
|
365
|
-
# does it need further processing ?
|
|
366
|
-
case current_file_info['type']
|
|
367
|
-
when 'file'
|
|
368
|
-
Log.log.debug("testing : #{current_file_info['name']}")
|
|
369
|
-
when 'folder'
|
|
370
|
-
items_to_explore.push({:node_api=>current_item[:node_api],:folder_id=>current_file_info['id'],:path=>item_path})
|
|
371
|
-
when 'link' # .*.asp-lnk
|
|
372
|
-
items_to_explore.push(read_asplnk(current_file_info).merge({:path=>item_path}))
|
|
373
|
-
else
|
|
374
|
-
Log.log.error("unknown folder item type: #{current_file_info['type']}")
|
|
375
|
-
end
|
|
376
|
-
rescue => e
|
|
377
|
-
Log.log.error("#{item_path}: #{e.message}")
|
|
378
|
-
end
|
|
375
|
+
when 'folder'
|
|
376
|
+
if @resolve_state[:path].empty?
|
|
377
|
+
# found: store
|
|
378
|
+
@resolve_state[:result][:file_id]=entry['id']
|
|
379
|
+
return false
|
|
379
380
|
end
|
|
381
|
+
else
|
|
382
|
+
Log.log.warn("unknown element type: #{entry['type']}")
|
|
380
383
|
end
|
|
381
|
-
|
|
384
|
+
# continue to dig folder
|
|
385
|
+
return true
|
|
382
386
|
end
|
|
383
387
|
|
|
384
|
-
# @return
|
|
388
|
+
# @return Array(node_info,file_id) for the given path
|
|
389
|
+
# @param top_node_file Array [root node,file id]
|
|
390
|
+
# @param element_path_string String path of element
|
|
385
391
|
# supports links to secondary nodes
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
current_node_api=get_node_api(current_node_info,SCOPE_NODE_USER) if current_node_api.nil?
|
|
399
|
-
# get folder content
|
|
400
|
-
folder_contents = current_node_api.read("files/#{current_file_id}/files")
|
|
401
|
-
Log.dump(:folder_contents,folder_contents)
|
|
402
|
-
matching_folders = folder_contents[:data].select { |i| i['name'].eql?(current_item)}
|
|
403
|
-
#Log.log.debug "matching_folders: #{matching_folders}"
|
|
404
|
-
raise "no such folder: #{current_item} in #{folder_contents[:data].map { |i| i['name']}}" if matching_folders.empty?
|
|
405
|
-
current_file_info = matching_folders.first
|
|
406
|
-
# process type of file
|
|
407
|
-
case current_file_info['type']
|
|
408
|
-
when 'file'
|
|
409
|
-
current_file_id=current_file_info['id']
|
|
410
|
-
# a file shall be terminal
|
|
411
|
-
if !items_to_explore.empty? then
|
|
412
|
-
raise "#{current_item} is a file, expecting folder to find: #{items_to_explore}"
|
|
413
|
-
end
|
|
414
|
-
when 'link'
|
|
415
|
-
current_node_info=self.read("nodes/#{current_file_info['target_node_id']}")[:data]
|
|
416
|
-
current_file_id=current_file_info['target_id']
|
|
417
|
-
# need to switch node
|
|
418
|
-
current_node_api=nil
|
|
419
|
-
when 'folder'
|
|
420
|
-
current_file_id=current_file_info['id']
|
|
421
|
-
else
|
|
422
|
-
Log.log.warn("unknown element type: #{current_file_info['type']}")
|
|
423
|
-
end
|
|
392
|
+
def resolve_node_file( top_node_file, element_path_string )
|
|
393
|
+
top_node_info,top_file_id=check_get_node_file(top_node_file)
|
|
394
|
+
path_elements=element_path_string.split(PATH_SEPARATOR).select{|i| !i.empty?}
|
|
395
|
+
result={node_info: top_node_info, file_id: nil}
|
|
396
|
+
if path_elements.empty?
|
|
397
|
+
result[:file_id]=top_file_id
|
|
398
|
+
else
|
|
399
|
+
@resolve_state={path: path_elements, result: result}
|
|
400
|
+
get_node_api(top_node_info,SCOPE_NODE_USER).crawl(self,{method: :process_resolve_node_file, top_file_id: top_file_id})
|
|
401
|
+
not_found=@resolve_state[:path]
|
|
402
|
+
@resolve_state=nil
|
|
403
|
+
raise "entry not found: #{not_found}" if result[:file_id].nil?
|
|
424
404
|
end
|
|
425
|
-
|
|
426
|
-
return {node_info: current_node_info, file_id: current_file_id}
|
|
405
|
+
return result
|
|
427
406
|
end
|
|
428
407
|
|
|
429
408
|
end # AoC
|
data/lib/aspera/cli/formater.rb
CHANGED
|
@@ -151,6 +151,8 @@ module Aspera
|
|
|
151
151
|
else
|
|
152
152
|
if user_asked_fields_list_str.start_with?('+')
|
|
153
153
|
result_default_fields(results,table_rows_hash_val).push(*user_asked_fields_list_str.gsub(/^\+/,'').split(','))
|
|
154
|
+
elsif user_asked_fields_list_str.start_with?('-')
|
|
155
|
+
result_default_fields(results,table_rows_hash_val).select{|i| ! user_asked_fields_list_str.gsub(/^\-/,'').split(',').include?(i)}
|
|
154
156
|
else
|
|
155
157
|
user_asked_fields_list_str.split(',')
|
|
156
158
|
end
|
data/lib/aspera/cli/main.rb
CHANGED
|
@@ -10,6 +10,7 @@ require 'aspera/persistency_folder'
|
|
|
10
10
|
require 'aspera/log'
|
|
11
11
|
require 'aspera/rest'
|
|
12
12
|
require 'aspera/nagios'
|
|
13
|
+
require 'aspera/secrets'
|
|
13
14
|
|
|
14
15
|
module Aspera
|
|
15
16
|
module Cli
|
|
@@ -21,18 +22,9 @@ module Aspera
|
|
|
21
22
|
# name of application, also foldername where config is stored
|
|
22
23
|
PROGRAM_NAME = 'ascli'
|
|
23
24
|
GEM_NAME = 'aspera-cli'
|
|
24
|
-
# Container module of current class : Aspera::Cli
|
|
25
|
-
CLI_MODULE=Module.nesting[1].to_s
|
|
26
|
-
# Path to Plugin classes: Aspera::Cli::Plugins
|
|
27
|
-
PLUGINS_MODULE=CLI_MODULE+'::Plugins'
|
|
28
25
|
VERBOSE_LEVELS=[:normal,:minimal,:quiet]
|
|
29
26
|
|
|
30
|
-
private_constant :PROGRAM_NAME,:GEM_NAME,:
|
|
31
|
-
|
|
32
|
-
# find the root folder of gem where this class is
|
|
33
|
-
def self.gem_root
|
|
34
|
-
File.expand_path(CLI_MODULE.to_s.gsub('::','/').gsub(%r([^/]+),'..'),File.dirname(__FILE__))
|
|
35
|
-
end
|
|
27
|
+
private_constant :PROGRAM_NAME,:GEM_NAME,:VERBOSE_LEVELS
|
|
36
28
|
|
|
37
29
|
# =============================================================
|
|
38
30
|
# Parameter handlers
|
|
@@ -62,13 +54,22 @@ module Aspera
|
|
|
62
54
|
@help_url='http://www.rubydoc.info/gems/'+GEM_NAME
|
|
63
55
|
@gem_url='https://rubygems.org/gems/'+GEM_NAME
|
|
64
56
|
# give command line arguments to option manager (no parsing)
|
|
65
|
-
|
|
57
|
+
app_main_folder=ENV[conf_dir_env_var]
|
|
58
|
+
# if env var undefined or empty
|
|
59
|
+
if app_main_folder.nil? or app_main_folder.empty?
|
|
60
|
+
user_home_folder=Dir.home
|
|
61
|
+
raise CliError,"Home folder does not exist: #{user_home_folder}. Check your user environment or use #{conf_dir_env_var}." unless Dir.exist?(user_home_folder)
|
|
62
|
+
app_main_folder=File.join(user_home_folder,Plugins::Config::ASPERA_HOME_FOLDER_NAME,PROGRAM_NAME)
|
|
63
|
+
end
|
|
64
|
+
@plugin_env[:options]=@opt_mgr=Manager.new(PROGRAM_NAME,argv,app_banner())
|
|
66
65
|
@plugin_env[:formater]=Formater.new(@plugin_env[:options])
|
|
67
|
-
Rest.user_agent=
|
|
66
|
+
Rest.user_agent=PROGRAM_NAME
|
|
68
67
|
# must override help methods before parser called (in other constructors)
|
|
69
68
|
init_global_options()
|
|
69
|
+
# secret manager
|
|
70
|
+
@plugin_env[:secret]=Aspera::Secrets.new
|
|
70
71
|
# the Config plugin adds the @preset parser
|
|
71
|
-
@plugin_env[:config]=Plugins::Config.new(@plugin_env,
|
|
72
|
+
@plugin_env[:config]=Plugins::Config.new(@plugin_env,PROGRAM_NAME,@help_url,Aspera::Cli::VERSION,app_main_folder)
|
|
72
73
|
# the TransferAgent plugin may use the @preset parser
|
|
73
74
|
@plugin_env[:transfer]=TransferAgent.new(@plugin_env)
|
|
74
75
|
Log.log.debug('created plugin env'.red)
|
|
@@ -82,21 +83,21 @@ module Aspera
|
|
|
82
83
|
end
|
|
83
84
|
|
|
84
85
|
def app_banner
|
|
85
|
-
banner = "NAME\n\t#{
|
|
86
|
+
banner = "NAME\n\t#{PROGRAM_NAME} -- a command line tool for Aspera Applications (v#{Aspera::Cli::VERSION})\n\n"
|
|
86
87
|
banner << "SYNOPSIS\n"
|
|
87
|
-
banner << "\t#{
|
|
88
|
-
banner << "\n"
|
|
89
|
-
banner << "DESCRIPTION\n"
|
|
88
|
+
banner << "\t#{PROGRAM_NAME} COMMANDS [OPTIONS] [ARGS]\n"
|
|
89
|
+
banner << "\nDESCRIPTION\n"
|
|
90
90
|
banner << "\tUse Aspera application to perform operations on command line.\n"
|
|
91
91
|
banner << "\tDocumentation and examples: #{@gem_url}\n"
|
|
92
|
-
banner << "\texecute: #{
|
|
92
|
+
banner << "\texecute: #{PROGRAM_NAME} conf doc\n"
|
|
93
93
|
banner << "\tor visit: #{@help_url}\n"
|
|
94
|
-
banner << "\n"
|
|
95
|
-
banner << "
|
|
96
|
-
banner << "\
|
|
94
|
+
banner << "\nENVIRONMENT VARIABLES\n"
|
|
95
|
+
banner << "\t#{conf_dir_env_var} config folder, default: $HOME/#{Plugins::Config::ASPERA_HOME_FOLDER_NAME}/#{PROGRAM_NAME}\n"
|
|
96
|
+
banner << "\t#any option can be set as an environment variable, refer to the manual\n"
|
|
97
|
+
banner << "\nCOMMANDS\n"
|
|
98
|
+
banner << "\tTo list first level commands, execute: #{PROGRAM_NAME}\n"
|
|
97
99
|
banner << "\tNote that commands can be written shortened (provided it is unique).\n"
|
|
98
|
-
banner << "\n"
|
|
99
|
-
banner << "OPTIONS\n"
|
|
100
|
+
banner << "\nOPTIONS\n"
|
|
100
101
|
banner << "\tOptions begin with a '-' (minus), and value is provided on command line.\n"
|
|
101
102
|
banner << "\tSpecial values are supported beginning with special prefix, like: #{ExtendedValue.instance.modifiers.map{|m|"@#{m}:"}.join(' ')}.\n"
|
|
102
103
|
banner << "\tDates format is 'DD-MM-YY HH:MM:SS', or 'now' or '-<num>h'\n\n"
|
|
@@ -140,7 +141,7 @@ module Aspera
|
|
|
140
141
|
require @plugin_env[:config].plugins[plugin_name_sym][:require_stanza]
|
|
141
142
|
# load default params only if no param already loaded before plugin instanciation
|
|
142
143
|
env[:config].add_plugin_default_preset(plugin_name_sym)
|
|
143
|
-
command_plugin=
|
|
144
|
+
command_plugin=Plugins::Config.plugin_new(plugin_name_sym,env)
|
|
144
145
|
Log.log.debug("got #{command_plugin.class}")
|
|
145
146
|
# TODO: check that ancestor is Plugin?
|
|
146
147
|
return command_plugin
|
|
@@ -176,7 +177,7 @@ module Aspera
|
|
|
176
177
|
# override main option parser with a brand new, to avoid having global options
|
|
177
178
|
plugin_env=@plugin_env.clone
|
|
178
179
|
plugin_env[:man_only]=true
|
|
179
|
-
plugin_env[:options]=Manager.new(
|
|
180
|
+
plugin_env[:options]=Manager.new(PROGRAM_NAME,[],'')
|
|
180
181
|
get_plugin_instance_with_options(plugin_name_sym,plugin_env)
|
|
181
182
|
# display generated help for plugin options
|
|
182
183
|
@plugin_env[:formater].display_message(:error,plugin_env[:options].parser.to_s)
|
|
@@ -187,10 +188,14 @@ module Aspera
|
|
|
187
188
|
|
|
188
189
|
protected
|
|
189
190
|
|
|
191
|
+
def conf_dir_env_var
|
|
192
|
+
return "#{PROGRAM_NAME}_home".upcase
|
|
193
|
+
end
|
|
194
|
+
|
|
190
195
|
# early debug for parser
|
|
191
196
|
# Note: does not accept shortcuts
|
|
192
197
|
def early_debug_setup(argv)
|
|
193
|
-
Log.instance.program_name=
|
|
198
|
+
Log.instance.program_name=PROGRAM_NAME
|
|
194
199
|
argv.each do |arg|
|
|
195
200
|
case arg
|
|
196
201
|
when '--'
|
|
@@ -214,10 +219,6 @@ module Aspera
|
|
|
214
219
|
return Main.result_nothing
|
|
215
220
|
end
|
|
216
221
|
|
|
217
|
-
def options;@opt_mgr;end
|
|
218
|
-
|
|
219
|
-
def program_name;PROGRAM_NAME;end
|
|
220
|
-
|
|
221
222
|
# this is the main function called by initial script just after constructor
|
|
222
223
|
def process_command_line
|
|
223
224
|
Log.log.debug('process_command_line')
|
|
@@ -231,16 +232,7 @@ module Aspera
|
|
|
231
232
|
# load global default options and process
|
|
232
233
|
@plugin_env[:config].add_plugin_default_preset(Plugins::Config::CONF_GLOBAL_SYM)
|
|
233
234
|
@opt_mgr.parse_options!
|
|
234
|
-
|
|
235
|
-
lock_port=@opt_mgr.get_option(:lock_port,:optional)
|
|
236
|
-
if !lock_port.nil?
|
|
237
|
-
begin
|
|
238
|
-
# no need to close later, will be freed on process exit. must save in member else it is garbage collected
|
|
239
|
-
@tcp_server=TCPServer.new('127.0.0.1',lock_port.to_i)
|
|
240
|
-
rescue => e
|
|
241
|
-
raise CliError,"Another instance is already running (lock port=#{lock_port})."
|
|
242
|
-
end
|
|
243
|
-
end
|
|
235
|
+
@plugin_env[:config].periodic_check_newer_gem_version
|
|
244
236
|
if @option_show_config and @opt_mgr.command_or_arg_empty?
|
|
245
237
|
command_sym=Plugins::Config::CONF_PLUGIN_SYM
|
|
246
238
|
else
|
|
@@ -264,6 +256,17 @@ module Aspera
|
|
|
264
256
|
@plugin_env[:formater].display_results({:type=>:single_object,:data=>@opt_mgr.declared_options(false)})
|
|
265
257
|
Process.exit(0)
|
|
266
258
|
end
|
|
259
|
+
# locking for singkle execution (only after "per plugin" option, in case lock port is there)
|
|
260
|
+
lock_port=@opt_mgr.get_option(:lock_port,:optional)
|
|
261
|
+
if !lock_port.nil?
|
|
262
|
+
begin
|
|
263
|
+
# no need to close later, will be freed on process exit. must save in member else it is garbage collected
|
|
264
|
+
Log.log.debug("Opening lock port #{lock_port.to_i}")
|
|
265
|
+
@tcp_server=TCPServer.new('127.0.0.1',lock_port.to_i)
|
|
266
|
+
rescue => e
|
|
267
|
+
raise CliError,"Another instance is already running (#{e.message})."
|
|
268
|
+
end
|
|
269
|
+
end
|
|
267
270
|
# execute and display
|
|
268
271
|
@plugin_env[:formater].display_results(command_plugin.execute_action)
|
|
269
272
|
# finish
|
|
@@ -271,11 +274,11 @@ module Aspera
|
|
|
271
274
|
rescue CliBadArgument => e; exception_info=[e,'Argument',:usage]
|
|
272
275
|
rescue CliNoSuchId => e; exception_info=[e,'Identifier']
|
|
273
276
|
rescue CliError => e; exception_info=[e,'Tool',:usage]
|
|
274
|
-
rescue Fasp::Error => e; exception_info=[e,
|
|
275
|
-
rescue Aspera::RestCallError => e;
|
|
276
|
-
rescue SocketError => e; exception_info=[e,
|
|
277
|
-
rescue StandardError => e; exception_info=[e,
|
|
278
|
-
rescue Interrupt => e; exception_info=[e,
|
|
277
|
+
rescue Fasp::Error => e; exception_info=[e,'FASP(ascp)']
|
|
278
|
+
rescue Aspera::RestCallError => e; exception_info=[e,'Rest']
|
|
279
|
+
rescue SocketError => e; exception_info=[e,'Network']
|
|
280
|
+
rescue StandardError => e; exception_info=[e,'Other',:debug]
|
|
281
|
+
rescue Interrupt => e; exception_info=[e,'Interruption',:debug]
|
|
279
282
|
end
|
|
280
283
|
# cleanup file list files
|
|
281
284
|
TempFileManager.instance.cleanup
|