aspera-cli 4.2.1 → 4.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +1580 -946
- data/bin/ascli +1 -1
- data/bin/asession +3 -5
- data/docs/Makefile +8 -11
- data/docs/README.erb.md +1521 -829
- data/docs/doc_tools.rb +58 -0
- data/docs/test_env.conf +3 -1
- data/examples/faspex4.rb +28 -19
- data/examples/transfer.rb +2 -2
- data/lib/aspera/aoc.rb +157 -134
- data/lib/aspera/cli/listener/progress_multi.rb +5 -5
- data/lib/aspera/cli/main.rb +106 -48
- data/lib/aspera/cli/manager.rb +19 -20
- data/lib/aspera/cli/plugin.rb +22 -7
- data/lib/aspera/cli/plugins/aoc.rb +260 -208
- data/lib/aspera/cli/plugins/ats.rb +11 -10
- data/lib/aspera/cli/plugins/bss.rb +2 -2
- data/lib/aspera/cli/plugins/config.rb +360 -189
- data/lib/aspera/cli/plugins/faspex.rb +119 -56
- data/lib/aspera/cli/plugins/faspex5.rb +32 -17
- data/lib/aspera/cli/plugins/node.rb +72 -31
- data/lib/aspera/cli/plugins/orchestrator.rb +5 -3
- data/lib/aspera/cli/plugins/preview.rb +94 -68
- data/lib/aspera/cli/plugins/server.rb +16 -5
- data/lib/aspera/cli/plugins/shares.rb +17 -0
- data/lib/aspera/cli/transfer_agent.rb +64 -82
- data/lib/aspera/cli/version.rb +1 -1
- data/lib/aspera/command_line_builder.rb +48 -31
- data/lib/aspera/cos_node.rb +4 -3
- data/lib/aspera/environment.rb +4 -4
- data/lib/aspera/fasp/{manager.rb → agent_base.rb} +7 -6
- data/lib/aspera/fasp/{connect.rb → agent_connect.rb} +46 -39
- data/lib/aspera/fasp/{local.rb → agent_direct.rb} +42 -38
- data/lib/aspera/fasp/{http_gw.rb → agent_httpgw.rb} +50 -29
- data/lib/aspera/fasp/{node.rb → agent_node.rb} +43 -4
- data/lib/aspera/fasp/agent_trsdk.rb +106 -0
- data/lib/aspera/fasp/default.rb +17 -0
- data/lib/aspera/fasp/installation.rb +64 -48
- data/lib/aspera/fasp/parameters.rb +78 -91
- data/lib/aspera/fasp/parameters.yaml +531 -0
- data/lib/aspera/fasp/uri.rb +1 -1
- data/lib/aspera/faspex_gw.rb +12 -11
- data/lib/aspera/id_generator.rb +22 -0
- data/lib/aspera/keychain/encrypted_hash.rb +120 -0
- data/lib/aspera/keychain/macos_security.rb +94 -0
- data/lib/aspera/log.rb +45 -32
- data/lib/aspera/node.rb +9 -4
- data/lib/aspera/oauth.rb +116 -100
- data/lib/aspera/persistency_action_once.rb +11 -7
- data/lib/aspera/persistency_folder.rb +6 -26
- data/lib/aspera/rest.rb +66 -50
- data/lib/aspera/sync.rb +40 -35
- data/lib/aspera/timer_limiter.rb +22 -0
- metadata +86 -29
- data/docs/transfer_spec.html +0 -99
- data/lib/aspera/api_detector.rb +0 -60
- data/lib/aspera/fasp/aoc.rb +0 -24
- data/lib/aspera/secrets.rb +0 -20
@@ -7,8 +7,10 @@ require 'aspera/persistency_action_once'
|
|
7
7
|
require 'aspera/open_application'
|
8
8
|
require 'aspera/fasp/uri'
|
9
9
|
require 'aspera/nagios'
|
10
|
+
require 'aspera/id_generator'
|
10
11
|
require 'xmlsimple'
|
11
12
|
require 'json'
|
13
|
+
require 'cgi'
|
12
14
|
|
13
15
|
module Aspera
|
14
16
|
module Cli
|
@@ -17,7 +19,33 @@ module Aspera
|
|
17
19
|
KEY_NODE='node'
|
18
20
|
KEY_PATH='path'
|
19
21
|
VAL_ALL='ALL'
|
20
|
-
|
22
|
+
# added field in result that identifies the package
|
23
|
+
PACKAGE_MATCH_FIELD='package_id'
|
24
|
+
# list of supported atoms
|
25
|
+
ATOM_MAILBOXES=[:inbox, :archive, :sent]
|
26
|
+
# allowed parameters for inbox.atom
|
27
|
+
ATOM_PARAMS=['page', 'count', 'startIndex']
|
28
|
+
# with special parameters (from Plugin class)
|
29
|
+
ATOM_EXT_PARAMS=ATOM_PARAMS+[MAX_ITEMS, MAX_PAGES]
|
30
|
+
# sub path in url for public link delivery
|
31
|
+
PUB_LINK_EXTERNAL_MATCH='external_deliveries/'
|
32
|
+
private_constant :KEY_NODE,:KEY_PATH,:VAL_ALL,:PACKAGE_MATCH_FIELD,:ATOM_MAILBOXES,
|
33
|
+
:ATOM_PARAMS,:ATOM_EXT_PARAMS,:PUB_LINK_EXTERNAL_MATCH
|
34
|
+
|
35
|
+
class << self
|
36
|
+
def detect(base_url)
|
37
|
+
api=Rest.new({:base_url=>base_url})
|
38
|
+
result=api.call({:operation=>'POST',:subpath=>'aspera/faspex',:headers=>{'Accept'=>'application/xrds+xml'},:text_body_params=>''})
|
39
|
+
# 4.x
|
40
|
+
if result[:http].body.start_with?('<?xml')
|
41
|
+
res_s=XmlSimple.xml_in(result[:http].body, {'ForceArray' => false})
|
42
|
+
version=res_s['XRD']['application']['version']
|
43
|
+
return {version: version}
|
44
|
+
end
|
45
|
+
return nil
|
46
|
+
end
|
47
|
+
end
|
48
|
+
|
21
49
|
def initialize(env)
|
22
50
|
@api_v3=nil
|
23
51
|
@api_v4=nil
|
@@ -27,7 +55,7 @@ module Aspera
|
|
27
55
|
self.options.add_opt_simple(:source_name,'create package from remote source (by name)')
|
28
56
|
self.options.add_opt_simple(:storage,'Faspex local storage definition')
|
29
57
|
self.options.add_opt_simple(:recipient,'use if recipient is a dropbox (with *)')
|
30
|
-
self.options.add_opt_list(:box,
|
58
|
+
self.options.add_opt_list(:box,ATOM_MAILBOXES,'package box')
|
31
59
|
self.options.set_option(:box,:inbox)
|
32
60
|
self.options.parse_options!
|
33
61
|
end
|
@@ -102,29 +130,64 @@ module Aspera
|
|
102
130
|
return @api_v4
|
103
131
|
end
|
104
132
|
|
105
|
-
|
106
|
-
|
107
|
-
# we match recv command on atom feed on this field
|
108
|
-
PACKAGE_MATCH_FIELD='package_id'
|
109
|
-
|
133
|
+
# query supports : {"startIndex":10,"count":1,"page":109}
|
110
134
|
def mailbox_all_entries
|
111
135
|
recipient_name=self.options.get_option(:recipient,:optional) || self.options.get_option(:username,:mandatory)
|
136
|
+
# mailbox is in ATOM_MAILBOXES
|
112
137
|
mailbox=self.options.get_option(:box,:mandatory)
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
result
|
118
|
-
|
119
|
-
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
|
138
|
+
# parameters
|
139
|
+
mailbox_query=self.options.get_option(:query,:optional)
|
140
|
+
max_items=nil
|
141
|
+
max_pages=nil
|
142
|
+
result=[]
|
143
|
+
if !mailbox_query.nil?
|
144
|
+
raise "query: must be Hash or nil" unless mailbox_query.is_a?(Hash)
|
145
|
+
raise "query: supported params: #{ATOM_EXT_PARAMS}" unless (mailbox_query.keys-ATOM_EXT_PARAMS).empty?
|
146
|
+
raise "query: startIndex and page are exclusive" if mailbox_query.has_key?('startIndex') and mailbox_query.has_key?('page')
|
147
|
+
max_items=mailbox_query[MAX_ITEMS]
|
148
|
+
mailbox_query.delete(MAX_ITEMS)
|
149
|
+
max_pages=mailbox_query[MAX_PAGES]
|
150
|
+
mailbox_query.delete(MAX_PAGES)
|
151
|
+
end
|
152
|
+
loop do
|
153
|
+
# get a batch of package information
|
154
|
+
# order: first batch is latest packages, and then in a batch ids are increasing
|
155
|
+
atom_xml=api_v3.call({operation: 'GET',subpath: "#{mailbox}.atom",headers: {'Accept'=>'application/xml'},url_params: mailbox_query})[:http].body
|
156
|
+
box_data=XmlSimple.xml_in(atom_xml, {'ForceArray' => true})
|
157
|
+
Log.dump(:box_data,box_data)
|
158
|
+
items=box_data.has_key?('entry') ? box_data['entry'] : []
|
159
|
+
Log.log.debug("new items: #{items.count}")
|
160
|
+
# it is the end if page is empty
|
161
|
+
break if items.empty?
|
162
|
+
# results will be sorted in reverse id
|
163
|
+
items.reverse.each do |package|
|
164
|
+
package[PACKAGE_MATCH_FIELD]=case mailbox
|
165
|
+
when :inbox,:archive
|
166
|
+
recipient=package['to'].select{|i|i['name'].first.eql?(recipient_name)}.first
|
167
|
+
recipient.nil? ? nil : recipient['recipient_delivery_id'].first
|
168
|
+
else # :sent
|
169
|
+
package['delivery_id'].first
|
170
|
+
end
|
171
|
+
# keep only those for the specified recipient,
|
172
|
+
result.push(package) unless package[PACKAGE_MATCH_FIELD].nil?
|
124
173
|
end
|
174
|
+
#result.push({PACKAGE_MATCH_FIELD=>'======'})
|
175
|
+
Log.log.debug("total items: #{result.count}")
|
176
|
+
# reach the limit ?
|
177
|
+
if !max_items.nil? and result.count >= max_items
|
178
|
+
result=result.slice(0,max_items) if result.count > max_items
|
179
|
+
break
|
180
|
+
end
|
181
|
+
link=box_data['link'].select{|i|i['rel'].eql?('next')}.first
|
182
|
+
Log.log.debug("link: #{link}")
|
183
|
+
# no next link
|
184
|
+
break if link.nil?
|
185
|
+
# replace parameters with the ones from next link
|
186
|
+
params=CGI.parse(URI.parse(link['href']).query)
|
187
|
+
mailbox_query=params.keys.inject({}){|m,i|;m[i]=params[i].first;m}
|
188
|
+
Log.log.debug("query: #{mailbox_query}")
|
189
|
+
break if !max_pages.nil? and mailbox_query['page'].to_i > max_pages
|
125
190
|
end
|
126
|
-
# remove dropbox packages
|
127
|
-
result.select!{|p|p['metadata'].first['field'].select{|j|j['name'].eql?('_dropbox_name')}.empty? rescue false}
|
128
191
|
return result
|
129
192
|
end
|
130
193
|
|
@@ -160,6 +223,8 @@ module Aspera
|
|
160
223
|
return pkgdatares.first
|
161
224
|
end
|
162
225
|
|
226
|
+
ACTIONS=[ :health,:package, :source, :me, :dropbox, :v4, :address_book, :login_methods ]
|
227
|
+
|
163
228
|
def execute_action
|
164
229
|
command=self.options.get_next_command(ACTIONS)
|
165
230
|
case command
|
@@ -215,44 +280,27 @@ module Aspera
|
|
215
280
|
skip_ids_data=[]
|
216
281
|
skip_ids_persistency=nil
|
217
282
|
case link_url
|
218
|
-
when nil
|
219
|
-
# usual case: no link
|
220
|
-
when /^faspe:/
|
221
|
-
pkg_id_uri=[{:id=>'package',:uri=>link_url}]
|
222
|
-
else
|
223
|
-
link_data=self.class.get_link_data(link_url)
|
224
|
-
if !link_data[:subpath].match(%r{external_deliveries/})
|
225
|
-
raise CliBadArgument,"pub link is #{link_data[:subpath]}, expecting external_deliveries/"
|
226
|
-
end
|
227
|
-
# Note: unauthenticated API
|
228
|
-
api_public_link=Rest.new({:base_url=>link_data[:base_url]})
|
229
|
-
pkgdatares=api_public_link.call({:operation=>'GET',:subpath=>link_data[:subpath],:url_params=>{:passcode=>link_data[:query]['passcode']},:headers=>{'Accept'=>'application/xml'}})
|
230
|
-
if !pkgdatares[:http].body.start_with?('<?xml ')
|
231
|
-
OpenApplication.instance.uri(link_url)
|
232
|
-
raise CliError, 'no such package'
|
233
|
-
end
|
234
|
-
package_entry=XmlSimple.xml_in(pkgdatares[:http].body, {'ForceArray' => false})
|
235
|
-
transfer_uri=self.class.get_fasp_uri_from_entry(package_entry)
|
236
|
-
transfer_spec=Fasp::Uri.new(transfer_uri).transfer_spec
|
237
|
-
transfer_spec['direction']='receive'
|
238
|
-
return Main.result_transfer(self.transfer.start(transfer_spec,{:src=>:node_gen3}))
|
239
|
-
end # public link
|
240
|
-
if pkg_id_uri.nil?
|
241
|
-
# get command line parameters
|
242
|
-
delivid=self.options.get_option(:id,:mandatory)
|
283
|
+
when nil # usual case: no link
|
243
284
|
if self.options.get_option(:once_only,:mandatory)
|
244
285
|
skip_ids_persistency=PersistencyActionOnce.new(
|
245
286
|
manager: @agents[:persistency],
|
246
|
-
data:
|
247
|
-
|
287
|
+
data: skip_ids_data,
|
288
|
+
id: IdGenerator.from_list(['faspex_recv',self.options.get_option(:url,:mandatory),self.options.get_option(:username,:mandatory),self.options.get_option(:box,:mandatory).to_s]))
|
248
289
|
end
|
290
|
+
# get command line parameters
|
291
|
+
delivid=self.instance_identifier()
|
292
|
+
raise "empty id" if delivid.empty?
|
249
293
|
if delivid.eql?(VAL_ALL)
|
250
294
|
pkg_id_uri=mailbox_all_entries.map{|i|{:id=>i[PACKAGE_MATCH_FIELD],:uri=>self.class.get_fasp_uri_from_entry(i)}}
|
251
|
-
# TODO : remove ids from skip not present in inbox
|
295
|
+
# TODO : remove ids from skip not present in inbox to avoid growing too big
|
252
296
|
# skip_ids_data.select!{|id|pkg_id_uri.select{|p|p[:id].eql?(id)}}
|
253
297
|
pkg_id_uri.select!{|i|!skip_ids_data.include?(i[:id])}
|
254
298
|
else
|
255
|
-
|
299
|
+
recipient=options.get_option(:recipient,:optional)
|
300
|
+
if !recipient.nil? and recipient.start_with?('*')
|
301
|
+
raise "Dropbox and Workgroup packages should use link option with faspe:"
|
302
|
+
end
|
303
|
+
# TODO: delivery id is the right one if package was receive by workgroup
|
256
304
|
endpoint=case self.options.get_option(:box,:mandatory)
|
257
305
|
when :inbox,:archive;'received'
|
258
306
|
when :sent; 'sent'
|
@@ -261,7 +309,25 @@ module Aspera
|
|
261
309
|
package_entry=XmlSimple.xml_in(entry_xml, {'ForceArray' => true})
|
262
310
|
pkg_id_uri=[{:id=>delivid,:uri=>self.class.get_fasp_uri_from_entry(package_entry)}]
|
263
311
|
end
|
264
|
-
|
312
|
+
when /^faspe:/
|
313
|
+
pkg_id_uri=[{:id=>'package',:uri=>link_url}]
|
314
|
+
else
|
315
|
+
link_data=self.class.get_link_data(link_url)
|
316
|
+
if !link_data[:subpath].start_with?(PUB_LINK_EXTERNAL_MATCH)
|
317
|
+
raise CliBadArgument,"Pub link is #{link_data[:subpath]}. Expecting #{PUB_LINK_EXTERNAL_MATCH}"
|
318
|
+
end
|
319
|
+
# Note: unauthenticated API (authorization is in url params)
|
320
|
+
api_public_link=Rest.new({:base_url=>link_data[:base_url]})
|
321
|
+
pkgdatares=api_public_link.call({:operation=>'GET',:subpath=>link_data[:subpath],:url_params=>{:passcode=>link_data[:query]['passcode']},:headers=>{'Accept'=>'application/xml'}})
|
322
|
+
if !pkgdatares[:http].body.start_with?('<?xml ')
|
323
|
+
OpenApplication.instance.uri(link_url)
|
324
|
+
raise CliError, 'no such package'
|
325
|
+
end
|
326
|
+
package_entry=XmlSimple.xml_in(pkgdatares[:http].body, {'ForceArray' => false})
|
327
|
+
Log.dump(:package_entry,package_entry)
|
328
|
+
transfer_uri=self.class.get_fasp_uri_from_entry(package_entry)
|
329
|
+
pkg_id_uri=[{:id=>package_entry['id'],:uri=>transfer_uri}]
|
330
|
+
end # public link
|
265
331
|
Log.dump(:pkg_id_uri,pkg_id_uri)
|
266
332
|
return Main.result_status('no package') if pkg_id_uri.empty?
|
267
333
|
result_transfer=[]
|
@@ -270,20 +336,17 @@ module Aspera
|
|
270
336
|
# NOTE: only external users have token in faspe: link !
|
271
337
|
if !transfer_spec.has_key?('token')
|
272
338
|
sanitized=id_uri[:uri].gsub('&','&')
|
273
|
-
# TODO: file jira
|
274
|
-
#XXsanitized.gsub!(/%3D%3D$/,'==')
|
275
|
-
#XXsanitized.gsub!(/%3D$/,'=')
|
276
339
|
xmlpayload='<?xml version="1.0" encoding="UTF-8"?><url-list xmlns="http://schemas.asperasoft.com/xml/url-list"><url href="'+sanitized+'"/></url-list>'
|
277
340
|
transfer_spec['token']=api_v3.call({:operation=>'POST',:subpath=>'issue-token?direction=down',:headers=>{'Accept'=>'text/plain','Content-Type'=>'application/vnd.aspera.url-list+xml'},:text_body_params=>xmlpayload})[:http].body
|
278
341
|
end
|
279
342
|
transfer_spec['direction']='receive'
|
280
343
|
statuses=self.transfer.start(transfer_spec,{:src=>:node_gen3})
|
281
|
-
result_transfer.push({'package'=>id_uri[:id],
|
344
|
+
result_transfer.push({'package'=>id_uri[:id],Main::STATUS_FIELD=>statuses})
|
282
345
|
# skip only if all sessions completed
|
283
346
|
skip_ids_data.push(id_uri[:id]) if TransferAgent.session_status(statuses).eql?(:success)
|
284
347
|
end
|
285
348
|
skip_ids_persistency.save unless skip_ids_persistency.nil?
|
286
|
-
return
|
349
|
+
return Main.result_transfer_multiple(result_transfer)
|
287
350
|
end
|
288
351
|
when :source
|
289
352
|
command_source=self.options.get_next_command([ :list, :id, :name ])
|
@@ -359,7 +422,7 @@ module Aspera
|
|
359
422
|
return self.entity_action(api_v4,'metadata_profiles',nil,:id)
|
360
423
|
when :package
|
361
424
|
pkg_box_type=self.options.get_next_command([:users])
|
362
|
-
pkg_box_id=self.
|
425
|
+
pkg_box_id=self.instance_identifier()
|
363
426
|
return self.entity_action(api_v4,"#{pkg_box_type}/#{pkg_box_id}/packages",nil,:id)
|
364
427
|
end
|
365
428
|
when :address_book
|
@@ -1,27 +1,41 @@
|
|
1
1
|
require 'aspera/cli/basic_auth_plugin'
|
2
2
|
require 'aspera/persistency_action_once'
|
3
|
+
require 'aspera/id_generator'
|
3
4
|
require 'securerandom'
|
4
5
|
|
5
6
|
module Aspera
|
6
7
|
module Cli
|
7
8
|
module Plugins
|
8
9
|
class Faspex5 < BasicAuthPlugin
|
10
|
+
class << self
|
11
|
+
def detect(base_url)
|
12
|
+
api=Rest.new({:base_url=>base_url})
|
13
|
+
result=api.read('api/v5/configuration/ping')
|
14
|
+
if result[:http].code.start_with?('2') and result[:http].body.strip.empty?
|
15
|
+
return {version: '5'}
|
16
|
+
end
|
17
|
+
return nil
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
9
21
|
VAL_ALL='ALL'
|
22
|
+
private_constant :VAL_ALL
|
23
|
+
|
10
24
|
def initialize(env)
|
11
25
|
super(env)
|
12
|
-
options.add_opt_simple(:client_id,'
|
13
|
-
options.add_opt_simple(:client_secret,'
|
14
|
-
options.add_opt_simple(:redirect_uri,'
|
15
|
-
options.add_opt_list(:auth,Oauth.auth_types.clone.push(:boot),'type of
|
16
|
-
options.add_opt_simple(:private_key,'RSA private key PEM value for JWT (prefix file path with @val:@file:)')
|
26
|
+
options.add_opt_simple(:client_id,'OAuth client identifier')
|
27
|
+
options.add_opt_simple(:client_secret,'OAuth client secret')
|
28
|
+
options.add_opt_simple(:redirect_uri,'OAuth redirect URI')
|
29
|
+
options.add_opt_list(:auth,Oauth.auth_types.clone.push(:boot),'OAuth type of authentication')
|
30
|
+
options.add_opt_simple(:private_key,'Oauth RSA private key PEM value for JWT (prefix file path with @val:@file:)')
|
17
31
|
options.set_option(:auth,:jwt)
|
18
32
|
options.parse_options!
|
19
33
|
end
|
20
34
|
|
21
35
|
def set_api
|
22
|
-
faxpex5_api_base_url=options.get_option(:url,:mandatory)
|
23
|
-
faxpex5_api_v5_url="#{faxpex5_api_base_url}/api/v5"
|
24
|
-
faxpex5_api_auth_url="#{faxpex5_api_base_url}/auth"
|
36
|
+
@faxpex5_api_base_url=options.get_option(:url,:mandatory)
|
37
|
+
faxpex5_api_v5_url="#{@faxpex5_api_base_url}/api/v5"
|
38
|
+
faxpex5_api_auth_url="#{@faxpex5_api_base_url}/auth"
|
25
39
|
case options.get_option(:auth,:mandatory)
|
26
40
|
when :boot
|
27
41
|
# the password here is the token copied directly from browser in developer mode
|
@@ -42,7 +56,7 @@ module Aspera
|
|
42
56
|
:redirect_uri => options.get_option(:redirect_uri,:mandatory),
|
43
57
|
}})
|
44
58
|
when :jwt
|
45
|
-
# currently Faspex 5 beta
|
59
|
+
# currently Faspex 5 beta 4 only supports non-user based apis (e.g. jobs)
|
46
60
|
app_client_id=options.get_option(:client_id,:mandatory)
|
47
61
|
@api_v5=Rest.new({
|
48
62
|
:base_url => faxpex5_api_v5_url,
|
@@ -50,12 +64,13 @@ module Aspera
|
|
50
64
|
:type => :oauth2,
|
51
65
|
:base_url => faxpex5_api_auth_url,
|
52
66
|
:grant => :jwt,
|
67
|
+
:f5_username => options.get_option(:username,:mandatory),
|
68
|
+
:f5_password => options.get_option(:password,:mandatory),
|
53
69
|
:client_id => app_client_id,
|
54
70
|
:client_secret => options.get_option(:client_secret,:mandatory),
|
55
71
|
:jwt_subject => "client:#{app_client_id}", # TODO Mmmm
|
56
72
|
:jwt_audience => app_client_id, # TODO Mmmm
|
57
73
|
:jwt_private_key_obj => OpenSSL::PKey::RSA.new(options.get_option(:private_key,:mandatory)),
|
58
|
-
:jwt_is_f5 => true, # TODO: remove when clarified
|
59
74
|
:jwt_headers => {typ: 'JWT'}
|
60
75
|
}})
|
61
76
|
end
|
@@ -69,7 +84,7 @@ module Aspera
|
|
69
84
|
command=options.get_next_command(ACTIONS)
|
70
85
|
case command
|
71
86
|
when :auth_client
|
72
|
-
api_auth=Rest.new(@api_v5.params.merge({base_url: @
|
87
|
+
api_auth=Rest.new(@api_v5.params.merge({base_url: "#{@faxpex5_api_base_url}/auth"}))
|
73
88
|
return self.entity_action(api_auth,'oauth_clients',nil,:id,nil,true)
|
74
89
|
when :node
|
75
90
|
return self.entity_action(@api_v5,'nodes',nil,:id,nil,true)
|
@@ -83,7 +98,7 @@ module Aspera
|
|
83
98
|
parameters=options.get_option(:value,:optional)
|
84
99
|
return {:type => :object_list, :data=>@api_v5.read('packages',parameters)[:data]['packages']}
|
85
100
|
when :show
|
86
|
-
id=
|
101
|
+
id=instance_identifier()
|
87
102
|
return {:type => :single_object, :data=>@api_v5.read("packages/#{id}")[:data]}
|
88
103
|
when :send
|
89
104
|
parameters=options.get_option(:value,:mandatory)
|
@@ -94,7 +109,7 @@ module Aspera
|
|
94
109
|
return Main.result_transfer(self.transfer.start(transfer_spec,{:src=>:node_gen3}))
|
95
110
|
when :receive
|
96
111
|
pkg_type='received'
|
97
|
-
pack_id=
|
112
|
+
pack_id=instance_identifier()
|
98
113
|
package_ids=[pack_id]
|
99
114
|
skip_ids_data=[]
|
100
115
|
skip_ids_persistency=nil
|
@@ -102,8 +117,8 @@ module Aspera
|
|
102
117
|
# read ids from persistency
|
103
118
|
skip_ids_persistency=PersistencyActionOnce.new(
|
104
119
|
manager: @agents[:persistency],
|
105
|
-
data:
|
106
|
-
|
120
|
+
data: skip_ids_data,
|
121
|
+
id: IdGenerator.from_list(['faspex_recv',options.get_option(:url,:mandatory),options.get_option(:username,:mandatory),pkg_type]))
|
107
122
|
end
|
108
123
|
if pack_id.eql?(VAL_ALL)
|
109
124
|
# TODO: if packages have same name, they will overwrite
|
@@ -119,12 +134,12 @@ module Aspera
|
|
119
134
|
transfer_spec=@api_v5.create("packages/#{id}/transfer_spec/download",{transfer_type: 'Connect', type: pkg_type})[:data]
|
120
135
|
transfer_spec.delete('authentication')
|
121
136
|
statuses=self.transfer.start(transfer_spec,{:src=>:node_gen3})
|
122
|
-
result_transfer.push({'package'=>id,
|
137
|
+
result_transfer.push({'package'=>id,Main::STATUS_FIELD=>statuses})
|
123
138
|
# skip only if all sessions completed
|
124
139
|
skip_ids_data.push(id) if TransferAgent.session_status(statuses).eql?(:success)
|
125
140
|
end
|
126
141
|
skip_ids_persistency.save unless skip_ids_persistency.nil?
|
127
|
-
return
|
142
|
+
return Main.result_transfer_multiple(result_transfer)
|
128
143
|
end
|
129
144
|
end
|
130
145
|
end
|
@@ -1,6 +1,8 @@
|
|
1
1
|
require 'aspera/cli/basic_auth_plugin'
|
2
2
|
require 'aspera/nagios'
|
3
3
|
require 'aspera/hash_ext'
|
4
|
+
require 'aspera/id_generator'
|
5
|
+
require 'aspera/node'
|
4
6
|
require 'base64'
|
5
7
|
require 'zlib'
|
6
8
|
|
@@ -8,26 +10,49 @@ module Aspera
|
|
8
10
|
module Cli
|
9
11
|
module Plugins
|
10
12
|
class Node < BasicAuthPlugin
|
13
|
+
class << self
|
14
|
+
def detect(base_url)
|
15
|
+
api=Rest.new({:base_url=>base_url})
|
16
|
+
result=api.call({:operation=>'GET',:subpath=>'ping'})
|
17
|
+
if result[:http].body.eql?('')
|
18
|
+
return {:product=>:node,:version=>'unknown'}
|
19
|
+
end
|
20
|
+
return nil
|
21
|
+
end
|
22
|
+
end
|
11
23
|
SAMPLE_SOAP_CALL='<?xml version="1.0" encoding="UTF-8"?><soapenv:Envelope xmlns:soapenv="http://schemas.xmlsoap.org/soap/envelope/" xmlns:typ="urn:Aspera:XML:FASPSessionNET:2009/11:Types"><soapenv:Header></soapenv:Header><soapenv:Body><typ:GetSessionInfoRequest><SessionFilter><SessionStatus>running</SessionStatus></SessionFilter></typ:GetSessionInfoRequest></soapenv:Body></soapenv:Envelope>'
|
12
24
|
private_constant :SAMPLE_SOAP_CALL
|
25
|
+
|
13
26
|
def initialize(env)
|
14
27
|
super(env)
|
15
|
-
# this is added to some requests , for instance to add tags
|
28
|
+
# this is added to some requests , for instance to add tags (COS)
|
16
29
|
@add_request_param = env[:add_request_param] || {}
|
17
30
|
unless env[:skip_basic_auth_options]
|
18
31
|
self.options.add_opt_simple(:validator,"identifier of validator (optional for central)")
|
19
32
|
self.options.add_opt_simple(:asperabrowserurl,"URL for simple aspera web ui")
|
20
33
|
self.options.add_opt_simple(:name,"sync name")
|
21
|
-
self.options.add_opt_list(:
|
34
|
+
self.options.add_opt_list(:token_type,[:aspera,:basic,:hybrid],'Type of token used for transfers')
|
22
35
|
self.options.set_option(:asperabrowserurl,'https://asperabrowser.mybluemix.net')
|
23
|
-
self.options.set_option(:
|
36
|
+
self.options.set_option(:token_type,:aspera)
|
24
37
|
self.options.parse_options!
|
25
38
|
end
|
26
39
|
return if env[:man_only]
|
27
40
|
if env.has_key?(:node_api)
|
28
41
|
@api_node=env[:node_api]
|
29
42
|
else
|
30
|
-
|
43
|
+
if self.options.get_option(:password,:mandatory).start_with?('Bearer ')
|
44
|
+
# info is provided like node_info of aoc
|
45
|
+
@api_node=Rest.new({
|
46
|
+
base_url: self.options.get_option(:url,:mandatory),
|
47
|
+
headers: {
|
48
|
+
'Authorization' => self.options.get_option(:password,:mandatory),
|
49
|
+
'X-Aspera-AccessKey' => self.options.get_option(:username,:mandatory),
|
50
|
+
}
|
51
|
+
})
|
52
|
+
else
|
53
|
+
# this is normal case
|
54
|
+
@api_node=basic_auth_api
|
55
|
+
end
|
31
56
|
end
|
32
57
|
end
|
33
58
|
|
@@ -152,7 +177,7 @@ module Aspera
|
|
152
177
|
when :mkdir
|
153
178
|
path_list=get_next_arg_add_prefix(prefix_path,"folder path or ext.val. list")
|
154
179
|
path_list=[path_list] unless path_list.is_a?(Array)
|
155
|
-
#TODO
|
180
|
+
#TODO: a command for that ?
|
156
181
|
#resp=@api_node.create('space',{ "paths" => path_list.map {|i| {:type=>:directory,:path=>i} } } )
|
157
182
|
resp=@api_node.create('files/create',{ "paths" => [{ :type => :directory, :path => path_list } ] } )
|
158
183
|
return c_result_translate_rem_prefix(resp,'folder','created',prefix_path)
|
@@ -190,22 +215,36 @@ module Aspera
|
|
190
215
|
#raise "unknown type: #{send_result['self']['type']}"
|
191
216
|
end
|
192
217
|
return c_result_remove_prefix_path(result,'path',prefix_path)
|
193
|
-
when :upload
|
194
|
-
|
195
|
-
|
196
|
-
|
197
|
-
|
198
|
-
|
199
|
-
|
200
|
-
|
201
|
-
|
202
|
-
|
203
|
-
|
204
|
-
|
205
|
-
|
206
|
-
|
207
|
-
|
208
|
-
|
218
|
+
when :upload,:download
|
219
|
+
token_type=self.options.get_option(:token_type,:optional)
|
220
|
+
# nil if Shares 1.x
|
221
|
+
token_type=:aspera if token_type.nil?
|
222
|
+
case token_type
|
223
|
+
when :aspera,:hybrid
|
224
|
+
transfer_paths=case command
|
225
|
+
when :upload;[ { :destination => self.transfer.destination_folder('send') } ]
|
226
|
+
when :download;self.transfer.ts_source_paths
|
227
|
+
end
|
228
|
+
# only one request, so only one answer
|
229
|
+
transfer_spec=@api_node.create("files/#{command}_setup",{:transfer_requests => [ { transfer_request: {
|
230
|
+
paths: transfer_paths
|
231
|
+
}.deep_merge(@add_request_param) } ] } )[:data]['transfer_specs'].first['transfer_spec']
|
232
|
+
# delete this part, as the returned value contains only destination, and not sources
|
233
|
+
transfer_spec.delete('paths') if command.eql?(:upload)
|
234
|
+
when :basic
|
235
|
+
raise "shall have auth" unless @api_node.params[:auth].is_a?(Hash)
|
236
|
+
raise "shall be basic auth" unless @api_node.params[:auth][:type].eql?(:basic)
|
237
|
+
transfer_spec={
|
238
|
+
'remote_host'=>URI.parse(@api_node.params[:base_url]).host,
|
239
|
+
'remote_user'=>Aspera::Fasp::Default::ACCESS_KEY_TRANSFER_USER,
|
240
|
+
'ssh_port' =>Aspera::Fasp::Default::SSH_PORT,
|
241
|
+
'direction' =>case command;when :upload;'send';when :download;'recv';else raise "Error";end
|
242
|
+
}
|
243
|
+
else raise "ERROR: token_type #{tt}"
|
244
|
+
end
|
245
|
+
if [:basic,:hybrid].include?(token_type)
|
246
|
+
Aspera::Node.set_ak_basic_token(transfer_spec,@api_node.params[:auth][:username],@api_node.params[:auth][:password])
|
247
|
+
end
|
209
248
|
return Main.result_transfer(self.transfer.start(transfer_spec,{:src=>:node_gen3}))
|
210
249
|
when :api_details
|
211
250
|
return { :type=>:single_object, :data => @api_node.params }
|
@@ -217,7 +256,7 @@ module Aspera
|
|
217
256
|
unless command.eql?(:list)
|
218
257
|
asyncname=self.options.get_option(:name,:optional)
|
219
258
|
if asyncname.nil?
|
220
|
-
asyncid=self.
|
259
|
+
asyncid=self.instance_identifier()
|
221
260
|
if asyncid.eql?('ALL') and [:show,:delete].include?(command)
|
222
261
|
asyncids=@api_node.read('async/list')[:data]['sync_ids']
|
223
262
|
else
|
@@ -271,8 +310,8 @@ module Aspera
|
|
271
310
|
if self.options.get_option(:once_only,:mandatory)
|
272
311
|
skip_ids_persistency=PersistencyActionOnce.new(
|
273
312
|
manager: @agents[:persistency],
|
274
|
-
data:
|
275
|
-
|
313
|
+
data: iteration_data,
|
314
|
+
id: IdGenerator.from_list(['sync_files',self.options.get_option(:url,:mandatory),self.options.get_option(:username,:mandatory),asyncid]))
|
276
315
|
unless iteration_data.first.nil?
|
277
316
|
data.select!{|l| l['fnid'].to_i>iteration_data.first}
|
278
317
|
end
|
@@ -300,7 +339,7 @@ module Aspera
|
|
300
339
|
case command
|
301
340
|
when :list
|
302
341
|
resp=@api_node.read('ops/transfers',self.options.get_option(:value,:optional))
|
303
|
-
return { :type => :object_list, :data => resp[:data], :fields=>['id','status'] } # TODO
|
342
|
+
return { :type => :object_list, :data => resp[:data], :fields=>['id','status'] } # TODO: useful?
|
304
343
|
when :create
|
305
344
|
resp=@api_node.create('streams',self.options.get_option(:value,:mandatory))
|
306
345
|
return { :type => :single_object, :data => resp[:data] }
|
@@ -323,7 +362,7 @@ module Aspera
|
|
323
362
|
command=self.options.get_next_command([ :list, :cancel, :show ])
|
324
363
|
res_class_path='ops/transfers'
|
325
364
|
if [:cancel, :show].include?(command)
|
326
|
-
one_res_id=self.
|
365
|
+
one_res_id=self.instance_identifier()
|
327
366
|
one_res_path="#{res_class_path}/#{one_res_id}"
|
328
367
|
end
|
329
368
|
case command
|
@@ -345,7 +384,7 @@ module Aspera
|
|
345
384
|
when :service
|
346
385
|
command=self.options.get_next_command([ :list, :create, :delete])
|
347
386
|
if [:delete].include?(command)
|
348
|
-
svcid=self.
|
387
|
+
svcid=self.instance_identifier()
|
349
388
|
end
|
350
389
|
case command
|
351
390
|
when :list
|
@@ -365,7 +404,7 @@ module Aspera
|
|
365
404
|
#return entity_action(@api_node,'v3/watchfolders',nil,:id)
|
366
405
|
command=self.options.get_next_command([ :create, :list, :show, :modify, :delete, :state])
|
367
406
|
if [:show,:modify,:delete,:state].include?(command)
|
368
|
-
one_res_id=self.
|
407
|
+
one_res_id=self.instance_identifier()
|
369
408
|
one_res_path="#{res_class_path}/#{one_res_id}"
|
370
409
|
end
|
371
410
|
# hum, to avoid: Unable to convert 2016_09_14 configuration
|
@@ -408,9 +447,11 @@ module Aspera
|
|
408
447
|
command=self.options.get_next_command([ :list, :modify])
|
409
448
|
case command
|
410
449
|
when :list
|
411
|
-
request_data.deep_merge!({
|
412
|
-
resp=@api_node.create('services/rest/transfers/v1/files',request_data)
|
413
|
-
|
450
|
+
request_data.deep_merge!({'validation'=>validation}) unless validation.nil?
|
451
|
+
resp=@api_node.create('services/rest/transfers/v1/files',request_data)[:data]
|
452
|
+
resp=JSON.parse(resp) if resp.is_a?(String)
|
453
|
+
Log.dump(:resp,resp)
|
454
|
+
return {:type=>:object_list,:data=>resp['file_transfer_info_result']['file_transfer_info'],:fields=>["session_uuid","file_id","status","path"]}
|
414
455
|
when :modify
|
415
456
|
request_data.deep_merge!(validation) unless validation.nil?
|
416
457
|
@api_node.update('services/rest/transfers/v1/files',request_data)
|
@@ -110,12 +110,14 @@ module Aspera
|
|
110
110
|
return {:type=>:object_list,:data=>result['Plugin']}
|
111
111
|
when :workflow
|
112
112
|
command=self.options.get_next_command([:list, :status, :inputs, :details, :start, :export])
|
113
|
-
unless [:list
|
114
|
-
wf_id=self.
|
113
|
+
unless [:list].include?(command)
|
114
|
+
wf_id=self.instance_identifier()
|
115
115
|
end
|
116
116
|
case command
|
117
117
|
when :status
|
118
|
-
|
118
|
+
options={}
|
119
|
+
options[:id]=wf_id unless wf_id.eql?('ALL')
|
120
|
+
result=call_API('workflows_status',options)[:data]
|
119
121
|
return {:type=>:object_list,:data=>result['workflows']['workflow']}
|
120
122
|
when :list
|
121
123
|
result=call_API('workflows_list',id: 0)[:data]
|