aspera-cli 4.2.1 → 4.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (59) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +1580 -946
  3. data/bin/ascli +1 -1
  4. data/bin/asession +3 -5
  5. data/docs/Makefile +8 -11
  6. data/docs/README.erb.md +1521 -829
  7. data/docs/doc_tools.rb +58 -0
  8. data/docs/test_env.conf +3 -1
  9. data/examples/faspex4.rb +28 -19
  10. data/examples/transfer.rb +2 -2
  11. data/lib/aspera/aoc.rb +157 -134
  12. data/lib/aspera/cli/listener/progress_multi.rb +5 -5
  13. data/lib/aspera/cli/main.rb +106 -48
  14. data/lib/aspera/cli/manager.rb +19 -20
  15. data/lib/aspera/cli/plugin.rb +22 -7
  16. data/lib/aspera/cli/plugins/aoc.rb +260 -208
  17. data/lib/aspera/cli/plugins/ats.rb +11 -10
  18. data/lib/aspera/cli/plugins/bss.rb +2 -2
  19. data/lib/aspera/cli/plugins/config.rb +360 -189
  20. data/lib/aspera/cli/plugins/faspex.rb +119 -56
  21. data/lib/aspera/cli/plugins/faspex5.rb +32 -17
  22. data/lib/aspera/cli/plugins/node.rb +72 -31
  23. data/lib/aspera/cli/plugins/orchestrator.rb +5 -3
  24. data/lib/aspera/cli/plugins/preview.rb +94 -68
  25. data/lib/aspera/cli/plugins/server.rb +16 -5
  26. data/lib/aspera/cli/plugins/shares.rb +17 -0
  27. data/lib/aspera/cli/transfer_agent.rb +64 -82
  28. data/lib/aspera/cli/version.rb +1 -1
  29. data/lib/aspera/command_line_builder.rb +48 -31
  30. data/lib/aspera/cos_node.rb +4 -3
  31. data/lib/aspera/environment.rb +4 -4
  32. data/lib/aspera/fasp/{manager.rb → agent_base.rb} +7 -6
  33. data/lib/aspera/fasp/{connect.rb → agent_connect.rb} +46 -39
  34. data/lib/aspera/fasp/{local.rb → agent_direct.rb} +42 -38
  35. data/lib/aspera/fasp/{http_gw.rb → agent_httpgw.rb} +50 -29
  36. data/lib/aspera/fasp/{node.rb → agent_node.rb} +43 -4
  37. data/lib/aspera/fasp/agent_trsdk.rb +106 -0
  38. data/lib/aspera/fasp/default.rb +17 -0
  39. data/lib/aspera/fasp/installation.rb +64 -48
  40. data/lib/aspera/fasp/parameters.rb +78 -91
  41. data/lib/aspera/fasp/parameters.yaml +531 -0
  42. data/lib/aspera/fasp/uri.rb +1 -1
  43. data/lib/aspera/faspex_gw.rb +12 -11
  44. data/lib/aspera/id_generator.rb +22 -0
  45. data/lib/aspera/keychain/encrypted_hash.rb +120 -0
  46. data/lib/aspera/keychain/macos_security.rb +94 -0
  47. data/lib/aspera/log.rb +45 -32
  48. data/lib/aspera/node.rb +9 -4
  49. data/lib/aspera/oauth.rb +116 -100
  50. data/lib/aspera/persistency_action_once.rb +11 -7
  51. data/lib/aspera/persistency_folder.rb +6 -26
  52. data/lib/aspera/rest.rb +66 -50
  53. data/lib/aspera/sync.rb +40 -35
  54. data/lib/aspera/timer_limiter.rb +22 -0
  55. metadata +86 -29
  56. data/docs/transfer_spec.html +0 -99
  57. data/lib/aspera/api_detector.rb +0 -60
  58. data/lib/aspera/fasp/aoc.rb +0 -24
  59. data/lib/aspera/secrets.rb +0 -20
@@ -7,8 +7,10 @@ require 'aspera/persistency_action_once'
7
7
  require 'aspera/open_application'
8
8
  require 'aspera/fasp/uri'
9
9
  require 'aspera/nagios'
10
+ require 'aspera/id_generator'
10
11
  require 'xmlsimple'
11
12
  require 'json'
13
+ require 'cgi'
12
14
 
13
15
  module Aspera
14
16
  module Cli
@@ -17,7 +19,33 @@ module Aspera
17
19
  KEY_NODE='node'
18
20
  KEY_PATH='path'
19
21
  VAL_ALL='ALL'
20
- private_constant :KEY_NODE,:KEY_PATH,:VAL_ALL
22
+ # added field in result that identifies the package
23
+ PACKAGE_MATCH_FIELD='package_id'
24
+ # list of supported atoms
25
+ ATOM_MAILBOXES=[:inbox, :archive, :sent]
26
+ # allowed parameters for inbox.atom
27
+ ATOM_PARAMS=['page', 'count', 'startIndex']
28
+ # with special parameters (from Plugin class)
29
+ ATOM_EXT_PARAMS=ATOM_PARAMS+[MAX_ITEMS, MAX_PAGES]
30
+ # sub path in url for public link delivery
31
+ PUB_LINK_EXTERNAL_MATCH='external_deliveries/'
32
+ private_constant :KEY_NODE,:KEY_PATH,:VAL_ALL,:PACKAGE_MATCH_FIELD,:ATOM_MAILBOXES,
33
+ :ATOM_PARAMS,:ATOM_EXT_PARAMS,:PUB_LINK_EXTERNAL_MATCH
34
+
35
+ class << self
36
+ def detect(base_url)
37
+ api=Rest.new({:base_url=>base_url})
38
+ result=api.call({:operation=>'POST',:subpath=>'aspera/faspex',:headers=>{'Accept'=>'application/xrds+xml'},:text_body_params=>''})
39
+ # 4.x
40
+ if result[:http].body.start_with?('<?xml')
41
+ res_s=XmlSimple.xml_in(result[:http].body, {'ForceArray' => false})
42
+ version=res_s['XRD']['application']['version']
43
+ return {version: version}
44
+ end
45
+ return nil
46
+ end
47
+ end
48
+
21
49
  def initialize(env)
22
50
  @api_v3=nil
23
51
  @api_v4=nil
@@ -27,7 +55,7 @@ module Aspera
27
55
  self.options.add_opt_simple(:source_name,'create package from remote source (by name)')
28
56
  self.options.add_opt_simple(:storage,'Faspex local storage definition')
29
57
  self.options.add_opt_simple(:recipient,'use if recipient is a dropbox (with *)')
30
- self.options.add_opt_list(:box,[:inbox,:sent,:archive],'package box')
58
+ self.options.add_opt_list(:box,ATOM_MAILBOXES,'package box')
31
59
  self.options.set_option(:box,:inbox)
32
60
  self.options.parse_options!
33
61
  end
@@ -102,29 +130,64 @@ module Aspera
102
130
  return @api_v4
103
131
  end
104
132
 
105
- ACTIONS=[ :health,:package, :source, :me, :dropbox, :v4, :address_book, :login_methods ]
106
-
107
- # we match recv command on atom feed on this field
108
- PACKAGE_MATCH_FIELD='package_id'
109
-
133
+ # query supports : {"startIndex":10,"count":1,"page":109}
110
134
  def mailbox_all_entries
111
135
  recipient_name=self.options.get_option(:recipient,:optional) || self.options.get_option(:username,:mandatory)
136
+ # mailbox is in ATOM_MAILBOXES
112
137
  mailbox=self.options.get_option(:box,:mandatory)
113
- all_inbox_xml=api_v3.call({:operation=>'GET',:subpath=>"#{mailbox}.atom",:headers=>{'Accept'=>'application/xml'}})[:http].body
114
- all_inbox_data=XmlSimple.xml_in(all_inbox_xml, {'ForceArray' => true})
115
- Log.dump(:all_inbox_data,all_inbox_data)
116
- result=all_inbox_data.has_key?('entry') ? all_inbox_data['entry'] : []
117
- result.each do |e|
118
- case mailbox
119
- when :inbox,:archive
120
- recipient=e['to'].select{|i|i['name'].first.eql?(recipient_name)}.first
121
- e[PACKAGE_MATCH_FIELD]=recipient.nil? ? 'n/a' : recipient['recipient_delivery_id'].first
122
- when :sent
123
- e[PACKAGE_MATCH_FIELD]=e['delivery_id'].first
138
+ # parameters
139
+ mailbox_query=self.options.get_option(:query,:optional)
140
+ max_items=nil
141
+ max_pages=nil
142
+ result=[]
143
+ if !mailbox_query.nil?
144
+ raise "query: must be Hash or nil" unless mailbox_query.is_a?(Hash)
145
+ raise "query: supported params: #{ATOM_EXT_PARAMS}" unless (mailbox_query.keys-ATOM_EXT_PARAMS).empty?
146
+ raise "query: startIndex and page are exclusive" if mailbox_query.has_key?('startIndex') and mailbox_query.has_key?('page')
147
+ max_items=mailbox_query[MAX_ITEMS]
148
+ mailbox_query.delete(MAX_ITEMS)
149
+ max_pages=mailbox_query[MAX_PAGES]
150
+ mailbox_query.delete(MAX_PAGES)
151
+ end
152
+ loop do
153
+ # get a batch of package information
154
+ # order: first batch is latest packages, and then in a batch ids are increasing
155
+ atom_xml=api_v3.call({operation: 'GET',subpath: "#{mailbox}.atom",headers: {'Accept'=>'application/xml'},url_params: mailbox_query})[:http].body
156
+ box_data=XmlSimple.xml_in(atom_xml, {'ForceArray' => true})
157
+ Log.dump(:box_data,box_data)
158
+ items=box_data.has_key?('entry') ? box_data['entry'] : []
159
+ Log.log.debug("new items: #{items.count}")
160
+ # it is the end if page is empty
161
+ break if items.empty?
162
+ # results will be sorted in reverse id
163
+ items.reverse.each do |package|
164
+ package[PACKAGE_MATCH_FIELD]=case mailbox
165
+ when :inbox,:archive
166
+ recipient=package['to'].select{|i|i['name'].first.eql?(recipient_name)}.first
167
+ recipient.nil? ? nil : recipient['recipient_delivery_id'].first
168
+ else # :sent
169
+ package['delivery_id'].first
170
+ end
171
+ # keep only those for the specified recipient,
172
+ result.push(package) unless package[PACKAGE_MATCH_FIELD].nil?
124
173
  end
174
+ #result.push({PACKAGE_MATCH_FIELD=>'======'})
175
+ Log.log.debug("total items: #{result.count}")
176
+ # reach the limit ?
177
+ if !max_items.nil? and result.count >= max_items
178
+ result=result.slice(0,max_items) if result.count > max_items
179
+ break
180
+ end
181
+ link=box_data['link'].select{|i|i['rel'].eql?('next')}.first
182
+ Log.log.debug("link: #{link}")
183
+ # no next link
184
+ break if link.nil?
185
+ # replace parameters with the ones from next link
186
+ params=CGI.parse(URI.parse(link['href']).query)
187
+ mailbox_query=params.keys.inject({}){|m,i|;m[i]=params[i].first;m}
188
+ Log.log.debug("query: #{mailbox_query}")
189
+ break if !max_pages.nil? and mailbox_query['page'].to_i > max_pages
125
190
  end
126
- # remove dropbox packages
127
- result.select!{|p|p['metadata'].first['field'].select{|j|j['name'].eql?('_dropbox_name')}.empty? rescue false}
128
191
  return result
129
192
  end
130
193
 
@@ -160,6 +223,8 @@ module Aspera
160
223
  return pkgdatares.first
161
224
  end
162
225
 
226
+ ACTIONS=[ :health,:package, :source, :me, :dropbox, :v4, :address_book, :login_methods ]
227
+
163
228
  def execute_action
164
229
  command=self.options.get_next_command(ACTIONS)
165
230
  case command
@@ -215,44 +280,27 @@ module Aspera
215
280
  skip_ids_data=[]
216
281
  skip_ids_persistency=nil
217
282
  case link_url
218
- when nil
219
- # usual case: no link
220
- when /^faspe:/
221
- pkg_id_uri=[{:id=>'package',:uri=>link_url}]
222
- else
223
- link_data=self.class.get_link_data(link_url)
224
- if !link_data[:subpath].match(%r{external_deliveries/})
225
- raise CliBadArgument,"pub link is #{link_data[:subpath]}, expecting external_deliveries/"
226
- end
227
- # Note: unauthenticated API
228
- api_public_link=Rest.new({:base_url=>link_data[:base_url]})
229
- pkgdatares=api_public_link.call({:operation=>'GET',:subpath=>link_data[:subpath],:url_params=>{:passcode=>link_data[:query]['passcode']},:headers=>{'Accept'=>'application/xml'}})
230
- if !pkgdatares[:http].body.start_with?('<?xml ')
231
- OpenApplication.instance.uri(link_url)
232
- raise CliError, 'no such package'
233
- end
234
- package_entry=XmlSimple.xml_in(pkgdatares[:http].body, {'ForceArray' => false})
235
- transfer_uri=self.class.get_fasp_uri_from_entry(package_entry)
236
- transfer_spec=Fasp::Uri.new(transfer_uri).transfer_spec
237
- transfer_spec['direction']='receive'
238
- return Main.result_transfer(self.transfer.start(transfer_spec,{:src=>:node_gen3}))
239
- end # public link
240
- if pkg_id_uri.nil?
241
- # get command line parameters
242
- delivid=self.options.get_option(:id,:mandatory)
283
+ when nil # usual case: no link
243
284
  if self.options.get_option(:once_only,:mandatory)
244
285
  skip_ids_persistency=PersistencyActionOnce.new(
245
286
  manager: @agents[:persistency],
246
- data: skip_ids_data,
247
- ids: ['faspex_recv',self.options.get_option(:url,:mandatory),self.options.get_option(:username,:mandatory),self.options.get_option(:box,:mandatory).to_s])
287
+ data: skip_ids_data,
288
+ id: IdGenerator.from_list(['faspex_recv',self.options.get_option(:url,:mandatory),self.options.get_option(:username,:mandatory),self.options.get_option(:box,:mandatory).to_s]))
248
289
  end
290
+ # get command line parameters
291
+ delivid=self.instance_identifier()
292
+ raise "empty id" if delivid.empty?
249
293
  if delivid.eql?(VAL_ALL)
250
294
  pkg_id_uri=mailbox_all_entries.map{|i|{:id=>i[PACKAGE_MATCH_FIELD],:uri=>self.class.get_fasp_uri_from_entry(i)}}
251
- # TODO : remove ids from skip not present in inbox
295
+ # TODO : remove ids from skip not present in inbox to avoid growing too big
252
296
  # skip_ids_data.select!{|id|pkg_id_uri.select{|p|p[:id].eql?(id)}}
253
297
  pkg_id_uri.select!{|i|!skip_ids_data.include?(i[:id])}
254
298
  else
255
- # TODO: delivery id is the right one if package was receive by group
299
+ recipient=options.get_option(:recipient,:optional)
300
+ if !recipient.nil? and recipient.start_with?('*')
301
+ raise "Dropbox and Workgroup packages should use link option with faspe:"
302
+ end
303
+ # TODO: delivery id is the right one if package was receive by workgroup
256
304
  endpoint=case self.options.get_option(:box,:mandatory)
257
305
  when :inbox,:archive;'received'
258
306
  when :sent; 'sent'
@@ -261,7 +309,25 @@ module Aspera
261
309
  package_entry=XmlSimple.xml_in(entry_xml, {'ForceArray' => true})
262
310
  pkg_id_uri=[{:id=>delivid,:uri=>self.class.get_fasp_uri_from_entry(package_entry)}]
263
311
  end
264
- end
312
+ when /^faspe:/
313
+ pkg_id_uri=[{:id=>'package',:uri=>link_url}]
314
+ else
315
+ link_data=self.class.get_link_data(link_url)
316
+ if !link_data[:subpath].start_with?(PUB_LINK_EXTERNAL_MATCH)
317
+ raise CliBadArgument,"Pub link is #{link_data[:subpath]}. Expecting #{PUB_LINK_EXTERNAL_MATCH}"
318
+ end
319
+ # Note: unauthenticated API (authorization is in url params)
320
+ api_public_link=Rest.new({:base_url=>link_data[:base_url]})
321
+ pkgdatares=api_public_link.call({:operation=>'GET',:subpath=>link_data[:subpath],:url_params=>{:passcode=>link_data[:query]['passcode']},:headers=>{'Accept'=>'application/xml'}})
322
+ if !pkgdatares[:http].body.start_with?('<?xml ')
323
+ OpenApplication.instance.uri(link_url)
324
+ raise CliError, 'no such package'
325
+ end
326
+ package_entry=XmlSimple.xml_in(pkgdatares[:http].body, {'ForceArray' => false})
327
+ Log.dump(:package_entry,package_entry)
328
+ transfer_uri=self.class.get_fasp_uri_from_entry(package_entry)
329
+ pkg_id_uri=[{:id=>package_entry['id'],:uri=>transfer_uri}]
330
+ end # public link
265
331
  Log.dump(:pkg_id_uri,pkg_id_uri)
266
332
  return Main.result_status('no package') if pkg_id_uri.empty?
267
333
  result_transfer=[]
@@ -270,20 +336,17 @@ module Aspera
270
336
  # NOTE: only external users have token in faspe: link !
271
337
  if !transfer_spec.has_key?('token')
272
338
  sanitized=id_uri[:uri].gsub('&','&amp;')
273
- # TODO: file jira
274
- #XXsanitized.gsub!(/%3D%3D$/,'==')
275
- #XXsanitized.gsub!(/%3D$/,'=')
276
339
  xmlpayload='<?xml version="1.0" encoding="UTF-8"?><url-list xmlns="http://schemas.asperasoft.com/xml/url-list"><url href="'+sanitized+'"/></url-list>'
277
340
  transfer_spec['token']=api_v3.call({:operation=>'POST',:subpath=>'issue-token?direction=down',:headers=>{'Accept'=>'text/plain','Content-Type'=>'application/vnd.aspera.url-list+xml'},:text_body_params=>xmlpayload})[:http].body
278
341
  end
279
342
  transfer_spec['direction']='receive'
280
343
  statuses=self.transfer.start(transfer_spec,{:src=>:node_gen3})
281
- result_transfer.push({'package'=>id_uri[:id],'status'=>statuses.map{|i|i.to_s}.join(',')})
344
+ result_transfer.push({'package'=>id_uri[:id],Main::STATUS_FIELD=>statuses})
282
345
  # skip only if all sessions completed
283
346
  skip_ids_data.push(id_uri[:id]) if TransferAgent.session_status(statuses).eql?(:success)
284
347
  end
285
348
  skip_ids_persistency.save unless skip_ids_persistency.nil?
286
- return {:type=>:object_list,:data=>result_transfer}
349
+ return Main.result_transfer_multiple(result_transfer)
287
350
  end
288
351
  when :source
289
352
  command_source=self.options.get_next_command([ :list, :id, :name ])
@@ -359,7 +422,7 @@ module Aspera
359
422
  return self.entity_action(api_v4,'metadata_profiles',nil,:id)
360
423
  when :package
361
424
  pkg_box_type=self.options.get_next_command([:users])
362
- pkg_box_id=self.options.get_option(:id,:mandatory)
425
+ pkg_box_id=self.instance_identifier()
363
426
  return self.entity_action(api_v4,"#{pkg_box_type}/#{pkg_box_id}/packages",nil,:id)
364
427
  end
365
428
  when :address_book
@@ -1,27 +1,41 @@
1
1
  require 'aspera/cli/basic_auth_plugin'
2
2
  require 'aspera/persistency_action_once'
3
+ require 'aspera/id_generator'
3
4
  require 'securerandom'
4
5
 
5
6
  module Aspera
6
7
  module Cli
7
8
  module Plugins
8
9
  class Faspex5 < BasicAuthPlugin
10
+ class << self
11
+ def detect(base_url)
12
+ api=Rest.new({:base_url=>base_url})
13
+ result=api.read('api/v5/configuration/ping')
14
+ if result[:http].code.start_with?('2') and result[:http].body.strip.empty?
15
+ return {version: '5'}
16
+ end
17
+ return nil
18
+ end
19
+ end
20
+
9
21
  VAL_ALL='ALL'
22
+ private_constant :VAL_ALL
23
+
10
24
  def initialize(env)
11
25
  super(env)
12
- options.add_opt_simple(:client_id,'API client identifier in application')
13
- options.add_opt_simple(:client_secret,'API client secret in application')
14
- options.add_opt_simple(:redirect_uri,'API client redirect URI')
15
- options.add_opt_list(:auth,Oauth.auth_types.clone.push(:boot),'type of Oauth authentication')
16
- options.add_opt_simple(:private_key,'RSA private key PEM value for JWT (prefix file path with @val:@file:)')
26
+ options.add_opt_simple(:client_id,'OAuth client identifier')
27
+ options.add_opt_simple(:client_secret,'OAuth client secret')
28
+ options.add_opt_simple(:redirect_uri,'OAuth redirect URI')
29
+ options.add_opt_list(:auth,Oauth.auth_types.clone.push(:boot),'OAuth type of authentication')
30
+ options.add_opt_simple(:private_key,'Oauth RSA private key PEM value for JWT (prefix file path with @val:@file:)')
17
31
  options.set_option(:auth,:jwt)
18
32
  options.parse_options!
19
33
  end
20
34
 
21
35
  def set_api
22
- faxpex5_api_base_url=options.get_option(:url,:mandatory)
23
- faxpex5_api_v5_url="#{faxpex5_api_base_url}/api/v5"
24
- faxpex5_api_auth_url="#{faxpex5_api_base_url}/auth"
36
+ @faxpex5_api_base_url=options.get_option(:url,:mandatory)
37
+ faxpex5_api_v5_url="#{@faxpex5_api_base_url}/api/v5"
38
+ faxpex5_api_auth_url="#{@faxpex5_api_base_url}/auth"
25
39
  case options.get_option(:auth,:mandatory)
26
40
  when :boot
27
41
  # the password here is the token copied directly from browser in developer mode
@@ -42,7 +56,7 @@ module Aspera
42
56
  :redirect_uri => options.get_option(:redirect_uri,:mandatory),
43
57
  }})
44
58
  when :jwt
45
- # currently Faspex 5 beta 3 only supports non-user based apis (e.g. jobs)
59
+ # currently Faspex 5 beta 4 only supports non-user based apis (e.g. jobs)
46
60
  app_client_id=options.get_option(:client_id,:mandatory)
47
61
  @api_v5=Rest.new({
48
62
  :base_url => faxpex5_api_v5_url,
@@ -50,12 +64,13 @@ module Aspera
50
64
  :type => :oauth2,
51
65
  :base_url => faxpex5_api_auth_url,
52
66
  :grant => :jwt,
67
+ :f5_username => options.get_option(:username,:mandatory),
68
+ :f5_password => options.get_option(:password,:mandatory),
53
69
  :client_id => app_client_id,
54
70
  :client_secret => options.get_option(:client_secret,:mandatory),
55
71
  :jwt_subject => "client:#{app_client_id}", # TODO Mmmm
56
72
  :jwt_audience => app_client_id, # TODO Mmmm
57
73
  :jwt_private_key_obj => OpenSSL::PKey::RSA.new(options.get_option(:private_key,:mandatory)),
58
- :jwt_is_f5 => true, # TODO: remove when clarified
59
74
  :jwt_headers => {typ: 'JWT'}
60
75
  }})
61
76
  end
@@ -69,7 +84,7 @@ module Aspera
69
84
  command=options.get_next_command(ACTIONS)
70
85
  case command
71
86
  when :auth_client
72
- api_auth=Rest.new(@api_v5.params.merge({base_url: @api_v5.params[:base_url].gsub(/api\/v5$/,'auth')}))
87
+ api_auth=Rest.new(@api_v5.params.merge({base_url: "#{@faxpex5_api_base_url}/auth"}))
73
88
  return self.entity_action(api_auth,'oauth_clients',nil,:id,nil,true)
74
89
  when :node
75
90
  return self.entity_action(@api_v5,'nodes',nil,:id,nil,true)
@@ -83,7 +98,7 @@ module Aspera
83
98
  parameters=options.get_option(:value,:optional)
84
99
  return {:type => :object_list, :data=>@api_v5.read('packages',parameters)[:data]['packages']}
85
100
  when :show
86
- id=options.get_option(:id,:mandatory)
101
+ id=instance_identifier()
87
102
  return {:type => :single_object, :data=>@api_v5.read("packages/#{id}")[:data]}
88
103
  when :send
89
104
  parameters=options.get_option(:value,:mandatory)
@@ -94,7 +109,7 @@ module Aspera
94
109
  return Main.result_transfer(self.transfer.start(transfer_spec,{:src=>:node_gen3}))
95
110
  when :receive
96
111
  pkg_type='received'
97
- pack_id=options.get_option(:id,:mandatory)
112
+ pack_id=instance_identifier()
98
113
  package_ids=[pack_id]
99
114
  skip_ids_data=[]
100
115
  skip_ids_persistency=nil
@@ -102,8 +117,8 @@ module Aspera
102
117
  # read ids from persistency
103
118
  skip_ids_persistency=PersistencyActionOnce.new(
104
119
  manager: @agents[:persistency],
105
- data: skip_ids_data,
106
- ids: ['faspex_recv',options.get_option(:url,:mandatory),options.get_option(:username,:mandatory),pkg_type])
120
+ data: skip_ids_data,
121
+ id: IdGenerator.from_list(['faspex_recv',options.get_option(:url,:mandatory),options.get_option(:username,:mandatory),pkg_type]))
107
122
  end
108
123
  if pack_id.eql?(VAL_ALL)
109
124
  # TODO: if packages have same name, they will overwrite
@@ -119,12 +134,12 @@ module Aspera
119
134
  transfer_spec=@api_v5.create("packages/#{id}/transfer_spec/download",{transfer_type: 'Connect', type: pkg_type})[:data]
120
135
  transfer_spec.delete('authentication')
121
136
  statuses=self.transfer.start(transfer_spec,{:src=>:node_gen3})
122
- result_transfer.push({'package'=>id,'status'=>statuses.map{|i|i.to_s}.join(',')})
137
+ result_transfer.push({'package'=>id,Main::STATUS_FIELD=>statuses})
123
138
  # skip only if all sessions completed
124
139
  skip_ids_data.push(id) if TransferAgent.session_status(statuses).eql?(:success)
125
140
  end
126
141
  skip_ids_persistency.save unless skip_ids_persistency.nil?
127
- return {:type=>:object_list,:data=>result_transfer}
142
+ return Main.result_transfer_multiple(result_transfer)
128
143
  end
129
144
  end
130
145
  end
@@ -1,6 +1,8 @@
1
1
  require 'aspera/cli/basic_auth_plugin'
2
2
  require 'aspera/nagios'
3
3
  require 'aspera/hash_ext'
4
+ require 'aspera/id_generator'
5
+ require 'aspera/node'
4
6
  require 'base64'
5
7
  require 'zlib'
6
8
 
@@ -8,26 +10,49 @@ module Aspera
8
10
  module Cli
9
11
  module Plugins
10
12
  class Node < BasicAuthPlugin
13
+ class << self
14
+ def detect(base_url)
15
+ api=Rest.new({:base_url=>base_url})
16
+ result=api.call({:operation=>'GET',:subpath=>'ping'})
17
+ if result[:http].body.eql?('')
18
+ return {:product=>:node,:version=>'unknown'}
19
+ end
20
+ return nil
21
+ end
22
+ end
11
23
  SAMPLE_SOAP_CALL='<?xml version="1.0" encoding="UTF-8"?><soapenv:Envelope xmlns:soapenv="http://schemas.xmlsoap.org/soap/envelope/" xmlns:typ="urn:Aspera:XML:FASPSessionNET:2009/11:Types"><soapenv:Header></soapenv:Header><soapenv:Body><typ:GetSessionInfoRequest><SessionFilter><SessionStatus>running</SessionStatus></SessionFilter></typ:GetSessionInfoRequest></soapenv:Body></soapenv:Envelope>'
12
24
  private_constant :SAMPLE_SOAP_CALL
25
+
13
26
  def initialize(env)
14
27
  super(env)
15
- # this is added to some requests , for instance to add tags
28
+ # this is added to some requests , for instance to add tags (COS)
16
29
  @add_request_param = env[:add_request_param] || {}
17
30
  unless env[:skip_basic_auth_options]
18
31
  self.options.add_opt_simple(:validator,"identifier of validator (optional for central)")
19
32
  self.options.add_opt_simple(:asperabrowserurl,"URL for simple aspera web ui")
20
33
  self.options.add_opt_simple(:name,"sync name")
21
- self.options.add_opt_list(:token,[:aspera,:basic,:auto],'todo: type of token used for transfers')
34
+ self.options.add_opt_list(:token_type,[:aspera,:basic,:hybrid],'Type of token used for transfers')
22
35
  self.options.set_option(:asperabrowserurl,'https://asperabrowser.mybluemix.net')
23
- self.options.set_option(:token,:aspera)
36
+ self.options.set_option(:token_type,:aspera)
24
37
  self.options.parse_options!
25
38
  end
26
39
  return if env[:man_only]
27
40
  if env.has_key?(:node_api)
28
41
  @api_node=env[:node_api]
29
42
  else
30
- @api_node=basic_auth_api unless env[:man_only]
43
+ if self.options.get_option(:password,:mandatory).start_with?('Bearer ')
44
+ # info is provided like node_info of aoc
45
+ @api_node=Rest.new({
46
+ base_url: self.options.get_option(:url,:mandatory),
47
+ headers: {
48
+ 'Authorization' => self.options.get_option(:password,:mandatory),
49
+ 'X-Aspera-AccessKey' => self.options.get_option(:username,:mandatory),
50
+ }
51
+ })
52
+ else
53
+ # this is normal case
54
+ @api_node=basic_auth_api
55
+ end
31
56
  end
32
57
  end
33
58
 
@@ -152,7 +177,7 @@ module Aspera
152
177
  when :mkdir
153
178
  path_list=get_next_arg_add_prefix(prefix_path,"folder path or ext.val. list")
154
179
  path_list=[path_list] unless path_list.is_a?(Array)
155
- #TODO
180
+ #TODO: a command for that ?
156
181
  #resp=@api_node.create('space',{ "paths" => path_list.map {|i| {:type=>:directory,:path=>i} } } )
157
182
  resp=@api_node.create('files/create',{ "paths" => [{ :type => :directory, :path => path_list } ] } )
158
183
  return c_result_translate_rem_prefix(resp,'folder','created',prefix_path)
@@ -190,22 +215,36 @@ module Aspera
190
215
  #raise "unknown type: #{send_result['self']['type']}"
191
216
  end
192
217
  return c_result_remove_prefix_path(result,'path',prefix_path)
193
- when :upload
194
- # we send only a list of one transfer request
195
- transfer_request = { :paths => [ { :destination => self.transfer.destination_folder('send') } ] }
196
- transfer_request.deep_merge!(@add_request_param)
197
- send_result=@api_node.create('files/upload_setup',{:transfer_requests => [ { :transfer_request => transfer_request } ] } )[:data]
198
- # only one request, so only one answer
199
- transfer_spec=send_result['transfer_specs'].first['transfer_spec']
200
- # delete this part, as the returned value contains only destination, and not sources
201
- transfer_spec.delete('paths')
202
- return Main.result_transfer(self.transfer.start(transfer_spec,{:src=>:node_gen3}))
203
- when :download
204
- transfer_request = {:paths => self.transfer.ts_source_paths }
205
- transfer_request.deep_merge!(@add_request_param)
206
- send_result=@api_node.create('files/download_setup',{:transfer_requests => [ { :transfer_request => transfer_request } ] } )[:data]
207
- # only one request, so only one answer
208
- transfer_spec=send_result['transfer_specs'].first['transfer_spec']
218
+ when :upload,:download
219
+ token_type=self.options.get_option(:token_type,:optional)
220
+ # nil if Shares 1.x
221
+ token_type=:aspera if token_type.nil?
222
+ case token_type
223
+ when :aspera,:hybrid
224
+ transfer_paths=case command
225
+ when :upload;[ { :destination => self.transfer.destination_folder('send') } ]
226
+ when :download;self.transfer.ts_source_paths
227
+ end
228
+ # only one request, so only one answer
229
+ transfer_spec=@api_node.create("files/#{command}_setup",{:transfer_requests => [ { transfer_request: {
230
+ paths: transfer_paths
231
+ }.deep_merge(@add_request_param) } ] } )[:data]['transfer_specs'].first['transfer_spec']
232
+ # delete this part, as the returned value contains only destination, and not sources
233
+ transfer_spec.delete('paths') if command.eql?(:upload)
234
+ when :basic
235
+ raise "shall have auth" unless @api_node.params[:auth].is_a?(Hash)
236
+ raise "shall be basic auth" unless @api_node.params[:auth][:type].eql?(:basic)
237
+ transfer_spec={
238
+ 'remote_host'=>URI.parse(@api_node.params[:base_url]).host,
239
+ 'remote_user'=>Aspera::Fasp::Default::ACCESS_KEY_TRANSFER_USER,
240
+ 'ssh_port' =>Aspera::Fasp::Default::SSH_PORT,
241
+ 'direction' =>case command;when :upload;'send';when :download;'recv';else raise "Error";end
242
+ }
243
+ else raise "ERROR: token_type #{tt}"
244
+ end
245
+ if [:basic,:hybrid].include?(token_type)
246
+ Aspera::Node.set_ak_basic_token(transfer_spec,@api_node.params[:auth][:username],@api_node.params[:auth][:password])
247
+ end
209
248
  return Main.result_transfer(self.transfer.start(transfer_spec,{:src=>:node_gen3}))
210
249
  when :api_details
211
250
  return { :type=>:single_object, :data => @api_node.params }
@@ -217,7 +256,7 @@ module Aspera
217
256
  unless command.eql?(:list)
218
257
  asyncname=self.options.get_option(:name,:optional)
219
258
  if asyncname.nil?
220
- asyncid=self.options.get_option(:id,:mandatory)
259
+ asyncid=self.instance_identifier()
221
260
  if asyncid.eql?('ALL') and [:show,:delete].include?(command)
222
261
  asyncids=@api_node.read('async/list')[:data]['sync_ids']
223
262
  else
@@ -271,8 +310,8 @@ module Aspera
271
310
  if self.options.get_option(:once_only,:mandatory)
272
311
  skip_ids_persistency=PersistencyActionOnce.new(
273
312
  manager: @agents[:persistency],
274
- data: iteration_data,
275
- ids: ['sync_files',self.options.get_option(:url,:mandatory),self.options.get_option(:username,:mandatory),asyncid])
313
+ data: iteration_data,
314
+ id: IdGenerator.from_list(['sync_files',self.options.get_option(:url,:mandatory),self.options.get_option(:username,:mandatory),asyncid]))
276
315
  unless iteration_data.first.nil?
277
316
  data.select!{|l| l['fnid'].to_i>iteration_data.first}
278
317
  end
@@ -300,7 +339,7 @@ module Aspera
300
339
  case command
301
340
  when :list
302
341
  resp=@api_node.read('ops/transfers',self.options.get_option(:value,:optional))
303
- return { :type => :object_list, :data => resp[:data], :fields=>['id','status'] } # TODO
342
+ return { :type => :object_list, :data => resp[:data], :fields=>['id','status'] } # TODO: useful?
304
343
  when :create
305
344
  resp=@api_node.create('streams',self.options.get_option(:value,:mandatory))
306
345
  return { :type => :single_object, :data => resp[:data] }
@@ -323,7 +362,7 @@ module Aspera
323
362
  command=self.options.get_next_command([ :list, :cancel, :show ])
324
363
  res_class_path='ops/transfers'
325
364
  if [:cancel, :show].include?(command)
326
- one_res_id=self.options.get_option(:id,:mandatory)
365
+ one_res_id=self.instance_identifier()
327
366
  one_res_path="#{res_class_path}/#{one_res_id}"
328
367
  end
329
368
  case command
@@ -345,7 +384,7 @@ module Aspera
345
384
  when :service
346
385
  command=self.options.get_next_command([ :list, :create, :delete])
347
386
  if [:delete].include?(command)
348
- svcid=self.options.get_option(:id,:mandatory)
387
+ svcid=self.instance_identifier()
349
388
  end
350
389
  case command
351
390
  when :list
@@ -365,7 +404,7 @@ module Aspera
365
404
  #return entity_action(@api_node,'v3/watchfolders',nil,:id)
366
405
  command=self.options.get_next_command([ :create, :list, :show, :modify, :delete, :state])
367
406
  if [:show,:modify,:delete,:state].include?(command)
368
- one_res_id=self.options.get_option(:id,:mandatory)
407
+ one_res_id=self.instance_identifier()
369
408
  one_res_path="#{res_class_path}/#{one_res_id}"
370
409
  end
371
410
  # hum, to avoid: Unable to convert 2016_09_14 configuration
@@ -408,9 +447,11 @@ module Aspera
408
447
  command=self.options.get_next_command([ :list, :modify])
409
448
  case command
410
449
  when :list
411
- request_data.deep_merge!({"validation"=>validation}) unless validation.nil?
412
- resp=@api_node.create('services/rest/transfers/v1/files',request_data)
413
- return {:type=>:object_list,:data=>resp[:data]["file_transfer_info_result"]["file_transfer_info"],:fields=>["session_uuid","file_id","status","path"]}
450
+ request_data.deep_merge!({'validation'=>validation}) unless validation.nil?
451
+ resp=@api_node.create('services/rest/transfers/v1/files',request_data)[:data]
452
+ resp=JSON.parse(resp) if resp.is_a?(String)
453
+ Log.dump(:resp,resp)
454
+ return {:type=>:object_list,:data=>resp['file_transfer_info_result']['file_transfer_info'],:fields=>["session_uuid","file_id","status","path"]}
414
455
  when :modify
415
456
  request_data.deep_merge!(validation) unless validation.nil?
416
457
  @api_node.update('services/rest/transfers/v1/files',request_data)
@@ -110,12 +110,14 @@ module Aspera
110
110
  return {:type=>:object_list,:data=>result['Plugin']}
111
111
  when :workflow
112
112
  command=self.options.get_next_command([:list, :status, :inputs, :details, :start, :export])
113
- unless [:list, :status].include?(command)
114
- wf_id=self.options.get_option(:id,:mandatory)
113
+ unless [:list].include?(command)
114
+ wf_id=self.instance_identifier()
115
115
  end
116
116
  case command
117
117
  when :status
118
- result=call_API('workflows_status')[:data]
118
+ options={}
119
+ options[:id]=wf_id unless wf_id.eql?('ALL')
120
+ result=call_API('workflows_status',options)[:data]
119
121
  return {:type=>:object_list,:data=>result['workflows']['workflow']}
120
122
  when :list
121
123
  result=call_API('workflows_list',id: 0)[:data]