aspera-cli 4.24.1 → 4.25.0.pre

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (99) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/CHANGELOG.md +1064 -745
  4. data/CONTRIBUTING.md +43 -100
  5. data/README.md +1281 -720
  6. data/bin/ascli +20 -1
  7. data/bin/asession +23 -27
  8. data/lib/aspera/agent/base.rb +10 -21
  9. data/lib/aspera/agent/connect.rb +2 -3
  10. data/lib/aspera/agent/desktop.rb +2 -2
  11. data/lib/aspera/agent/direct.rb +49 -32
  12. data/lib/aspera/agent/factory.rb +31 -0
  13. data/lib/aspera/api/aoc.rb +134 -76
  14. data/lib/aspera/api/cos_node.rb +3 -2
  15. data/lib/aspera/api/faspex.rb +213 -0
  16. data/lib/aspera/api/node.rb +107 -94
  17. data/lib/aspera/ascmd.rb +1 -2
  18. data/lib/aspera/ascp/installation.rb +73 -58
  19. data/lib/aspera/ascp/management.rb +119 -23
  20. data/lib/aspera/assert.rb +39 -11
  21. data/lib/aspera/cli/error.rb +4 -2
  22. data/lib/aspera/cli/extended_value.rb +91 -67
  23. data/lib/aspera/cli/formatter.rb +62 -27
  24. data/lib/aspera/cli/hints.rb +8 -0
  25. data/lib/aspera/cli/info.rb +4 -4
  26. data/lib/aspera/cli/main.rb +76 -84
  27. data/lib/aspera/cli/manager.rb +352 -248
  28. data/lib/aspera/cli/plugins/alee.rb +5 -4
  29. data/lib/aspera/cli/plugins/aoc.rb +175 -195
  30. data/lib/aspera/cli/plugins/ats.rb +4 -4
  31. data/lib/aspera/cli/plugins/base.rb +343 -0
  32. data/lib/aspera/cli/plugins/basic_auth.rb +45 -0
  33. data/lib/aspera/cli/plugins/config.rb +283 -269
  34. data/lib/aspera/cli/plugins/console.rb +27 -22
  35. data/lib/aspera/cli/plugins/cos.rb +3 -3
  36. data/lib/aspera/cli/plugins/factory.rb +78 -0
  37. data/lib/aspera/cli/plugins/faspex.rb +49 -46
  38. data/lib/aspera/cli/plugins/faspex5.rb +113 -225
  39. data/lib/aspera/cli/plugins/faspio.rb +19 -18
  40. data/lib/aspera/cli/plugins/httpgw.rb +14 -13
  41. data/lib/aspera/cli/plugins/node.rb +162 -149
  42. data/lib/aspera/cli/plugins/oauth.rb +48 -0
  43. data/lib/aspera/cli/plugins/orchestrator.rb +129 -45
  44. data/lib/aspera/cli/plugins/preview.rb +30 -50
  45. data/lib/aspera/cli/plugins/server.rb +21 -21
  46. data/lib/aspera/cli/plugins/shares.rb +45 -47
  47. data/lib/aspera/cli/sync_actions.rb +50 -39
  48. data/lib/aspera/cli/transfer_agent.rb +35 -49
  49. data/lib/aspera/cli/transfer_progress.rb +6 -6
  50. data/lib/aspera/cli/version.rb +3 -3
  51. data/lib/aspera/cli/wizard.rb +70 -55
  52. data/lib/aspera/colors.rb +6 -0
  53. data/lib/aspera/command_line_builder.rb +59 -61
  54. data/lib/aspera/command_line_converter.rb +2 -1
  55. data/lib/aspera/coverage.rb +2 -2
  56. data/lib/aspera/data_repository.rb +1 -1
  57. data/lib/aspera/environment.rb +51 -41
  58. data/lib/aspera/faspex_gw.rb +7 -5
  59. data/lib/aspera/faspex_postproc.rb +1 -1
  60. data/lib/aspera/keychain/factory.rb +1 -2
  61. data/lib/aspera/keychain/macos_security.rb +1 -1
  62. data/lib/aspera/log.rb +37 -9
  63. data/lib/aspera/markdown.rb +31 -0
  64. data/lib/aspera/nagios.rb +7 -6
  65. data/lib/aspera/oauth/base.rb +25 -28
  66. data/lib/aspera/oauth/factory.rb +9 -9
  67. data/lib/aspera/oauth/url_json.rb +2 -1
  68. data/lib/aspera/oauth/web.rb +2 -2
  69. data/lib/aspera/preview/file_types.rb +23 -37
  70. data/lib/aspera/products/connect.rb +7 -6
  71. data/lib/aspera/products/desktop.rb +1 -4
  72. data/lib/aspera/products/other.rb +9 -1
  73. data/lib/aspera/products/transferd.rb +0 -1
  74. data/lib/aspera/rest.rb +168 -113
  75. data/lib/aspera/rest_error_analyzer.rb +4 -4
  76. data/lib/aspera/ssh.rb +7 -4
  77. data/lib/aspera/ssl.rb +41 -0
  78. data/lib/aspera/sync/args.schema.yaml +46 -3
  79. data/lib/aspera/sync/conf.schema.yaml +307 -123
  80. data/lib/aspera/sync/database.rb +2 -1
  81. data/lib/aspera/sync/operations.rb +135 -79
  82. data/lib/aspera/temp_file_manager.rb +17 -5
  83. data/lib/aspera/transfer/error.rb +16 -7
  84. data/lib/aspera/transfer/parameters.rb +35 -22
  85. data/lib/aspera/transfer/resumer.rb +74 -0
  86. data/lib/aspera/transfer/spec.rb +5 -5
  87. data/lib/aspera/transfer/spec.schema.yaml +170 -59
  88. data/lib/aspera/transfer/spec_doc.rb +49 -43
  89. data/lib/aspera/uri_reader.rb +2 -2
  90. data/lib/aspera/web_auth.rb +6 -6
  91. data/lib/transferd_pb.rb +2 -2
  92. data.tar.gz.sig +0 -0
  93. metadata +26 -11
  94. metadata.gz.sig +0 -0
  95. data/lib/aspera/cli/basic_auth_plugin.rb +0 -43
  96. data/lib/aspera/cli/plugin.rb +0 -333
  97. data/lib/aspera/cli/plugin_factory.rb +0 -81
  98. data/lib/aspera/resumer.rb +0 -77
  99. data/lib/aspera/transfer/error_info.rb +0 -91
@@ -1,7 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  # cspell:ignore snid fnid bidi ssync asyncs rund asnodeadmin mkfile mklink asperabrowser asperabrowserurl watchfolders watchfolderd entsrv
4
- require 'aspera/cli/basic_auth_plugin'
4
+ require 'aspera/cli/plugins/basic_auth'
5
5
  require 'aspera/cli/sync_actions'
6
6
  require 'aspera/cli/special_values'
7
7
  require 'aspera/transfer/spec'
@@ -9,7 +9,6 @@ require 'aspera/nagios'
9
9
  require 'aspera/hash_ext'
10
10
  require 'aspera/id_generator'
11
11
  require 'aspera/api/node'
12
- require 'aspera/api/aoc'
13
12
  require 'aspera/oauth'
14
13
  require 'aspera/node_simulator'
15
14
  require 'aspera/assert'
@@ -19,9 +18,33 @@ require 'zlib'
19
18
  module Aspera
20
19
  module Cli
21
20
  module Plugins
22
- class Node < Cli::BasicAuthPlugin
21
+ class Node < BasicAuth
23
22
  include SyncActions
24
23
 
24
+ # Processing of paths in arguments and results
25
+ # Used only by Faspex4 to browse packages
26
+ class NodePathPrefix
27
+ def initialize(path)
28
+ @root = path
29
+ end
30
+
31
+ # get next path argument from command line, and add prefix
32
+ def add_to_path(path_arg)
33
+ File.join(@root, path_arg)
34
+ end
35
+
36
+ # get remaining path arguments from command line, and add prefix
37
+ def add_to_paths!(path_args)
38
+ path_args.map!{ |p| add_to_path(p)}
39
+ end
40
+
41
+ def remove_in_object_list!(obj_list)
42
+ obj_list.each do |item|
43
+ item['path'] = item['path'][@root.length..-1] if item['path'].start_with?(@root)
44
+ end
45
+ end
46
+ end
47
+
25
48
  class << self
26
49
  # directory: node, container: shares
27
50
  FOLDER_TYPES = %w[directory container].freeze
@@ -46,12 +69,12 @@ module Aspera
46
69
  next unless base_url.match?('https?://')
47
70
  api = Rest.new(base_url: base_url)
48
71
  test_endpoint = 'ping'
49
- result = api.call(operation: 'GET', subpath: test_endpoint)
50
- next unless result[:http].body.eql?('')
72
+ http = api.read(test_endpoint, ret: :resp)
73
+ next unless http.body.eql?('')
51
74
  # also remove "/"
52
75
  url_end = -2 - test_endpoint.length
53
76
  return {
54
- url: result[:http].uri.to_s[0..url_end],
77
+ url: http.uri.to_s[0..url_end],
55
78
  version: 'requires authentication'
56
79
  }
57
80
  rescue StandardError => e
@@ -62,18 +85,6 @@ module Aspera
62
85
  return
63
86
  end
64
87
 
65
- def wizard(object:, _private_key_path: nil, _pub_key_pem: nil)
66
- options = object.options
67
- return {
68
- preset_value: {
69
- url: options.get_option(:url, mandatory: true),
70
- username: options.get_option(:username, mandatory: true),
71
- password: options.get_option(:password, mandatory: true)
72
- },
73
- test_args: 'info'
74
- }
75
- end
76
-
77
88
  def declare_options(options)
78
89
  return if @options_declared
79
90
  @options_declared = true
@@ -81,11 +92,11 @@ module Aspera
81
92
  options.declare(:validator, 'Identifier of validator (optional for central)')
82
93
  options.declare(:asperabrowserurl, 'URL for simple aspera web ui', default: 'https://asperabrowser.mybluemix.net')
83
94
  options.declare(
84
- :default_ports, 'Gen4: Use standard FASP ports (true) or get from node API (false)', values: :bool, default: :yes,
95
+ :default_ports, 'Gen4: Use standard FASP ports (true) or get from node API (false)', allowed: Allowed::TYPES_BOOLEAN, default: true,
85
96
  handler: {o: Api::Node, m: :use_standard_ports}
86
97
  )
87
98
  options.declare(
88
- :node_cache, 'Gen4: Set to no to force actual file system read', values: :bool,
99
+ :node_cache, 'Gen4: Set to no to force actual file system read', allowed: Allowed::TYPES_BOOLEAN,
89
100
  handler: {o: Api::Node, m: :use_node_cache}
90
101
  )
91
102
  options.declare(:root_id, 'Gen4: File id of top folder when using access key (override AK root id)')
@@ -100,6 +111,20 @@ module Aspera
100
111
  end
101
112
  end
102
113
 
114
+ # @param wizard [Wizard] The wizard object
115
+ # @param app_url [Wizard] The wizard object
116
+ # @return [Hash] :preset_value, :test_args
117
+ def wizard(wizard, app_url)
118
+ return {
119
+ preset_value: {
120
+ url: app_url,
121
+ username: options.get_option(:username, mandatory: true),
122
+ password: options.get_option(:password, mandatory: true)
123
+ },
124
+ test_args: 'info'
125
+ }
126
+ end
127
+
103
128
  # spellchecker: disable
104
129
  # SOAP API call to test central API
105
130
  CENTRAL_SOAP_API_TEST = '<?xml version="1.0" encoding="UTF-8"?>' \
@@ -113,7 +138,7 @@ module Aspera
113
138
  SEARCH_REMOVE_FIELDS = %w[basename permissions].freeze
114
139
 
115
140
  # Actions in execute_command_gen3
116
- COMMANDS_GEN3 = %i[search space mkdir mklink mkfile rename delete browse upload download cat sync transport]
141
+ COMMANDS_GEN3 = %i[search space mkdir mklink mkfile rename delete browse upload download cat sync transport spec]
117
142
 
118
143
  BASE_ACTIONS = %i[api_details].concat(COMMANDS_GEN3).freeze
119
144
 
@@ -141,9 +166,9 @@ module Aspera
141
166
  GEN4_LS_FIELDS = %w[name type recursive_size size modified_time access_level].freeze
142
167
 
143
168
  # @param api [Rest] an existing API object for the Node API
144
- # @param prefix_path [String,nil] for Faspex 4, allows browsing a package
169
+ # @param prefix_path [String,nil] for Faspex 4, allows browsing a package without full path in node (removes storage prefix)
145
170
  def initialize(context:, api: nil, prefix_path: nil)
146
- @prefix_path = prefix_path
171
+ @prefixer = prefix_path ? NodePathPrefix.new(prefix_path) : nil
147
172
  super(context: context, basic_options: api.nil?)
148
173
  Node.declare_options(options)
149
174
  return if context.only_manual?
@@ -170,43 +195,12 @@ module Aspera
170
195
  end
171
196
  end
172
197
 
173
- # reduce the path from a result on given named column
174
- def c_result_remove_prefix_path(result, column)
175
- return result if @prefix_path.nil?
176
- case result[:type]
177
- when :object_list
178
- result[:data].each do |item|
179
- item[column] = item[column][@prefix_path.length..-1] if item[column].start_with?(@prefix_path)
180
- end
181
- when :single_object
182
- item = result[:data]
183
- item[column] = item[column][@prefix_path.length..-1] if item[column].start_with?(@prefix_path)
184
- end
185
- return result
186
- end
187
-
188
- # translates paths results into CLI result, and removes prefix
189
- def c_result_translate_rem_prefix(response, type, success_msg)
190
- errors = []
191
- final_result = {type: :object_list, data: [], fields: [type, 'result']}
192
- response['paths'].each do |p|
193
- result = success_msg
194
- if p.key?('error')
195
- Log.log.error{"#{p['error']['user_message']} : #{p['path']}"}
196
- result = p['error']['user_message']
197
- errors.push([p['path'], p['error']['user_message']])
198
- end
199
- final_result[:data].push({type => p['path'], 'result' => result})
200
- end
201
- # one error make all fail
202
- raise errors.map{ |i| "#{i.first}: #{i.last}"}.join(', ') unless errors.empty?
203
- return c_result_remove_prefix_path(final_result, type)
204
- end
205
-
206
198
  # Gen3 API
207
199
  def browse_gen3
208
- folders_to_process = [get_one_argument_with_prefix('path')]
209
- query = options.get_option(:query, default: {})
200
+ folders_to_process = options.get_next_argument('path', validation: String)
201
+ folders_to_process = @prefixer.add_to_path(folders_to_process) unless @prefixer.nil?
202
+ folders_to_process = [folders_to_process]
203
+ query = options.get_option(:query) || {}
210
204
  # special parameter: max number of entries in result
211
205
  max_items = query.delete(MAX_ITEMS)
212
206
  # special parameter: recursive browsing
@@ -224,14 +218,13 @@ module Aspera
224
218
  query['path'] = path
225
219
  offset = 0
226
220
  total_count = nil
227
- result = nil
228
221
  loop do
229
222
  # example: send_result={'items'=>[{'file'=>"filename1","permissions"=>[{'name'=>'read'},{'name'=>'write'}]}]}
230
223
  response = @api_node.create('files/browse', query)
231
224
  # 'file','symbolic_link'
232
225
  if !Node.gen3_entry_folder?(response['self']) || only_path
233
- result = {type: :single_object, data: response['self']}
234
- break
226
+ @prefixer&.remove_in_object_list!([response['self']])
227
+ return Main.result_single_object(response['self'])
235
228
  end
236
229
  items = response['items']
237
230
  total_count ||= response['total_count']
@@ -252,9 +245,10 @@ module Aspera
252
245
  end
253
246
  query.delete('skip')
254
247
  end
255
- result ||= {type: :object_list, data: all_items}
248
+ @prefixer&.remove_in_object_list!(all_items)
249
+ return Main.result_object_list(all_items)
250
+ ensure
256
251
  formatter.long_operation_terminated
257
- return c_result_remove_prefix_path(result, 'path')
258
252
  end
259
253
 
260
254
  # Create async transfer spec request from direction and folders
@@ -293,48 +287,57 @@ module Aspera
293
287
  case command
294
288
  when :delete
295
289
  # TODO: add query for recursive
296
- paths_to_delete = get_all_arguments_with_prefix('file list')
290
+ paths_to_delete = options.get_next_argument('file list', multiple: true)
291
+ @prefixer&.add_to_paths!(paths_to_delete)
297
292
  resp = @api_node.create('files/delete', {paths: paths_to_delete.map{ |i| {'path' => i.start_with?('/') ? i : "/#{i}"}}})
298
- return c_result_translate_rem_prefix(resp, 'file', 'deleted')
293
+ return cli_result_from_paths_response(resp, 'file deleted')
299
294
  when :search
300
- search_root = get_one_argument_with_prefix('search root')
295
+ search_root = options.get_next_argument('search root', validation: String)
296
+ search_root = @prefixer.add_to_path(search_root) unless @prefixer.nil?
301
297
  parameters = {'path' => search_root}
302
298
  other_options = options.get_option(:query)
303
299
  parameters.merge!(other_options) unless other_options.nil?
304
300
  resp = @api_node.create('files/search', parameters)
305
- result = {type: :object_list, data: resp['items']}
306
- return Main.result_empty if result[:data].empty?
307
- result[:fields] = result[:data].first.keys.reject{ |i| SEARCH_REMOVE_FIELDS.include?(i)}
301
+ return Main.result_empty if resp['items'].empty?
302
+ fields = resp['items'].first.keys.reject{ |i| SEARCH_REMOVE_FIELDS.include?(i)}
308
303
  formatter.display_item_count(resp['item_count'], resp['total_count'])
309
304
  formatter.display_status("params: #{resp['parameters'].keys.map{ |k| "#{k}:#{resp['parameters'][k]}"}.join(',')}")
310
- return c_result_remove_prefix_path(result, 'path')
305
+ @prefixer&.remove_in_object_list!(resp['items'])
306
+ return Main.result_object_list(resp['items'], fields: fields)
311
307
  when :space
312
- path_list = get_all_arguments_with_prefix('folder path or ext.val. list')
308
+ path_list = options.get_next_argument('folder path or ext.val. list', multiple: true)
309
+ @prefixer&.add_to_paths!(path_list)
313
310
  resp = @api_node.create('space', {'paths' => path_list.map{ |i| {path: i}}})
314
- result = {type: :object_list, data: resp['paths']}
315
- # return c_result_translate_rem_prefix(resp,'folder','created',@prefix_path)
316
- return c_result_remove_prefix_path(result, 'path')
311
+ @prefixer&.remove_in_object_list!(resp['paths'])
312
+ return Main.result_object_list(resp['paths'])
317
313
  when :mkdir
318
- path_list = get_all_arguments_with_prefix('folder path or ext.val. list')
314
+ path_list = options.get_next_argument('folder path or ext.val. list', multiple: true)
315
+ @prefixer&.add_to_paths!(path_list)
319
316
  resp = @api_node.create('files/create', {'paths' => path_list.map{ |i| {type: :directory, path: i}}})
320
- return c_result_translate_rem_prefix(resp, 'folder', 'created')
317
+ return cli_result_from_paths_response(resp, 'folder created')
321
318
  when :mklink
322
- target = get_one_argument_with_prefix('target')
323
- one_path = get_one_argument_with_prefix('link path')
319
+ target = options.get_next_argument('target', validation: String)
320
+ target = @prefixer.add_to_path(target) unless @prefixer.nil?
321
+ one_path = options.get_next_argument('link path', validation: String)
322
+ one_path = @prefixer.add_to_path(one_path) unless @prefixer.nil?
324
323
  resp = @api_node.create('files/create', {'paths' => [{type: :symbolic_link, path: one_path, target: {path: target}}]})
325
- return c_result_translate_rem_prefix(resp, 'folder', 'created')
324
+ return cli_result_from_paths_response(resp, 'link created')
326
325
  when :mkfile
327
- one_path = get_one_argument_with_prefix('file path')
326
+ one_path = options.get_next_argument('file path', validation: String)
327
+ one_path = @prefixer.add_to_path(one_path) unless @prefixer.nil?
328
328
  contents64 = Base64.strict_encode64(options.get_next_argument('contents'))
329
329
  resp = @api_node.create('files/create', {'paths' => [{type: :file, path: one_path, contents: contents64}]})
330
- return c_result_translate_rem_prefix(resp, 'folder', 'created')
330
+ return cli_result_from_paths_response(resp, 'file created')
331
331
  when :rename
332
332
  # TODO: multiple ?
333
- path_base = get_one_argument_with_prefix('path_base')
334
- path_src = get_one_argument_with_prefix('path_src')
335
- path_dst = get_one_argument_with_prefix('path_dst')
333
+ path_base = options.get_next_argument('path_base', validation: String)
334
+ path_base = @prefixer.add_to_path(path_base) unless @prefixer.nil?
335
+ path_src = options.get_next_argument('path_src', validation: String)
336
+ path_src = @prefixer.add_to_path(path_src) unless @prefixer.nil?
337
+ path_dst = options.get_next_argument('path_dst', validation: String)
338
+ path_dst = @prefixer.add_to_path(path_dst) unless @prefixer.nil?
336
339
  resp = @api_node.create('files/rename', {'paths' => [{'path' => path_base, 'source' => path_src, 'destination' => path_dst}]})
337
- return c_result_translate_rem_prefix(resp, 'entry', 'moved')
340
+ return cli_result_from_paths_response(resp, 'entry moved')
338
341
  when :browse
339
342
  return browse_gen3
340
343
  when :sync
@@ -376,15 +379,15 @@ module Aspera
376
379
  transfer_spec.delete('paths') if command.eql?(:upload)
377
380
  return Main.result_transfer(transfer.start(transfer_spec))
378
381
  when :cat
379
- remote_path = get_one_argument_with_prefix('remote path')
382
+ remote_path = options.get_next_argument('remote path', validation: String)
383
+ remote_path = @prefixer.add_to_path(remote_path) unless @prefixer.nil?
380
384
  File.basename(remote_path)
381
- result = @api_node.call(
382
- operation: 'GET',
383
- subpath: "files/#{URI.encode_www_form_component(remote_path)}/contents"
384
- )
385
- return Main.result_text(result[:http].body)
385
+ http = @api_node.read("files/#{URI.encode_www_form_component(remote_path)}/contents", ret: :resp)
386
+ return Main.result_text(http.body)
386
387
  when :transport
387
388
  return Main.result_single_object(@api_node.transport_params)
389
+ when :spec
390
+ return Main.result_single_object(@api_node.base_spec)
388
391
  end
389
392
  Aspera.error_unreachable_line
390
393
  end
@@ -395,9 +398,9 @@ module Aspera
395
398
  when *COMMANDS_GEN3
396
399
  execute_command_gen3(command)
397
400
  when :access_keys
398
- ak_command = options.get_next_command(%i[do set_bearer_key].concat(Plugin::ALL_OPS))
401
+ ak_command = options.get_next_command(%i[do set_bearer_key].concat(ALL_OPS))
399
402
  case ak_command
400
- when *Plugin::ALL_OPS
403
+ when *ALL_OPS
401
404
  return entity_execute(
402
405
  api: @api_node,
403
406
  entity: 'access_keys',
@@ -446,13 +449,14 @@ module Aspera
446
449
  subpath: 'services/soap/Transfer-201210',
447
450
  content_type: Rest::MIME_TEXT,
448
451
  body: CENTRAL_SOAP_API_TEST,
449
- headers: {'Content-Type' => 'text/xml;charset=UTF-8', 'SOAPAction' => 'FASPSessionNET-200911#GetSessionInfo'}
450
- )[:http].body
452
+ headers: {'Content-Type' => 'text/xml;charset=UTF-8', 'SOAPAction' => 'FASPSessionNET-200911#GetSessionInfo'},
453
+ ret: :resp
454
+ ).body
451
455
  nagios.add_ok('central', 'accessible by node')
452
456
  rescue StandardError => e
453
457
  nagios.add_critical('central', e.to_s)
454
458
  end
455
- return nagios.result
459
+ Main.result_object_list(nagios.status_list)
456
460
  when :events
457
461
  events = @api_node.read('events', query_read_delete)
458
462
  return Main.result_object_list(events, fields: ->(f){!f.start_with?('data')})
@@ -475,7 +479,7 @@ module Aspera
475
479
  # Allows to specify a file by its path or by its id on the node in command line
476
480
  # @return [Hash] api and main file id for given path or id in next argument
477
481
  def apifid_from_next_arg(top_file_id)
478
- file_path = instance_identifier(description: 'path or %id:<id>') do |attribute, value|
482
+ file_path = instance_identifier(description: 'path or %id:<id> or %id:') do |attribute, value|
479
483
  raise BadArgument, 'Only selection "id" is supported (file id)' unless attribute.eql?('id')
480
484
  # directly return result for method
481
485
  return {api: @api_node, file_id: value}
@@ -512,17 +516,17 @@ module Aspera
512
516
  else Aspera.error_unreachable_line
513
517
  end
514
518
  return Main.result_single_object(result) if command_repo.eql?(:node_info)
515
- # check format of bearer token
516
- OAuth::Factory.bearer_token(result[:password])
519
+ raise BadArgument, 'Cannot get bearer token if authenticating with secret' unless apifid[:api].auth_params[:type].eql?(:oauth2)
520
+ Aspera.assert(OAuth::Factory.bearer_auth?(result[:password])){'Not using bearer token auth'}
517
521
  return Main.result_text(result[:password])
518
522
  when :browse
519
523
  apifid = apifid_from_next_arg(top_file_id)
520
- file_info = apifid[:api].read_with_cache("files/#{apifid[:file_id]}")
524
+ file_info = apifid[:api].read("files/#{apifid[:file_id]}", **Api::Node.cache_control)
521
525
  unless file_info['type'].eql?('folder')
522
526
  # a single file
523
527
  return Main.result_object_list([file_info], fields: GEN4_LS_FIELDS)
524
528
  end
525
- return Main.result_object_list(apifid[:api].list_files(apifid[:file_id]), fields: GEN4_LS_FIELDS)
529
+ return Main.result_object_list(apifid[:api].list_files(apifid[:file_id], query: query_read_delete), fields: GEN4_LS_FIELDS)
526
530
  when :find
527
531
  apifid = apifid_from_next_arg(top_file_id)
528
532
  find_lambda = Api::Node.file_matcher_from_argument(options)
@@ -530,7 +534,7 @@ module Aspera
530
534
  when :mkdir, :mklink, :mkfile
531
535
  containing_folder_path, new_item = Api::Node.split_folder(options.get_next_argument('path'))
532
536
  apifid = @api_node.resolve_api_fid(top_file_id, containing_folder_path, true)
533
- query = options.get_option(:query, mandatory: false)
537
+ query = options.get_option(:query)
534
538
  check_exists = true
535
539
  payload = {name: new_item}
536
540
  if query
@@ -569,10 +573,11 @@ module Aspera
569
573
  return Main.result_status("renamed to #{newname}")
570
574
  when :delete
571
575
  return do_bulk_operation(command: command_repo, descr: 'path', values: String, id_result: 'path') do |l_path|
572
- apifid = if (m = l_path.match(REGEX_LOOKUP_ID_BY_FIELD))
576
+ apifid = if (m = Base.percent_selector(l_path))
577
+ Aspera.assert_values(m[:field], ['id'], type: BadIdentifier)
573
578
  {
574
579
  api: @api_node,
575
- file_id: m[2]
580
+ file_id: m[:value]
576
581
  }
577
582
  else
578
583
  @api_node.resolve_api_fid(top_file_id, l_path)
@@ -601,11 +606,8 @@ module Aspera
601
606
  return Main.result_transfer(transfer.start(apifid[:api].transfer_spec_gen4(apifid[:file_id], Transfer::Spec::DIRECTION_RECEIVE, {'paths'=>source_paths})))
602
607
  when :cat
603
608
  apifid = apifid_from_next_arg(top_file_id)
604
- result = apifid[:api].call(
605
- operation: 'GET',
606
- subpath: "files/#{apifid[:file_id]}/content"
607
- )
608
- return Main.result_text(result[:http].body)
609
+ http = apifid[:api].read("files/#{apifid[:file_id]}/content", ret: :resp)
610
+ return Main.result_text(http.body)
609
611
  when :show
610
612
  apifid = apifid_from_next_arg(top_file_id)
611
613
  items = apifid[:api].read("files/#{apifid[:file_id]}")
@@ -617,18 +619,14 @@ module Aspera
617
619
  return Main.result_status('Done')
618
620
  when :thumbnail
619
621
  apifid = apifid_from_next_arg(top_file_id)
620
- result = apifid[:api].call(
621
- operation: 'GET',
622
- subpath: "files/#{apifid[:file_id]}/preview",
623
- headers: {'Accept' => 'image/png'}
624
- )
625
- return Main.result_image(result[:http].body)
622
+ http = apifid[:api].read("files/#{apifid[:file_id]}/preview", headers: {'Accept' => 'image/png'}, ret: :resp)
623
+ return Main.result_image(http.body)
626
624
  when :permission
627
625
  apifid = apifid_from_next_arg(top_file_id)
628
626
  command_perm = options.get_next_command(%i[list show create delete])
629
627
  case command_perm
630
628
  when :list
631
- list_query = query_read_delete(default: {'include' => Rest.array_params(%w[access_level permission_count])})
629
+ list_query = query_read_delete(default: Rest.php_style({'include' => %w[access_level permission_count]}))
632
630
  # specify file to get permissions for unless not specified
633
631
  list_query['file_id'] = apifid[:file_id] unless apifid[:file_id].to_s.empty?
634
632
  list_query['inherited'] = false if list_query.key?('file_id') && !list_query.key?('inherited')
@@ -676,7 +674,7 @@ module Aspera
676
674
  async_ids = @api_node.read('async/list')['sync_ids']
677
675
  summaries = @api_node.create('async/summary', {'syncs' => async_ids})['sync_summaries']
678
676
  selected = summaries.find{ |s| s['name'].eql?(value)}
679
- raise Cli::BadIdentifier.new('sync', "#{field}=#{value}") if selected.nil?
677
+ raise Cli::BadIdentifier.new('sync', value, field: field) if selected.nil?
680
678
  return selected['snid']
681
679
  end
682
680
 
@@ -761,7 +759,7 @@ module Aspera
761
759
  # name is unique, so we can return
762
760
  return id if sync_info[field].eql?(value)
763
761
  end
764
- raise Cli::BadIdentifier.new('ssync', "#{field}=#{value}")
762
+ raise Cli::BadIdentifier.new('ssync', value, field: field)
765
763
  end
766
764
 
767
765
  WATCH_FOLDER_MUL = %i[create list].freeze
@@ -788,7 +786,7 @@ module Aspera
788
786
  when :show
789
787
  return Main.result_single_object(@api_node.read(one_res_path))
790
788
  when :modify
791
- @api_node.update(one_res_path, options.get_option(:query, mandatory: true))
789
+ @api_node.update(one_res_path, value_create_modify(command: 'watch_folder'))
792
790
  return Main.result_status("#{one_res_id} updated")
793
791
  when :delete
794
792
  @api_node.delete(one_res_path)
@@ -820,9 +818,9 @@ module Aspera
820
818
  when :async then return execute_async # former API
821
819
  when :ssync
822
820
  # Node API: /asyncs (newer)
823
- sync_command = options.get_next_command(%i[start stop bandwidth counters files state summary] + Plugin::ALL_OPS - %i[modify])
821
+ sync_command = options.get_next_command(%i[start stop bandwidth counters files state summary] + ALL_OPS - %i[modify])
824
822
  case sync_command
825
- when *Plugin::ALL_OPS
823
+ when *ALL_OPS
826
824
  return entity_execute(
827
825
  api: @api_node,
828
826
  entity: :asyncs,
@@ -836,12 +834,12 @@ module Aspera
836
834
  operation: 'POST',
837
835
  subpath: "asyncs/#{asyncs_id}/#{sync_command}",
838
836
  content_type: Rest::MIME_TEXT,
839
- body: ''
840
- )[:http].body
837
+ body: '',
838
+ ret: :resp
839
+ ).body
841
840
  return Main.result_status('Done')
842
841
  end
843
- parameters = nil
844
- parameters = options.get_option(:query, default: {}) if %i[bandwidth counters files].include?(sync_command)
842
+ parameters = options.get_option(:query) || {} if %i[bandwidth counters files].include?(sync_command)
845
843
  return Main.result_single_object(@api_node.read("asyncs/#{asyncs_id}/#{sync_command}", parameters))
846
844
  end
847
845
  when :stream
@@ -893,21 +891,23 @@ module Aspera
893
891
  return Main.result_object_list(transfers_data, fields: %w[id status start_spec.direction start_spec.remote_user start_spec.remote_host start_spec.destination_path])
894
892
  when :sessions
895
893
  transfers_data = @api_node.read('ops/transfers', query_read_delete)
896
- sessions = transfers_data.map{ |t| t['sessions']}.flatten
894
+ sessions = transfers_data.flat_map{ |t| t['sessions']}
895
+ start_end = %i[start end].freeze
897
896
  sessions.each do |session|
898
- session['start_time'] = Time.at(session['start_time_usec'] / 1_000_000.0).utc.iso8601(0)
899
- session['end_time'] = Time.at(session['end_time_usec'] / 1_000_000.0).utc.iso8601(0)
897
+ start_end.each do |what|
898
+ session["#{what}_time"] = session["#{what}_time_usec"] ? Time.at(session["#{what}_time_usec"] / 1_000_000.0).utc.iso8601(0) : nil
899
+ end
900
900
  end
901
901
  return Main.result_object_list(sessions, fields: %w[id status start_time end_time target_rate_kbps])
902
902
  when :cancel
903
- resp = @api_node.cancel("ops/transfers/#{instance_identifier}")
904
- return Main.result_single_object(resp)
903
+ @api_node.cancel("ops/transfers/#{instance_identifier}")
904
+ return Main.result_status('Cancelled')
905
905
  when :show
906
906
  resp = @api_node.read("ops/transfers/#{instance_identifier}")
907
907
  return Main.result_single_object(resp)
908
908
  when :modify
909
- resp = @api_node.update("ops/transfers/#{instance_identifier}", options.get_next_argument('update value', validation: Hash))
910
- return Main.result_single_object(resp)
909
+ @api_node.update("ops/transfers/#{instance_identifier}", options.get_next_argument('update value', validation: Hash))
910
+ return Main.result_status('Modified')
911
911
  when :bandwidth_average
912
912
  transfers_data = @api_node.read('ops/transfers', query_read_delete)
913
913
  # collect all key dates
@@ -977,7 +977,7 @@ module Aspera
977
977
  command = options.get_next_command(%i[session file])
978
978
  validator_id = options.get_option(:validator)
979
979
  validation = {'validator_id' => validator_id} unless validator_id.nil?
980
- request_data = options.get_option(:query, default: {})
980
+ request_data = options.get_option(:query) || {}
981
981
  case command
982
982
  when :session
983
983
  command = options.get_next_command([:list])
@@ -1110,18 +1110,31 @@ module Aspera
1110
1110
 
1111
1111
  private
1112
1112
 
1113
- # get remaining path arguments from command line, and add prefix
1114
- def get_all_arguments_with_prefix(name)
1115
- path_args = options.get_next_argument(name, multiple: true)
1116
- return path_args if @prefix_path.nil?
1117
- return path_args.map{ |p| File.join(@prefix_path, p)}
1113
+ # Response has key `paths`.
1114
+ # From those, check if there is an error
1115
+ # @return [Array] of Hash with 2 keys: `path` and `result`
1116
+ def response_to_result(response, success_msg)
1117
+ errors = []
1118
+ obj_list = []
1119
+ response['paths'].each do |p|
1120
+ result = success_msg
1121
+ if p.key?('error')
1122
+ Log.log.error{"#{p['error']['user_message']} : #{p['path']}"}
1123
+ result = p['error']['user_message']
1124
+ errors.push([p['path'], p['error']['user_message']])
1125
+ end
1126
+ obj_list.push({'path' => p['path'], 'result' => result})
1127
+ end
1128
+ # one error make all fail
1129
+ raise errors.map{ |i| "#{i.first}: #{i.last}"}.join(', ') unless errors.empty?
1130
+ obj_list
1118
1131
  end
1119
1132
 
1120
- # get next path argument from command line, and add prefix
1121
- def get_one_argument_with_prefix(name)
1122
- path_arg = options.get_next_argument(name, validation: String)
1123
- return path_arg if @prefix_path.nil?
1124
- return File.join(@prefix_path, path_arg)
1133
+ # Translates paths results into CLI result, and removes prefix
1134
+ def cli_result_from_paths_response(response, success_msg)
1135
+ obj_list = response_to_result(response, success_msg)
1136
+ @prefixer&.remove_in_object_list!(obj_list)
1137
+ return Main.result_object_list(obj_list, fields: %w[path result])
1125
1138
  end
1126
1139
 
1127
1140
  # Executes the provided API call in loop
@@ -1138,22 +1151,22 @@ module Aspera
1138
1151
  item_list = []
1139
1152
  query_token[:iteration_token] = iteration[0] unless iteration.nil?
1140
1153
  loop do
1141
- result = api.call(**call_args, query: query_token)
1142
- Aspera.assert_type(result[:data], Array){"Expected data to be an Array, got: #{result[:data].class}"}
1154
+ data, http = api.call(**call_args, query: query_token, ret: :both)
1155
+ Aspera.assert_type(data, Array){"Expected data to be an Array, got: #{data.class}"}
1143
1156
  # no data
1144
- break if result[:data].empty?
1157
+ break if data.empty?
1145
1158
  # get next iteration token from link
1146
1159
  next_iteration_token = nil
1147
- link_info = result[:http]['Link']
1160
+ link_info = http['Link']
1148
1161
  unless link_info.nil?
1149
1162
  m = link_info.match(/<([^>]+)>/)
1150
1163
  Aspera.assert(m){"Cannot parse iteration in Link: #{link_info}"}
1151
- next_iteration_token = CGI.parse(URI.parse(m[1]).query)['iteration_token']&.first
1164
+ next_iteration_token = Rest.query_to_h(URI.parse(m[1]).query)['iteration_token']
1152
1165
  end
1153
1166
  # same as last iteration: stop
1154
1167
  break if next_iteration_token&.eql?(query_token[:iteration_token])
1155
1168
  query_token[:iteration_token] = next_iteration_token
1156
- item_list.concat(result[:data])
1169
+ item_list.concat(data)
1157
1170
  if max&.<=(item_list.length)
1158
1171
  item_list = item_list.slice(0, max)
1159
1172
  break
@@ -0,0 +1,48 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'aspera/cli/plugins/basic_auth'
4
+
5
+ module Aspera
6
+ module Cli
7
+ module Plugins
8
+ # base class for applications supporting OAuth 2.0 authentication
9
+ class Oauth < BasicAuth
10
+ # OAuth methods supported
11
+ AUTH_TYPES = %i[web jwt boot].freeze
12
+ # Options used for authentication
13
+ AUTH_OPTIONS = %i[url auth client_id client_secret scope redirect_uri private_key passphrase username password].freeze
14
+ def initialize(**_)
15
+ super
16
+ options.declare(:auth, 'OAuth type of authentication', allowed: AUTH_TYPES, default: :jwt)
17
+ options.declare(:client_id, 'OAuth client identifier')
18
+ options.declare(:client_secret, 'OAuth client secret')
19
+ options.declare(:redirect_uri, 'OAuth (Web) redirect URI for web authentication')
20
+ options.declare(:private_key, 'OAuth (JWT) RSA private key PEM value (prefix file path with @file:)')
21
+ options.declare(:passphrase, 'OAuth (JWT) RSA private key passphrase')
22
+ options.declare(:scope, 'OAuth scope for API calls')
23
+ end
24
+
25
+ # Get all options specified by AUTH_OPTIONS and add.keys
26
+ # Adds those not nil to the `base`.
27
+ # Instantiate the provided `klass` with those kwargs.
28
+ # `add` can specify a default value (not `nil`)
29
+ # @param klass [Class] API object to create
30
+ # @param base [Hash] The base options for creation
31
+ # @param add [Hash] Additional options, key=symbol, value:default value or nil
32
+ def new_with_options(klass, base: {}, add: {})
33
+ klass.new(**
34
+ (AUTH_OPTIONS + add.keys).each_with_object(base) do |i, m|
35
+ v = options.get_option(i)
36
+ m[i] = v unless v.nil?
37
+ m[i] = add[i] unless !m[i].nil? || add[i].nil?
38
+ end)
39
+ rescue ::ArgumentError => e
40
+ if (m = e.message.match(/missing keyword: :(.*)$/))
41
+ raise Cli::Error, "Missing option: #{m[1]}"
42
+ end
43
+ raise
44
+ end
45
+ end
46
+ end
47
+ end
48
+ end