aspera-cli 4.23.0 → 4.24.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (109) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/CHANGELOG.md +32 -1
  4. data/CONTRIBUTING.md +86 -29
  5. data/README.md +1651 -856
  6. data/bin/ascli +2 -1
  7. data/bin/asession +4 -4
  8. data/lib/aspera/agent/base.rb +4 -0
  9. data/lib/aspera/agent/connect.rb +20 -18
  10. data/lib/aspera/agent/desktop.rb +14 -11
  11. data/lib/aspera/agent/direct.rb +39 -31
  12. data/lib/aspera/agent/httpgw.rb +2 -2
  13. data/lib/aspera/agent/node.rb +9 -11
  14. data/lib/aspera/agent/transferd.rb +18 -11
  15. data/lib/aspera/api/aoc.rb +44 -31
  16. data/lib/aspera/api/cos_node.rb +7 -5
  17. data/lib/aspera/api/httpgw.rb +15 -18
  18. data/lib/aspera/api/node.rb +104 -22
  19. data/lib/aspera/ascmd.rb +22 -16
  20. data/lib/aspera/ascp/installation.rb +37 -40
  21. data/lib/aspera/ascp/management.rb +5 -4
  22. data/lib/aspera/assert.rb +54 -23
  23. data/lib/aspera/cli/basic_auth_plugin.rb +8 -7
  24. data/lib/aspera/cli/error.rb +1 -1
  25. data/lib/aspera/cli/extended_value.rb +28 -29
  26. data/lib/aspera/cli/formatter.rb +191 -168
  27. data/lib/aspera/cli/hints.rb +29 -3
  28. data/lib/aspera/cli/main.rb +138 -107
  29. data/lib/aspera/cli/manager.rb +50 -30
  30. data/lib/aspera/cli/plugin.rb +148 -77
  31. data/lib/aspera/cli/plugin_factory.rb +2 -2
  32. data/lib/aspera/cli/plugins/aoc.rb +189 -70
  33. data/lib/aspera/cli/plugins/ats.rb +15 -13
  34. data/lib/aspera/cli/plugins/config.rb +86 -213
  35. data/lib/aspera/cli/plugins/console.rb +49 -18
  36. data/lib/aspera/cli/plugins/cos.rb +4 -4
  37. data/lib/aspera/cli/plugins/faspex.rb +45 -51
  38. data/lib/aspera/cli/plugins/faspex5.rb +162 -163
  39. data/lib/aspera/cli/plugins/faspio.rb +6 -5
  40. data/lib/aspera/cli/plugins/httpgw.rb +2 -2
  41. data/lib/aspera/cli/plugins/node.rb +144 -162
  42. data/lib/aspera/cli/plugins/orchestrator.rb +10 -14
  43. data/lib/aspera/cli/plugins/preview.rb +26 -29
  44. data/lib/aspera/cli/plugins/server.rb +28 -28
  45. data/lib/aspera/cli/plugins/shares.rb +40 -28
  46. data/lib/aspera/cli/sync_actions.rb +101 -80
  47. data/lib/aspera/cli/transfer_agent.rb +51 -50
  48. data/lib/aspera/cli/transfer_progress.rb +29 -20
  49. data/lib/aspera/cli/version.rb +1 -1
  50. data/lib/aspera/cli/wizard.rb +160 -0
  51. data/lib/aspera/colors.rb +13 -8
  52. data/lib/aspera/command_line_builder.rb +28 -22
  53. data/lib/aspera/command_line_converter.rb +31 -0
  54. data/lib/aspera/environment.rb +144 -101
  55. data/lib/aspera/faspex_gw.rb +1 -1
  56. data/lib/aspera/faspex_postproc.rb +3 -2
  57. data/lib/aspera/hash_ext.rb +1 -1
  58. data/lib/aspera/id_generator.rb +10 -10
  59. data/lib/aspera/keychain/base.rb +18 -0
  60. data/lib/aspera/keychain/encrypted_hash.rb +6 -12
  61. data/lib/aspera/keychain/factory.rb +9 -3
  62. data/lib/aspera/keychain/hashicorp_vault.rb +9 -6
  63. data/lib/aspera/keychain/macos_security.rb +13 -13
  64. data/lib/aspera/log.rb +69 -20
  65. data/lib/aspera/nagios.rb +5 -6
  66. data/lib/aspera/node_simulator.rb +12 -7
  67. data/lib/aspera/oauth/base.rb +5 -3
  68. data/lib/aspera/oauth/factory.rb +24 -18
  69. data/lib/aspera/oauth/jwt.rb +13 -1
  70. data/lib/aspera/oauth/url_json.rb +3 -3
  71. data/lib/aspera/oauth/web.rb +5 -3
  72. data/lib/aspera/persistency_folder.rb +2 -2
  73. data/lib/aspera/preview/file_types.rb +4 -3
  74. data/lib/aspera/preview/generator.rb +25 -12
  75. data/lib/aspera/preview/terminal.rb +10 -7
  76. data/lib/aspera/preview/utils.rb +11 -9
  77. data/lib/aspera/products/connect.rb +1 -1
  78. data/lib/aspera/products/desktop.rb +1 -1
  79. data/lib/aspera/products/other.rb +2 -2
  80. data/lib/aspera/products/transferd.rb +8 -6
  81. data/lib/aspera/proxy_auto_config.rb +1 -1
  82. data/lib/aspera/rest.rb +29 -22
  83. data/lib/aspera/rest_call_error.rb +1 -1
  84. data/lib/aspera/resumer.rb +1 -1
  85. data/lib/aspera/secret_hider.rb +46 -40
  86. data/lib/aspera/ssh.rb +13 -3
  87. data/lib/aspera/sync/args.schema.yaml +102 -0
  88. data/lib/aspera/sync/conf.schema.yaml +701 -0
  89. data/lib/aspera/sync/database.rb +83 -0
  90. data/lib/aspera/{transfer/sync.rb → sync/operations.rb} +132 -65
  91. data/lib/aspera/temp_file_manager.rb +3 -2
  92. data/lib/aspera/transfer/error.rb +1 -1
  93. data/lib/aspera/transfer/error_info.rb +1 -2
  94. data/lib/aspera/transfer/faux_file.rb +11 -10
  95. data/lib/aspera/transfer/parameters.rb +6 -5
  96. data/lib/aspera/transfer/spec.rb +15 -1
  97. data/lib/aspera/transfer/spec.schema.yaml +316 -293
  98. data/lib/aspera/transfer/spec_doc.rb +34 -16
  99. data/lib/aspera/transfer/uri.rb +5 -5
  100. data/lib/aspera/uri_reader.rb +14 -10
  101. data/lib/aspera/web_auth.rb +2 -2
  102. data/lib/aspera/web_server_simple.rb +2 -2
  103. data.tar.gz.sig +0 -0
  104. metadata +15 -13
  105. metadata.gz.sig +2 -2
  106. data/lib/aspera/transfer/async_conf.schema.yaml +0 -716
  107. data/lib/aspera/transfer/convert.rb +0 -29
  108. data/lib/aspera/transfer/sync_instance.schema.yaml +0 -20
  109. data/lib/aspera/transfer/sync_session.schema.yaml +0 -86
@@ -59,10 +59,10 @@ module Aspera
59
59
  Log.log.debug{"detect error: #{e}"}
60
60
  end
61
61
  raise error if error
62
- return nil
62
+ return
63
63
  end
64
64
 
65
- def wizard(object:, private_key_path: nil, pub_key_pem: nil)
65
+ def wizard(object:, _private_key_path: nil, _pub_key_pem: nil)
66
66
  options = object.options
67
67
  return {
68
68
  preset_value: {
@@ -77,20 +77,24 @@ module Aspera
77
77
  def declare_options(options)
78
78
  return if @options_declared
79
79
  @options_declared = true
80
+ @dynamic_key = nil
80
81
  options.declare(:validator, 'Identifier of validator (optional for central)')
81
82
  options.declare(:asperabrowserurl, 'URL for simple aspera web ui', default: 'https://asperabrowser.mybluemix.net')
82
- options.declare(:sync_name, 'Sync name')
83
83
  options.declare(
84
- :default_ports, 'Use standard FASP ports or get from node api (gen4)', values: :bool, default: :yes,
85
- handler: {o: Api::Node, m: :use_standard_ports})
84
+ :default_ports, 'Gen4: Use standard FASP ports (true) or get from node API (false)', values: :bool, default: :yes,
85
+ handler: {o: Api::Node, m: :use_standard_ports}
86
+ )
86
87
  options.declare(
87
- :node_cache, 'Set to no to force actual file system read (gen4)', values: :bool,
88
- handler: {o: Api::Node, m: :use_node_cache})
89
- options.declare(:root_id, 'File id of top folder when using access key (override AK root id)')
88
+ :node_cache, 'Gen4: Set to no to force actual file system read', values: :bool,
89
+ handler: {o: Api::Node, m: :use_node_cache}
90
+ )
91
+ options.declare(:root_id, 'Gen4: File id of top folder when using access key (override AK root id)')
92
+ options.declare(:dynamic_key, 'Private key PEM to use for dynamic key auth', handler: {o: Api::Node, m: :use_dynamic_key})
90
93
  SyncActions.declare_options(options)
91
94
  options.parse_options!
92
95
  end
93
96
 
97
+ # Using /files/browse: is it a folder (node and shares)
94
98
  def gen3_entry_folder?(entry)
95
99
  FOLDER_TYPES.include?(entry['type'])
96
100
  end
@@ -105,10 +109,10 @@ module Aspera
105
109
  '</soapenv:Envelope>'
106
110
  # spellchecker: enable
107
111
 
108
- # fields removed in result of search
112
+ # Fields removed in result of search
109
113
  SEARCH_REMOVE_FIELDS = %w[basename permissions].freeze
110
114
 
111
- # actions in execute_command_gen3
115
+ # Actions in execute_command_gen3
112
116
  COMMANDS_GEN3 = %i[search space mkdir mklink mkfile rename delete browse upload download cat sync transport]
113
117
 
114
118
  BASE_ACTIONS = %i[api_details].concat(COMMANDS_GEN3).freeze
@@ -121,7 +125,7 @@ module Aspera
121
125
  # actions used commonly when a node is involved
122
126
  COMMON_ACTIONS = %i[access_keys].concat(BASE_ACTIONS).concat(SPECIAL_ACTIONS).freeze
123
127
 
124
- private_constant(*%i[CENTRAL_SOAP_API_TEST SEARCH_REMOVE_FIELDS BASE_ACTIONS SPECIAL_ACTIONS V3_IN_V4_ACTIONS COMMON_ACTIONS])
128
+ private_constant :CENTRAL_SOAP_API_TEST, :SEARCH_REMOVE_FIELDS, :BASE_ACTIONS, :SPECIAL_ACTIONS, :V3_IN_V4_ACTIONS, :COMMON_ACTIONS
125
129
 
126
130
  # used in aoc
127
131
  NODE4_READ_ACTIONS = %i[bearer_token_node node_info browse find].freeze
@@ -138,16 +142,16 @@ module Aspera
138
142
 
139
143
  # @param api [Rest] an existing API object for the Node API
140
144
  # @param prefix_path [String,nil] for Faspex 4, allows browsing a package
141
- def initialize(api: nil, prefix_path: nil, **env)
145
+ def initialize(context:, api: nil, prefix_path: nil)
142
146
  @prefix_path = prefix_path
143
- super(**env, basic_options: api.nil?)
147
+ super(context: context, basic_options: api.nil?)
144
148
  Node.declare_options(options)
145
- return if env[:broker].only_manual?
149
+ return if context.only_manual?
146
150
  @api_node =
147
151
  if !api.nil?
148
152
  # this can be Api::Node or Rest (Shares)
149
153
  api
150
- elsif OAuth::Factory.bearer?(options.get_option(:password, mandatory: true))
154
+ elsif OAuth::Factory.bearer_auth?(options.get_option(:password, mandatory: true))
151
155
  # info is provided like node_info of aoc
152
156
  Api::Node.new(
153
157
  base_url: options.get_option(:url, mandatory: true),
@@ -161,7 +165,8 @@ module Aspera
161
165
  type: :basic,
162
166
  username: options.get_option(:username, mandatory: true),
163
167
  password: options.get_option(:password, mandatory: true)
164
- })
168
+ }
169
+ )
165
170
  end
166
171
  end
167
172
 
@@ -188,18 +193,17 @@ module Aspera
188
193
  result = success_msg
189
194
  if p.key?('error')
190
195
  Log.log.error{"#{p['error']['user_message']} : #{p['path']}"}
191
- result = "ERROR: #{p['error']['user_message']}"
196
+ result = p['error']['user_message']
192
197
  errors.push([p['path'], p['error']['user_message']])
193
198
  end
194
199
  final_result[:data].push({type => p['path'], 'result' => result})
195
200
  end
196
201
  # one error make all fail
197
- unless errors.empty?
198
- raise errors.map{ |i| "#{i.first}: #{i.last}"}.join(', ')
199
- end
202
+ raise errors.map{ |i| "#{i.first}: #{i.last}"}.join(', ') unless errors.empty?
200
203
  return c_result_remove_prefix_path(final_result, type)
201
204
  end
202
205
 
206
+ # Gen3 API
203
207
  def browse_gen3
204
208
  folders_to_process = [get_one_argument_with_prefix('path')]
205
209
  query = options.get_option(:query, default: {})
@@ -236,9 +240,7 @@ module Aspera
236
240
  formatter.display_item_count(response['item_count'], total_count)
237
241
  break
238
242
  end
239
- if recursive
240
- folders_to_process.concat(items.select{ |i| Node.gen3_entry_folder?(i)}.map{ |i| i['path']})
241
- end
243
+ folders_to_process.concat(items.select{ |i| Node.gen3_entry_folder?(i)}.map{ |i| i['path']}) if recursive
242
244
  if !max_items.nil? && (all_items.count >= max_items)
243
245
  all_items = all_items.slice(0, max_items) if all_items.count > max_items
244
246
  break
@@ -350,7 +352,7 @@ module Aspera
350
352
  transfer_spec.delete_if{ |_k, v| v.nil?}
351
353
  # delete this part, as the returned value contains only destination, and not sources
352
354
  # transfer_spec.delete('paths') if command.eql?(:upload)
353
- Log.log.debug{Log.dump(:ts, transfer_spec)}
355
+ Log.dump(:ts, transfer_spec)
354
356
  transfer_spec
355
357
  end
356
358
  when :upload, :download
@@ -364,10 +366,12 @@ module Aspera
364
366
  end
365
367
  # add fixed parameters if any (for COS)
366
368
  @api_node.add_tspec_info(request_transfer_spec) if @api_node.respond_to?(:add_tspec_info)
369
+ Api::Node.add_public_key(request_transfer_spec)
367
370
  # prepare payload for single request
368
371
  setup_payload = {transfer_requests: [{transfer_request: request_transfer_spec}]}
369
372
  # only one request, so only one answer
370
373
  transfer_spec = @api_node.create("files/#{command}_setup", setup_payload)['transfer_specs'].first['transfer_spec']
374
+ Api::Node.add_private_key(transfer_spec)
371
375
  # delete this part, as the returned value contains only destination, and not sources
372
376
  transfer_spec.delete('paths') if command.eql?(:upload)
373
377
  return Main.result_transfer(transfer.start(transfer_spec))
@@ -376,7 +380,8 @@ module Aspera
376
380
  File.basename(remote_path)
377
381
  result = @api_node.call(
378
382
  operation: 'GET',
379
- subpath: "files/#{URI.encode_www_form_component(remote_path)}/contents")
383
+ subpath: "files/#{URI.encode_www_form_component(remote_path)}/contents"
384
+ )
380
385
  return Main.result_text(result[:http].body)
381
386
  when :transport
382
387
  return Main.result_single_object(@api_node.transport_params)
@@ -393,8 +398,12 @@ module Aspera
393
398
  ak_command = options.get_next_command(%i[do set_bearer_key].concat(Plugin::ALL_OPS))
394
399
  case ak_command
395
400
  when *Plugin::ALL_OPS
396
- return entity_command(ak_command, @api_node, 'access_keys') do |field, value|
397
- raise Cli::BadIdentifier, 'only selector: %id:self' unless field.eql?('id') && value.eql?('self')
401
+ return entity_execute(
402
+ api: @api_node,
403
+ entity: 'access_keys',
404
+ command: ak_command
405
+ ) do |field, value|
406
+ raise BadArgument, 'only selector: %id:self' unless field.eql?('id') && value.eql?('self')
398
407
  @api_node.read('access_keys/self')['id']
399
408
  end
400
409
  when :do
@@ -456,9 +465,7 @@ module Aspera
456
465
  when :license
457
466
  # requires: asnodeadmin -mu <node user> --acl-add=internal --internal
458
467
  node_license = @api_node.read('license')
459
- if node_license['failure'].is_a?(String) && node_license['failure'].include?('ACL')
460
- Log.log.error('server must have: asnodeadmin -mu <node user> --acl-add=internal --internal')
461
- end
468
+ Log.log.error('server must have: asnodeadmin -mu <node user> --acl-add=internal --internal') if node_license['failure'].is_a?(String) && node_license['failure'].include?('ACL')
462
469
  return Main.result_single_object(node_license)
463
470
  when :api_details
464
471
  return Main.result_single_object({base_url: @api_node.base_url}.merge(@api_node.params))
@@ -469,7 +476,7 @@ module Aspera
469
476
  # @return [Hash] api and main file id for given path or id in next argument
470
477
  def apifid_from_next_arg(top_file_id)
471
478
  file_path = instance_identifier(description: 'path or %id:<id>') do |attribute, value|
472
- raise Cli::BadIdentifier, 'Only selection "id" is supported (file id)' unless attribute.eql?('id')
479
+ raise BadArgument, 'Only selection "id" is supported (file id)' unless attribute.eql?('id')
473
480
  # directly return result for method
474
481
  return {api: @api_node, file_id: value}
475
482
  end
@@ -487,7 +494,7 @@ module Aspera
487
494
  command_legacy = options.get_next_command(V3_IN_V4_ACTIONS)
488
495
  # TODO: shall we support all methods here ? what if there is a link ?
489
496
  apifid = @api_node.resolve_api_fid(top_file_id, '')
490
- return Node.new(**init_params, api: apifid[:api]).execute_action(command_legacy)
497
+ return Node.new(context: context, api: apifid[:api]).execute_action(command_legacy)
491
498
  when :node_info, :bearer_token_node
492
499
  apifid = apifid_from_next_arg(top_file_id)
493
500
  result = {
@@ -506,8 +513,8 @@ module Aspera
506
513
  end
507
514
  return Main.result_single_object(result) if command_repo.eql?(:node_info)
508
515
  # check format of bearer token
509
- OAuth::Factory.bearer_extract(result[:password])
510
- return Main.result_status(result[:password])
516
+ OAuth::Factory.bearer_token(result[:password])
517
+ return Main.result_text(result[:password])
511
518
  when :browse
512
519
  apifid = apifid_from_next_arg(top_file_id)
513
520
  file_info = apifid[:api].read_with_cache("files/#{apifid[:file_id]}")
@@ -521,9 +528,8 @@ module Aspera
521
528
  find_lambda = Api::Node.file_matcher_from_argument(options)
522
529
  return Main.result_object_list(@api_node.find_files(apifid[:file_id], find_lambda), fields: ['path'])
523
530
  when :mkdir, :mklink, :mkfile
524
- containing_folder_path = options.get_next_argument('path').split(Api::Node::PATH_SEPARATOR)
525
- new_item = containing_folder_path.pop
526
- apifid = @api_node.resolve_api_fid(top_file_id, containing_folder_path.join(Api::Node::PATH_SEPARATOR), true)
531
+ containing_folder_path, new_item = Api::Node.split_folder(options.get_next_argument('path'))
532
+ apifid = @api_node.resolve_api_fid(top_file_id, containing_folder_path, true)
527
533
  query = options.get_option(:query, mandatory: false)
528
534
  check_exists = true
529
535
  payload = {name: new_item}
@@ -577,61 +583,28 @@ module Aspera
577
583
  when :sync
578
584
  return execute_sync_action do |sync_direction, _local_path, remote_path|
579
585
  # Gen4 API
580
- # direction is push pull, bidi
581
586
  Aspera.assert_values(sync_direction, %i[push pull bidi])
582
587
  ts_direction = case sync_direction
583
588
  when :push, :bidi then Transfer::Spec::DIRECTION_SEND
584
589
  when :pull then Transfer::Spec::DIRECTION_RECEIVE
585
590
  else Aspera.error_unreachable_line
586
591
  end
587
- # remote is specified by option to_folder
592
+ # remote is specified by option: `to_folder`
588
593
  apifid = @api_node.resolve_api_fid(top_file_id, remote_path)
589
- transfer_spec = apifid[:api].transfer_spec_gen4(apifid[:file_id], ts_direction)
590
- Log.log.debug{Log.dump(:ts, transfer_spec)}
591
- transfer_spec
594
+ apifid[:api].transfer_spec_gen4(apifid[:file_id], ts_direction)
592
595
  end
593
596
  when :upload
594
597
  apifid = @api_node.resolve_api_fid(top_file_id, transfer.destination_folder(Transfer::Spec::DIRECTION_SEND), true)
595
598
  return Main.result_transfer(transfer.start(apifid[:api].transfer_spec_gen4(apifid[:file_id], Transfer::Spec::DIRECTION_SEND)))
596
599
  when :download
597
- source_paths = transfer.ts_source_paths
598
- # special case for AoC : all files must be in same folder
599
- source_folder = source_paths.shift['source']
600
- # if a single file: split into folder and path
601
- apifid = @api_node.resolve_api_fid(top_file_id, source_folder, true)
602
- if source_paths.empty?
603
- # get precise info in this element
604
- file_info = apifid[:api].read("files/#{apifid[:file_id]}")
605
- case file_info['type']
606
- when 'file'
607
- # if the single source is a file, we need to split into folder path and filename
608
- src_dir_elements = source_folder.split(Api::Node::PATH_SEPARATOR)
609
- # filename is the last one, source folder is what remains
610
- source_paths = [{'source' => src_dir_elements.pop}]
611
- apifid = @api_node.resolve_api_fid(top_file_id, src_dir_elements.join(Api::Node::PATH_SEPARATOR), true)
612
- when 'link', 'folder'
613
- # single source is 'folder' or 'link'
614
- # TODO: add this ? , 'destination'=>file_info['name']
615
- source_paths = [{'source' => '.'}]
616
- else
617
- raise BadArgument, "Unknown source type: #{file_info['type']}"
618
- end
619
- end
600
+ apifid, source_paths = @api_node.resolve_api_fid_paths(top_file_id, transfer.ts_source_paths)
620
601
  return Main.result_transfer(transfer.start(apifid[:api].transfer_spec_gen4(apifid[:file_id], Transfer::Spec::DIRECTION_RECEIVE, {'paths'=>source_paths})))
621
602
  when :cat
622
- source_paths = transfer.ts_source_paths
623
- source_folder = source_paths.shift['source']
624
- if source_paths.empty?
625
- source_folder = source_folder.split(Api::Node::PATH_SEPARATOR)
626
- source_paths = [{'source' => source_folder.pop}]
627
- source_folder = source_folder.join(Api::Node::PATH_SEPARATOR)
628
- end
629
- raise Cli::BadArgument, 'one file at a time only in HTTP mode' if source_paths.length > 1
630
- file_name = source_paths.first['source']
631
- apifid = @api_node.resolve_api_fid(top_file_id, File.join(source_folder, file_name))
603
+ apifid = apifid_from_next_arg(top_file_id)
632
604
  result = apifid[:api].call(
633
605
  operation: 'GET',
634
- subpath: "files/#{apifid[:file_id]}/content")
606
+ subpath: "files/#{apifid[:file_id]}/content"
607
+ )
635
608
  return Main.result_text(result[:http].body)
636
609
  when :show
637
610
  apifid = apifid_from_next_arg(top_file_id)
@@ -649,7 +622,7 @@ module Aspera
649
622
  subpath: "files/#{apifid[:file_id]}/preview",
650
623
  headers: {'Accept' => 'image/png'}
651
624
  )
652
- return Main.result_image(result[:http].body, formatter: formatter)
625
+ return Main.result_image(result[:http].body)
653
626
  when :permission
654
627
  apifid = apifid_from_next_arg(top_file_id)
655
628
  command_perm = options.get_next_command(%i[list show create delete])
@@ -666,7 +639,7 @@ module Aspera
666
639
  perm_id = instance_identifier
667
640
  return Main.result_single_object(apifid[:api].read("permissions/#{perm_id}"))
668
641
  when :delete
669
- return do_bulk_operation(command: command_perm, descr: 'identifier', values: :identifier) do |one_id|
642
+ return do_bulk_operation(command: command_perm, values: :identifier) do |one_id|
670
643
  apifid[:api].delete("permissions/#{one_id}")
671
644
  # notify application of deletion
672
645
  the_app = apifid[:api].app_info
@@ -693,25 +666,30 @@ module Aspera
693
666
  Aspera.error_unreachable_line
694
667
  end
695
668
 
696
- # This is older API
669
+ # Search /async by name
670
+ # @param field [String] name of the field to search
671
+ # @param value [String] value of the field to search
672
+ # @return [Integer] id of the sync
673
+ # @raise [Cli::BadArgument] if no such sync, or not by name
674
+ def async_lookup(field, value)
675
+ raise Cli::BadArgument, "Only search by name is supported (#{field})" unless field.eql?('name')
676
+ async_ids = @api_node.read('async/list')['sync_ids']
677
+ summaries = @api_node.create('async/summary', {'syncs' => async_ids})['sync_summaries']
678
+ selected = summaries.find{ |s| s['name'].eql?(value)}
679
+ raise Cli::BadIdentifier.new('sync', "#{field}=#{value}") if selected.nil?
680
+ return selected['snid']
681
+ end
682
+
683
+ # Node API: /async (stats only)
697
684
  def execute_async
698
685
  command = options.get_next_command(%i[list delete files show counters bandwidth])
699
686
  unless command.eql?(:list)
700
- async_name = options.get_option(:sync_name)
701
- if async_name.nil?
702
- async_id = instance_identifier
703
- if async_id.eql?(SpecialValues::ALL) && %i[show delete].include?(command)
704
- async_ids = @api_node.read('async/list')['sync_ids']
705
- else
706
- Integer(async_id) # must be integer
707
- async_ids = [async_id]
708
- end
709
- else
687
+ async_id = instance_identifier{ |field, value| async_lookup(field, value)}
688
+ if async_id.eql?(SpecialValues::ALL)
689
+ raise Cli::BadArgument, 'ALL only for show and delete' unless %i[show delete].include?(command)
710
690
  async_ids = @api_node.read('async/list')['sync_ids']
711
- summaries = @api_node.create('async/summary', {'syncs' => async_ids})['sync_summaries']
712
- selected = summaries.find{ |s| s['name'].eql?(async_name)}
713
- raise Cli::BadIdentifier, "no such sync: #{async_name}" if selected.nil?
714
- async_id = selected['snid']
691
+ else
692
+ Integer(async_id) # must be integer
715
693
  async_ids = [async_id]
716
694
  end
717
695
  post_data = {'syncs' => async_ids}
@@ -719,7 +697,7 @@ module Aspera
719
697
  case command
720
698
  when :list
721
699
  resp = @api_node.read('async/list')['sync_ids']
722
- return Main.result_value_list(resp, name: 'id')
700
+ return Main.result_value_list(resp)
723
701
  when :show
724
702
  resp = @api_node.create('async/summary', post_data)['sync_summaries']
725
703
  return Main.result_empty if resp.empty?
@@ -729,7 +707,7 @@ module Aspera
729
707
  resp = @api_node.create('async/delete', post_data)
730
708
  return Main.result_single_object(resp)
731
709
  when :bandwidth
732
- post_data['seconds'] = 100 # TODO: as parameter with --value
710
+ post_data['seconds'] = 100 # TODO: as parameter with --query
733
711
  resp = @api_node.create('async/bandwidth', post_data)
734
712
  data = resp['bandwidth_data']
735
713
  return Main.result_empty if data.empty?
@@ -755,10 +733,10 @@ module Aspera
755
733
  'sync_files',
756
734
  options.get_option(:url, mandatory: true),
757
735
  options.get_option(:username, mandatory: true),
758
- async_id]))
759
- unless iteration_data.first.nil?
760
- data.select!{ |l| l['fnid'].to_i > iteration_data.first}
761
- end
736
+ async_id
737
+ ])
738
+ )
739
+ data.select!{ |l| l['fnid'].to_i > iteration_data.first} unless iteration_data.first.nil?
762
740
  iteration_data[0] = data.last['fnid'].to_i unless data.empty?
763
741
  end
764
742
  return Main.result_empty if data.empty?
@@ -771,10 +749,11 @@ module Aspera
771
749
  end
772
750
  end
773
751
 
752
+ # Search /asyncs by name
753
+ # @param field [String] name of the field to search
754
+ # @param value [String] value of the field to search
774
755
  # @return [Integer] id of the sync
775
756
  # @raise [Cli::BadArgument] if no such sync, or not by name
776
- # @param [String] field name of the field to search
777
- # @param [String] value value of the field to search
778
757
  def ssync_lookup(field, value)
779
758
  raise Cli::BadArgument, "Only search by name is supported (#{field})" unless field.eql?('name')
780
759
  @api_node.read('asyncs')['ids'].each do |id|
@@ -782,7 +761,41 @@ module Aspera
782
761
  # name is unique, so we can return
783
762
  return id if sync_info[field].eql?(value)
784
763
  end
785
- raise Cli::BadIdentifier, "no such sync: #{field}=#{value}"
764
+ raise Cli::BadIdentifier.new('ssync', "#{field}=#{value}")
765
+ end
766
+
767
+ WATCH_FOLDER_MUL = %i[create list].freeze
768
+ WATCH_FOLDER_SING = %i[show modify delete state].freeze
769
+ private_constant :WATCH_FOLDER_MUL, :WATCH_FOLDER_SING
770
+
771
+ def watch_folder_action
772
+ res_class_path = 'v3/watchfolders'
773
+ command = options.get_next_command(WATCH_FOLDER_MUL + WATCH_FOLDER_SING)
774
+ if WATCH_FOLDER_SING.include?(command)
775
+ one_res_id = instance_identifier
776
+ one_res_path = "#{res_class_path}/#{one_res_id}"
777
+ end
778
+ # hum, to avoid: Unable to convert 2016_09_14 configuration
779
+ @api_node.params[:headers] ||= {}
780
+ @api_node.params[:headers]['X-aspera-WF-version'] = '2017_10_23'
781
+ case command
782
+ when :create
783
+ resp = @api_node.create(res_class_path, value_create_modify(command: command))
784
+ return Main.result_status("#{resp['id']} created")
785
+ when :list
786
+ resp = @api_node.read(res_class_path, query_read_delete)
787
+ return Main.result_value_list(resp['ids'])
788
+ when :show
789
+ return Main.result_single_object(@api_node.read(one_res_path))
790
+ when :modify
791
+ @api_node.update(one_res_path, options.get_option(:query, mandatory: true))
792
+ return Main.result_status("#{one_res_id} updated")
793
+ when :delete
794
+ @api_node.delete(one_res_path)
795
+ return Main.result_status("#{one_res_id} deleted")
796
+ when :state
797
+ return Main.result_single_object(@api_node.read("#{one_res_path}/state"))
798
+ end
786
799
  end
787
800
 
788
801
  ACTIONS = %i[
@@ -800,16 +813,22 @@ module Aspera
800
813
  telemetry
801
814
  ].concat(COMMON_ACTIONS).freeze
802
815
 
803
- def execute_action(command=nil)
816
+ def execute_action(command = nil)
804
817
  command ||= options.get_next_command(ACTIONS)
805
818
  case command
806
819
  when *COMMON_ACTIONS then return execute_simple_common(command)
807
820
  when :async then return execute_async # former API
808
821
  when :ssync
809
- # newer API
810
- sync_command = options.get_next_command(%i[start stop bandwidth counters files state summary].concat(Plugin::ALL_OPS) - %i[modify])
822
+ # Node API: /asyncs (newer)
823
+ sync_command = options.get_next_command(%i[start stop bandwidth counters files state summary] + Plugin::ALL_OPS - %i[modify])
811
824
  case sync_command
812
- when *Plugin::ALL_OPS then return entity_command(sync_command, @api_node, 'asyncs', item_list_key: 'ids'){ |field, value| ssync_lookup(field, value)}
825
+ when *Plugin::ALL_OPS
826
+ return entity_execute(
827
+ api: @api_node,
828
+ entity: :asyncs,
829
+ command: sync_command,
830
+ items_key: 'ids'
831
+ ){ |field, value| ssync_lookup(field, value)}
813
832
  else
814
833
  asyncs_id = instance_identifier{ |field, value| ssync_lookup(field, value)}
815
834
  if %i[start stop].include?(sync_command)
@@ -859,7 +878,8 @@ module Aspera
859
878
  'node_transfers',
860
879
  options.get_option(:url, mandatory: true),
861
880
  options.get_option(:username, mandatory: true)
862
- ]))
881
+ ])
882
+ )
863
883
  if transfer_filter.delete('reset')
864
884
  iteration_persistency.data.clear
865
885
  iteration_persistency.save
@@ -870,11 +890,7 @@ module Aspera
870
890
  max_items = transfer_filter.delete(MAX_ITEMS)
871
891
  transfers_data = call_with_iteration(api: @api_node, operation: 'GET', subpath: 'ops/transfers', max: max_items, query: transfer_filter, iteration: iteration_persistency&.data)
872
892
  iteration_persistency&.save
873
- return {
874
- type: :object_list,
875
- data: transfers_data,
876
- fields: %w[id status start_spec.direction start_spec.remote_user start_spec.remote_host start_spec.destination_path]
877
- }
893
+ return Main.result_object_list(transfers_data, fields: %w[id status start_spec.direction start_spec.remote_user start_spec.remote_host start_spec.destination_path])
878
894
  when :sessions
879
895
  transfers_data = @api_node.read('ops/transfers', query_read_delete)
880
896
  sessions = transfers_data.map{ |t| t['sessions']}.flatten
@@ -882,11 +898,7 @@ module Aspera
882
898
  session['start_time'] = Time.at(session['start_time_usec'] / 1_000_000.0).utc.iso8601(0)
883
899
  session['end_time'] = Time.at(session['end_time_usec'] / 1_000_000.0).utc.iso8601(0)
884
900
  end
885
- return {
886
- type: :object_list,
887
- data: sessions,
888
- fields: %w[id status start_time end_time target_rate_kbps]
889
- }
901
+ return Main.result_object_list(sessions, fields: %w[id status start_time end_time target_rate_kbps])
890
902
  when :cancel
891
903
  resp = @api_node.cancel("ops/transfers/#{instance_identifier}")
892
904
  return Main.result_single_object(resp)
@@ -945,9 +957,7 @@ module Aspera
945
957
  end
946
958
  when :service
947
959
  command = options.get_next_command(%i[list create delete])
948
- if [:delete].include?(command)
949
- service_id = instance_identifier
950
- end
960
+ service_id = instance_identifier if [:delete].include?(command)
951
961
  case command
952
962
  when :list
953
963
  resp = @api_node.read('rund/services')
@@ -962,33 +972,7 @@ module Aspera
962
972
  return Main.result_status("#{service_id} deleted")
963
973
  end
964
974
  when :watch_folder
965
- res_class_path = 'v3/watchfolders'
966
- command = options.get_next_command(%i[create list show modify delete state])
967
- if %i[show modify delete state].include?(command)
968
- one_res_id = instance_identifier
969
- one_res_path = "#{res_class_path}/#{one_res_id}"
970
- end
971
- # hum, to avoid: Unable to convert 2016_09_14 configuration
972
- @api_node.params[:headers] ||= {}
973
- @api_node.params[:headers]['X-aspera-WF-version'] = '2017_10_23'
974
- case command
975
- when :create
976
- resp = @api_node.create(res_class_path, value_create_modify(command: command))
977
- return Main.result_status("#{resp['id']} created")
978
- when :list
979
- resp = @api_node.read(res_class_path, query_read_delete)
980
- return Main.result_value_list(resp['ids'], name: 'id')
981
- when :show
982
- return Main.result_single_object(@api_node.read(one_res_path))
983
- when :modify
984
- @api_node.update(one_res_path, options.get_option(:query, mandatory: true))
985
- return Main.result_status("#{one_res_id} updated")
986
- when :delete
987
- @api_node.delete(one_res_path)
988
- return Main.result_status("#{one_res_id} deleted")
989
- when :state
990
- return Main.result_single_object(@api_node.read("#{one_res_path}/state"))
991
- end
975
+ return watch_folder_action
992
976
  when :central
993
977
  command = options.get_next_command(%i[session file])
994
978
  validator_id = options.get_option(:validator)
@@ -1002,11 +986,7 @@ module Aspera
1002
986
  request_data = options.get_next_argument('request data', mandatory: false, validation: Hash, default: {})
1003
987
  request_data.deep_merge!({'validation' => validation}) unless validation.nil?
1004
988
  resp = @api_node.create('services/rest/transfers/v1/sessions', request_data)
1005
- return {
1006
- type: :object_list,
1007
- data: resp['session_info_result']['session_info'],
1008
- fields: %w[session_uuid status transport direction bytes_transferred]
1009
- }
989
+ return Main.result_object_list(resp['session_info_result']['session_info'], fields: %w[session_uuid status transport direction bytes_transferred])
1010
990
  end
1011
991
  when :file
1012
992
  command = options.get_next_command(%i[list modify])
@@ -1016,7 +996,7 @@ module Aspera
1016
996
  request_data.deep_merge!({'validation' => validation}) unless validation.nil?
1017
997
  resp = @api_node.create('services/rest/transfers/v1/files', request_data)
1018
998
  resp = JSON.parse(resp) if resp.is_a?(String)
1019
- Log.log.debug{Log.dump(:resp, resp)}
999
+ Log.dump(:resp, resp)
1020
1000
  return Main.result_object_list(resp['file_transfer_info_result']['file_transfer_info'], fields: %w[session_uuid file_id status path])
1021
1001
  when :modify
1022
1002
  request_data = options.get_next_argument('request data', mandatory: false, validation: Hash, default: {})
@@ -1036,12 +1016,12 @@ module Aspera
1036
1016
  Environment.instance.open_uri("#{options.get_option(:asperabrowserurl)}?goto=#{encoded_params}")
1037
1017
  return Main.result_status('done')
1038
1018
  when :basic_token
1039
- return Main.result_status(Rest.basic_authorization(options.get_option(:username, mandatory: true), options.get_option(:password, mandatory: true)))
1019
+ return Main.result_text(Rest.basic_authorization(options.get_option(:username, mandatory: true), options.get_option(:password, mandatory: true)))
1040
1020
  when :bearer_token
1041
1021
  private_key = OpenSSL::PKey::RSA.new(options.get_next_argument('private RSA key PEM value', validation: String))
1042
1022
  token_info = options.get_next_argument('user and group identification', validation: Hash)
1043
1023
  access_key = options.get_option(:username, mandatory: true)
1044
- return Main.result_status(Api::Node.bearer_token(payload: token_info, access_key: access_key, private_key: private_key))
1024
+ return Main.result_text(Api::Node.bearer_token(payload: token_info, access_key: access_key, private_key: private_key))
1045
1025
  when :simulator
1046
1026
  require 'aspera/node_simulator'
1047
1027
  parameters = value_create_modify(command: command, default: {}).symbolize_keys
@@ -1145,16 +1125,18 @@ module Aspera
1145
1125
  end
1146
1126
 
1147
1127
  # Executes the provided API call in loop
1148
- # @param api [Rest] the API to call
1149
- # @param iteration [Array] a single element array with the iteration token or nil
1150
- # @param max [Integer] maximum number of items to return, or nil for no limit
1151
- # @param query [Hash] query parameters to use for the API call
1152
- # @param call_args [Hash] additional arguments to pass to the API call
1128
+ # @param api [Rest] the API to call
1129
+ # @param iteration [Array] a single element array with the iteration token or nil
1130
+ # @param max [Integer] maximum number of items to return, or nil for no limit
1131
+ # @param query [Hash] query parameters to use for the API call
1132
+ # @param call_args [Hash] additional arguments to pass to the API call
1153
1133
  # @return [Array] list of items returned by the API call
1154
1134
  def call_with_iteration(api:, iteration: nil, max: nil, query: nil, **call_args)
1155
- query_token = query.clone || {}
1135
+ Aspera.assert_type(iteration, Array, NilClass){'iteration'}
1136
+ Aspera.assert_type(query, Hash, NilClass){'query'}
1137
+ query_token = query&.dup || {}
1156
1138
  item_list = []
1157
- query_token[:iteration_token] = iteration.first if iteration.is_a?(Array)
1139
+ query_token[:iteration_token] = iteration[0] unless iteration.nil?
1158
1140
  loop do
1159
1141
  result = api.call(**call_args, query: query_token)
1160
1142
  Aspera.assert_type(result[:data], Array){"Expected data to be an Array, got: #{result[:data].class}"}