aspera-cli 4.12.0 → 4.13.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/CHANGELOG.md +17 -0
  4. data/CONTRIBUTING.md +97 -22
  5. data/README.md +548 -394
  6. data/bin/ascli +3 -3
  7. data/docs/test_env.conf +12 -5
  8. data/lib/aspera/aoc.rb +42 -42
  9. data/lib/aspera/ascmd.rb +4 -3
  10. data/lib/aspera/cli/extended_value.rb +24 -37
  11. data/lib/aspera/cli/formatter.rb +6 -0
  12. data/lib/aspera/cli/info.rb +2 -4
  13. data/lib/aspera/cli/main.rb +6 -0
  14. data/lib/aspera/cli/manager.rb +15 -6
  15. data/lib/aspera/cli/plugin.rb +1 -5
  16. data/lib/aspera/cli/plugins/aoc.rb +23 -6
  17. data/lib/aspera/cli/plugins/config.rb +13 -6
  18. data/lib/aspera/cli/plugins/faspex.rb +4 -3
  19. data/lib/aspera/cli/plugins/faspex5.rb +175 -42
  20. data/lib/aspera/cli/plugins/node.rb +107 -50
  21. data/lib/aspera/cli/plugins/preview.rb +3 -3
  22. data/lib/aspera/cli/plugins/server.rb +11 -1
  23. data/lib/aspera/cli/plugins/sync.rb +3 -3
  24. data/lib/aspera/cli/transfer_agent.rb +24 -10
  25. data/lib/aspera/cli/version.rb +2 -1
  26. data/lib/aspera/command_line_builder.rb +2 -1
  27. data/lib/aspera/cos_node.rb +1 -1
  28. data/lib/aspera/fasp/agent_connect.rb +1 -1
  29. data/lib/aspera/fasp/agent_direct.rb +12 -12
  30. data/lib/aspera/fasp/agent_node.rb +14 -4
  31. data/lib/aspera/fasp/installation.rb +1 -0
  32. data/lib/aspera/fasp/parameters.rb +11 -3
  33. data/lib/aspera/fasp/parameters.yaml +3 -1
  34. data/lib/aspera/fasp/transfer_spec.rb +3 -1
  35. data/lib/aspera/faspex_gw.rb +1 -0
  36. data/lib/aspera/faspex_postproc.rb +2 -2
  37. data/lib/aspera/node.rb +11 -4
  38. data/lib/aspera/oauth.rb +3 -2
  39. data/lib/aspera/preview/file_types.rb +8 -6
  40. data/lib/aspera/preview/generator.rb +23 -11
  41. data/lib/aspera/preview/options.rb +3 -2
  42. data/lib/aspera/preview/terminal.rb +34 -0
  43. data/lib/aspera/preview/utils.rb +8 -8
  44. data/lib/aspera/rest.rb +5 -4
  45. data/lib/aspera/rest_call_error.rb +3 -1
  46. data/lib/aspera/secret_hider.rb +4 -4
  47. data/lib/aspera/sync.rb +39 -33
  48. data/lib/aspera/web_server_simple.rb +22 -18
  49. data.tar.gz.sig +0 -0
  50. metadata +39 -46
  51. metadata.gz.sig +0 -0
  52. data/examples/aoc.rb +0 -30
  53. data/examples/faspex4.rb +0 -94
  54. data/examples/node.rb +0 -96
  55. data/examples/server.rb +0 -93
@@ -7,6 +7,7 @@ require 'aspera/hash_ext'
7
7
  require 'aspera/id_generator'
8
8
  require 'aspera/node'
9
9
  require 'aspera/aoc'
10
+ require 'aspera/sync'
10
11
  require 'aspera/fasp/transfer_spec'
11
12
  require 'base64'
12
13
  require 'zlib'
@@ -14,6 +15,51 @@ require 'zlib'
14
15
  module Aspera
15
16
  module Cli
16
17
  module Plugins
18
+ class SyncSpecGen3
19
+ def initialize(api_node)
20
+ @api_node = api_node
21
+ end
22
+
23
+ def transfer_spec(direction, local_path, remote_path)
24
+ # empty transfer spec for authorization request
25
+ direction_sym = direction.to_sym
26
+ request_transfer_spec = {
27
+ type: Aspera::Sync::DIRECTION_TO_REQUEST_TYPE[direction_sym],
28
+ paths: {
29
+ source: remote_path,
30
+ destination: local_path
31
+ }
32
+ }
33
+ # add fixed parameters if any (for COS)
34
+ @api_node.add_tspec_info(request_transfer_spec) if @api_node.respond_to?(:add_tspec_info)
35
+ # prepare payload for single request
36
+ setup_payload = {transfer_requests: [{transfer_request: request_transfer_spec}]}
37
+ # only one request, so only one answer
38
+ transfer_spec = @api_node.create('files/sync_setup', setup_payload)[:data]['transfer_specs'].first['transfer_spec']
39
+ # API returns null tag... but async does not like it
40
+ transfer_spec.delete_if{ |_k, v| v.nil? }
41
+ # delete this part, as the returned value contains only destination, and not sources
42
+ # transfer_spec.delete('paths') if command.eql?(:upload)
43
+ Log.dump(:ts, transfer_spec)
44
+ return transfer_spec
45
+ end
46
+ end
47
+
48
+ class SyncSpecGen4
49
+ def initialize(api_node, top_file_id)
50
+ @api_node = api_node
51
+ @top_file_id = top_file_id
52
+ end
53
+
54
+ def transfer_spec(direction, local_path, remote_path)
55
+ # remote is specified by option to_folder
56
+ apifid = @api_node.resolve_api_fid(@top_file_id, remote_path)
57
+ transfer_spec = apifid[:api].transfer_spec_gen4(apifid[:file_id], Fasp::TransferSpec::DIRECTION_SEND)
58
+ Log.dump(:ts, transfer_spec)
59
+ return transfer_spec
60
+ end
61
+ end
62
+
17
63
  class Node < Aspera::Cli::BasicAuthPlugin
18
64
  class << self
19
65
  def detect(base_url)
@@ -30,7 +76,7 @@ module Aspera
30
76
  env[:options].add_opt_simple(:asperabrowserurl, 'URL for simple aspera web ui')
31
77
  env[:options].add_opt_simple(:sync_name, 'sync name')
32
78
  env[:options].add_opt_simple(:path, 'file or folder path for gen4 operation "file"')
33
- env[:options].add_opt_list(:token_type, %i[aspera basic hybrid], 'Type of token used for transfers')
79
+ env[:options].add_opt_list(:token_type, %i[aspera basic hybrid], 'type of token used for transfers')
34
80
  env[:options].add_opt_boolean(:default_ports, 'use standard FASP ports or get from node api (gen4)')
35
81
  env[:options].set_option(:asperabrowserurl, 'https://asperabrowser.mybluemix.net')
36
82
  env[:options].set_option(:token_type, :aspera)
@@ -53,7 +99,7 @@ module Aspera
53
99
  SEARCH_REMOVE_FIELDS = %w[basename permissions].freeze
54
100
 
55
101
  # actions in execute_command_gen3
56
- COMMANDS_GEN3 = %i[search space mkdir mklink mkfile rename delete browse upload download]
102
+ COMMANDS_GEN3 = %i[search space mkdir mklink mkfile rename delete browse upload download sync]
57
103
 
58
104
  BASE_ACTIONS = %i[api_details].concat(COMMANDS_GEN3).freeze
59
105
 
@@ -145,7 +191,7 @@ module Aspera
145
191
  # translates paths results into CLI result, and removes prefix
146
192
  def c_result_translate_rem_prefix(response, type, success_msg, path_prefix)
147
193
  errors = []
148
- resres = { data: [], type: :object_list, fields: [type, 'result']}
194
+ final_result = { data: [], type: :object_list, fields: [type, 'result']}
149
195
  JSON.parse(response[:http].body)['paths'].each do |p|
150
196
  result = success_msg
151
197
  if p.key?('error')
@@ -153,13 +199,13 @@ module Aspera
153
199
  result = 'ERROR: ' + p['error']['user_message']
154
200
  errors.push([p['path'], p['error']['user_message']])
155
201
  end
156
- resres[:data].push({type => p['path'], 'result' => result})
202
+ final_result[:data].push({type => p['path'], 'result' => result})
157
203
  end
158
204
  # one error make all fail
159
205
  unless errors.empty?
160
206
  raise errors.map{|i|"#{i.first}: #{i.last}"}.join(', ')
161
207
  end
162
- return c_result_remove_prefix_path(resres, type, path_prefix)
208
+ return c_result_remove_prefix_path(final_result, type, path_prefix)
163
209
  end
164
210
 
165
211
  # get path arguments from command line, and add prefix
@@ -187,7 +233,7 @@ module Aspera
187
233
  result = { type: :object_list, data: resp[:data]['items']}
188
234
  return Main.result_empty if result[:data].empty?
189
235
  result[:fields] = result[:data].first.keys.reject{|i|SEARCH_REMOVE_FIELDS.include?(i)}
190
- formatter.display_status("Items: #{resp[:data]['item_count']}/#{resp[:data]['total_count']}")
236
+ formatter.display_item_count(resp[:data]['item_count'], resp[:data]['total_count'])
191
237
  formatter.display_status("params: #{resp[:data]['parameters'].keys.map{|k|"#{k}:#{resp[:data]['parameters'][k]}"}.join(',')}")
192
238
  return c_result_remove_prefix_path(result, 'path', prefix_path)
193
239
  when :space
@@ -228,13 +274,16 @@ module Aspera
228
274
  case send_result['self']['type']
229
275
  when 'directory', 'container' # directory: node, container: shares
230
276
  result = { data: send_result['items'], type: :object_list, textify: lambda { |table_data| c_textify_browse(table_data) } }
231
- formatter.display_status("Items: #{send_result['item_count']}/#{send_result['total_count']}")
277
+ formatter.display_item_count(send_result['item_count'], send_result['total_count'])
232
278
  else # 'file','symbolic_link'
233
279
  result = { data: send_result['self'], type: :single_object}
234
280
  # result={ data: [send_result['self']] , type: :object_list, textify: lambda { |table_data| c_textify_browse(table_data) } }
235
281
  # raise "unknown type: #{send_result['self']['type']}"
236
282
  end
237
283
  return c_result_remove_prefix_path(result, 'path', prefix_path)
284
+ when :sync
285
+ node_sync = SyncSpecGen3.new(@api_node)
286
+ return Plugins::Sync.new(@agents, sync_spec: node_sync).execute_action
238
287
  when :upload, :download
239
288
  token_type = options.get_option(:token_type)
240
289
  # nil if Shares 1.x
@@ -244,7 +293,11 @@ module Aspera
244
293
  # empty transfer spec for authorization request
245
294
  request_transfer_spec = {}
246
295
  # set requested paths depending on direction
247
- request_transfer_spec[:paths] = command.eql?(:download) ? transfer.ts_source_paths : [{ destination: transfer.destination_folder('send') }]
296
+ request_transfer_spec[:paths] = if command.eql?(:download)
297
+ transfer.ts_source_paths
298
+ else
299
+ [{ destination: transfer.destination_folder(Fasp::TransferSpec::DIRECTION_SEND) }]
300
+ end
248
301
  # add fixed parameters if any (for COS)
249
302
  @api_node.add_tspec_info(request_transfer_spec) if @api_node.respond_to?(:add_tspec_info)
250
303
  # prepare payload for single request
@@ -330,7 +383,7 @@ module Aspera
330
383
  return { type: :single_object, data: @api_node.params }
331
384
  end
332
385
  end
333
-
386
+ GEN4_FILE_COMMANDS = %i[show modify permission thumbnail].freeze
334
387
  def execute_node_gen4_file_command(command_node_file, top_file_id)
335
388
  file_path = options.get_option(:path)
336
389
  apifid =
@@ -347,6 +400,14 @@ module Aspera
347
400
  update_param = options.get_next_argument('update data', type: Hash)
348
401
  apifid[:api].update("files/#{apifid[:file_id]}", update_param)[:data]
349
402
  return Main.result_status('Done')
403
+ when :thumbnail
404
+ result = apifid[:api].call(
405
+ operation: 'GET',
406
+ subpath: "files/#{apifid[:file_id]}/preview",
407
+ headers: {'Accept' => 'image/png'}
408
+ )
409
+ require 'aspera/preview/terminal'
410
+ return Main.result_status(Preview::Terminal.build(result[:http].body, reserved_lines: 3))
350
411
  when :permission
351
412
  command_perm = options.get_next_command(%i[list create delete])
352
413
  case command_perm
@@ -375,7 +436,7 @@ module Aspera
375
436
  the_app[:api].permissions_create_params(create_param: create_param, app_info: the_app) unless the_app.nil?
376
437
  # create permission
377
438
  created_data = apifid[:api].create('permissions', create_param)[:data]
378
- # bnotify application of creation
439
+ # notify application of creation
379
440
  the_app[:api].permissions_create_event(created_data: created_data, app_info: the_app) unless the_app.nil?
380
441
  return { type: :single_object, data: created_data}
381
442
  else raise "internal error:shall not reach here (#{command_perm})"
@@ -417,7 +478,7 @@ module Aspera
417
478
  if file_info['type'].eql?('folder')
418
479
  result = apifid[:api].read("files/#{apifid[:file_id]}/files", options.get_option(:value))
419
480
  items = result[:data]
420
- formatter.display_status("Items: #{result[:data].length}/#{result[:http]['X-Total-Count']}")
481
+ formatter.display_item_count(result[:data].length, result[:http]['X-Total-Count'])
421
482
  else
422
483
  items = [file_info]
423
484
  end
@@ -446,12 +507,8 @@ module Aspera
446
507
  {'path' => l_path}
447
508
  end
448
509
  when :sync
449
- # remote is specified by option to_folder
450
- apifid = @api_node.resolve_api_fid(top_file_id, transfer.destination_folder(Fasp::TransferSpec::DIRECTION_SEND))
451
- transfer_spec = apifid[:api].transfer_spec_gen4(apifid[:file_id], Fasp::TransferSpec::DIRECTION_SEND)
452
- Log.dump(:ts, transfer_spec)
453
- sync_plugin = Plugins::Sync.new(@agents, transfer_spec: transfer_spec)
454
- return sync_plugin.execute_action
510
+ node_sync = SyncSpecGen4.new(@api_node, top_file_id)
511
+ return Plugins::Sync.new(@agents, sync_spec: node_sync).execute_action
455
512
  when :upload
456
513
  apifid = @api_node.resolve_api_fid(top_file_id, transfer.destination_folder(Fasp::TransferSpec::DIRECTION_SEND))
457
514
  return Main.result_transfer(transfer.start(apifid[:api].transfer_spec_gen4(apifid[:file_id], Fasp::TransferSpec::DIRECTION_SEND)))
@@ -499,7 +556,7 @@ module Aspera
499
556
  save_to_file: File.join(transfer.destination_folder(Fasp::TransferSpec::DIRECTION_RECEIVE), file_name))
500
557
  return Main.result_status("downloaded: #{file_name}")
501
558
  when :file
502
- command_node_file = options.get_next_command(%i[show modify permission])
559
+ command_node_file = options.get_next_command(GEN4_FILE_COMMANDS)
503
560
  return execute_node_gen4_file_command(command_node_file, top_file_id)
504
561
  else raise "INTERNAL ERROR: no case for #{command_repo}"
505
562
  end # command_repo
@@ -510,43 +567,43 @@ module Aspera
510
567
  def execute_async
511
568
  command = options.get_next_command(%i[list delete files show counters bandwidth])
512
569
  unless command.eql?(:list)
513
- asyncname = options.get_option(:sync_name)
514
- if asyncname.nil?
515
- asyncid = instance_identifier
516
- if asyncid.eql?(VAL_ALL) && %i[show delete].include?(command)
517
- asyncids = @api_node.read('async/list')[:data]['sync_ids']
570
+ async_name = options.get_option(:sync_name)
571
+ if async_name.nil?
572
+ async_id = instance_identifier
573
+ if async_id.eql?(VAL_ALL) && %i[show delete].include?(command)
574
+ async_ids = @api_node.read('async/list')[:data]['sync_ids']
518
575
  else
519
- Integer(asyncid) # must be integer
520
- asyncids = [asyncid]
576
+ Integer(async_id) # must be integer
577
+ async_ids = [async_id]
521
578
  end
522
579
  else
523
- asyncids = @api_node.read('async/list')[:data]['sync_ids']
524
- summaries = @api_node.create('async/summary', {'syncs' => asyncids})[:data]['sync_summaries']
525
- selected = summaries.find{|s|s['name'].eql?(asyncname)}
526
- raise "no such sync: #{asyncname}" if selected.nil?
527
- asyncid = selected['snid']
528
- asyncids = [asyncid]
580
+ async_ids = @api_node.read('async/list')[:data]['sync_ids']
581
+ summaries = @api_node.create('async/summary', {'syncs' => async_ids})[:data]['sync_summaries']
582
+ selected = summaries.find{|s|s['name'].eql?(async_name)}
583
+ raise "no such sync: #{async_name}" if selected.nil?
584
+ async_id = selected['snid']
585
+ async_ids = [async_id]
529
586
  end
530
- pdata = {'syncs' => asyncids}
587
+ post_data = {'syncs' => async_ids}
531
588
  end
532
589
  case command
533
590
  when :list
534
591
  resp = @api_node.read('async/list')[:data]['sync_ids']
535
592
  return { type: :value_list, data: resp, name: 'id' }
536
593
  when :show
537
- resp = @api_node.create('async/summary', pdata)[:data]['sync_summaries']
594
+ resp = @api_node.create('async/summary', post_data)[:data]['sync_summaries']
538
595
  return Main.result_empty if resp.empty?
539
- return { type: :object_list, data: resp, fields: %w[snid name local_dir remote_dir] } if asyncid.eql?(VAL_ALL)
596
+ return { type: :object_list, data: resp, fields: %w[snid name local_dir remote_dir] } if async_id.eql?(VAL_ALL)
540
597
  return { type: :single_object, data: resp.first }
541
598
  when :delete
542
- resp = @api_node.create('async/delete', pdata)[:data]
599
+ resp = @api_node.create('async/delete', post_data)[:data]
543
600
  return { type: :single_object, data: resp, name: 'id' }
544
601
  when :bandwidth
545
- pdata['seconds'] = 100 # TODO: as parameter with --value
546
- resp = @api_node.create('async/bandwidth', pdata)[:data]
602
+ post_data['seconds'] = 100 # TODO: as parameter with --value
603
+ resp = @api_node.create('async/bandwidth', post_data)[:data]
547
604
  data = resp['bandwidth_data']
548
605
  return Main.result_empty if data.empty?
549
- data = data.first[asyncid]['data']
606
+ data = data.first[async_id]['data']
550
607
  return { type: :object_list, data: data, name: 'id' }
551
608
  when :files
552
609
  # count int
@@ -554,10 +611,10 @@ module Aspera
554
611
  # skip int
555
612
  # status int
556
613
  filter = options.get_option(:value)
557
- pdata.merge!(filter) unless filter.nil?
558
- resp = @api_node.create('async/files', pdata)[:data]
614
+ post_data.merge!(filter) unless filter.nil?
615
+ resp = @api_node.create('async/files', post_data)[:data]
559
616
  data = resp['sync_files']
560
- data = data.first[asyncid] unless data.empty?
617
+ data = data.first[async_id] unless data.empty?
561
618
  iteration_data = []
562
619
  skip_ids_persistency = nil
563
620
  if options.get_option(:once_only, is_type: :mandatory)
@@ -568,7 +625,7 @@ module Aspera
568
625
  'sync_files',
569
626
  options.get_option(:url, is_type: :mandatory),
570
627
  options.get_option(:username, is_type: :mandatory),
571
- asyncid]))
628
+ async_id]))
572
629
  unless iteration_data.first.nil?
573
630
  data.select!{|l| l['fnid'].to_i > iteration_data.first}
574
631
  end
@@ -578,7 +635,7 @@ module Aspera
578
635
  skip_ids_persistency&.save
579
636
  return { type: :object_list, data: data, name: 'id' }
580
637
  when :counters
581
- resp = @api_node.create('async/counters', pdata)[:data]['sync_counters'].first[asyncid].last
638
+ resp = @api_node.create('async/counters', post_data)[:data]['sync_counters'].first[async_id].last
582
639
  return Main.result_empty if resp.nil?
583
640
  return { type: :single_object, data: resp }
584
641
  end
@@ -586,7 +643,7 @@ module Aspera
586
643
 
587
644
  ACTIONS = %i[
588
645
  async
589
- sync
646
+ ssync
590
647
  stream
591
648
  transfer
592
649
  service
@@ -599,9 +656,9 @@ module Aspera
599
656
  command ||= options.get_next_command(ACTIONS)
600
657
  case command
601
658
  when *COMMON_ACTIONS then return execute_simple_common(command, prefix_path)
602
- when :async then return execute_async
603
- when :sync
604
- # newer api
659
+ when :async then return execute_async # former API
660
+ when :ssync
661
+ # newer API
605
662
  sync_command = options.get_next_command(%i[bandwidth counters files start state stop summary].concat(Plugin::ALL_OPS) - %i[modify])
606
663
  case sync_command
607
664
  when *Plugin::ALL_OPS then return entity_command(sync_command, @api_node, 'asyncs', item_list_key: 'ids')
@@ -665,7 +722,7 @@ module Aspera
665
722
  when :service
666
723
  command = options.get_next_command(%i[list create delete])
667
724
  if [:delete].include?(command)
668
- svcid = instance_identifier
725
+ service_id = instance_identifier
669
726
  end
670
727
  case command
671
728
  when :list
@@ -677,8 +734,8 @@ module Aspera
677
734
  resp = @api_node.create('rund/services', params)
678
735
  return Main.result_status("#{resp[:data]['id']} created")
679
736
  when :delete
680
- @api_node.delete("rund/services/#{svcid}")
681
- return Main.result_status("#{svcid} deleted")
737
+ @api_node.delete("rund/services/#{service_id}")
738
+ return Main.result_status("#{service_id} deleted")
682
739
  end
683
740
  when :watch_folder
684
741
  res_class_path = 'v3/watchfolders'
@@ -149,8 +149,8 @@ module Aspera
149
149
  if event['data']['direction'].eql?(Fasp::TransferSpec::DIRECTION_RECEIVE) &&
150
150
  event['data']['status'].eql?('completed') &&
151
151
  event['data']['error_code'].eql?(0) &&
152
- event['data'].dig('tags', 'aspera', PREV_GEN_TAG).nil?
153
- folder_id = event.dig('data', 'tags', 'aspera', 'node', 'file_id')
152
+ event['data'].dig('tags', Fasp::TransferSpec::TAG_RESERVED, PREV_GEN_TAG).nil?
153
+ folder_id = event.dig('data', 'tags', Fasp::TransferSpec::TAG_RESERVED, 'node', 'file_id')
154
154
  folder_id ||= event.dig('data', 'file_id')
155
155
  if !folder_id.nil?
156
156
  folder_entry = @api_node.read("files/#{folder_id}")[:data] rescue nil
@@ -226,7 +226,7 @@ module Aspera
226
226
  'direction' => direction,
227
227
  'paths' => [{'source' => source_filename}],
228
228
  'tags' => {
229
- 'aspera' => {
229
+ Fasp::TransferSpec::TAG_RESERVED => {
230
230
  PREV_GEN_TAG => true,
231
231
  'node' => {
232
232
  'access_key' => @access_key_self['id'],
@@ -13,6 +13,16 @@ module Aspera
13
13
  module Cli
14
14
  module Plugins
15
15
  # implement basic remote access with FASP/SSH
16
+ class SyncSpecServer
17
+ def initialize(transfer_spec)
18
+ @transfer_spec = transfer_spec
19
+ end
20
+
21
+ def transfer_spec(direction, local_path, remote_path)
22
+ return @transfer_spec
23
+ end
24
+ end
25
+
16
26
  class Server < Aspera::Cli::BasicAuthPlugin
17
27
  SSH_SCHEME = 'ssh'
18
28
  URI_SCHEMES = %w[https local].push(SSH_SCHEME).freeze
@@ -121,7 +131,7 @@ module Aspera
121
131
  Fasp::TransferSpec.action_to_direction(transfer_spec, command)
122
132
  return Main.result_transfer(transfer.start(transfer_spec))
123
133
  when :sync
124
- sync_plugin = Sync.new(@agents, transfer_spec: transfer_spec)
134
+ sync_plugin = Plugins::Sync.new(@agents, sync_spec: SyncSpecServer.new(transfer_spec))
125
135
  return sync_plugin.execute_action
126
136
  end
127
137
  end
@@ -11,14 +11,14 @@ module Aspera
11
11
  module Plugins
12
12
  # Execute Aspera Sync
13
13
  class Sync < Aspera::Cli::Plugin
14
- def initialize(env, transfer_spec: nil)
14
+ def initialize(env, sync_spec: nil)
15
15
  super(env)
16
16
  options.add_opt_simple(:sync_info, 'Information for sync instance and sessions (Hash)')
17
17
  options.add_opt_simple(:sync_session, 'Name of session to use for admin commands. default: first in parameters')
18
18
  options.parse_options!
19
19
  return if env[:man_only]
20
20
  @params = options.get_option(:sync_info, is_type: :mandatory)
21
- Aspera::Sync.update_parameters_with_transfer_spec(@params, transfer_spec) unless transfer_spec.nil?
21
+ @sync_spec = sync_spec
22
22
  end
23
23
 
24
24
  ACTIONS = %i[start admin].freeze
@@ -27,7 +27,7 @@ module Aspera
27
27
  command = options.get_next_command(ACTIONS)
28
28
  case command
29
29
  when :start
30
- Aspera::Sync.new(@params).start
30
+ Aspera::Sync.new(@params, @sync_spec).start
31
31
  return Main.result_success
32
32
  when :admin
33
33
  sync_admin = Aspera::SyncAdmin.new(@params, options.get_option(:sync_session))
@@ -48,18 +48,20 @@ module Aspera
48
48
  @config = config
49
49
  # command line can override transfer spec
50
50
  @transfer_spec_cmdline = {'create_dir' => true}
51
+ @transfer_info = {}
51
52
  # the currently selected transfer agent
52
53
  @agent = nil
53
54
  @progress_listener = Listener::ProgressMulti.new
54
55
  # source/destination pair, like "paths" of transfer spec
55
56
  @transfer_paths = nil
56
57
  @opt_mgr.set_obj_attr(:ts, self, :option_transfer_spec)
58
+ @opt_mgr.set_obj_attr(:transfer_info, self, :option_transfer_info)
57
59
  @opt_mgr.add_opt_simple(:ts, "Override transfer spec values (Hash, e.g. use @json: prefix), current=#{@opt_mgr.get_option(:ts)}")
58
60
  @opt_mgr.add_opt_simple(:to_folder, 'Destination folder for transferred files')
59
61
  @opt_mgr.add_opt_simple(:sources, "How list of transferred files is provided (#{FILE_LIST_OPTIONS.join(',')})")
60
62
  @opt_mgr.add_opt_list(:src_type, %i[list pair], 'Type of file list')
61
63
  @opt_mgr.add_opt_list(:transfer, TRANSFER_AGENTS, 'Type of transfer agent')
62
- @opt_mgr.add_opt_simple(:transfer_info, 'Parameters for transfer agent')
64
+ @opt_mgr.add_opt_simple(:transfer_info, 'Parameters for transfer agent (Hash)')
63
65
  @opt_mgr.add_opt_list(:progress, %i[none native multi], 'Type of progress bar')
64
66
  @opt_mgr.set_option(:transfer, :direct)
65
67
  @opt_mgr.set_option(:src_type, :list)
@@ -70,7 +72,18 @@ module Aspera
70
72
  def option_transfer_spec; @transfer_spec_cmdline; end
71
73
 
72
74
  # multiple option are merged
73
- def option_transfer_spec=(value); @transfer_spec_cmdline.merge!(value); end
75
+ def option_transfer_spec=(value)
76
+ raise 'option ts shall be a Hash' unless value.is_a?(Hash)
77
+ @transfer_spec_cmdline.merge!(value)
78
+ end
79
+
80
+ def option_transfer_info; @transfer_info; end
81
+
82
+ # multiple option are merged
83
+ def option_transfer_info=(value)
84
+ raise 'option transfer_info shall be a Hash' unless value.is_a?(Hash)
85
+ @transfer_info.merge!(value)
86
+ end
74
87
 
75
88
  def option_transfer_spec_deep_merge(ts); @transfer_spec_cmdline.deep_merge!(ts); end
76
89
 
@@ -92,19 +105,19 @@ module Aspera
92
105
  require "aspera/fasp/agent_#{agent_type}"
93
106
  agent_options = @opt_mgr.get_option(:transfer_info)
94
107
  raise CliBadArgument, "the transfer agent configuration shall be Hash, not #{agent_options.class} (#{agent_options}), "\
95
- 'use either @json:<json> or @preset:<parameter set name>' unless [Hash, NilClass].include?(agent_options.class)
96
- # special case
97
- if agent_type.eql?(:node) && agent_options.nil?
108
+ 'e.g. use @json:<json>' unless agent_options.is_a?(Hash)
109
+ # special case: use default node
110
+ if agent_type.eql?(:node) && agent_options.empty?
98
111
  param_set_name = @config.get_plugin_default_config_name(:node)
99
- raise CliBadArgument, "No default node configured, Please specify --#{:transfer_info.to_s.tr('_', '-')}" if param_set_name.nil?
112
+ raise CliBadArgument, "No default node configured. Please specify #{Manager.option_name_to_line(:transfer_info)}" if param_set_name.nil?
100
113
  agent_options = @config.preset_by_name(param_set_name)
101
114
  end
102
- # special case
115
+ # special case: native progress bar
103
116
  if agent_type.eql?(:direct) && @opt_mgr.get_option(:progress, is_type: :mandatory).eql?(:native)
104
- agent_options = {} if agent_options.nil?
105
117
  agent_options[:quiet] = false
106
118
  end
107
- agent_options = agent_options.symbolize_keys if agent_options.is_a?(Hash)
119
+ # normalize after getting from user or default node
120
+ agent_options = agent_options.symbolize_keys
108
121
  # get agent instance
109
122
  new_agent = Kernel.const_get("Aspera::Fasp::Agent#{agent_type.capitalize}").new(agent_options)
110
123
  self.agent_instance = new_agent
@@ -129,6 +142,7 @@ module Aspera
129
142
  return dest_folder
130
143
  end
131
144
 
145
+ # @return [Array] list of source files
132
146
  def source_list
133
147
  return ts_source_paths.map do |i|
134
148
  i['source']
@@ -196,7 +210,7 @@ module Aspera
196
210
  # init default if required in any case
197
211
  @transfer_spec_cmdline['destination_root'] ||= destination_folder(transfer_spec['direction'])
198
212
  when Fasp::TransferSpec::DIRECTION_SEND
199
- if transfer_spec.dig('tags', 'aspera', 'node', 'access_key')
213
+ if transfer_spec.dig('tags', Fasp::TransferSpec::TAG_RESERVED, 'node', 'access_key')
200
214
  # gen4
201
215
  @transfer_spec_cmdline.delete('destination_root') if @transfer_spec_cmdline.key?('destination_root_id')
202
216
  elsif transfer_spec.key?('token')
@@ -3,6 +3,7 @@
3
3
  module Aspera
4
4
  module Cli
5
5
  # for beta add extension : .beta1
6
- VERSION = '4.12.0'
6
+ # for dev version add extension : .pre
7
+ VERSION = '4.13.0'
7
8
  end
8
9
  end
@@ -9,7 +9,7 @@ module Aspera
9
9
  # parameter with one of those tags is a command line option with --
10
10
  CLI_OPTION_TYPE_SWITCH = %i[opt_without_arg opt_with_arg].freeze
11
11
  CLI_OPTION_TYPES = %i[special ignore envvar].concat(CLI_OPTION_TYPE_SWITCH).freeze
12
- OPTIONS_KEYS = %i[desc accepted_types default enum agents required cli ts].freeze
12
+ OPTIONS_KEYS = %i[desc accepted_types default enum agents required cli ts deprecation].freeze
13
13
  CLI_KEYS = %i[type switch convert variable].freeze
14
14
 
15
15
  private_constant :CLI_OPTION_TYPE_SWITCH, :OPTIONS_KEYS, :CLI_KEYS
@@ -37,6 +37,7 @@ module Aspera
37
37
  # by default : optional
38
38
  options[:mandatory] ||= false
39
39
  options[:desc] ||= ''
40
+ options[:desc] = "DEPRECATED: #{options[:deprecation]}\n#{options[:desc]}" if options.key?(:deprecation)
40
41
  cli = options[:cli]
41
42
  unsupported_cli_keys = cli.keys - CLI_KEYS
42
43
  raise "Unsupported cli keys: #{unsupported_cli_keys}" unless unsupported_cli_keys.empty?
@@ -65,7 +65,7 @@ module Aspera
65
65
  type: :basic,
66
66
  username: ats_info['AccessKey']['Id'],
67
67
  password: ats_info['AccessKey']['Secret']}},
68
- add_tspec: {'tags'=>{'aspera'=>{'node'=>{'storage_credentials'=>@storage_credentials}}}})
68
+ add_tspec: {'tags'=>{Fasp::TransferSpec::TAG_RESERVED=>{'node'=>{'storage_credentials'=>@storage_credentials}}}})
69
69
  # update storage_credentials AND Rest params
70
70
  generate_token
71
71
  end
@@ -66,7 +66,7 @@ module Aspera
66
66
  }]}
67
67
  # asynchronous anyway
68
68
  res = @connect_api.create('transfers/start', connect_transfer_args)[:data]
69
- @xfer_id = res['transfer_specs'].first['transfer_spec']['tags']['aspera']['xfer_id']
69
+ @xfer_id = res['transfer_specs'].first['transfer_spec']['tags'][Fasp::TransferSpec::TAG_RESERVED]['xfer_id']
70
70
  end
71
71
 
72
72
  def wait_for_transfers_completion
@@ -37,15 +37,15 @@ module Aspera
37
37
  # clone transfer spec because we modify it (first level keys)
38
38
  transfer_spec = transfer_spec.clone
39
39
  # if there is aspera tags
40
- if transfer_spec['tags'].is_a?(Hash) && transfer_spec['tags']['aspera'].is_a?(Hash)
40
+ if transfer_spec['tags'].is_a?(Hash) && transfer_spec['tags'][Fasp::TransferSpec::TAG_RESERVED].is_a?(Hash)
41
41
  # TODO: what is this for ? only on local ascp ?
42
42
  # NOTE: important: transfer id must be unique: generate random id
43
43
  # using a non unique id results in discard of tags in AoC, and a package is never finalized
44
44
  # all sessions in a multi-session transfer must have the same xfer_id (see admin manual)
45
- transfer_spec['tags']['aspera']['xfer_id'] ||= SecureRandom.uuid
45
+ transfer_spec['tags'][Fasp::TransferSpec::TAG_RESERVED]['xfer_id'] ||= SecureRandom.uuid
46
46
  Log.log.debug{"xfer id=#{transfer_spec['xfer_id']}"}
47
47
  # TODO: useful ? node only ?
48
- transfer_spec['tags']['aspera']['xfer_retry'] ||= 3600
48
+ transfer_spec['tags'][Fasp::TransferSpec::TAG_RESERVED]['xfer_retry'] ||= 3600
49
49
  end
50
50
  Log.dump('ts', transfer_spec)
51
51
 
@@ -85,7 +85,7 @@ module Aspera
85
85
  env_args = Parameters.ts_to_env_args(transfer_spec, wss: @options[:wss], ascp_args: @options[:ascp_args])
86
86
 
87
87
  # add fallback cert and key as arguments if needed
88
- if %w[1 force].include?(transfer_spec['http_fallback'])
88
+ if ['1', 1, true, 'force'].include?(transfer_spec['http_fallback'])
89
89
  env_args[:args].unshift('-Y', Installation.instance.path(:fallback_key))
90
90
  env_args[:args].unshift('-I', Installation.instance.path(:fallback_cert))
91
91
  end
@@ -183,20 +183,20 @@ module Aspera
183
183
  end
184
184
  # (optional) check it exists
185
185
  raise Fasp::Error, "no such file: #{ascp_path}" unless File.exist?(ascp_path)
186
- # open random local TCP port for listening for ascp management
186
+ # open an available (0) local TCP port as ascp management
187
187
  mgt_sock = TCPServer.new('127.0.0.1', 0)
188
188
  # clone arguments as we eed to modify with mgt port
189
189
  ascp_arguments = env_args[:args].clone
190
- # add management port
190
+ # add management port on the selected local port
191
191
  ascp_arguments.unshift('-M', mgt_sock.addr[1].to_s)
192
192
  # start ascp in sub process
193
193
  Log.log.debug do
194
- 'execute: ' +
195
- env_args[:env].map{|k, v| "#{k}=#{Shellwords.shellescape(v)}"}.join(' ') +
196
- ' ' +
197
- Shellwords.shellescape(ascp_path) +
198
- ' ' +
199
- ascp_arguments.map{|a|Shellwords.shellescape(a)}.join(' ')
194
+ [
195
+ 'execute:',
196
+ env_args[:env].map{|k, v| "#{k}=#{Shellwords.shellescape(v)}"},
197
+ Shellwords.shellescape(ascp_path),
198
+ ascp_arguments.map{|a|Shellwords.shellescape(a)}
199
+ ].flatten.join(' ')
200
200
  end
201
201
  # start process
202
202
  ascp_pid = Process.spawn(env_args[:env], [ascp_path, ascp_path], *ascp_arguments)
@@ -76,10 +76,13 @@ module Aspera
76
76
  transfer_spec.delete('EX_ssh_key_paths')
77
77
  end
78
78
  end
79
- if transfer_spec['tags'].is_a?(Hash) && transfer_spec['tags']['aspera'].is_a?(Hash)
80
- transfer_spec['tags']['aspera']['xfer_retry'] ||= 150
79
+ # add mandatory retry parameter for node api
80
+ ts_tags = transfer_spec['tags']
81
+ if ts_tags.is_a?(Hash) && ts_tags[Fasp::TransferSpec::TAG_RESERVED].is_a?(Hash)
82
+ ts_tags[Fasp::TransferSpec::TAG_RESERVED]['xfer_retry'] ||= 150
81
83
  end
82
- # Optimization in case of sending to the same node (TODO: probably remove this, as /etc/hosts shall be used for that)
84
+ # Optimization in case of sending to the same node
85
+ # TODO: probably remove this, as /etc/hosts shall be used for that
83
86
  if !transfer_spec['wss_enabled'] && transfer_spec['remote_host'].eql?(URI.parse(node_api_.params[:base_url]).host)
84
87
  transfer_spec['remote_host'] = '127.0.0.1'
85
88
  end
@@ -116,9 +119,16 @@ module Aspera
116
119
  else
117
120
  notify_progress(@transfer_id, transfer_data['bytes_transferred'])
118
121
  end
122
+ when 'failed'
123
+ # Bug in HSTS ? transfer is marked failed, but there is no reason
124
+ if transfer_data['error_code'].eql?(0) && transfer_data['error_desc'].empty?
125
+ notify_end(@transfer_id)
126
+ break
127
+ end
128
+ raise Fasp::Error, "status: #{transfer_data['status']}. code: #{transfer_data['error_code']}. description: #{transfer_data['error_desc']}"
119
129
  else
120
130
  Log.log.warn{"transfer_data -> #{transfer_data}"}
121
- raise Fasp::Error, "#{transfer_data['status']}: #{transfer_data['error_desc']}"
131
+ raise Fasp::Error, "status: #{transfer_data['status']}. code: #{transfer_data['error_code']}. description: #{transfer_data['error_desc']}"
122
132
  end
123
133
  sleep(1)
124
134
  end
@@ -62,6 +62,7 @@ module Aspera
62
62
 
63
63
  # location of SDK files
64
64
  def sdk_folder=(v)
65
+ Log.log.debug{"sdk_folder=#{v}"}
65
66
  @sdk_dir = v
66
67
  sdk_folder
67
68
  end