aspera-cli 4.12.0 → 4.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/CHANGELOG.md +17 -0
- data/CONTRIBUTING.md +97 -22
- data/README.md +548 -394
- data/bin/ascli +3 -3
- data/docs/test_env.conf +12 -5
- data/lib/aspera/aoc.rb +42 -42
- data/lib/aspera/ascmd.rb +4 -3
- data/lib/aspera/cli/extended_value.rb +24 -37
- data/lib/aspera/cli/formatter.rb +6 -0
- data/lib/aspera/cli/info.rb +2 -4
- data/lib/aspera/cli/main.rb +6 -0
- data/lib/aspera/cli/manager.rb +15 -6
- data/lib/aspera/cli/plugin.rb +1 -5
- data/lib/aspera/cli/plugins/aoc.rb +23 -6
- data/lib/aspera/cli/plugins/config.rb +13 -6
- data/lib/aspera/cli/plugins/faspex.rb +4 -3
- data/lib/aspera/cli/plugins/faspex5.rb +175 -42
- data/lib/aspera/cli/plugins/node.rb +107 -50
- data/lib/aspera/cli/plugins/preview.rb +3 -3
- data/lib/aspera/cli/plugins/server.rb +11 -1
- data/lib/aspera/cli/plugins/sync.rb +3 -3
- data/lib/aspera/cli/transfer_agent.rb +24 -10
- data/lib/aspera/cli/version.rb +2 -1
- data/lib/aspera/command_line_builder.rb +2 -1
- data/lib/aspera/cos_node.rb +1 -1
- data/lib/aspera/fasp/agent_connect.rb +1 -1
- data/lib/aspera/fasp/agent_direct.rb +12 -12
- data/lib/aspera/fasp/agent_node.rb +14 -4
- data/lib/aspera/fasp/installation.rb +1 -0
- data/lib/aspera/fasp/parameters.rb +11 -3
- data/lib/aspera/fasp/parameters.yaml +3 -1
- data/lib/aspera/fasp/transfer_spec.rb +3 -1
- data/lib/aspera/faspex_gw.rb +1 -0
- data/lib/aspera/faspex_postproc.rb +2 -2
- data/lib/aspera/node.rb +11 -4
- data/lib/aspera/oauth.rb +3 -2
- data/lib/aspera/preview/file_types.rb +8 -6
- data/lib/aspera/preview/generator.rb +23 -11
- data/lib/aspera/preview/options.rb +3 -2
- data/lib/aspera/preview/terminal.rb +34 -0
- data/lib/aspera/preview/utils.rb +8 -8
- data/lib/aspera/rest.rb +5 -4
- data/lib/aspera/rest_call_error.rb +3 -1
- data/lib/aspera/secret_hider.rb +4 -4
- data/lib/aspera/sync.rb +39 -33
- data/lib/aspera/web_server_simple.rb +22 -18
- data.tar.gz.sig +0 -0
- metadata +39 -46
- metadata.gz.sig +0 -0
- data/examples/aoc.rb +0 -30
- data/examples/faspex4.rb +0 -94
- data/examples/node.rb +0 -96
- data/examples/server.rb +0 -93
@@ -7,6 +7,7 @@ require 'aspera/hash_ext'
|
|
7
7
|
require 'aspera/id_generator'
|
8
8
|
require 'aspera/node'
|
9
9
|
require 'aspera/aoc'
|
10
|
+
require 'aspera/sync'
|
10
11
|
require 'aspera/fasp/transfer_spec'
|
11
12
|
require 'base64'
|
12
13
|
require 'zlib'
|
@@ -14,6 +15,51 @@ require 'zlib'
|
|
14
15
|
module Aspera
|
15
16
|
module Cli
|
16
17
|
module Plugins
|
18
|
+
class SyncSpecGen3
|
19
|
+
def initialize(api_node)
|
20
|
+
@api_node = api_node
|
21
|
+
end
|
22
|
+
|
23
|
+
def transfer_spec(direction, local_path, remote_path)
|
24
|
+
# empty transfer spec for authorization request
|
25
|
+
direction_sym = direction.to_sym
|
26
|
+
request_transfer_spec = {
|
27
|
+
type: Aspera::Sync::DIRECTION_TO_REQUEST_TYPE[direction_sym],
|
28
|
+
paths: {
|
29
|
+
source: remote_path,
|
30
|
+
destination: local_path
|
31
|
+
}
|
32
|
+
}
|
33
|
+
# add fixed parameters if any (for COS)
|
34
|
+
@api_node.add_tspec_info(request_transfer_spec) if @api_node.respond_to?(:add_tspec_info)
|
35
|
+
# prepare payload for single request
|
36
|
+
setup_payload = {transfer_requests: [{transfer_request: request_transfer_spec}]}
|
37
|
+
# only one request, so only one answer
|
38
|
+
transfer_spec = @api_node.create('files/sync_setup', setup_payload)[:data]['transfer_specs'].first['transfer_spec']
|
39
|
+
# API returns null tag... but async does not like it
|
40
|
+
transfer_spec.delete_if{ |_k, v| v.nil? }
|
41
|
+
# delete this part, as the returned value contains only destination, and not sources
|
42
|
+
# transfer_spec.delete('paths') if command.eql?(:upload)
|
43
|
+
Log.dump(:ts, transfer_spec)
|
44
|
+
return transfer_spec
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
class SyncSpecGen4
|
49
|
+
def initialize(api_node, top_file_id)
|
50
|
+
@api_node = api_node
|
51
|
+
@top_file_id = top_file_id
|
52
|
+
end
|
53
|
+
|
54
|
+
def transfer_spec(direction, local_path, remote_path)
|
55
|
+
# remote is specified by option to_folder
|
56
|
+
apifid = @api_node.resolve_api_fid(@top_file_id, remote_path)
|
57
|
+
transfer_spec = apifid[:api].transfer_spec_gen4(apifid[:file_id], Fasp::TransferSpec::DIRECTION_SEND)
|
58
|
+
Log.dump(:ts, transfer_spec)
|
59
|
+
return transfer_spec
|
60
|
+
end
|
61
|
+
end
|
62
|
+
|
17
63
|
class Node < Aspera::Cli::BasicAuthPlugin
|
18
64
|
class << self
|
19
65
|
def detect(base_url)
|
@@ -30,7 +76,7 @@ module Aspera
|
|
30
76
|
env[:options].add_opt_simple(:asperabrowserurl, 'URL for simple aspera web ui')
|
31
77
|
env[:options].add_opt_simple(:sync_name, 'sync name')
|
32
78
|
env[:options].add_opt_simple(:path, 'file or folder path for gen4 operation "file"')
|
33
|
-
env[:options].add_opt_list(:token_type, %i[aspera basic hybrid], '
|
79
|
+
env[:options].add_opt_list(:token_type, %i[aspera basic hybrid], 'type of token used for transfers')
|
34
80
|
env[:options].add_opt_boolean(:default_ports, 'use standard FASP ports or get from node api (gen4)')
|
35
81
|
env[:options].set_option(:asperabrowserurl, 'https://asperabrowser.mybluemix.net')
|
36
82
|
env[:options].set_option(:token_type, :aspera)
|
@@ -53,7 +99,7 @@ module Aspera
|
|
53
99
|
SEARCH_REMOVE_FIELDS = %w[basename permissions].freeze
|
54
100
|
|
55
101
|
# actions in execute_command_gen3
|
56
|
-
COMMANDS_GEN3 = %i[search space mkdir mklink mkfile rename delete browse upload download]
|
102
|
+
COMMANDS_GEN3 = %i[search space mkdir mklink mkfile rename delete browse upload download sync]
|
57
103
|
|
58
104
|
BASE_ACTIONS = %i[api_details].concat(COMMANDS_GEN3).freeze
|
59
105
|
|
@@ -145,7 +191,7 @@ module Aspera
|
|
145
191
|
# translates paths results into CLI result, and removes prefix
|
146
192
|
def c_result_translate_rem_prefix(response, type, success_msg, path_prefix)
|
147
193
|
errors = []
|
148
|
-
|
194
|
+
final_result = { data: [], type: :object_list, fields: [type, 'result']}
|
149
195
|
JSON.parse(response[:http].body)['paths'].each do |p|
|
150
196
|
result = success_msg
|
151
197
|
if p.key?('error')
|
@@ -153,13 +199,13 @@ module Aspera
|
|
153
199
|
result = 'ERROR: ' + p['error']['user_message']
|
154
200
|
errors.push([p['path'], p['error']['user_message']])
|
155
201
|
end
|
156
|
-
|
202
|
+
final_result[:data].push({type => p['path'], 'result' => result})
|
157
203
|
end
|
158
204
|
# one error make all fail
|
159
205
|
unless errors.empty?
|
160
206
|
raise errors.map{|i|"#{i.first}: #{i.last}"}.join(', ')
|
161
207
|
end
|
162
|
-
return c_result_remove_prefix_path(
|
208
|
+
return c_result_remove_prefix_path(final_result, type, path_prefix)
|
163
209
|
end
|
164
210
|
|
165
211
|
# get path arguments from command line, and add prefix
|
@@ -187,7 +233,7 @@ module Aspera
|
|
187
233
|
result = { type: :object_list, data: resp[:data]['items']}
|
188
234
|
return Main.result_empty if result[:data].empty?
|
189
235
|
result[:fields] = result[:data].first.keys.reject{|i|SEARCH_REMOVE_FIELDS.include?(i)}
|
190
|
-
formatter.
|
236
|
+
formatter.display_item_count(resp[:data]['item_count'], resp[:data]['total_count'])
|
191
237
|
formatter.display_status("params: #{resp[:data]['parameters'].keys.map{|k|"#{k}:#{resp[:data]['parameters'][k]}"}.join(',')}")
|
192
238
|
return c_result_remove_prefix_path(result, 'path', prefix_path)
|
193
239
|
when :space
|
@@ -228,13 +274,16 @@ module Aspera
|
|
228
274
|
case send_result['self']['type']
|
229
275
|
when 'directory', 'container' # directory: node, container: shares
|
230
276
|
result = { data: send_result['items'], type: :object_list, textify: lambda { |table_data| c_textify_browse(table_data) } }
|
231
|
-
formatter.
|
277
|
+
formatter.display_item_count(send_result['item_count'], send_result['total_count'])
|
232
278
|
else # 'file','symbolic_link'
|
233
279
|
result = { data: send_result['self'], type: :single_object}
|
234
280
|
# result={ data: [send_result['self']] , type: :object_list, textify: lambda { |table_data| c_textify_browse(table_data) } }
|
235
281
|
# raise "unknown type: #{send_result['self']['type']}"
|
236
282
|
end
|
237
283
|
return c_result_remove_prefix_path(result, 'path', prefix_path)
|
284
|
+
when :sync
|
285
|
+
node_sync = SyncSpecGen3.new(@api_node)
|
286
|
+
return Plugins::Sync.new(@agents, sync_spec: node_sync).execute_action
|
238
287
|
when :upload, :download
|
239
288
|
token_type = options.get_option(:token_type)
|
240
289
|
# nil if Shares 1.x
|
@@ -244,7 +293,11 @@ module Aspera
|
|
244
293
|
# empty transfer spec for authorization request
|
245
294
|
request_transfer_spec = {}
|
246
295
|
# set requested paths depending on direction
|
247
|
-
request_transfer_spec[:paths] = command.eql?(:download)
|
296
|
+
request_transfer_spec[:paths] = if command.eql?(:download)
|
297
|
+
transfer.ts_source_paths
|
298
|
+
else
|
299
|
+
[{ destination: transfer.destination_folder(Fasp::TransferSpec::DIRECTION_SEND) }]
|
300
|
+
end
|
248
301
|
# add fixed parameters if any (for COS)
|
249
302
|
@api_node.add_tspec_info(request_transfer_spec) if @api_node.respond_to?(:add_tspec_info)
|
250
303
|
# prepare payload for single request
|
@@ -330,7 +383,7 @@ module Aspera
|
|
330
383
|
return { type: :single_object, data: @api_node.params }
|
331
384
|
end
|
332
385
|
end
|
333
|
-
|
386
|
+
GEN4_FILE_COMMANDS = %i[show modify permission thumbnail].freeze
|
334
387
|
def execute_node_gen4_file_command(command_node_file, top_file_id)
|
335
388
|
file_path = options.get_option(:path)
|
336
389
|
apifid =
|
@@ -347,6 +400,14 @@ module Aspera
|
|
347
400
|
update_param = options.get_next_argument('update data', type: Hash)
|
348
401
|
apifid[:api].update("files/#{apifid[:file_id]}", update_param)[:data]
|
349
402
|
return Main.result_status('Done')
|
403
|
+
when :thumbnail
|
404
|
+
result = apifid[:api].call(
|
405
|
+
operation: 'GET',
|
406
|
+
subpath: "files/#{apifid[:file_id]}/preview",
|
407
|
+
headers: {'Accept' => 'image/png'}
|
408
|
+
)
|
409
|
+
require 'aspera/preview/terminal'
|
410
|
+
return Main.result_status(Preview::Terminal.build(result[:http].body, reserved_lines: 3))
|
350
411
|
when :permission
|
351
412
|
command_perm = options.get_next_command(%i[list create delete])
|
352
413
|
case command_perm
|
@@ -375,7 +436,7 @@ module Aspera
|
|
375
436
|
the_app[:api].permissions_create_params(create_param: create_param, app_info: the_app) unless the_app.nil?
|
376
437
|
# create permission
|
377
438
|
created_data = apifid[:api].create('permissions', create_param)[:data]
|
378
|
-
#
|
439
|
+
# notify application of creation
|
379
440
|
the_app[:api].permissions_create_event(created_data: created_data, app_info: the_app) unless the_app.nil?
|
380
441
|
return { type: :single_object, data: created_data}
|
381
442
|
else raise "internal error:shall not reach here (#{command_perm})"
|
@@ -417,7 +478,7 @@ module Aspera
|
|
417
478
|
if file_info['type'].eql?('folder')
|
418
479
|
result = apifid[:api].read("files/#{apifid[:file_id]}/files", options.get_option(:value))
|
419
480
|
items = result[:data]
|
420
|
-
formatter.
|
481
|
+
formatter.display_item_count(result[:data].length, result[:http]['X-Total-Count'])
|
421
482
|
else
|
422
483
|
items = [file_info]
|
423
484
|
end
|
@@ -446,12 +507,8 @@ module Aspera
|
|
446
507
|
{'path' => l_path}
|
447
508
|
end
|
448
509
|
when :sync
|
449
|
-
|
450
|
-
|
451
|
-
transfer_spec = apifid[:api].transfer_spec_gen4(apifid[:file_id], Fasp::TransferSpec::DIRECTION_SEND)
|
452
|
-
Log.dump(:ts, transfer_spec)
|
453
|
-
sync_plugin = Plugins::Sync.new(@agents, transfer_spec: transfer_spec)
|
454
|
-
return sync_plugin.execute_action
|
510
|
+
node_sync = SyncSpecGen4.new(@api_node, top_file_id)
|
511
|
+
return Plugins::Sync.new(@agents, sync_spec: node_sync).execute_action
|
455
512
|
when :upload
|
456
513
|
apifid = @api_node.resolve_api_fid(top_file_id, transfer.destination_folder(Fasp::TransferSpec::DIRECTION_SEND))
|
457
514
|
return Main.result_transfer(transfer.start(apifid[:api].transfer_spec_gen4(apifid[:file_id], Fasp::TransferSpec::DIRECTION_SEND)))
|
@@ -499,7 +556,7 @@ module Aspera
|
|
499
556
|
save_to_file: File.join(transfer.destination_folder(Fasp::TransferSpec::DIRECTION_RECEIVE), file_name))
|
500
557
|
return Main.result_status("downloaded: #{file_name}")
|
501
558
|
when :file
|
502
|
-
command_node_file = options.get_next_command(
|
559
|
+
command_node_file = options.get_next_command(GEN4_FILE_COMMANDS)
|
503
560
|
return execute_node_gen4_file_command(command_node_file, top_file_id)
|
504
561
|
else raise "INTERNAL ERROR: no case for #{command_repo}"
|
505
562
|
end # command_repo
|
@@ -510,43 +567,43 @@ module Aspera
|
|
510
567
|
def execute_async
|
511
568
|
command = options.get_next_command(%i[list delete files show counters bandwidth])
|
512
569
|
unless command.eql?(:list)
|
513
|
-
|
514
|
-
if
|
515
|
-
|
516
|
-
if
|
517
|
-
|
570
|
+
async_name = options.get_option(:sync_name)
|
571
|
+
if async_name.nil?
|
572
|
+
async_id = instance_identifier
|
573
|
+
if async_id.eql?(VAL_ALL) && %i[show delete].include?(command)
|
574
|
+
async_ids = @api_node.read('async/list')[:data]['sync_ids']
|
518
575
|
else
|
519
|
-
Integer(
|
520
|
-
|
576
|
+
Integer(async_id) # must be integer
|
577
|
+
async_ids = [async_id]
|
521
578
|
end
|
522
579
|
else
|
523
|
-
|
524
|
-
summaries = @api_node.create('async/summary', {'syncs' =>
|
525
|
-
selected = summaries.find{|s|s['name'].eql?(
|
526
|
-
raise "no such sync: #{
|
527
|
-
|
528
|
-
|
580
|
+
async_ids = @api_node.read('async/list')[:data]['sync_ids']
|
581
|
+
summaries = @api_node.create('async/summary', {'syncs' => async_ids})[:data]['sync_summaries']
|
582
|
+
selected = summaries.find{|s|s['name'].eql?(async_name)}
|
583
|
+
raise "no such sync: #{async_name}" if selected.nil?
|
584
|
+
async_id = selected['snid']
|
585
|
+
async_ids = [async_id]
|
529
586
|
end
|
530
|
-
|
587
|
+
post_data = {'syncs' => async_ids}
|
531
588
|
end
|
532
589
|
case command
|
533
590
|
when :list
|
534
591
|
resp = @api_node.read('async/list')[:data]['sync_ids']
|
535
592
|
return { type: :value_list, data: resp, name: 'id' }
|
536
593
|
when :show
|
537
|
-
resp = @api_node.create('async/summary',
|
594
|
+
resp = @api_node.create('async/summary', post_data)[:data]['sync_summaries']
|
538
595
|
return Main.result_empty if resp.empty?
|
539
|
-
return { type: :object_list, data: resp, fields: %w[snid name local_dir remote_dir] } if
|
596
|
+
return { type: :object_list, data: resp, fields: %w[snid name local_dir remote_dir] } if async_id.eql?(VAL_ALL)
|
540
597
|
return { type: :single_object, data: resp.first }
|
541
598
|
when :delete
|
542
|
-
resp = @api_node.create('async/delete',
|
599
|
+
resp = @api_node.create('async/delete', post_data)[:data]
|
543
600
|
return { type: :single_object, data: resp, name: 'id' }
|
544
601
|
when :bandwidth
|
545
|
-
|
546
|
-
resp = @api_node.create('async/bandwidth',
|
602
|
+
post_data['seconds'] = 100 # TODO: as parameter with --value
|
603
|
+
resp = @api_node.create('async/bandwidth', post_data)[:data]
|
547
604
|
data = resp['bandwidth_data']
|
548
605
|
return Main.result_empty if data.empty?
|
549
|
-
data = data.first[
|
606
|
+
data = data.first[async_id]['data']
|
550
607
|
return { type: :object_list, data: data, name: 'id' }
|
551
608
|
when :files
|
552
609
|
# count int
|
@@ -554,10 +611,10 @@ module Aspera
|
|
554
611
|
# skip int
|
555
612
|
# status int
|
556
613
|
filter = options.get_option(:value)
|
557
|
-
|
558
|
-
resp = @api_node.create('async/files',
|
614
|
+
post_data.merge!(filter) unless filter.nil?
|
615
|
+
resp = @api_node.create('async/files', post_data)[:data]
|
559
616
|
data = resp['sync_files']
|
560
|
-
data = data.first[
|
617
|
+
data = data.first[async_id] unless data.empty?
|
561
618
|
iteration_data = []
|
562
619
|
skip_ids_persistency = nil
|
563
620
|
if options.get_option(:once_only, is_type: :mandatory)
|
@@ -568,7 +625,7 @@ module Aspera
|
|
568
625
|
'sync_files',
|
569
626
|
options.get_option(:url, is_type: :mandatory),
|
570
627
|
options.get_option(:username, is_type: :mandatory),
|
571
|
-
|
628
|
+
async_id]))
|
572
629
|
unless iteration_data.first.nil?
|
573
630
|
data.select!{|l| l['fnid'].to_i > iteration_data.first}
|
574
631
|
end
|
@@ -578,7 +635,7 @@ module Aspera
|
|
578
635
|
skip_ids_persistency&.save
|
579
636
|
return { type: :object_list, data: data, name: 'id' }
|
580
637
|
when :counters
|
581
|
-
resp = @api_node.create('async/counters',
|
638
|
+
resp = @api_node.create('async/counters', post_data)[:data]['sync_counters'].first[async_id].last
|
582
639
|
return Main.result_empty if resp.nil?
|
583
640
|
return { type: :single_object, data: resp }
|
584
641
|
end
|
@@ -586,7 +643,7 @@ module Aspera
|
|
586
643
|
|
587
644
|
ACTIONS = %i[
|
588
645
|
async
|
589
|
-
|
646
|
+
ssync
|
590
647
|
stream
|
591
648
|
transfer
|
592
649
|
service
|
@@ -599,9 +656,9 @@ module Aspera
|
|
599
656
|
command ||= options.get_next_command(ACTIONS)
|
600
657
|
case command
|
601
658
|
when *COMMON_ACTIONS then return execute_simple_common(command, prefix_path)
|
602
|
-
when :async then return execute_async
|
603
|
-
when :
|
604
|
-
# newer
|
659
|
+
when :async then return execute_async # former API
|
660
|
+
when :ssync
|
661
|
+
# newer API
|
605
662
|
sync_command = options.get_next_command(%i[bandwidth counters files start state stop summary].concat(Plugin::ALL_OPS) - %i[modify])
|
606
663
|
case sync_command
|
607
664
|
when *Plugin::ALL_OPS then return entity_command(sync_command, @api_node, 'asyncs', item_list_key: 'ids')
|
@@ -665,7 +722,7 @@ module Aspera
|
|
665
722
|
when :service
|
666
723
|
command = options.get_next_command(%i[list create delete])
|
667
724
|
if [:delete].include?(command)
|
668
|
-
|
725
|
+
service_id = instance_identifier
|
669
726
|
end
|
670
727
|
case command
|
671
728
|
when :list
|
@@ -677,8 +734,8 @@ module Aspera
|
|
677
734
|
resp = @api_node.create('rund/services', params)
|
678
735
|
return Main.result_status("#{resp[:data]['id']} created")
|
679
736
|
when :delete
|
680
|
-
@api_node.delete("rund/services/#{
|
681
|
-
return Main.result_status("#{
|
737
|
+
@api_node.delete("rund/services/#{service_id}")
|
738
|
+
return Main.result_status("#{service_id} deleted")
|
682
739
|
end
|
683
740
|
when :watch_folder
|
684
741
|
res_class_path = 'v3/watchfolders'
|
@@ -149,8 +149,8 @@ module Aspera
|
|
149
149
|
if event['data']['direction'].eql?(Fasp::TransferSpec::DIRECTION_RECEIVE) &&
|
150
150
|
event['data']['status'].eql?('completed') &&
|
151
151
|
event['data']['error_code'].eql?(0) &&
|
152
|
-
event['data'].dig('tags',
|
153
|
-
folder_id = event.dig('data', 'tags',
|
152
|
+
event['data'].dig('tags', Fasp::TransferSpec::TAG_RESERVED, PREV_GEN_TAG).nil?
|
153
|
+
folder_id = event.dig('data', 'tags', Fasp::TransferSpec::TAG_RESERVED, 'node', 'file_id')
|
154
154
|
folder_id ||= event.dig('data', 'file_id')
|
155
155
|
if !folder_id.nil?
|
156
156
|
folder_entry = @api_node.read("files/#{folder_id}")[:data] rescue nil
|
@@ -226,7 +226,7 @@ module Aspera
|
|
226
226
|
'direction' => direction,
|
227
227
|
'paths' => [{'source' => source_filename}],
|
228
228
|
'tags' => {
|
229
|
-
|
229
|
+
Fasp::TransferSpec::TAG_RESERVED => {
|
230
230
|
PREV_GEN_TAG => true,
|
231
231
|
'node' => {
|
232
232
|
'access_key' => @access_key_self['id'],
|
@@ -13,6 +13,16 @@ module Aspera
|
|
13
13
|
module Cli
|
14
14
|
module Plugins
|
15
15
|
# implement basic remote access with FASP/SSH
|
16
|
+
class SyncSpecServer
|
17
|
+
def initialize(transfer_spec)
|
18
|
+
@transfer_spec = transfer_spec
|
19
|
+
end
|
20
|
+
|
21
|
+
def transfer_spec(direction, local_path, remote_path)
|
22
|
+
return @transfer_spec
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
16
26
|
class Server < Aspera::Cli::BasicAuthPlugin
|
17
27
|
SSH_SCHEME = 'ssh'
|
18
28
|
URI_SCHEMES = %w[https local].push(SSH_SCHEME).freeze
|
@@ -121,7 +131,7 @@ module Aspera
|
|
121
131
|
Fasp::TransferSpec.action_to_direction(transfer_spec, command)
|
122
132
|
return Main.result_transfer(transfer.start(transfer_spec))
|
123
133
|
when :sync
|
124
|
-
sync_plugin = Sync.new(@agents,
|
134
|
+
sync_plugin = Plugins::Sync.new(@agents, sync_spec: SyncSpecServer.new(transfer_spec))
|
125
135
|
return sync_plugin.execute_action
|
126
136
|
end
|
127
137
|
end
|
@@ -11,14 +11,14 @@ module Aspera
|
|
11
11
|
module Plugins
|
12
12
|
# Execute Aspera Sync
|
13
13
|
class Sync < Aspera::Cli::Plugin
|
14
|
-
def initialize(env,
|
14
|
+
def initialize(env, sync_spec: nil)
|
15
15
|
super(env)
|
16
16
|
options.add_opt_simple(:sync_info, 'Information for sync instance and sessions (Hash)')
|
17
17
|
options.add_opt_simple(:sync_session, 'Name of session to use for admin commands. default: first in parameters')
|
18
18
|
options.parse_options!
|
19
19
|
return if env[:man_only]
|
20
20
|
@params = options.get_option(:sync_info, is_type: :mandatory)
|
21
|
-
|
21
|
+
@sync_spec = sync_spec
|
22
22
|
end
|
23
23
|
|
24
24
|
ACTIONS = %i[start admin].freeze
|
@@ -27,7 +27,7 @@ module Aspera
|
|
27
27
|
command = options.get_next_command(ACTIONS)
|
28
28
|
case command
|
29
29
|
when :start
|
30
|
-
Aspera::Sync.new(@params).start
|
30
|
+
Aspera::Sync.new(@params, @sync_spec).start
|
31
31
|
return Main.result_success
|
32
32
|
when :admin
|
33
33
|
sync_admin = Aspera::SyncAdmin.new(@params, options.get_option(:sync_session))
|
@@ -48,18 +48,20 @@ module Aspera
|
|
48
48
|
@config = config
|
49
49
|
# command line can override transfer spec
|
50
50
|
@transfer_spec_cmdline = {'create_dir' => true}
|
51
|
+
@transfer_info = {}
|
51
52
|
# the currently selected transfer agent
|
52
53
|
@agent = nil
|
53
54
|
@progress_listener = Listener::ProgressMulti.new
|
54
55
|
# source/destination pair, like "paths" of transfer spec
|
55
56
|
@transfer_paths = nil
|
56
57
|
@opt_mgr.set_obj_attr(:ts, self, :option_transfer_spec)
|
58
|
+
@opt_mgr.set_obj_attr(:transfer_info, self, :option_transfer_info)
|
57
59
|
@opt_mgr.add_opt_simple(:ts, "Override transfer spec values (Hash, e.g. use @json: prefix), current=#{@opt_mgr.get_option(:ts)}")
|
58
60
|
@opt_mgr.add_opt_simple(:to_folder, 'Destination folder for transferred files')
|
59
61
|
@opt_mgr.add_opt_simple(:sources, "How list of transferred files is provided (#{FILE_LIST_OPTIONS.join(',')})")
|
60
62
|
@opt_mgr.add_opt_list(:src_type, %i[list pair], 'Type of file list')
|
61
63
|
@opt_mgr.add_opt_list(:transfer, TRANSFER_AGENTS, 'Type of transfer agent')
|
62
|
-
@opt_mgr.add_opt_simple(:transfer_info, 'Parameters for transfer agent')
|
64
|
+
@opt_mgr.add_opt_simple(:transfer_info, 'Parameters for transfer agent (Hash)')
|
63
65
|
@opt_mgr.add_opt_list(:progress, %i[none native multi], 'Type of progress bar')
|
64
66
|
@opt_mgr.set_option(:transfer, :direct)
|
65
67
|
@opt_mgr.set_option(:src_type, :list)
|
@@ -70,7 +72,18 @@ module Aspera
|
|
70
72
|
def option_transfer_spec; @transfer_spec_cmdline; end
|
71
73
|
|
72
74
|
# multiple option are merged
|
73
|
-
def option_transfer_spec=(value)
|
75
|
+
def option_transfer_spec=(value)
|
76
|
+
raise 'option ts shall be a Hash' unless value.is_a?(Hash)
|
77
|
+
@transfer_spec_cmdline.merge!(value)
|
78
|
+
end
|
79
|
+
|
80
|
+
def option_transfer_info; @transfer_info; end
|
81
|
+
|
82
|
+
# multiple option are merged
|
83
|
+
def option_transfer_info=(value)
|
84
|
+
raise 'option transfer_info shall be a Hash' unless value.is_a?(Hash)
|
85
|
+
@transfer_info.merge!(value)
|
86
|
+
end
|
74
87
|
|
75
88
|
def option_transfer_spec_deep_merge(ts); @transfer_spec_cmdline.deep_merge!(ts); end
|
76
89
|
|
@@ -92,19 +105,19 @@ module Aspera
|
|
92
105
|
require "aspera/fasp/agent_#{agent_type}"
|
93
106
|
agent_options = @opt_mgr.get_option(:transfer_info)
|
94
107
|
raise CliBadArgument, "the transfer agent configuration shall be Hash, not #{agent_options.class} (#{agent_options}), "\
|
95
|
-
'use
|
96
|
-
# special case
|
97
|
-
if agent_type.eql?(:node) && agent_options.
|
108
|
+
'e.g. use @json:<json>' unless agent_options.is_a?(Hash)
|
109
|
+
# special case: use default node
|
110
|
+
if agent_type.eql?(:node) && agent_options.empty?
|
98
111
|
param_set_name = @config.get_plugin_default_config_name(:node)
|
99
|
-
raise CliBadArgument, "No default node configured
|
112
|
+
raise CliBadArgument, "No default node configured. Please specify #{Manager.option_name_to_line(:transfer_info)}" if param_set_name.nil?
|
100
113
|
agent_options = @config.preset_by_name(param_set_name)
|
101
114
|
end
|
102
|
-
# special case
|
115
|
+
# special case: native progress bar
|
103
116
|
if agent_type.eql?(:direct) && @opt_mgr.get_option(:progress, is_type: :mandatory).eql?(:native)
|
104
|
-
agent_options = {} if agent_options.nil?
|
105
117
|
agent_options[:quiet] = false
|
106
118
|
end
|
107
|
-
|
119
|
+
# normalize after getting from user or default node
|
120
|
+
agent_options = agent_options.symbolize_keys
|
108
121
|
# get agent instance
|
109
122
|
new_agent = Kernel.const_get("Aspera::Fasp::Agent#{agent_type.capitalize}").new(agent_options)
|
110
123
|
self.agent_instance = new_agent
|
@@ -129,6 +142,7 @@ module Aspera
|
|
129
142
|
return dest_folder
|
130
143
|
end
|
131
144
|
|
145
|
+
# @return [Array] list of source files
|
132
146
|
def source_list
|
133
147
|
return ts_source_paths.map do |i|
|
134
148
|
i['source']
|
@@ -196,7 +210,7 @@ module Aspera
|
|
196
210
|
# init default if required in any case
|
197
211
|
@transfer_spec_cmdline['destination_root'] ||= destination_folder(transfer_spec['direction'])
|
198
212
|
when Fasp::TransferSpec::DIRECTION_SEND
|
199
|
-
if transfer_spec.dig('tags',
|
213
|
+
if transfer_spec.dig('tags', Fasp::TransferSpec::TAG_RESERVED, 'node', 'access_key')
|
200
214
|
# gen4
|
201
215
|
@transfer_spec_cmdline.delete('destination_root') if @transfer_spec_cmdline.key?('destination_root_id')
|
202
216
|
elsif transfer_spec.key?('token')
|
data/lib/aspera/cli/version.rb
CHANGED
@@ -9,7 +9,7 @@ module Aspera
|
|
9
9
|
# parameter with one of those tags is a command line option with --
|
10
10
|
CLI_OPTION_TYPE_SWITCH = %i[opt_without_arg opt_with_arg].freeze
|
11
11
|
CLI_OPTION_TYPES = %i[special ignore envvar].concat(CLI_OPTION_TYPE_SWITCH).freeze
|
12
|
-
OPTIONS_KEYS = %i[desc accepted_types default enum agents required cli ts].freeze
|
12
|
+
OPTIONS_KEYS = %i[desc accepted_types default enum agents required cli ts deprecation].freeze
|
13
13
|
CLI_KEYS = %i[type switch convert variable].freeze
|
14
14
|
|
15
15
|
private_constant :CLI_OPTION_TYPE_SWITCH, :OPTIONS_KEYS, :CLI_KEYS
|
@@ -37,6 +37,7 @@ module Aspera
|
|
37
37
|
# by default : optional
|
38
38
|
options[:mandatory] ||= false
|
39
39
|
options[:desc] ||= ''
|
40
|
+
options[:desc] = "DEPRECATED: #{options[:deprecation]}\n#{options[:desc]}" if options.key?(:deprecation)
|
40
41
|
cli = options[:cli]
|
41
42
|
unsupported_cli_keys = cli.keys - CLI_KEYS
|
42
43
|
raise "Unsupported cli keys: #{unsupported_cli_keys}" unless unsupported_cli_keys.empty?
|
data/lib/aspera/cos_node.rb
CHANGED
@@ -65,7 +65,7 @@ module Aspera
|
|
65
65
|
type: :basic,
|
66
66
|
username: ats_info['AccessKey']['Id'],
|
67
67
|
password: ats_info['AccessKey']['Secret']}},
|
68
|
-
add_tspec: {'tags'=>{
|
68
|
+
add_tspec: {'tags'=>{Fasp::TransferSpec::TAG_RESERVED=>{'node'=>{'storage_credentials'=>@storage_credentials}}}})
|
69
69
|
# update storage_credentials AND Rest params
|
70
70
|
generate_token
|
71
71
|
end
|
@@ -66,7 +66,7 @@ module Aspera
|
|
66
66
|
}]}
|
67
67
|
# asynchronous anyway
|
68
68
|
res = @connect_api.create('transfers/start', connect_transfer_args)[:data]
|
69
|
-
@xfer_id = res['transfer_specs'].first['transfer_spec']['tags'][
|
69
|
+
@xfer_id = res['transfer_specs'].first['transfer_spec']['tags'][Fasp::TransferSpec::TAG_RESERVED]['xfer_id']
|
70
70
|
end
|
71
71
|
|
72
72
|
def wait_for_transfers_completion
|
@@ -37,15 +37,15 @@ module Aspera
|
|
37
37
|
# clone transfer spec because we modify it (first level keys)
|
38
38
|
transfer_spec = transfer_spec.clone
|
39
39
|
# if there is aspera tags
|
40
|
-
if transfer_spec['tags'].is_a?(Hash) && transfer_spec['tags'][
|
40
|
+
if transfer_spec['tags'].is_a?(Hash) && transfer_spec['tags'][Fasp::TransferSpec::TAG_RESERVED].is_a?(Hash)
|
41
41
|
# TODO: what is this for ? only on local ascp ?
|
42
42
|
# NOTE: important: transfer id must be unique: generate random id
|
43
43
|
# using a non unique id results in discard of tags in AoC, and a package is never finalized
|
44
44
|
# all sessions in a multi-session transfer must have the same xfer_id (see admin manual)
|
45
|
-
transfer_spec['tags'][
|
45
|
+
transfer_spec['tags'][Fasp::TransferSpec::TAG_RESERVED]['xfer_id'] ||= SecureRandom.uuid
|
46
46
|
Log.log.debug{"xfer id=#{transfer_spec['xfer_id']}"}
|
47
47
|
# TODO: useful ? node only ?
|
48
|
-
transfer_spec['tags'][
|
48
|
+
transfer_spec['tags'][Fasp::TransferSpec::TAG_RESERVED]['xfer_retry'] ||= 3600
|
49
49
|
end
|
50
50
|
Log.dump('ts', transfer_spec)
|
51
51
|
|
@@ -85,7 +85,7 @@ module Aspera
|
|
85
85
|
env_args = Parameters.ts_to_env_args(transfer_spec, wss: @options[:wss], ascp_args: @options[:ascp_args])
|
86
86
|
|
87
87
|
# add fallback cert and key as arguments if needed
|
88
|
-
if
|
88
|
+
if ['1', 1, true, 'force'].include?(transfer_spec['http_fallback'])
|
89
89
|
env_args[:args].unshift('-Y', Installation.instance.path(:fallback_key))
|
90
90
|
env_args[:args].unshift('-I', Installation.instance.path(:fallback_cert))
|
91
91
|
end
|
@@ -183,20 +183,20 @@ module Aspera
|
|
183
183
|
end
|
184
184
|
# (optional) check it exists
|
185
185
|
raise Fasp::Error, "no such file: #{ascp_path}" unless File.exist?(ascp_path)
|
186
|
-
# open
|
186
|
+
# open an available (0) local TCP port as ascp management
|
187
187
|
mgt_sock = TCPServer.new('127.0.0.1', 0)
|
188
188
|
# clone arguments as we eed to modify with mgt port
|
189
189
|
ascp_arguments = env_args[:args].clone
|
190
|
-
# add management port
|
190
|
+
# add management port on the selected local port
|
191
191
|
ascp_arguments.unshift('-M', mgt_sock.addr[1].to_s)
|
192
192
|
# start ascp in sub process
|
193
193
|
Log.log.debug do
|
194
|
-
|
195
|
-
|
196
|
-
|
197
|
-
Shellwords.shellescape(ascp_path)
|
198
|
-
|
199
|
-
|
194
|
+
[
|
195
|
+
'execute:',
|
196
|
+
env_args[:env].map{|k, v| "#{k}=#{Shellwords.shellescape(v)}"},
|
197
|
+
Shellwords.shellescape(ascp_path),
|
198
|
+
ascp_arguments.map{|a|Shellwords.shellescape(a)}
|
199
|
+
].flatten.join(' ')
|
200
200
|
end
|
201
201
|
# start process
|
202
202
|
ascp_pid = Process.spawn(env_args[:env], [ascp_path, ascp_path], *ascp_arguments)
|
@@ -76,10 +76,13 @@ module Aspera
|
|
76
76
|
transfer_spec.delete('EX_ssh_key_paths')
|
77
77
|
end
|
78
78
|
end
|
79
|
-
|
80
|
-
|
79
|
+
# add mandatory retry parameter for node api
|
80
|
+
ts_tags = transfer_spec['tags']
|
81
|
+
if ts_tags.is_a?(Hash) && ts_tags[Fasp::TransferSpec::TAG_RESERVED].is_a?(Hash)
|
82
|
+
ts_tags[Fasp::TransferSpec::TAG_RESERVED]['xfer_retry'] ||= 150
|
81
83
|
end
|
82
|
-
# Optimization in case of sending to the same node
|
84
|
+
# Optimization in case of sending to the same node
|
85
|
+
# TODO: probably remove this, as /etc/hosts shall be used for that
|
83
86
|
if !transfer_spec['wss_enabled'] && transfer_spec['remote_host'].eql?(URI.parse(node_api_.params[:base_url]).host)
|
84
87
|
transfer_spec['remote_host'] = '127.0.0.1'
|
85
88
|
end
|
@@ -116,9 +119,16 @@ module Aspera
|
|
116
119
|
else
|
117
120
|
notify_progress(@transfer_id, transfer_data['bytes_transferred'])
|
118
121
|
end
|
122
|
+
when 'failed'
|
123
|
+
# Bug in HSTS ? transfer is marked failed, but there is no reason
|
124
|
+
if transfer_data['error_code'].eql?(0) && transfer_data['error_desc'].empty?
|
125
|
+
notify_end(@transfer_id)
|
126
|
+
break
|
127
|
+
end
|
128
|
+
raise Fasp::Error, "status: #{transfer_data['status']}. code: #{transfer_data['error_code']}. description: #{transfer_data['error_desc']}"
|
119
129
|
else
|
120
130
|
Log.log.warn{"transfer_data -> #{transfer_data}"}
|
121
|
-
raise Fasp::Error, "#{transfer_data['status']}: #{transfer_data['error_desc']}"
|
131
|
+
raise Fasp::Error, "status: #{transfer_data['status']}. code: #{transfer_data['error_code']}. description: #{transfer_data['error_desc']}"
|
122
132
|
end
|
123
133
|
sleep(1)
|
124
134
|
end
|