aspera-cli 4.24.2 → 4.25.0.pre2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/CHANGELOG.md +1067 -758
- data/CONTRIBUTING.md +93 -120
- data/README.md +817 -510
- data/lib/aspera/agent/direct.rb +14 -12
- data/lib/aspera/agent/transferd.rb +4 -4
- data/lib/aspera/api/aoc.rb +71 -43
- data/lib/aspera/api/cos_node.rb +3 -2
- data/lib/aspera/api/faspex.rb +6 -5
- data/lib/aspera/api/node.rb +10 -12
- data/lib/aspera/ascmd.rb +1 -2
- data/lib/aspera/ascp/installation.rb +55 -41
- data/lib/aspera/ascp/management.rb +9 -5
- data/lib/aspera/assert.rb +28 -6
- data/lib/aspera/cli/error.rb +4 -2
- data/lib/aspera/cli/extended_value.rb +94 -62
- data/lib/aspera/cli/formatter.rb +55 -22
- data/lib/aspera/cli/main.rb +21 -14
- data/lib/aspera/cli/manager.rb +349 -248
- data/lib/aspera/cli/plugins/alee.rb +3 -3
- data/lib/aspera/cli/plugins/aoc.rb +94 -51
- data/lib/aspera/cli/plugins/base.rb +62 -49
- data/lib/aspera/cli/plugins/config.rb +85 -96
- data/lib/aspera/cli/plugins/console.rb +15 -9
- data/lib/aspera/cli/plugins/cos.rb +1 -1
- data/lib/aspera/cli/plugins/faspex.rb +34 -27
- data/lib/aspera/cli/plugins/faspex5.rb +47 -44
- data/lib/aspera/cli/plugins/faspio.rb +7 -6
- data/lib/aspera/cli/plugins/httpgw.rb +3 -2
- data/lib/aspera/cli/plugins/node.rb +132 -120
- data/lib/aspera/cli/plugins/oauth.rb +1 -1
- data/lib/aspera/cli/plugins/orchestrator.rb +116 -33
- data/lib/aspera/cli/plugins/preview.rb +26 -46
- data/lib/aspera/cli/plugins/server.rb +9 -10
- data/lib/aspera/cli/plugins/shares.rb +77 -43
- data/lib/aspera/cli/sync_actions.rb +49 -38
- data/lib/aspera/cli/transfer_agent.rb +16 -34
- data/lib/aspera/cli/version.rb +1 -1
- data/lib/aspera/cli/wizard.rb +8 -5
- data/lib/aspera/command_line_builder.rb +20 -17
- data/lib/aspera/coverage.rb +6 -2
- data/lib/aspera/environment.rb +71 -84
- data/lib/aspera/faspex_gw.rb +1 -1
- data/lib/aspera/faspex_postproc.rb +1 -1
- data/lib/aspera/keychain/factory.rb +1 -2
- data/lib/aspera/keychain/macos_security.rb +2 -2
- data/lib/aspera/log.rb +2 -1
- data/lib/aspera/markdown.rb +31 -0
- data/lib/aspera/nagios.rb +6 -5
- data/lib/aspera/oauth/base.rb +17 -27
- data/lib/aspera/oauth/factory.rb +1 -1
- data/lib/aspera/oauth/url_json.rb +2 -1
- data/lib/aspera/preview/file_types.rb +23 -37
- data/lib/aspera/preview/terminal.rb +95 -29
- data/lib/aspera/preview/utils.rb +6 -5
- data/lib/aspera/products/connect.rb +3 -3
- data/lib/aspera/rest.rb +51 -39
- data/lib/aspera/rest_error_analyzer.rb +4 -4
- data/lib/aspera/ssh.rb +5 -2
- data/lib/aspera/ssl.rb +41 -0
- data/lib/aspera/sync/conf.schema.yaml +182 -34
- data/lib/aspera/sync/database.rb +2 -1
- data/lib/aspera/sync/operations.rb +128 -72
- data/lib/aspera/transfer/parameters.rb +3 -4
- data/lib/aspera/transfer/spec.rb +2 -3
- data/lib/aspera/transfer/spec.schema.yaml +49 -19
- data/lib/aspera/transfer/spec_doc.rb +14 -14
- data/lib/aspera/uri_reader.rb +1 -1
- data/lib/transferd_pb.rb +2 -2
- data.tar.gz.sig +0 -0
- metadata +33 -6
- metadata.gz.sig +0 -0
|
@@ -23,6 +23,7 @@ module Aspera
|
|
|
23
23
|
'Faspex'
|
|
24
24
|
end
|
|
25
25
|
|
|
26
|
+
# @return [Hash,NilClass]
|
|
26
27
|
def detect(address_or_url)
|
|
27
28
|
# add scheme if missing
|
|
28
29
|
address_or_url = "https://#{address_or_url}" unless address_or_url.match?(%r{^[a-z]{1,6}://})
|
|
@@ -33,7 +34,7 @@ module Aspera
|
|
|
33
34
|
# Faspex is always HTTPS
|
|
34
35
|
next unless base_url.start_with?('https://')
|
|
35
36
|
api = Rest.new(base_url: base_url, redirect_max: 1)
|
|
36
|
-
response = api.
|
|
37
|
+
response = api.read(Api::Faspex::PATH_API_DETECT, ret: :resp)
|
|
37
38
|
next unless response.code.start_with?('2') && response.body.strip.empty?
|
|
38
39
|
# end is at -1, and subtract 1 for "/"
|
|
39
40
|
url_length = -2 - Api::Faspex::PATH_API_DETECT.length
|
|
@@ -52,7 +53,7 @@ module Aspera
|
|
|
52
53
|
end
|
|
53
54
|
|
|
54
55
|
# @param wizard [Wizard] The wizard object
|
|
55
|
-
# @param app_url [
|
|
56
|
+
# @param app_url [String] Tested URL
|
|
56
57
|
# @return [Hash] :preset_value, :test_args
|
|
57
58
|
def wizard(wizard, app_url)
|
|
58
59
|
client_id = options.get_option(:client_id)
|
|
@@ -93,7 +94,7 @@ module Aspera
|
|
|
93
94
|
super
|
|
94
95
|
options.declare(:box, "Package inbox, either shared inbox name or one of: #{Api::Faspex::API_LIST_MAILBOX_TYPES.join(', ')} or #{SpecialValues::ALL}", default: 'inbox')
|
|
95
96
|
options.declare(:shared_folder, 'Send package with files from shared folder')
|
|
96
|
-
options.declare(:group_type, 'Type of shared box',
|
|
97
|
+
options.declare(:group_type, 'Type of shared box', allowed: %i[shared_inboxes workgroups], default: :shared_inboxes)
|
|
97
98
|
options.parse_options!
|
|
98
99
|
end
|
|
99
100
|
|
|
@@ -222,8 +223,7 @@ module Aspera
|
|
|
222
223
|
else
|
|
223
224
|
# a single id was provided, or a list of ids
|
|
224
225
|
package_ids = [package_ids] unless package_ids.is_a?(Array)
|
|
225
|
-
Aspera.
|
|
226
|
-
Aspera.assert(package_ids.all?(String)){'Package id shall be String'}
|
|
226
|
+
Aspera.assert_array_all(package_ids, String){'Package id(s)'}
|
|
227
227
|
# packages = package_ids.map{|pkg_id|@api_v5.read("packages/#{pkg_id}")}
|
|
228
228
|
packages = package_ids.map{ |pkg_id| {'id'=>pkg_id}}
|
|
229
229
|
end
|
|
@@ -256,7 +256,7 @@ module Aspera
|
|
|
256
256
|
content_type: Rest::MIME_JSON,
|
|
257
257
|
body: param_file_list,
|
|
258
258
|
headers: {'Accept' => Rest::MIME_JSON}
|
|
259
|
-
)
|
|
259
|
+
)
|
|
260
260
|
# delete flag for Connect Client
|
|
261
261
|
transfer_spec.delete('authentication')
|
|
262
262
|
statuses = transfer.start(transfer_spec)
|
|
@@ -294,28 +294,29 @@ module Aspera
|
|
|
294
294
|
until folders_to_process.empty?
|
|
295
295
|
path = folders_to_process.shift
|
|
296
296
|
loop do
|
|
297
|
-
|
|
297
|
+
data, http = @api_v5.call(
|
|
298
298
|
operation: 'POST',
|
|
299
299
|
subpath: browse_endpoint,
|
|
300
300
|
query: query,
|
|
301
301
|
content_type: Rest::MIME_JSON,
|
|
302
302
|
body: {'path' => path, 'filters' => filters},
|
|
303
|
-
headers: {'Accept' => Rest::MIME_JSON}
|
|
303
|
+
headers: {'Accept' => Rest::MIME_JSON},
|
|
304
|
+
ret: :both
|
|
304
305
|
)
|
|
305
|
-
all_items.concat(
|
|
306
|
+
all_items.concat(data['items'])
|
|
306
307
|
if !max_items.nil? && (all_items.count >= max_items)
|
|
307
308
|
all_items = all_items.slice(0, max_items) if all_items.count > max_items
|
|
308
309
|
break
|
|
309
310
|
end
|
|
310
|
-
folders_to_process.concat(
|
|
311
|
+
folders_to_process.concat(data['items'].select{ |i| i['type'].eql?('directory')}.map{ |i| i['path']}) if recursive
|
|
311
312
|
if use_paging
|
|
312
|
-
iteration_token =
|
|
313
|
+
iteration_token = http[Api::Faspex::HEADER_ITERATION_TOKEN]
|
|
313
314
|
break if iteration_token.nil? || iteration_token.empty?
|
|
314
315
|
query['iteration_token'] = iteration_token
|
|
315
316
|
else
|
|
316
|
-
total_count =
|
|
317
|
-
break if
|
|
318
|
-
query['offset'] +=
|
|
317
|
+
total_count = data['total_count'] if total_count.nil?
|
|
318
|
+
break if data['item_count'].eql?(0)
|
|
319
|
+
query['offset'] += data['item_count']
|
|
319
320
|
end
|
|
320
321
|
formatter.long_operation_running(all_items.count)
|
|
321
322
|
end
|
|
@@ -349,8 +350,7 @@ module Aspera
|
|
|
349
350
|
when :delete
|
|
350
351
|
ids = package_id
|
|
351
352
|
ids = [ids] unless ids.is_a?(Array)
|
|
352
|
-
Aspera.
|
|
353
|
-
Aspera.assert(ids.all?(String)){"Package id(s) shall be String, but have: #{ids.map(&:class).uniq.join(', ')}"}
|
|
353
|
+
Aspera.assert_array_all(ids, String){'Package id(s)'}
|
|
354
354
|
# API returns 204, empty on success
|
|
355
355
|
@api_v5.call(
|
|
356
356
|
operation: 'DELETE',
|
|
@@ -372,29 +372,32 @@ module Aspera
|
|
|
372
372
|
}]
|
|
373
373
|
end
|
|
374
374
|
normalize_recipients(parameters)
|
|
375
|
+
# User specified content prot in tspec, but faspex requires in package creation
|
|
376
|
+
# `transfer_spec/upload` will set `content_protection`
|
|
377
|
+
if transfer.user_transfer_spec['content_protection'] && !parameters.key?('ear_enabled')
|
|
378
|
+
transfer.user_transfer_spec.delete('content_protection')
|
|
379
|
+
parameters['ear_enabled'] = true
|
|
380
|
+
end
|
|
375
381
|
package = @api_v5.create('packages', parameters)
|
|
376
382
|
shared_folder = options.get_option(:shared_folder)
|
|
377
383
|
if shared_folder.nil?
|
|
378
384
|
# send from local files
|
|
379
|
-
transfer_spec = @api_v5.
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
query:
|
|
383
|
-
|
|
384
|
-
body: {paths: transfer.source_list},
|
|
385
|
-
headers: {'Accept' => Rest::MIME_JSON}
|
|
386
|
-
)[:data]
|
|
385
|
+
transfer_spec = @api_v5.create(
|
|
386
|
+
"packages/#{package['id']}/transfer_spec/upload",
|
|
387
|
+
{paths: transfer.source_list},
|
|
388
|
+
query: {transfer_type: Api::Faspex::TRANSFER_CONNECT}
|
|
389
|
+
)
|
|
387
390
|
# well, we asked a TS for connect, but we actually want a generic one
|
|
388
391
|
transfer_spec.delete('authentication')
|
|
389
392
|
return Main.result_transfer(transfer.start(transfer_spec))
|
|
390
393
|
else
|
|
391
394
|
# send from remote shared folder
|
|
392
|
-
if (m =
|
|
395
|
+
if (m = Base.percent_selector(shared_folder))
|
|
393
396
|
shared_folder = lookup_entity_by_field(
|
|
394
397
|
api: @api_v5,
|
|
395
398
|
entity: 'shared_folders',
|
|
396
|
-
field: m[
|
|
397
|
-
value:
|
|
399
|
+
field: m[:field],
|
|
400
|
+
value: m[:value]
|
|
398
401
|
)['id']
|
|
399
402
|
end
|
|
400
403
|
transfer_request = {shared_folder_id: shared_folder, paths: transfer.source_list}
|
|
@@ -512,20 +515,20 @@ module Aspera
|
|
|
512
515
|
users = options.get_next_argument('user id, %name:, or Array')
|
|
513
516
|
users = [users] unless users.is_a?(Array)
|
|
514
517
|
users = users.map do |user|
|
|
515
|
-
if (m =
|
|
518
|
+
if (m = Base.percent_selector(user))
|
|
516
519
|
lookup_entity_by_field(
|
|
517
520
|
api: @api_v5,
|
|
518
521
|
entity: 'accounts',
|
|
519
|
-
field: m[
|
|
520
|
-
value:
|
|
521
|
-
query: Rest.php_style({type:
|
|
522
|
+
field: m[:field],
|
|
523
|
+
value: m[:value],
|
|
524
|
+
query: Rest.php_style({type: ACCOUNT_TYPES})
|
|
522
525
|
)['id']
|
|
523
526
|
else
|
|
524
527
|
# it's the user id (not member id...)
|
|
525
528
|
user
|
|
526
529
|
end
|
|
527
530
|
end
|
|
528
|
-
access = options.get_next_argument('level', mandatory: false, accept_list:
|
|
531
|
+
access = options.get_next_argument('level', mandatory: false, accept_list: SHARED_INBOX_MEMBER_LEVELS, default: :standard)
|
|
529
532
|
options.unshift_next_argument({user: users.map{ |u| {id: u, access: access}}})
|
|
530
533
|
end
|
|
531
534
|
return entity_execute(
|
|
@@ -536,10 +539,10 @@ module Aspera
|
|
|
536
539
|
) do |field, value|
|
|
537
540
|
lookup_entity_by_field(
|
|
538
541
|
api: @api_v5,
|
|
539
|
-
entity: '
|
|
542
|
+
entity: 'contacts',
|
|
540
543
|
field: field,
|
|
541
544
|
value: value,
|
|
542
|
-
query: Rest.php_style({type: %w
|
|
545
|
+
query: Rest.php_style({type: %w[user]})
|
|
543
546
|
)['id']
|
|
544
547
|
end
|
|
545
548
|
when :reset_password
|
|
@@ -552,12 +555,8 @@ module Aspera
|
|
|
552
555
|
end
|
|
553
556
|
|
|
554
557
|
def execute_admin
|
|
555
|
-
command = options.get_next_command(%i[configuration smtp
|
|
558
|
+
command = options.get_next_command(%i[configuration smtp events clean_deleted].concat(Api::Faspex::ADMIN_RESOURCES).freeze)
|
|
556
559
|
case command
|
|
557
|
-
when :resource
|
|
558
|
-
# resource will be deprecated
|
|
559
|
-
Log.log.warn('resource command is deprecated (4.18), directly use the specific command instead')
|
|
560
|
-
return execute_resource(options.get_next_command(Api::Faspex::ADMIN_RESOURCES))
|
|
561
560
|
when *Api::Faspex::ADMIN_RESOURCES
|
|
562
561
|
return execute_resource(command)
|
|
563
562
|
when :clean_deleted
|
|
@@ -630,16 +629,16 @@ module Aspera
|
|
|
630
629
|
when :health
|
|
631
630
|
nagios = Nagios.new
|
|
632
631
|
begin
|
|
633
|
-
|
|
634
|
-
.
|
|
635
|
-
|
|
632
|
+
data, http = Rest.new(base_url: options.get_option(:url, mandatory: true))
|
|
633
|
+
.read('health', ret: :both)
|
|
634
|
+
data.each do |k, v|
|
|
636
635
|
nagios.add_ok(k, v.to_s)
|
|
637
636
|
end
|
|
638
|
-
nagios.add_ok('version',
|
|
637
|
+
nagios.add_ok('version', http['X-IBM-Aspera']) if http['X-IBM-Aspera']
|
|
639
638
|
rescue StandardError => e
|
|
640
639
|
nagios.add_critical('core', e.to_s)
|
|
641
640
|
end
|
|
642
|
-
|
|
641
|
+
Main.result_object_list(nagios.status_list)
|
|
643
642
|
when :user
|
|
644
643
|
case options.get_next_command(%i[account profile])
|
|
645
644
|
when :account
|
|
@@ -718,6 +717,10 @@ module Aspera
|
|
|
718
717
|
return Main.result_status('Gateway terminated')
|
|
719
718
|
end
|
|
720
719
|
end
|
|
720
|
+
SHARED_INBOX_MEMBER_LEVELS = %i[submit_only standard shared_inbox_admin].freeze
|
|
721
|
+
ACCOUNT_TYPES = %w{local_user saml_user self_registered_user external_user}.freeze
|
|
722
|
+
CONTACT_TYPES = %w{workgroup shared_inbox distribution_list user external_user}.freeze
|
|
723
|
+
private_constant :SHARED_INBOX_MEMBER_LEVELS, :ACCOUNT_TYPES, :CONTACT_TYPES
|
|
721
724
|
end
|
|
722
725
|
end
|
|
723
726
|
end
|
|
@@ -13,11 +13,12 @@ module Aspera
|
|
|
13
13
|
'faspio Gateway'
|
|
14
14
|
end
|
|
15
15
|
|
|
16
|
+
# @return [Hash,NilClass]
|
|
16
17
|
def detect(base_url)
|
|
17
18
|
api = Rest.new(base_url: base_url)
|
|
18
|
-
|
|
19
|
-
server_type =
|
|
20
|
-
return unless
|
|
19
|
+
data, http = api.read('ping', ret: :both)
|
|
20
|
+
server_type = http['Server']
|
|
21
|
+
return unless data.is_a?(Hash) && data.empty?
|
|
21
22
|
return unless server_type.is_a?(String) && server_type.include?('faspio')
|
|
22
23
|
return {
|
|
23
24
|
version: server_type.gsub(%r{^.*/}, ''),
|
|
@@ -27,7 +28,7 @@ module Aspera
|
|
|
27
28
|
end
|
|
28
29
|
|
|
29
30
|
# @param wizard [Wizard] The wizard object
|
|
30
|
-
# @param app_url [
|
|
31
|
+
# @param app_url [String] Tested URL
|
|
31
32
|
# @return [Hash] :preset_value, :test_args
|
|
32
33
|
def wizard(wizard, app_url)
|
|
33
34
|
return {
|
|
@@ -42,7 +43,7 @@ module Aspera
|
|
|
42
43
|
|
|
43
44
|
def initialize(**_)
|
|
44
45
|
super
|
|
45
|
-
options.declare(:auth, 'OAuth type of authentication',
|
|
46
|
+
options.declare(:auth, 'OAuth type of authentication', allowed: %i[jwt basic])
|
|
46
47
|
options.declare(:client_id, 'OAuth client identifier')
|
|
47
48
|
options.declare(:private_key, 'OAuth JWT RSA private key PEM value (prefix file path with @file:)')
|
|
48
49
|
options.declare(:passphrase, 'OAuth JWT RSA private key passphrase')
|
|
@@ -88,7 +89,7 @@ module Aspera
|
|
|
88
89
|
rescue StandardError => e
|
|
89
90
|
nagios.add_critical('api', e.to_s)
|
|
90
91
|
end
|
|
91
|
-
|
|
92
|
+
Main.result_object_list(nagios.status_list)
|
|
92
93
|
when :bridges
|
|
93
94
|
return entity_execute(api: api, entity: 'bridges')
|
|
94
95
|
end
|
|
@@ -14,6 +14,7 @@ module Aspera
|
|
|
14
14
|
'HTTP Gateway'
|
|
15
15
|
end
|
|
16
16
|
|
|
17
|
+
# @return [Hash,NilClass]
|
|
17
18
|
def detect(base_url)
|
|
18
19
|
api = Api::Httpgw.new(url: base_url)
|
|
19
20
|
api_info = api.info
|
|
@@ -26,7 +27,7 @@ module Aspera
|
|
|
26
27
|
end
|
|
27
28
|
|
|
28
29
|
# @param wizard [Wizard] The wizard object
|
|
29
|
-
# @param app_url [
|
|
30
|
+
# @param app_url [String] Tested URL
|
|
30
31
|
# @return [Hash] :preset_value, :test_args
|
|
31
32
|
def wizard(wizard, app_url)
|
|
32
33
|
return {
|
|
@@ -56,7 +57,7 @@ module Aspera
|
|
|
56
57
|
rescue StandardError => e
|
|
57
58
|
nagios.add_critical('api', e.to_s)
|
|
58
59
|
end
|
|
59
|
-
|
|
60
|
+
Main.result_object_list(nagios.status_list)
|
|
60
61
|
when :info
|
|
61
62
|
api_v1 = Api::Httpgw.new(url: base_url)
|
|
62
63
|
return Main.result_single_object(api_v1.info)
|