aspera-cli 4.15.0 → 4.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/BUGS.md +29 -3
- data/CHANGELOG.md +292 -228
- data/CONTRIBUTING.md +69 -18
- data/README.md +1102 -952
- data/bin/ascli +13 -31
- data/bin/asession +3 -1
- data/examples/dascli +2 -2
- data/lib/aspera/aoc.rb +28 -33
- data/lib/aspera/ascmd.rb +3 -6
- data/lib/aspera/assert.rb +45 -0
- data/lib/aspera/cli/extended_value.rb +5 -5
- data/lib/aspera/cli/formatter.rb +26 -13
- data/lib/aspera/cli/hints.rb +4 -3
- data/lib/aspera/cli/main.rb +16 -3
- data/lib/aspera/cli/manager.rb +45 -36
- data/lib/aspera/cli/plugin.rb +20 -13
- data/lib/aspera/cli/plugins/aoc.rb +103 -73
- data/lib/aspera/cli/plugins/ats.rb +4 -3
- data/lib/aspera/cli/plugins/config.rb +114 -119
- data/lib/aspera/cli/plugins/cos.rb +2 -2
- data/lib/aspera/cli/plugins/faspex.rb +23 -19
- data/lib/aspera/cli/plugins/faspex5.rb +75 -43
- data/lib/aspera/cli/plugins/node.rb +28 -15
- data/lib/aspera/cli/plugins/orchestrator.rb +4 -2
- data/lib/aspera/cli/plugins/preview.rb +9 -7
- data/lib/aspera/cli/plugins/server.rb +6 -3
- data/lib/aspera/cli/plugins/shares.rb +30 -26
- data/lib/aspera/cli/sync_actions.rb +9 -9
- data/lib/aspera/cli/transfer_agent.rb +21 -14
- data/lib/aspera/cli/transfer_progress.rb +2 -3
- data/lib/aspera/cli/version.rb +1 -1
- data/lib/aspera/command_line_builder.rb +13 -11
- data/lib/aspera/cos_node.rb +3 -2
- data/lib/aspera/coverage.rb +22 -0
- data/lib/aspera/data_repository.rb +33 -2
- data/lib/aspera/environment.rb +4 -2
- data/lib/aspera/fasp/{agent_aspera.rb → agent_alpha.rb} +29 -39
- data/lib/aspera/fasp/agent_base.rb +17 -7
- data/lib/aspera/fasp/agent_direct.rb +88 -84
- data/lib/aspera/fasp/agent_httpgw.rb +4 -3
- data/lib/aspera/fasp/agent_node.rb +3 -2
- data/lib/aspera/fasp/agent_trsdk.rb +79 -37
- data/lib/aspera/fasp/installation.rb +51 -12
- data/lib/aspera/fasp/management.rb +11 -6
- data/lib/aspera/fasp/parameters.rb +53 -47
- data/lib/aspera/fasp/resume_policy.rb +7 -5
- data/lib/aspera/fasp/sync.rb +273 -0
- data/lib/aspera/fasp/transfer_spec.rb +10 -8
- data/lib/aspera/fasp/uri.rb +2 -2
- data/lib/aspera/faspex_gw.rb +11 -8
- data/lib/aspera/faspex_postproc.rb +6 -5
- data/lib/aspera/id_generator.rb +3 -1
- data/lib/aspera/json_rpc.rb +10 -8
- data/lib/aspera/keychain/encrypted_hash.rb +46 -11
- data/lib/aspera/keychain/macos_security.rb +15 -13
- data/lib/aspera/log.rb +4 -3
- data/lib/aspera/nagios.rb +7 -2
- data/lib/aspera/node.rb +17 -16
- data/lib/aspera/node_simulator.rb +214 -0
- data/lib/aspera/oauth.rb +22 -19
- data/lib/aspera/persistency_action_once.rb +13 -14
- data/lib/aspera/persistency_folder.rb +3 -2
- data/lib/aspera/preview/file_types.rb +53 -267
- data/lib/aspera/preview/generator.rb +7 -5
- data/lib/aspera/preview/terminal.rb +14 -5
- data/lib/aspera/preview/utils.rb +8 -7
- data/lib/aspera/proxy_auto_config.rb +6 -3
- data/lib/aspera/rest.rb +29 -13
- data/lib/aspera/rest_error_analyzer.rb +1 -0
- data/lib/aspera/rest_errors_aspera.rb +2 -0
- data/lib/aspera/secret_hider.rb +5 -2
- data/lib/aspera/ssh.rb +10 -8
- data/lib/aspera/temp_file_manager.rb +1 -1
- data/lib/aspera/web_server_simple.rb +2 -1
- data.tar.gz.sig +0 -0
- metadata +96 -45
- metadata.gz.sig +0 -0
- data/lib/aspera/sync.rb +0 -219
@@ -12,6 +12,8 @@ require 'aspera/persistency_action_once'
|
|
12
12
|
require 'aspera/open_application'
|
13
13
|
require 'aspera/nagios'
|
14
14
|
require 'aspera/id_generator'
|
15
|
+
require 'aspera/log'
|
16
|
+
require 'aspera/assert'
|
15
17
|
require 'xmlsimple'
|
16
18
|
require 'json'
|
17
19
|
require 'cgi'
|
@@ -75,10 +77,10 @@ module Aspera
|
|
75
77
|
}
|
76
78
|
end
|
77
79
|
|
78
|
-
# extract elements from
|
80
|
+
# extract elements from faspex public link
|
79
81
|
def get_link_data(public_url)
|
80
82
|
public_uri = URI.parse(public_url)
|
81
|
-
|
83
|
+
assert((m = public_uri.path.match(%r{^(.*)/(external.*)$})), exception_class: Cli::BadArgument){'Public link does not match Faspex format'}
|
82
84
|
base = m[1]
|
83
85
|
subpath = m[2]
|
84
86
|
port_add = public_uri.port.eql?(public_uri.default_port) ? '' : ":#{public_uri.port}"
|
@@ -91,16 +93,13 @@ module Aspera
|
|
91
93
|
return result
|
92
94
|
end
|
93
95
|
|
94
|
-
# get Fasp::Uri::SCHEME URI from entry in xml, and fix problems
|
96
|
+
# get Fasp::Uri::SCHEME URI from entry in xml, and fix problems.
|
95
97
|
def get_fasp_uri_from_entry(entry, raise_no_link: true)
|
96
98
|
unless entry.key?('link')
|
97
99
|
raise Cli::BadArgument, 'package has no link (deleted?)' if raise_no_link
|
98
100
|
return nil
|
99
101
|
end
|
100
102
|
result = entry['link'].find{|e| e['rel'].eql?('package')}['href']
|
101
|
-
# tags in the end of URL is not well % encoded... there are "=" that should be %3D
|
102
|
-
# TODO: enter ticket to Faspex ?
|
103
|
-
# ##XXif m=result.match(/(=+)$/);result.gsub!(/=+$/,"#{"%3D"*m[1].length}");end
|
104
103
|
return result
|
105
104
|
end
|
106
105
|
|
@@ -162,9 +161,9 @@ module Aspera
|
|
162
161
|
max_pages = nil
|
163
162
|
result = []
|
164
163
|
if !mailbox_query.nil?
|
165
|
-
|
166
|
-
|
167
|
-
|
164
|
+
assert_type(mailbox_query, Hash){'query'}
|
165
|
+
assert((mailbox_query.keys - ATOM_EXT_PARAMS).empty?){"query: supported params: #{ATOM_EXT_PARAMS}"}
|
166
|
+
assert(!(mailbox_query.key?('startIndex') && mailbox_query.key?('page'))){'query: startIndex and page are exclusive'}
|
168
167
|
max_items = mailbox_query[MAX_ITEMS]
|
169
168
|
mailbox_query.delete(MAX_ITEMS)
|
170
169
|
max_pages = mailbox_query[MAX_PAGES]
|
@@ -271,7 +270,7 @@ module Aspera
|
|
271
270
|
end
|
272
271
|
return nagios.result
|
273
272
|
when :package
|
274
|
-
command_pkg = options.get_next_command(%i[send
|
273
|
+
command_pkg = options.get_next_command(%i[send receive list show], aliases: {recv: :receive})
|
275
274
|
case command_pkg
|
276
275
|
when :show
|
277
276
|
delivery_id = instance_identifier
|
@@ -284,7 +283,7 @@ module Aspera
|
|
284
283
|
}
|
285
284
|
when :send
|
286
285
|
delivery_info = options.get_option(:delivery_info, mandatory: true)
|
287
|
-
|
286
|
+
assert_type(delivery_info, Hash, exception_class: Cli::BadArgument){'delivery_info'}
|
288
287
|
# actual parameter to faspex API
|
289
288
|
package_create_params = {'delivery' => delivery_info}
|
290
289
|
public_link_url = options.get_option(:link)
|
@@ -294,7 +293,7 @@ module Aspera
|
|
294
293
|
first_source = delivery_info['sources'].first
|
295
294
|
first_source['paths'].push(*transfer.source_list)
|
296
295
|
source_id = instance_identifier(as_option: :remote_source) do |field, value|
|
297
|
-
|
296
|
+
assert(field.eql?('name'), exception_class: Cli::BadArgument){'only name as selector, or give id'}
|
298
297
|
source_list = api_v3.call({operation: 'GET', subpath: 'source_shares', headers: {'Accept' => 'application/json'}})[:data]['items']
|
299
298
|
self.class.get_source_id_by_name(value, source_list)
|
300
299
|
end
|
@@ -318,7 +317,7 @@ module Aspera
|
|
318
317
|
end
|
319
318
|
# Log.log.debug{Log.dump('transfer_spec',transfer_spec)}
|
320
319
|
return Main.result_transfer(transfer.start(transfer_spec))
|
321
|
-
when :
|
320
|
+
when :receive
|
322
321
|
link_url = options.get_option(:link)
|
323
322
|
# list of faspex ID/URI to download
|
324
323
|
pkg_id_uri = nil
|
@@ -341,8 +340,13 @@ module Aspera
|
|
341
340
|
delivery_id = instance_identifier
|
342
341
|
raise 'empty id' if delivery_id.empty?
|
343
342
|
recipient = options.get_option(:recipient)
|
344
|
-
if
|
343
|
+
if delivery_id.eql?(ExtendedValue::ALL)
|
345
344
|
pkg_id_uri = mailbox_filtered_entries.map{|i|{id: i[PACKAGE_MATCH_FIELD], uri: self.class.get_fasp_uri_from_entry(i, raise_no_link: false)}}
|
345
|
+
elsif delivery_id.eql?(ExtendedValue::INIT)
|
346
|
+
assert(skip_ids_persistency){'Only with option once_only'}
|
347
|
+
skip_ids_persistency.data.clear.concat(mailbox_filtered_entries.map{|i|{id: i[PACKAGE_MATCH_FIELD]}})
|
348
|
+
skip_ids_persistency.save
|
349
|
+
return Main.result_status("Initialized skip for #{skip_ids_persistency.data.count} package(s)")
|
346
350
|
elsif !recipient.nil? && recipient.start_with?('*')
|
347
351
|
found_package_link = mailbox_filtered_entries(stop_at_id: delivery_id).find{|p|p[PACKAGE_MATCH_FIELD].eql?(delivery_id)}['link'].first['href']
|
348
352
|
raise "Not Found. Dropbox and Workgroup packages can use the link option with #{Fasp::Uri::SCHEME}" if found_package_link.nil?
|
@@ -423,17 +427,17 @@ module Aspera
|
|
423
427
|
return {type: :object_list, data: source_list}
|
424
428
|
else # :info :node
|
425
429
|
source_id = instance_identifier do |field, value|
|
426
|
-
|
430
|
+
assert(field.eql?('name'), exception_class: Cli::BadArgument){'only name as selector, or give id'}
|
427
431
|
self.class.get_source_id_by_name(value, source_list)
|
428
432
|
end.to_i
|
429
433
|
source_name = source_list.find{|i|i['id'].eql?(source_id)}['name']
|
430
434
|
source_hash = options.get_option(:storage, mandatory: true)
|
431
435
|
# check value of option
|
432
|
-
|
436
|
+
assert_type(source_hash, Hash, exception_class: Cli::Error){'storage option'}
|
433
437
|
source_hash.each do |name, storage|
|
434
|
-
|
438
|
+
assert_type(storage, Hash, exception_class: Cli::Error){"storage '#{name}'"}
|
435
439
|
[KEY_NODE, KEY_PATH].each do |key|
|
436
|
-
|
440
|
+
assert(storage.key?(key), exception_class: Cli::Error){"storage '#{name}' must have a '#{key}'"}
|
437
441
|
end
|
438
442
|
end
|
439
443
|
if !source_hash.key?(source_name)
|
@@ -446,8 +450,8 @@ module Aspera
|
|
446
450
|
return {data: source_info, type: :single_object}
|
447
451
|
when :node
|
448
452
|
node_config = ExtendedValue.instance.evaluate(source_info[KEY_NODE])
|
449
|
-
raise Cli::Error, "bad type for: \"#{source_info[KEY_NODE]}\"" unless node_config.is_a?(Hash)
|
450
453
|
Log.log.debug{"node=#{node_config}"}
|
454
|
+
assert_type(node_config, Hash, exception_class: Cli::Error){source_info[KEY_NODE]}
|
451
455
|
api_node = Rest.new({
|
452
456
|
base_url: node_config['url'],
|
453
457
|
auth: {
|
@@ -3,10 +3,12 @@
|
|
3
3
|
# spellchecker: ignore workgroups mypackages passcode
|
4
4
|
|
5
5
|
require 'aspera/cli/basic_auth_plugin'
|
6
|
+
require 'aspera/cli/extended_value'
|
6
7
|
require 'aspera/persistency_action_once'
|
7
8
|
require 'aspera/id_generator'
|
8
9
|
require 'aspera/nagios'
|
9
10
|
require 'aspera/environment'
|
11
|
+
require 'aspera/assert'
|
10
12
|
require 'securerandom'
|
11
13
|
require 'tty-spinner'
|
12
14
|
|
@@ -19,17 +21,17 @@ module Aspera
|
|
19
21
|
API_DETECT = 'api/v5/configuration/ping'
|
20
22
|
# list of supported mailbox types (to list packages)
|
21
23
|
API_LIST_MAILBOX_TYPES = %w[inbox inbox_history inbox_all inbox_all_history outbox outbox_history pending pending_history all].freeze
|
22
|
-
PACKAGE_ALL_INIT = 'INIT'
|
23
24
|
PACKAGE_SEND_FROM_REMOTE_SOURCE = 'remote_source'
|
24
25
|
# Faspex API v5: get transfer spec for connect
|
25
26
|
TRANSFER_CONNECT = 'connect'
|
26
27
|
ADMIN_RESOURCES = %i[
|
27
28
|
accounts contacts jobs workgroups shared_inboxes nodes oauth_clients registrations saml_configs
|
28
|
-
metadata_profiles email_notifications
|
29
|
+
metadata_profiles email_notifications alternate_addresses
|
29
30
|
].freeze
|
30
31
|
JOB_RUNNING = %w[queued working].freeze
|
31
32
|
STANDARD_PATH = '/aspera/faspex'
|
32
|
-
|
33
|
+
PER_PAGE_DEFAULT = 100
|
34
|
+
private_constant(*%i[JOB_RUNNING RECIPIENT_TYPES PACKAGE_TERMINATED API_DETECT API_LIST_MAILBOX_TYPES PACKAGE_SEND_FROM_REMOTE_SOURCE PER_PAGE_DEFAULT])
|
33
35
|
class << self
|
34
36
|
def application_name
|
35
37
|
'Faspex'
|
@@ -102,7 +104,7 @@ module Aspera
|
|
102
104
|
options.declare(:auth, 'OAuth type of authentication', values: %i[boot].concat(Oauth::STD_AUTH_TYPES), default: :jwt)
|
103
105
|
options.declare(:private_key, 'OAuth JWT RSA private key PEM value (prefix file path with @file:)')
|
104
106
|
options.declare(:passphrase, 'OAuth JWT RSA private key passphrase')
|
105
|
-
options.declare(:box, "Package inbox, either shared inbox name or one of #{API_LIST_MAILBOX_TYPES} or #{ExtendedValue::ALL}", default: 'inbox')
|
107
|
+
options.declare(:box, "Package inbox, either shared inbox name or one of: #{API_LIST_MAILBOX_TYPES.join(', ')} or #{ExtendedValue::ALL}", default: 'inbox')
|
106
108
|
options.declare(:shared_folder, 'Send package with files from shared folder')
|
107
109
|
options.declare(:group_type, 'Type of shared box', values: %i[shared_inboxes workgroups], default: :shared_inboxes)
|
108
110
|
options.parse_options!
|
@@ -168,14 +170,14 @@ module Aspera
|
|
168
170
|
headers: {typ: 'JWT'}
|
169
171
|
}
|
170
172
|
}})
|
171
|
-
else
|
173
|
+
else error_unexpected_value(auth_type)
|
172
174
|
end
|
173
175
|
end
|
174
176
|
|
175
177
|
# if recipient is just an email, then convert to expected API hash : name and type
|
176
178
|
def normalize_recipients(parameters)
|
177
179
|
return unless parameters.key?('recipients')
|
178
|
-
|
180
|
+
assert_type(parameters['recipients'], Array){'recipients'}
|
179
181
|
recipient_types = RECIPIENT_TYPES
|
180
182
|
if parameters.key?('recipient_types')
|
181
183
|
recipient_types = parameters['recipient_types']
|
@@ -236,27 +238,25 @@ module Aspera
|
|
236
238
|
spinner.spin
|
237
239
|
sleep(0.5)
|
238
240
|
end
|
239
|
-
|
241
|
+
error_unreachable_line
|
240
242
|
end
|
241
243
|
|
242
|
-
#
|
244
|
+
# Get a (full or partial) list of all entities of a given type
|
243
245
|
# @param type [String] the type of entity to list (just a name)
|
244
246
|
# @param query [Hash,nil] additional query parameters
|
245
|
-
# @param
|
247
|
+
# @param real_path [String] real path if it's n ot just the type
|
246
248
|
# @param item_list_key [String] key in the result to get the list of items
|
247
|
-
def list_entities(type:,
|
248
|
-
query = {} if query.nil?
|
249
|
+
def list_entities(type:, real_path: nil, query: {}, item_list_key: nil)
|
249
250
|
type = type.to_s if type.is_a?(Symbol)
|
251
|
+
assert_type(type, String)
|
250
252
|
item_list_key = type if item_list_key.nil?
|
251
|
-
|
252
|
-
full_path = type
|
253
|
-
full_path = "#{path}/#{full_path}" unless path.nil? || path.empty?
|
253
|
+
full_path = real_path.nil? ? type : real_path
|
254
254
|
result = []
|
255
255
|
offset = 0
|
256
256
|
max_items = query.delete(MAX_ITEMS)
|
257
257
|
remain_pages = query.delete(MAX_PAGES)
|
258
258
|
# merge default parameters, by default 100 per page
|
259
|
-
query = {'limit'=>
|
259
|
+
query = {'limit'=> PER_PAGE_DEFAULT}.merge(query)
|
260
260
|
loop do
|
261
261
|
query['offset'] = offset
|
262
262
|
page_result = @api_v5.read(full_path, query)[:data]
|
@@ -275,33 +275,36 @@ module Aspera
|
|
275
275
|
end
|
276
276
|
|
277
277
|
# lookup an entity id from its name
|
278
|
-
def lookup_entity_by_field(type:, value:, field: 'name', query: :default,
|
279
|
-
|
280
|
-
|
278
|
+
def lookup_entity_by_field(type:, value:, field: 'name', query: :default, real_path: nil, item_list_key: nil)
|
279
|
+
if query.eql?(:default)
|
280
|
+
assert(field.eql?('name')){'Default query is on name only'}
|
281
|
+
query = {'q'=> value}
|
282
|
+
end
|
283
|
+
found = list_entities(type: type, real_path: real_path, query: query, item_list_key: item_list_key).select{|i|i[field].eql?(value)}
|
281
284
|
case found.length
|
282
285
|
when 0 then raise "No #{type} with #{field} = #{value}"
|
283
286
|
when 1 then return found.first
|
284
|
-
else raise "Found #{found.length} #{
|
287
|
+
else raise "Found #{found.length} #{real_path} with #{field} = #{value}"
|
285
288
|
end
|
286
289
|
end
|
287
290
|
|
288
291
|
# list all packages with optional filter
|
289
|
-
def list_packages_with_filter
|
292
|
+
def list_packages_with_filter(query: {})
|
290
293
|
filter = options.get_next_argument('filter', mandatory: false, type: Proc, default: ->(_x){true})
|
291
294
|
# translate box name to API prefix (with ending slash)
|
292
295
|
box = options.get_option(:box)
|
293
|
-
|
296
|
+
real_path =
|
294
297
|
case box
|
295
|
-
when ExtendedValue::ALL then '' # only admin can list all packages globally
|
296
|
-
when *API_LIST_MAILBOX_TYPES then box
|
298
|
+
when ExtendedValue::ALL then 'packages' # only admin can list all packages globally
|
299
|
+
when *API_LIST_MAILBOX_TYPES then "#{box}/packages"
|
297
300
|
else
|
298
301
|
group_type = options.get_option(:group_type)
|
299
|
-
"#{group_type}/#{lookup_entity_by_field(type: group_type, value: box)['id']}"
|
302
|
+
"#{group_type}/#{lookup_entity_by_field(type: group_type, value: box)['id']}/packages"
|
300
303
|
end
|
301
304
|
return list_entities(
|
302
305
|
type: 'packages',
|
303
|
-
query: query_read_delete(default:
|
304
|
-
|
306
|
+
query: query_read_delete(default: query),
|
307
|
+
real_path: real_path).select(&filter)
|
305
308
|
end
|
306
309
|
|
307
310
|
def package_receive(package_ids)
|
@@ -315,25 +318,32 @@ module Aspera
|
|
315
318
|
id: IdGenerator.from_list([
|
316
319
|
'faspex_recv',
|
317
320
|
options.get_option(:url, mandatory: true),
|
318
|
-
options.get_option(:username, mandatory: true)
|
321
|
+
options.get_option(:username, mandatory: true),
|
322
|
+
options.get_option(:box, mandatory: true)
|
323
|
+
]))
|
319
324
|
end
|
325
|
+
packages = []
|
320
326
|
case package_ids
|
321
|
-
when
|
322
|
-
|
327
|
+
when ExtendedValue::INIT
|
328
|
+
assert(skip_ids_persistency){'Only with option once_only'}
|
323
329
|
skip_ids_persistency.data.clear.concat(list_packages_with_filter.map{|p|p['id']})
|
324
330
|
skip_ids_persistency.save
|
325
331
|
return Main.result_status("Initialized skip for #{skip_ids_persistency.data.count} package(s)")
|
326
332
|
when ExtendedValue::ALL
|
327
333
|
# TODO: if packages have same name, they will overwrite ?
|
328
|
-
|
329
|
-
Log.log.
|
330
|
-
Log.log.
|
331
|
-
|
332
|
-
Log.log.
|
334
|
+
packages = list_packages_with_filter(query: {'status' => 'completed'})
|
335
|
+
Log.log.trace1{Log.dump(:package_ids, packages.map{|p|p['id']})}
|
336
|
+
Log.log.trace1{Log.dump(:skip_ids, skip_ids_persistency.data)}
|
337
|
+
packages.reject!{|p|skip_ids_persistency.data.include?(p['id'])} if skip_ids_persistency
|
338
|
+
Log.log.trace1{Log.dump(:package_ids, packages.map{|p|p['id']})}
|
339
|
+
else
|
340
|
+
# a single id was provided, or a list of ids
|
341
|
+
package_ids = [package_ids] unless package_ids.is_a?(Array)
|
342
|
+
assert_type(package_ids, Array){'Expecting a single package id or a list of ids'}
|
343
|
+
assert(package_ids.all?(String)){'Package id shall be String'}
|
344
|
+
# packages = package_ids.map{|pkg_id|@api_v5.read("packages/#{pkg_id}")[:data]}
|
345
|
+
packages = package_ids.map{|pkg_id|{'id'=>pkg_id}}
|
333
346
|
end
|
334
|
-
# a single id was provided
|
335
|
-
# TODO: check package_ids is a list of strings
|
336
|
-
package_ids = [package_ids] if package_ids.is_a?(String)
|
337
347
|
result_transfer = []
|
338
348
|
param_file_list = {}
|
339
349
|
begin
|
@@ -352,7 +362,8 @@ module Aspera
|
|
352
362
|
else # shared inbox / workgroup
|
353
363
|
download_params[:recipient_workgroup_id] = lookup_entity_by_field(type: options.get_option(:group_type), value: box)['id']
|
354
364
|
end
|
355
|
-
|
365
|
+
packages.each do |package|
|
366
|
+
pkg_id = package['id']
|
356
367
|
formatter.display_status("Receiving package #{pkg_id}")
|
357
368
|
# TODO: allow from sent as well ?
|
358
369
|
transfer_spec = @api_v5.call(
|
@@ -406,7 +417,8 @@ module Aspera
|
|
406
417
|
when :delete
|
407
418
|
ids = package_id
|
408
419
|
ids = [ids] unless ids.is_a?(Array)
|
409
|
-
|
420
|
+
assert_type(ids, Array){'Package identifier'}
|
421
|
+
assert(ids.all?(String)){'Package id shall be String'}
|
410
422
|
# API returns 204, empty on success
|
411
423
|
@api_v5.call({operation: 'DELETE', subpath: 'packages', headers: {'Accept' => 'application/json'}, json_params: {ids: ids}})
|
412
424
|
return Main.result_status('Package(s) deleted')
|
@@ -523,11 +535,14 @@ module Aspera
|
|
523
535
|
id_as_arg = false
|
524
536
|
display_fields = nil
|
525
537
|
adm_api = @api_v5
|
538
|
+
special_query = :default
|
526
539
|
available_commands = [].concat(Plugin::ALL_OPS)
|
527
540
|
case res_type
|
528
541
|
when :metadata_profiles
|
529
542
|
res_path = 'configuration/metadata_profiles'
|
530
543
|
list_key = 'profiles'
|
544
|
+
when :alternate_addresses
|
545
|
+
res_path = 'configuration/alternate_addresses'
|
531
546
|
when :email_notifications
|
532
547
|
list_key = false
|
533
548
|
id_as_arg = 'type'
|
@@ -538,11 +553,26 @@ module Aspera
|
|
538
553
|
adm_api = Rest.new(@api_v5.params.merge({base_url: auth_api_url}))
|
539
554
|
when :shared_inboxes, :workgroups
|
540
555
|
available_commands.push(:members, :saml_groups, :invite_external_collaborator)
|
556
|
+
special_query = {'all': true}
|
557
|
+
when :nodes
|
558
|
+
available_commands.push(:shared_folders)
|
541
559
|
end
|
542
560
|
res_command = options.get_next_command(available_commands)
|
543
561
|
case res_command
|
544
562
|
when *Plugin::ALL_OPS
|
545
|
-
return entity_command(res_command, adm_api, res_path, item_list_key: list_key, display_fields: display_fields, id_as_arg: id_as_arg)
|
563
|
+
return entity_command(res_command, adm_api, res_path, item_list_key: list_key, display_fields: display_fields, id_as_arg: id_as_arg) do |field, value|
|
564
|
+
lookup_entity_by_field(
|
565
|
+
type: res_type, real_path: res_path, field: field, value: value, query: special_query)['id']
|
566
|
+
end
|
567
|
+
when :shared_folders
|
568
|
+
node_id = instance_identifier do |field, value|
|
569
|
+
lookup_entity_by_field(type: res_type.to_s, field: field, value: value)['id']
|
570
|
+
end
|
571
|
+
sh_path = "#{res_path}/#{node_id}/shared_folders"
|
572
|
+
return entity_action(adm_api, sh_path, item_list_key: 'shared_folders') do |field, value|
|
573
|
+
lookup_entity_by_field(
|
574
|
+
type: 'shared_folders', real_path: sh_path, field: field, value: value)['id']
|
575
|
+
end
|
546
576
|
when :invite_external_collaborator
|
547
577
|
shared_inbox_id = instance_identifier { |field, value| lookup_entity_by_field(type: res_type.to_s, field: field, value: value)['id']}
|
548
578
|
creation_payload = value_create_modify(command: res_command, type: [Hash, String])
|
@@ -550,7 +580,9 @@ module Aspera
|
|
550
580
|
res_path = "#{res_type}/#{shared_inbox_id}/external_collaborator"
|
551
581
|
result = adm_api.create(res_path, creation_payload)[:data]
|
552
582
|
formatter.display_status(result['message'])
|
553
|
-
result = lookup_entity_by_field(
|
583
|
+
result = lookup_entity_by_field(
|
584
|
+
type: 'members', real_path: "#{res_type}/#{shared_inbox_id}/members", value: creation_payload['email_address'],
|
585
|
+
query: {})
|
554
586
|
return {type: :single_object, data: result}
|
555
587
|
when :members, :saml_groups
|
556
588
|
res_id = instance_identifier { |field, value| lookup_entity_by_field(type: res_type.to_s, field: field, value: value)['id']}
|
@@ -607,7 +639,7 @@ module Aspera
|
|
607
639
|
end
|
608
640
|
when :gateway
|
609
641
|
require 'aspera/faspex_gw'
|
610
|
-
url = value_create_modify(command: command, type: String)
|
642
|
+
url = value_create_modify(command: command, description: 'listening url (e.g. https://localhost:12345)', type: String)
|
611
643
|
uri = URI.parse(url)
|
612
644
|
server = WebServerSimple.new(uri)
|
613
645
|
server.mount(uri.path, Faspex4GWServlet, @api_v5, nil)
|
@@ -622,7 +654,7 @@ module Aspera
|
|
622
654
|
require 'aspera/faspex_postproc' # cspell:disable-line
|
623
655
|
parameters = value_create_modify(command: command)
|
624
656
|
parameters = parameters.symbolize_keys
|
625
|
-
|
657
|
+
assert(parameters.key?(:url)){'Missing key: url'}
|
626
658
|
uri = URI.parse(parameters[:url])
|
627
659
|
parameters[:processing] ||= {}
|
628
660
|
parameters[:processing][:root] = uri.path
|
@@ -9,8 +9,9 @@ require 'aspera/hash_ext'
|
|
9
9
|
require 'aspera/id_generator'
|
10
10
|
require 'aspera/node'
|
11
11
|
require 'aspera/aoc'
|
12
|
-
require 'aspera/sync'
|
13
12
|
require 'aspera/oauth'
|
13
|
+
require 'aspera/node_simulator'
|
14
|
+
require 'aspera/assert'
|
14
15
|
require 'base64'
|
15
16
|
require 'zlib'
|
16
17
|
|
@@ -303,7 +304,7 @@ module Aspera
|
|
303
304
|
save_to_file: File.join(transfer.destination_folder(Fasp::TransferSpec::DIRECTION_RECEIVE), file_name))
|
304
305
|
return Main.result_status("downloaded: #{file_name}")
|
305
306
|
end
|
306
|
-
|
307
|
+
error_unreachable_line
|
307
308
|
end
|
308
309
|
|
309
310
|
# common API to node and Shares
|
@@ -412,7 +413,7 @@ module Aspera
|
|
412
413
|
url: apifid[:api].params[:base_url],
|
413
414
|
root_id: apifid[:file_id]
|
414
415
|
}
|
415
|
-
|
416
|
+
assert_values(apifid[:api].params[:auth][:type], %i[basic oauth2])
|
416
417
|
case apifid[:api].params[:auth][:type]
|
417
418
|
when :basic
|
418
419
|
result[:username] = apifid[:api].params[:auth][:username]
|
@@ -420,7 +421,7 @@ module Aspera
|
|
420
421
|
when :oauth2
|
421
422
|
result[:username] = apifid[:api].params[:headers][Aspera::Node::HEADER_X_ASPERA_ACCESS_KEY]
|
422
423
|
result[:password] = apifid[:api].oauth_token
|
423
|
-
else
|
424
|
+
else error_unreachable_line
|
424
425
|
end
|
425
426
|
return {type: :single_object, data: result} if command_repo.eql?(:node_info)
|
426
427
|
# check format of bearer token
|
@@ -463,10 +464,11 @@ module Aspera
|
|
463
464
|
return execute_sync_action do |sync_direction, _local_path, remote_path|
|
464
465
|
# Gen4 API
|
465
466
|
# direction is push pull, bidi
|
467
|
+
assert_values(sync_direction, %i[push pull bidi])
|
466
468
|
ts_direction = case sync_direction
|
467
469
|
when :push, :bidi then Fasp::TransferSpec::DIRECTION_SEND
|
468
470
|
when :pull then Fasp::TransferSpec::DIRECTION_RECEIVE
|
469
|
-
else
|
471
|
+
else error_unreachable_line
|
470
472
|
end
|
471
473
|
# remote is specified by option to_folder
|
472
474
|
apifid = @api_node.resolve_api_fid(top_file_id, remote_path)
|
@@ -536,12 +538,7 @@ module Aspera
|
|
536
538
|
subpath: "files/#{apifid[:file_id]}/preview",
|
537
539
|
headers: {'Accept' => 'image/png'}
|
538
540
|
)
|
539
|
-
|
540
|
-
terminal_options = options.get_option(:query, default: {}).symbolize_keys
|
541
|
-
allowed_options = Preview::Terminal.method(:build).parameters.select{|i|i[0].eql?(:key)}.map{|i|i[1]}
|
542
|
-
unknown_options = terminal_options.keys - allowed_options
|
543
|
-
raise "invalid options: #{unknown_options.join(', ')}, use #{allowed_options.join(', ')}" unless unknown_options.empty?
|
544
|
-
return Main.result_status(Preview::Terminal.build(result[:http].body, **terminal_options))
|
541
|
+
return Main.result_picture_in_terminal(options, result[:http].body)
|
545
542
|
when :permission
|
546
543
|
apifid = apifid_from_next_arg(top_file_id)
|
547
544
|
command_perm = options.get_next_command(%i[list create delete])
|
@@ -574,11 +571,11 @@ module Aspera
|
|
574
571
|
# notify application of creation
|
575
572
|
the_app[:api].permissions_send_event(created_data: created_data, app_info: the_app) unless the_app.nil?
|
576
573
|
return { type: :single_object, data: created_data}
|
577
|
-
else
|
574
|
+
else error_unreachable_line
|
578
575
|
end
|
579
|
-
else
|
576
|
+
else error_unreachable_line
|
580
577
|
end # command_repo
|
581
|
-
|
578
|
+
error_unreachable_line
|
582
579
|
end # execute_command_gen4
|
583
580
|
|
584
581
|
# This is older API
|
@@ -683,7 +680,8 @@ module Aspera
|
|
683
680
|
central
|
684
681
|
asperabrowser
|
685
682
|
basic_token
|
686
|
-
bearer_token
|
683
|
+
bearer_token
|
684
|
+
simulator].concat(COMMON_ACTIONS).freeze
|
687
685
|
|
688
686
|
def execute_action(command=nil, prefix_path=nil)
|
689
687
|
command ||= options.get_next_command(ACTIONS)
|
@@ -906,6 +904,21 @@ module Aspera
|
|
906
904
|
token_info = options.get_next_argument('user and group identification', type: Hash)
|
907
905
|
access_key = options.get_option(:username, mandatory: true)
|
908
906
|
return Main.result_status(Aspera::Node.bearer_token(payload: token_info, access_key: access_key, private_key: private_key))
|
907
|
+
when :simulator
|
908
|
+
require 'aspera/node_simulator'
|
909
|
+
parameters = value_create_modify(command: command)
|
910
|
+
parameters = parameters.symbolize_keys
|
911
|
+
raise 'Missing key: url' unless parameters.key?(:url)
|
912
|
+
uri = URI.parse(parameters[:url])
|
913
|
+
server = WebServerSimple.new(uri, certificate: parameters[:certificate])
|
914
|
+
server.mount(uri.path, NodeSimulatorServlet, parameters[:credentials], transfer)
|
915
|
+
# on ctrl-c, tell server main loop to exit
|
916
|
+
trap('INT') { server.shutdown }
|
917
|
+
formatter.display_status("Node Simulator listening on #{uri.port}")
|
918
|
+
Log.log.info{"Listening on #{uri.port}"}
|
919
|
+
# this is blocking until server exits
|
920
|
+
server.start
|
921
|
+
return Main.result_status('Simulator terminated')
|
909
922
|
end # case command
|
910
923
|
raise 'ERROR: shall not reach this line'
|
911
924
|
end # execute_action
|
@@ -2,6 +2,8 @@
|
|
2
2
|
|
3
3
|
require 'aspera/cli/basic_auth_plugin'
|
4
4
|
require 'aspera/nagios'
|
5
|
+
require 'aspera/log'
|
6
|
+
require 'aspera/assert'
|
5
7
|
require 'xmlsimple'
|
6
8
|
|
7
9
|
module Aspera
|
@@ -95,7 +97,7 @@ module Aspera
|
|
95
97
|
call_args[:url_params][:format] = format
|
96
98
|
when :ext
|
97
99
|
call_args[:subpath] = "#{call_args[:subpath]}.#{format}"
|
98
|
-
else
|
100
|
+
else error_unexpected_value(call_type)
|
99
101
|
end
|
100
102
|
end
|
101
103
|
result = @api_orch.call(call_args)
|
@@ -205,7 +207,7 @@ module Aspera
|
|
205
207
|
result[:data] = call_ao('initiate', id: wf_id, args: call_params, accept: override_accept)[:data]
|
206
208
|
return result
|
207
209
|
end # wf command
|
208
|
-
else
|
210
|
+
else error_unexpected_value(command)
|
209
211
|
end # case command
|
210
212
|
end # execute_action
|
211
213
|
end # Orchestrator
|
@@ -13,6 +13,8 @@ require 'aspera/node'
|
|
13
13
|
require 'aspera/hash_ext'
|
14
14
|
require 'aspera/timer_limiter'
|
15
15
|
require 'aspera/id_generator'
|
16
|
+
require 'aspera/log'
|
17
|
+
require 'aspera/assert'
|
16
18
|
require 'securerandom'
|
17
19
|
|
18
20
|
module Aspera
|
@@ -94,7 +96,7 @@ module Aspera
|
|
94
96
|
end
|
95
97
|
|
96
98
|
options.parse_options!
|
97
|
-
|
99
|
+
assert_type(@option_skip_folders, Array){'skip_folder'}
|
98
100
|
@tmp_folder = File.join(options.get_option(:temp_folder, mandatory: true), "#{TMP_DIR_PREFIX}.#{SecureRandom.uuid}")
|
99
101
|
FileUtils.mkdir_p(@tmp_folder)
|
100
102
|
Log.log.debug{"tmpdir: #{@tmp_folder}"}
|
@@ -104,7 +106,7 @@ module Aspera
|
|
104
106
|
@skip_types = []
|
105
107
|
value.split(',').each do |v|
|
106
108
|
s = v.to_sym
|
107
|
-
|
109
|
+
assert_values(s, Aspera::Preview::FileTypes::CONVERSION_TYPES){'skip_types'}
|
108
110
|
@skip_types.push(s)
|
109
111
|
end
|
110
112
|
end
|
@@ -211,7 +213,7 @@ module Aspera
|
|
211
213
|
end
|
212
214
|
|
213
215
|
def do_transfer(direction, folder_id, source_filename, destination='/')
|
214
|
-
|
216
|
+
assert(!(destination.nil? && direction.eql?(Fasp::TransferSpec::DIRECTION_RECEIVE)))
|
215
217
|
t_spec = @api_node.transfer_spec_gen4(folder_id, direction, {
|
216
218
|
'paths' => [{'source' => source_filename}],
|
217
219
|
'tags' => {Fasp::TransferSpec::TAG_RESERVED => {PREV_GEN_TAG => true}}
|
@@ -420,13 +422,13 @@ module Aspera
|
|
420
422
|
raise Cli::Error, "Folder #{@option_previews_folder} does not exist on node. " \
|
421
423
|
'Please create it in the storage root, or specify an alternate name.' if @previews_folder_entry.nil?
|
422
424
|
else
|
423
|
-
|
425
|
+
assert(@access_key_self['storage']['type'].eql?('local')){'only local storage allowed in this mode'}
|
424
426
|
@local_storage_root = @access_key_self['storage']['path']
|
425
427
|
# TODO: option to override @local_storage_root='xxx'
|
426
428
|
@local_storage_root = @local_storage_root[PVCL_LOCAL_STORAGE.length..-1] if @local_storage_root.start_with?(PVCL_LOCAL_STORAGE)
|
427
429
|
# TODO: windows could have "C:" ?
|
428
|
-
|
429
|
-
|
430
|
+
assert(@local_storage_root.start_with?('/')){"not local storage: #{@local_storage_root}"}
|
431
|
+
assert(File.directory?(@local_storage_root), exception_class: Cli::Error){"Local storage root folder #{@local_storage_root} does not exist."}
|
430
432
|
@local_preview_folder = File.join(@local_storage_root, @option_previews_folder)
|
431
433
|
raise Cli::Error, "Folder #{@local_preview_folder} does not exist locally. " \
|
432
434
|
'Please create it, or specify an alternate name.' unless File.directory?(@local_preview_folder)
|
@@ -435,7 +437,7 @@ module Aspera
|
|
435
437
|
Log.log.debug{"marker file: #{marker_file}"}
|
436
438
|
if File.exist?(marker_file)
|
437
439
|
ak = File.read(marker_file).chomp
|
438
|
-
|
440
|
+
assert(@access_key_self['id'].eql?(ak)){"mismatch access key in #{marker_file}: contains #{ak}, using #{@access_key_self['id']}"}
|
439
441
|
else
|
440
442
|
File.write(marker_file, @access_key_self['id'])
|
441
443
|
end
|
@@ -7,6 +7,8 @@ require 'aspera/fasp/transfer_spec'
|
|
7
7
|
require 'aspera/ascmd'
|
8
8
|
require 'aspera/ssh'
|
9
9
|
require 'aspera/nagios'
|
10
|
+
require 'aspera/log'
|
11
|
+
require 'aspera/assert'
|
10
12
|
require 'tempfile'
|
11
13
|
require 'open3'
|
12
14
|
|
@@ -148,8 +150,9 @@ module Aspera
|
|
148
150
|
end
|
149
151
|
ssh_key_list = options.get_option(:ssh_keys)
|
150
152
|
if !ssh_key_list.nil?
|
151
|
-
raise 'Expecting single value or array for ssh_keys' unless ssh_key_list.is_a?(Array) || ssh_key_list.is_a?(String)
|
152
153
|
ssh_key_list = [ssh_key_list] if ssh_key_list.is_a?(String)
|
154
|
+
assert_type(ssh_key_list, Array){'ssh_keys'}
|
155
|
+
assert(ssh_key_list.all?(String))
|
153
156
|
ssh_key_list.map!{|p|File.expand_path(p)}
|
154
157
|
Log.log.debug{"SSH keys=#{ssh_key_list}"}
|
155
158
|
if !ssh_key_list.empty?
|
@@ -225,7 +228,7 @@ module Aspera
|
|
225
228
|
else
|
226
229
|
nagios.add_critical('transfer', statuses.reject{|i|i.eql?(:success)}.first.to_s)
|
227
230
|
end
|
228
|
-
else
|
231
|
+
else error_unexpected_value(command_nagios)
|
229
232
|
end
|
230
233
|
return nagios.result
|
231
234
|
when *TRANSFER_COMMANDS
|
@@ -248,7 +251,7 @@ module Aspera
|
|
248
251
|
rescue Aspera::AsCmd::Error => e
|
249
252
|
raise Cli::BadArgument, e.extended_message
|
250
253
|
end
|
251
|
-
else
|
254
|
+
else error_unreachable_line
|
252
255
|
end
|
253
256
|
end # execute_action
|
254
257
|
end # Server
|