aspera-cli 4.13.0 → 4.14.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/CHANGELOG.md +28 -5
- data/CONTRIBUTING.md +17 -1
- data/README.md +782 -401
- data/examples/dascli +1 -1
- data/examples/rubyc +24 -0
- data/lib/aspera/aoc.rb +21 -32
- data/lib/aspera/ascmd.rb +1 -0
- data/lib/aspera/cli/basic_auth_plugin.rb +6 -6
- data/lib/aspera/cli/formatter.rb +17 -25
- data/lib/aspera/cli/main.rb +21 -27
- data/lib/aspera/cli/manager.rb +128 -114
- data/lib/aspera/cli/plugin.rb +87 -38
- data/lib/aspera/cli/plugins/alee.rb +2 -2
- data/lib/aspera/cli/plugins/aoc.rb +216 -102
- data/lib/aspera/cli/plugins/ats.rb +16 -18
- data/lib/aspera/cli/plugins/bss.rb +3 -3
- data/lib/aspera/cli/plugins/config.rb +177 -367
- data/lib/aspera/cli/plugins/console.rb +4 -6
- data/lib/aspera/cli/plugins/cos.rb +12 -13
- data/lib/aspera/cli/plugins/faspex.rb +17 -18
- data/lib/aspera/cli/plugins/faspex5.rb +332 -216
- data/lib/aspera/cli/plugins/node.rb +171 -142
- data/lib/aspera/cli/plugins/orchestrator.rb +15 -18
- data/lib/aspera/cli/plugins/preview.rb +38 -60
- data/lib/aspera/cli/plugins/server.rb +22 -15
- data/lib/aspera/cli/plugins/shares.rb +24 -33
- data/lib/aspera/cli/plugins/sync.rb +3 -3
- data/lib/aspera/cli/transfer_agent.rb +29 -26
- data/lib/aspera/cli/version.rb +1 -1
- data/lib/aspera/colors.rb +9 -7
- data/lib/aspera/data/6 +0 -0
- data/lib/aspera/environment.rb +7 -3
- data/lib/aspera/fasp/agent_connect.rb +5 -0
- data/lib/aspera/fasp/agent_direct.rb +5 -5
- data/lib/aspera/fasp/agent_httpgw.rb +138 -60
- data/lib/aspera/fasp/agent_trsdk.rb +2 -0
- data/lib/aspera/fasp/error_info.rb +2 -0
- data/lib/aspera/fasp/installation.rb +18 -19
- data/lib/aspera/fasp/parameters.rb +18 -17
- data/lib/aspera/fasp/parameters.yaml +2 -1
- data/lib/aspera/fasp/resume_policy.rb +3 -3
- data/lib/aspera/fasp/transfer_spec.rb +6 -5
- data/lib/aspera/fasp/uri.rb +23 -21
- data/lib/aspera/faspex_postproc.rb +1 -1
- data/lib/aspera/hash_ext.rb +12 -2
- data/lib/aspera/keychain/macos_security.rb +13 -13
- data/lib/aspera/log.rb +1 -0
- data/lib/aspera/node.rb +62 -80
- data/lib/aspera/oauth.rb +1 -1
- data/lib/aspera/persistency_action_once.rb +1 -1
- data/lib/aspera/preview/terminal.rb +61 -15
- data/lib/aspera/preview/utils.rb +3 -3
- data/lib/aspera/proxy_auto_config.js +2 -2
- data/lib/aspera/rest.rb +37 -0
- data/lib/aspera/secret_hider.rb +6 -1
- data/lib/aspera/ssh.rb +1 -1
- data/lib/aspera/sync.rb +2 -0
- data.tar.gz.sig +0 -0
- metadata +3 -4
- metadata.gz.sig +0 -0
- data/docs/test_env.conf +0 -186
- data/lib/aspera/data/7 +0 -0
@@ -18,11 +18,13 @@ module Aspera
|
|
18
18
|
RECIPIENT_TYPES = %w[user workgroup external_user distribution_list shared_inbox].freeze
|
19
19
|
PACKAGE_TERMINATED = %w[completed failed].freeze
|
20
20
|
API_DETECT = 'api/v5/configuration/ping'
|
21
|
-
# list of supported mailbox types
|
22
|
-
|
23
|
-
PACKAGE_TYPE_RECEIVED = 'received'
|
21
|
+
# list of supported mailbox types (to list packages)
|
22
|
+
API_LIST_MAILBOX_TYPES = %w[inbox inbox_history inbox_all inbox_all_history outbox outbox_history pending pending_history all].freeze
|
24
23
|
PACKAGE_ALL_INIT = 'INIT'
|
25
|
-
|
24
|
+
PACKAGE_SEND_FROM_REMOTE_SOURCE = 'remote_source'
|
25
|
+
ADMIN_RESOURCES = %i[accounts contacts jobs workgroups shared_inboxes nodes oauth_clients registrations saml_configs metadata_profiles
|
26
|
+
email_notifications].freeze
|
27
|
+
private_constant(*%i[RECIPIENT_TYPES PACKAGE_TERMINATED API_DETECT API_LIST_MAILBOX_TYPES PACKAGE_SEND_FROM_REMOTE_SOURCE])
|
26
28
|
class << self
|
27
29
|
def detect(base_url)
|
28
30
|
api = Rest.new(base_url: base_url, redirect_max: 1)
|
@@ -31,27 +33,29 @@ module Aspera
|
|
31
33
|
suffix_length = -2 - API_DETECT.length
|
32
34
|
return {
|
33
35
|
version: result[:http]['x-ibm-aspera'] || '5',
|
34
|
-
url: result[:http].uri.to_s[0..suffix_length]
|
36
|
+
url: result[:http].uri.to_s[0..suffix_length],
|
37
|
+
name: 'Faspex 5'
|
38
|
+
}
|
35
39
|
end
|
36
40
|
return nil
|
37
41
|
end
|
38
42
|
end
|
39
43
|
|
44
|
+
# Faspex API v5: get transfer spec for connect
|
40
45
|
TRANSFER_CONNECT = 'connect'
|
41
46
|
|
42
47
|
def initialize(env)
|
43
48
|
super(env)
|
44
|
-
options.
|
45
|
-
options.
|
46
|
-
options.
|
47
|
-
options.
|
48
|
-
options.
|
49
|
-
options.
|
50
|
-
options.
|
51
|
-
options.
|
52
|
-
options.
|
53
|
-
options.
|
54
|
-
options.set_option(:box, 'inbox')
|
49
|
+
options.declare(:client_id, 'OAuth client identifier')
|
50
|
+
options.declare(:client_secret, 'OAuth client secret')
|
51
|
+
options.declare(:redirect_uri, 'OAuth redirect URI for web authentication')
|
52
|
+
options.declare(:auth, 'OAuth type of authentication', values: %i[boot link].concat(Oauth::STD_AUTH_TYPES), default: :jwt)
|
53
|
+
options.declare(:private_key, 'OAuth JWT RSA private key PEM value (prefix file path with @file:)')
|
54
|
+
options.declare(:passphrase, 'OAuth JWT RSA private key passphrase')
|
55
|
+
options.declare(:link, 'Public link authorization (specific operations)')
|
56
|
+
options.declare(:box, "Package inbox, either shared inbox name or one of #{API_LIST_MAILBOX_TYPES} or #{VAL_ALL}", default: 'inbox')
|
57
|
+
options.declare(:shared_folder, 'Send package with files from shared folder')
|
58
|
+
options.declare(:group_type, 'Shared inbox or workgroup', values: %i[shared_inboxes workgroups], default: :shared_inboxes)
|
55
59
|
options.parse_options!
|
56
60
|
end
|
57
61
|
|
@@ -61,10 +65,10 @@ module Aspera
|
|
61
65
|
@faspex5_api_base_url = public_link.gsub(%r{/public/.*}, '').gsub(/\?.*/, '')
|
62
66
|
options.set_option(:auth, :link)
|
63
67
|
end
|
64
|
-
@faspex5_api_base_url ||= options.get_option(:url,
|
68
|
+
@faspex5_api_base_url ||= options.get_option(:url, mandatory: true).gsub(%r{/+$}, '')
|
65
69
|
@faspex5_api_auth_url = "#{@faspex5_api_base_url}/auth"
|
66
70
|
faspex5_api_v5_url = "#{@faspex5_api_base_url}/api/v5"
|
67
|
-
case options.get_option(:auth,
|
71
|
+
case options.get_option(:auth, mandatory: true)
|
68
72
|
when :link
|
69
73
|
uri = URI.parse(public_link)
|
70
74
|
args = URI.decode_www_form(uri.query).each_with_object({}){|v, h|h[v.first] = v.last; }
|
@@ -81,7 +85,7 @@ module Aspera
|
|
81
85
|
# the password here is the token copied directly from browser in developer mode
|
82
86
|
@api_v5 = Rest.new({
|
83
87
|
base_url: faspex5_api_v5_url,
|
84
|
-
headers: {'Authorization' => options.get_option(:password,
|
88
|
+
headers: {'Authorization' => options.get_option(:password, mandatory: true)}
|
85
89
|
})
|
86
90
|
when :web
|
87
91
|
# opens a browser and ask user to auth using web
|
@@ -91,11 +95,11 @@ module Aspera
|
|
91
95
|
type: :oauth2,
|
92
96
|
base_url: @faspex5_api_auth_url,
|
93
97
|
grant_method: :web,
|
94
|
-
client_id: options.get_option(:client_id,
|
95
|
-
web: {redirect_uri: options.get_option(:redirect_uri,
|
98
|
+
client_id: options.get_option(:client_id, mandatory: true),
|
99
|
+
web: {redirect_uri: options.get_option(:redirect_uri, mandatory: true)}
|
96
100
|
}})
|
97
101
|
when :jwt
|
98
|
-
app_client_id = options.get_option(:client_id,
|
102
|
+
app_client_id = options.get_option(:client_id, mandatory: true)
|
99
103
|
@api_v5 = Rest.new({
|
100
104
|
base_url: faspex5_api_v5_url,
|
101
105
|
auth: {
|
@@ -107,9 +111,9 @@ module Aspera
|
|
107
111
|
payload: {
|
108
112
|
iss: app_client_id, # issuer
|
109
113
|
aud: app_client_id, # audience (this field is not clear...)
|
110
|
-
sub: "user:#{options.get_option(:username,
|
114
|
+
sub: "user:#{options.get_option(:username, mandatory: true)}" # subject is a user
|
111
115
|
},
|
112
|
-
private_key_obj: OpenSSL::PKey::RSA.new(options.get_option(:private_key,
|
116
|
+
private_key_obj: OpenSSL::PKey::RSA.new(options.get_option(:private_key, mandatory: true), options.get_option(:passphrase)),
|
113
117
|
headers: {typ: 'JWT'}
|
114
118
|
}
|
115
119
|
}})
|
@@ -121,13 +125,16 @@ module Aspera
|
|
121
125
|
def normalize_recipients(parameters)
|
122
126
|
return unless parameters.key?('recipients')
|
123
127
|
raise 'Field recipients must be an Array' unless parameters['recipients'].is_a?(Array)
|
124
|
-
|
128
|
+
recipient_types = RECIPIENT_TYPES
|
129
|
+
if parameters.key?('recipient_types')
|
130
|
+
recipient_types = parameters['recipient_types']
|
131
|
+
parameters.delete('recipient_types')
|
132
|
+
recipient_types = [recipient_types] unless recipient_types.is_a?(Array)
|
133
|
+
end
|
134
|
+
parameters['recipients'].map! do |recipient_data|
|
125
135
|
# if just a string, assume it is the name
|
126
136
|
if recipient_data.is_a?(String)
|
127
|
-
|
128
|
-
raise "No matching contact for #{recipient_data}" if 0.eql?(result['total_count'])
|
129
|
-
raise "Multiple matching contact for #{recipient_data} : #{result['contacts'].map{|i|i['name']}.join(', ')}" unless 1.eql?(result['total_count'])
|
130
|
-
matched = result['contacts'].first
|
137
|
+
matched = @api_v5.lookup_by_name('contacts', recipient_data, {context: 'packages', type: Rest.array_params(recipient_types)})
|
131
138
|
recipient_data = {
|
132
139
|
name: matched['name'],
|
133
140
|
recipient_type: matched['type']
|
@@ -139,14 +146,13 @@ module Aspera
|
|
139
146
|
end
|
140
147
|
|
141
148
|
# wait for package status to be in provided list
|
142
|
-
def wait_package_status(id, status_list
|
143
|
-
parameters = options.get_option(:value)
|
149
|
+
def wait_package_status(id, status_list: PACKAGE_TERMINATED)
|
144
150
|
spinner = nil
|
145
151
|
progress = nil
|
146
152
|
while true
|
147
153
|
status = @api_v5.read("packages/#{id}/upload_details")[:data]
|
148
154
|
# user asked to not follow
|
149
|
-
break
|
155
|
+
break if status_list.nil? || status_list.include?(status['upload_status'])
|
150
156
|
if status['upload_status'].eql?('submitted')
|
151
157
|
if spinner.nil?
|
152
158
|
spinner = TTY::Spinner.new('[:spinner] :title', format: :classic)
|
@@ -163,20 +169,24 @@ module Aspera
|
|
163
169
|
else
|
164
170
|
progress.progress = status['bytes_written'].to_i
|
165
171
|
end
|
166
|
-
break if status_list.include?(status['upload_status'])
|
167
172
|
sleep(0.5)
|
168
173
|
end
|
169
174
|
status['id'] = id
|
170
175
|
return status
|
171
176
|
end
|
172
177
|
|
173
|
-
# get a list of all entities of a given type
|
174
|
-
# @param
|
175
|
-
# @param query [Hash] additional query parameters
|
176
|
-
# @param
|
177
|
-
|
178
|
-
|
179
|
-
|
178
|
+
# get a (full or partial) list of all entities of a given type
|
179
|
+
# @param type [String] the type of entity to list (just a name)
|
180
|
+
# @param query [Hash,nil] additional query parameters
|
181
|
+
# @param path [String] optional prefix to add to the path (nil or empty string: no prefix)
|
182
|
+
# @param item_list_key [String] key in the result to get the list of items
|
183
|
+
def list_entities(type:, path: nil, query: nil, item_list_key: nil)
|
184
|
+
query = {} if query.nil?
|
185
|
+
type = type.to_s if type.is_a?(Symbol)
|
186
|
+
item_list_key = type if item_list_key.nil?
|
187
|
+
raise "internal error: Invalid type #{type.class}" unless type.is_a?(String)
|
188
|
+
full_path = type
|
189
|
+
full_path = "#{path}/#{full_path}" unless path.nil? || path.empty?
|
180
190
|
result = []
|
181
191
|
offset = 0
|
182
192
|
max_items = query.delete(MAX_ITEMS)
|
@@ -185,45 +195,204 @@ module Aspera
|
|
185
195
|
query = {'limit'=> 100}.merge(query)
|
186
196
|
loop do
|
187
197
|
query['offset'] = offset
|
188
|
-
|
189
|
-
result.concat(
|
198
|
+
page_result = @api_v5.read(full_path, query)[:data]
|
199
|
+
result.concat(page_result[item_list_key])
|
190
200
|
# reach the limit set by user ?
|
191
201
|
if !max_items.nil? && (result.length >= max_items)
|
192
202
|
result = result.slice(0, max_items)
|
193
203
|
break
|
194
204
|
end
|
195
|
-
break if result.length >=
|
205
|
+
break if result.length >= page_result['total_count']
|
196
206
|
remain_pages -= 1 unless remain_pages.nil?
|
197
207
|
break if remain_pages == 0
|
198
|
-
offset +=
|
208
|
+
offset += page_result[item_list_key].length
|
199
209
|
end
|
200
210
|
return result
|
201
211
|
end
|
202
212
|
|
203
213
|
# lookup an entity id from its name
|
204
|
-
def
|
205
|
-
|
214
|
+
def lookup_entity_by_field(type:, value:, field: 'name', query: :default, path: nil, item_list_key: nil)
|
215
|
+
query = {'q'=> value} if query.eql?(:default)
|
216
|
+
found = list_entities(type: type, path: path, query: query, item_list_key: item_list_key).select{|i|i[field].eql?(value)}
|
206
217
|
case found.length
|
207
|
-
when 0 then raise "No #{
|
208
|
-
when 1 then return found.first
|
209
|
-
else raise "
|
218
|
+
when 0 then raise "No #{type} with #{field} = #{value}"
|
219
|
+
when 1 then return found.first
|
220
|
+
else raise "Found #{found.length} #{path} with #{field} = #{value}"
|
210
221
|
end
|
211
222
|
end
|
212
223
|
|
213
|
-
#
|
214
|
-
def
|
215
|
-
|
216
|
-
|
217
|
-
|
218
|
-
|
219
|
-
|
224
|
+
# list all packages with optional filter
|
225
|
+
def list_packages_with_filter
|
226
|
+
filter = options.get_next_argument('filter', mandatory: false, type: Proc, default: ->(_x){true})
|
227
|
+
# translate box name to API prefix (with ending slash)
|
228
|
+
box = options.get_option(:box)
|
229
|
+
api_path =
|
230
|
+
case box
|
231
|
+
when VAL_ALL then '' # only admin can list all packages globally
|
232
|
+
when *API_LIST_MAILBOX_TYPES then box
|
233
|
+
else
|
234
|
+
group_type = options.get_option(:group_type)
|
235
|
+
"#{group_type}/#{lookup_entity_by_field(type: group_type, value: box)['id']}"
|
236
|
+
end
|
237
|
+
return list_entities(
|
238
|
+
type: 'packages',
|
239
|
+
query: query_read_delete(default: {}),
|
240
|
+
path: api_path).select(&filter)
|
241
|
+
end
|
242
|
+
|
243
|
+
def package_receive
|
244
|
+
# prepare persistency if needed
|
245
|
+
skip_ids_persistency = nil
|
246
|
+
if options.get_option(:once_only, mandatory: true)
|
247
|
+
# read ids from persistency
|
248
|
+
skip_ids_persistency = PersistencyActionOnce.new(
|
249
|
+
manager: @agents[:persistency],
|
250
|
+
data: [],
|
251
|
+
id: IdGenerator.from_list([
|
252
|
+
'faspex_recv',
|
253
|
+
options.get_option(:url, mandatory: true),
|
254
|
+
options.get_option(:username, mandatory: true)]))
|
255
|
+
end
|
256
|
+
package_ids =
|
257
|
+
if @pub_link_context&.key?('package_id')
|
258
|
+
@pub_link_context['package_id']
|
259
|
+
else
|
260
|
+
# one or several packages
|
261
|
+
instance_identifier
|
262
|
+
end
|
263
|
+
case package_ids
|
264
|
+
when PACKAGE_ALL_INIT
|
265
|
+
raise 'Only with option once_only' unless skip_ids_persistency
|
266
|
+
skip_ids_persistency.data.clear.concat(list_packages_with_filter.map{|p|p['id']})
|
267
|
+
skip_ids_persistency.save
|
268
|
+
return Main.result_status("Initialized skip for #{skip_ids_persistency.data.count} package(s)")
|
269
|
+
when VAL_ALL
|
270
|
+
# TODO: if packages have same name, they will overwrite ?
|
271
|
+
package_ids = list_packages_with_filter.map{|p|p['id']}
|
272
|
+
Log.dump(:package_ids, package_ids)
|
273
|
+
Log.dump(:package_ids, skip_ids_persistency.data)
|
274
|
+
package_ids.reject!{|i|skip_ids_persistency.data.include?(i)} if skip_ids_persistency
|
275
|
+
Log.dump(:package_ids, package_ids)
|
276
|
+
end
|
277
|
+
# a single id was provided
|
278
|
+
# TODO: check package_ids is a list of strings
|
279
|
+
package_ids = [package_ids] if package_ids.is_a?(String)
|
280
|
+
result_transfer = []
|
281
|
+
package_ids.each do |pkg_id|
|
282
|
+
formatter.display_status("Receiving package #{pkg_id}")
|
283
|
+
param_file_list = {}
|
284
|
+
begin
|
285
|
+
param_file_list['paths'] = transfer.source_list.map{|source|{'path'=>source}}
|
286
|
+
rescue Aspera::Cli::CliBadArgument
|
287
|
+
# paths is optional
|
288
|
+
end
|
289
|
+
download_params = {
|
290
|
+
type: 'received',
|
291
|
+
transfer_type: TRANSFER_CONNECT
|
292
|
+
}
|
293
|
+
box = options.get_option(:box)
|
294
|
+
case box
|
295
|
+
when /outbox/ then download_params[:type] = 'sent'
|
296
|
+
when *API_LIST_MAILBOX_TYPES then nil # nothing to do
|
297
|
+
else # shared inbox / workgroup
|
298
|
+
download_params[:recipient_workgroup_id] = lookup_entity_by_field(type: options.get_option(:group_type), value: box)['id']
|
299
|
+
end
|
300
|
+
# TODO: allow from sent as well ?
|
301
|
+
transfer_spec = @api_v5.call(
|
302
|
+
operation: 'POST',
|
303
|
+
subpath: "packages/#{pkg_id}/transfer_spec/download",
|
304
|
+
headers: {'Accept' => 'application/json'},
|
305
|
+
url_params: download_params,
|
306
|
+
json_params: param_file_list
|
307
|
+
)[:data]
|
308
|
+
# delete flag for Connect Client
|
309
|
+
transfer_spec.delete('authentication')
|
310
|
+
statuses = transfer.start(transfer_spec)
|
311
|
+
result_transfer.push({'package' => pkg_id, Main::STATUS_FIELD => statuses})
|
312
|
+
# skip only if all sessions completed
|
313
|
+
if TransferAgent.session_status(statuses).eql?(:success) && skip_ids_persistency
|
314
|
+
skip_ids_persistency.data.push(pkg_id)
|
315
|
+
skip_ids_persistency.save
|
316
|
+
end
|
220
317
|
end
|
318
|
+
return Main.result_transfer_multiple(result_transfer)
|
221
319
|
end
|
222
320
|
|
223
|
-
|
224
|
-
|
225
|
-
|
226
|
-
|
321
|
+
def package_action
|
322
|
+
command = options.get_next_command(%i[list show browse status delete send receive])
|
323
|
+
case command
|
324
|
+
when :list
|
325
|
+
return {
|
326
|
+
type: :object_list,
|
327
|
+
data: list_packages_with_filter,
|
328
|
+
fields: %w[id title release_date total_bytes total_files created_time state]
|
329
|
+
}
|
330
|
+
when :show
|
331
|
+
id = @pub_link_context['package_id'] if @pub_link_context&.key?('package_id')
|
332
|
+
id ||= instance_identifier
|
333
|
+
return {type: :single_object, data: @api_v5.read("packages/#{id}")[:data]}
|
334
|
+
when :browse
|
335
|
+
id = @pub_link_context['package_id'] if @pub_link_context&.key?('package_id')
|
336
|
+
id ||= instance_identifier
|
337
|
+
path = options.get_next_argument('path', expected: :single, mandatory: false) || '/'
|
338
|
+
# TODO: support multi-page listing ?
|
339
|
+
params = {
|
340
|
+
# recipient_user_id: 25,
|
341
|
+
# offset: 0,
|
342
|
+
# limit: 25
|
343
|
+
}
|
344
|
+
result = @api_v5.call({
|
345
|
+
operation: 'POST',
|
346
|
+
subpath: "packages/#{id}/files/received",
|
347
|
+
headers: {'Accept' => 'application/json'},
|
348
|
+
url_params: params,
|
349
|
+
json_params: {'path' => path, 'filters' => {'basenames'=>[]}}})[:data]
|
350
|
+
formatter.display_item_count(result['item_count'], result['total_count'])
|
351
|
+
return {type: :object_list, data: result['items']}
|
352
|
+
when :status
|
353
|
+
status = wait_package_status(instance_identifier, status_list: nil)
|
354
|
+
return {type: :single_object, data: status}
|
355
|
+
when :delete
|
356
|
+
ids = instance_identifier
|
357
|
+
ids = [ids] unless ids.is_a?(Array)
|
358
|
+
raise 'Package identifier must be a single id or an Array' unless ids.is_a?(Array) && ids.all?(String)
|
359
|
+
# API returns 204, empty on success
|
360
|
+
@api_v5.call({operation: 'DELETE', subpath: 'packages', headers: {'Accept' => 'application/json'}, json_params: {ids: ids}})
|
361
|
+
return Main.result_status('Package(s) deleted')
|
362
|
+
when :send
|
363
|
+
parameters = value_create_modify(command: command, type: Hash)
|
364
|
+
normalize_recipients(parameters)
|
365
|
+
package = @api_v5.create('packages', parameters)[:data]
|
366
|
+
shared_folder = options.get_option(:shared_folder)
|
367
|
+
if shared_folder.nil?
|
368
|
+
# TODO: option to send from remote source or httpgw
|
369
|
+
transfer_spec = @api_v5.call(
|
370
|
+
operation: 'POST',
|
371
|
+
subpath: "packages/#{package['id']}/transfer_spec/upload",
|
372
|
+
headers: {'Accept' => 'application/json'},
|
373
|
+
url_params: {transfer_type: TRANSFER_CONNECT},
|
374
|
+
json_params: {paths: transfer.source_list}
|
375
|
+
)[:data]
|
376
|
+
# well, we asked a TS for connect, but we actually want a generic one
|
377
|
+
transfer_spec.delete('authentication')
|
378
|
+
return Main.result_transfer(transfer.start(transfer_spec))
|
379
|
+
else
|
380
|
+
if (m = shared_folder.match(REGEX_LOOKUP_ID_BY_FIELD))
|
381
|
+
shared_folder = lookup_entity_by_field(type: 'shared_folders', value: m[2])['id']
|
382
|
+
end
|
383
|
+
transfer_request = {shared_folder_id: shared_folder, paths: transfer.source_list}
|
384
|
+
# start remote transfer and get first status
|
385
|
+
result = @api_v5.create("packages/#{package['id']}/remote_transfer", transfer_request)[:data]
|
386
|
+
result['id'] = package['id']
|
387
|
+
unless result['status'].eql?('completed')
|
388
|
+
formatter.display_status("Package #{package['id']}")
|
389
|
+
result = wait_package_status(package['id'])
|
390
|
+
end
|
391
|
+
return {type: :single_object, data: result}
|
392
|
+
end
|
393
|
+
when :receive
|
394
|
+
return package_receive
|
395
|
+
end # case package
|
227
396
|
end
|
228
397
|
|
229
398
|
ACTIONS = %i[health version user bearer_token packages shared_folders admin gateway postprocessing].freeze
|
@@ -259,148 +428,19 @@ module Aspera
|
|
259
428
|
when :bearer_token
|
260
429
|
return {type: :text, data: @api_v5.oauth_token}
|
261
430
|
when :packages
|
262
|
-
|
263
|
-
case command
|
264
|
-
when :list
|
265
|
-
return {
|
266
|
-
type: :object_list,
|
267
|
-
data: list_packages,
|
268
|
-
fields: %w[id title release_date total_bytes total_files created_time state]
|
269
|
-
}
|
270
|
-
when :show
|
271
|
-
id = @pub_link_context['package_id'] if @pub_link_context&.key?('package_id')
|
272
|
-
id ||= instance_identifier
|
273
|
-
return {type: :single_object, data: @api_v5.read("packages/#{id}")[:data]}
|
274
|
-
when :browse
|
275
|
-
id = @pub_link_context['package_id'] if @pub_link_context&.key?('package_id')
|
276
|
-
id ||= instance_identifier
|
277
|
-
path = options.get_next_argument('path', expected: :single, mandatory: false) || '/'
|
278
|
-
# TODO: support multi-page listing ?
|
279
|
-
params = {
|
280
|
-
# recipient_user_id: 25,
|
281
|
-
# offset: 0,
|
282
|
-
# limit: 25
|
283
|
-
}
|
284
|
-
result = @api_v5.call({
|
285
|
-
operation: 'POST',
|
286
|
-
subpath: "packages/#{id}/files/received",
|
287
|
-
headers: {'Accept' => 'application/json'},
|
288
|
-
url_params: params,
|
289
|
-
json_params: {'path' => path, 'filters' => {'basenames'=>[]}}})[:data]
|
290
|
-
formatter.display_item_count(result['item_count'], result['total_count'])
|
291
|
-
return {type: :object_list, data: result['items']}
|
292
|
-
when :status
|
293
|
-
status = wait_package_status(instance_identifier)
|
294
|
-
return {type: :single_object, data: status}
|
295
|
-
when :delete
|
296
|
-
ids = instance_identifier
|
297
|
-
ids = [ids] unless ids.is_a?(Array)
|
298
|
-
raise 'Package identifier must be a single id or an Array' unless ids.is_a?(Array) && ids.all?(String)
|
299
|
-
# API returns 204, empty on success
|
300
|
-
@api_v5.call({operation: 'DELETE', subpath: 'packages', headers: {'Accept' => 'application/json'}, json_params: {ids: ids}})
|
301
|
-
return Main.result_status('Package(s) deleted')
|
302
|
-
when :send
|
303
|
-
parameters = options.get_option(:value, is_type: :mandatory)
|
304
|
-
raise CliBadArgument, 'Value must be Hash, refer to API' unless parameters.is_a?(Hash)
|
305
|
-
normalize_recipients(parameters)
|
306
|
-
package = @api_v5.create('packages', parameters)[:data]
|
307
|
-
shared_folder = options.get_option(:shared_folder)
|
308
|
-
if shared_folder.nil?
|
309
|
-
# TODO: option to send from remote source or httpgw
|
310
|
-
transfer_spec = @api_v5.call(
|
311
|
-
operation: 'POST',
|
312
|
-
subpath: "packages/#{package['id']}/transfer_spec/upload",
|
313
|
-
headers: {'Accept' => 'application/json'},
|
314
|
-
url_params: {transfer_type: TRANSFER_CONNECT},
|
315
|
-
json_params: {paths: transfer.source_list}
|
316
|
-
)[:data]
|
317
|
-
# well, we asked a TS for connect, but we actually want a generic one
|
318
|
-
transfer_spec.delete('authentication')
|
319
|
-
return Main.result_transfer(transfer.start(transfer_spec))
|
320
|
-
else
|
321
|
-
if !shared_folder.to_i.to_s.eql?(shared_folder)
|
322
|
-
shared_folder = lookup_name_to_id('shared_folders', shared_folder)
|
323
|
-
end
|
324
|
-
transfer_request = {shared_folder_id: shared_folder, paths: transfer.source_list}
|
325
|
-
# start remote transfer and get first status
|
326
|
-
result = @api_v5.create("packages/#{package['id']}/remote_transfer", transfer_request)[:data]
|
327
|
-
result['id'] = package['id']
|
328
|
-
unless result['status'].eql?('completed')
|
329
|
-
formatter.display_status("Package #{package['id']}")
|
330
|
-
result = wait_package_status(package['id'])
|
331
|
-
end
|
332
|
-
return {type: :single_object, data: result}
|
333
|
-
end
|
334
|
-
when :receive
|
335
|
-
# prepare persistency if needed
|
336
|
-
skip_ids_persistency = nil
|
337
|
-
if options.get_option(:once_only, is_type: :mandatory)
|
338
|
-
# read ids from persistency
|
339
|
-
skip_ids_persistency = PersistencyActionOnce.new(
|
340
|
-
manager: @agents[:persistency],
|
341
|
-
data: [],
|
342
|
-
id: IdGenerator.from_list([
|
343
|
-
'faspex_recv',
|
344
|
-
options.get_option(:url, is_type: :mandatory),
|
345
|
-
options.get_option(:username, is_type: :mandatory),
|
346
|
-
PACKAGE_TYPE_RECEIVED]))
|
347
|
-
end
|
348
|
-
# one or several packages
|
349
|
-
package_ids = @pub_link_context['package_id'] if @pub_link_context&.key?('package_id')
|
350
|
-
package_ids ||= instance_identifier
|
351
|
-
case package_ids
|
352
|
-
when PACKAGE_ALL_INIT
|
353
|
-
raise 'Only with option once_only' unless skip_ids_persistency
|
354
|
-
skip_ids_persistency.data.clear.concat(list_packages.map{|p|p['id']})
|
355
|
-
skip_ids_persistency.save
|
356
|
-
return Main.result_status("Initialized skip for #{skip_ids_persistency.data.count} package(s)")
|
357
|
-
when VAL_ALL
|
358
|
-
# TODO: if packages have same name, they will overwrite ?
|
359
|
-
package_ids = list_packages.map{|p|p['id']}
|
360
|
-
Log.dump(:package_ids, package_ids)
|
361
|
-
Log.dump(:package_ids, skip_ids_persistency.data)
|
362
|
-
package_ids.reject!{|i|skip_ids_persistency.data.include?(i)} if skip_ids_persistency
|
363
|
-
Log.dump(:package_ids, package_ids)
|
364
|
-
end
|
365
|
-
# a single id was provided
|
366
|
-
# TODO: check package_ids is a list of strings
|
367
|
-
package_ids = [package_ids] if package_ids.is_a?(String)
|
368
|
-
result_transfer = []
|
369
|
-
package_ids.each do |pkg_id|
|
370
|
-
formatter.display_status("Receiving package #{pkg_id}")
|
371
|
-
param_file_list = {}
|
372
|
-
begin
|
373
|
-
param_file_list['paths'] = transfer.source_list.map{|source|{'path'=>source}}
|
374
|
-
rescue Aspera::Cli::CliBadArgument
|
375
|
-
# paths is optional
|
376
|
-
end
|
377
|
-
# TODO: allow from sent as well ?
|
378
|
-
transfer_spec = @api_v5.call(
|
379
|
-
operation: 'POST',
|
380
|
-
subpath: "packages/#{pkg_id}/transfer_spec/download",
|
381
|
-
headers: {'Accept' => 'application/json'},
|
382
|
-
url_params: {transfer_type: TRANSFER_CONNECT, type: PACKAGE_TYPE_RECEIVED},
|
383
|
-
json_params: param_file_list
|
384
|
-
)[:data]
|
385
|
-
# delete flag for Connect Client
|
386
|
-
transfer_spec.delete('authentication')
|
387
|
-
statuses = transfer.start(transfer_spec)
|
388
|
-
result_transfer.push({'package' => pkg_id, Main::STATUS_FIELD => statuses})
|
389
|
-
# skip only if all sessions completed
|
390
|
-
if TransferAgent.session_status(statuses).eql?(:success) && skip_ids_persistency
|
391
|
-
skip_ids_persistency.data.push(pkg_id)
|
392
|
-
skip_ids_persistency.save
|
393
|
-
end
|
394
|
-
end
|
395
|
-
return Main.result_transfer_multiple(result_transfer)
|
396
|
-
end # case package
|
431
|
+
return package_action
|
397
432
|
when :shared_folders
|
398
433
|
all_shared_folders = @api_v5.read('shared_folders')[:data]['shared_folders']
|
399
434
|
case options.get_next_command(%i[list browse])
|
400
435
|
when :list
|
401
436
|
return {type: :object_list, data: all_shared_folders}
|
402
437
|
when :browse
|
403
|
-
shared_folder_id = instance_identifier
|
438
|
+
shared_folder_id = instance_identifier do |field, value|
|
439
|
+
matches = all_shared_folders.select{|i|i[field].eql?(value)}
|
440
|
+
raise "no match for #{field} = #{value}" if matches.empty?
|
441
|
+
raise "multiple matches for #{field} = #{value}" if matches.length > 1
|
442
|
+
matches.first['id']
|
443
|
+
end
|
404
444
|
path = options.get_next_argument('folder path', mandatory: false) || '/'
|
405
445
|
node = all_shared_folders.find{|i|i['id'].eql?(shared_folder_id)}
|
406
446
|
raise "No such shared folder id #{shared_folder_id}" if node.nil?
|
@@ -418,12 +458,14 @@ module Aspera
|
|
418
458
|
end
|
419
459
|
end
|
420
460
|
when :admin
|
421
|
-
case options.get_next_command(%i[resource smtp])
|
461
|
+
case options.get_next_command(%i[resource smtp].freeze)
|
422
462
|
when :resource
|
423
|
-
res_type = options.get_next_command(
|
424
|
-
email_notifications])
|
463
|
+
res_type = options.get_next_command(ADMIN_RESOURCES)
|
425
464
|
res_path = list_key = res_type.to_s
|
426
465
|
id_as_arg = false
|
466
|
+
display_fields = nil
|
467
|
+
adm_api = @api_v5
|
468
|
+
available_commands = [].concat(Plugin::ALL_OPS)
|
427
469
|
case res_type
|
428
470
|
when :metadata_profiles
|
429
471
|
res_path = 'configuration/metadata_profiles'
|
@@ -431,26 +473,69 @@ module Aspera
|
|
431
473
|
when :email_notifications
|
432
474
|
list_key = false
|
433
475
|
id_as_arg = 'type'
|
476
|
+
when :accounts
|
477
|
+
display_fields = [:all_but, 'user_profile_data_attributes']
|
478
|
+
when :oauth_clients
|
479
|
+
display_fields = [:all_but, 'public_key']
|
480
|
+
adm_api = Rest.new(@api_v5.params.merge({base_url: @faspex5_api_auth_url}))
|
481
|
+
when :shared_inboxes, :workgroups
|
482
|
+
available_commands.push(:members, :saml_groups, :invite_external_collaborator)
|
434
483
|
end
|
435
|
-
|
436
|
-
|
437
|
-
|
438
|
-
|
484
|
+
res_command = options.get_next_command(available_commands)
|
485
|
+
case res_command
|
486
|
+
when *Plugin::ALL_OPS
|
487
|
+
return entity_command(res_command, adm_api, res_path, item_list_key: list_key, display_fields: display_fields, id_as_arg: id_as_arg)
|
488
|
+
when :invite_external_collaborator
|
489
|
+
shared_inbox_id = instance_identifier { |field, value| lookup_entity_by_field(type: res_type.to_s, field: field, value: value)['id']}
|
490
|
+
creation_payload = value_create_modify(command: res_command, type: [Hash, String])
|
491
|
+
creation_payload = {'email_address' => creation_payload} if creation_payload.is_a?(String)
|
492
|
+
res_path = "#{res_type}/#{shared_inbox_id}/external_collaborator"
|
493
|
+
result = adm_api.create(res_path, creation_payload)[:data]
|
494
|
+
formatter.display_status(result['message'])
|
495
|
+
result = lookup_entity_by_field(type: 'members', path: "#{res_type}/#{shared_inbox_id}", value: creation_payload['email_address'], query: {})
|
496
|
+
return {type: :single_object, data: result}
|
497
|
+
when :members, :saml_groups
|
498
|
+
res_id = instance_identifier { |field, value| lookup_entity_by_field(type: res_type.to_s, field: field, value: value)['id']}
|
499
|
+
res_prefix = "#{res_type}/#{res_id}"
|
500
|
+
res_path = "#{res_prefix}/#{res_command}"
|
501
|
+
list_key = res_command.to_s
|
502
|
+
list_key = 'groups' if res_command.eql?(:saml_groups)
|
503
|
+
sub_command = options.get_next_command(%i[create list modify delete])
|
504
|
+
if sub_command.eql?(:create) && options.get_option(:value).nil?
|
505
|
+
raise "use option 'value' to provide saml group_id and access (refer to API)" unless res_command.eql?(:members)
|
506
|
+
# first arg is one user name or list of users
|
507
|
+
users = options.get_next_argument('user id, or email, or list of')
|
508
|
+
users = [users] unless users.is_a?(Array)
|
509
|
+
users = users.map do |user|
|
510
|
+
if (m = user.match(REGEX_LOOKUP_ID_BY_FIELD))
|
511
|
+
lookup_entity_by_field(
|
512
|
+
type: 'accounts', field: m[1], value: m[2],
|
513
|
+
query: {type: Rest.array_params(%w{local_user saml_user self_registered_user external_user})})['id']
|
514
|
+
else
|
515
|
+
# it's the user id (not member id...)
|
516
|
+
user
|
517
|
+
end
|
518
|
+
end
|
519
|
+
access = options.get_next_argument('level', mandatory: false, expected: %i[submit_only standard shared_inbox_admin], default: :standard)
|
520
|
+
# TODO: unshift to command line parameters instead of using deprecated option "value"
|
521
|
+
options.set_option(:value, {user: users.map{|u|{id: u, access: access}}})
|
439
522
|
end
|
440
|
-
|
441
|
-
|
442
|
-
|
523
|
+
return entity_command(sub_command, adm_api, res_path, item_list_key: list_key) do |field, value|
|
524
|
+
lookup_entity_by_field(
|
525
|
+
type: 'accounts', field: field, value: value,
|
526
|
+
query: {type: Rest.array_params(%w{local_user saml_user self_registered_user external_user})})['id']
|
527
|
+
end
|
443
528
|
end
|
444
|
-
return entity_action(adm_api, res_path, item_list_key: list_key, display_fields: display_fields, id_as_arg: id_as_arg)
|
445
529
|
when :smtp
|
446
530
|
smtp_path = 'configuration/smtp'
|
447
|
-
|
531
|
+
smtp_cmd = options.get_next_command(%i[show create modify delete test])
|
532
|
+
case smtp_cmd
|
448
533
|
when :show
|
449
534
|
return { type: :single_object, data: @api_v5.read(smtp_path)[:data] }
|
450
535
|
when :create
|
451
|
-
return { type: :single_object, data: @api_v5.create(smtp_path,
|
536
|
+
return { type: :single_object, data: @api_v5.create(smtp_path, value_create_modify(command: smtp_cmd, type: Hash))[:data] }
|
452
537
|
when :modify
|
453
|
-
return { type: :single_object, data: @api_v5.modify(smtp_path,
|
538
|
+
return { type: :single_object, data: @api_v5.modify(smtp_path, value_create_modify(command: smtp_cmd, type: Hash))[:data] }
|
454
539
|
when :delete
|
455
540
|
return { type: :single_object, data: @api_v5.delete(smtp_path)[:data] }
|
456
541
|
when :test
|
@@ -461,20 +546,20 @@ module Aspera
|
|
461
546
|
end
|
462
547
|
when :gateway
|
463
548
|
require 'aspera/faspex_gw'
|
464
|
-
url =
|
549
|
+
url = value_create_modify(type: String)
|
465
550
|
uri = URI.parse(url)
|
466
551
|
server = WebServerSimple.new(uri)
|
467
552
|
server.mount(uri.path, Faspex4GWServlet, @api_v5, nil)
|
553
|
+
# on ctrl-c, tell server main loop to exit
|
468
554
|
trap('INT') { server.shutdown }
|
469
|
-
formatter.display_status("Faspex 4
|
555
|
+
formatter.display_status("Gateway for Faspex 4-style API listening on #{url}")
|
470
556
|
Log.log.info("Listening on #{url}")
|
471
557
|
# this is blocking until server exits
|
472
558
|
server.start
|
473
559
|
return Main.result_status('Gateway terminated')
|
474
560
|
when :postprocessing
|
475
|
-
require 'aspera/faspex_postproc'
|
476
|
-
parameters =
|
477
|
-
raise 'parameters must be Hash' unless parameters.is_a?(Hash)
|
561
|
+
require 'aspera/faspex_postproc' # cspell:disable-line
|
562
|
+
parameters = value_create_modify(type: Hash)
|
478
563
|
parameters = parameters.symbolize_keys
|
479
564
|
raise 'Missing key: url' unless parameters.key?(:url)
|
480
565
|
uri = URI.parse(parameters[:url])
|
@@ -482,14 +567,45 @@ module Aspera
|
|
482
567
|
parameters[:processing][:root] = uri.path
|
483
568
|
server = WebServerSimple.new(uri, certificate: parameters[:certificate])
|
484
569
|
server.mount(uri.path, Faspex4PostProcServlet, parameters[:processing])
|
570
|
+
# on ctrl-c, tell server main loop to exit
|
485
571
|
trap('INT') { server.shutdown }
|
486
|
-
formatter.display_status("Faspex 4 post processing listening on #{uri.port}")
|
572
|
+
formatter.display_status("Web-hook for Faspex 4-style post processing listening on #{uri.port}")
|
487
573
|
Log.log.info("Listening on #{uri.port}")
|
488
574
|
# this is blocking until server exits
|
489
575
|
server.start
|
490
576
|
return Main.result_status('Gateway terminated')
|
491
577
|
end # case command
|
492
578
|
end # action
|
579
|
+
|
580
|
+
def wizard(params)
|
581
|
+
if params[:prepare]
|
582
|
+
# if not defined by user, generate unique name
|
583
|
+
params[:preset_name] ||= [params[:plugin_sym]].concat(URI.parse(params[:instance_url]).host.gsub(/[^a-z0-9.]/, '').split('.')).join('_')
|
584
|
+
params[:need_private_key] = true
|
585
|
+
return
|
586
|
+
end
|
587
|
+
formatter.display_status('Ask the ascli client id and secret to your Administrator, or ask them to go to:'.red)
|
588
|
+
OpenApplication.instance.uri(params[:instance_url])
|
589
|
+
formatter.display_status('Then: 𓃑 → Admin → Configurations → API clients')
|
590
|
+
formatter.display_status('Create an API client with:')
|
591
|
+
formatter.display_status('- name: ascli')
|
592
|
+
formatter.display_status('- JWT: enabled')
|
593
|
+
formatter.display_status('Then, logged in as user go to your profile:')
|
594
|
+
formatter.display_status('👤 → Account Settings → Preferences -> Public Key in PEM:')
|
595
|
+
formatter.display_status(params[:pub_key_pem])
|
596
|
+
formatter.display_status('Once set, fill in the parameters:')
|
597
|
+
return {
|
598
|
+
preset_value: {
|
599
|
+
url: params[:instance_url],
|
600
|
+
username: options.get_option(:username, mandatory: true),
|
601
|
+
auth: :jwt.to_s,
|
602
|
+
private_key: '@file:' + params[:private_key_path],
|
603
|
+
client_id: options.get_option(:client_id, mandatory: true),
|
604
|
+
client_secret: options.get_option(:client_secret, mandatory: true)
|
605
|
+
},
|
606
|
+
test_args: "#{params[:plugin_sym]} user profile show"
|
607
|
+
}
|
608
|
+
end
|
493
609
|
end # Faspex5
|
494
610
|
end # Plugins
|
495
611
|
end # Cli
|