aspera-cli 4.24.2 → 4.25.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/CHANGELOG.md +1070 -758
- data/CONTRIBUTING.md +130 -115
- data/README.md +961 -623
- data/lib/aspera/agent/direct.rb +14 -12
- data/lib/aspera/agent/factory.rb +9 -6
- data/lib/aspera/agent/transferd.rb +8 -8
- data/lib/aspera/api/aoc.rb +104 -67
- data/lib/aspera/api/ats.rb +1 -0
- data/lib/aspera/api/cos_node.rb +3 -2
- data/lib/aspera/api/faspex.rb +17 -10
- data/lib/aspera/api/node.rb +10 -12
- data/lib/aspera/ascmd.rb +2 -3
- data/lib/aspera/ascp/installation.rb +60 -46
- data/lib/aspera/ascp/management.rb +9 -5
- data/lib/aspera/assert.rb +28 -6
- data/lib/aspera/cli/error.rb +4 -2
- data/lib/aspera/cli/extended_value.rb +94 -62
- data/lib/aspera/cli/formatter.rb +44 -58
- data/lib/aspera/cli/main.rb +21 -14
- data/lib/aspera/cli/manager.rb +317 -250
- data/lib/aspera/cli/plugins/alee.rb +3 -3
- data/lib/aspera/cli/plugins/aoc.rb +139 -78
- data/lib/aspera/cli/plugins/ats.rb +30 -36
- data/lib/aspera/cli/plugins/base.rb +68 -55
- data/lib/aspera/cli/plugins/config.rb +90 -100
- data/lib/aspera/cli/plugins/console.rb +15 -9
- data/lib/aspera/cli/plugins/cos.rb +1 -1
- data/lib/aspera/cli/plugins/faspex.rb +39 -30
- data/lib/aspera/cli/plugins/faspex5.rb +57 -52
- data/lib/aspera/cli/plugins/faspio.rb +10 -7
- data/lib/aspera/cli/plugins/httpgw.rb +3 -2
- data/lib/aspera/cli/plugins/node.rb +140 -125
- data/lib/aspera/cli/plugins/oauth.rb +13 -12
- data/lib/aspera/cli/plugins/orchestrator.rb +116 -33
- data/lib/aspera/cli/plugins/preview.rb +28 -48
- data/lib/aspera/cli/plugins/server.rb +9 -10
- data/lib/aspera/cli/plugins/shares.rb +77 -43
- data/lib/aspera/cli/sync_actions.rb +49 -38
- data/lib/aspera/cli/transfer_agent.rb +16 -35
- data/lib/aspera/cli/version.rb +1 -1
- data/lib/aspera/cli/wizard.rb +8 -5
- data/lib/aspera/command_line_builder.rb +24 -21
- data/lib/aspera/coverage.rb +6 -2
- data/lib/aspera/dot_container.rb +108 -0
- data/lib/aspera/environment.rb +71 -84
- data/lib/aspera/faspex_gw.rb +1 -1
- data/lib/aspera/faspex_postproc.rb +1 -1
- data/lib/aspera/id_generator.rb +7 -10
- data/lib/aspera/keychain/factory.rb +1 -2
- data/lib/aspera/keychain/macos_security.rb +2 -2
- data/lib/aspera/log.rb +2 -1
- data/lib/aspera/markdown.rb +31 -0
- data/lib/aspera/nagios.rb +6 -5
- data/lib/aspera/oauth/base.rb +41 -64
- data/lib/aspera/oauth/factory.rb +6 -7
- data/lib/aspera/oauth/generic.rb +1 -1
- data/lib/aspera/oauth/jwt.rb +1 -1
- data/lib/aspera/oauth/url_json.rb +6 -4
- data/lib/aspera/oauth/web.rb +2 -2
- data/lib/aspera/preview/file_types.rb +24 -38
- data/lib/aspera/preview/terminal.rb +95 -29
- data/lib/aspera/preview/utils.rb +6 -5
- data/lib/aspera/products/connect.rb +3 -3
- data/lib/aspera/rest.rb +54 -39
- data/lib/aspera/rest_error_analyzer.rb +4 -4
- data/lib/aspera/ssh.rb +10 -6
- data/lib/aspera/ssl.rb +41 -0
- data/lib/aspera/sync/conf.schema.yaml +184 -36
- data/lib/aspera/sync/database.rb +2 -1
- data/lib/aspera/sync/operations.rb +128 -72
- data/lib/aspera/transfer/parameters.rb +9 -10
- data/lib/aspera/transfer/spec.rb +2 -3
- data/lib/aspera/transfer/spec.schema.yaml +52 -22
- data/lib/aspera/transfer/spec_doc.rb +20 -30
- data/lib/aspera/uri_reader.rb +18 -4
- data/lib/transferd_pb.rb +2 -2
- data.tar.gz.sig +0 -0
- metadata +34 -6
- metadata.gz.sig +0 -0
data/lib/aspera/agent/direct.rb
CHANGED
|
@@ -186,20 +186,21 @@ module Aspera
|
|
|
186
186
|
Log.log.debug('fasp local shutdown')
|
|
187
187
|
end
|
|
188
188
|
|
|
189
|
+
# @param id [String] Transfer session identifier
|
|
189
190
|
# @return [Array] list of sessions for a job
|
|
190
|
-
def sessions_by_job(
|
|
191
|
-
@sessions.select{ |session| session[:job_id].eql?(
|
|
191
|
+
def sessions_by_job(id)
|
|
192
|
+
@sessions.select{ |session| session[:job_id].eql?(id)}
|
|
192
193
|
end
|
|
193
194
|
|
|
194
195
|
# Send command to management port of command (used in `asession).
|
|
195
196
|
# Examples:
|
|
196
197
|
# {'type'=>'START','source'=>_path_,'destination'=>_path_}
|
|
197
198
|
# {'type'=>'DONE'}
|
|
198
|
-
# @param data [Hash] Command on mgt port
|
|
199
|
-
# @param id [String] Optional identifier
|
|
199
|
+
# @param data [Hash] Command on mgt port (snake case)
|
|
200
|
+
# @param id [String] Optional identifier of transfer session
|
|
200
201
|
def send_command(data, id: nil)
|
|
201
202
|
Log.dump(:command, data)
|
|
202
|
-
sessions = id ?
|
|
203
|
+
sessions = id ? sessions_by_job(id) : @sessions
|
|
203
204
|
if sessions.empty?
|
|
204
205
|
Log.log.warn('No transfer session')
|
|
205
206
|
return
|
|
@@ -251,11 +252,14 @@ module Aspera
|
|
|
251
252
|
Aspera.assert_type(session, Hash)
|
|
252
253
|
notify_progress(:sessions_init, info: 'starting')
|
|
253
254
|
begin
|
|
255
|
+
# do not use
|
|
254
256
|
capture_stderr = false
|
|
255
257
|
stderr_r, stderr_w = nil
|
|
256
258
|
spawn_args = {}
|
|
257
259
|
command_pid = nil
|
|
258
|
-
|
|
260
|
+
# get location of command executable (ascp, async)
|
|
261
|
+
command_path = Ascp::Installation.instance.path(name)
|
|
262
|
+
command_arguments = [command_path]
|
|
259
263
|
if @monitor
|
|
260
264
|
# we use Socket directly, instead of TCPServer, as it gives access to lower level options
|
|
261
265
|
socket_class = defined?(JRUBY_VERSION) ? ServerSocket : Socket
|
|
@@ -266,20 +270,18 @@ module Aspera
|
|
|
266
270
|
mgt_server_socket.listen(1)
|
|
267
271
|
# build arguments and add mgt port
|
|
268
272
|
command_arguments = if name.eql?(:async)
|
|
269
|
-
["--exclusive-mgmt-port=#{mgt_server_socket.local_address.ip_port}"]
|
|
273
|
+
[command_path, "--exclusive-mgmt-port=#{mgt_server_socket.local_address.ip_port}"]
|
|
270
274
|
else
|
|
271
|
-
['-M', mgt_server_socket.local_address.ip_port.to_s]
|
|
275
|
+
[command_path, '-M', mgt_server_socket.local_address.ip_port.to_s]
|
|
272
276
|
end
|
|
273
277
|
end
|
|
274
278
|
command_arguments.concat(args)
|
|
275
279
|
if capture_stderr
|
|
276
280
|
# capture process stderr
|
|
277
281
|
stderr_r, stderr_w = IO.pipe
|
|
278
|
-
spawn_args[err] = stderr_w
|
|
282
|
+
spawn_args[:err] = stderr_w
|
|
279
283
|
end
|
|
280
|
-
|
|
281
|
-
command_path = Ascp::Installation.instance.path(name)
|
|
282
|
-
command_pid = Environment.secure_spawn(env: env, exec: command_path, args: command_arguments, **spawn_args)
|
|
284
|
+
command_pid = Environment.secure_execute(*command_arguments, mode: :background, env: env, **spawn_args)
|
|
283
285
|
# close here, but still used in other process (pipe)
|
|
284
286
|
stderr_w&.close
|
|
285
287
|
notify_progress(:sessions_init, info: "waiting for #{name} to start")
|
data/lib/aspera/agent/factory.rb
CHANGED
|
@@ -16,15 +16,18 @@ module Aspera
|
|
|
16
16
|
Aspera::Agent.const_get(agent.to_s.capitalize).new(**options)
|
|
17
17
|
end
|
|
18
18
|
|
|
19
|
-
|
|
20
|
-
#
|
|
21
|
-
|
|
19
|
+
IGNORED_ITEMS = %i[factory base]
|
|
20
|
+
# Available agents: :long : Capitalized name string, :short : single character symbol
|
|
21
|
+
ALL =
|
|
22
22
|
Dir.children(File.dirname(File.expand_path(__FILE__)))
|
|
23
23
|
.select{ |file| file.end_with?(Environment::RB_EXT)}
|
|
24
24
|
.map{ |file| File.basename(file, Environment::RB_EXT).to_sym}
|
|
25
|
-
.reject{ |item| IGNORED_ITEMS.include?(item)}
|
|
26
|
-
|
|
27
|
-
|
|
25
|
+
.reject{ |item| IGNORED_ITEMS.include?(item)}.each_with_object({}) do |agent_sym, hash|
|
|
26
|
+
hash[agent_sym] = {
|
|
27
|
+
long: agent_sym.to_s.capitalize,
|
|
28
|
+
short: agent_sym.eql?(:direct) ? :a : agent_sym.to_s[0].to_sym
|
|
29
|
+
}.freeze
|
|
30
|
+
end.freeze
|
|
28
31
|
private_constant :IGNORED_ITEMS
|
|
29
32
|
end
|
|
30
33
|
end
|
|
@@ -21,10 +21,10 @@ module Aspera
|
|
|
21
21
|
|
|
22
22
|
private_constant :LOCAL_SOCKET_ADDR, :PORT_SEP, :AUTO_LOCAL_TCP_PORT
|
|
23
23
|
|
|
24
|
-
# @param url [String]
|
|
25
|
-
# @param start [
|
|
26
|
-
# @param stop [
|
|
27
|
-
# @param base [Hash]
|
|
24
|
+
# @param url [String] URL of the transfer manager daemon
|
|
25
|
+
# @param start [Boolean] If `false`, expect that an external daemon is already running
|
|
26
|
+
# @param stop [Boolean] If `false`, do not shutdown daemon on exit
|
|
27
|
+
# @param base [Hash] Base class options
|
|
28
28
|
def initialize(
|
|
29
29
|
url: AUTO_LOCAL_TCP_PORT,
|
|
30
30
|
start: true,
|
|
@@ -79,9 +79,9 @@ module Aspera
|
|
|
79
79
|
log_stdout = "#{transferd_base_tmp}.out"
|
|
80
80
|
log_stderr = "#{transferd_base_tmp}.err"
|
|
81
81
|
File.write(conf_file, config.to_json)
|
|
82
|
-
@daemon_pid = Environment.
|
|
83
|
-
|
|
84
|
-
|
|
82
|
+
@daemon_pid = Environment.secure_execute(
|
|
83
|
+
Ascp::Installation.instance.path(:transferd), '--config', conf_file,
|
|
84
|
+
mode: :background,
|
|
85
85
|
out: log_stdout,
|
|
86
86
|
err: log_stderr
|
|
87
87
|
)
|
|
@@ -170,7 +170,7 @@ module Aspera
|
|
|
170
170
|
def stop_daemon
|
|
171
171
|
if !@daemon_pid.nil?
|
|
172
172
|
Log.log.debug("Stopping daemon #{@daemon_pid}")
|
|
173
|
-
Process.kill(:
|
|
173
|
+
Process.kill(:KILL, @daemon_pid)
|
|
174
174
|
_, status = Process.wait2(@daemon_pid)
|
|
175
175
|
Log.log.debug("daemon stopped #{status}")
|
|
176
176
|
@daemon_pid = nil
|
data/lib/aspera/api/aoc.rb
CHANGED
|
@@ -33,7 +33,7 @@ module Aspera
|
|
|
33
33
|
# types of events for shared folder creation
|
|
34
34
|
# Node events: permission.created permission.modified permission.deleted
|
|
35
35
|
PERMISSIONS_CREATED = ['permission.created'].freeze
|
|
36
|
-
# Special
|
|
36
|
+
# Special user identifier when creating workspace shared folders
|
|
37
37
|
ID_AK_ADMIN = 'ASPERA_ACCESS_KEY_ADMIN'
|
|
38
38
|
|
|
39
39
|
private_constant :MAX_AOC_URL_REDIRECT,
|
|
@@ -48,11 +48,13 @@ module Aspera
|
|
|
48
48
|
:ID_AK_ADMIN
|
|
49
49
|
|
|
50
50
|
# various API scopes supported
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
51
|
+
module Scope
|
|
52
|
+
SELF = 'self'
|
|
53
|
+
USER = 'user:all'
|
|
54
|
+
ADMIN = 'admin:all'
|
|
55
|
+
ADMIN_USER = 'admin-user:all'
|
|
56
|
+
ADMIN_USER_USER = "#{ADMIN_USER}+#{USER}"
|
|
57
|
+
end
|
|
56
58
|
FILES_APP = 'files'
|
|
57
59
|
PACKAGES_APP = 'packages'
|
|
58
60
|
API_V1 = 'api/v1'
|
|
@@ -73,7 +75,7 @@ module Aspera
|
|
|
73
75
|
# split host of URL into organization and domain
|
|
74
76
|
def split_org_domain(uri)
|
|
75
77
|
Aspera.assert_type(uri, URI)
|
|
76
|
-
|
|
78
|
+
Aspera.assert(!uri.host.nil?){"No host found in URL. Please check URL format: https://myorg.#{SAAS_DOMAIN_PROD}"}
|
|
77
79
|
parts = uri.host.split('.', 2)
|
|
78
80
|
Aspera.assert(parts.length == 2){"expecting a public FQDN for #{PRODUCT_NAME}"}
|
|
79
81
|
parts[0] = nil if parts[0].eql?('api')
|
|
@@ -89,7 +91,7 @@ module Aspera
|
|
|
89
91
|
# @param url [String] URL of AoC public link
|
|
90
92
|
# @return [Hash] information about public link, or nil if not a public link
|
|
91
93
|
def link_info(url)
|
|
92
|
-
final_uri = Rest.new(base_url: url, redirect_max: MAX_AOC_URL_REDIRECT).call(operation: 'GET')
|
|
94
|
+
final_uri = Rest.new(base_url: url, redirect_max: MAX_AOC_URL_REDIRECT).call(operation: 'GET', ret: :resp).uri
|
|
93
95
|
Log.dump(:final_uri, final_uri, level: :trace1)
|
|
94
96
|
org_domain = split_org_domain(final_uri)
|
|
95
97
|
if (m = final_uri.path.match(%r{/oauth2/([^/]+)/login$}))
|
|
@@ -97,7 +99,7 @@ module Aspera
|
|
|
97
99
|
else
|
|
98
100
|
Log.log.debug{"path=#{final_uri.path} does not end with /login"}
|
|
99
101
|
end
|
|
100
|
-
|
|
102
|
+
Aspera.assert(!final_uri.query.nil?, 'AoC shall redirect to login page with a query', type: Error)
|
|
101
103
|
query = Rest.query_to_h(final_uri.query)
|
|
102
104
|
Log.dump(:query, query, level: :trace1)
|
|
103
105
|
# is that a public link ?
|
|
@@ -141,9 +143,10 @@ module Aspera
|
|
|
141
143
|
end
|
|
142
144
|
|
|
143
145
|
# Call block with same query using paging and response information.
|
|
144
|
-
# Block must return
|
|
146
|
+
# Block must return an Array with data and http response
|
|
145
147
|
# @return [Hash] {items: , total: }
|
|
146
|
-
def call_paging(query:
|
|
148
|
+
def call_paging(query: nil, formatter: nil)
|
|
149
|
+
query = {} if query.nil?
|
|
147
150
|
Aspera.assert_type(query, Hash){'query'}
|
|
148
151
|
Aspera.assert(block_given?)
|
|
149
152
|
# set default large page if user does not specify own parameters. AoC Caps to 1000 anyway
|
|
@@ -158,33 +161,62 @@ module Aspera
|
|
|
158
161
|
loop do
|
|
159
162
|
new_query = query.clone
|
|
160
163
|
new_query['page'] = current_page
|
|
161
|
-
|
|
162
|
-
Aspera.assert(
|
|
163
|
-
|
|
164
|
-
total_count = result[:http]['X-Total-Count']
|
|
164
|
+
result_data, result_http = yield(new_query)
|
|
165
|
+
Aspera.assert(result_http)
|
|
166
|
+
total_count = result_http['X-Total-Count']&.to_i
|
|
165
167
|
page_count += 1
|
|
166
168
|
current_page += 1
|
|
167
|
-
add_items =
|
|
169
|
+
add_items = result_data
|
|
170
|
+
break if add_items.nil?
|
|
168
171
|
break if add_items.empty?
|
|
169
172
|
# append new items to full list
|
|
170
173
|
item_list += add_items
|
|
171
174
|
break if !max_items.nil? && item_list.count >= max_items
|
|
172
175
|
break if !max_pages.nil? && page_count >= max_pages
|
|
176
|
+
break if total_count&.<=(item_list.count)
|
|
173
177
|
formatter&.long_operation_running("#{item_list.count} / #{total_count}") unless total_count.eql?(item_list.count.to_s)
|
|
174
178
|
end
|
|
175
179
|
formatter&.long_operation_terminated
|
|
176
180
|
item_list = item_list[0..max_items - 1] if !max_items.nil? && item_list.count > max_items
|
|
177
181
|
return {items: item_list, total: total_count}
|
|
178
182
|
end
|
|
183
|
+
|
|
184
|
+
# @param id [String] Identifier or workspace
|
|
185
|
+
# @return [Hash] suitable for permission filtering
|
|
186
|
+
def workspace_access(id)
|
|
187
|
+
{
|
|
188
|
+
'access_type' => 'user',
|
|
189
|
+
'access_id' => "#{ID_AK_ADMIN}_WS_#{id}"
|
|
190
|
+
}
|
|
191
|
+
end
|
|
192
|
+
|
|
193
|
+
# @param permission [Hash] Shared folder information
|
|
194
|
+
# @return [Boolean] `true` if internal access
|
|
195
|
+
def workspace_access?(permission)
|
|
196
|
+
permission['access_id'].start_with?("#{ID_AK_ADMIN}_WS_")
|
|
197
|
+
end
|
|
179
198
|
end
|
|
180
199
|
|
|
181
200
|
attr_reader :private_link
|
|
182
201
|
|
|
183
|
-
def initialize(
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
202
|
+
def initialize(
|
|
203
|
+
url:,
|
|
204
|
+
auth:,
|
|
205
|
+
subpath: API_V1,
|
|
206
|
+
client_id: nil,
|
|
207
|
+
client_secret: nil,
|
|
208
|
+
scope: nil,
|
|
209
|
+
redirect_uri: nil,
|
|
210
|
+
private_key: nil,
|
|
211
|
+
passphrase: nil,
|
|
212
|
+
username: nil,
|
|
213
|
+
password: nil,
|
|
214
|
+
workspace: nil,
|
|
215
|
+
secret_finder: nil
|
|
216
|
+
)
|
|
217
|
+
# Test here because link may set url
|
|
218
|
+
Aspera.assert(url, 'Missing mandatory option: url', type: ParameterError)
|
|
219
|
+
Aspera.assert(scope, 'Missing mandatory option: scope', type: ParameterError)
|
|
188
220
|
# default values for client id
|
|
189
221
|
client_id, client_secret = self.class.get_client_info if client_id.nil?
|
|
190
222
|
# access key secrets are provided out of band to get node api access
|
|
@@ -197,10 +229,12 @@ module Aspera
|
|
|
197
229
|
@workspace_info = nil
|
|
198
230
|
@home_info = nil
|
|
199
231
|
auth_params = {
|
|
200
|
-
type:
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
232
|
+
type: :oauth2,
|
|
233
|
+
params: {
|
|
234
|
+
client_id: client_id,
|
|
235
|
+
client_secret: client_secret,
|
|
236
|
+
scope: scope
|
|
237
|
+
}
|
|
204
238
|
}
|
|
205
239
|
# analyze type of url
|
|
206
240
|
url_info = AoC.link_info(url)
|
|
@@ -209,7 +243,7 @@ module Aspera
|
|
|
209
243
|
auth_params[:grant_method] = if url_info.key?(:token)
|
|
210
244
|
:url_json
|
|
211
245
|
else
|
|
212
|
-
|
|
246
|
+
Aspera.assert(auth, 'Missing mandatory option: auth', type: ParameterError)
|
|
213
247
|
auth
|
|
214
248
|
end
|
|
215
249
|
# this is the base API url
|
|
@@ -219,27 +253,27 @@ module Aspera
|
|
|
219
253
|
# fill other auth parameters based on OAuth method
|
|
220
254
|
case auth_params[:grant_method]
|
|
221
255
|
when :web
|
|
222
|
-
|
|
256
|
+
Aspera.assert(redirect_uri, 'Missing mandatory option: redirect_uri', type: ParameterError)
|
|
223
257
|
auth_params[:redirect_uri] = redirect_uri
|
|
224
258
|
when :jwt
|
|
225
|
-
|
|
226
|
-
|
|
259
|
+
Aspera.assert(private_key, 'Missing mandatory option: private_key', type: ParameterError)
|
|
260
|
+
Aspera.assert(username, 'Missing mandatory option: username', type: ParameterError)
|
|
227
261
|
auth_params[:private_key_obj] = OpenSSL::PKey::RSA.new(private_key, passphrase)
|
|
228
262
|
auth_params[:payload] = {
|
|
229
|
-
iss:
|
|
263
|
+
iss: client_id, # issuer
|
|
230
264
|
sub: username, # subject
|
|
231
265
|
aud: JWT_AUDIENCE
|
|
232
266
|
}
|
|
233
267
|
# add jwt payload for global client id
|
|
234
|
-
auth_params[:payload][:org] = url_info[:organization] if GLOBAL_CLIENT_APPS.include?(
|
|
268
|
+
auth_params[:payload][:org] = url_info[:organization] if GLOBAL_CLIENT_APPS.include?(client_id)
|
|
235
269
|
auth_params[:cache_ids] = [url_info[:organization]]
|
|
236
270
|
when :url_json
|
|
237
|
-
auth_params[:url] = {grant_type: 'url_token'} #
|
|
271
|
+
auth_params[:url] = {grant_type: 'url_token'} # Query arguments
|
|
238
272
|
auth_params[:json] = {url_token: url_info[:token]} # JSON body
|
|
239
273
|
# password protection of link
|
|
240
274
|
auth_params[:json][:password] = password unless password.nil?
|
|
241
275
|
# basic auth required for /token
|
|
242
|
-
auth_params[:auth] = {type: :basic, username:
|
|
276
|
+
auth_params[:auth] = {type: :basic, username: client_id, password: client_secret}
|
|
243
277
|
else Aspera.error_unexpected_value(auth_params[:grant_method]){'auth, use one of: :web, :jwt'}
|
|
244
278
|
end
|
|
245
279
|
super(
|
|
@@ -250,10 +284,9 @@ module Aspera
|
|
|
250
284
|
|
|
251
285
|
# read using the query and paging
|
|
252
286
|
# @return [Hash] {items: , total: }
|
|
253
|
-
def read_with_paging(subpath, query
|
|
287
|
+
def read_with_paging(subpath, query = nil, formatter: nil)
|
|
254
288
|
return self.class.call_paging(query: query, formatter: formatter) do |paged_query|
|
|
255
|
-
|
|
256
|
-
call(operation: 'GET', subpath: subpath, headers: {'Accept' => Rest::MIME_JSON}, query: paged_query)
|
|
289
|
+
read(subpath, query: paged_query, ret: :both)
|
|
257
290
|
end
|
|
258
291
|
end
|
|
259
292
|
|
|
@@ -352,16 +385,17 @@ module Aspera
|
|
|
352
385
|
file_id: user_info['read_only_home_file_id']
|
|
353
386
|
}
|
|
354
387
|
end
|
|
355
|
-
|
|
388
|
+
Aspera.assert(!@home_info[:node_id].to_s.empty?, "Cannot get user's home node id, check your default workspace or specify one", type: Error)
|
|
356
389
|
Log.dump(:context, @home_info)
|
|
357
390
|
@home_info
|
|
358
391
|
end
|
|
359
392
|
|
|
360
|
-
#
|
|
361
|
-
# @param
|
|
362
|
-
# @param
|
|
363
|
-
# @param
|
|
364
|
-
# @param
|
|
393
|
+
# Return a Node API for given node id, in a given context (files, packages), for the given scope.
|
|
394
|
+
# @param node_id [String] identifier of node in AoC
|
|
395
|
+
# @param workspace_id [String,nil] workspace identifier
|
|
396
|
+
# @param workspace_name [String,nil] workspace name
|
|
397
|
+
# @param scope [String,nil] e.g. Node::SCOPE_USER, or Node::SCOPE_ADMIN, or nil (requires secret)
|
|
398
|
+
# @param package_info [Hash,nil] created package information
|
|
365
399
|
# @returns [Node] a node API for access key
|
|
366
400
|
def node_api_from(node_id:, workspace_id: nil, workspace_name: nil, scope: Node::SCOPE_USER, package_info: nil)
|
|
367
401
|
Aspera.assert_type(node_id, String)
|
|
@@ -380,18 +414,22 @@ module Aspera
|
|
|
380
414
|
app_info[:package_name] = package_info['name']
|
|
381
415
|
end
|
|
382
416
|
node_params = {base_url: node_info['url']}
|
|
383
|
-
|
|
384
|
-
|
|
417
|
+
ak_secret = @secret_finder&.lookup_secret(url: node_info['url'], username: node_info['access_key'])
|
|
418
|
+
# If secret is available, or no scope, use basic auth
|
|
419
|
+
if scope.nil? || ak_secret
|
|
420
|
+
Aspera.assert(ak_secret, "Secret not found for access key #{node_info['access_key']}@#{node_info['url']}", type: Error)
|
|
385
421
|
node_params[:auth] = {
|
|
386
422
|
type: :basic,
|
|
387
423
|
username: node_info['access_key'],
|
|
388
|
-
password:
|
|
424
|
+
password: ak_secret
|
|
389
425
|
}
|
|
390
426
|
else
|
|
391
427
|
# OAuth bearer token
|
|
392
428
|
node_params[:auth] = auth_params.clone
|
|
393
|
-
node_params[:auth][:
|
|
394
|
-
|
|
429
|
+
node_params[:auth][:params] ||= {}
|
|
430
|
+
node_params[:auth][:params][:scope] = Node.token_scope(node_info['access_key'], scope)
|
|
431
|
+
node_params[:auth][:params][:owner_access] = true if scope.eql?(Node::SCOPE_ADMIN)
|
|
432
|
+
# Special header required for bearer token only
|
|
395
433
|
node_params[:headers] = {Node::HEADER_X_ASPERA_ACCESS_KEY => node_info['access_key']}
|
|
396
434
|
end
|
|
397
435
|
node_params[:app_info] = app_info
|
|
@@ -423,37 +461,38 @@ module Aspera
|
|
|
423
461
|
end
|
|
424
462
|
meta_schema.each do |field|
|
|
425
463
|
provided = pkg_meta.select{ |i| i['name'].eql?(field['name'])}
|
|
426
|
-
|
|
427
|
-
|
|
464
|
+
Aspera.assert(provided.count <= 1, type: Error){"only one field with name #{field['name']} allowed"}
|
|
465
|
+
Aspera.assert(!provided.empty?, type: Error){"missing mandatory field: #{field['name']}"} if field['required']
|
|
428
466
|
end
|
|
429
467
|
end
|
|
430
468
|
|
|
431
469
|
# Normalize package creation recipient lists as expected by AoC API
|
|
432
470
|
# AoC expects {type: , id: }, but ascli allows providing either the native values or just a name
|
|
433
471
|
# in that case, the name is resolved and replaced with {type: , id: }
|
|
434
|
-
# @param package_data
|
|
435
|
-
# @param
|
|
436
|
-
# @
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
472
|
+
# @param package_data [Hash] The whole package creation payload
|
|
473
|
+
# @param rcpt_lst_field [String] The field in structure, i.e. recipients or bcc_recipients
|
|
474
|
+
# @param new_user_option [Hash] Additionnal fields for contact creation
|
|
475
|
+
# @return nil, `package_data` is modified
|
|
476
|
+
def resolve_package_recipients(package_data, rcpt_lst_field, new_user_option)
|
|
477
|
+
return unless package_data.key?(rcpt_lst_field)
|
|
478
|
+
Aspera.assert_type(package_data[rcpt_lst_field], Array){rcpt_lst_field}
|
|
440
479
|
new_user_option = {'package_contact' => true} if new_user_option.nil?
|
|
441
480
|
Aspera.assert_type(new_user_option, Hash){'new_user_option'}
|
|
481
|
+
ws_id = package_data['workspace_id']
|
|
442
482
|
# list with resolved elements
|
|
443
483
|
resolved_list = []
|
|
444
|
-
package_data[
|
|
484
|
+
package_data[rcpt_lst_field].each do |short_recipient_info|
|
|
445
485
|
case short_recipient_info
|
|
446
486
|
when Hash # native API information, check keys
|
|
447
|
-
Aspera.assert(short_recipient_info.keys.sort.eql?(%w[id type])){"#{
|
|
487
|
+
Aspera.assert(short_recipient_info.keys.sort.eql?(%w[id type])){"#{rcpt_lst_field} element shall have fields: id and type"}
|
|
448
488
|
when String # CLI helper: need to resolve provided name to type/id
|
|
449
489
|
# email: user, else dropbox
|
|
450
490
|
entity_type = short_recipient_info.include?('@') ? 'contacts' : 'dropboxes'
|
|
451
491
|
begin
|
|
452
492
|
full_recipient_info = lookup_by_name(entity_type, short_recipient_info, query: {'current_workspace_id' => ws_id})
|
|
453
|
-
rescue
|
|
454
|
-
raise e unless e.message.start_with?(ENTITY_NOT_FOUND)
|
|
493
|
+
rescue EntityNotFound
|
|
455
494
|
# dropboxes cannot be created on the fly
|
|
456
|
-
|
|
495
|
+
Aspera.assert_values(entity_type, 'contacts', type: Error){"No such shared inbox in workspace #{ws_id}"}
|
|
457
496
|
# unknown user: create it as external user
|
|
458
497
|
full_recipient_info = create('contacts', {
|
|
459
498
|
'current_workspace_id' => ws_id,
|
|
@@ -465,14 +504,13 @@ module Aspera
|
|
|
465
504
|
else
|
|
466
505
|
{'id' => full_recipient_info['source_id'], 'type' => full_recipient_info['source_type']}
|
|
467
506
|
end
|
|
468
|
-
else #
|
|
469
|
-
raise "#{recipient_list_field} item must be a String (email, shared inbox) or Hash (id,type)"
|
|
507
|
+
else Aspera.error_unexpected_value(short_recipient_info.class.name){"#{rcpt_lst_field} item must be a String (email, shared inbox) or Hash (id,type)"}
|
|
470
508
|
end
|
|
471
509
|
# add original or resolved recipient info
|
|
472
510
|
resolved_list.push(short_recipient_info)
|
|
473
511
|
end
|
|
474
512
|
# replace with resolved elements
|
|
475
|
-
package_data[
|
|
513
|
+
package_data[rcpt_lst_field] = resolved_list
|
|
476
514
|
return
|
|
477
515
|
end
|
|
478
516
|
|
|
@@ -505,8 +543,8 @@ module Aspera
|
|
|
505
543
|
# package_data['file_names']||=[..list of filenames to transfer...]
|
|
506
544
|
|
|
507
545
|
# lookup users
|
|
508
|
-
resolve_package_recipients(package_data,
|
|
509
|
-
resolve_package_recipients(package_data,
|
|
546
|
+
resolve_package_recipients(package_data, 'recipients', new_user_option)
|
|
547
|
+
resolve_package_recipients(package_data, 'bcc_recipients', new_user_option)
|
|
510
548
|
|
|
511
549
|
validate_metadata(package_data) if validate_meta
|
|
512
550
|
|
|
@@ -622,8 +660,7 @@ module Aspera
|
|
|
622
660
|
when NilClass
|
|
623
661
|
when ''
|
|
624
662
|
# workspace shared folder
|
|
625
|
-
perm_data[
|
|
626
|
-
perm_data['access_id'] = "#{ID_AK_ADMIN}_WS_#{app_info[:workspace_id]}"
|
|
663
|
+
perm_data.merge!(self.class.workspace_access(app_info[:workspace_id]))
|
|
627
664
|
tag_workspace['shared_with_name'] = perm_data['access_id']
|
|
628
665
|
else
|
|
629
666
|
entity_info = lookup_by_name('contacts', shared_with, query: {'current_workspace_id' => app_info[:workspace_id]})
|
data/lib/aspera/api/ats.rb
CHANGED
data/lib/aspera/api/cos_node.rb
CHANGED
|
@@ -55,8 +55,9 @@ module Aspera
|
|
|
55
55
|
operation: 'GET',
|
|
56
56
|
subpath: bucket,
|
|
57
57
|
headers: {'Accept' => 'application/xml'},
|
|
58
|
-
query: {'faspConnectionInfo' => nil}
|
|
59
|
-
|
|
58
|
+
query: {'faspConnectionInfo' => nil},
|
|
59
|
+
ret: :resp
|
|
60
|
+
).body
|
|
60
61
|
ats_info = XmlSimple.xml_in(xml_result_text, {'ForceArray' => false})
|
|
61
62
|
Log.dump(:ats_info, ats_info)
|
|
62
63
|
@storage_credentials = {
|
data/lib/aspera/api/faspex.rb
CHANGED
|
@@ -28,24 +28,25 @@ module Aspera
|
|
|
28
28
|
|
|
29
29
|
def create_token
|
|
30
30
|
# Exchange context (passcode) for code
|
|
31
|
-
|
|
31
|
+
http = api.call(
|
|
32
32
|
operation: 'GET',
|
|
33
33
|
subpath: @path_authorize,
|
|
34
34
|
query: {
|
|
35
35
|
response_type: :code,
|
|
36
36
|
state: @context,
|
|
37
|
-
client_id: client_id,
|
|
37
|
+
client_id: params[:client_id],
|
|
38
38
|
redirect_uri: @redirect_uri
|
|
39
39
|
},
|
|
40
|
-
exception: false
|
|
40
|
+
exception: false,
|
|
41
|
+
ret: :resp
|
|
41
42
|
)
|
|
42
43
|
# code / state located in redirected URL query
|
|
43
|
-
info = Rest.query_to_h(URI.parse(
|
|
44
|
+
info = Rest.query_to_h(URI.parse(http['Location']).query)
|
|
44
45
|
Log.dump(:info, info)
|
|
45
46
|
raise Error, info['action_message'] if info['action_message']
|
|
46
47
|
Aspera.assert(info['code']){'Missing code in answer'}
|
|
47
48
|
# Exchange code for token
|
|
48
|
-
return create_token_call(
|
|
49
|
+
return create_token_call(base_params.merge(
|
|
49
50
|
grant_type: 'authorization_code',
|
|
50
51
|
code: info['code'],
|
|
51
52
|
redirect_uri: @redirect_uri
|
|
@@ -133,7 +134,7 @@ module Aspera
|
|
|
133
134
|
case auth
|
|
134
135
|
when :public_link
|
|
135
136
|
# Get URL of final redirect of public link
|
|
136
|
-
redir_url = Rest.new(base_url: url, redirect_max: 3).call(operation: 'GET')
|
|
137
|
+
redir_url = Rest.new(base_url: url, redirect_max: 3).call(operation: 'GET', ret: :resp).uri.to_s
|
|
137
138
|
Log.dump(:redir_url, redir_url, level: :trace1)
|
|
138
139
|
# get context from query
|
|
139
140
|
encoded_context = Rest.query_to_h(URI.parse(redir_url).query)['context']
|
|
@@ -145,7 +146,7 @@ module Aspera
|
|
|
145
146
|
base_url = redir_url.gsub(%r{/public/.*}, '').gsub(/\?.*/, '')
|
|
146
147
|
# Get web UI client_id and redirect_uri
|
|
147
148
|
# TODO: change this for something more reliable
|
|
148
|
-
config = JSON.parse(Rest.new(base_url: "#{base_url}/config.js", redirect_max: 3).call(operation: 'GET')
|
|
149
|
+
config = JSON.parse(Rest.new(base_url: "#{base_url}/config.js", redirect_max: 3).call(operation: 'GET').sub(/^[^=]+=/, '').gsub(/([a-z_]+):/, '"\1":').delete("\n ").tr("'", '"')).symbolize_keys
|
|
149
150
|
Log.dump(:configjs, config)
|
|
150
151
|
{
|
|
151
152
|
base_url: "#{base_url}/#{PATH_API_V5}",
|
|
@@ -154,7 +155,9 @@ module Aspera
|
|
|
154
155
|
base_url: "#{base_url}/#{PATH_AUTH}",
|
|
155
156
|
grant_method: :faspex_pub_link,
|
|
156
157
|
context: encoded_context,
|
|
157
|
-
|
|
158
|
+
params: {
|
|
159
|
+
client_id: config[:client_id]
|
|
160
|
+
},
|
|
158
161
|
redirect_uri: config[:redirect_uri]
|
|
159
162
|
}
|
|
160
163
|
}
|
|
@@ -176,7 +179,9 @@ module Aspera
|
|
|
176
179
|
type: :oauth2,
|
|
177
180
|
base_url: "#{url}/#{PATH_AUTH}",
|
|
178
181
|
grant_method: :web,
|
|
179
|
-
|
|
182
|
+
params: {
|
|
183
|
+
client_id: client_id
|
|
184
|
+
},
|
|
180
185
|
redirect_uri: redirect_uri
|
|
181
186
|
}
|
|
182
187
|
}
|
|
@@ -189,7 +194,9 @@ module Aspera
|
|
|
189
194
|
type: :oauth2,
|
|
190
195
|
base_url: "#{url}/#{PATH_AUTH}",
|
|
191
196
|
grant_method: :jwt,
|
|
192
|
-
|
|
197
|
+
params: {
|
|
198
|
+
client_id: client_id
|
|
199
|
+
},
|
|
193
200
|
payload: {
|
|
194
201
|
iss: client_id, # issuer
|
|
195
202
|
aud: client_id, # audience (this field is not clear...)
|
data/lib/aspera/api/node.rb
CHANGED
|
@@ -386,21 +386,19 @@ module Aspera
|
|
|
386
386
|
return oauth.authorization(refresh: true)
|
|
387
387
|
end
|
|
388
388
|
|
|
389
|
+
# Get a base download transfer spec (gen3)
|
|
390
|
+
# @return [Hash] Base transfer spec
|
|
391
|
+
def base_spec
|
|
392
|
+
create(
|
|
393
|
+
'files/download_setup',
|
|
394
|
+
{transfer_requests: [{transfer_request: {paths: [{source: '/'}]}}]}
|
|
395
|
+
)['transfer_specs'].first['transfer_spec']
|
|
396
|
+
end
|
|
397
|
+
|
|
389
398
|
# Get generic part of transfer spec with transport parameters only
|
|
390
399
|
# @return [Hash] Base transfer spec
|
|
391
400
|
def transport_params
|
|
392
|
-
|
|
393
|
-
# Retrieve values from API (and keep a copy/cache)
|
|
394
|
-
full_spec = create(
|
|
395
|
-
'files/download_setup',
|
|
396
|
-
{transfer_requests: [{transfer_request: {paths: [{source: '/'}]}}]}
|
|
397
|
-
)['transfer_specs'].first['transfer_spec']
|
|
398
|
-
# Set available fields
|
|
399
|
-
@std_t_spec_cache = Transfer::Spec::TRANSPORT_FIELDS.each_with_object({}) do |i, h|
|
|
400
|
-
h[i] = full_spec[i] if full_spec.key?(i)
|
|
401
|
-
end
|
|
402
|
-
end
|
|
403
|
-
return @std_t_spec_cache
|
|
401
|
+
@std_t_spec_cache ||= base_spec.slice(*Transfer::Spec::TRANSPORT_FIELDS).freeze
|
|
404
402
|
end
|
|
405
403
|
|
|
406
404
|
# Create transfer spec for gen4
|
data/lib/aspera/ascmd.rb
CHANGED
|
@@ -89,13 +89,12 @@ module Aspera
|
|
|
89
89
|
# Version 2 allows use of reverse proxy with multiple addresses.
|
|
90
90
|
# @param [Symbol] one of OPERATIONS
|
|
91
91
|
# @param [Array] parameters for "as" command
|
|
92
|
-
# @return result of command, type depends on command
|
|
92
|
+
# @return [Boolean,Array,Hash] result of command, type depends on command
|
|
93
93
|
def execute_single(action_sym, arguments, version: 1, host: nil)
|
|
94
94
|
arguments = [] if arguments.nil?
|
|
95
95
|
Log.log.debug{"execute_single:#{action_sym}:#{arguments}"}
|
|
96
96
|
Aspera.assert_type(action_sym, Symbol)
|
|
97
|
-
Aspera.
|
|
98
|
-
Aspera.assert(arguments.all?(String), 'arguments must be strings')
|
|
97
|
+
Aspera.assert_array_all(arguments, String){'arguments'}
|
|
99
98
|
remote_cmd = 'ascmd'
|
|
100
99
|
# lines of commands (String's)
|
|
101
100
|
command_lines = []
|