aspera-cli 4.26.2 → 4.27.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/CHANGELOG.md +310 -21
- data/CONTRIBUTING.md +99 -14
- data/TODO.md +50 -0
- data/bin/ascli +3 -3
- data/docs/README.md +10977 -0
- data/docs/test-mcp-with-ai.md +202 -0
- data/lib/aspera/agent/base.rb +29 -7
- data/lib/aspera/agent/connect.rb +58 -21
- data/lib/aspera/agent/desktop.rb +53 -18
- data/lib/aspera/agent/direct.rb +71 -40
- data/lib/aspera/agent/factory.rb +3 -3
- data/lib/aspera/agent/httpgw.rb +55 -14
- data/lib/aspera/agent/node.rb +34 -2
- data/lib/aspera/agent/transferd.rb +48 -12
- data/lib/aspera/api/alee.rb +1 -1
- data/lib/aspera/api/aoc.rb +42 -44
- data/lib/aspera/api/cos_node.rb +10 -9
- data/lib/aspera/api/faspex.rb +16 -15
- data/lib/aspera/api/httpgw.rb +23 -23
- data/lib/aspera/api/node.rb +51 -70
- data/lib/aspera/api/queries/bss_subscription_account.graphql +18 -0
- data/lib/aspera/api/queries/bss_subscription_usage.graphql +18 -0
- data/lib/aspera/ascmd.rb +32 -30
- data/lib/aspera/ascp/installation.rb +75 -69
- data/lib/aspera/ascp/management.rb +7 -8
- data/lib/aspera/assert.rb +12 -13
- data/lib/aspera/cli/ascp_actions.rb +155 -0
- data/lib/aspera/cli/async_transfer_store.rb +125 -0
- data/lib/aspera/cli/bootstrapper.rb +197 -0
- data/lib/aspera/cli/command_registry.rb +145 -0
- data/lib/aspera/cli/command_spec.rb +141 -0
- data/lib/aspera/cli/context.rb +19 -6
- data/lib/aspera/cli/error.rb +16 -0
- data/lib/aspera/cli/extended_value.rb +27 -26
- data/lib/aspera/cli/formatter.rb +95 -59
- data/lib/aspera/cli/gem_checker.rb +65 -0
- data/lib/aspera/cli/hints.rb +3 -3
- data/lib/aspera/cli/http.rb +74 -30
- data/lib/aspera/cli/info.rb +2 -0
- data/lib/aspera/cli/mailer.rb +97 -0
- data/lib/aspera/cli/mcp_tool.rb +225 -0
- data/lib/aspera/cli/option_declarator.rb +77 -0
- data/lib/aspera/cli/options.schema.yaml +634 -7
- data/lib/aspera/cli/parser.rb +1217 -0
- data/lib/aspera/cli/plugins/alee.rb +20 -22
- data/lib/aspera/cli/plugins/aoc.rb +1178 -881
- data/lib/aspera/cli/plugins/ats.rb +205 -162
- data/lib/aspera/cli/plugins/base.rb +703 -175
- data/lib/aspera/cli/plugins/basic_auth.rb +7 -9
- data/lib/aspera/cli/plugins/config.rb +434 -757
- data/lib/aspera/cli/plugins/console.rb +107 -65
- data/lib/aspera/cli/plugins/cos.rb +46 -33
- data/lib/aspera/cli/plugins/factory.rb +6 -6
- data/lib/aspera/cli/plugins/faspex5.rb +627 -388
- data/lib/aspera/cli/plugins/faspio.rb +54 -51
- data/lib/aspera/cli/plugins/httpgw.rb +18 -25
- data/lib/aspera/cli/plugins/mcp.rb +279 -0
- data/lib/aspera/cli/plugins/node.rb +988 -840
- data/lib/aspera/cli/plugins/oauth.rb +7 -10
- data/lib/aspera/cli/plugins/orchestrator.rb +112 -135
- data/lib/aspera/cli/plugins/preview.rb +195 -157
- data/lib/aspera/cli/plugins/server.rb +141 -91
- data/lib/aspera/cli/plugins/shares.rb +343 -110
- data/lib/aspera/cli/preset_actions.rb +159 -0
- data/lib/aspera/cli/preset_manager.rb +82 -39
- data/lib/aspera/cli/result.rb +78 -26
- data/lib/aspera/cli/runner.rb +323 -153
- data/lib/aspera/cli/secret_finder.rb +40 -0
- data/lib/aspera/cli/special_values.rb +1 -0
- data/lib/aspera/cli/sync_actions.rb +89 -64
- data/lib/aspera/cli/terminal_formatter.rb +3 -3
- data/lib/aspera/cli/transfer_actions.rb +90 -0
- data/lib/aspera/cli/transfer_agent.rb +129 -64
- data/lib/aspera/cli/transfer_progress.rb +9 -9
- data/lib/aspera/cli/vault_manager.rb +79 -0
- data/lib/aspera/cli/version.rb +1 -1
- data/lib/aspera/cli/wizard.rb +28 -30
- data/lib/aspera/colors.rb +3 -3
- data/lib/aspera/command_line_builder.rb +34 -34
- data/lib/aspera/command_line_converter.rb +1 -1
- data/lib/aspera/coverage.rb +1 -2
- data/lib/aspera/data_repository.rb +2 -1
- data/lib/aspera/dot_container.rb +12 -12
- data/lib/aspera/environment.rb +30 -23
- data/lib/aspera/exec_spec.rb +13 -0
- data/lib/aspera/faspex_gw.rb +5 -5
- data/lib/aspera/faspex_postproc.rb +16 -10
- data/lib/aspera/graphql.rb +37 -0
- data/lib/aspera/hash_ext.rb +8 -2
- data/lib/aspera/json_rpc/client.rb +62 -0
- data/lib/aspera/json_rpc/version.rb +7 -0
- data/lib/aspera/keychain/base.rb +10 -3
- data/lib/aspera/keychain/encrypted_hash.rb +92 -17
- data/lib/aspera/keychain/factory.rb +20 -8
- data/lib/aspera/keychain/hashicorp_vault.rb +1 -1
- data/lib/aspera/keychain/macos_security.rb +23 -25
- data/lib/aspera/keychain/one_password_api.rb +86 -0
- data/lib/aspera/keychain/one_password_base.rb +34 -0
- data/lib/aspera/keychain/one_password_cli.rb +98 -0
- data/lib/aspera/link_header.rb +82 -0
- data/lib/aspera/log.rb +29 -15
- data/lib/aspera/markdown.rb +90 -7
- data/lib/aspera/nagios.rb +8 -8
- data/lib/aspera/node_simulator.rb +35 -27
- data/lib/aspera/oauth/base.rb +9 -10
- data/lib/aspera/oauth/boot.rb +5 -5
- data/lib/aspera/oauth/factory.rb +12 -10
- data/lib/aspera/oauth/jwt.rb +9 -11
- data/lib/aspera/oauth/web.rb +6 -6
- data/lib/aspera/persistency_action_once.rb +12 -12
- data/lib/aspera/persistency_folder.rb +18 -11
- data/lib/aspera/preview/file_types.rb +5 -5
- data/lib/aspera/preview/generator.rb +52 -49
- data/lib/aspera/preview/options.rb +3 -2
- data/lib/aspera/preview/terminal.rb +5 -5
- data/lib/aspera/preview/utils.rb +66 -19
- data/lib/aspera/products/connect.rb +2 -2
- data/lib/aspera/products/desktop.rb +1 -1
- data/lib/aspera/products/other.rb +3 -3
- data/lib/aspera/products/transferd.rb +4 -4
- data/lib/aspera/proxy_auto_config.rb +15 -14
- data/lib/aspera/rest.rb +159 -151
- data/lib/aspera/rest_error_analyzer.rb +6 -6
- data/lib/aspera/rest_errors_aspera.rb +0 -10
- data/lib/aspera/rest_list.rb +11 -10
- data/lib/aspera/schema/IBM Aspera Node API-4.4.6.yaml +6232 -0
- data/lib/aspera/schema/IBM Aspera faspio Gateway API-1.0.0.yaml +249 -0
- data/lib/aspera/schema/IBM Aspera on Cloud API-0.2.6-enhanced.yaml +1853 -510
- data/lib/aspera/schema/IBM_Aspera_Shares.yaml +4653 -0
- data/lib/aspera/schema/documentation.rb +65 -28
- data/lib/aspera/schema/reader.rb +136 -13
- data/lib/aspera/schema/registry.rb +60 -8
- data/lib/aspera/secret_hider.rb +12 -7
- data/lib/aspera/ssh.rb +64 -31
- data/lib/aspera/ssl.rb +5 -5
- data/lib/aspera/sync/conf.schema.yaml +2 -2
- data/lib/aspera/sync/database.rb +2 -2
- data/lib/aspera/sync/operations.rb +22 -25
- data/lib/aspera/temp_file_manager.rb +25 -5
- data/lib/aspera/timer_limiter.rb +1 -1
- data/lib/aspera/transfer/faux_file.rb +24 -11
- data/lib/aspera/transfer/parameters.rb +36 -34
- data/lib/aspera/transfer/result.rb +74 -0
- data/lib/aspera/transfer/resumer.rb +10 -10
- data/lib/aspera/transfer/spec.rb +18 -1
- data/lib/aspera/transfer/spec.schema.yaml +12 -3
- data/lib/aspera/uri_reader.rb +52 -15
- data/lib/aspera/web_auth.rb +8 -7
- data/lib/aspera/web_server_simple.rb +15 -12
- data/lib/aspera/yaml.rb +5 -4
- data.tar.gz.sig +0 -0
- metadata +34 -6
- metadata.gz.sig +0 -0
- data/lib/aspera/cli/manager.rb +0 -766
- data/lib/aspera/cli/plugins/faspex.rb +0 -539
- data/lib/aspera/json_rpc.rb +0 -52
- data/lib/aspera/transfer/uri.rb +0 -56
data/lib/aspera/rest.rb
CHANGED
|
@@ -77,6 +77,9 @@ module Aspera
|
|
|
77
77
|
# and error are analyzed in RestErrorAnalyzer
|
|
78
78
|
class Rest
|
|
79
79
|
class << self
|
|
80
|
+
# Build a Basic authentication header value
|
|
81
|
+
# @param user [String] Username
|
|
82
|
+
# @param pass [String] Password
|
|
80
83
|
# @return [String] Basic auth token
|
|
81
84
|
def basic_authorization(user, pass); return "Basic #{Base64.strict_encode64("#{user}:#{pass}")}"; end
|
|
82
85
|
|
|
@@ -84,7 +87,7 @@ module Aspera
|
|
|
84
87
|
# @param query [Hash] A key can have Array value and result will use PHP format: a[]=1&a[]=2
|
|
85
88
|
# @return [Hash] The query parameters.
|
|
86
89
|
def php_style(query)
|
|
87
|
-
Aspera.assert_type(query, Hash){'query'}
|
|
90
|
+
Aspera.assert_type(query, Hash) { 'query' }
|
|
88
91
|
query[:x_array_php_style] = true
|
|
89
92
|
query
|
|
90
93
|
end
|
|
@@ -93,10 +96,11 @@ module Aspera
|
|
|
93
96
|
# Check if php style is specified.
|
|
94
97
|
# `nil` values in query result in key without value, e.g. `?a`, while empty string values result in `?a=`.
|
|
95
98
|
# @param url [String] The URL without query.
|
|
96
|
-
# @param query [Hash,Array,String] The query.
|
|
99
|
+
# @param query [Hash,Array,String] The query parameters.
|
|
100
|
+
# @return [URI] The built URI.
|
|
97
101
|
def build_uri(url, query)
|
|
98
102
|
uri = URI.parse(url)
|
|
99
|
-
Aspera.assert_values(uri.scheme, %w[http https]){'URI scheme'}
|
|
103
|
+
Aspera.assert_values(uri.scheme, %w[http https]) { 'URI scheme' }
|
|
100
104
|
return uri if query.nil? || query.respond_to?(:empty?) && query.empty?
|
|
101
105
|
Log.dump(:query, query)
|
|
102
106
|
uri.query =
|
|
@@ -106,9 +110,9 @@ module Aspera
|
|
|
106
110
|
when Hash
|
|
107
111
|
URI.encode_www_form(h_to_query_array(query))
|
|
108
112
|
when Array
|
|
109
|
-
Aspera.assert(query.all?{ |i| i.is_a?(Array) && i.length.eql?(2)}, 'Query must be array of arrays of 2 elements')
|
|
113
|
+
Aspera.assert(query.all? { |i| i.is_a?(Array) && i.length.eql?(2) }, 'Query must be array of arrays of 2 elements')
|
|
110
114
|
URI.encode_www_form(query) # remove nil values
|
|
111
|
-
else Aspera.error_unexpected_value(query.class){'query type'}
|
|
115
|
+
else Aspera.error_unexpected_value(query.class) { 'query type' }
|
|
112
116
|
end.gsub('%5B%5D=', '[]=')
|
|
113
117
|
# [] is allowed in url parameters
|
|
114
118
|
uri
|
|
@@ -118,6 +122,7 @@ module Aspera
|
|
|
118
122
|
# Either p=1&p=2 (default)
|
|
119
123
|
# or p[]=1&p[]=2 (if `:x_array_php_style` is set to true in query)
|
|
120
124
|
# @param query [Hash] HTTP query as hash
|
|
125
|
+
# @return [Array<Array>] Array of [key, value] pairs suitable for URI.encode_www_form
|
|
121
126
|
def h_to_query_array(query)
|
|
122
127
|
Aspera.assert_type(query, Hash)
|
|
123
128
|
suffix = query.delete(:x_array_php_style) ? '[]' : nil
|
|
@@ -155,43 +160,12 @@ module Aspera
|
|
|
155
160
|
end
|
|
156
161
|
end
|
|
157
162
|
|
|
158
|
-
# Parse Link header according to RFC 8288 to extract a specific relation
|
|
159
|
-
# @param link_header [String, nil] The Link header value
|
|
160
|
-
# @param rel [String] The relation to look for (default: 'next')
|
|
161
|
-
# @return [String, nil] The URL of the link with the specified relation, or nil
|
|
162
|
-
def parse_link_header(link_header, rel: 'next')
|
|
163
|
-
return if link_header.nil? || link_header.empty?
|
|
164
|
-
# RFC 8288: Link header format is: <URI>; param1=value1; param2=value2, <URI2>; ...
|
|
165
|
-
# We look for the link with the specified rel
|
|
166
|
-
link_header.split(',').each do |link_part|
|
|
167
|
-
link_part = link_part.strip
|
|
168
|
-
# Extract URL between < and >
|
|
169
|
-
url_match = link_part.match(/<([^>]+)>/)
|
|
170
|
-
next unless url_match
|
|
171
|
-
url = url_match[1]
|
|
172
|
-
# Extract parameters after the URL
|
|
173
|
-
params_str = link_part[url_match.end(0)..]
|
|
174
|
-
# Check if this link has the specified rel (with or without quotes, case insensitive)
|
|
175
|
-
next unless /;\s*rel\s*=\s*"?#{Regexp.escape(rel)}"?/i.match?(params_str)
|
|
176
|
-
return url
|
|
177
|
-
end
|
|
178
|
-
# Fallback: if no rel found and looking for 'next', try the first link (backward compatibility)
|
|
179
|
-
if rel.eql?('next')
|
|
180
|
-
first_link = link_header.split(',').first&.strip
|
|
181
|
-
if first_link
|
|
182
|
-
url_match = first_link.match(/<([^>]+)>/)
|
|
183
|
-
return url_match[1] if url_match
|
|
184
|
-
end
|
|
185
|
-
end
|
|
186
|
-
nil
|
|
187
|
-
end
|
|
188
|
-
|
|
189
163
|
# Start a HTTP/S session, also used for web sockets
|
|
190
164
|
# @param base_url [String] Base url of HTTP/S session
|
|
191
165
|
# @return [Net::HTTP] A started HTTP session
|
|
192
166
|
def start_http_session(base_url)
|
|
193
167
|
uri = URI.parse(base_url)
|
|
194
|
-
Aspera.assert_values(uri.scheme, %w[http https]){'URI scheme'}
|
|
168
|
+
Aspera.assert_values(uri.scheme, %w[http https]) { 'URI scheme' }
|
|
195
169
|
# This honors http_proxy env var
|
|
196
170
|
http_session = Net::HTTP.new(uri.host, uri.port)
|
|
197
171
|
http_session.use_ssl = uri.scheme.eql?('https')
|
|
@@ -211,7 +185,7 @@ module Aspera
|
|
|
211
185
|
Aspera.assert_type(http_session, Net::HTTP)
|
|
212
186
|
# Net::BufferedIO in net/protocol.rb
|
|
213
187
|
result = http_session.instance_variable_get(:@socket)
|
|
214
|
-
Aspera.assert(!result.nil?){"no socket for #{http_session}"}
|
|
188
|
+
Aspera.assert(!result.nil?) { "no socket for #{http_session}" }
|
|
215
189
|
return result
|
|
216
190
|
end
|
|
217
191
|
|
|
@@ -279,7 +253,7 @@ module Aspera
|
|
|
279
253
|
# Base common headers of API
|
|
280
254
|
attr_reader :headers
|
|
281
255
|
|
|
282
|
-
# @return creation parameters
|
|
256
|
+
# @return [Hash] creation parameters
|
|
283
257
|
def params
|
|
284
258
|
return {
|
|
285
259
|
base_url: @base_url, # String
|
|
@@ -316,7 +290,7 @@ module Aspera
|
|
|
316
290
|
# remove trailing port if it is 443 and scheme is https
|
|
317
291
|
@base_url = @base_url.gsub(/:443$/, '') if @base_url.start_with?('https://')
|
|
318
292
|
@base_url = @base_url.gsub(/:80$/, '') if @base_url.start_with?('http://')
|
|
319
|
-
Log.log.debug{"Rest.new(#{@base_url})"}
|
|
293
|
+
Log.log.debug { "Rest.new(#{@base_url})" }
|
|
320
294
|
# default is no auth
|
|
321
295
|
@auth_params = auth
|
|
322
296
|
Aspera.assert_type(@auth_params, Hash)
|
|
@@ -334,11 +308,11 @@ module Aspera
|
|
|
334
308
|
@oauth = nil
|
|
335
309
|
end
|
|
336
310
|
|
|
337
|
-
# @return the OAuth object (create, or cached if already created)
|
|
311
|
+
# @return [OAuth::Base] the OAuth object (create, or cached if already created)
|
|
338
312
|
def oauth
|
|
339
313
|
if @oauth.nil?
|
|
340
314
|
Aspera.assert(@auth_params[:type].eql?(:oauth2), 'no OAuth defined')
|
|
341
|
-
oauth_parameters = @auth_params.reject{ |k, _v| k.eql?(:type)}
|
|
315
|
+
oauth_parameters = @auth_params.reject { |k, _v| k.eql?(:type) }
|
|
342
316
|
Log.dump(:oauth_parameters, oauth_parameters)
|
|
343
317
|
@oauth = OAuth::Factory.instance.create(**oauth_parameters)
|
|
344
318
|
end
|
|
@@ -352,9 +326,9 @@ module Aspera
|
|
|
352
326
|
# @param content_type [String, nil] Type of body parameters (one of MIME_*) and serialization, else use headers
|
|
353
327
|
# @param body [Hash, String, nil] Body parameters
|
|
354
328
|
# @param headers [Hash{String => String}] Additional headers (override Content-Type)
|
|
355
|
-
# @param
|
|
329
|
+
# @param save_to [String, IO, nil] File path or IO object to save response body; progress bar is used when set
|
|
356
330
|
# @param exception [Boolean] Whether to raise an exception on HTTP error
|
|
357
|
-
# @param ret [Symbol] One of :data, :resp, :both
|
|
331
|
+
# @param ret [Symbol] One of :data, :resp, :both - controls return value
|
|
358
332
|
# @return [Array(Hash, Net::HTTPResponse)] When `ret` is :both
|
|
359
333
|
# @return [Net::HTTPResponse] When `ret` is :resp
|
|
360
334
|
# @return [Hash] When `ret` is :data
|
|
@@ -366,42 +340,20 @@ module Aspera
|
|
|
366
340
|
content_type: nil,
|
|
367
341
|
body: nil,
|
|
368
342
|
headers: nil,
|
|
369
|
-
|
|
343
|
+
save_to: nil,
|
|
370
344
|
exception: true,
|
|
371
345
|
ret: :data
|
|
372
346
|
)
|
|
373
347
|
subpath = subpath.to_s if subpath.is_a?(Symbol)
|
|
374
348
|
subpath = '' if subpath.nil?
|
|
375
|
-
Log.log.debug{"call #{operation} [#{subpath}]".red.bold.bg_green}
|
|
349
|
+
Log.log.debug { "call #{operation} [#{subpath}]".red.bold.bg_green }
|
|
376
350
|
Log.dump(:body, body, level: :trace1)
|
|
377
351
|
Log.dump(:query, query, level: :trace1)
|
|
378
352
|
Log.dump(:headers, headers, level: :trace1)
|
|
379
353
|
Aspera.assert_type(subpath, String)
|
|
380
354
|
# We must have a way to check return code
|
|
381
355
|
Aspera.assert(exception || !ret.eql?(:data), 'ret: :data requires exception handler')
|
|
382
|
-
|
|
383
|
-
headers = @headers.clone
|
|
384
|
-
else
|
|
385
|
-
h = headers
|
|
386
|
-
headers = @headers.clone
|
|
387
|
-
headers.merge!(h)
|
|
388
|
-
end
|
|
389
|
-
Aspera.assert_type(headers, Hash)
|
|
390
|
-
case @auth_params[:type]
|
|
391
|
-
when :none
|
|
392
|
-
# no auth
|
|
393
|
-
when :basic
|
|
394
|
-
Log.log.debug('using Basic auth')
|
|
395
|
-
# done in build_req
|
|
396
|
-
when :oauth2
|
|
397
|
-
headers['Authorization'] = oauth.authorization unless headers.key?('Authorization')
|
|
398
|
-
when :url
|
|
399
|
-
query ||= {}
|
|
400
|
-
@auth_params[:url_query].each do |key, value|
|
|
401
|
-
query[key] = value
|
|
402
|
-
end
|
|
403
|
-
else Aspera.error_unexpected_value(@auth_params[:type])
|
|
404
|
-
end
|
|
356
|
+
headers, query = prepare_call(headers, query)
|
|
405
357
|
result_http = nil
|
|
406
358
|
result_data = nil
|
|
407
359
|
# initialize with number of initial retries allowed, nil gives zero
|
|
@@ -409,37 +361,7 @@ module Aspera
|
|
|
409
361
|
# start a block to be able to retry the actual HTTP request in case of OAuth token expiration
|
|
410
362
|
begin
|
|
411
363
|
Log.log.debug("send request (retries=#{tries_remain_redirect})")
|
|
412
|
-
|
|
413
|
-
# URI.escape()
|
|
414
|
-
separator = ['', '/'].include?(subpath) ? '' : '/'
|
|
415
|
-
uri = self.class.build_uri("#{@base_url}#{separator}#{subpath}", query)
|
|
416
|
-
Log.log.debug{"URI=#{uri}"}
|
|
417
|
-
begin
|
|
418
|
-
# instantiate request object based on string name
|
|
419
|
-
req = Net::HTTP.const_get(operation.capitalize).new(uri)
|
|
420
|
-
rescue NameError
|
|
421
|
-
raise "unsupported operation : #{operation}"
|
|
422
|
-
end
|
|
423
|
-
case content_type
|
|
424
|
-
when nil # ignore
|
|
425
|
-
when Mime::JSON
|
|
426
|
-
req.body = JSON.generate(body) # , ascii_only: true
|
|
427
|
-
req['Content-Type'] = Mime::JSON
|
|
428
|
-
when Mime::WWW
|
|
429
|
-
req.body = URI.encode_www_form(body)
|
|
430
|
-
req['Content-Type'] = Mime::WWW
|
|
431
|
-
when Mime::TEXT
|
|
432
|
-
req.body = body
|
|
433
|
-
req['Content-Type'] = Mime::TEXT
|
|
434
|
-
else Aspera.error_unexpected_value(content_type){'body type'}
|
|
435
|
-
end
|
|
436
|
-
# set headers
|
|
437
|
-
headers.each do |key, value|
|
|
438
|
-
req[key] = value
|
|
439
|
-
end
|
|
440
|
-
# :type = :basic
|
|
441
|
-
req.basic_auth(@auth_params[:username], @auth_params[:password]) if @auth_params[:type].eql?(:basic)
|
|
442
|
-
Log.dump(:req_body, req.body, level: :trace1)
|
|
364
|
+
req = build_request(operation, subpath, query, content_type, body, headers)
|
|
443
365
|
# we try the call, and will retry on some error types
|
|
444
366
|
error_tries ||= 1 + RestParameters.instance.retry_max
|
|
445
367
|
result_mime = nil
|
|
@@ -448,58 +370,21 @@ module Aspera
|
|
|
448
370
|
http_session.request(req) do |response|
|
|
449
371
|
result_http = response
|
|
450
372
|
result_mime = self.class.parse_header(result_http['Content-Type'] || Mime::TEXT)[:type]
|
|
451
|
-
Log.log.debug{"response: code=#{result_http.code}, mime=#{result_mime}, content-type=#{response['Content-Type']}"}
|
|
373
|
+
Log.log.debug { "response: code=#{result_http.code}, mime=#{result_mime}, content-type=#{response['Content-Type']}" }
|
|
452
374
|
# JSON data needs to be parsed, in case it contains an error code
|
|
453
|
-
|
|
454
|
-
result_http.code.to_s.start_with?('2') &&
|
|
455
|
-
!Mime.json?(result_mime)
|
|
456
|
-
total_size = result_http['Content-Length']&.to_i
|
|
457
|
-
Log.log.debug('before write file')
|
|
458
|
-
target_file = save_to_file
|
|
459
|
-
# override user's path to path in header
|
|
460
|
-
if !response['Content-Disposition'].nil?
|
|
461
|
-
disposition = self.class.parse_header(response['Content-Disposition'])
|
|
462
|
-
target_file = File.join(File.dirname(target_file), disposition[:parameters][:filename]) if disposition[:parameters].key?(:filename) && !disposition[:parameters][:filename].eql?('.')
|
|
463
|
-
end
|
|
464
|
-
# download with temp filename
|
|
465
|
-
target_file_tmp = "#{target_file}#{RestParameters.instance.download_partial_suffix}"
|
|
466
|
-
Log.log.debug{"saving to: #{target_file}"}
|
|
467
|
-
written_size = 0
|
|
468
|
-
session_id = SecureRandom.uuid.freeze
|
|
469
|
-
RestParameters.instance.progress_bar&.event(:session_start, session_id: session_id)
|
|
470
|
-
RestParameters.instance.progress_bar&.event(:session_size, session_id: session_id, info: total_size) if total_size
|
|
471
|
-
FileUtils.mkdir_p(File.dirname(target_file_tmp))
|
|
472
|
-
limiter = TimerLimiter.new(0.5)
|
|
473
|
-
File.open(target_file_tmp, 'wb') do |file|
|
|
474
|
-
result_http.read_body do |fragment|
|
|
475
|
-
file.write(fragment)
|
|
476
|
-
written_size += fragment.length
|
|
477
|
-
RestParameters.instance.progress_bar&.event(:transfer, session_id: session_id, info: written_size) if limiter.trigger?
|
|
478
|
-
end
|
|
479
|
-
end
|
|
480
|
-
RestParameters.instance.progress_bar&.event(:session_end, session_id: session_id)
|
|
481
|
-
RestParameters.instance.progress_bar&.event(:end)
|
|
482
|
-
# rename at the end
|
|
483
|
-
File.rename(target_file_tmp, target_file)
|
|
484
|
-
file_saved = true
|
|
485
|
-
end
|
|
375
|
+
file_saved = save_response(response, result_http, result_mime, save_to)
|
|
486
376
|
end
|
|
487
|
-
Log.log.debug{"result: code=#{result_http.code} mime=#{result_mime}"}
|
|
488
|
-
# sometimes there is a UTF8 char (e.g.
|
|
377
|
+
Log.log.debug { "result: code=#{result_http.code} mime=#{result_mime}" }
|
|
378
|
+
# sometimes there is a UTF8 char (e.g. (c) )
|
|
489
379
|
# TODO : related to mime type encoding ?
|
|
490
380
|
# result_http.body.force_encoding('UTF-8') if result_http.body.is_a?(String)
|
|
491
381
|
# Log.log.debug{"result: body=#{result_http.body}"}
|
|
492
|
-
result_data = result_http
|
|
493
|
-
Log.dump(:result_data_raw, result_data, level: :trace1)
|
|
494
|
-
# TODO: Remove next 2 lines when bug in async node api is fixed. (Aspera/core/issues/4490)
|
|
495
|
-
node_api_bug = result_data&.index('}HTTP/1.1 400 Bad Request') if result_data.is_a?(String)
|
|
496
|
-
result_data = result_data[0..node_api_bug] if node_api_bug
|
|
497
|
-
result_data = JSON.parse(result_data) if Mime.json?(result_mime) && !result_data.nil? && !result_data.empty?
|
|
498
|
-
Log.dump(:result_data, result_data)
|
|
382
|
+
result_data = parse_response(result_http, result_mime)
|
|
499
383
|
RestErrorAnalyzer.instance.raise_on_error(req, result_data, result_http)
|
|
500
|
-
unless file_saved ||
|
|
501
|
-
|
|
502
|
-
File.
|
|
384
|
+
unless file_saved || save_to.nil?
|
|
385
|
+
raise 'save_to: IO object requires a streaming response' if save_to.respond_to?(:write)
|
|
386
|
+
FileUtils.mkdir_p(File.dirname(save_to))
|
|
387
|
+
File.write(save_to, result_http.body, binmode: true)
|
|
503
388
|
end
|
|
504
389
|
rescue RestCallError => e
|
|
505
390
|
do_retry = false
|
|
@@ -520,7 +405,7 @@ module Aspera
|
|
|
520
405
|
# regenerate a brand new token
|
|
521
406
|
req['Authorization'] = oauth.authorization(cache: false)
|
|
522
407
|
end
|
|
523
|
-
Log.log.debug
|
|
408
|
+
Log.log.debug('using new token')
|
|
524
409
|
do_retry ||= true
|
|
525
410
|
end
|
|
526
411
|
if do_retry && (error_tries -= 1).positive?
|
|
@@ -535,7 +420,7 @@ module Aspera
|
|
|
535
420
|
# special case: relative redirect
|
|
536
421
|
if URI.parse(new_url).host.nil?
|
|
537
422
|
# we don't manage relative redirects with non-absolute path
|
|
538
|
-
Aspera.assert(new_url.start_with?('/')){"redirect location is relative: #{new_url}, but does not start with /."}
|
|
423
|
+
Aspera.assert(new_url.start_with?('/')) { "redirect location is relative: #{new_url}, but does not start with /." }
|
|
539
424
|
new_url = "#{current_uri.scheme}://#{current_uri.host}#{new_url}"
|
|
540
425
|
end
|
|
541
426
|
# forwards the request to the new location
|
|
@@ -548,7 +433,7 @@ module Aspera
|
|
|
548
433
|
query: query,
|
|
549
434
|
body: body,
|
|
550
435
|
content_type: content_type,
|
|
551
|
-
|
|
436
|
+
save_to: save_to,
|
|
552
437
|
exception: exception,
|
|
553
438
|
headers: headers,
|
|
554
439
|
ret: ret
|
|
@@ -557,15 +442,138 @@ module Aspera
|
|
|
557
442
|
# raise exception if could not retry and not return error in result
|
|
558
443
|
raise e if exception
|
|
559
444
|
end
|
|
560
|
-
Log.log.debug{"result=http:#{result_http}, data:#{result_data.class}"}
|
|
445
|
+
Log.log.debug { "result=http:#{result_http}, data:#{result_data.class}" }
|
|
561
446
|
return case ret
|
|
562
447
|
when :data then result_data
|
|
563
448
|
when :resp then result_http
|
|
564
449
|
when :both then [result_data, result_http]
|
|
565
|
-
else Aspera.error_unexpected_value(ret){'Type of result for REST'}
|
|
450
|
+
else Aspera.error_unexpected_value(ret) { 'Type of result for REST' }
|
|
566
451
|
end
|
|
567
452
|
end
|
|
568
453
|
|
|
454
|
+
private
|
|
455
|
+
|
|
456
|
+
def prepare_call(headers, query)
|
|
457
|
+
if headers.nil?
|
|
458
|
+
headers = @headers.clone
|
|
459
|
+
else
|
|
460
|
+
h = headers
|
|
461
|
+
headers = @headers.clone
|
|
462
|
+
headers.merge!(h)
|
|
463
|
+
end
|
|
464
|
+
Aspera.assert_type(headers, Hash)
|
|
465
|
+
case @auth_params[:type]
|
|
466
|
+
when :none
|
|
467
|
+
# no auth
|
|
468
|
+
when :basic
|
|
469
|
+
Log.log.debug('using Basic auth')
|
|
470
|
+
# done in build_req
|
|
471
|
+
when :oauth2
|
|
472
|
+
headers['Authorization'] = oauth.authorization unless headers.key?('Authorization')
|
|
473
|
+
when :url
|
|
474
|
+
query ||= {}
|
|
475
|
+
@auth_params[:url_query].each do |key, value|
|
|
476
|
+
query[key] = value
|
|
477
|
+
end
|
|
478
|
+
else Aspera.error_unexpected_value(@auth_params[:type])
|
|
479
|
+
end
|
|
480
|
+
[headers, query]
|
|
481
|
+
end
|
|
482
|
+
|
|
483
|
+
def build_request(operation, subpath, query, content_type, body, headers)
|
|
484
|
+
# TODO: shall we percent encode subpath (spaces) test with access key delete with space in id
|
|
485
|
+
# URI.escape()
|
|
486
|
+
separator = ['', '/'].include?(subpath) ? '' : '/'
|
|
487
|
+
uri = self.class.build_uri("#{@base_url}#{separator}#{subpath}", query)
|
|
488
|
+
Log.log.debug { "URI=#{uri}" }
|
|
489
|
+
begin
|
|
490
|
+
# instantiate request object based on string name
|
|
491
|
+
req = Net::HTTP.const_get(operation.capitalize).new(uri)
|
|
492
|
+
rescue NameError
|
|
493
|
+
raise "unsupported operation : #{operation}"
|
|
494
|
+
end
|
|
495
|
+
case content_type
|
|
496
|
+
when nil # ignore
|
|
497
|
+
when Mime::JSON
|
|
498
|
+
req.body = JSON.generate(body) # , ascii_only: true
|
|
499
|
+
req['Content-Type'] = Mime::JSON
|
|
500
|
+
when Mime::WWW
|
|
501
|
+
req.body = URI.encode_www_form(body)
|
|
502
|
+
req['Content-Type'] = Mime::WWW
|
|
503
|
+
when Mime::TEXT
|
|
504
|
+
req.body = body
|
|
505
|
+
req['Content-Type'] = Mime::TEXT
|
|
506
|
+
else Aspera.error_unexpected_value(content_type) { 'body type' }
|
|
507
|
+
end
|
|
508
|
+
# set headers
|
|
509
|
+
headers.each do |key, value|
|
|
510
|
+
req[key] = value
|
|
511
|
+
end
|
|
512
|
+
# :type = :basic
|
|
513
|
+
req.basic_auth(@auth_params[:username], @auth_params[:password]) if @auth_params[:type].eql?(:basic)
|
|
514
|
+
Log.dump(:req_body, req.body, level: :trace1)
|
|
515
|
+
req
|
|
516
|
+
end
|
|
517
|
+
|
|
518
|
+
def save_response(response, result_http, result_mime, save_to)
|
|
519
|
+
return false unless !save_to.nil? && result_http.code.to_s.start_with?('2') && !Mime.json?(result_mime)
|
|
520
|
+
|
|
521
|
+
total_size = result_http['Content-Length']&.to_i
|
|
522
|
+
Log.log.debug('before write file')
|
|
523
|
+
target_file = save_to
|
|
524
|
+
# override user's path to path in header
|
|
525
|
+
unless response['Content-Disposition'].nil?
|
|
526
|
+
disposition = self.class.parse_header(response['Content-Disposition'])
|
|
527
|
+
if disposition[:parameters].key?(:filename) && !disposition[:parameters][:filename].eql?('.')
|
|
528
|
+
# Use only the basename to prevent path traversal via a server-controlled Content-Disposition header
|
|
529
|
+
safe_filename = File.basename(disposition[:parameters][:filename])
|
|
530
|
+
target_file = File.join(File.dirname(target_file), safe_filename) unless safe_filename.empty?
|
|
531
|
+
end
|
|
532
|
+
end
|
|
533
|
+
Log.log.debug { "saving to: #{target_file}" }
|
|
534
|
+
written_size = 0
|
|
535
|
+
session_id = SecureRandom.uuid.freeze
|
|
536
|
+
RestParameters.instance.progress_bar&.event(:session_start, session_id: session_id)
|
|
537
|
+
RestParameters.instance.progress_bar&.event(:session_size, session_id: session_id, info: total_size) if total_size
|
|
538
|
+
limiter = TimerLimiter.new(0.5)
|
|
539
|
+
if target_file.respond_to?(:write)
|
|
540
|
+
# IO object: stream directly into it
|
|
541
|
+
result_http.read_body do |fragment|
|
|
542
|
+
target_file.write(fragment)
|
|
543
|
+
written_size += fragment.length
|
|
544
|
+
RestParameters.instance.progress_bar&.event(:transfer, session_id: session_id, info: written_size) if limiter.trigger?
|
|
545
|
+
end
|
|
546
|
+
else
|
|
547
|
+
# file path: download to partial name first, then rename atomically
|
|
548
|
+
target_file_tmp = "#{target_file}#{RestParameters.instance.download_partial_suffix}"
|
|
549
|
+
FileUtils.mkdir_p(File.dirname(target_file_tmp))
|
|
550
|
+
File.open(target_file_tmp, 'wb') do |file|
|
|
551
|
+
result_http.read_body do |fragment|
|
|
552
|
+
file.write(fragment)
|
|
553
|
+
written_size += fragment.length
|
|
554
|
+
RestParameters.instance.progress_bar&.event(:transfer, session_id: session_id, info: written_size) if limiter.trigger?
|
|
555
|
+
end
|
|
556
|
+
end
|
|
557
|
+
File.rename(target_file_tmp, target_file)
|
|
558
|
+
end
|
|
559
|
+
RestParameters.instance.progress_bar&.event(:session_end, session_id: session_id)
|
|
560
|
+
RestParameters.instance.progress_bar&.event(:end)
|
|
561
|
+
true
|
|
562
|
+
end
|
|
563
|
+
|
|
564
|
+
def parse_response(result_http, result_mime)
|
|
565
|
+
result_data = result_http.body
|
|
566
|
+
Log.dump(:result_data_raw, result_data, level: :trace1)
|
|
567
|
+
# TODO: Remove next 2 lines when bug in async node api is fixed. (Aspera/core/issues/4490)
|
|
568
|
+
node_api_bug = result_data&.index('}HTTP/1.1 400 Bad Request') if result_data.is_a?(String)
|
|
569
|
+
result_data = result_data[0..node_api_bug] if node_api_bug
|
|
570
|
+
result_data = JSON.parse(result_data) if Mime.json?(result_mime) && !result_data.nil? && !result_data.empty?
|
|
571
|
+
Log.dump(:result_data, result_data)
|
|
572
|
+
result_data
|
|
573
|
+
end
|
|
574
|
+
|
|
575
|
+
public
|
|
576
|
+
|
|
569
577
|
#
|
|
570
578
|
# CRUD simplified methods here
|
|
571
579
|
# If specific elements are needed, then use the full `call` method
|
|
@@ -32,11 +32,11 @@ module Aspera
|
|
|
32
32
|
# Use this method to analyze a EST result and raise an exception
|
|
33
33
|
# Analyzes REST call response and raises a RestCallError exception
|
|
34
34
|
# if HTTP result code is not 2XX
|
|
35
|
-
# @param req [Net::HTTPRequest]
|
|
36
|
-
# @param data [Object]
|
|
37
|
-
# @param http [Net::HTTPResponse]
|
|
35
|
+
# @param req [Net::HTTPRequest] the HTTP request
|
|
36
|
+
# @param data [Object] the parsed response body
|
|
37
|
+
# @param http [Net::HTTPResponse] the raw HTTP response
|
|
38
38
|
def raise_on_error(req, data, http)
|
|
39
|
-
Log.log.debug{"raise_on_error #{req.method} #{req.path} #{http.code}"}
|
|
39
|
+
Log.log.debug { "raise_on_error #{req.method} #{req.path} #{http.code}" }
|
|
40
40
|
context = {
|
|
41
41
|
messages: [],
|
|
42
42
|
request: req,
|
|
@@ -49,7 +49,7 @@ module Aspera
|
|
|
49
49
|
@error_handlers.each do |handler|
|
|
50
50
|
handler[:block].call(handler[:name], context)
|
|
51
51
|
rescue StandardError => e
|
|
52
|
-
Log.log.error{"ERROR in handler:\n#{e.message}\n#{e.backtrace}"}
|
|
52
|
+
Log.log.error { "ERROR in handler:\n#{e.message}\n#{e.backtrace}" }
|
|
53
53
|
end
|
|
54
54
|
raise RestCallError, context unless context[:messages].empty?
|
|
55
55
|
end
|
|
@@ -97,7 +97,7 @@ module Aspera
|
|
|
97
97
|
# @param message [String] one error message to add to list
|
|
98
98
|
def add_error(context, type, message)
|
|
99
99
|
context[:messages].push(message)
|
|
100
|
-
Log.log.trace1{"Found error: #{type}: #{message}"}
|
|
100
|
+
Log.log.trace1 { "Found error: #{type}: #{message}" }
|
|
101
101
|
log_file = instance.log_file
|
|
102
102
|
# log error for further analysis (file must exist to activate)
|
|
103
103
|
return if log_file.nil? || !File.exist?(log_file)
|
|
@@ -41,16 +41,6 @@ module Aspera
|
|
|
41
41
|
end
|
|
42
42
|
RestErrorAnalyzer.instance.add_simple_handler(name: 'T9:IBM cloud IAM', path: ['errorMessage'])
|
|
43
43
|
RestErrorAnalyzer.instance.add_simple_handler(name: 'T10:faspex v4', path: ['user_message'])
|
|
44
|
-
RestErrorAnalyzer.instance.add_handler('bss graphql') do |type, context|
|
|
45
|
-
next unless context[:data].is_a?(Hash)
|
|
46
|
-
d_t_s = context[:data]['errors']
|
|
47
|
-
next unless d_t_s.is_a?(Array)
|
|
48
|
-
d_t_s.each do |res|
|
|
49
|
-
r_err = res['message']
|
|
50
|
-
next unless r_err.is_a?(String)
|
|
51
|
-
RestErrorAnalyzer.add_error(context, type, r_err)
|
|
52
|
-
end
|
|
53
|
-
end
|
|
54
44
|
RestErrorAnalyzer.instance.add_handler('Orchestrator') do |type, context|
|
|
55
45
|
next if context[:response].code.start_with?('2')
|
|
56
46
|
data = context[:data]
|
data/lib/aspera/rest_list.rb
CHANGED
|
@@ -15,10 +15,10 @@ module Aspera
|
|
|
15
15
|
# @param value [String] Value of field of searched entity
|
|
16
16
|
# @param field [String] Field of searched entity
|
|
17
17
|
# @param query [Hash] Optional additional search query parameters
|
|
18
|
-
# @
|
|
18
|
+
# @return [Hash] A single entity matching the search, or an exception if not found or multiple found
|
|
19
19
|
def lookup_with_q(entity, value:, field: 'name', query: {})
|
|
20
|
-
Aspera.assert_type(query, Hash){'query'}
|
|
21
|
-
Aspera.assert_type(field, String){'field'}
|
|
20
|
+
Aspera.assert_type(query, Hash) { 'query' }
|
|
21
|
+
Aspera.assert_type(field, String) { 'field' }
|
|
22
22
|
# returns entities matching the query (it matches against several fields in case insensitive way)
|
|
23
23
|
# We don't do paging, as anyway, we look for only one match
|
|
24
24
|
matching_items = read(entity, query.merge({'q' => value}))
|
|
@@ -32,11 +32,11 @@ module Aspera
|
|
|
32
32
|
else
|
|
33
33
|
# multiple case insensitive partial matches, try case insensitive full match
|
|
34
34
|
# (anyway AoC does not allow creation of 2 entities with same case insensitive field value)
|
|
35
|
-
value_matches = matching_items.select{ |i| i[field].casecmp?(value)}
|
|
35
|
+
value_matches = matching_items.select { |i| i[field].casecmp?(value) }
|
|
36
36
|
case value_matches.length
|
|
37
37
|
when 1 then return value_matches.first
|
|
38
|
-
when 0 then raise
|
|
39
|
-
else raise "Two entities cannot have the same case insensitive #{field}: #{value_matches.map{ |i| i[field]}}"
|
|
38
|
+
when 0 then raise Error, "#{entity}: Multiple case insensitive partial match for: \"#{value}\" in #{matching_items.map { |i| i[field] }.join(', ')} but no case insensitive full match. Please be more specific or give exact #{field}."
|
|
39
|
+
else raise Error, "Two entities cannot have the same case insensitive #{field}: #{value_matches.map { |i| i[field] }}"
|
|
40
40
|
end
|
|
41
41
|
end
|
|
42
42
|
end
|
|
@@ -60,7 +60,8 @@ module Aspera
|
|
|
60
60
|
Aspera.assert_type(entity, String)
|
|
61
61
|
Aspera.assert_type(items_key, String)
|
|
62
62
|
Aspera.assert_type(query, Hash)
|
|
63
|
-
Log.log.debug{"list_entities t=#{entity} k=#{items_key}
|
|
63
|
+
Log.log.debug { "list_entities t=#{entity} k=#{items_key}" }
|
|
64
|
+
Log.dump(:query, query)
|
|
64
65
|
result = []
|
|
65
66
|
offset = 0
|
|
66
67
|
max_items = query.delete(MAX_ITEMS)
|
|
@@ -104,10 +105,10 @@ module Aspera
|
|
|
104
105
|
# @param query [Hash] Additional query parameters (Default: `:default`)
|
|
105
106
|
def lookup_entity_by_field(entity:, value:, field: 'name', items_key: nil, query: :default)
|
|
106
107
|
if query.eql?(:default)
|
|
107
|
-
Aspera.assert_values(field, ['name']){'Default query field'}
|
|
108
|
+
Aspera.assert_values(field, ['name']) { 'Default query field' }
|
|
108
109
|
query = {'q'=> value}
|
|
109
110
|
end
|
|
110
|
-
lookup_entity_generic(entity: entity, field: field, value: value){list_entities_limit_offset_total_count(entity: entity, items_key: items_key, query: query).first}
|
|
111
|
+
lookup_entity_generic(entity: entity, field: field, value: value) { list_entities_limit_offset_total_count(entity: entity, items_key: items_key, query: query).first }
|
|
111
112
|
end
|
|
112
113
|
|
|
113
114
|
# Lookup entity by field and value.
|
|
@@ -123,7 +124,7 @@ module Aspera
|
|
|
123
124
|
Aspera.assert(block_given?, 'block required for lookup_entity_generic')
|
|
124
125
|
found = yield
|
|
125
126
|
Aspera.assert_array_all(found, Hash)
|
|
126
|
-
found = found.select{ |i| i[field].eql?(value)}
|
|
127
|
+
found = found.select { |i| i[field].eql?(value) }
|
|
127
128
|
return found.first if found.length.eql?(1)
|
|
128
129
|
raise Cli::BadIdentifier.new(entity, value, field: field, count: found.length)
|
|
129
130
|
end
|