aspera-cli 4.27.1 → 4.27.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/CHANGELOG.md +67 -1
- data/bin/ascli +2 -1
- data/docs/README.md +805 -747
- data/lib/aspera/agent/connect.rb +6 -4
- data/lib/aspera/agent/desktop.rb +2 -2
- data/lib/aspera/agent/direct.rb +3 -1
- data/lib/aspera/agent/node.rb +3 -3
- data/lib/aspera/api/alee.rb +1 -1
- data/lib/aspera/api/aoc.rb +14 -12
- data/lib/aspera/api/ats.rb +1 -1
- data/lib/aspera/api/cos_node.rb +2 -2
- data/lib/aspera/api/faspex.rb +9 -7
- data/lib/aspera/api/httpgw.rb +37 -33
- data/lib/aspera/api/node.rb +38 -33
- data/lib/aspera/ascmd.rb +3 -1
- data/lib/aspera/ascp/installation.rb +62 -27
- data/lib/aspera/ascp/management.rb +1 -0
- data/lib/aspera/assert.rb +4 -0
- data/lib/aspera/cli/ascp_actions.rb +20 -41
- data/lib/aspera/cli/async_transfer_store.rb +2 -2
- data/lib/aspera/cli/bootstrapper.rb +11 -15
- data/lib/aspera/cli/command_line.rb +252 -0
- data/lib/aspera/cli/command_registry.rb +149 -33
- data/lib/aspera/cli/command_spec.rb +103 -14
- data/lib/aspera/cli/completion/ascli.bash +12 -0
- data/lib/aspera/cli/completion/ascli.fish +16 -0
- data/lib/aspera/cli/completion/ascli.zsh +19 -0
- data/lib/aspera/cli/context.rb +3 -0
- data/lib/aspera/cli/deprecation.rb +37 -0
- data/lib/aspera/cli/extended_value.rb +2 -0
- data/lib/aspera/cli/formatter.rb +87 -75
- data/lib/aspera/cli/gem_checker.rb +1 -1
- data/lib/aspera/cli/hints.rb +7 -6
- data/lib/aspera/cli/http.rb +21 -21
- data/lib/aspera/cli/info.rb +3 -0
- data/lib/aspera/cli/mcp_tool.rb +47 -83
- data/lib/aspera/cli/option_declarator.rb +33 -42
- data/lib/aspera/cli/option_registry.rb +69 -0
- data/lib/aspera/cli/option_types.rb +103 -0
- data/lib/aspera/cli/option_value.rb +281 -0
- data/lib/aspera/cli/options.schema.yaml +38 -5
- data/lib/aspera/cli/parser.rb +307 -848
- data/lib/aspera/cli/plugins/alee.rb +7 -4
- data/lib/aspera/cli/plugins/aoc.rb +435 -380
- data/lib/aspera/cli/plugins/ats.rb +58 -73
- data/lib/aspera/cli/plugins/base.rb +190 -240
- data/lib/aspera/cli/plugins/basic_auth.rb +2 -10
- data/lib/aspera/cli/plugins/config.rb +244 -178
- data/lib/aspera/cli/plugins/console.rb +102 -38
- data/lib/aspera/cli/plugins/cos.rb +6 -23
- data/lib/aspera/cli/plugins/factory.rb +3 -0
- data/lib/aspera/cli/plugins/faspex5.rb +176 -173
- data/lib/aspera/cli/plugins/faspio.rb +5 -10
- data/lib/aspera/cli/plugins/httpgw.rb +8 -11
- data/lib/aspera/cli/plugins/mcp.rb +20 -55
- data/lib/aspera/cli/plugins/node.rb +277 -311
- data/lib/aspera/cli/plugins/orchestrator.rb +90 -77
- data/lib/aspera/cli/plugins/preview.rb +79 -90
- data/lib/aspera/cli/plugins/server.rb +76 -50
- data/lib/aspera/cli/plugins/shares.rb +68 -116
- data/lib/aspera/cli/preset_actions.rb +17 -10
- data/lib/aspera/cli/preset_manager.rb +12 -2
- data/lib/aspera/cli/prompt.rb +35 -0
- data/lib/aspera/cli/result.rb +13 -18
- data/lib/aspera/cli/runner.rb +31 -54
- data/lib/aspera/cli/special_values.rb +5 -0
- data/lib/aspera/cli/sync_actions.rb +41 -37
- data/lib/aspera/cli/terminal_formatter.rb +9 -3
- data/lib/aspera/cli/transfer_actions.rb +0 -6
- data/lib/aspera/cli/transfer_agent.rb +29 -35
- data/lib/aspera/cli/vault_manager.rb +0 -17
- data/lib/aspera/cli/version.rb +1 -1
- data/lib/aspera/cli/wizard.rb +4 -2
- data/lib/aspera/command_line_builder.rb +1 -0
- data/lib/aspera/coverage.rb +1 -0
- data/lib/aspera/environment.rb +7 -1
- data/lib/aspera/faspex_gw.rb +2 -1
- data/lib/aspera/faspex_postproc.rb +1 -0
- data/lib/aspera/graphql.rb +5 -5
- data/lib/aspera/json_rpc/client.rb +5 -5
- data/lib/aspera/keychain/encrypted_hash.rb +1 -1
- data/lib/aspera/keychain/one_password_api.rb +1 -1
- data/lib/aspera/link_header.rb +2 -2
- data/lib/aspera/log.rb +22 -25
- data/lib/aspera/markdown.rb +2 -0
- data/lib/aspera/mime.rb +25 -0
- data/lib/aspera/node_simulator.rb +1 -0
- data/lib/aspera/oauth/base.rb +35 -25
- data/lib/aspera/oauth/factory.rb +1 -0
- data/lib/aspera/oauth/generic.rb +1 -1
- data/lib/aspera/oauth/jwt.rb +1 -1
- data/lib/aspera/oauth/web.rb +9 -8
- data/lib/aspera/preview/file_types.rb +4 -4
- data/lib/aspera/preview/generator.rb +7 -0
- data/lib/aspera/preview/options.rb +4 -4
- data/lib/aspera/preview/terminal.rb +4 -3
- data/lib/aspera/preview/utils.rb +9 -6
- data/lib/aspera/products/connect.rb +1 -1
- data/lib/aspera/rainbow.rb +7 -0
- data/lib/aspera/rest/aspera_errors.rb +60 -0
- data/lib/aspera/rest/call_error.rb +27 -0
- data/lib/aspera/rest/client.rb +514 -0
- data/lib/aspera/rest/error_analyzer.rb +113 -0
- data/lib/aspera/rest/list.rb +143 -0
- data/lib/aspera/rest/parameters.rb +55 -0
- data/lib/aspera/rest/util.rb +176 -0
- data/lib/aspera/rest.rb +7 -621
- data/lib/aspera/schema/IBM Aspera Console-enhanced.yaml +1125 -0
- data/lib/aspera/schema/IBM Aspera on Cloud API-0.2.6-enhanced.yaml +39 -0
- data/lib/aspera/schema/IBM Aspera on Cloud Automation API-1.0.5-enhanced.yaml +2395 -0
- data/lib/aspera/schema/documentation.rb +13 -3
- data/lib/aspera/schema/registry.rb +18 -1
- data/lib/aspera/schema/validator.rb +92 -0
- data/lib/aspera/secret_hider.rb +36 -25
- data/lib/aspera/string_ext.rb +15 -0
- data/lib/aspera/temp_file_manager.rb +6 -5
- data/lib/aspera/transfer/parameters.rb +2 -0
- data/lib/aspera/transfer/spec.rb +1 -0
- data/lib/aspera/transfer/spec.schema.yaml +1 -0
- data/lib/aspera/uri_reader.rb +11 -11
- data/lib/aspera/web_auth/index.html +147 -0
- data/lib/aspera/web_auth/server.rb +81 -0
- data.tar.gz.sig +0 -0
- metadata +39 -7
- metadata.gz.sig +0 -0
- data/lib/aspera/colors.rb +0 -79
- data/lib/aspera/rest_call_error.rb +0 -25
- data/lib/aspera/rest_error_analyzer.rb +0 -111
- data/lib/aspera/rest_errors_aspera.rb +0 -58
- data/lib/aspera/rest_list.rb +0 -136
- data/lib/aspera/web_auth.rb +0 -211
data/lib/aspera/rest.rb
CHANGED
|
@@ -1,623 +1,9 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
require 'aspera/
|
|
4
|
-
require 'aspera/
|
|
5
|
-
require 'aspera/
|
|
6
|
-
require 'aspera/
|
|
7
|
-
require 'aspera/
|
|
8
|
-
require 'aspera/
|
|
9
|
-
require 'aspera/
|
|
10
|
-
require 'net/http'
|
|
11
|
-
require 'net/https'
|
|
12
|
-
require 'json'
|
|
13
|
-
require 'base64'
|
|
14
|
-
require 'singleton'
|
|
15
|
-
require 'securerandom'
|
|
16
|
-
require 'fileutils'
|
|
17
|
-
|
|
18
|
-
# Cancel method for HTTP
|
|
19
|
-
class Net::HTTP::Cancel < Net::HTTPRequest # rubocop:disable Style/ClassAndModuleChildren
|
|
20
|
-
METHOD = 'CANCEL'
|
|
21
|
-
REQUEST_HAS_BODY = false
|
|
22
|
-
RESPONSE_HAS_BODY = false
|
|
23
|
-
end
|
|
24
|
-
|
|
25
|
-
module Aspera
|
|
26
|
-
# Global settings for Rest object
|
|
27
|
-
# For example to remove certificate verification globally:
|
|
28
|
-
# `RestParameters.instance.session_cb = lambda{|http|http.verify_mode=OpenSSL::SSL::VERIFY_NONE}`
|
|
29
|
-
# @param user_agent [String] HTTP request header: 'User-Agent'
|
|
30
|
-
# @param download_partial_suffix [String] suffix for partial download
|
|
31
|
-
# @param session_cb [lambda] lambda called on new HTTP session. Takes the Net::HTTP as arg. Used to change parameters on creation.
|
|
32
|
-
# @param progress_bar [Object] progress bar object called for file transfer
|
|
33
|
-
#
|
|
34
|
-
# @!method self.instance
|
|
35
|
-
# Returns the singleton instance of RestParameters
|
|
36
|
-
# @return [RestParameters] the singleton instance
|
|
37
|
-
class RestParameters
|
|
38
|
-
include Singleton
|
|
39
|
-
|
|
40
|
-
attr_accessor :user_agent, :download_partial_suffix, :retry_on_error, :retry_on_timeout, :retry_on_unavailable, :retry_max, :retry_sleep, :session_cb, :progress_bar, :spinner_cb
|
|
41
|
-
|
|
42
|
-
private
|
|
43
|
-
|
|
44
|
-
def initialize
|
|
45
|
-
@user_agent = 'RubyAsperaRest'
|
|
46
|
-
@download_partial_suffix = '.http_partial'
|
|
47
|
-
@retry_on_error = false
|
|
48
|
-
@retry_on_timeout = true
|
|
49
|
-
@retry_on_unavailable = true
|
|
50
|
-
@retry_max = 1
|
|
51
|
-
@retry_sleep = 4
|
|
52
|
-
@session_cb = nil
|
|
53
|
-
@progress_bar = nil
|
|
54
|
-
@spinner_cb = nil
|
|
55
|
-
end
|
|
56
|
-
end
|
|
57
|
-
|
|
58
|
-
class EntityNotFound < Error
|
|
59
|
-
end
|
|
60
|
-
|
|
61
|
-
module Mime
|
|
62
|
-
JSON = 'application/json'
|
|
63
|
-
WWW = 'application/x-www-form-urlencoded'
|
|
64
|
-
TEXT = 'text/plain'
|
|
65
|
-
# Check if a MIME type is JSON, including parameters, e.g. application/json; charset=utf-8
|
|
66
|
-
def json?(mime)
|
|
67
|
-
JSON_LIST.include?(mime)
|
|
68
|
-
end
|
|
69
|
-
module_function :json?
|
|
70
|
-
# Content-Type that are JSON
|
|
71
|
-
JSON_LIST = [JSON, 'application/vnd.api+json', 'application/x-javascript'].freeze
|
|
72
|
-
private_constant :JSON_LIST
|
|
73
|
-
end
|
|
74
|
-
|
|
75
|
-
# Make HTTP calls, equivalent to rest-client
|
|
76
|
-
# rest call errors are raised as exception RestCallError
|
|
77
|
-
# and error are analyzed in RestErrorAnalyzer
|
|
78
|
-
class Rest
|
|
79
|
-
class << self
|
|
80
|
-
# Build a Basic authentication header value
|
|
81
|
-
# @param user [String] Username
|
|
82
|
-
# @param pass [String] Password
|
|
83
|
-
# @return [String] Basic auth token
|
|
84
|
-
def basic_authorization(user, pass); return "Basic #{Base64.strict_encode64("#{user}:#{pass}")}"; end
|
|
85
|
-
|
|
86
|
-
# Indicate that the given Hash query uses php style for array parameters
|
|
87
|
-
# @param query [Hash] A key can have Array value and result will use PHP format: a[]=1&a[]=2
|
|
88
|
-
# @return [Hash] The query parameters.
|
|
89
|
-
def php_style(query)
|
|
90
|
-
Aspera.assert_type(query, Hash) { 'query' }
|
|
91
|
-
query[:x_array_php_style] = true
|
|
92
|
-
query
|
|
93
|
-
end
|
|
94
|
-
|
|
95
|
-
# Build URI from URL and parameters and check it is `http` or `https`.
|
|
96
|
-
# Check if php style is specified.
|
|
97
|
-
# `nil` values in query result in key without value, e.g. `?a`, while empty string values result in `?a=`.
|
|
98
|
-
# @param url [String] The URL without query.
|
|
99
|
-
# @param query [Hash,Array,String] The query parameters.
|
|
100
|
-
# @return [URI] The built URI.
|
|
101
|
-
def build_uri(url, query)
|
|
102
|
-
uri = URI.parse(url)
|
|
103
|
-
Aspera.assert_values(uri.scheme, %w[http https]) { 'URI scheme' }
|
|
104
|
-
return uri if query.nil? || query.respond_to?(:empty?) && query.empty?
|
|
105
|
-
Log.dump(:query, query)
|
|
106
|
-
uri.query =
|
|
107
|
-
case query
|
|
108
|
-
when String
|
|
109
|
-
query
|
|
110
|
-
when Hash
|
|
111
|
-
URI.encode_www_form(h_to_query_array(query))
|
|
112
|
-
when Array
|
|
113
|
-
Aspera.assert(query.all? { |i| i.is_a?(Array) && i.length.eql?(2) }, 'Query must be array of arrays of 2 elements')
|
|
114
|
-
URI.encode_www_form(query) # remove nil values
|
|
115
|
-
else Aspera.error_unexpected_value(query.class) { 'query type' }
|
|
116
|
-
end.gsub('%5B%5D=', '[]=')
|
|
117
|
-
# [] is allowed in url parameters
|
|
118
|
-
uri
|
|
119
|
-
end
|
|
120
|
-
|
|
121
|
-
# Support array for query parameter, there is no standard.
|
|
122
|
-
# Either p=1&p=2 (default)
|
|
123
|
-
# or p[]=1&p[]=2 (if `:x_array_php_style` is set to true in query)
|
|
124
|
-
# @param query [Hash] HTTP query as hash
|
|
125
|
-
# @return [Array<Array>] Array of [key, value] pairs suitable for URI.encode_www_form
|
|
126
|
-
def h_to_query_array(query)
|
|
127
|
-
Aspera.assert_type(query, Hash)
|
|
128
|
-
suffix = query.delete(:x_array_php_style) ? '[]' : nil
|
|
129
|
-
query.each_with_object([]) do |(k, v), query_array|
|
|
130
|
-
case v
|
|
131
|
-
when Array
|
|
132
|
-
v.each do |e|
|
|
133
|
-
query_array.push(["#{k}#{suffix}", e])
|
|
134
|
-
end
|
|
135
|
-
else
|
|
136
|
-
query_array.push([k, v])
|
|
137
|
-
end
|
|
138
|
-
end
|
|
139
|
-
end
|
|
140
|
-
|
|
141
|
-
# Decode query string as Hash
|
|
142
|
-
# if parameter is only once, then it's a scalar
|
|
143
|
-
# if a parameter is several, then it's array
|
|
144
|
-
# if parameter has [] then it's an array, and [] is removed
|
|
145
|
-
# Support arrays in query string, e.g. PHP's way is p[]=1&p[]=2
|
|
146
|
-
# @param query [String] query string as in URI.query
|
|
147
|
-
# @return [Hash] decoded query
|
|
148
|
-
def query_to_h(query)
|
|
149
|
-
URI.decode_www_form(query).each_with_object({}) do |(key, value), h|
|
|
150
|
-
if key.end_with?('[]')
|
|
151
|
-
key = key[..-3]
|
|
152
|
-
h[key] = [] unless h.key?(key)
|
|
153
|
-
end
|
|
154
|
-
if h.key?(key)
|
|
155
|
-
h[key] = [h[key]] if !h[key].is_a?(Array)
|
|
156
|
-
h[key].push(value)
|
|
157
|
-
else
|
|
158
|
-
h[key] = value
|
|
159
|
-
end
|
|
160
|
-
end
|
|
161
|
-
end
|
|
162
|
-
|
|
163
|
-
# Start a HTTP/S session, also used for web sockets
|
|
164
|
-
# @param base_url [String] Base url of HTTP/S session
|
|
165
|
-
# @return [Net::HTTP] A started HTTP session
|
|
166
|
-
def start_http_session(base_url)
|
|
167
|
-
uri = URI.parse(base_url)
|
|
168
|
-
Aspera.assert_values(uri.scheme, %w[http https]) { 'URI scheme' }
|
|
169
|
-
# This honors http_proxy env var
|
|
170
|
-
http_session = Net::HTTP.new(uri.host, uri.port)
|
|
171
|
-
http_session.use_ssl = uri.scheme.eql?('https')
|
|
172
|
-
# Set http options in callback, such as timeout and cert. verification
|
|
173
|
-
RestParameters.instance.session_cb&.call(http_session)
|
|
174
|
-
# Manually start session for keep alive (if supported by server, else, session is closed every time)
|
|
175
|
-
http_session.start
|
|
176
|
-
return http_session
|
|
177
|
-
end
|
|
178
|
-
|
|
179
|
-
# get Net::HTTP underlying socket i/o
|
|
180
|
-
# little hack, handy because HTTP debug, proxy, etc... will be available
|
|
181
|
-
# used implement web sockets after `start_http_session`
|
|
182
|
-
# @param http_session [Net::HTTP] the session object
|
|
183
|
-
# @return [Net::BufferedIO] The underlying socket i/o
|
|
184
|
-
def io_http_session(http_session)
|
|
185
|
-
Aspera.assert_type(http_session, Net::HTTP)
|
|
186
|
-
# Net::BufferedIO in net/protocol.rb
|
|
187
|
-
result = http_session.instance_variable_get(:@socket)
|
|
188
|
-
Aspera.assert(!result.nil?) { "no socket for #{http_session}" }
|
|
189
|
-
return result
|
|
190
|
-
end
|
|
191
|
-
|
|
192
|
-
# @return [String] PEM certificates of remote server
|
|
193
|
-
def remote_certificate_chain(url, as_string: true)
|
|
194
|
-
result = []
|
|
195
|
-
# initiate a session to retrieve remote certificate
|
|
196
|
-
http_session = Rest.start_http_session(url)
|
|
197
|
-
begin
|
|
198
|
-
# retrieve underlying openssl socket
|
|
199
|
-
result = Rest.io_http_session(http_session).io.peer_cert_chain
|
|
200
|
-
rescue
|
|
201
|
-
result = http_session.peer_cert
|
|
202
|
-
ensure
|
|
203
|
-
http_session.finish
|
|
204
|
-
end
|
|
205
|
-
result = result.map(&:to_pem).join("\n") if as_string
|
|
206
|
-
return result
|
|
207
|
-
end
|
|
208
|
-
|
|
209
|
-
# Parses an HTTP Content-Type header string into its media type and parameters
|
|
210
|
-
# according to RFC 9110 and RFC 6838.
|
|
211
|
-
# TODO: use gem: content_type
|
|
212
|
-
#
|
|
213
|
-
# @param header [String] The Content-Type header string, e.g., "application/json; charset=utf-8"
|
|
214
|
-
# @return [Hash] A hash with :type and :parameters keys.
|
|
215
|
-
# Example:
|
|
216
|
-
# {
|
|
217
|
-
# type: "application/json",
|
|
218
|
-
# parameters: {
|
|
219
|
-
# charset: "utf-8",
|
|
220
|
-
# version: "1.0"
|
|
221
|
-
# }
|
|
222
|
-
# }
|
|
223
|
-
def parse_header(header)
|
|
224
|
-
parts = header.split(';').map(&:strip)
|
|
225
|
-
media_type = parts.shift.downcase
|
|
226
|
-
parameters = parts.filter_map do |param|
|
|
227
|
-
key, value = param.split('=', 2)
|
|
228
|
-
next unless key && value
|
|
229
|
-
key = key.strip.downcase.to_sym
|
|
230
|
-
value = value.strip.gsub(/\A"|"\z/, '')
|
|
231
|
-
[key, value]
|
|
232
|
-
end.to_h
|
|
233
|
-
{type: media_type, parameters: parameters}
|
|
234
|
-
end
|
|
235
|
-
end
|
|
236
|
-
|
|
237
|
-
private
|
|
238
|
-
|
|
239
|
-
# create and start keep alive connection on demand
|
|
240
|
-
def http_session
|
|
241
|
-
@http_session = self.class.start_http_session(@base_url) if @http_session.nil?
|
|
242
|
-
return @http_session
|
|
243
|
-
end
|
|
244
|
-
|
|
245
|
-
public
|
|
246
|
-
|
|
247
|
-
# All original constructor parameters
|
|
248
|
-
attr_reader :auth_params
|
|
249
|
-
|
|
250
|
-
# The root URL for the API
|
|
251
|
-
attr_reader :base_url
|
|
252
|
-
|
|
253
|
-
# Base common headers of API
|
|
254
|
-
attr_reader :headers
|
|
255
|
-
|
|
256
|
-
# @return [Hash] creation parameters
|
|
257
|
-
def params
|
|
258
|
-
return {
|
|
259
|
-
base_url: @base_url, # String
|
|
260
|
-
auth: @auth_params, # Hash
|
|
261
|
-
not_auth_codes: @not_auth_codes, # Array
|
|
262
|
-
redirect_max: @redirect_max, # Integer
|
|
263
|
-
headers: @headers # Hash
|
|
264
|
-
}
|
|
265
|
-
end
|
|
266
|
-
|
|
267
|
-
# Create a REST object for API calls
|
|
268
|
-
# HTTP sessions parameters can be modified using global parameters in RestParameters
|
|
269
|
-
# For example, TLS verification can be skipped.
|
|
270
|
-
# @param base_url [String] base URL of REST API
|
|
271
|
-
# @param auth [Hash] authentication parameters:
|
|
272
|
-
# :type (:none, :basic, :url, :oauth2)
|
|
273
|
-
# :username [:basic]
|
|
274
|
-
# :password [:basic]
|
|
275
|
-
# :url_query [:url] a hash
|
|
276
|
-
# :* [:oauth2] see OAuth::Factory class
|
|
277
|
-
# @param not_auth_codes [Array] codes that trigger a refresh/regeneration of bearer token
|
|
278
|
-
# @param redirect_max [Integer] max redirection allowed
|
|
279
|
-
# @param headers [Hash] default headers to include in all calls
|
|
280
|
-
def initialize(
|
|
281
|
-
base_url:,
|
|
282
|
-
auth: {type: :none},
|
|
283
|
-
not_auth_codes: ['401'],
|
|
284
|
-
redirect_max: 0,
|
|
285
|
-
headers: {}
|
|
286
|
-
)
|
|
287
|
-
Aspera.assert_type(base_url, String)
|
|
288
|
-
# base url with no trailing slashes (note: string may be frozen)
|
|
289
|
-
@base_url = base_url.chomp('/')
|
|
290
|
-
# remove trailing port if it is 443 and scheme is https
|
|
291
|
-
@base_url = @base_url.gsub(/:443$/, '') if @base_url.start_with?('https://')
|
|
292
|
-
@base_url = @base_url.gsub(/:80$/, '') if @base_url.start_with?('http://')
|
|
293
|
-
Log.log.debug { "Rest.new(#{@base_url})" }
|
|
294
|
-
# default is no auth
|
|
295
|
-
@auth_params = auth
|
|
296
|
-
Aspera.assert_type(@auth_params, Hash)
|
|
297
|
-
Aspera.assert(@auth_params.key?(:type), 'no auth type defined')
|
|
298
|
-
@not_auth_codes = not_auth_codes
|
|
299
|
-
Aspera.assert_type(@not_auth_codes, Array)
|
|
300
|
-
# persistent session
|
|
301
|
-
@http_session = nil
|
|
302
|
-
@redirect_max = redirect_max
|
|
303
|
-
Aspera.assert_type(@redirect_max, Integer)
|
|
304
|
-
@headers = headers.clone
|
|
305
|
-
Aspera.assert_type(@headers, Hash)
|
|
306
|
-
@headers['User-Agent'] ||= RestParameters.instance.user_agent
|
|
307
|
-
# OAuth object (created on demand)
|
|
308
|
-
@oauth = nil
|
|
309
|
-
end
|
|
310
|
-
|
|
311
|
-
# @return [OAuth::Base] the OAuth object (create, or cached if already created)
|
|
312
|
-
def oauth
|
|
313
|
-
if @oauth.nil?
|
|
314
|
-
Aspera.assert(@auth_params[:type].eql?(:oauth2), 'no OAuth defined')
|
|
315
|
-
oauth_parameters = @auth_params.reject { |k, _v| k.eql?(:type) }
|
|
316
|
-
Log.dump(:oauth_parameters, oauth_parameters)
|
|
317
|
-
@oauth = OAuth::Factory.instance.create(**oauth_parameters)
|
|
318
|
-
end
|
|
319
|
-
return @oauth
|
|
320
|
-
end
|
|
321
|
-
|
|
322
|
-
# HTTP/S REST call
|
|
323
|
-
# @param operation [String] HTTP operation (GET, POST, PUT, DELETE)
|
|
324
|
-
# @param subpath [String] subpath of REST API
|
|
325
|
-
# @param query [Hash{String,Symbol => Object}] URL parameters
|
|
326
|
-
# @param content_type [String, nil] Type of body parameters (one of MIME_*) and serialization, else use headers
|
|
327
|
-
# @param body [Hash, String, nil] Body parameters
|
|
328
|
-
# @param headers [Hash{String => String}] Additional headers (override Content-Type)
|
|
329
|
-
# @param save_to [String, IO, nil] File path or IO object to save response body; progress bar is used when set
|
|
330
|
-
# @param exception [Boolean] Whether to raise an exception on HTTP error
|
|
331
|
-
# @param ret [Symbol] One of :data, :resp, :both - controls return value
|
|
332
|
-
# @return [Array(Hash, Net::HTTPResponse)] When `ret` is :both
|
|
333
|
-
# @return [Net::HTTPResponse] When `ret` is :resp
|
|
334
|
-
# @return [Hash] When `ret` is :data
|
|
335
|
-
# @raise [RestCallError] on error if `exception` is true
|
|
336
|
-
def call(
|
|
337
|
-
operation:,
|
|
338
|
-
subpath: nil,
|
|
339
|
-
query: nil,
|
|
340
|
-
content_type: nil,
|
|
341
|
-
body: nil,
|
|
342
|
-
headers: nil,
|
|
343
|
-
save_to: nil,
|
|
344
|
-
exception: true,
|
|
345
|
-
ret: :data
|
|
346
|
-
)
|
|
347
|
-
subpath = subpath.to_s if subpath.is_a?(Symbol)
|
|
348
|
-
subpath = '' if subpath.nil?
|
|
349
|
-
Log.log.debug { "call #{operation} [#{subpath}]".red.bold.bg_green }
|
|
350
|
-
Log.dump(:body, body, level: :trace1)
|
|
351
|
-
Log.dump(:query, query, level: :trace1)
|
|
352
|
-
Log.dump(:headers, headers, level: :trace1)
|
|
353
|
-
Aspera.assert_type(subpath, String)
|
|
354
|
-
# We must have a way to check return code
|
|
355
|
-
Aspera.assert(exception || !ret.eql?(:data), 'ret: :data requires exception handler')
|
|
356
|
-
headers, query = prepare_call(headers, query)
|
|
357
|
-
result_http = nil
|
|
358
|
-
result_data = nil
|
|
359
|
-
# initialize with number of initial retries allowed, nil gives zero
|
|
360
|
-
tries_remain_redirect = @redirect_max
|
|
361
|
-
# start a block to be able to retry the actual HTTP request in case of OAuth token expiration
|
|
362
|
-
begin
|
|
363
|
-
Log.log.debug("send request (retries=#{tries_remain_redirect})")
|
|
364
|
-
req = build_request(operation, subpath, query, content_type, body, headers)
|
|
365
|
-
# we try the call, and will retry on some error types
|
|
366
|
-
error_tries ||= 1 + RestParameters.instance.retry_max
|
|
367
|
-
result_mime = nil
|
|
368
|
-
file_saved = false
|
|
369
|
-
# make http request (pipelined)
|
|
370
|
-
http_session.request(req) do |response|
|
|
371
|
-
result_http = response
|
|
372
|
-
result_mime = self.class.parse_header(result_http['Content-Type'] || Mime::TEXT)[:type]
|
|
373
|
-
Log.log.debug { "response: code=#{result_http.code}, mime=#{result_mime}, content-type=#{response['Content-Type']}" }
|
|
374
|
-
# JSON data needs to be parsed, in case it contains an error code
|
|
375
|
-
file_saved = save_response(response, result_http, result_mime, save_to)
|
|
376
|
-
end
|
|
377
|
-
Log.log.debug { "result: code=#{result_http.code} mime=#{result_mime}" }
|
|
378
|
-
# sometimes there is a UTF8 char (e.g. (c) )
|
|
379
|
-
# TODO : related to mime type encoding ?
|
|
380
|
-
# result_http.body.force_encoding('UTF-8') if result_http.body.is_a?(String)
|
|
381
|
-
# Log.log.debug{"result: body=#{result_http.body}"}
|
|
382
|
-
result_data = parse_response(result_http, result_mime)
|
|
383
|
-
RestErrorAnalyzer.instance.raise_on_error(req, result_data, result_http)
|
|
384
|
-
unless file_saved || save_to.nil?
|
|
385
|
-
raise 'save_to: IO object requires a streaming response' if save_to.respond_to?(:write)
|
|
386
|
-
FileUtils.mkdir_p(File.dirname(save_to))
|
|
387
|
-
File.write(save_to, result_http.body, binmode: true)
|
|
388
|
-
end
|
|
389
|
-
rescue RestCallError => e
|
|
390
|
-
do_retry = false
|
|
391
|
-
# AoC have some timeout , like Connect to platform.bss.asperasoft.com:443 ...
|
|
392
|
-
do_retry ||= true if e.response.body.include?('failed: connect timed out') && RestParameters.instance.retry_on_timeout
|
|
393
|
-
# AoC sometimes not available
|
|
394
|
-
do_retry ||= true if RestParameters.instance.retry_on_unavailable && UNAVAILABLE_CODES.include?(result_http.code.to_s)
|
|
395
|
-
# possibility to retry anything if it fails
|
|
396
|
-
do_retry ||= true if RestParameters.instance.retry_on_error
|
|
397
|
-
# not authorized: oauth token expired
|
|
398
|
-
if @not_auth_codes.include?(result_http.code.to_s) && @auth_params[:type].eql?(:oauth2)
|
|
399
|
-
begin
|
|
400
|
-
# try to use refresh token
|
|
401
|
-
req['Authorization'] = oauth.authorization(refresh: true)
|
|
402
|
-
rescue RestCallError => e_tok
|
|
403
|
-
e = e_tok
|
|
404
|
-
Log.log.error('refresh failed'.bg_red)
|
|
405
|
-
# regenerate a brand new token
|
|
406
|
-
req['Authorization'] = oauth.authorization(cache: false)
|
|
407
|
-
end
|
|
408
|
-
Log.log.debug('using new token')
|
|
409
|
-
do_retry ||= true
|
|
410
|
-
end
|
|
411
|
-
if do_retry && (error_tries -= 1).positive?
|
|
412
|
-
sleep(RestParameters.instance.retry_sleep) unless RestParameters.instance.retry_sleep.eql?(0)
|
|
413
|
-
retry
|
|
414
|
-
end
|
|
415
|
-
# redirect ? (any code beginning with 3)
|
|
416
|
-
if e.response.is_a?(Net::HTTPRedirection) && tries_remain_redirect.positive?
|
|
417
|
-
tries_remain_redirect -= 1
|
|
418
|
-
current_uri = URI.parse(@base_url)
|
|
419
|
-
new_url = e.response['Location']
|
|
420
|
-
# special case: relative redirect
|
|
421
|
-
if URI.parse(new_url).host.nil?
|
|
422
|
-
# we don't manage relative redirects with non-absolute path
|
|
423
|
-
Aspera.assert(new_url.start_with?('/')) { "redirect location is relative: #{new_url}, but does not start with /." }
|
|
424
|
-
new_url = "#{current_uri.scheme}://#{current_uri.host}#{new_url}"
|
|
425
|
-
end
|
|
426
|
-
# forwards the request to the new location
|
|
427
|
-
return self.class.new(
|
|
428
|
-
base_url: new_url,
|
|
429
|
-
redirect_max: tries_remain_redirect
|
|
430
|
-
).call(
|
|
431
|
-
operation: operation,
|
|
432
|
-
subpath: new_url.end_with?('/') ? '/' : nil,
|
|
433
|
-
query: query,
|
|
434
|
-
body: body,
|
|
435
|
-
content_type: content_type,
|
|
436
|
-
save_to: save_to,
|
|
437
|
-
exception: exception,
|
|
438
|
-
headers: headers,
|
|
439
|
-
ret: ret
|
|
440
|
-
)
|
|
441
|
-
end
|
|
442
|
-
# raise exception if could not retry and not return error in result
|
|
443
|
-
raise e if exception
|
|
444
|
-
end
|
|
445
|
-
Log.log.debug { "result=http:#{result_http}, data:#{result_data.class}" }
|
|
446
|
-
return case ret
|
|
447
|
-
when :data then result_data
|
|
448
|
-
when :resp then result_http
|
|
449
|
-
when :both then [result_data, result_http]
|
|
450
|
-
else Aspera.error_unexpected_value(ret) { 'Type of result for REST' }
|
|
451
|
-
end
|
|
452
|
-
end
|
|
453
|
-
|
|
454
|
-
private
|
|
455
|
-
|
|
456
|
-
def prepare_call(headers, query)
|
|
457
|
-
if headers.nil?
|
|
458
|
-
headers = @headers.clone
|
|
459
|
-
else
|
|
460
|
-
h = headers
|
|
461
|
-
headers = @headers.clone
|
|
462
|
-
headers.merge!(h)
|
|
463
|
-
end
|
|
464
|
-
Aspera.assert_type(headers, Hash)
|
|
465
|
-
case @auth_params[:type]
|
|
466
|
-
when :none
|
|
467
|
-
# no auth
|
|
468
|
-
when :basic
|
|
469
|
-
Log.log.debug('using Basic auth')
|
|
470
|
-
# done in build_req
|
|
471
|
-
when :oauth2
|
|
472
|
-
headers['Authorization'] = oauth.authorization unless headers.key?('Authorization')
|
|
473
|
-
when :url
|
|
474
|
-
query ||= {}
|
|
475
|
-
@auth_params[:url_query].each do |key, value|
|
|
476
|
-
query[key] = value
|
|
477
|
-
end
|
|
478
|
-
else Aspera.error_unexpected_value(@auth_params[:type])
|
|
479
|
-
end
|
|
480
|
-
[headers, query]
|
|
481
|
-
end
|
|
482
|
-
|
|
483
|
-
def build_request(operation, subpath, query, content_type, body, headers)
|
|
484
|
-
# TODO: shall we percent encode subpath (spaces) test with access key delete with space in id
|
|
485
|
-
# URI.escape()
|
|
486
|
-
separator = ['', '/'].include?(subpath) ? '' : '/'
|
|
487
|
-
uri = self.class.build_uri("#{@base_url}#{separator}#{subpath}", query)
|
|
488
|
-
Log.log.debug { "URI=#{uri}" }
|
|
489
|
-
begin
|
|
490
|
-
# instantiate request object based on string name
|
|
491
|
-
req = Net::HTTP.const_get(operation.capitalize).new(uri)
|
|
492
|
-
rescue NameError
|
|
493
|
-
raise "unsupported operation : #{operation}"
|
|
494
|
-
end
|
|
495
|
-
case content_type
|
|
496
|
-
when nil # ignore
|
|
497
|
-
when Mime::JSON
|
|
498
|
-
req.body = JSON.generate(body) # , ascii_only: true
|
|
499
|
-
req['Content-Type'] = Mime::JSON
|
|
500
|
-
when Mime::WWW
|
|
501
|
-
req.body = URI.encode_www_form(body)
|
|
502
|
-
req['Content-Type'] = Mime::WWW
|
|
503
|
-
when Mime::TEXT
|
|
504
|
-
req.body = body
|
|
505
|
-
req['Content-Type'] = Mime::TEXT
|
|
506
|
-
else Aspera.error_unexpected_value(content_type) { 'body type' }
|
|
507
|
-
end
|
|
508
|
-
# set headers
|
|
509
|
-
headers.each do |key, value|
|
|
510
|
-
req[key] = value
|
|
511
|
-
end
|
|
512
|
-
# :type = :basic
|
|
513
|
-
req.basic_auth(@auth_params[:username], @auth_params[:password]) if @auth_params[:type].eql?(:basic)
|
|
514
|
-
Log.dump(:req_body, req.body, level: :trace1)
|
|
515
|
-
req
|
|
516
|
-
end
|
|
517
|
-
|
|
518
|
-
def save_response(response, result_http, result_mime, save_to)
|
|
519
|
-
return false unless !save_to.nil? && result_http.code.to_s.start_with?('2') && !Mime.json?(result_mime)
|
|
520
|
-
|
|
521
|
-
total_size = result_http['Content-Length']&.to_i
|
|
522
|
-
Log.log.debug('before write file')
|
|
523
|
-
target_file = save_to
|
|
524
|
-
# override user's path to path in header
|
|
525
|
-
unless response['Content-Disposition'].nil?
|
|
526
|
-
disposition = self.class.parse_header(response['Content-Disposition'])
|
|
527
|
-
if disposition[:parameters].key?(:filename) && !disposition[:parameters][:filename].eql?('.')
|
|
528
|
-
# Use only the basename to prevent path traversal via a server-controlled Content-Disposition header
|
|
529
|
-
safe_filename = File.basename(disposition[:parameters][:filename])
|
|
530
|
-
target_file = File.join(File.dirname(target_file), safe_filename) unless safe_filename.empty?
|
|
531
|
-
end
|
|
532
|
-
end
|
|
533
|
-
Log.log.debug { "saving to: #{target_file}" }
|
|
534
|
-
written_size = 0
|
|
535
|
-
session_id = SecureRandom.uuid.freeze
|
|
536
|
-
RestParameters.instance.progress_bar&.event(:session_start, session_id: session_id)
|
|
537
|
-
RestParameters.instance.progress_bar&.event(:session_size, session_id: session_id, info: total_size) if total_size
|
|
538
|
-
limiter = TimerLimiter.new(0.5)
|
|
539
|
-
if target_file.respond_to?(:write)
|
|
540
|
-
# IO object: stream directly into it
|
|
541
|
-
result_http.read_body do |fragment|
|
|
542
|
-
target_file.write(fragment)
|
|
543
|
-
written_size += fragment.length
|
|
544
|
-
RestParameters.instance.progress_bar&.event(:transfer, session_id: session_id, info: written_size) if limiter.trigger?
|
|
545
|
-
end
|
|
546
|
-
else
|
|
547
|
-
# file path: download to partial name first, then rename atomically
|
|
548
|
-
target_file_tmp = "#{target_file}#{RestParameters.instance.download_partial_suffix}"
|
|
549
|
-
FileUtils.mkdir_p(File.dirname(target_file_tmp))
|
|
550
|
-
File.open(target_file_tmp, 'wb') do |file|
|
|
551
|
-
result_http.read_body do |fragment|
|
|
552
|
-
file.write(fragment)
|
|
553
|
-
written_size += fragment.length
|
|
554
|
-
RestParameters.instance.progress_bar&.event(:transfer, session_id: session_id, info: written_size) if limiter.trigger?
|
|
555
|
-
end
|
|
556
|
-
end
|
|
557
|
-
File.rename(target_file_tmp, target_file)
|
|
558
|
-
end
|
|
559
|
-
RestParameters.instance.progress_bar&.event(:session_end, session_id: session_id)
|
|
560
|
-
RestParameters.instance.progress_bar&.event(:end)
|
|
561
|
-
true
|
|
562
|
-
end
|
|
563
|
-
|
|
564
|
-
def parse_response(result_http, result_mime)
|
|
565
|
-
result_data = result_http.body
|
|
566
|
-
Log.dump(:result_data_raw, result_data, level: :trace1)
|
|
567
|
-
# TODO: Remove next 2 lines when bug in async node api is fixed. (Aspera/core/issues/4490)
|
|
568
|
-
node_api_bug = result_data&.index('}HTTP/1.1 400 Bad Request') if result_data.is_a?(String)
|
|
569
|
-
result_data = result_data[0..node_api_bug] if node_api_bug
|
|
570
|
-
result_data = JSON.parse(result_data) if Mime.json?(result_mime) && !result_data.nil? && !result_data.empty?
|
|
571
|
-
Log.dump(:result_data, result_data)
|
|
572
|
-
result_data
|
|
573
|
-
end
|
|
574
|
-
|
|
575
|
-
public
|
|
576
|
-
|
|
577
|
-
#
|
|
578
|
-
# CRUD simplified methods here
|
|
579
|
-
# If specific elements are needed, then use the full `call` method
|
|
580
|
-
#
|
|
581
|
-
|
|
582
|
-
# Create: `POST`
|
|
583
|
-
def create(subpath, params, **kwargs)
|
|
584
|
-
kwargs[:headers] ||= {}
|
|
585
|
-
kwargs[:headers]['Accept'] = Mime::JSON unless kwargs[:headers].key?('Accept')
|
|
586
|
-
kwargs[:content_type] = Mime::JSON unless kwargs.key?(:content_type)
|
|
587
|
-
return call(operation: 'POST', subpath: subpath, body: params, **kwargs)
|
|
588
|
-
end
|
|
589
|
-
|
|
590
|
-
# Read: `GET`
|
|
591
|
-
def read(subpath, query = nil, **kwargs)
|
|
592
|
-
kwargs[:headers] ||= {}
|
|
593
|
-
kwargs[:headers]['Accept'] = Mime::JSON unless kwargs[:headers].key?('Accept')
|
|
594
|
-
return call(operation: 'GET', subpath: subpath, query: query, **kwargs)
|
|
595
|
-
end
|
|
596
|
-
|
|
597
|
-
# Update: `PUT`
|
|
598
|
-
def update(subpath, params, **kwargs)
|
|
599
|
-
kwargs[:headers] ||= {}
|
|
600
|
-
kwargs[:headers]['Accept'] = Mime::JSON unless kwargs[:headers].key?('Accept')
|
|
601
|
-
kwargs[:content_type] = Mime::JSON unless kwargs.key?(:content_type)
|
|
602
|
-
return call(operation: 'PUT', subpath: subpath, body: params, **kwargs)
|
|
603
|
-
end
|
|
604
|
-
|
|
605
|
-
# Delete: `DELETE`
|
|
606
|
-
def delete(subpath, params = nil, **kwargs)
|
|
607
|
-
kwargs[:headers] ||= {}
|
|
608
|
-
kwargs[:headers]['Accept'] = Mime::JSON unless kwargs[:headers].key?('Accept')
|
|
609
|
-
return call(operation: 'DELETE', subpath: subpath, query: params, **kwargs)
|
|
610
|
-
end
|
|
611
|
-
|
|
612
|
-
# Cancel: `CANCEL`
|
|
613
|
-
def cancel(subpath, **kwargs)
|
|
614
|
-
kwargs[:headers] ||= {}
|
|
615
|
-
kwargs[:headers]['Accept'] = Mime::JSON unless kwargs[:headers].key?('Accept')
|
|
616
|
-
return call(operation: 'CANCEL', subpath: subpath, **kwargs)
|
|
617
|
-
end
|
|
618
|
-
|
|
619
|
-
UNAVAILABLE_CODES = ['503']
|
|
620
|
-
|
|
621
|
-
private_constant :UNAVAILABLE_CODES
|
|
622
|
-
end
|
|
623
|
-
end
|
|
3
|
+
require 'aspera/rest/parameters'
|
|
4
|
+
require 'aspera/rest/util'
|
|
5
|
+
require 'aspera/rest/call_error'
|
|
6
|
+
require 'aspera/rest/error_analyzer'
|
|
7
|
+
require 'aspera/rest/aspera_errors'
|
|
8
|
+
require 'aspera/rest/client'
|
|
9
|
+
require 'aspera/rest/list'
|