aspera-cli 4.16.0 → 4.17.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/CHANGELOG.md +50 -19
- data/CONTRIBUTING.md +3 -1
- data/README.md +965 -793
- data/bin/asession +29 -21
- data/lib/aspera/{fasp/agent_alpha.rb → agent/alpha.rb} +26 -25
- data/lib/aspera/{fasp/agent_base.rb → agent/base.rb} +15 -12
- data/lib/aspera/{fasp/agent_connect.rb → agent/connect.rb} +13 -11
- data/lib/aspera/{fasp/agent_direct.rb → agent/direct.rb} +49 -53
- data/lib/aspera/{fasp/agent_httpgw.rb → agent/httpgw.rb} +20 -19
- data/lib/aspera/{fasp/agent_node.rb → agent/node.rb} +20 -33
- data/lib/aspera/{fasp/agent_trsdk.rb → agent/trsdk.rb} +11 -11
- data/lib/aspera/api/aoc.rb +586 -0
- data/lib/aspera/api/ats.rb +46 -0
- data/lib/aspera/api/cos_node.rb +95 -0
- data/lib/aspera/api/node.rb +344 -0
- data/lib/aspera/ascmd.rb +46 -10
- data/lib/aspera/{fasp → ascp}/installation.rb +5 -5
- data/lib/aspera/{fasp → ascp}/management.rb +3 -8
- data/lib/aspera/{fasp → ascp}/products.rb +1 -1
- data/lib/aspera/assert.rb +30 -30
- data/lib/aspera/cli/basic_auth_plugin.rb +11 -10
- data/lib/aspera/cli/extended_value.rb +1 -1
- data/lib/aspera/cli/formatter.rb +13 -13
- data/lib/aspera/cli/hints.rb +5 -5
- data/lib/aspera/cli/main.rb +35 -28
- data/lib/aspera/cli/manager.rb +25 -24
- data/lib/aspera/cli/plugin.rb +22 -15
- data/lib/aspera/cli/plugin_factory.rb +61 -0
- data/lib/aspera/cli/plugins/alee.rb +7 -7
- data/lib/aspera/cli/plugins/aoc.rb +83 -77
- data/lib/aspera/cli/plugins/ats.rb +32 -33
- data/lib/aspera/cli/plugins/bss.rb +3 -4
- data/lib/aspera/cli/plugins/config.rb +169 -186
- data/lib/aspera/cli/plugins/console.rb +8 -6
- data/lib/aspera/cli/plugins/cos.rb +19 -18
- data/lib/aspera/cli/plugins/faspex.rb +61 -54
- data/lib/aspera/cli/plugins/faspex5.rb +150 -103
- data/lib/aspera/cli/plugins/node.rb +68 -73
- data/lib/aspera/cli/plugins/orchestrator.rb +34 -44
- data/lib/aspera/cli/plugins/preview.rb +31 -31
- data/lib/aspera/cli/plugins/server.rb +31 -33
- data/lib/aspera/cli/plugins/shares.rb +13 -11
- data/lib/aspera/cli/sync_actions.rb +8 -8
- data/lib/aspera/cli/transfer_agent.rb +32 -19
- data/lib/aspera/cli/transfer_progress.rb +1 -1
- data/lib/aspera/cli/version.rb +1 -1
- data/lib/aspera/colors.rb +5 -0
- data/lib/aspera/command_line_builder.rb +14 -14
- data/lib/aspera/coverage.rb +1 -2
- data/lib/aspera/data_repository.rb +1 -1
- data/lib/aspera/environment.rb +2 -3
- data/lib/aspera/faspex_gw.rb +5 -6
- data/lib/aspera/faspex_postproc.rb +1 -1
- data/lib/aspera/id_generator.rb +2 -2
- data/lib/aspera/json_rpc.rb +5 -5
- data/lib/aspera/keychain/encrypted_hash.rb +6 -6
- data/lib/aspera/keychain/macos_security.rb +27 -22
- data/lib/aspera/log.rb +2 -2
- data/lib/aspera/nagios.rb +3 -3
- data/lib/aspera/node_simulator.rb +5 -6
- data/lib/aspera/oauth/base.rb +143 -0
- data/lib/aspera/oauth/factory.rb +124 -0
- data/lib/aspera/oauth/generic.rb +34 -0
- data/lib/aspera/oauth/jwt.rb +51 -0
- data/lib/aspera/oauth/url_json.rb +31 -0
- data/lib/aspera/oauth/web.rb +50 -0
- data/lib/aspera/oauth.rb +5 -331
- data/lib/aspera/open_application.rb +7 -7
- data/lib/aspera/persistency_action_once.rb +4 -4
- data/lib/aspera/persistency_folder.rb +2 -2
- data/lib/aspera/preview/generator.rb +5 -5
- data/lib/aspera/preview/terminal.rb +3 -2
- data/lib/aspera/preview/utils.rb +3 -3
- data/lib/aspera/proxy_auto_config.rb +4 -4
- data/lib/aspera/rest.rb +175 -144
- data/lib/aspera/rest_errors_aspera.rb +3 -3
- data/lib/aspera/resumer.rb +77 -0
- data/lib/aspera/ssh.rb +6 -1
- data/lib/aspera/{fasp → transfer}/error.rb +3 -3
- data/lib/aspera/{fasp → transfer}/error_info.rb +1 -1
- data/lib/aspera/{fasp → transfer}/faux_file.rb +1 -1
- data/lib/aspera/{fasp → transfer}/parameters.rb +58 -89
- data/lib/aspera/{fasp/transfer_spec.rb → transfer/spec.rb} +18 -16
- data/lib/aspera/{fasp/parameters.yaml → transfer/spec.yaml} +4 -99
- data/lib/aspera/{fasp → transfer}/sync.rb +32 -32
- data/lib/aspera/{fasp → transfer}/uri.rb +9 -8
- data/lib/aspera/web_server_simple.rb +11 -3
- data.tar.gz.sig +0 -0
- metadata +36 -63
- metadata.gz.sig +0 -0
- data/lib/aspera/aoc.rb +0 -601
- data/lib/aspera/ats_api.rb +0 -47
- data/lib/aspera/cos_node.rb +0 -94
- data/lib/aspera/fasp/resume_policy.rb +0 -79
- data/lib/aspera/node.rb +0 -339
@@ -35,7 +35,7 @@ module Aspera
|
|
35
35
|
end
|
36
36
|
|
37
37
|
def put(object_id, value)
|
38
|
-
assert_type(value, String)
|
38
|
+
Aspera.assert_type(value, String)
|
39
39
|
persist_filepath = id_to_filepath(object_id)
|
40
40
|
Log.log.debug{"persistency saving: #{persist_filepath}"}
|
41
41
|
FileUtils.rm_f(persist_filepath)
|
@@ -68,7 +68,7 @@ module Aspera
|
|
68
68
|
|
69
69
|
# @param object_id String or Array
|
70
70
|
def id_to_filepath(object_id)
|
71
|
-
assert_type(object_id, String)
|
71
|
+
Aspera.assert_type(object_id, String)
|
72
72
|
FileUtils.mkdir_p(@folder)
|
73
73
|
Environment.restrict_file_access(@folder)
|
74
74
|
return File.join(@folder, "#{object_id}#{FILE_SUFFIX}")
|
@@ -54,7 +54,7 @@ module Aspera
|
|
54
54
|
end
|
55
55
|
@processing_method = @processing_method.to_sym
|
56
56
|
Log.log.debug{"method: #{@processing_method}"}
|
57
|
-
assert(respond_to?(@processing_method, true)){"no processing know for #{conversion_type} -> #{@preview_format_sym}"}
|
57
|
+
Aspera.assert(respond_to?(@processing_method, true)){"no processing know for #{conversion_type} -> #{@preview_format_sym}"}
|
58
58
|
end
|
59
59
|
|
60
60
|
# create preview as specified in constructor
|
@@ -88,7 +88,7 @@ module Aspera
|
|
88
88
|
# @param total_count of parts
|
89
89
|
# @param index of part (start at 1)
|
90
90
|
def get_offset(duration, start_offset, total_count, index)
|
91
|
-
assert_type(duration, Float){'duration'}
|
91
|
+
Aspera.assert_type(duration, Float){'duration'}
|
92
92
|
return start_offset + ((index - 1) * (duration - start_offset) / total_count)
|
93
93
|
end
|
94
94
|
|
@@ -151,10 +151,10 @@ module Aspera
|
|
151
151
|
# do a simple re-encoding
|
152
152
|
def convert_video_to_mp4_using_reencode
|
153
153
|
options = @options.reencode_ffmpeg
|
154
|
-
assert_type(options, Hash){'reencode_ffmpeg'}
|
154
|
+
Aspera.assert_type(options, Hash){'reencode_ffmpeg'}
|
155
155
|
options.each do |k, v|
|
156
|
-
assert_values(k, FFMPEG_OPTIONS_LIST){'key'}
|
157
|
-
assert_type(v, Array){k}
|
156
|
+
Aspera.assert_values(k, FFMPEG_OPTIONS_LIST){'key'}
|
157
|
+
Aspera.assert_type(v, Array){k}
|
158
158
|
end
|
159
159
|
Utils.ffmpeg(
|
160
160
|
in_f: @source_file_path,
|
@@ -4,6 +4,7 @@
|
|
4
4
|
|
5
5
|
require 'rainbow'
|
6
6
|
require 'io/console'
|
7
|
+
require 'aspera/log'
|
7
8
|
module Aspera
|
8
9
|
module Preview
|
9
10
|
# Display a picture in the terminal, either using coloured characters or iTerm2
|
@@ -40,9 +41,9 @@ module Aspera
|
|
40
41
|
(term_rows, term_columns) = IO.console.winsize
|
41
42
|
term_rows -= reserve
|
42
43
|
# compute scaling to fit terminal
|
43
|
-
fit_term_ratio = [term_rows / image.rows.to_f, term_columns / image.columns.to_f].min
|
44
|
+
fit_term_ratio = [term_rows.to_f * font_ratio / image.rows.to_f, term_columns.to_f / image.columns.to_f].min
|
44
45
|
height_ratio = double ? 2.0 : 1.0
|
45
|
-
image = image.scale((image.columns * fit_term_ratio
|
46
|
+
image = image.scale((image.columns * fit_term_ratio).to_i, (image.rows * fit_term_ratio * height_ratio / font_ratio).to_i)
|
46
47
|
# quantum depth is 8 or 16, see: `convert xc: -format "%q" info:`
|
47
48
|
shift_for_8_bit = Magick::MAGICKCORE_QUANTUM_DEPTH - 8
|
48
49
|
# get all pixel colors, adjusted for Rainbow
|
data/lib/aspera/preview/utils.rb
CHANGED
@@ -44,7 +44,7 @@ module Aspera
|
|
44
44
|
# one could use "system", but we would need to redirect stdout/err
|
45
45
|
# @return true if su
|
46
46
|
def external_command(command_sym, command_args, check_code: true)
|
47
|
-
assert_values(command_sym, EXTERNAL_TOOLS){'command'}
|
47
|
+
Aspera.assert_values(command_sym, EXTERNAL_TOOLS){'command'}
|
48
48
|
# build command line, and quote special characters
|
49
49
|
command_line = command_args.clone.unshift(command_sym).map{|i| shell_quote(i.to_s)}.join(' ')
|
50
50
|
Log.log.debug{"cmd=#{command_line}".blue}
|
@@ -59,7 +59,7 @@ module Aspera
|
|
59
59
|
end
|
60
60
|
|
61
61
|
def ffmpeg(a)
|
62
|
-
assert_type(a, Hash)
|
62
|
+
Aspera.assert_type(a, Hash)
|
63
63
|
# input_file,input_args,output_file,output_args
|
64
64
|
a[:gl_p] ||= [
|
65
65
|
'-y', # overwrite output without asking
|
@@ -67,7 +67,7 @@ module Aspera
|
|
67
67
|
]
|
68
68
|
a[:in_p] ||= []
|
69
69
|
a[:out_p] ||= []
|
70
|
-
assert(%i[gl_p in_f in_p out_f out_p].eql?(a.keys.sort)){"wrong params (#{a.keys.sort})"}
|
70
|
+
Aspera.assert(%i[gl_p in_f in_p out_f out_p].eql?(a.keys.sort)){"wrong params (#{a.keys.sort})"}
|
71
71
|
external_command(:ffmpeg, [a[:gl_p], a[:in_p], '-i', a[:in_f], a[:out_p], a[:out_f]].flatten)
|
72
72
|
end
|
73
73
|
|
@@ -11,7 +11,7 @@ module URI
|
|
11
11
|
alias_method :find_proxy_orig, :find_proxy
|
12
12
|
class << self
|
13
13
|
def register_proxy_finder
|
14
|
-
assert(block_given?)
|
14
|
+
Aspera.assert(block_given?)
|
15
15
|
# overload the method in URI : call user's provided block and fallback to original method
|
16
16
|
define_method(:find_proxy) {|env_vars=ENV| yield(to_s) || find_proxy_orig(env_vars)}
|
17
17
|
end
|
@@ -109,12 +109,12 @@ END_OF_JAVASCRIPT
|
|
109
109
|
parts = item.strip.split
|
110
110
|
case parts.shift
|
111
111
|
when 'DIRECT'
|
112
|
-
assert(parts.empty?){'DIRECT has no param'}
|
112
|
+
Aspera.assert(parts.empty?){'DIRECT has no param'}
|
113
113
|
Log.log.debug('ignoring proxy DIRECT')
|
114
114
|
when 'PROXY'
|
115
115
|
addr_port = parts.shift
|
116
|
-
assert_type(addr_port, String)
|
117
|
-
assert(parts.empty?){'PROXY shall have one param'}
|
116
|
+
Aspera.assert_type(addr_port, String)
|
117
|
+
Aspera.assert(parts.empty?){'PROXY shall have one param'}
|
118
118
|
begin
|
119
119
|
# PAC proxy addresses are <host>:<port>
|
120
120
|
if /:[0-9]+$/.match?(addr_port)
|
data/lib/aspera/rest.rb
CHANGED
@@ -56,19 +56,19 @@ module Aspera
|
|
56
56
|
return values.first.eql?(ARRAY_PARAMS)
|
57
57
|
end
|
58
58
|
|
59
|
-
# build URI from URL and parameters and check it is http or https
|
60
|
-
def build_uri(url,
|
59
|
+
# build URI from URL and parameters and check it is http or https, encode array [] parameters
|
60
|
+
def build_uri(url, query_hash=nil)
|
61
61
|
uri = URI.parse(url)
|
62
|
-
assert(%w[http https].include?(uri.scheme)){"REST endpoint shall be http/s not #{uri.scheme}"}
|
63
|
-
return uri if
|
64
|
-
Log.log.debug{Log.dump('
|
65
|
-
assert_type(
|
66
|
-
return uri if
|
62
|
+
Aspera.assert(%w[http https].include?(uri.scheme)){"REST endpoint shall be http/s not #{uri.scheme}"}
|
63
|
+
return uri if query_hash.nil?
|
64
|
+
Log.log.debug{Log.dump('query', query_hash)}
|
65
|
+
Aspera.assert_type(query_hash, Hash)
|
66
|
+
return uri if query_hash.empty?
|
67
67
|
query = []
|
68
|
-
|
68
|
+
query_hash.each do |k, v|
|
69
69
|
case v
|
70
70
|
when Array
|
71
|
-
# support array
|
71
|
+
# support array for query parameter, there is no standard. Either p[]=1&p[]=2, or p=1&p=2
|
72
72
|
suffix = array_params?(v) ? v.shift : ''
|
73
73
|
v.each do |e|
|
74
74
|
query.push(["#{k}#{suffix}", e])
|
@@ -86,7 +86,7 @@ module Aspera
|
|
86
86
|
URI.decode_www_form(query).each_with_object({}){|v, h|h[v.first] = v.last }
|
87
87
|
end
|
88
88
|
|
89
|
-
#
|
89
|
+
# Start a HTTP/S session, also used for web sockets
|
90
90
|
# @param base_url [String] base url of HTTP/S session
|
91
91
|
# @return [Net::HTTP] a started HTTP session
|
92
92
|
def start_http_session(base_url)
|
@@ -105,31 +105,34 @@ module Aspera
|
|
105
105
|
# little hack, handy because HTTP debug, proxy, etc... will be available
|
106
106
|
# used implement web sockets after `start_http_session`
|
107
107
|
def io_http_session(http_session)
|
108
|
-
assert_type(http_session, Net::HTTP)
|
108
|
+
Aspera.assert_type(http_session, Net::HTTP)
|
109
109
|
# Net::BufferedIO in net/protocol.rb
|
110
110
|
result = http_session.instance_variable_get(:@socket)
|
111
|
-
assert(!result.nil?){"no socket for #{http_session}"}
|
111
|
+
Aspera.assert(!result.nil?){"no socket for #{http_session}"}
|
112
112
|
return result
|
113
113
|
end
|
114
114
|
|
115
115
|
# @return [String] PEM certificates of remote server
|
116
|
-
def
|
116
|
+
def remote_certificate_chain(url, as_string: true)
|
117
|
+
result = []
|
117
118
|
# initiate a session to retrieve remote certificate
|
118
119
|
http_session = Rest.start_http_session(url)
|
119
120
|
begin
|
120
121
|
# retrieve underlying openssl socket
|
121
|
-
|
122
|
+
result = Rest.io_http_session(http_session).io.peer_cert_chain
|
122
123
|
rescue
|
123
|
-
|
124
|
+
result = http_session.peer_cert
|
124
125
|
ensure
|
125
126
|
http_session.finish
|
126
127
|
end
|
128
|
+
result = result.map(&:to_pem).join("\n") if as_string
|
129
|
+
return result
|
127
130
|
end
|
128
131
|
|
129
132
|
# set global parameters
|
130
133
|
def set_parameters(**options)
|
131
134
|
options.each do |key, value|
|
132
|
-
assert(@@global.key?(key)){"
|
135
|
+
Aspera.assert(@@global.key?(key)){"Unknown Rest option #{key}"}
|
133
136
|
@@global[key] = value
|
134
137
|
end
|
135
138
|
end
|
@@ -145,135 +148,169 @@ module Aspera
|
|
145
148
|
# create and start keep alive connection on demand
|
146
149
|
def http_session
|
147
150
|
if @http_session.nil?
|
148
|
-
@http_session = self.class.start_http_session(@
|
151
|
+
@http_session = self.class.start_http_session(@base_url)
|
149
152
|
end
|
150
153
|
return @http_session
|
151
154
|
end
|
152
155
|
|
153
|
-
|
154
|
-
|
155
|
-
|
156
|
-
|
157
|
-
|
158
|
-
|
159
|
-
|
160
|
-
|
161
|
-
|
162
|
-
return @oauth
|
163
|
-
end
|
164
|
-
|
165
|
-
# @param a_rest_params [Hash] default call parameters (merged at call)
|
166
|
-
def initialize(a_rest_params)
|
167
|
-
assert_type(a_rest_params, Hash)
|
168
|
-
assert_type(a_rest_params[:base_url], String)
|
169
|
-
@params = a_rest_params.clone
|
170
|
-
Log.log.debug{Log.dump('REST params', @params)}
|
171
|
-
# base url without trailing slashes (note: string may be frozen)
|
172
|
-
@params[:base_url] = @params[:base_url].gsub(%r{/+$}, '')
|
173
|
-
@http_session = nil
|
174
|
-
# default is no auth
|
175
|
-
@params[:auth] ||= {type: :none}
|
176
|
-
@params[:not_auth_codes] ||= ['401']
|
177
|
-
@oauth = nil
|
178
|
-
Log.log.debug{Log.dump('REST params(2)', @params)}
|
179
|
-
end
|
180
|
-
|
181
|
-
def oauth_token(force_refresh: false)
|
182
|
-
assert_values(force_refresh, [true, false])
|
183
|
-
return oauth.get_authorization(use_refresh_token: force_refresh)
|
184
|
-
end
|
185
|
-
|
186
|
-
def build_request(call_data)
|
156
|
+
def build_request(
|
157
|
+
operation:,
|
158
|
+
subpath:,
|
159
|
+
url_params:,
|
160
|
+
json_params:,
|
161
|
+
www_body_params:,
|
162
|
+
text_body_params:,
|
163
|
+
headers:
|
164
|
+
)
|
187
165
|
# TODO: shall we percent encode subpath (spaces) test with access key delete with space in id
|
188
166
|
# URI.escape()
|
189
|
-
|
167
|
+
separator = !['', '/'].include?(subpath) || @base_url.end_with?('/') ? '/' : ''
|
168
|
+
uri = self.class.build_uri("#{@base_url}#{separator}#{subpath}", url_params)
|
190
169
|
Log.log.debug{"URI=#{uri}"}
|
191
170
|
begin
|
192
171
|
# instantiate request object based on string name
|
193
|
-
req = Net::HTTP.const_get(
|
172
|
+
req = Net::HTTP.const_get(operation.capitalize).new(uri)
|
194
173
|
rescue NameError
|
195
|
-
raise "unsupported operation : #{
|
174
|
+
raise "unsupported operation : #{operation}"
|
196
175
|
end
|
197
|
-
if
|
198
|
-
req.body = JSON.generate(
|
199
|
-
Log.log.debug{Log.dump('body JSON data', call_data[:json_params])}
|
176
|
+
if !json_params.nil?
|
177
|
+
req.body = JSON.generate(json_params) # , ascii_only: true
|
200
178
|
req['Content-Type'] = 'application/json'
|
201
|
-
# call_data[:headers]['Accept']='application/json'
|
202
179
|
end
|
203
|
-
if
|
204
|
-
req.body = URI.encode_www_form(
|
205
|
-
Log.log.debug{"body www data=#{req.body.chomp}"}
|
180
|
+
if !www_body_params.nil?
|
181
|
+
req.body = URI.encode_www_form(www_body_params)
|
206
182
|
req['Content-Type'] = 'application/x-www-form-urlencoded'
|
207
183
|
end
|
208
|
-
if
|
209
|
-
req.body =
|
210
|
-
Log.log.debug{"body data=#{req.body.chomp}"}
|
184
|
+
if !text_body_params.nil?
|
185
|
+
req.body = text_body_params
|
211
186
|
end
|
212
187
|
# set headers
|
213
|
-
|
214
|
-
|
215
|
-
req[key] = call_data[:headers][key]
|
216
|
-
end
|
188
|
+
headers.each do |key, value|
|
189
|
+
req[key] = value
|
217
190
|
end
|
218
191
|
# :type = :basic
|
219
|
-
req.basic_auth(
|
192
|
+
req.basic_auth(@auth_params[:username], @auth_params[:password]) if @auth_params[:type].eql?(:basic)
|
220
193
|
Log.log.debug{Log.dump(:req_body, req.body)}
|
221
194
|
return req
|
222
195
|
end
|
223
196
|
|
197
|
+
public
|
198
|
+
|
199
|
+
attr_reader :auth_params
|
200
|
+
attr_reader :base_url
|
201
|
+
|
202
|
+
def params
|
203
|
+
return {
|
204
|
+
base_url: @base_url,
|
205
|
+
auth: @auth_params,
|
206
|
+
not_auth_codes: @not_auth_codes,
|
207
|
+
redirect_max: @redirect_max,
|
208
|
+
headers: @headers
|
209
|
+
}
|
210
|
+
end
|
211
|
+
|
212
|
+
# @param base_url [String] base URL of REST API
|
213
|
+
# @param auth [Hash] authentication parameters:
|
214
|
+
# :type (:none, :basic, :url, :oauth2)
|
215
|
+
# :username [:basic]
|
216
|
+
# :password [:basic]
|
217
|
+
# :url_query [:url] a hash
|
218
|
+
# :* [:oauth2] see OAuth::Factory class
|
219
|
+
# @param not_auth_codes [Array] codes that trigger a refresh/regeneration of bearer token
|
220
|
+
# @param redirect_max [int] max redirections allowed
|
221
|
+
def initialize(
|
222
|
+
base_url:,
|
223
|
+
auth: nil,
|
224
|
+
not_auth_codes: nil,
|
225
|
+
redirect_max: 0,
|
226
|
+
headers: nil
|
227
|
+
)
|
228
|
+
Aspera.assert_type(base_url, String)
|
229
|
+
# base url with max one trailing slashes (note: string may be frozen)
|
230
|
+
@base_url = base_url.gsub(%r{//+$}, '/')
|
231
|
+
# default is no auth
|
232
|
+
@auth_params = auth.nil? ? {type: :none} : auth
|
233
|
+
Aspera.assert_type(@auth_params, Hash)
|
234
|
+
Aspera.assert(@auth_params.key?(:type)){'no auth type defined'}
|
235
|
+
@not_auth_codes = not_auth_codes.nil? ? ['401'] : not_auth_codes
|
236
|
+
Aspera.assert_type(@not_auth_codes, Array)
|
237
|
+
# persistent session
|
238
|
+
@http_session = nil
|
239
|
+
# OAuth object (created on demand)
|
240
|
+
@oauth = nil
|
241
|
+
@redirect_max = redirect_max
|
242
|
+
@headers = headers.nil? ? {} : headers
|
243
|
+
Aspera.assert_type(@headers, Hash)
|
244
|
+
@headers['User-Agent'] ||= @@global[:user_agent]
|
245
|
+
end
|
246
|
+
|
247
|
+
# @return the OAuth object (create, or cached if already created)
|
248
|
+
def oauth
|
249
|
+
if @oauth.nil?
|
250
|
+
Aspera.assert(@auth_params[:type].eql?(:oauth2)){'no OAuth defined'}
|
251
|
+
oauth_parameters = @auth_params.reject { |k, _v| k.eql?(:type) }
|
252
|
+
Log.log.debug{Log.dump('oauth parameters', oauth_parameters)}
|
253
|
+
@oauth = OAuth::Factory.instance.create(**oauth_parameters)
|
254
|
+
end
|
255
|
+
return @oauth
|
256
|
+
end
|
257
|
+
|
258
|
+
def oauth_token(force_refresh: false)
|
259
|
+
Aspera.assert_values(force_refresh, [true, false])
|
260
|
+
return oauth.get_authorization(use_refresh_token: force_refresh)
|
261
|
+
end
|
262
|
+
|
224
263
|
# HTTP/S REST call
|
225
|
-
#
|
226
|
-
#
|
227
|
-
|
228
|
-
|
229
|
-
|
230
|
-
|
231
|
-
|
232
|
-
|
233
|
-
|
234
|
-
|
235
|
-
|
236
|
-
|
237
|
-
|
238
|
-
|
239
|
-
|
240
|
-
|
241
|
-
|
242
|
-
|
243
|
-
|
244
|
-
|
245
|
-
|
246
|
-
|
247
|
-
|
248
|
-
|
249
|
-
|
250
|
-
|
251
|
-
# defaults from @params are overridden by call data
|
252
|
-
call_data = @params.deep_merge(call_data)
|
253
|
-
case call_data[:auth][:type]
|
264
|
+
# @param save_to_file (filepath)
|
265
|
+
# @param return_error (bool)
|
266
|
+
def call(
|
267
|
+
operation:,
|
268
|
+
subpath: nil,
|
269
|
+
json_params: nil,
|
270
|
+
url_params: nil,
|
271
|
+
www_body_params: nil,
|
272
|
+
text_body_params: nil,
|
273
|
+
save_to_file: nil,
|
274
|
+
return_error: false,
|
275
|
+
headers: nil
|
276
|
+
)
|
277
|
+
subpath = subpath.to_s if subpath.is_a?(Symbol)
|
278
|
+
subpath = '' if subpath.nil?
|
279
|
+
Aspera.assert_type(subpath, String)
|
280
|
+
if headers.nil?
|
281
|
+
headers = @headers.clone
|
282
|
+
else
|
283
|
+
h = headers
|
284
|
+
headers = @headers.clone
|
285
|
+
headers.merge!(h)
|
286
|
+
end
|
287
|
+
Aspera.assert_type(headers, Hash)
|
288
|
+
Log.log.debug{"#{operation} [#{subpath}]".red.bold.bg_green}
|
289
|
+
case @auth_params[:type]
|
254
290
|
when :none
|
255
291
|
# no auth
|
256
292
|
when :basic
|
257
293
|
Log.log.debug('using Basic auth')
|
258
294
|
# done in build_req
|
259
295
|
when :oauth2
|
260
|
-
|
296
|
+
headers['Authorization'] = oauth_token unless headers.key?('Authorization')
|
261
297
|
when :url
|
262
|
-
|
263
|
-
|
264
|
-
|
298
|
+
url_params ||= {}
|
299
|
+
@auth_params[:url_query].each do |key, value|
|
300
|
+
url_params[key] = value
|
265
301
|
end
|
266
|
-
else error_unexpected_value(
|
302
|
+
else Aspera.error_unexpected_value(@auth_params[:type])
|
267
303
|
end
|
268
|
-
req = build_request(call_data)
|
269
|
-
Log.log.debug{"call_data = #{call_data}"}
|
270
304
|
result = {http: nil}
|
271
305
|
# start a block to be able to retry the actual HTTP request
|
272
306
|
begin
|
307
|
+
req = build_request(
|
308
|
+
operation: operation, subpath: subpath, json_params: json_params, url_params: url_params, www_body_params: www_body_params,
|
309
|
+
text_body_params: text_body_params, headers: headers)
|
273
310
|
# we try the call, and will retry only if oauth, as we can, first with refresh, and then re-auth if refresh is bad
|
274
311
|
oauth_tries ||= 2
|
275
312
|
# initialize with number of initial retries allowed, nil gives zero
|
276
|
-
tries_remain_redirect =
|
313
|
+
tries_remain_redirect = @redirect_max.to_i if tries_remain_redirect.nil?
|
277
314
|
Log.log.debug("send request (retries=#{tries_remain_redirect})")
|
278
315
|
result_mime = nil
|
279
316
|
file_saved = false
|
@@ -282,13 +319,13 @@ module Aspera
|
|
282
319
|
result[:http] = response
|
283
320
|
result_mime = (result[:http]['Content-Type'] || 'text/plain').split(';').first.downcase
|
284
321
|
# JSON data needs to be parsed, in case it contains an error code
|
285
|
-
if !
|
322
|
+
if !save_to_file.nil? &&
|
286
323
|
result[:http].code.to_s.start_with?('2') &&
|
287
324
|
!result[:http]['Content-Length'].nil? &&
|
288
325
|
!JSON_DECODE.include?(result_mime)
|
289
326
|
total_size = result[:http]['Content-Length'].to_i
|
290
327
|
Log.log.debug('before write file')
|
291
|
-
target_file =
|
328
|
+
target_file = save_to_file
|
292
329
|
# override user's path to path in header
|
293
330
|
if !response['Content-Disposition'].nil? && (m = response['Content-Disposition'].match(/filename="([^"]+)"/))
|
294
331
|
target_file = File.join(File.dirname(target_file), m[1])
|
@@ -321,13 +358,13 @@ module Aspera
|
|
321
358
|
else # when 'text/plain'
|
322
359
|
result[:http].body
|
323
360
|
end
|
324
|
-
Log.log.debug{
|
325
|
-
Log.log.debug{
|
361
|
+
Log.log.debug{"result: code=#{result[:http].code} mime=#{result_mime}"}
|
362
|
+
Log.log.debug{Log.dump('data', result[:data])}
|
326
363
|
RestErrorAnalyzer.instance.raise_on_error(req, result)
|
327
|
-
File.write(
|
364
|
+
File.write(save_to_file, result[:http].body) unless file_saved || save_to_file.nil?
|
328
365
|
rescue RestCallError => e
|
329
366
|
# not authorized: oauth token expired
|
330
|
-
if
|
367
|
+
if @not_auth_codes.include?(result[:http].code.to_s) && @auth_params[:type].eql?(:oauth2)
|
331
368
|
begin
|
332
369
|
# try to use refresh token
|
333
370
|
req['Authorization'] = oauth_token(force_refresh: true)
|
@@ -337,35 +374,28 @@ module Aspera
|
|
337
374
|
# regenerate a brand new token
|
338
375
|
req['Authorization'] = oauth_token(force_refresh: true)
|
339
376
|
end
|
340
|
-
Log.log.debug{"using new token=#{
|
341
|
-
retry
|
377
|
+
Log.log.debug{"using new token=#{headers['Authorization']}"}
|
378
|
+
retry if (oauth_tries -= 1).nonzero?
|
342
379
|
end # if oauth
|
343
380
|
# redirect ? (any code beginning with 3)
|
344
|
-
if
|
381
|
+
if e.response.is_a?(Net::HTTPRedirection) && tries_remain_redirect.positive?
|
345
382
|
tries_remain_redirect -= 1
|
346
|
-
current_uri = URI.parse(
|
383
|
+
current_uri = URI.parse(@base_url)
|
347
384
|
new_url = e.response['location']
|
348
385
|
# special case: relative redirect
|
349
386
|
if URI.parse(new_url).host.nil?
|
350
387
|
# we don't manage relative redirects with non-absolute path
|
351
|
-
assert(new_url.start_with?('/')){"redirect location is relative: #{new_url}, but does not start with /."}
|
352
|
-
new_url = current_uri.scheme
|
353
|
-
end
|
354
|
-
Log.log.info{"URL is moved: #{new_url}"}
|
355
|
-
redirection_uri = URI.parse(new_url)
|
356
|
-
call_data[:base_url] = new_url
|
357
|
-
call_data[:subpath] = ''
|
358
|
-
if current_uri.host.eql?(redirection_uri.host) && current_uri.port.eql?(redirection_uri.port)
|
359
|
-
req = build_request(call_data)
|
360
|
-
retry
|
361
|
-
else
|
362
|
-
# change host
|
363
|
-
Log.log.info{"Redirect changes host: #{current_uri.host} -> #{redirection_uri.host}"}
|
364
|
-
return self.class.new(call_data).call(call_data)
|
388
|
+
Aspera.assert(new_url.start_with?('/')){"redirect location is relative: #{new_url}, but does not start with /."}
|
389
|
+
new_url = "#{current_uri.scheme}://#{current_uri.host}#{new_url}"
|
365
390
|
end
|
391
|
+
# forwards the request to the new location
|
392
|
+
return self.class.new(base_url: new_url, redirect_max: tries_remain_redirect).call(
|
393
|
+
operation: operation, json_params: json_params,
|
394
|
+
url_params: url_params, www_body_params: www_body_params, text_body_params: text_body_params,
|
395
|
+
save_to_file: save_to_file, return_error: return_error, headers: headers)
|
366
396
|
end
|
367
397
|
# raise exception if could not retry and not return error in result
|
368
|
-
raise e unless
|
398
|
+
raise e unless return_error
|
369
399
|
end # begin request
|
370
400
|
Log.log.debug{"result=#{result}"}
|
371
401
|
return result
|
@@ -377,36 +407,37 @@ module Aspera
|
|
377
407
|
|
378
408
|
# @param encoding : one of: :json_params, :url_params
|
379
409
|
def create(subpath, params, encoding=:json_params)
|
380
|
-
return call(
|
410
|
+
return call(operation: 'POST', subpath: subpath, headers: {'Accept' => 'application/json'}, encoding => params)
|
381
411
|
end
|
382
412
|
|
383
|
-
def read(subpath,
|
384
|
-
return call(
|
413
|
+
def read(subpath, query=nil)
|
414
|
+
return call(operation: 'GET', subpath: subpath, headers: {'Accept' => 'application/json'}, url_params: query)
|
385
415
|
end
|
386
416
|
|
387
417
|
def update(subpath, params)
|
388
|
-
return call(
|
418
|
+
return call(operation: 'PUT', subpath: subpath, headers: {'Accept' => 'application/json'}, json_params: params)
|
389
419
|
end
|
390
420
|
|
391
421
|
def delete(subpath, params=nil)
|
392
|
-
return call(
|
422
|
+
return call(operation: 'DELETE', subpath: subpath, headers: {'Accept' => 'application/json'}, url_params: params)
|
393
423
|
end
|
394
424
|
|
395
425
|
def cancel(subpath)
|
396
|
-
return call(
|
426
|
+
return call(operation: 'CANCEL', subpath: subpath, headers: {'Accept' => 'application/json'})
|
397
427
|
end
|
398
428
|
|
399
|
-
# Query by
|
429
|
+
# Query entity by general search (read with parameter `q`)
|
430
|
+
# TODO: not generic enough ? move somewhere ? inheritance ?
|
400
431
|
# @param subpath path of entity in API
|
401
432
|
# @param search_name name of searched entity
|
402
|
-
# @param
|
403
|
-
|
404
|
-
|
405
|
-
|
433
|
+
# @param query additional search query parameters
|
434
|
+
# @returns [Hash] A single entity matching the search, or an exception if not found or multiple found
|
435
|
+
def lookup_by_name(subpath, search_name, query={})
|
436
|
+
# returns entities matching the query (it matches against several fields in case insensitive way)
|
437
|
+
matching_items = read(subpath, query.merge({'q' => search_name}))[:data]
|
406
438
|
# API style: {totalcount:, ...} cspell: disable-line
|
407
|
-
# TODO: not generic enough ? move somewhere ? inheritance ?
|
408
439
|
matching_items = matching_items[subpath] if matching_items.is_a?(Hash)
|
409
|
-
assert_type(matching_items, Array)
|
440
|
+
Aspera.assert_type(matching_items, Array)
|
410
441
|
case matching_items.length
|
411
442
|
when 1 then return matching_items.first
|
412
443
|
when 0 then raise %Q{#{ENTITY_NOT_FOUND} #{subpath}: "#{search_name}"}
|
@@ -15,10 +15,10 @@ module Aspera
|
|
15
15
|
RestErrorAnalyzer.instance.add_simple_handler(name: 'Type 1: error:user_message', path: %w[error user_message], always: true)
|
16
16
|
RestErrorAnalyzer.instance.add_simple_handler(name: 'Type 2: error:description', path: %w[error description])
|
17
17
|
RestErrorAnalyzer.instance.add_simple_handler(name: 'Type 3: error:internal_message', path: %w[error internal_message])
|
18
|
-
# AoC Automation
|
19
|
-
RestErrorAnalyzer.instance.add_simple_handler(name: 'AoC Automation', path: ['error'])
|
20
18
|
RestErrorAnalyzer.instance.add_simple_handler(name: 'Type 5', path: ['error_description'])
|
21
19
|
RestErrorAnalyzer.instance.add_simple_handler(name: 'Type 6', path: ['message'])
|
20
|
+
# AoC Automation
|
21
|
+
RestErrorAnalyzer.instance.add_simple_handler(name: 'AoC Automation', path: ['error'])
|
22
22
|
RestErrorAnalyzer.instance.add_handler('Type 7: errors[]') do |type, call_context|
|
23
23
|
next unless call_context[:data].is_a?(Hash) && call_context[:data]['errors'].is_a?(Hash)
|
24
24
|
# special for Shares: false positive ? (update global transfer_settings)
|
@@ -33,7 +33,7 @@ module Aspera
|
|
33
33
|
d_t_s = call_context[:data]['transfer_specs']
|
34
34
|
next unless d_t_s.is_a?(Array)
|
35
35
|
d_t_s.each do |res|
|
36
|
-
r_err = res.dig(*%w[transfer_spec error])
|
36
|
+
r_err = res.dig(*%w[transfer_spec error]) || res['error']
|
37
37
|
next unless r_err.is_a?(Hash)
|
38
38
|
RestErrorAnalyzer.add_error(call_context, type, "#{r_err['code']}: #{r_err['reason']}: #{r_err['user_message']}")
|
39
39
|
end
|