aspera-cli 4.16.0 → 4.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/CHANGELOG.md +50 -19
- data/CONTRIBUTING.md +3 -1
- data/README.md +965 -793
- data/bin/asession +29 -21
- data/lib/aspera/{fasp/agent_alpha.rb → agent/alpha.rb} +26 -25
- data/lib/aspera/{fasp/agent_base.rb → agent/base.rb} +15 -12
- data/lib/aspera/{fasp/agent_connect.rb → agent/connect.rb} +13 -11
- data/lib/aspera/{fasp/agent_direct.rb → agent/direct.rb} +49 -53
- data/lib/aspera/{fasp/agent_httpgw.rb → agent/httpgw.rb} +20 -19
- data/lib/aspera/{fasp/agent_node.rb → agent/node.rb} +20 -33
- data/lib/aspera/{fasp/agent_trsdk.rb → agent/trsdk.rb} +11 -11
- data/lib/aspera/api/aoc.rb +586 -0
- data/lib/aspera/api/ats.rb +46 -0
- data/lib/aspera/api/cos_node.rb +95 -0
- data/lib/aspera/api/node.rb +344 -0
- data/lib/aspera/ascmd.rb +46 -10
- data/lib/aspera/{fasp → ascp}/installation.rb +5 -5
- data/lib/aspera/{fasp → ascp}/management.rb +3 -8
- data/lib/aspera/{fasp → ascp}/products.rb +1 -1
- data/lib/aspera/assert.rb +30 -30
- data/lib/aspera/cli/basic_auth_plugin.rb +11 -10
- data/lib/aspera/cli/extended_value.rb +1 -1
- data/lib/aspera/cli/formatter.rb +13 -13
- data/lib/aspera/cli/hints.rb +5 -5
- data/lib/aspera/cli/main.rb +35 -28
- data/lib/aspera/cli/manager.rb +25 -24
- data/lib/aspera/cli/plugin.rb +22 -15
- data/lib/aspera/cli/plugin_factory.rb +61 -0
- data/lib/aspera/cli/plugins/alee.rb +7 -7
- data/lib/aspera/cli/plugins/aoc.rb +83 -77
- data/lib/aspera/cli/plugins/ats.rb +32 -33
- data/lib/aspera/cli/plugins/bss.rb +3 -4
- data/lib/aspera/cli/plugins/config.rb +169 -186
- data/lib/aspera/cli/plugins/console.rb +8 -6
- data/lib/aspera/cli/plugins/cos.rb +19 -18
- data/lib/aspera/cli/plugins/faspex.rb +61 -54
- data/lib/aspera/cli/plugins/faspex5.rb +150 -103
- data/lib/aspera/cli/plugins/node.rb +68 -73
- data/lib/aspera/cli/plugins/orchestrator.rb +34 -44
- data/lib/aspera/cli/plugins/preview.rb +31 -31
- data/lib/aspera/cli/plugins/server.rb +31 -33
- data/lib/aspera/cli/plugins/shares.rb +13 -11
- data/lib/aspera/cli/sync_actions.rb +8 -8
- data/lib/aspera/cli/transfer_agent.rb +32 -19
- data/lib/aspera/cli/transfer_progress.rb +1 -1
- data/lib/aspera/cli/version.rb +1 -1
- data/lib/aspera/colors.rb +5 -0
- data/lib/aspera/command_line_builder.rb +14 -14
- data/lib/aspera/coverage.rb +1 -2
- data/lib/aspera/data_repository.rb +1 -1
- data/lib/aspera/environment.rb +2 -3
- data/lib/aspera/faspex_gw.rb +5 -6
- data/lib/aspera/faspex_postproc.rb +1 -1
- data/lib/aspera/id_generator.rb +2 -2
- data/lib/aspera/json_rpc.rb +5 -5
- data/lib/aspera/keychain/encrypted_hash.rb +6 -6
- data/lib/aspera/keychain/macos_security.rb +27 -22
- data/lib/aspera/log.rb +2 -2
- data/lib/aspera/nagios.rb +3 -3
- data/lib/aspera/node_simulator.rb +5 -6
- data/lib/aspera/oauth/base.rb +143 -0
- data/lib/aspera/oauth/factory.rb +124 -0
- data/lib/aspera/oauth/generic.rb +34 -0
- data/lib/aspera/oauth/jwt.rb +51 -0
- data/lib/aspera/oauth/url_json.rb +31 -0
- data/lib/aspera/oauth/web.rb +50 -0
- data/lib/aspera/oauth.rb +5 -331
- data/lib/aspera/open_application.rb +7 -7
- data/lib/aspera/persistency_action_once.rb +4 -4
- data/lib/aspera/persistency_folder.rb +2 -2
- data/lib/aspera/preview/generator.rb +5 -5
- data/lib/aspera/preview/terminal.rb +3 -2
- data/lib/aspera/preview/utils.rb +3 -3
- data/lib/aspera/proxy_auto_config.rb +4 -4
- data/lib/aspera/rest.rb +175 -144
- data/lib/aspera/rest_errors_aspera.rb +3 -3
- data/lib/aspera/resumer.rb +77 -0
- data/lib/aspera/ssh.rb +6 -1
- data/lib/aspera/{fasp → transfer}/error.rb +3 -3
- data/lib/aspera/{fasp → transfer}/error_info.rb +1 -1
- data/lib/aspera/{fasp → transfer}/faux_file.rb +1 -1
- data/lib/aspera/{fasp → transfer}/parameters.rb +58 -89
- data/lib/aspera/{fasp/transfer_spec.rb → transfer/spec.rb} +18 -16
- data/lib/aspera/{fasp/parameters.yaml → transfer/spec.yaml} +4 -99
- data/lib/aspera/{fasp → transfer}/sync.rb +32 -32
- data/lib/aspera/{fasp → transfer}/uri.rb +9 -8
- data/lib/aspera/web_server_simple.rb +11 -3
- data.tar.gz.sig +0 -0
- metadata +36 -63
- metadata.gz.sig +0 -0
- data/lib/aspera/aoc.rb +0 -601
- data/lib/aspera/ats_api.rb +0 -47
- data/lib/aspera/cos_node.rb +0 -94
- data/lib/aspera/fasp/resume_policy.rb +0 -79
- data/lib/aspera/node.rb +0 -339
|
@@ -35,7 +35,7 @@ module Aspera
|
|
|
35
35
|
end
|
|
36
36
|
|
|
37
37
|
def put(object_id, value)
|
|
38
|
-
assert_type(value, String)
|
|
38
|
+
Aspera.assert_type(value, String)
|
|
39
39
|
persist_filepath = id_to_filepath(object_id)
|
|
40
40
|
Log.log.debug{"persistency saving: #{persist_filepath}"}
|
|
41
41
|
FileUtils.rm_f(persist_filepath)
|
|
@@ -68,7 +68,7 @@ module Aspera
|
|
|
68
68
|
|
|
69
69
|
# @param object_id String or Array
|
|
70
70
|
def id_to_filepath(object_id)
|
|
71
|
-
assert_type(object_id, String)
|
|
71
|
+
Aspera.assert_type(object_id, String)
|
|
72
72
|
FileUtils.mkdir_p(@folder)
|
|
73
73
|
Environment.restrict_file_access(@folder)
|
|
74
74
|
return File.join(@folder, "#{object_id}#{FILE_SUFFIX}")
|
|
@@ -54,7 +54,7 @@ module Aspera
|
|
|
54
54
|
end
|
|
55
55
|
@processing_method = @processing_method.to_sym
|
|
56
56
|
Log.log.debug{"method: #{@processing_method}"}
|
|
57
|
-
assert(respond_to?(@processing_method, true)){"no processing know for #{conversion_type} -> #{@preview_format_sym}"}
|
|
57
|
+
Aspera.assert(respond_to?(@processing_method, true)){"no processing know for #{conversion_type} -> #{@preview_format_sym}"}
|
|
58
58
|
end
|
|
59
59
|
|
|
60
60
|
# create preview as specified in constructor
|
|
@@ -88,7 +88,7 @@ module Aspera
|
|
|
88
88
|
# @param total_count of parts
|
|
89
89
|
# @param index of part (start at 1)
|
|
90
90
|
def get_offset(duration, start_offset, total_count, index)
|
|
91
|
-
assert_type(duration, Float){'duration'}
|
|
91
|
+
Aspera.assert_type(duration, Float){'duration'}
|
|
92
92
|
return start_offset + ((index - 1) * (duration - start_offset) / total_count)
|
|
93
93
|
end
|
|
94
94
|
|
|
@@ -151,10 +151,10 @@ module Aspera
|
|
|
151
151
|
# do a simple re-encoding
|
|
152
152
|
def convert_video_to_mp4_using_reencode
|
|
153
153
|
options = @options.reencode_ffmpeg
|
|
154
|
-
assert_type(options, Hash){'reencode_ffmpeg'}
|
|
154
|
+
Aspera.assert_type(options, Hash){'reencode_ffmpeg'}
|
|
155
155
|
options.each do |k, v|
|
|
156
|
-
assert_values(k, FFMPEG_OPTIONS_LIST){'key'}
|
|
157
|
-
assert_type(v, Array){k}
|
|
156
|
+
Aspera.assert_values(k, FFMPEG_OPTIONS_LIST){'key'}
|
|
157
|
+
Aspera.assert_type(v, Array){k}
|
|
158
158
|
end
|
|
159
159
|
Utils.ffmpeg(
|
|
160
160
|
in_f: @source_file_path,
|
|
@@ -4,6 +4,7 @@
|
|
|
4
4
|
|
|
5
5
|
require 'rainbow'
|
|
6
6
|
require 'io/console'
|
|
7
|
+
require 'aspera/log'
|
|
7
8
|
module Aspera
|
|
8
9
|
module Preview
|
|
9
10
|
# Display a picture in the terminal, either using coloured characters or iTerm2
|
|
@@ -40,9 +41,9 @@ module Aspera
|
|
|
40
41
|
(term_rows, term_columns) = IO.console.winsize
|
|
41
42
|
term_rows -= reserve
|
|
42
43
|
# compute scaling to fit terminal
|
|
43
|
-
fit_term_ratio = [term_rows / image.rows.to_f, term_columns / image.columns.to_f].min
|
|
44
|
+
fit_term_ratio = [term_rows.to_f * font_ratio / image.rows.to_f, term_columns.to_f / image.columns.to_f].min
|
|
44
45
|
height_ratio = double ? 2.0 : 1.0
|
|
45
|
-
image = image.scale((image.columns * fit_term_ratio
|
|
46
|
+
image = image.scale((image.columns * fit_term_ratio).to_i, (image.rows * fit_term_ratio * height_ratio / font_ratio).to_i)
|
|
46
47
|
# quantum depth is 8 or 16, see: `convert xc: -format "%q" info:`
|
|
47
48
|
shift_for_8_bit = Magick::MAGICKCORE_QUANTUM_DEPTH - 8
|
|
48
49
|
# get all pixel colors, adjusted for Rainbow
|
data/lib/aspera/preview/utils.rb
CHANGED
|
@@ -44,7 +44,7 @@ module Aspera
|
|
|
44
44
|
# one could use "system", but we would need to redirect stdout/err
|
|
45
45
|
# @return true if su
|
|
46
46
|
def external_command(command_sym, command_args, check_code: true)
|
|
47
|
-
assert_values(command_sym, EXTERNAL_TOOLS){'command'}
|
|
47
|
+
Aspera.assert_values(command_sym, EXTERNAL_TOOLS){'command'}
|
|
48
48
|
# build command line, and quote special characters
|
|
49
49
|
command_line = command_args.clone.unshift(command_sym).map{|i| shell_quote(i.to_s)}.join(' ')
|
|
50
50
|
Log.log.debug{"cmd=#{command_line}".blue}
|
|
@@ -59,7 +59,7 @@ module Aspera
|
|
|
59
59
|
end
|
|
60
60
|
|
|
61
61
|
def ffmpeg(a)
|
|
62
|
-
assert_type(a, Hash)
|
|
62
|
+
Aspera.assert_type(a, Hash)
|
|
63
63
|
# input_file,input_args,output_file,output_args
|
|
64
64
|
a[:gl_p] ||= [
|
|
65
65
|
'-y', # overwrite output without asking
|
|
@@ -67,7 +67,7 @@ module Aspera
|
|
|
67
67
|
]
|
|
68
68
|
a[:in_p] ||= []
|
|
69
69
|
a[:out_p] ||= []
|
|
70
|
-
assert(%i[gl_p in_f in_p out_f out_p].eql?(a.keys.sort)){"wrong params (#{a.keys.sort})"}
|
|
70
|
+
Aspera.assert(%i[gl_p in_f in_p out_f out_p].eql?(a.keys.sort)){"wrong params (#{a.keys.sort})"}
|
|
71
71
|
external_command(:ffmpeg, [a[:gl_p], a[:in_p], '-i', a[:in_f], a[:out_p], a[:out_f]].flatten)
|
|
72
72
|
end
|
|
73
73
|
|
|
@@ -11,7 +11,7 @@ module URI
|
|
|
11
11
|
alias_method :find_proxy_orig, :find_proxy
|
|
12
12
|
class << self
|
|
13
13
|
def register_proxy_finder
|
|
14
|
-
assert(block_given?)
|
|
14
|
+
Aspera.assert(block_given?)
|
|
15
15
|
# overload the method in URI : call user's provided block and fallback to original method
|
|
16
16
|
define_method(:find_proxy) {|env_vars=ENV| yield(to_s) || find_proxy_orig(env_vars)}
|
|
17
17
|
end
|
|
@@ -109,12 +109,12 @@ END_OF_JAVASCRIPT
|
|
|
109
109
|
parts = item.strip.split
|
|
110
110
|
case parts.shift
|
|
111
111
|
when 'DIRECT'
|
|
112
|
-
assert(parts.empty?){'DIRECT has no param'}
|
|
112
|
+
Aspera.assert(parts.empty?){'DIRECT has no param'}
|
|
113
113
|
Log.log.debug('ignoring proxy DIRECT')
|
|
114
114
|
when 'PROXY'
|
|
115
115
|
addr_port = parts.shift
|
|
116
|
-
assert_type(addr_port, String)
|
|
117
|
-
assert(parts.empty?){'PROXY shall have one param'}
|
|
116
|
+
Aspera.assert_type(addr_port, String)
|
|
117
|
+
Aspera.assert(parts.empty?){'PROXY shall have one param'}
|
|
118
118
|
begin
|
|
119
119
|
# PAC proxy addresses are <host>:<port>
|
|
120
120
|
if /:[0-9]+$/.match?(addr_port)
|
data/lib/aspera/rest.rb
CHANGED
|
@@ -56,19 +56,19 @@ module Aspera
|
|
|
56
56
|
return values.first.eql?(ARRAY_PARAMS)
|
|
57
57
|
end
|
|
58
58
|
|
|
59
|
-
# build URI from URL and parameters and check it is http or https
|
|
60
|
-
def build_uri(url,
|
|
59
|
+
# build URI from URL and parameters and check it is http or https, encode array [] parameters
|
|
60
|
+
def build_uri(url, query_hash=nil)
|
|
61
61
|
uri = URI.parse(url)
|
|
62
|
-
assert(%w[http https].include?(uri.scheme)){"REST endpoint shall be http/s not #{uri.scheme}"}
|
|
63
|
-
return uri if
|
|
64
|
-
Log.log.debug{Log.dump('
|
|
65
|
-
assert_type(
|
|
66
|
-
return uri if
|
|
62
|
+
Aspera.assert(%w[http https].include?(uri.scheme)){"REST endpoint shall be http/s not #{uri.scheme}"}
|
|
63
|
+
return uri if query_hash.nil?
|
|
64
|
+
Log.log.debug{Log.dump('query', query_hash)}
|
|
65
|
+
Aspera.assert_type(query_hash, Hash)
|
|
66
|
+
return uri if query_hash.empty?
|
|
67
67
|
query = []
|
|
68
|
-
|
|
68
|
+
query_hash.each do |k, v|
|
|
69
69
|
case v
|
|
70
70
|
when Array
|
|
71
|
-
# support array
|
|
71
|
+
# support array for query parameter, there is no standard. Either p[]=1&p[]=2, or p=1&p=2
|
|
72
72
|
suffix = array_params?(v) ? v.shift : ''
|
|
73
73
|
v.each do |e|
|
|
74
74
|
query.push(["#{k}#{suffix}", e])
|
|
@@ -86,7 +86,7 @@ module Aspera
|
|
|
86
86
|
URI.decode_www_form(query).each_with_object({}){|v, h|h[v.first] = v.last }
|
|
87
87
|
end
|
|
88
88
|
|
|
89
|
-
#
|
|
89
|
+
# Start a HTTP/S session, also used for web sockets
|
|
90
90
|
# @param base_url [String] base url of HTTP/S session
|
|
91
91
|
# @return [Net::HTTP] a started HTTP session
|
|
92
92
|
def start_http_session(base_url)
|
|
@@ -105,31 +105,34 @@ module Aspera
|
|
|
105
105
|
# little hack, handy because HTTP debug, proxy, etc... will be available
|
|
106
106
|
# used implement web sockets after `start_http_session`
|
|
107
107
|
def io_http_session(http_session)
|
|
108
|
-
assert_type(http_session, Net::HTTP)
|
|
108
|
+
Aspera.assert_type(http_session, Net::HTTP)
|
|
109
109
|
# Net::BufferedIO in net/protocol.rb
|
|
110
110
|
result = http_session.instance_variable_get(:@socket)
|
|
111
|
-
assert(!result.nil?){"no socket for #{http_session}"}
|
|
111
|
+
Aspera.assert(!result.nil?){"no socket for #{http_session}"}
|
|
112
112
|
return result
|
|
113
113
|
end
|
|
114
114
|
|
|
115
115
|
# @return [String] PEM certificates of remote server
|
|
116
|
-
def
|
|
116
|
+
def remote_certificate_chain(url, as_string: true)
|
|
117
|
+
result = []
|
|
117
118
|
# initiate a session to retrieve remote certificate
|
|
118
119
|
http_session = Rest.start_http_session(url)
|
|
119
120
|
begin
|
|
120
121
|
# retrieve underlying openssl socket
|
|
121
|
-
|
|
122
|
+
result = Rest.io_http_session(http_session).io.peer_cert_chain
|
|
122
123
|
rescue
|
|
123
|
-
|
|
124
|
+
result = http_session.peer_cert
|
|
124
125
|
ensure
|
|
125
126
|
http_session.finish
|
|
126
127
|
end
|
|
128
|
+
result = result.map(&:to_pem).join("\n") if as_string
|
|
129
|
+
return result
|
|
127
130
|
end
|
|
128
131
|
|
|
129
132
|
# set global parameters
|
|
130
133
|
def set_parameters(**options)
|
|
131
134
|
options.each do |key, value|
|
|
132
|
-
assert(@@global.key?(key)){"
|
|
135
|
+
Aspera.assert(@@global.key?(key)){"Unknown Rest option #{key}"}
|
|
133
136
|
@@global[key] = value
|
|
134
137
|
end
|
|
135
138
|
end
|
|
@@ -145,135 +148,169 @@ module Aspera
|
|
|
145
148
|
# create and start keep alive connection on demand
|
|
146
149
|
def http_session
|
|
147
150
|
if @http_session.nil?
|
|
148
|
-
@http_session = self.class.start_http_session(@
|
|
151
|
+
@http_session = self.class.start_http_session(@base_url)
|
|
149
152
|
end
|
|
150
153
|
return @http_session
|
|
151
154
|
end
|
|
152
155
|
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
return @oauth
|
|
163
|
-
end
|
|
164
|
-
|
|
165
|
-
# @param a_rest_params [Hash] default call parameters (merged at call)
|
|
166
|
-
def initialize(a_rest_params)
|
|
167
|
-
assert_type(a_rest_params, Hash)
|
|
168
|
-
assert_type(a_rest_params[:base_url], String)
|
|
169
|
-
@params = a_rest_params.clone
|
|
170
|
-
Log.log.debug{Log.dump('REST params', @params)}
|
|
171
|
-
# base url without trailing slashes (note: string may be frozen)
|
|
172
|
-
@params[:base_url] = @params[:base_url].gsub(%r{/+$}, '')
|
|
173
|
-
@http_session = nil
|
|
174
|
-
# default is no auth
|
|
175
|
-
@params[:auth] ||= {type: :none}
|
|
176
|
-
@params[:not_auth_codes] ||= ['401']
|
|
177
|
-
@oauth = nil
|
|
178
|
-
Log.log.debug{Log.dump('REST params(2)', @params)}
|
|
179
|
-
end
|
|
180
|
-
|
|
181
|
-
def oauth_token(force_refresh: false)
|
|
182
|
-
assert_values(force_refresh, [true, false])
|
|
183
|
-
return oauth.get_authorization(use_refresh_token: force_refresh)
|
|
184
|
-
end
|
|
185
|
-
|
|
186
|
-
def build_request(call_data)
|
|
156
|
+
def build_request(
|
|
157
|
+
operation:,
|
|
158
|
+
subpath:,
|
|
159
|
+
url_params:,
|
|
160
|
+
json_params:,
|
|
161
|
+
www_body_params:,
|
|
162
|
+
text_body_params:,
|
|
163
|
+
headers:
|
|
164
|
+
)
|
|
187
165
|
# TODO: shall we percent encode subpath (spaces) test with access key delete with space in id
|
|
188
166
|
# URI.escape()
|
|
189
|
-
|
|
167
|
+
separator = !['', '/'].include?(subpath) || @base_url.end_with?('/') ? '/' : ''
|
|
168
|
+
uri = self.class.build_uri("#{@base_url}#{separator}#{subpath}", url_params)
|
|
190
169
|
Log.log.debug{"URI=#{uri}"}
|
|
191
170
|
begin
|
|
192
171
|
# instantiate request object based on string name
|
|
193
|
-
req = Net::HTTP.const_get(
|
|
172
|
+
req = Net::HTTP.const_get(operation.capitalize).new(uri)
|
|
194
173
|
rescue NameError
|
|
195
|
-
raise "unsupported operation : #{
|
|
174
|
+
raise "unsupported operation : #{operation}"
|
|
196
175
|
end
|
|
197
|
-
if
|
|
198
|
-
req.body = JSON.generate(
|
|
199
|
-
Log.log.debug{Log.dump('body JSON data', call_data[:json_params])}
|
|
176
|
+
if !json_params.nil?
|
|
177
|
+
req.body = JSON.generate(json_params) # , ascii_only: true
|
|
200
178
|
req['Content-Type'] = 'application/json'
|
|
201
|
-
# call_data[:headers]['Accept']='application/json'
|
|
202
179
|
end
|
|
203
|
-
if
|
|
204
|
-
req.body = URI.encode_www_form(
|
|
205
|
-
Log.log.debug{"body www data=#{req.body.chomp}"}
|
|
180
|
+
if !www_body_params.nil?
|
|
181
|
+
req.body = URI.encode_www_form(www_body_params)
|
|
206
182
|
req['Content-Type'] = 'application/x-www-form-urlencoded'
|
|
207
183
|
end
|
|
208
|
-
if
|
|
209
|
-
req.body =
|
|
210
|
-
Log.log.debug{"body data=#{req.body.chomp}"}
|
|
184
|
+
if !text_body_params.nil?
|
|
185
|
+
req.body = text_body_params
|
|
211
186
|
end
|
|
212
187
|
# set headers
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
req[key] = call_data[:headers][key]
|
|
216
|
-
end
|
|
188
|
+
headers.each do |key, value|
|
|
189
|
+
req[key] = value
|
|
217
190
|
end
|
|
218
191
|
# :type = :basic
|
|
219
|
-
req.basic_auth(
|
|
192
|
+
req.basic_auth(@auth_params[:username], @auth_params[:password]) if @auth_params[:type].eql?(:basic)
|
|
220
193
|
Log.log.debug{Log.dump(:req_body, req.body)}
|
|
221
194
|
return req
|
|
222
195
|
end
|
|
223
196
|
|
|
197
|
+
public
|
|
198
|
+
|
|
199
|
+
attr_reader :auth_params
|
|
200
|
+
attr_reader :base_url
|
|
201
|
+
|
|
202
|
+
def params
|
|
203
|
+
return {
|
|
204
|
+
base_url: @base_url,
|
|
205
|
+
auth: @auth_params,
|
|
206
|
+
not_auth_codes: @not_auth_codes,
|
|
207
|
+
redirect_max: @redirect_max,
|
|
208
|
+
headers: @headers
|
|
209
|
+
}
|
|
210
|
+
end
|
|
211
|
+
|
|
212
|
+
# @param base_url [String] base URL of REST API
|
|
213
|
+
# @param auth [Hash] authentication parameters:
|
|
214
|
+
# :type (:none, :basic, :url, :oauth2)
|
|
215
|
+
# :username [:basic]
|
|
216
|
+
# :password [:basic]
|
|
217
|
+
# :url_query [:url] a hash
|
|
218
|
+
# :* [:oauth2] see OAuth::Factory class
|
|
219
|
+
# @param not_auth_codes [Array] codes that trigger a refresh/regeneration of bearer token
|
|
220
|
+
# @param redirect_max [int] max redirections allowed
|
|
221
|
+
def initialize(
|
|
222
|
+
base_url:,
|
|
223
|
+
auth: nil,
|
|
224
|
+
not_auth_codes: nil,
|
|
225
|
+
redirect_max: 0,
|
|
226
|
+
headers: nil
|
|
227
|
+
)
|
|
228
|
+
Aspera.assert_type(base_url, String)
|
|
229
|
+
# base url with max one trailing slashes (note: string may be frozen)
|
|
230
|
+
@base_url = base_url.gsub(%r{//+$}, '/')
|
|
231
|
+
# default is no auth
|
|
232
|
+
@auth_params = auth.nil? ? {type: :none} : auth
|
|
233
|
+
Aspera.assert_type(@auth_params, Hash)
|
|
234
|
+
Aspera.assert(@auth_params.key?(:type)){'no auth type defined'}
|
|
235
|
+
@not_auth_codes = not_auth_codes.nil? ? ['401'] : not_auth_codes
|
|
236
|
+
Aspera.assert_type(@not_auth_codes, Array)
|
|
237
|
+
# persistent session
|
|
238
|
+
@http_session = nil
|
|
239
|
+
# OAuth object (created on demand)
|
|
240
|
+
@oauth = nil
|
|
241
|
+
@redirect_max = redirect_max
|
|
242
|
+
@headers = headers.nil? ? {} : headers
|
|
243
|
+
Aspera.assert_type(@headers, Hash)
|
|
244
|
+
@headers['User-Agent'] ||= @@global[:user_agent]
|
|
245
|
+
end
|
|
246
|
+
|
|
247
|
+
# @return the OAuth object (create, or cached if already created)
|
|
248
|
+
def oauth
|
|
249
|
+
if @oauth.nil?
|
|
250
|
+
Aspera.assert(@auth_params[:type].eql?(:oauth2)){'no OAuth defined'}
|
|
251
|
+
oauth_parameters = @auth_params.reject { |k, _v| k.eql?(:type) }
|
|
252
|
+
Log.log.debug{Log.dump('oauth parameters', oauth_parameters)}
|
|
253
|
+
@oauth = OAuth::Factory.instance.create(**oauth_parameters)
|
|
254
|
+
end
|
|
255
|
+
return @oauth
|
|
256
|
+
end
|
|
257
|
+
|
|
258
|
+
def oauth_token(force_refresh: false)
|
|
259
|
+
Aspera.assert_values(force_refresh, [true, false])
|
|
260
|
+
return oauth.get_authorization(use_refresh_token: force_refresh)
|
|
261
|
+
end
|
|
262
|
+
|
|
224
263
|
# HTTP/S REST call
|
|
225
|
-
#
|
|
226
|
-
#
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
# defaults from @params are overridden by call data
|
|
252
|
-
call_data = @params.deep_merge(call_data)
|
|
253
|
-
case call_data[:auth][:type]
|
|
264
|
+
# @param save_to_file (filepath)
|
|
265
|
+
# @param return_error (bool)
|
|
266
|
+
def call(
|
|
267
|
+
operation:,
|
|
268
|
+
subpath: nil,
|
|
269
|
+
json_params: nil,
|
|
270
|
+
url_params: nil,
|
|
271
|
+
www_body_params: nil,
|
|
272
|
+
text_body_params: nil,
|
|
273
|
+
save_to_file: nil,
|
|
274
|
+
return_error: false,
|
|
275
|
+
headers: nil
|
|
276
|
+
)
|
|
277
|
+
subpath = subpath.to_s if subpath.is_a?(Symbol)
|
|
278
|
+
subpath = '' if subpath.nil?
|
|
279
|
+
Aspera.assert_type(subpath, String)
|
|
280
|
+
if headers.nil?
|
|
281
|
+
headers = @headers.clone
|
|
282
|
+
else
|
|
283
|
+
h = headers
|
|
284
|
+
headers = @headers.clone
|
|
285
|
+
headers.merge!(h)
|
|
286
|
+
end
|
|
287
|
+
Aspera.assert_type(headers, Hash)
|
|
288
|
+
Log.log.debug{"#{operation} [#{subpath}]".red.bold.bg_green}
|
|
289
|
+
case @auth_params[:type]
|
|
254
290
|
when :none
|
|
255
291
|
# no auth
|
|
256
292
|
when :basic
|
|
257
293
|
Log.log.debug('using Basic auth')
|
|
258
294
|
# done in build_req
|
|
259
295
|
when :oauth2
|
|
260
|
-
|
|
296
|
+
headers['Authorization'] = oauth_token unless headers.key?('Authorization')
|
|
261
297
|
when :url
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
298
|
+
url_params ||= {}
|
|
299
|
+
@auth_params[:url_query].each do |key, value|
|
|
300
|
+
url_params[key] = value
|
|
265
301
|
end
|
|
266
|
-
else error_unexpected_value(
|
|
302
|
+
else Aspera.error_unexpected_value(@auth_params[:type])
|
|
267
303
|
end
|
|
268
|
-
req = build_request(call_data)
|
|
269
|
-
Log.log.debug{"call_data = #{call_data}"}
|
|
270
304
|
result = {http: nil}
|
|
271
305
|
# start a block to be able to retry the actual HTTP request
|
|
272
306
|
begin
|
|
307
|
+
req = build_request(
|
|
308
|
+
operation: operation, subpath: subpath, json_params: json_params, url_params: url_params, www_body_params: www_body_params,
|
|
309
|
+
text_body_params: text_body_params, headers: headers)
|
|
273
310
|
# we try the call, and will retry only if oauth, as we can, first with refresh, and then re-auth if refresh is bad
|
|
274
311
|
oauth_tries ||= 2
|
|
275
312
|
# initialize with number of initial retries allowed, nil gives zero
|
|
276
|
-
tries_remain_redirect =
|
|
313
|
+
tries_remain_redirect = @redirect_max.to_i if tries_remain_redirect.nil?
|
|
277
314
|
Log.log.debug("send request (retries=#{tries_remain_redirect})")
|
|
278
315
|
result_mime = nil
|
|
279
316
|
file_saved = false
|
|
@@ -282,13 +319,13 @@ module Aspera
|
|
|
282
319
|
result[:http] = response
|
|
283
320
|
result_mime = (result[:http]['Content-Type'] || 'text/plain').split(';').first.downcase
|
|
284
321
|
# JSON data needs to be parsed, in case it contains an error code
|
|
285
|
-
if !
|
|
322
|
+
if !save_to_file.nil? &&
|
|
286
323
|
result[:http].code.to_s.start_with?('2') &&
|
|
287
324
|
!result[:http]['Content-Length'].nil? &&
|
|
288
325
|
!JSON_DECODE.include?(result_mime)
|
|
289
326
|
total_size = result[:http]['Content-Length'].to_i
|
|
290
327
|
Log.log.debug('before write file')
|
|
291
|
-
target_file =
|
|
328
|
+
target_file = save_to_file
|
|
292
329
|
# override user's path to path in header
|
|
293
330
|
if !response['Content-Disposition'].nil? && (m = response['Content-Disposition'].match(/filename="([^"]+)"/))
|
|
294
331
|
target_file = File.join(File.dirname(target_file), m[1])
|
|
@@ -321,13 +358,13 @@ module Aspera
|
|
|
321
358
|
else # when 'text/plain'
|
|
322
359
|
result[:http].body
|
|
323
360
|
end
|
|
324
|
-
Log.log.debug{
|
|
325
|
-
Log.log.debug{
|
|
361
|
+
Log.log.debug{"result: code=#{result[:http].code} mime=#{result_mime}"}
|
|
362
|
+
Log.log.debug{Log.dump('data', result[:data])}
|
|
326
363
|
RestErrorAnalyzer.instance.raise_on_error(req, result)
|
|
327
|
-
File.write(
|
|
364
|
+
File.write(save_to_file, result[:http].body) unless file_saved || save_to_file.nil?
|
|
328
365
|
rescue RestCallError => e
|
|
329
366
|
# not authorized: oauth token expired
|
|
330
|
-
if
|
|
367
|
+
if @not_auth_codes.include?(result[:http].code.to_s) && @auth_params[:type].eql?(:oauth2)
|
|
331
368
|
begin
|
|
332
369
|
# try to use refresh token
|
|
333
370
|
req['Authorization'] = oauth_token(force_refresh: true)
|
|
@@ -337,35 +374,28 @@ module Aspera
|
|
|
337
374
|
# regenerate a brand new token
|
|
338
375
|
req['Authorization'] = oauth_token(force_refresh: true)
|
|
339
376
|
end
|
|
340
|
-
Log.log.debug{"using new token=#{
|
|
341
|
-
retry
|
|
377
|
+
Log.log.debug{"using new token=#{headers['Authorization']}"}
|
|
378
|
+
retry if (oauth_tries -= 1).nonzero?
|
|
342
379
|
end # if oauth
|
|
343
380
|
# redirect ? (any code beginning with 3)
|
|
344
|
-
if
|
|
381
|
+
if e.response.is_a?(Net::HTTPRedirection) && tries_remain_redirect.positive?
|
|
345
382
|
tries_remain_redirect -= 1
|
|
346
|
-
current_uri = URI.parse(
|
|
383
|
+
current_uri = URI.parse(@base_url)
|
|
347
384
|
new_url = e.response['location']
|
|
348
385
|
# special case: relative redirect
|
|
349
386
|
if URI.parse(new_url).host.nil?
|
|
350
387
|
# we don't manage relative redirects with non-absolute path
|
|
351
|
-
assert(new_url.start_with?('/')){"redirect location is relative: #{new_url}, but does not start with /."}
|
|
352
|
-
new_url = current_uri.scheme
|
|
353
|
-
end
|
|
354
|
-
Log.log.info{"URL is moved: #{new_url}"}
|
|
355
|
-
redirection_uri = URI.parse(new_url)
|
|
356
|
-
call_data[:base_url] = new_url
|
|
357
|
-
call_data[:subpath] = ''
|
|
358
|
-
if current_uri.host.eql?(redirection_uri.host) && current_uri.port.eql?(redirection_uri.port)
|
|
359
|
-
req = build_request(call_data)
|
|
360
|
-
retry
|
|
361
|
-
else
|
|
362
|
-
# change host
|
|
363
|
-
Log.log.info{"Redirect changes host: #{current_uri.host} -> #{redirection_uri.host}"}
|
|
364
|
-
return self.class.new(call_data).call(call_data)
|
|
388
|
+
Aspera.assert(new_url.start_with?('/')){"redirect location is relative: #{new_url}, but does not start with /."}
|
|
389
|
+
new_url = "#{current_uri.scheme}://#{current_uri.host}#{new_url}"
|
|
365
390
|
end
|
|
391
|
+
# forwards the request to the new location
|
|
392
|
+
return self.class.new(base_url: new_url, redirect_max: tries_remain_redirect).call(
|
|
393
|
+
operation: operation, json_params: json_params,
|
|
394
|
+
url_params: url_params, www_body_params: www_body_params, text_body_params: text_body_params,
|
|
395
|
+
save_to_file: save_to_file, return_error: return_error, headers: headers)
|
|
366
396
|
end
|
|
367
397
|
# raise exception if could not retry and not return error in result
|
|
368
|
-
raise e unless
|
|
398
|
+
raise e unless return_error
|
|
369
399
|
end # begin request
|
|
370
400
|
Log.log.debug{"result=#{result}"}
|
|
371
401
|
return result
|
|
@@ -377,36 +407,37 @@ module Aspera
|
|
|
377
407
|
|
|
378
408
|
# @param encoding : one of: :json_params, :url_params
|
|
379
409
|
def create(subpath, params, encoding=:json_params)
|
|
380
|
-
return call(
|
|
410
|
+
return call(operation: 'POST', subpath: subpath, headers: {'Accept' => 'application/json'}, encoding => params)
|
|
381
411
|
end
|
|
382
412
|
|
|
383
|
-
def read(subpath,
|
|
384
|
-
return call(
|
|
413
|
+
def read(subpath, query=nil)
|
|
414
|
+
return call(operation: 'GET', subpath: subpath, headers: {'Accept' => 'application/json'}, url_params: query)
|
|
385
415
|
end
|
|
386
416
|
|
|
387
417
|
def update(subpath, params)
|
|
388
|
-
return call(
|
|
418
|
+
return call(operation: 'PUT', subpath: subpath, headers: {'Accept' => 'application/json'}, json_params: params)
|
|
389
419
|
end
|
|
390
420
|
|
|
391
421
|
def delete(subpath, params=nil)
|
|
392
|
-
return call(
|
|
422
|
+
return call(operation: 'DELETE', subpath: subpath, headers: {'Accept' => 'application/json'}, url_params: params)
|
|
393
423
|
end
|
|
394
424
|
|
|
395
425
|
def cancel(subpath)
|
|
396
|
-
return call(
|
|
426
|
+
return call(operation: 'CANCEL', subpath: subpath, headers: {'Accept' => 'application/json'})
|
|
397
427
|
end
|
|
398
428
|
|
|
399
|
-
# Query by
|
|
429
|
+
# Query entity by general search (read with parameter `q`)
|
|
430
|
+
# TODO: not generic enough ? move somewhere ? inheritance ?
|
|
400
431
|
# @param subpath path of entity in API
|
|
401
432
|
# @param search_name name of searched entity
|
|
402
|
-
# @param
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
433
|
+
# @param query additional search query parameters
|
|
434
|
+
# @returns [Hash] A single entity matching the search, or an exception if not found or multiple found
|
|
435
|
+
def lookup_by_name(subpath, search_name, query={})
|
|
436
|
+
# returns entities matching the query (it matches against several fields in case insensitive way)
|
|
437
|
+
matching_items = read(subpath, query.merge({'q' => search_name}))[:data]
|
|
406
438
|
# API style: {totalcount:, ...} cspell: disable-line
|
|
407
|
-
# TODO: not generic enough ? move somewhere ? inheritance ?
|
|
408
439
|
matching_items = matching_items[subpath] if matching_items.is_a?(Hash)
|
|
409
|
-
assert_type(matching_items, Array)
|
|
440
|
+
Aspera.assert_type(matching_items, Array)
|
|
410
441
|
case matching_items.length
|
|
411
442
|
when 1 then return matching_items.first
|
|
412
443
|
when 0 then raise %Q{#{ENTITY_NOT_FOUND} #{subpath}: "#{search_name}"}
|
|
@@ -15,10 +15,10 @@ module Aspera
|
|
|
15
15
|
RestErrorAnalyzer.instance.add_simple_handler(name: 'Type 1: error:user_message', path: %w[error user_message], always: true)
|
|
16
16
|
RestErrorAnalyzer.instance.add_simple_handler(name: 'Type 2: error:description', path: %w[error description])
|
|
17
17
|
RestErrorAnalyzer.instance.add_simple_handler(name: 'Type 3: error:internal_message', path: %w[error internal_message])
|
|
18
|
-
# AoC Automation
|
|
19
|
-
RestErrorAnalyzer.instance.add_simple_handler(name: 'AoC Automation', path: ['error'])
|
|
20
18
|
RestErrorAnalyzer.instance.add_simple_handler(name: 'Type 5', path: ['error_description'])
|
|
21
19
|
RestErrorAnalyzer.instance.add_simple_handler(name: 'Type 6', path: ['message'])
|
|
20
|
+
# AoC Automation
|
|
21
|
+
RestErrorAnalyzer.instance.add_simple_handler(name: 'AoC Automation', path: ['error'])
|
|
22
22
|
RestErrorAnalyzer.instance.add_handler('Type 7: errors[]') do |type, call_context|
|
|
23
23
|
next unless call_context[:data].is_a?(Hash) && call_context[:data]['errors'].is_a?(Hash)
|
|
24
24
|
# special for Shares: false positive ? (update global transfer_settings)
|
|
@@ -33,7 +33,7 @@ module Aspera
|
|
|
33
33
|
d_t_s = call_context[:data]['transfer_specs']
|
|
34
34
|
next unless d_t_s.is_a?(Array)
|
|
35
35
|
d_t_s.each do |res|
|
|
36
|
-
r_err = res.dig(*%w[transfer_spec error])
|
|
36
|
+
r_err = res.dig(*%w[transfer_spec error]) || res['error']
|
|
37
37
|
next unless r_err.is_a?(Hash)
|
|
38
38
|
RestErrorAnalyzer.add_error(call_context, type, "#{r_err['code']}: #{r_err['reason']}: #{r_err['user_message']}")
|
|
39
39
|
end
|