aspera-cli 4.10.0 → 4.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/BUGS.md +19 -0
- data/CHANGELOG.md +528 -0
- data/CONTRIBUTING.md +143 -0
- data/README.md +977 -589
- data/bin/ascli +4 -4
- data/bin/asession +12 -12
- data/docs/test_env.conf +29 -19
- data/examples/aoc.rb +6 -6
- data/examples/dascli +18 -16
- data/examples/faspex4.rb +15 -15
- data/examples/node.rb +12 -12
- data/examples/proxy.pac +2 -2
- data/examples/server.rb +12 -12
- data/lib/aspera/aoc.rb +344 -272
- data/lib/aspera/ascmd.rb +56 -54
- data/lib/aspera/ats_api.rb +4 -4
- data/lib/aspera/cli/basic_auth_plugin.rb +15 -12
- data/lib/aspera/cli/extended_value.rb +9 -9
- data/lib/aspera/cli/{formater.rb → formatter.rb} +69 -69
- data/lib/aspera/cli/listener/line_dump.rb +1 -1
- data/lib/aspera/cli/listener/logger.rb +1 -1
- data/lib/aspera/cli/listener/progress.rb +5 -6
- data/lib/aspera/cli/listener/progress_multi.rb +16 -21
- data/lib/aspera/cli/main.rb +72 -73
- data/lib/aspera/cli/manager.rb +112 -112
- data/lib/aspera/cli/plugin.rb +68 -48
- data/lib/aspera/cli/plugins/alee.rb +4 -4
- data/lib/aspera/cli/plugins/aoc.rb +322 -720
- data/lib/aspera/cli/plugins/ats.rb +50 -52
- data/lib/aspera/cli/plugins/bss.rb +10 -10
- data/lib/aspera/cli/plugins/config.rb +514 -410
- data/lib/aspera/cli/plugins/console.rb +12 -12
- data/lib/aspera/cli/plugins/cos.rb +18 -20
- data/lib/aspera/cli/plugins/faspex.rb +134 -136
- data/lib/aspera/cli/plugins/faspex5.rb +235 -70
- data/lib/aspera/cli/plugins/node.rb +378 -309
- data/lib/aspera/cli/plugins/orchestrator.rb +52 -49
- data/lib/aspera/cli/plugins/preview.rb +129 -120
- data/lib/aspera/cli/plugins/server.rb +137 -83
- data/lib/aspera/cli/plugins/shares.rb +77 -52
- data/lib/aspera/cli/plugins/sync.rb +13 -33
- data/lib/aspera/cli/transfer_agent.rb +61 -61
- data/lib/aspera/cli/version.rb +2 -1
- data/lib/aspera/colors.rb +3 -3
- data/lib/aspera/command_line_builder.rb +78 -74
- data/lib/aspera/cos_node.rb +31 -29
- data/lib/aspera/data_repository.rb +1 -1
- data/lib/aspera/environment.rb +30 -28
- data/lib/aspera/fasp/agent_base.rb +17 -15
- data/lib/aspera/fasp/agent_connect.rb +34 -32
- data/lib/aspera/fasp/agent_direct.rb +70 -73
- data/lib/aspera/fasp/agent_httpgw.rb +79 -74
- data/lib/aspera/fasp/agent_node.rb +26 -26
- data/lib/aspera/fasp/agent_trsdk.rb +20 -20
- data/lib/aspera/fasp/error.rb +3 -2
- data/lib/aspera/fasp/error_info.rb +11 -8
- data/lib/aspera/fasp/installation.rb +80 -80
- data/lib/aspera/fasp/listener.rb +2 -2
- data/lib/aspera/fasp/parameters.rb +103 -92
- data/lib/aspera/fasp/parameters.yaml +313 -214
- data/lib/aspera/fasp/resume_policy.rb +10 -10
- data/lib/aspera/fasp/transfer_spec.rb +22 -2
- data/lib/aspera/fasp/uri.rb +7 -7
- data/lib/aspera/faspex_gw.rb +80 -159
- data/lib/aspera/faspex_postproc.rb +77 -0
- data/lib/aspera/hash_ext.rb +3 -3
- data/lib/aspera/id_generator.rb +5 -5
- data/lib/aspera/keychain/encrypted_hash.rb +23 -28
- data/lib/aspera/keychain/macos_security.rb +21 -20
- data/lib/aspera/log.rb +13 -13
- data/lib/aspera/nagios.rb +24 -23
- data/lib/aspera/node.rb +217 -38
- data/lib/aspera/oauth.rb +78 -74
- data/lib/aspera/open_application.rb +19 -11
- data/lib/aspera/persistency_action_once.rb +4 -4
- data/lib/aspera/persistency_folder.rb +13 -13
- data/lib/aspera/preview/file_types.rb +8 -8
- data/lib/aspera/preview/generator.rb +67 -67
- data/lib/aspera/preview/utils.rb +27 -27
- data/lib/aspera/proxy_auto_config.js +63 -63
- data/lib/aspera/proxy_auto_config.rb +19 -19
- data/lib/aspera/rest.rb +65 -67
- data/lib/aspera/rest_call_error.rb +2 -1
- data/lib/aspera/rest_error_analyzer.rb +22 -21
- data/lib/aspera/rest_errors_aspera.rb +16 -16
- data/lib/aspera/secret_hider.rb +17 -14
- data/lib/aspera/ssh.rb +15 -14
- data/lib/aspera/sync.rb +177 -62
- data/lib/aspera/temp_file_manager.rb +2 -2
- data/lib/aspera/uri_reader.rb +4 -4
- data/lib/aspera/web_auth.rb +13 -64
- data/lib/aspera/web_server_simple.rb +76 -0
- data.tar.gz.sig +0 -0
- metadata +11 -6
- metadata.gz.sig +0 -0
@@ -11,20 +11,20 @@ module URI
|
|
11
11
|
def register_proxy_finder
|
12
12
|
raise 'mandatory block missing' unless Kernel.block_given?
|
13
13
|
# overload the method in URI : call user's provided block and fallback to original method
|
14
|
-
define_method(:find_proxy) {|
|
14
|
+
define_method(:find_proxy) {|env_vars=ENV| yield(to_s) || find_proxy_orig(env_vars)}
|
15
15
|
end
|
16
16
|
end
|
17
17
|
end
|
18
18
|
end
|
19
19
|
|
20
20
|
module Aspera
|
21
|
-
# Evaluate a proxy
|
21
|
+
# Evaluate a proxy auto config script
|
22
22
|
class ProxyAutoConfig
|
23
23
|
# template file is read once, it contains functions that can be used in a proxy autoconf script
|
24
24
|
# it is similar to mozilla ascii_pac_utils.inc
|
25
|
-
PAC_FUNCTIONS_FILE = __FILE__.gsub(/\.rb$/,'.js').freeze
|
25
|
+
PAC_FUNCTIONS_FILE = __FILE__.gsub(/\.rb$/, '.js').freeze
|
26
26
|
PAC_MAIN_FUNCTION = 'FindProxyForURL'
|
27
|
-
private_constant :PAC_FUNCTIONS_FILE
|
27
|
+
private_constant :PAC_FUNCTIONS_FILE, :PAC_MAIN_FUNCTION
|
28
28
|
|
29
29
|
private
|
30
30
|
|
@@ -52,7 +52,7 @@ END_OF_JAVASCRIPT
|
|
52
52
|
|
53
53
|
public
|
54
54
|
|
55
|
-
attr_writer :proxy_user
|
55
|
+
attr_writer :proxy_user, :proxy_pass
|
56
56
|
|
57
57
|
# @param proxy_auto_config the proxy auto config script to be evaluated
|
58
58
|
def initialize(proxy_auto_config)
|
@@ -60,7 +60,7 @@ END_OF_JAVASCRIPT
|
|
60
60
|
@proxy_auto_config = proxy_auto_config
|
61
61
|
# avoid multiple execution, this does not support load balancing
|
62
62
|
@cache = {}
|
63
|
-
@proxy_user
|
63
|
+
@proxy_user = @proxy_pass = @pac_functions = nil
|
64
64
|
end
|
65
65
|
|
66
66
|
def register_uri_generic
|
@@ -70,12 +70,12 @@ END_OF_JAVASCRIPT
|
|
70
70
|
end
|
71
71
|
|
72
72
|
# execute proxy auto config script for the given URL : https://en.wikipedia.org/wiki/Proxy_auto-config
|
73
|
-
# @return either nil, or a String
|
73
|
+
# @return either nil, or a String formatted following PAC standard
|
74
74
|
def find_proxy_for_url(service_url)
|
75
75
|
uri = URI.parse(service_url)
|
76
76
|
simple_url = "#{uri.scheme}://#{uri.host}"
|
77
|
-
if !@cache.
|
78
|
-
Log.log.debug
|
77
|
+
if !@cache.key?(simple_url)
|
78
|
+
Log.log.debug{"PAC: starting javascript for #{service_url}"}
|
79
79
|
# require at runtime, in case there is no js engine
|
80
80
|
require 'execjs'
|
81
81
|
# read template lib
|
@@ -84,7 +84,7 @@ END_OF_JAVASCRIPT
|
|
84
84
|
js_to_execute = "#{pac_dns_functions(uri.host)}#{@pac_functions}#{@proxy_auto_config}"
|
85
85
|
executable_js = ExecJS.compile(js_to_execute)
|
86
86
|
@cache[simple_url] = executable_js.call(PAC_MAIN_FUNCTION, simple_url, uri.host)
|
87
|
-
Log.log.debug
|
87
|
+
Log.log.debug{"PAC: result: #{@cache[simple_url]}"}
|
88
88
|
end
|
89
89
|
return @cache[simple_url]
|
90
90
|
end
|
@@ -97,11 +97,11 @@ END_OF_JAVASCRIPT
|
|
97
97
|
# execute PAC script
|
98
98
|
proxy_list_str = find_proxy_for_url(service_url)
|
99
99
|
if !proxy_list_str.is_a?(String)
|
100
|
-
Log.log.warn
|
100
|
+
Log.log.warn{"PAC: did not return a String, returned #{proxy_list_str.class}"}
|
101
101
|
return uri_list
|
102
102
|
end
|
103
103
|
proxy_list_str.strip!
|
104
|
-
proxy_list_str.gsub!(/\s+/,' ')
|
104
|
+
proxy_list_str.gsub!(/\s+/, ' ')
|
105
105
|
proxy_list_str.split(';').each do |item|
|
106
106
|
# strip and split by space
|
107
107
|
parts = item.strip.split
|
@@ -115,21 +115,21 @@ END_OF_JAVASCRIPT
|
|
115
115
|
begin
|
116
116
|
# PAC proxy addresses are <host>:<port>
|
117
117
|
if /:[0-9]+$/.match?(addr_port)
|
118
|
-
uri=URI.parse("proxy://#{addr_port}")
|
118
|
+
uri = URI.parse("proxy://#{addr_port}")
|
119
119
|
# ruby v>2.6 allows
|
120
|
-
uri.user
|
121
|
-
uri.password
|
120
|
+
uri.user = @proxy_user
|
121
|
+
uri.password = @proxy_pass
|
122
122
|
uri_list.push(uri)
|
123
123
|
else
|
124
|
-
Log.log.warn
|
124
|
+
Log.log.warn{"PAC: PROXY must be <address>:<port>, ignoring #{addr_port}"}
|
125
125
|
end
|
126
126
|
rescue StandardError => e
|
127
|
-
Log.log.warn
|
127
|
+
Log.log.warn{"PAC: cannot parse #{addr_port} #{e}"}
|
128
128
|
end
|
129
|
-
else Log.log.warn
|
129
|
+
else Log.log.warn{"PAC: ignoring proxy type #{parts.first}: not supported"}
|
130
130
|
end
|
131
131
|
end
|
132
|
-
Log.log.debug
|
132
|
+
Log.log.debug{"Proxies: #{uri_list}"}
|
133
133
|
return uri_list
|
134
134
|
end
|
135
135
|
end
|
data/lib/aspera/rest.rb
CHANGED
@@ -13,18 +13,12 @@ require 'cgi'
|
|
13
13
|
require 'ruby-progressbar'
|
14
14
|
|
15
15
|
# add cancel method to http
|
16
|
-
class Net::HTTP::Cancel < Net::HTTPRequest
|
16
|
+
class Net::HTTP::Cancel < Net::HTTPRequest # rubocop:disable Style/ClassAndModuleChildren
|
17
17
|
METHOD = 'CANCEL'
|
18
18
|
REQUEST_HAS_BODY = false
|
19
19
|
RESPONSE_HAS_BODY = false
|
20
20
|
end
|
21
21
|
|
22
|
-
#class Net::HTTP::Delete < Net::HTTPRequest
|
23
|
-
# METHOD = 'DELETE'
|
24
|
-
# REQUEST_HAS_BODY = false
|
25
|
-
# RESPONSE_HAS_BODY = false
|
26
|
-
#end
|
27
|
-
|
28
22
|
module Aspera
|
29
23
|
# a simple class to make HTTP calls, equivalent to rest-client
|
30
24
|
# rest call errors are raised as exception RestCallError
|
@@ -42,17 +36,22 @@ module Aspera
|
|
42
36
|
proxy_pass: nil
|
43
37
|
}
|
44
38
|
|
39
|
+
ARRAY_PARAMS = '[]'
|
40
|
+
|
45
41
|
class << self
|
46
42
|
# define accessors
|
47
|
-
@@global.
|
43
|
+
@@global.each_key do |p|
|
48
44
|
define_method(p){@@global[p]}
|
49
|
-
define_method("#{p}=")
|
45
|
+
define_method("#{p}=") do |val|
|
46
|
+
Log.log.debug{"#{p} => #{val}".red}
|
47
|
+
@@global[p] = val
|
48
|
+
end
|
50
49
|
end
|
51
50
|
|
52
|
-
def basic_creds(user,pass); return "Basic #{Base64.strict_encode64("#{user}:#{pass}")}";end
|
51
|
+
def basic_creds(user, pass); return "Basic #{Base64.strict_encode64("#{user}:#{pass}")}"; end
|
53
52
|
|
54
53
|
# build URI from URL and parameters and check it is http or https
|
55
|
-
def build_uri(url,params=nil)
|
54
|
+
def build_uri(url, params=nil)
|
56
55
|
uri = URI.parse(url)
|
57
56
|
raise "REST endpoint shall be http/s not #{uri.scheme}" unless %w[http https].include?(uri.scheme)
|
58
57
|
if !params.nil?
|
@@ -60,15 +59,15 @@ module Aspera
|
|
60
59
|
if params.is_a?(Hash)
|
61
60
|
orig = params
|
62
61
|
params = []
|
63
|
-
orig.each do |k,v|
|
62
|
+
orig.each do |k, v|
|
64
63
|
case v
|
65
64
|
when Array
|
66
|
-
suffix = v.first.eql?(
|
65
|
+
suffix = v.first.eql?(ARRAY_PARAMS) ? v.shift : ''
|
67
66
|
v.each do |e|
|
68
|
-
params.push([k + suffix,e])
|
67
|
+
params.push([k.to_s + suffix, e])
|
69
68
|
end
|
70
69
|
else
|
71
|
-
params.push([k,v])
|
70
|
+
params.push([k, v])
|
72
71
|
end
|
73
72
|
end
|
74
73
|
end
|
@@ -82,8 +81,8 @@ module Aspera
|
|
82
81
|
uri = build_uri(base_url)
|
83
82
|
# this honors http_proxy env var
|
84
83
|
http_session = Net::HTTP.new(uri.host, uri.port)
|
85
|
-
http_session.proxy_user=proxy_user
|
86
|
-
http_session.proxy_pass=proxy_pass
|
84
|
+
http_session.proxy_user = proxy_user
|
85
|
+
http_session.proxy_pass = proxy_pass
|
87
86
|
http_session.use_ssl = uri.scheme.eql?('https')
|
88
87
|
http_session.set_debug_output($stdout) if debug
|
89
88
|
# set http options in callback, such as timeout and cert. verification
|
@@ -99,7 +98,7 @@ module Aspera
|
|
99
98
|
# create and start keep alive connection on demand
|
100
99
|
def http_session
|
101
100
|
if @http_session.nil?
|
102
|
-
@http_session=self.class.start_http_session(@params[:base_url])
|
101
|
+
@http_session = self.class.start_http_session(@params[:base_url])
|
103
102
|
end
|
104
103
|
return @http_session
|
105
104
|
end
|
@@ -121,57 +120,56 @@ module Aspera
|
|
121
120
|
raise 'ERROR: expecting Hash' unless a_rest_params.is_a?(Hash)
|
122
121
|
raise 'ERROR: expecting base_url' unless a_rest_params[:base_url].is_a?(String)
|
123
122
|
@params = a_rest_params.clone
|
124
|
-
Log.dump('REST params'
|
123
|
+
Log.dump('REST params', @params)
|
125
124
|
# base url without trailing slashes (note: string may be frozen)
|
126
|
-
@params[:base_url] = @params[:base_url].gsub(
|
125
|
+
@params[:base_url] = @params[:base_url].gsub(%r{/+$}, '')
|
127
126
|
@http_session = nil
|
128
127
|
# default is no auth
|
129
128
|
@params[:auth] ||= {type: :none}
|
130
129
|
@params[:not_auth_codes] ||= ['401']
|
131
130
|
@oauth = nil
|
132
|
-
Log.dump('REST params(2)'
|
131
|
+
Log.dump('REST params(2)', @params)
|
133
132
|
end
|
134
133
|
|
135
134
|
def oauth_token(force_refresh: false)
|
136
|
-
raise "ERROR: expecting boolean, have #{force_refresh}" unless [true,false].include?(force_refresh)
|
135
|
+
raise "ERROR: expecting boolean, have #{force_refresh}" unless [true, false].include?(force_refresh)
|
137
136
|
return oauth.get_authorization(use_refresh_token: force_refresh)
|
138
137
|
end
|
139
138
|
|
140
139
|
def build_request(call_data)
|
141
140
|
# TODO: shall we percent encode subpath (spaces) test with access key delete with space in id
|
142
141
|
# URI.escape()
|
143
|
-
uri = self.class.build_uri("#{call_data[:base_url]}#{['','/'].include?(call_data[:subpath]) ? '' : '/'}#{call_data[:subpath]}",call_data[:url_params])
|
144
|
-
Log.log.debug
|
142
|
+
uri = self.class.build_uri("#{call_data[:base_url]}#{['', '/'].include?(call_data[:subpath]) ? '' : '/'}#{call_data[:subpath]}", call_data[:url_params])
|
143
|
+
Log.log.debug{"URI=#{uri}"}
|
145
144
|
begin
|
146
|
-
#
|
145
|
+
# instantiate request object based on string name
|
147
146
|
req = Net::HTTP.const_get(call_data[:operation].capitalize).new(uri)
|
148
147
|
rescue NameError
|
149
148
|
raise "unsupported operation : #{call_data[:operation]}"
|
150
149
|
end
|
151
|
-
if call_data.
|
150
|
+
if call_data.key?(:json_params) && !call_data[:json_params].nil?
|
152
151
|
req.body = JSON.generate(call_data[:json_params])
|
153
|
-
Log.dump('body JSON data',call_data[:json_params])
|
154
|
-
#Log.log.debug("body JSON data=#{JSON.pretty_generate(call_data[:json_params])}")
|
152
|
+
Log.dump('body JSON data', call_data[:json_params])
|
155
153
|
req['Content-Type'] = 'application/json'
|
156
|
-
#call_data[:headers]['Accept']='application/json'
|
154
|
+
# call_data[:headers]['Accept']='application/json'
|
157
155
|
end
|
158
|
-
if call_data.
|
156
|
+
if call_data.key?(:www_body_params)
|
159
157
|
req.body = URI.encode_www_form(call_data[:www_body_params])
|
160
|
-
Log.log.debug
|
158
|
+
Log.log.debug{"body www data=#{req.body.chomp}"}
|
161
159
|
req['Content-Type'] = 'application/x-www-form-urlencoded'
|
162
160
|
end
|
163
|
-
if call_data.
|
161
|
+
if call_data.key?(:text_body_params)
|
164
162
|
req.body = call_data[:text_body_params]
|
165
|
-
Log.log.debug
|
163
|
+
Log.log.debug{"body data=#{req.body.chomp}"}
|
166
164
|
end
|
167
165
|
# set headers
|
168
|
-
if call_data.
|
169
|
-
call_data[:headers].
|
166
|
+
if call_data.key?(:headers)
|
167
|
+
call_data[:headers].each_key do |key|
|
170
168
|
req[key] = call_data[:headers][key]
|
171
169
|
end
|
172
170
|
end
|
173
171
|
# :type = :basic
|
174
|
-
req.basic_auth(call_data[:auth][:username],call_data[:auth][:password]) if call_data[:auth][:type].eql?(:basic)
|
172
|
+
req.basic_auth(call_data[:auth][:username], call_data[:auth][:password]) if call_data[:auth][:type].eql?(:basic)
|
175
173
|
return req
|
176
174
|
end
|
177
175
|
|
@@ -199,10 +197,10 @@ module Aspera
|
|
199
197
|
def call(call_data)
|
200
198
|
raise "Hash call parameter is required (#{call_data.class})" unless call_data.is_a?(Hash)
|
201
199
|
call_data[:subpath] = '' if call_data[:subpath].nil?
|
202
|
-
Log.log.debug
|
200
|
+
Log.log.debug{"accessing #{call_data[:subpath]}".red.bold.bg_green}
|
203
201
|
call_data[:headers] ||= {}
|
204
202
|
call_data[:headers]['User-Agent'] ||= self.class.user_agent
|
205
|
-
# defaults from @params are
|
203
|
+
# defaults from @params are overridden by call data
|
206
204
|
call_data = @params.deep_merge(call_data)
|
207
205
|
case call_data[:auth][:type]
|
208
206
|
when :none
|
@@ -211,7 +209,7 @@ module Aspera
|
|
211
209
|
Log.log.debug('using Basic auth')
|
212
210
|
# done in build_req
|
213
211
|
when :oauth2
|
214
|
-
call_data[:headers]['Authorization'] = oauth_token unless call_data[:headers].
|
212
|
+
call_data[:headers]['Authorization'] = oauth_token unless call_data[:headers].key?('Authorization')
|
215
213
|
when :url
|
216
214
|
call_data[:url_params] ||= {}
|
217
215
|
call_data[:auth][:url_creds].each do |key, value|
|
@@ -220,7 +218,7 @@ module Aspera
|
|
220
218
|
else raise "unsupported auth type: [#{call_data[:auth][:type]}]"
|
221
219
|
end
|
222
220
|
req = build_request(call_data)
|
223
|
-
Log.log.debug
|
221
|
+
Log.log.debug{"call_data = #{call_data}"}
|
224
222
|
result = {http: nil}
|
225
223
|
# start a block to be able to retry the actual HTTP request
|
226
224
|
begin
|
@@ -242,11 +240,11 @@ module Aspera
|
|
242
240
|
target_file = call_data[:save_to_file]
|
243
241
|
# override user's path to path in header
|
244
242
|
if !response['Content-Disposition'].nil? && (m = response['Content-Disposition'].match(/filename="([^"]+)"/))
|
245
|
-
target_file = File.join(File.dirname(target_file),m[1])
|
243
|
+
target_file = File.join(File.dirname(target_file), m[1])
|
246
244
|
end
|
247
245
|
# download with temp filename
|
248
246
|
target_file_tmp = "#{target_file}#{self.class.download_partial_suffix}"
|
249
|
-
Log.log.debug
|
247
|
+
Log.log.debug{"saving to: #{target_file}"}
|
250
248
|
File.open(target_file_tmp, 'wb') do |file|
|
251
249
|
result[:http].read_body do |fragment|
|
252
250
|
file.write(fragment)
|
@@ -262,17 +260,17 @@ module Aspera
|
|
262
260
|
end
|
263
261
|
# sometimes there is a UTF8 char (e.g. (c) )
|
264
262
|
result[:http].body.force_encoding('UTF-8') if result[:http].body.is_a?(String)
|
265
|
-
Log.log.debug
|
263
|
+
Log.log.debug{"result: body=#{result[:http].body}"}
|
266
264
|
result_mime = (result[:http]['Content-Type'] || 'text/plain').split(';').first
|
267
265
|
result[:data] = case result_mime
|
268
|
-
when 'application/json','application/vnd.api+json'
|
266
|
+
when 'application/json', 'application/vnd.api+json'
|
269
267
|
JSON.parse(result[:http].body) rescue nil
|
270
|
-
else #when 'text/plain'
|
268
|
+
else # when 'text/plain'
|
271
269
|
result[:http].body
|
272
270
|
end
|
273
|
-
Log.dump("result: parsed: #{result_mime}",result[:data])
|
274
|
-
Log.log.debug
|
275
|
-
RestErrorAnalyzer.instance.raise_on_error(req,result)
|
271
|
+
Log.dump("result: parsed: #{result_mime}", result[:data])
|
272
|
+
Log.log.debug{"result: code=#{result[:http].code}"}
|
273
|
+
RestErrorAnalyzer.instance.raise_on_error(req, result)
|
276
274
|
rescue RestCallError => e
|
277
275
|
# not authorized: oauth token expired
|
278
276
|
if call_data[:not_auth_codes].include?(result[:http].code.to_s) && call_data[:auth][:type].eql?(:oauth2)
|
@@ -283,34 +281,34 @@ module Aspera
|
|
283
281
|
e = e_tok
|
284
282
|
Log.log.error('refresh failed'.bg_red)
|
285
283
|
# regenerate a brand new token
|
286
|
-
req['Authorization'] = oauth_token(
|
284
|
+
req['Authorization'] = oauth_token(force_refresh: true)
|
287
285
|
end
|
288
|
-
Log.log.debug
|
286
|
+
Log.log.debug{"using new token=#{call_data[:headers]['Authorization']}"}
|
289
287
|
retry unless (oauth_tries -= 1).zero?
|
290
288
|
end # if oauth
|
291
289
|
# moved ?
|
292
290
|
if e.response.is_a?(Net::HTTPRedirection) && tries_remain_redirect.positive?
|
293
291
|
tries_remain_redirect -= 1
|
294
292
|
current_uri = URI.parse(call_data[:base_url])
|
295
|
-
new_url=e.response['location']
|
296
|
-
new_url="#{current_uri.scheme}:#{new_url}" unless new_url.start_with?('http')
|
297
|
-
Log.log.info
|
298
|
-
|
293
|
+
new_url = e.response['location']
|
294
|
+
new_url = "#{current_uri.scheme}:#{new_url}" unless new_url.start_with?('http')
|
295
|
+
Log.log.info{"URL is moved: #{new_url}"}
|
296
|
+
redirection_uri = URI.parse(new_url)
|
299
297
|
call_data[:base_url] = new_url
|
300
298
|
call_data[:subpath] = ''
|
301
|
-
if current_uri.host.eql?(
|
299
|
+
if current_uri.host.eql?(redirection_uri.host) && current_uri.port.eql?(redirection_uri.port)
|
302
300
|
req = build_request(call_data)
|
303
301
|
retry
|
304
302
|
else
|
305
303
|
# change host
|
306
|
-
Log.log.info
|
304
|
+
Log.log.info{"Redirect changes host: #{current_uri.host} -> #{redirection_uri.host}"}
|
307
305
|
return self.class.new(call_data).call(call_data)
|
308
306
|
end
|
309
307
|
end
|
310
308
|
# raise exception if could not retry and not return error in result
|
311
309
|
raise e unless call_data[:return_error]
|
312
310
|
end # begin request
|
313
|
-
Log.log.debug
|
311
|
+
Log.log.debug{"result=#{result}"}
|
314
312
|
return result
|
315
313
|
end
|
316
314
|
|
@@ -319,24 +317,24 @@ module Aspera
|
|
319
317
|
#
|
320
318
|
|
321
319
|
# @param encoding : one of: :json_params, :url_params
|
322
|
-
def create(subpath,params,encoding=:json_params)
|
323
|
-
return call({operation: 'POST',subpath: subpath,headers: {'Accept' => 'application/json'},encoding => params})
|
320
|
+
def create(subpath, params, encoding=:json_params)
|
321
|
+
return call({operation: 'POST', subpath: subpath, headers: {'Accept' => 'application/json'}, encoding => params})
|
324
322
|
end
|
325
323
|
|
326
|
-
def read(subpath,args=nil)
|
327
|
-
return call({operation: 'GET',subpath: subpath,headers: {'Accept' => 'application/json'},url_params: args})
|
324
|
+
def read(subpath, args=nil)
|
325
|
+
return call({operation: 'GET', subpath: subpath, headers: {'Accept' => 'application/json'}, url_params: args})
|
328
326
|
end
|
329
327
|
|
330
|
-
def update(subpath,params)
|
331
|
-
return call({operation: 'PUT',subpath: subpath,headers: {'Accept' => 'application/json'},json_params: params})
|
328
|
+
def update(subpath, params)
|
329
|
+
return call({operation: 'PUT', subpath: subpath, headers: {'Accept' => 'application/json'}, json_params: params})
|
332
330
|
end
|
333
331
|
|
334
|
-
def delete(subpath)
|
335
|
-
return call({operation: 'DELETE',subpath: subpath,headers: {'Accept' => 'application/json'}})
|
332
|
+
def delete(subpath, args=nil)
|
333
|
+
return call({operation: 'DELETE', subpath: subpath, headers: {'Accept' => 'application/json'}, url_params: args})
|
336
334
|
end
|
337
335
|
|
338
336
|
def cancel(subpath)
|
339
|
-
return call({operation: 'CANCEL',subpath: subpath,headers: {'Accept' => 'application/json'}})
|
337
|
+
return call({operation: 'CANCEL', subpath: subpath, headers: {'Accept' => 'application/json'}})
|
340
338
|
end
|
341
339
|
end
|
342
|
-
end #module Aspera
|
340
|
+
end # module Aspera
|
@@ -4,8 +4,9 @@ module Aspera
|
|
4
4
|
# raised on error after REST call
|
5
5
|
class RestCallError < StandardError
|
6
6
|
attr_accessor :request, :response
|
7
|
+
|
7
8
|
# @param http response
|
8
|
-
def initialize(req,resp,msg)
|
9
|
+
def initialize(req, resp, msg)
|
9
10
|
@request = req
|
10
11
|
@response = resp
|
11
12
|
super(msg)
|
@@ -9,15 +9,16 @@ module Aspera
|
|
9
9
|
class RestErrorAnalyzer
|
10
10
|
include Singleton
|
11
11
|
attr_accessor :log_file
|
12
|
+
|
12
13
|
# the singleton object is registered with application specific handlers
|
13
14
|
def initialize
|
14
15
|
# list of handlers
|
15
16
|
@error_handlers = []
|
16
17
|
@log_file = nil
|
17
|
-
add_handler('Type Generic') do |type,call_context|
|
18
|
+
add_handler('Type Generic') do |type, call_context|
|
18
19
|
if !call_context[:response].code.start_with?('2')
|
19
20
|
# add generic information
|
20
|
-
RestErrorAnalyzer.add_error(call_context,type,"#{call_context[:request]['host']} #{call_context[:response].code} #{call_context[:response].message}")
|
21
|
+
RestErrorAnalyzer.add_error(call_context, type, "#{call_context[:request]['host']} #{call_context[:response].code} #{call_context[:response].message}")
|
21
22
|
end
|
22
23
|
end
|
23
24
|
end
|
@@ -25,7 +26,7 @@ module Aspera
|
|
25
26
|
# Use this method to analyze a EST result and raise an exception
|
26
27
|
# Analyzes REST call response and raises a RestCallError exception
|
27
28
|
# if HTTP result code is not 2XX
|
28
|
-
def raise_on_error(req,res)
|
29
|
+
def raise_on_error(req, res)
|
29
30
|
call_context = {
|
30
31
|
messages: [],
|
31
32
|
request: req,
|
@@ -36,44 +37,44 @@ module Aspera
|
|
36
37
|
# analyze errors from provided handlers
|
37
38
|
# note that there can be an error even if code is 2XX
|
38
39
|
@error_handlers.each do |handler|
|
39
|
-
begin
|
40
|
-
#Log.log.debug
|
41
|
-
handler[:block].call(handler[:name],call_context)
|
40
|
+
begin # rubocop:disable Style/RedundantBegin
|
41
|
+
# Log.log.debug{"test exception: #{handler[:name]}"}
|
42
|
+
handler[:block].call(handler[:name], call_context)
|
42
43
|
rescue StandardError => e
|
43
|
-
Log.log.error
|
44
|
+
Log.log.error{"ERROR in handler:\n#{e.message}\n#{e.backtrace}"}
|
44
45
|
end
|
45
46
|
end
|
46
|
-
raise RestCallError.new(call_context[:request],call_context[:response],call_context[:messages].join("\n")) unless call_context[:messages].empty?
|
47
|
+
raise RestCallError.new(call_context[:request], call_context[:response], call_context[:messages].join("\n")) unless call_context[:messages].empty?
|
47
48
|
end
|
48
49
|
|
49
|
-
# add a new error handler (done at application
|
50
|
+
# add a new error handler (done at application initialization)
|
50
51
|
# @param name : name of error handler (for logs)
|
51
52
|
# @param block : processing of response: takes two parameters: name, call_context
|
52
53
|
# name is the one provided here
|
53
54
|
# call_context is built in method raise_on_error
|
54
|
-
def add_handler(name
|
55
|
+
def add_handler(name, &block)
|
55
56
|
@error_handlers.unshift({name: name, block: block})
|
56
57
|
end
|
57
58
|
|
58
59
|
# add a simple error handler
|
59
60
|
# check that key exists and is string under specified path (hash)
|
60
61
|
# adds other keys as secondary information
|
61
|
-
def add_simple_handler(name
|
62
|
-
add_handler(name) do |type,call_context|
|
62
|
+
def add_simple_handler(name, *args)
|
63
|
+
add_handler(name) do |type, call_context|
|
63
64
|
# need to clone because we modify and same array is used subsequently
|
64
65
|
path = args.clone
|
65
|
-
#Log.log.debug
|
66
|
+
# Log.log.debug{"path=#{path}"}
|
66
67
|
# if last in path is boolean it tells if the error is only with http error code or always
|
67
68
|
always = [true, false].include?(path.last) ? path.pop : false
|
68
69
|
if call_context[:data].is_a?(Hash) && (!call_context[:response].code.start_with?('2') || always)
|
69
70
|
msg_key = path.pop
|
70
71
|
# dig and find sub entry corresponding to path in deep hash
|
71
|
-
error_struct = path.inject(call_context[:data]) { |
|
72
|
+
error_struct = path.inject(call_context[:data]) { |sub_hash, key| sub_hash.respond_to?(:keys) ? sub_hash[key] : nil }
|
72
73
|
if error_struct.is_a?(Hash) && error_struct[msg_key].is_a?(String)
|
73
|
-
RestErrorAnalyzer.add_error(call_context,type,error_struct[msg_key])
|
74
|
-
error_struct.each do |k,v|
|
74
|
+
RestErrorAnalyzer.add_error(call_context, type, error_struct[msg_key])
|
75
|
+
error_struct.each do |k, v|
|
75
76
|
next if k.eql?(msg_key)
|
76
|
-
RestErrorAnalyzer.add_error(call_context,"#{type}(sub)","#{k}: #{v}") if [String,Integer].include?(v.class)
|
77
|
+
RestErrorAnalyzer.add_error(call_context, "#{type}(sub)", "#{k}: #{v}") if [String, Integer].include?(v.class)
|
77
78
|
end
|
78
79
|
end
|
79
80
|
end
|
@@ -86,12 +87,12 @@ module Aspera
|
|
86
87
|
# @param call_context a Hash containing the result call_context, provided to handler
|
87
88
|
# @param type a string describing type of exception, for logging purpose
|
88
89
|
# @param msg one error message to add to list
|
89
|
-
def add_error(call_context,type,msg)
|
90
|
+
def add_error(call_context, type, msg)
|
90
91
|
call_context[:messages].push(msg)
|
91
|
-
|
92
|
+
log_file = instance.log_file
|
92
93
|
# log error for further analysis (file must exist to activate)
|
93
|
-
return if
|
94
|
-
File.open(
|
94
|
+
return if log_file.nil? || !File.exist?(log_file)
|
95
|
+
File.open(log_file, 'a+') do |f|
|
95
96
|
f.write("\n=#{type}=====\n#{call_context[:request].method} #{call_context[:request].path}\n#{call_context[:response].code}\n"\
|
96
97
|
"#{JSON.generate(call_context[:data])}\n#{call_context[:messages].join("\n")}")
|
97
98
|
end
|
@@ -12,45 +12,45 @@ module Aspera
|
|
12
12
|
Log.log.debug('registering Aspera REST error handlers')
|
13
13
|
# Faspex 4: both user_message and internal_message, and code 200
|
14
14
|
# example: missing meta data on package creation
|
15
|
-
RestErrorAnalyzer.instance.add_simple_handler('Type 1: error:user_message','error','user_message',true)
|
16
|
-
RestErrorAnalyzer.instance.add_simple_handler('Type 2: error:description','error','description')
|
17
|
-
RestErrorAnalyzer.instance.add_simple_handler('Type 3: error:internal_message','error','internal_message')
|
15
|
+
RestErrorAnalyzer.instance.add_simple_handler('Type 1: error:user_message', 'error', 'user_message', true)
|
16
|
+
RestErrorAnalyzer.instance.add_simple_handler('Type 2: error:description', 'error', 'description')
|
17
|
+
RestErrorAnalyzer.instance.add_simple_handler('Type 3: error:internal_message', 'error', 'internal_message')
|
18
18
|
# AoC Automation
|
19
|
-
RestErrorAnalyzer.instance.add_simple_handler('AoC Automation','error')
|
20
|
-
RestErrorAnalyzer.instance.add_simple_handler('Type 5','error_description')
|
21
|
-
RestErrorAnalyzer.instance.add_simple_handler('Type 6','message')
|
22
|
-
RestErrorAnalyzer.instance.add_handler('Type 7: errors[]') do |name,call_context|
|
19
|
+
RestErrorAnalyzer.instance.add_simple_handler('AoC Automation', 'error')
|
20
|
+
RestErrorAnalyzer.instance.add_simple_handler('Type 5', 'error_description')
|
21
|
+
RestErrorAnalyzer.instance.add_simple_handler('Type 6', 'message')
|
22
|
+
RestErrorAnalyzer.instance.add_handler('Type 7: errors[]') do |name, call_context|
|
23
23
|
if call_context[:data].is_a?(Hash) && call_context[:data]['errors'].is_a?(Hash)
|
24
|
-
call_context[:data]['errors'].each do |k,v|
|
25
|
-
RestErrorAnalyzer.add_error(call_context,name,"#{k}: #{v}")
|
24
|
+
call_context[:data]['errors'].each do |k, v|
|
25
|
+
RestErrorAnalyzer.add_error(call_context, name, "#{k}: #{v}")
|
26
26
|
end
|
27
27
|
end
|
28
28
|
end
|
29
29
|
# call to upload_setup and download_setup of node api
|
30
|
-
RestErrorAnalyzer.instance.add_handler('T8:node: *_setup') do |type,call_context|
|
30
|
+
RestErrorAnalyzer.instance.add_handler('T8:node: *_setup') do |type, call_context|
|
31
31
|
if call_context[:data].is_a?(Hash)
|
32
32
|
d_t_s = call_context[:data]['transfer_specs']
|
33
33
|
if d_t_s.is_a?(Array)
|
34
34
|
d_t_s.each do |res|
|
35
|
-
#r_err=res['transfer_spec']['error']
|
35
|
+
# r_err=res['transfer_spec']['error']
|
36
36
|
r_err = res['error']
|
37
37
|
if r_err.is_a?(Hash)
|
38
|
-
RestErrorAnalyzer.add_error(call_context,type,"#{r_err['code']}: #{r_err['reason']}: #{r_err['user_message']}")
|
38
|
+
RestErrorAnalyzer.add_error(call_context, type, "#{r_err['code']}: #{r_err['reason']}: #{r_err['user_message']}")
|
39
39
|
end
|
40
40
|
end
|
41
41
|
end
|
42
42
|
end
|
43
43
|
end
|
44
|
-
RestErrorAnalyzer.instance.add_simple_handler('T9:IBM cloud IAM','errorMessage')
|
45
|
-
RestErrorAnalyzer.instance.add_simple_handler('T10:faspex v4','user_message')
|
46
|
-
RestErrorAnalyzer.instance.add_handler('bss graphql') do |type,call_context|
|
44
|
+
RestErrorAnalyzer.instance.add_simple_handler('T9:IBM cloud IAM', 'errorMessage')
|
45
|
+
RestErrorAnalyzer.instance.add_simple_handler('T10:faspex v4', 'user_message')
|
46
|
+
RestErrorAnalyzer.instance.add_handler('bss graphql') do |type, call_context|
|
47
47
|
if call_context[:data].is_a?(Hash)
|
48
48
|
d_t_s = call_context[:data]['errors']
|
49
49
|
if d_t_s.is_a?(Array)
|
50
50
|
d_t_s.each do |res|
|
51
51
|
r_err = res['message']
|
52
52
|
if r_err.is_a?(String)
|
53
|
-
RestErrorAnalyzer.add_error(call_context,type,r_err)
|
53
|
+
RestErrorAnalyzer.add_error(call_context, type, r_err)
|
54
54
|
end
|
55
55
|
end
|
56
56
|
end
|