aspera-cli 4.6.0 → 4.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +427 -300
- data/bin/ascli +2 -1
- data/bin/asession +1 -0
- data/docs/test_env.conf +2 -0
- data/examples/aoc.rb +4 -3
- data/examples/faspex4.rb +21 -19
- data/examples/proxy.pac +1 -1
- data/examples/transfer.rb +15 -15
- data/lib/aspera/aoc.rb +135 -124
- data/lib/aspera/ascmd.rb +85 -75
- data/lib/aspera/ats_api.rb +11 -10
- data/lib/aspera/cli/basic_auth_plugin.rb +13 -14
- data/lib/aspera/cli/extended_value.rb +42 -33
- data/lib/aspera/cli/formater.rb +138 -111
- data/lib/aspera/cli/info.rb +17 -0
- data/lib/aspera/cli/listener/line_dump.rb +3 -2
- data/lib/aspera/cli/listener/logger.rb +2 -1
- data/lib/aspera/cli/listener/progress.rb +16 -18
- data/lib/aspera/cli/listener/progress_multi.rb +13 -16
- data/lib/aspera/cli/main.rb +122 -130
- data/lib/aspera/cli/manager.rb +146 -154
- data/lib/aspera/cli/plugin.rb +38 -34
- data/lib/aspera/cli/plugins/alee.rb +6 -6
- data/lib/aspera/cli/plugins/aoc.rb +273 -276
- data/lib/aspera/cli/plugins/ats.rb +82 -76
- data/lib/aspera/cli/plugins/bss.rb +14 -16
- data/lib/aspera/cli/plugins/config.rb +350 -306
- data/lib/aspera/cli/plugins/console.rb +23 -19
- data/lib/aspera/cli/plugins/cos.rb +18 -18
- data/lib/aspera/cli/plugins/faspex.rb +180 -159
- data/lib/aspera/cli/plugins/faspex5.rb +64 -54
- data/lib/aspera/cli/plugins/node.rb +147 -140
- data/lib/aspera/cli/plugins/orchestrator.rb +68 -66
- data/lib/aspera/cli/plugins/preview.rb +92 -96
- data/lib/aspera/cli/plugins/server.rb +79 -75
- data/lib/aspera/cli/plugins/shares.rb +23 -24
- data/lib/aspera/cli/plugins/sync.rb +20 -22
- data/lib/aspera/cli/transfer_agent.rb +40 -39
- data/lib/aspera/cli/version.rb +2 -1
- data/lib/aspera/colors.rb +35 -27
- data/lib/aspera/command_line_builder.rb +48 -34
- data/lib/aspera/cos_node.rb +29 -21
- data/lib/aspera/data_repository.rb +3 -2
- data/lib/aspera/environment.rb +50 -45
- data/lib/aspera/fasp/agent_base.rb +22 -20
- data/lib/aspera/fasp/agent_connect.rb +13 -11
- data/lib/aspera/fasp/agent_direct.rb +48 -59
- data/lib/aspera/fasp/agent_httpgw.rb +33 -39
- data/lib/aspera/fasp/agent_node.rb +15 -13
- data/lib/aspera/fasp/agent_trsdk.rb +12 -14
- data/lib/aspera/fasp/error.rb +2 -1
- data/lib/aspera/fasp/error_info.rb +68 -52
- data/lib/aspera/fasp/installation.rb +106 -94
- data/lib/aspera/fasp/listener.rb +1 -0
- data/lib/aspera/fasp/parameters.rb +83 -92
- data/lib/aspera/fasp/parameters.yaml +305 -249
- data/lib/aspera/fasp/resume_policy.rb +11 -14
- data/lib/aspera/fasp/transfer_spec.rb +26 -0
- data/lib/aspera/fasp/uri.rb +22 -21
- data/lib/aspera/faspex_gw.rb +55 -90
- data/lib/aspera/hash_ext.rb +4 -3
- data/lib/aspera/id_generator.rb +8 -7
- data/lib/aspera/keychain/encrypted_hash.rb +17 -16
- data/lib/aspera/keychain/macos_security.rb +6 -10
- data/lib/aspera/log.rb +25 -20
- data/lib/aspera/nagios.rb +13 -12
- data/lib/aspera/node.rb +30 -22
- data/lib/aspera/oauth.rb +175 -226
- data/lib/aspera/open_application.rb +4 -3
- data/lib/aspera/persistency_action_once.rb +6 -6
- data/lib/aspera/persistency_folder.rb +5 -9
- data/lib/aspera/preview/file_types.rb +6 -5
- data/lib/aspera/preview/generator.rb +25 -24
- data/lib/aspera/preview/options.rb +16 -14
- data/lib/aspera/preview/utils.rb +98 -98
- data/lib/aspera/{proxy_auto_config.erb.js → proxy_auto_config.js} +23 -31
- data/lib/aspera/proxy_auto_config.rb +111 -20
- data/lib/aspera/rest.rb +115 -113
- data/lib/aspera/rest_call_error.rb +2 -2
- data/lib/aspera/rest_error_analyzer.rb +23 -25
- data/lib/aspera/rest_errors_aspera.rb +15 -14
- data/lib/aspera/ssh.rb +12 -10
- data/lib/aspera/sync.rb +42 -41
- data/lib/aspera/temp_file_manager.rb +18 -14
- data/lib/aspera/timer_limiter.rb +2 -1
- data/lib/aspera/uri_reader.rb +7 -5
- data/lib/aspera/web_auth.rb +79 -76
- metadata +64 -21
- data/docs/Makefile +0 -65
- data/docs/README.erb.md +0 -4424
- data/docs/README.md +0 -13
- data/docs/diagrams.txt +0 -49
- data/docs/doc_tools.rb +0 -58
- data/lib/aspera/cli/plugins/shares2.rb +0 -114
- data/lib/aspera/fasp/default.rb +0 -17
|
@@ -1,34 +1,125 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
1
2
|
require 'uri'
|
|
2
3
|
require 'resolv'
|
|
3
|
-
|
|
4
|
+
|
|
5
|
+
module URI
|
|
6
|
+
class Generic
|
|
7
|
+
# save original method that finds proxy in URI::Generic, it uses env var http_proxy
|
|
8
|
+
alias :find_proxy_orig find_proxy
|
|
9
|
+
def self.register_proxy_finder(&block)
|
|
10
|
+
# overload the method
|
|
11
|
+
define_method(:find_proxy) {|envars=ENV| block.call(to_s) || find_proxy_orig(envars)}
|
|
12
|
+
end
|
|
13
|
+
end
|
|
14
|
+
end
|
|
4
15
|
|
|
5
16
|
module Aspera
|
|
6
|
-
#
|
|
17
|
+
# Evaluate a proxy autoconfig script
|
|
7
18
|
class ProxyAutoConfig
|
|
8
19
|
# template file is read once, it contains functions that can be used in a proxy autoconf script
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
20
|
+
# it is similar to mozilla ascii_pac_utils.inc
|
|
21
|
+
PAC_FUNCTIONS_FILE=__FILE__.gsub(/\.rb$/,'.js').freeze
|
|
22
|
+
PAC_MAIN_FUNCTION='FindProxyForURL'
|
|
23
|
+
private_constant :PAC_FUNCTIONS_FILE,:PAC_MAIN_FUNCTION
|
|
24
|
+
|
|
25
|
+
private
|
|
15
26
|
|
|
16
|
-
#
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
# variables starting with "context_" are replaced in the ERB template file
|
|
21
|
-
# I did not find an easy way for the javascript to callback ruby
|
|
22
|
-
# and anyway, it only needs to get DNS translation
|
|
27
|
+
# variables starting with "context_" are replaced in the ERB template file
|
|
28
|
+
# I did not find an easy way for the javascript to callback ruby
|
|
29
|
+
# and anyway, it only needs to get DNS translation
|
|
30
|
+
def pac_dns_functions(context_host)
|
|
23
31
|
context_self='127.0.0.1'
|
|
24
|
-
context_host=URI.parse(service_url).host
|
|
25
32
|
context_ip=nil
|
|
26
33
|
Resolv::DNS.open{|dns|dns.each_address(context_host){|r_addr|context_ip=r_addr.to_s if r_addr.is_a?(Resolv::IPv4)}}
|
|
27
34
|
raise "DNS name not found: #{context_host}" if context_ip.nil?
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
35
|
+
# NOTE: Javascript code here with string inclusions
|
|
36
|
+
javascript=<<END_OF_JAVASCRIPT
|
|
37
|
+
function dnsResolve(host) {
|
|
38
|
+
if (host == '#{context_host}' || host == '#{context_ip}')
|
|
39
|
+
return '#{context_ip}';
|
|
40
|
+
throw 'only DNS for initial host, not ' + host;
|
|
41
|
+
}
|
|
42
|
+
function myIpAddress() {
|
|
43
|
+
return '#{context_self}';
|
|
44
|
+
}
|
|
45
|
+
END_OF_JAVASCRIPT
|
|
46
|
+
return javascript
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
public
|
|
50
|
+
|
|
51
|
+
# @param proxy_auto_config the proxy auto config script to be evaluated
|
|
52
|
+
def initialize(proxy_auto_config)
|
|
53
|
+
# user provided javascript with FindProxyForURL function
|
|
54
|
+
@proxy_auto_config=proxy_auto_config
|
|
55
|
+
# avoid multiple execution, this does not support load balancing
|
|
56
|
+
@cache={}
|
|
57
|
+
@pac_functions=nil
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
def register_uri_generic
|
|
61
|
+
URI::Generic.register_proxy_finder{|url_str|get_proxies(url_str).first}
|
|
62
|
+
# allow chaining
|
|
63
|
+
return self
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
# execute proxy auto config script for the given URL : https://en.wikipedia.org/wiki/Proxy_auto-config
|
|
67
|
+
# @return either nil, or a String formated following PAC standard
|
|
68
|
+
def find_proxy_for_url(service_url)
|
|
69
|
+
uri=URI.parse(service_url)
|
|
70
|
+
simple_url="#{uri.scheme}://#{uri.host}"
|
|
71
|
+
if !@cache.has_key?(simple_url)
|
|
72
|
+
Log.log.debug("PAC: starting javascript for #{service_url}")
|
|
73
|
+
# require at runtime, in case there is no js engine
|
|
74
|
+
require 'execjs'
|
|
75
|
+
# read template lib
|
|
76
|
+
@pac_functions=File.read(PAC_FUNCTIONS_FILE).freeze if @pac_functions.nil?
|
|
77
|
+
# to be executed is dns + utils + user function
|
|
78
|
+
js_to_execute="#{pac_dns_functions(uri.host)}#{@pac_functions}#{@proxy_auto_config}"
|
|
79
|
+
executable_js = ExecJS.compile(js_to_execute)
|
|
80
|
+
@cache[simple_url]=executable_js.call(PAC_MAIN_FUNCTION, simple_url, uri.host)
|
|
81
|
+
Log.log.debug("PAC: result: #{@cache[simple_url]}")
|
|
82
|
+
end
|
|
83
|
+
return @cache[simple_url]
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
# used to replace URI::Generic.find_proxy
|
|
87
|
+
# @return Array of URI, possibly empty
|
|
88
|
+
def get_proxies(service_url)
|
|
89
|
+
# prepare result
|
|
90
|
+
uri_list=[]
|
|
91
|
+
# execute PAC script
|
|
92
|
+
proxy_list_str=find_proxy_for_url(service_url)
|
|
93
|
+
if !proxy_list_str.is_a?(String)
|
|
94
|
+
Log.log.warn("PAC: did not return a String, returned #{proxy_list_str.class}")
|
|
95
|
+
return uri_list
|
|
96
|
+
end
|
|
97
|
+
proxy_list_str.strip!
|
|
98
|
+
proxy_list_str.gsub!(/\s+/,' ')
|
|
99
|
+
proxy_list_str.split(';').each do |item|
|
|
100
|
+
# strip and split by space
|
|
101
|
+
parts=item.strip.split
|
|
102
|
+
case parts.shift
|
|
103
|
+
when 'DIRECT'
|
|
104
|
+
raise 'DIRECT has no param' unless parts.empty?
|
|
105
|
+
when 'PROXY'
|
|
106
|
+
addr_port=parts.shift
|
|
107
|
+
raise 'PROXY shall have one param' unless addr_port.is_a?(String) && parts.empty?
|
|
108
|
+
begin
|
|
109
|
+
# PAC proxy addresses are <host>:<port>
|
|
110
|
+
if addr_port.match(/:[0-9]+$/)
|
|
111
|
+
# we want to return URIs, so add dummy scheme
|
|
112
|
+
uri_list.push(URI.parse("proxy://#{addr_port}"))
|
|
113
|
+
else
|
|
114
|
+
Log.log.warn("PAC: PROXY must be <address>:<port>, ignoring #{addr_port}")
|
|
115
|
+
end
|
|
116
|
+
rescue StandardError
|
|
117
|
+
Log.log.warn("PAC: cannot parse #{addr_port}")
|
|
118
|
+
end
|
|
119
|
+
else Log.log.warn("PAC: ignoring proxy type #{parts.first}: not supported")
|
|
120
|
+
end
|
|
121
|
+
end
|
|
122
|
+
return uri_list
|
|
31
123
|
end
|
|
32
124
|
end
|
|
33
125
|
end
|
|
34
|
-
|
data/lib/aspera/rest.rb
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
1
2
|
require 'aspera/log'
|
|
2
3
|
require 'aspera/oauth'
|
|
3
4
|
require 'aspera/rest_error_analyzer'
|
|
@@ -28,58 +29,52 @@ module Aspera
|
|
|
28
29
|
# rest call errors are raised as exception RestCallError
|
|
29
30
|
# and error are analyzed in RestErrorAnalyzer
|
|
30
31
|
class Rest
|
|
31
|
-
|
|
32
|
-
#
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
public
|
|
42
|
-
def self.session_cb=(v); @@session_cb=v;Log.log.debug("session_cb => #{@@session_cb}".red);end
|
|
43
|
-
|
|
44
|
-
def self.session_cb; @@session_cb;end
|
|
45
|
-
|
|
46
|
-
def self.insecure=(v); @@insecure=v;Log.log.debug("insecure => #{@@insecure}".red);end
|
|
32
|
+
# global settings also valid for any subclass
|
|
33
|
+
@@global={ # rubocop:disable Style/ClassVars
|
|
34
|
+
debug: false,
|
|
35
|
+
# true if https ignore certificate
|
|
36
|
+
insecure: false,
|
|
37
|
+
user_agent: 'Ruby',
|
|
38
|
+
download_partial_suffix: '.http_partial',
|
|
39
|
+
# a lambda which takes the Net::HTTP as arg, use this to change parameters
|
|
40
|
+
session_cb: nil
|
|
41
|
+
}
|
|
47
42
|
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
def self.debug=(flag); @@debug=flag; Log.log.debug("debug http => #{flag}"); end
|
|
43
|
+
class<<self
|
|
44
|
+
# define accessors
|
|
45
|
+
@@global.keys.each do |p|
|
|
46
|
+
define_method(p){@@global[p]}
|
|
47
|
+
define_method("#{p}="){|val|Log.log.debug("#{p} => #{val}".red);@@global[p]=val}
|
|
48
|
+
end
|
|
55
49
|
|
|
56
|
-
|
|
50
|
+
def basic_creds(user,pass); return "Basic #{Base64.strict_encode64("#{user}:#{pass}")}";end
|
|
57
51
|
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
52
|
+
# build URI from URL and parameters and check it is http or https
|
|
53
|
+
def build_uri(url,params=nil)
|
|
54
|
+
uri=URI.parse(url)
|
|
55
|
+
raise 'REST endpoint shall be http(s)' unless ['http','https'].include?(uri.scheme)
|
|
56
|
+
if !params.nil?
|
|
57
|
+
# support array url params, there is no standard. Either p[]=1&p[]=2, or p=1&p=2
|
|
58
|
+
if params.is_a?(Hash)
|
|
59
|
+
orig=params
|
|
60
|
+
params=[]
|
|
61
|
+
orig.each do |k,v|
|
|
62
|
+
case v
|
|
63
|
+
when Array
|
|
64
|
+
suffix=v.first.eql?('[]') ? v.shift : ''
|
|
65
|
+
v.each do |e|
|
|
66
|
+
params.push([k+suffix,e])
|
|
67
|
+
end
|
|
68
|
+
else
|
|
69
|
+
params.push([k,v])
|
|
73
70
|
end
|
|
74
|
-
else
|
|
75
|
-
params.push([k,v])
|
|
76
71
|
end
|
|
77
72
|
end
|
|
73
|
+
# CGI.unescape to transform back %5D into []
|
|
74
|
+
uri.query=CGI.unescape(URI.encode_www_form(params))
|
|
78
75
|
end
|
|
79
|
-
|
|
80
|
-
uri.query=CGI.unescape(URI.encode_www_form(params))
|
|
76
|
+
return uri
|
|
81
77
|
end
|
|
82
|
-
return uri
|
|
83
78
|
end
|
|
84
79
|
|
|
85
80
|
private
|
|
@@ -91,10 +86,10 @@ module Aspera
|
|
|
91
86
|
# this honors http_proxy env var
|
|
92
87
|
@http_session=Net::HTTP.new(uri.host, uri.port)
|
|
93
88
|
@http_session.use_ssl = uri.scheme.eql?('https')
|
|
94
|
-
@http_session.verify_mode = OpenSSL::SSL::VERIFY_NONE if
|
|
95
|
-
@http_session.set_debug_output($stdout) if
|
|
89
|
+
@http_session.verify_mode = OpenSSL::SSL::VERIFY_NONE if self.class.insecure
|
|
90
|
+
@http_session.set_debug_output($stdout) if self.class.debug
|
|
96
91
|
# set http options in callback, such as timeout and cert. verification
|
|
97
|
-
|
|
92
|
+
self.class.session_cb.call(@http_session) unless self.class.session_cb.nil?
|
|
98
93
|
# manually start session for keep alive (if supported by server, else, session is closed every time)
|
|
99
94
|
@http_session.start
|
|
100
95
|
end
|
|
@@ -104,32 +99,34 @@ module Aspera
|
|
|
104
99
|
public
|
|
105
100
|
|
|
106
101
|
attr_reader :params
|
|
107
|
-
attr_reader :oauth
|
|
108
102
|
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
103
|
+
def oauth
|
|
104
|
+
if @oauth.nil?
|
|
105
|
+
raise 'ERROR: no OAuth defined' unless @params[:auth][:type].eql?(:oauth2)
|
|
106
|
+
@oauth=Oauth.new(@params[:auth])
|
|
107
|
+
end
|
|
108
|
+
return @oauth
|
|
109
|
+
end
|
|
110
|
+
|
|
111
|
+
# @param a_rest_params [Hash] default call parameters (merged at call)
|
|
115
112
|
def initialize(a_rest_params)
|
|
116
|
-
raise
|
|
117
|
-
raise
|
|
113
|
+
raise 'ERROR: expecting Hash' unless a_rest_params.is_a?(Hash)
|
|
114
|
+
raise 'ERROR: expecting base_url' unless a_rest_params[:base_url].is_a?(String)
|
|
118
115
|
@params=a_rest_params.clone
|
|
119
116
|
Log.dump('REST params',@params)
|
|
120
117
|
# base url without trailing slashes (note: string may be frozen)
|
|
121
118
|
@params[:base_url]=@params[:base_url].gsub(/\/+$/,'')
|
|
122
119
|
@http_session=nil
|
|
123
120
|
# default is no auth
|
|
124
|
-
@params[:auth]||={:
|
|
121
|
+
@params[:auth]||={type: :none}
|
|
125
122
|
@params[:not_auth_codes]||=['401']
|
|
126
|
-
@oauth=
|
|
123
|
+
@oauth=nil
|
|
127
124
|
Log.dump('REST params(2)',@params)
|
|
128
125
|
end
|
|
129
126
|
|
|
130
|
-
def oauth_token(
|
|
131
|
-
raise "ERROR: expecting
|
|
132
|
-
return
|
|
127
|
+
def oauth_token(force_refresh: false)
|
|
128
|
+
raise "ERROR: expecting boolean, have #{force_refresh}" unless [true,false].include?(force_refresh)
|
|
129
|
+
return oauth.get_authorization(use_refresh_token: force_refresh)
|
|
133
130
|
end
|
|
134
131
|
|
|
135
132
|
def build_request(call_data)
|
|
@@ -139,28 +136,28 @@ module Aspera
|
|
|
139
136
|
Log.log.debug("URI=#{uri}")
|
|
140
137
|
begin
|
|
141
138
|
# instanciate request object based on string name
|
|
142
|
-
req=
|
|
143
|
-
rescue NameError
|
|
139
|
+
req=Net::HTTP.const_get(call_data[:operation].capitalize).new(uri)
|
|
140
|
+
rescue NameError
|
|
144
141
|
raise "unsupported operation : #{call_data[:operation]}"
|
|
145
142
|
end
|
|
146
|
-
if call_data.has_key?(:json_params)
|
|
143
|
+
if call_data.has_key?(:json_params) && !call_data[:json_params].nil?
|
|
147
144
|
req.body=JSON.generate(call_data[:json_params])
|
|
148
145
|
Log.dump('body JSON data',call_data[:json_params])
|
|
149
146
|
#Log.log.debug("body JSON data=#{JSON.pretty_generate(call_data[:json_params])}")
|
|
150
147
|
req['Content-Type'] = 'application/json'
|
|
151
148
|
#call_data[:headers]['Accept']='application/json'
|
|
152
149
|
end
|
|
153
|
-
if call_data.has_key?(:www_body_params)
|
|
150
|
+
if call_data.has_key?(:www_body_params)
|
|
154
151
|
req.body=URI.encode_www_form(call_data[:www_body_params])
|
|
155
152
|
Log.log.debug("body www data=#{req.body.chomp}")
|
|
156
153
|
req['Content-Type'] = 'application/x-www-form-urlencoded'
|
|
157
154
|
end
|
|
158
|
-
if call_data.has_key?(:text_body_params)
|
|
155
|
+
if call_data.has_key?(:text_body_params)
|
|
159
156
|
req.body=call_data[:text_body_params]
|
|
160
157
|
Log.log.debug("body data=#{req.body.chomp}")
|
|
161
158
|
end
|
|
162
159
|
# set headers
|
|
163
|
-
if call_data.has_key?(:headers)
|
|
160
|
+
if call_data.has_key?(:headers)
|
|
164
161
|
call_data[:headers].keys.each do |key|
|
|
165
162
|
req[key] = call_data[:headers][key]
|
|
166
163
|
end
|
|
@@ -180,22 +177,30 @@ module Aspera
|
|
|
180
177
|
# :url_params
|
|
181
178
|
# :www_body_params
|
|
182
179
|
# :text_body_params
|
|
183
|
-
# :save_to_file (filepath)
|
|
184
|
-
# :return_error (bool)
|
|
185
|
-
# :redirect_max (int)
|
|
180
|
+
# :save_to_file (filepath) default: nil
|
|
181
|
+
# :return_error (bool) default: nil
|
|
182
|
+
# :redirect_max (int) default: 0
|
|
183
|
+
# :not_auth_codes (array)
|
|
184
|
+
# ----
|
|
185
|
+
# authentication (:auth) :
|
|
186
|
+
# :type (:none, :basic, :oauth2, :url)
|
|
187
|
+
# :username [:basic]
|
|
188
|
+
# :password [:basic]
|
|
189
|
+
# :url_creds [:url] a hash
|
|
190
|
+
# :* [:oauth2] see Oauth class
|
|
186
191
|
def call(call_data)
|
|
187
192
|
raise "Hash call parameter is required (#{call_data.class})" unless call_data.is_a?(Hash)
|
|
188
193
|
call_data[:subpath]='' if call_data[:subpath].nil?
|
|
189
194
|
Log.log.debug("accessing #{call_data[:subpath]}".red.bold.bg_green)
|
|
190
195
|
call_data[:headers]||={}
|
|
191
|
-
call_data[:headers]['User-Agent'] ||=
|
|
196
|
+
call_data[:headers]['User-Agent'] ||= self.class.user_agent
|
|
192
197
|
# defaults from @params are overriden by call data
|
|
193
198
|
call_data=@params.deep_merge(call_data)
|
|
194
199
|
case call_data[:auth][:type]
|
|
195
200
|
when :none
|
|
196
201
|
# no auth
|
|
197
202
|
when :basic
|
|
198
|
-
Log.log.debug(
|
|
203
|
+
Log.log.debug('using Basic auth')
|
|
199
204
|
# done in build_req
|
|
200
205
|
when :oauth2
|
|
201
206
|
call_data[:headers]['Authorization']=oauth_token unless call_data[:headers].has_key?('Authorization')
|
|
@@ -208,31 +213,31 @@ module Aspera
|
|
|
208
213
|
end
|
|
209
214
|
req=build_request(call_data)
|
|
210
215
|
Log.log.debug("call_data = #{call_data}")
|
|
211
|
-
result={:
|
|
216
|
+
result={http: nil}
|
|
212
217
|
# start a block to be able to retry the actual HTTP request
|
|
213
218
|
begin
|
|
214
219
|
# we try the call, and will retry only if oauth, as we can, first with refresh, and then re-auth if refresh is bad
|
|
215
220
|
oauth_tries ||= 2
|
|
216
221
|
tries_remain_redirect||=call_data[:redirect_max].nil? ? 0 : call_data[:redirect_max].to_i
|
|
217
|
-
Log.log.debug(
|
|
222
|
+
Log.log.debug('send request')
|
|
218
223
|
# make http request (pipelined)
|
|
219
224
|
http_session.request(req) do |response|
|
|
220
225
|
result[:http] = response
|
|
221
|
-
if call_data.
|
|
226
|
+
if !call_data[:save_to_file].nil? && result[:http].code.to_s.start_with?('2')
|
|
222
227
|
total_size=result[:http]['Content-Length'].to_i
|
|
223
228
|
progress=ProgressBar.create(
|
|
224
|
-
:
|
|
225
|
-
:
|
|
226
|
-
:
|
|
227
|
-
:
|
|
228
|
-
Log.log.debug(
|
|
229
|
+
format: '%a %B %p%% %r KB/sec %e',
|
|
230
|
+
rate_scale: lambda{|rate|rate/1024},
|
|
231
|
+
title: 'progress',
|
|
232
|
+
total: total_size)
|
|
233
|
+
Log.log.debug('before write file')
|
|
229
234
|
target_file=call_data[:save_to_file]
|
|
230
235
|
# override user's path to path in header
|
|
231
|
-
if !response['Content-Disposition'].nil?
|
|
236
|
+
if !response['Content-Disposition'].nil? && (m=response['Content-Disposition'].match(/filename="([^"]+)"/))
|
|
232
237
|
target_file=File.join(File.dirname(target_file),m[1])
|
|
233
238
|
end
|
|
234
239
|
# download with temp filename
|
|
235
|
-
target_file_tmp="#{target_file}#{
|
|
240
|
+
target_file_tmp="#{target_file}#{self.class.download_partial_suffix}"
|
|
236
241
|
Log.log.debug("saving to: #{target_file}")
|
|
237
242
|
File.open(target_file_tmp, 'wb') do |file|
|
|
238
243
|
result[:http].read_body do |fragment|
|
|
@@ -248,26 +253,26 @@ module Aspera
|
|
|
248
253
|
end # save_to_file
|
|
249
254
|
end
|
|
250
255
|
# sometimes there is a UTF8 char (e.g. (c) )
|
|
251
|
-
result[:http].body.force_encoding(
|
|
256
|
+
result[:http].body.force_encoding('UTF-8') if result[:http].body.is_a?(String)
|
|
252
257
|
Log.log.debug("result: body=#{result[:http].body}")
|
|
253
258
|
result_mime=(result[:http]['Content-Type']||'text/plain').split(';').first
|
|
254
|
-
case result_mime
|
|
259
|
+
result[:data]=case result_mime
|
|
255
260
|
when 'application/json','application/vnd.api+json'
|
|
256
|
-
|
|
261
|
+
JSON.parse(result[:http].body) rescue nil
|
|
257
262
|
else #when 'text/plain'
|
|
258
|
-
result[:
|
|
263
|
+
result[:http].body
|
|
259
264
|
end
|
|
260
265
|
Log.dump("result: parsed: #{result_mime}",result[:data])
|
|
261
266
|
Log.log.debug("result: code=#{result[:http].code}")
|
|
262
|
-
RestErrorAnalyzer.instance.
|
|
267
|
+
RestErrorAnalyzer.instance.raise_on_error(req,result)
|
|
263
268
|
rescue RestCallError => e
|
|
264
269
|
# not authorized: oauth token expired
|
|
265
|
-
if
|
|
270
|
+
if call_data[:not_auth_codes].include?(result[:http].code.to_s) && call_data[:auth][:type].eql?(:oauth2)
|
|
266
271
|
begin
|
|
267
272
|
# try to use refresh token
|
|
268
|
-
req['Authorization']=oauth_token(
|
|
273
|
+
req['Authorization']=oauth_token(force_refresh: true)
|
|
269
274
|
rescue RestCallError => e
|
|
270
|
-
Log.log.error(
|
|
275
|
+
Log.log.error('refresh failed'.bg_red)
|
|
271
276
|
# regenerate a brand new token
|
|
272
277
|
req['Authorization']=oauth_token
|
|
273
278
|
end
|
|
@@ -275,27 +280,24 @@ module Aspera
|
|
|
275
280
|
retry unless (oauth_tries -= 1).zero?
|
|
276
281
|
end # if oauth
|
|
277
282
|
# moved ?
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
else
|
|
290
|
-
# change host
|
|
291
|
-
Log.log.info("Redirect changes host: #{current_uri.host} -> #{redir_uri.host}")
|
|
292
|
-
return self.class.new(call_data).call(call_data)
|
|
293
|
-
end
|
|
283
|
+
raise e unless e.response.is_a?(Net::HTTPRedirection)
|
|
284
|
+
if tries_remain_redirect.positive?
|
|
285
|
+
tries_remain_redirect-=1
|
|
286
|
+
Log.log.info("URL is moved: #{e.response['location']}")
|
|
287
|
+
current_uri=URI.parse(call_data[:base_url])
|
|
288
|
+
redir_uri=URI.parse(e.response['location'])
|
|
289
|
+
call_data[:base_url]=e.response['location']
|
|
290
|
+
call_data[:subpath]=''
|
|
291
|
+
if current_uri.host.eql?(redir_uri.host) && current_uri.port.eql?(redir_uri.port)
|
|
292
|
+
req=build_request(call_data)
|
|
293
|
+
retry
|
|
294
294
|
else
|
|
295
|
-
|
|
295
|
+
# change host
|
|
296
|
+
Log.log.info("Redirect changes host: #{current_uri.host} -> #{redir_uri.host}")
|
|
297
|
+
return self.class.new(call_data).call(call_data)
|
|
296
298
|
end
|
|
297
299
|
else
|
|
298
|
-
raise e
|
|
300
|
+
raise e unless call_data[:return_error]
|
|
299
301
|
end
|
|
300
302
|
# raise exception if could not retry and not return error in result
|
|
301
303
|
raise e unless call_data[:return_error]
|
|
@@ -310,23 +312,23 @@ module Aspera
|
|
|
310
312
|
|
|
311
313
|
# @param encoding : one of: :json_params, :url_params
|
|
312
314
|
def create(subpath,params,encoding=:json_params)
|
|
313
|
-
return call({:
|
|
315
|
+
return call({operation: 'POST',subpath: subpath,headers: {'Accept'=>'application/json'},encoding=>params})
|
|
314
316
|
end
|
|
315
317
|
|
|
316
318
|
def read(subpath,args=nil)
|
|
317
|
-
return call({:
|
|
319
|
+
return call({operation: 'GET',subpath: subpath,headers: {'Accept'=>'application/json'},url_params: args})
|
|
318
320
|
end
|
|
319
321
|
|
|
320
322
|
def update(subpath,params)
|
|
321
|
-
return call({:
|
|
323
|
+
return call({operation: 'PUT',subpath: subpath,headers: {'Accept'=>'application/json'},json_params: params})
|
|
322
324
|
end
|
|
323
325
|
|
|
324
326
|
def delete(subpath)
|
|
325
|
-
return call({:
|
|
327
|
+
return call({operation: 'DELETE',subpath: subpath,headers: {'Accept'=>'application/json'}})
|
|
326
328
|
end
|
|
327
329
|
|
|
328
330
|
def cancel(subpath)
|
|
329
|
-
return call({:
|
|
331
|
+
return call({operation: 'CANCEL',subpath: subpath,headers: {'Accept'=>'application/json'}})
|
|
330
332
|
end
|
|
331
333
|
end
|
|
332
334
|
end #module Aspera
|
|
@@ -1,8 +1,8 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
1
2
|
module Aspera
|
|
2
3
|
# raised on error after REST call
|
|
3
4
|
class RestCallError < StandardError
|
|
4
|
-
attr_accessor :request
|
|
5
|
-
attr_accessor :response
|
|
5
|
+
attr_accessor :request, :response
|
|
6
6
|
# @param http response
|
|
7
7
|
def initialize(req,resp,msg)
|
|
8
8
|
@request = req
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
1
2
|
require 'aspera/log'
|
|
2
3
|
require 'aspera/rest_call_error'
|
|
3
4
|
require 'singleton'
|
|
@@ -12,10 +13,10 @@ module Aspera
|
|
|
12
13
|
# list of handlers
|
|
13
14
|
@error_handlers=[]
|
|
14
15
|
@log_file=nil
|
|
15
|
-
|
|
16
|
-
if !
|
|
16
|
+
add_handler('Type Generic') do |type,call_context|
|
|
17
|
+
if !call_context[:response].code.start_with?('2')
|
|
17
18
|
# add generic information
|
|
18
|
-
RestErrorAnalyzer.add_error(
|
|
19
|
+
RestErrorAnalyzer.add_error(call_context,type,"#{call_context[:request]['host']} #{call_context[:response].code} #{call_context[:response].message}")
|
|
19
20
|
end
|
|
20
21
|
end
|
|
21
22
|
end
|
|
@@ -23,8 +24,8 @@ module Aspera
|
|
|
23
24
|
# Use this method to analyze a EST result and raise an exception
|
|
24
25
|
# Analyzes REST call response and raises a RestCallError exception
|
|
25
26
|
# if HTTP result code is not 2XX
|
|
26
|
-
def
|
|
27
|
-
|
|
27
|
+
def raise_on_error(req,res)
|
|
28
|
+
call_context={
|
|
28
29
|
messages: [],
|
|
29
30
|
request: req,
|
|
30
31
|
response: res[:http],
|
|
@@ -36,21 +37,19 @@ module Aspera
|
|
|
36
37
|
@error_handlers.each do |handler|
|
|
37
38
|
begin
|
|
38
39
|
#Log.log.debug("test exception: #{handler[:name]}")
|
|
39
|
-
handler[:block].call(handler[:name],
|
|
40
|
-
rescue => e
|
|
40
|
+
handler[:block].call(handler[:name],call_context)
|
|
41
|
+
rescue StandardError => e
|
|
41
42
|
Log.log.error("ERROR in handler:\n#{e.message}\n#{e.backtrace}")
|
|
42
43
|
end
|
|
43
44
|
end
|
|
44
|
-
unless
|
|
45
|
-
raise RestCallError.new(context[:request],context[:response],context[:messages].join("\n"))
|
|
46
|
-
end
|
|
45
|
+
raise RestCallError.new(call_context[:request],call_context[:response],call_context[:messages].join("\n")) unless call_context[:messages].empty?
|
|
47
46
|
end
|
|
48
47
|
|
|
49
48
|
# add a new error handler (done at application initialisation)
|
|
50
49
|
# @param name : name of error handler (for logs)
|
|
51
|
-
# @param block : processing of response: takes two parameters: name,
|
|
50
|
+
# @param block : processing of response: takes two parameters: name, call_context
|
|
52
51
|
# name is the one provided here
|
|
53
|
-
#
|
|
52
|
+
# call_context is built in method raise_on_error
|
|
54
53
|
def add_handler(name,&block)
|
|
55
54
|
@error_handlers.unshift({name: name, block: block})
|
|
56
55
|
end
|
|
@@ -59,21 +58,21 @@ module Aspera
|
|
|
59
58
|
# check that key exists and is string under specified path (hash)
|
|
60
59
|
# adds other keys as secondary information
|
|
61
60
|
def add_simple_handler(name,*args)
|
|
62
|
-
add_handler(name) do |type,
|
|
61
|
+
add_handler(name) do |type,call_context|
|
|
63
62
|
# need to clone because we modify and same array is used subsequently
|
|
64
63
|
path=args.clone
|
|
65
64
|
#Log.log.debug("path=#{path}")
|
|
66
65
|
# if last in path is boolean it tells if the error is only with http error code or always
|
|
67
66
|
always=[true, false].include?(path.last) ? path.pop : false
|
|
68
|
-
if
|
|
67
|
+
if call_context[:data].is_a?(Hash) && (!call_context[:response].code.start_with?('2') || always)
|
|
69
68
|
msg_key=path.pop
|
|
70
69
|
# dig and find sub entry corresponding to path in deep hash
|
|
71
|
-
error_struct=path.inject(
|
|
72
|
-
if error_struct.is_a?(Hash)
|
|
73
|
-
RestErrorAnalyzer.add_error(
|
|
70
|
+
error_struct=path.inject(call_context[:data]) { |subhash, key| subhash.respond_to?(:keys) ? subhash[key] : nil }
|
|
71
|
+
if error_struct.is_a?(Hash) && error_struct[msg_key].is_a?(String)
|
|
72
|
+
RestErrorAnalyzer.add_error(call_context,type,error_struct[msg_key])
|
|
74
73
|
error_struct.each do |k,v|
|
|
75
74
|
next if k.eql?(msg_key)
|
|
76
|
-
RestErrorAnalyzer.add_error(
|
|
75
|
+
RestErrorAnalyzer.add_error(call_context,"#{type}(sub)","#{k}: #{v}") if [String,Integer].include?(v.class)
|
|
77
76
|
end
|
|
78
77
|
end
|
|
79
78
|
end
|
|
@@ -82,17 +81,16 @@ module Aspera
|
|
|
82
81
|
|
|
83
82
|
# used by handler to add an error description to list of errors
|
|
84
83
|
# for logging and tracing : collect error descriptions (create file to activate)
|
|
85
|
-
# @param
|
|
84
|
+
# @param call_context a Hash containing the result call_context, provided to handler
|
|
86
85
|
# @param type a string describing type of exception, for logging purpose
|
|
87
86
|
# @param msg one error message to add to list
|
|
88
|
-
def self.add_error(
|
|
89
|
-
|
|
87
|
+
def self.add_error(call_context,type,msg)
|
|
88
|
+
call_context[:messages].push(msg)
|
|
90
89
|
logfile=instance.log_file
|
|
91
90
|
# log error for further analysis (file must exist to activate)
|
|
92
|
-
if
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
end
|
|
91
|
+
return if logfile.nil? || !File.exist?(logfile)
|
|
92
|
+
File.open(logfile,'a+') do |f|
|
|
93
|
+
f.write("\n=#{type}=====\n#{call_context[:request].method} #{call_context[:request].path}\n#{call_context[:response].code}\n#{JSON.generate(call_context[:data])}\n#{call_context[:messages].join("\n")}")
|
|
96
94
|
end
|
|
97
95
|
end
|
|
98
96
|
end
|