aspera-cli 4.4.0 → 4.7.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +2095 -1503
- data/bin/ascli +2 -1
- data/bin/asession +4 -5
- data/docs/test_env.conf +3 -0
- data/examples/aoc.rb +4 -3
- data/examples/faspex4.rb +25 -25
- data/examples/proxy.pac +1 -1
- data/examples/transfer.rb +17 -17
- data/lib/aspera/aoc.rb +238 -185
- data/lib/aspera/ascmd.rb +93 -83
- data/lib/aspera/ats_api.rb +11 -10
- data/lib/aspera/cli/basic_auth_plugin.rb +13 -14
- data/lib/aspera/cli/extended_value.rb +42 -33
- data/lib/aspera/cli/formater.rb +142 -108
- data/lib/aspera/cli/info.rb +17 -0
- data/lib/aspera/cli/listener/line_dump.rb +3 -2
- data/lib/aspera/cli/listener/logger.rb +2 -1
- data/lib/aspera/cli/listener/progress.rb +16 -18
- data/lib/aspera/cli/listener/progress_multi.rb +18 -21
- data/lib/aspera/cli/main.rb +173 -149
- data/lib/aspera/cli/manager.rb +163 -168
- data/lib/aspera/cli/plugin.rb +43 -31
- data/lib/aspera/cli/plugins/alee.rb +6 -6
- data/lib/aspera/cli/plugins/aoc.rb +405 -370
- data/lib/aspera/cli/plugins/ats.rb +86 -79
- data/lib/aspera/cli/plugins/bss.rb +14 -16
- data/lib/aspera/cli/plugins/config.rb +580 -362
- data/lib/aspera/cli/plugins/console.rb +23 -19
- data/lib/aspera/cli/plugins/cos.rb +18 -18
- data/lib/aspera/cli/plugins/faspex.rb +201 -158
- data/lib/aspera/cli/plugins/faspex5.rb +80 -57
- data/lib/aspera/cli/plugins/node.rb +183 -166
- data/lib/aspera/cli/plugins/orchestrator.rb +71 -67
- data/lib/aspera/cli/plugins/preview.rb +92 -96
- data/lib/aspera/cli/plugins/server.rb +79 -75
- data/lib/aspera/cli/plugins/shares.rb +35 -19
- data/lib/aspera/cli/plugins/sync.rb +20 -22
- data/lib/aspera/cli/transfer_agent.rb +76 -113
- data/lib/aspera/cli/version.rb +2 -1
- data/lib/aspera/colors.rb +35 -27
- data/lib/aspera/command_line_builder.rb +48 -34
- data/lib/aspera/cos_node.rb +29 -21
- data/lib/aspera/data_repository.rb +3 -2
- data/lib/aspera/environment.rb +50 -45
- data/lib/aspera/fasp/{manager.rb → agent_base.rb} +28 -25
- data/lib/aspera/fasp/{connect.rb → agent_connect.rb} +52 -43
- data/lib/aspera/fasp/{local.rb → agent_direct.rb} +58 -72
- data/lib/aspera/fasp/{http_gw.rb → agent_httpgw.rb} +37 -43
- data/lib/aspera/fasp/{node.rb → agent_node.rb} +35 -16
- data/lib/aspera/fasp/agent_trsdk.rb +104 -0
- data/lib/aspera/fasp/error.rb +2 -1
- data/lib/aspera/fasp/error_info.rb +68 -52
- data/lib/aspera/fasp/installation.rb +152 -124
- data/lib/aspera/fasp/listener.rb +1 -0
- data/lib/aspera/fasp/parameters.rb +87 -92
- data/lib/aspera/fasp/parameters.yaml +305 -249
- data/lib/aspera/fasp/resume_policy.rb +11 -14
- data/lib/aspera/fasp/transfer_spec.rb +26 -0
- data/lib/aspera/fasp/uri.rb +22 -21
- data/lib/aspera/faspex_gw.rb +55 -89
- data/lib/aspera/hash_ext.rb +4 -3
- data/lib/aspera/id_generator.rb +8 -7
- data/lib/aspera/keychain/encrypted_hash.rb +121 -0
- data/lib/aspera/keychain/macos_security.rb +90 -0
- data/lib/aspera/log.rb +55 -37
- data/lib/aspera/nagios.rb +13 -12
- data/lib/aspera/node.rb +30 -25
- data/lib/aspera/oauth.rb +175 -226
- data/lib/aspera/open_application.rb +4 -3
- data/lib/aspera/persistency_action_once.rb +6 -6
- data/lib/aspera/persistency_folder.rb +5 -9
- data/lib/aspera/preview/file_types.rb +6 -5
- data/lib/aspera/preview/generator.rb +25 -24
- data/lib/aspera/preview/options.rb +16 -14
- data/lib/aspera/preview/utils.rb +98 -98
- data/lib/aspera/{proxy_auto_config.erb.js → proxy_auto_config.js} +23 -31
- data/lib/aspera/proxy_auto_config.rb +111 -20
- data/lib/aspera/rest.rb +154 -135
- data/lib/aspera/rest_call_error.rb +2 -2
- data/lib/aspera/rest_error_analyzer.rb +23 -25
- data/lib/aspera/rest_errors_aspera.rb +15 -14
- data/lib/aspera/ssh.rb +12 -10
- data/lib/aspera/sync.rb +42 -41
- data/lib/aspera/temp_file_manager.rb +18 -14
- data/lib/aspera/timer_limiter.rb +2 -1
- data/lib/aspera/uri_reader.rb +7 -5
- data/lib/aspera/web_auth.rb +79 -76
- metadata +116 -29
- data/docs/Makefile +0 -66
- data/docs/README.erb.md +0 -3973
- data/docs/README.md +0 -13
- data/docs/diagrams.txt +0 -49
- data/docs/doc_tools.rb +0 -58
- data/lib/aspera/api_detector.rb +0 -60
- data/lib/aspera/cli/plugins/shares2.rb +0 -114
- data/lib/aspera/secrets.rb +0 -20
@@ -1,34 +1,125 @@
|
|
1
|
+
# frozen_string_literal: true
|
1
2
|
require 'uri'
|
2
3
|
require 'resolv'
|
3
|
-
|
4
|
+
|
5
|
+
module URI
|
6
|
+
class Generic
|
7
|
+
# save original method that finds proxy in URI::Generic, it uses env var http_proxy
|
8
|
+
alias :find_proxy_orig find_proxy
|
9
|
+
def self.register_proxy_finder(&block)
|
10
|
+
# overload the method
|
11
|
+
define_method(:find_proxy) {|envars=ENV| block.call(to_s) || find_proxy_orig(envars)}
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
4
15
|
|
5
16
|
module Aspera
|
6
|
-
#
|
17
|
+
# Evaluate a proxy autoconfig script
|
7
18
|
class ProxyAutoConfig
|
8
19
|
# template file is read once, it contains functions that can be used in a proxy autoconf script
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
20
|
+
# it is similar to mozilla ascii_pac_utils.inc
|
21
|
+
PAC_FUNCTIONS_FILE=__FILE__.gsub(/\.rb$/,'.js').freeze
|
22
|
+
PAC_MAIN_FUNCTION='FindProxyForURL'
|
23
|
+
private_constant :PAC_FUNCTIONS_FILE,:PAC_MAIN_FUNCTION
|
24
|
+
|
25
|
+
private
|
15
26
|
|
16
|
-
#
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
# variables starting with "context_" are replaced in the ERB template file
|
21
|
-
# I did not find an easy way for the javascript to callback ruby
|
22
|
-
# and anyway, it only needs to get DNS translation
|
27
|
+
# variables starting with "context_" are replaced in the ERB template file
|
28
|
+
# I did not find an easy way for the javascript to callback ruby
|
29
|
+
# and anyway, it only needs to get DNS translation
|
30
|
+
def pac_dns_functions(context_host)
|
23
31
|
context_self='127.0.0.1'
|
24
|
-
context_host=URI.parse(service_url).host
|
25
32
|
context_ip=nil
|
26
33
|
Resolv::DNS.open{|dns|dns.each_address(context_host){|r_addr|context_ip=r_addr.to_s if r_addr.is_a?(Resolv::IPv4)}}
|
27
34
|
raise "DNS name not found: #{context_host}" if context_ip.nil?
|
28
|
-
|
29
|
-
|
30
|
-
|
35
|
+
# NOTE: Javascript code here with string inclusions
|
36
|
+
javascript=<<END_OF_JAVASCRIPT
|
37
|
+
function dnsResolve(host) {
|
38
|
+
if (host == '#{context_host}' || host == '#{context_ip}')
|
39
|
+
return '#{context_ip}';
|
40
|
+
throw 'only DNS for initial host, not ' + host;
|
41
|
+
}
|
42
|
+
function myIpAddress() {
|
43
|
+
return '#{context_self}';
|
44
|
+
}
|
45
|
+
END_OF_JAVASCRIPT
|
46
|
+
return javascript
|
47
|
+
end
|
48
|
+
|
49
|
+
public
|
50
|
+
|
51
|
+
# @param proxy_auto_config the proxy auto config script to be evaluated
|
52
|
+
def initialize(proxy_auto_config)
|
53
|
+
# user provided javascript with FindProxyForURL function
|
54
|
+
@proxy_auto_config=proxy_auto_config
|
55
|
+
# avoid multiple execution, this does not support load balancing
|
56
|
+
@cache={}
|
57
|
+
@pac_functions=nil
|
58
|
+
end
|
59
|
+
|
60
|
+
def register_uri_generic
|
61
|
+
URI::Generic.register_proxy_finder{|url_str|get_proxies(url_str).first}
|
62
|
+
# allow chaining
|
63
|
+
return self
|
64
|
+
end
|
65
|
+
|
66
|
+
# execute proxy auto config script for the given URL : https://en.wikipedia.org/wiki/Proxy_auto-config
|
67
|
+
# @return either nil, or a String formated following PAC standard
|
68
|
+
def find_proxy_for_url(service_url)
|
69
|
+
uri=URI.parse(service_url)
|
70
|
+
simple_url="#{uri.scheme}://#{uri.host}"
|
71
|
+
if !@cache.has_key?(simple_url)
|
72
|
+
Log.log.debug("PAC: starting javascript for #{service_url}")
|
73
|
+
# require at runtime, in case there is no js engine
|
74
|
+
require 'execjs'
|
75
|
+
# read template lib
|
76
|
+
@pac_functions=File.read(PAC_FUNCTIONS_FILE).freeze if @pac_functions.nil?
|
77
|
+
# to be executed is dns + utils + user function
|
78
|
+
js_to_execute="#{pac_dns_functions(uri.host)}#{@pac_functions}#{@proxy_auto_config}"
|
79
|
+
executable_js = ExecJS.compile(js_to_execute)
|
80
|
+
@cache[simple_url]=executable_js.call(PAC_MAIN_FUNCTION, simple_url, uri.host)
|
81
|
+
Log.log.debug("PAC: result: #{@cache[simple_url]}")
|
82
|
+
end
|
83
|
+
return @cache[simple_url]
|
84
|
+
end
|
85
|
+
|
86
|
+
# used to replace URI::Generic.find_proxy
|
87
|
+
# @return Array of URI, possibly empty
|
88
|
+
def get_proxies(service_url)
|
89
|
+
# prepare result
|
90
|
+
uri_list=[]
|
91
|
+
# execute PAC script
|
92
|
+
proxy_list_str=find_proxy_for_url(service_url)
|
93
|
+
if !proxy_list_str.is_a?(String)
|
94
|
+
Log.log.warn("PAC: did not return a String, returned #{proxy_list_str.class}")
|
95
|
+
return uri_list
|
96
|
+
end
|
97
|
+
proxy_list_str.strip!
|
98
|
+
proxy_list_str.gsub!(/\s+/,' ')
|
99
|
+
proxy_list_str.split(';').each do |item|
|
100
|
+
# strip and split by space
|
101
|
+
parts=item.strip.split
|
102
|
+
case parts.shift
|
103
|
+
when 'DIRECT'
|
104
|
+
raise 'DIRECT has no param' unless parts.empty?
|
105
|
+
when 'PROXY'
|
106
|
+
addr_port=parts.shift
|
107
|
+
raise 'PROXY shall have one param' unless addr_port.is_a?(String) && parts.empty?
|
108
|
+
begin
|
109
|
+
# PAC proxy addresses are <host>:<port>
|
110
|
+
if addr_port.match(/:[0-9]+$/)
|
111
|
+
# we want to return URIs, so add dummy scheme
|
112
|
+
uri_list.push(URI.parse("proxy://#{addr_port}"))
|
113
|
+
else
|
114
|
+
Log.log.warn("PAC: PROXY must be <address>:<port>, ignoring #{addr_port}")
|
115
|
+
end
|
116
|
+
rescue StandardError
|
117
|
+
Log.log.warn("PAC: cannot parse #{addr_port}")
|
118
|
+
end
|
119
|
+
else Log.log.warn("PAC: ignoring proxy type #{parts.first}: not supported")
|
120
|
+
end
|
121
|
+
end
|
122
|
+
return uri_list
|
31
123
|
end
|
32
124
|
end
|
33
125
|
end
|
34
|
-
|
data/lib/aspera/rest.rb
CHANGED
@@ -1,3 +1,4 @@
|
|
1
|
+
# frozen_string_literal: true
|
1
2
|
require 'aspera/log'
|
2
3
|
require 'aspera/oauth'
|
3
4
|
require 'aspera/rest_error_analyzer'
|
@@ -28,52 +29,52 @@ module Aspera
|
|
28
29
|
# rest call errors are raised as exception RestCallError
|
29
30
|
# and error are analyzed in RestErrorAnalyzer
|
30
31
|
class Rest
|
31
|
-
|
32
|
-
#
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
def self.insecure; @@insecure;end
|
43
|
-
|
44
|
-
def self.user_agent=(v); @@user_agent=v;Log.log.debug("user_agent => #{@@user_agent}".red);end
|
32
|
+
# global settings also valid for any subclass
|
33
|
+
@@global={ # rubocop:disable Style/ClassVars
|
34
|
+
debug: false,
|
35
|
+
# true if https ignore certificate
|
36
|
+
insecure: false,
|
37
|
+
user_agent: 'Ruby',
|
38
|
+
download_partial_suffix: '.http_partial',
|
39
|
+
# a lambda which takes the Net::HTTP as arg, use this to change parameters
|
40
|
+
session_cb: nil
|
41
|
+
}
|
45
42
|
|
46
|
-
|
47
|
-
|
48
|
-
|
43
|
+
class<<self
|
44
|
+
# define accessors
|
45
|
+
@@global.keys.each do |p|
|
46
|
+
define_method(p){@@global[p]}
|
47
|
+
define_method("#{p}="){|val|Log.log.debug("#{p} => #{val}".red);@@global[p]=val}
|
48
|
+
end
|
49
49
|
|
50
|
-
|
50
|
+
def basic_creds(user,pass); return "Basic #{Base64.strict_encode64("#{user}:#{pass}")}";end
|
51
51
|
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
52
|
+
# build URI from URL and parameters and check it is http or https
|
53
|
+
def build_uri(url,params=nil)
|
54
|
+
uri=URI.parse(url)
|
55
|
+
raise 'REST endpoint shall be http(s)' unless ['http','https'].include?(uri.scheme)
|
56
|
+
if !params.nil?
|
57
|
+
# support array url params, there is no standard. Either p[]=1&p[]=2, or p=1&p=2
|
58
|
+
if params.is_a?(Hash)
|
59
|
+
orig=params
|
60
|
+
params=[]
|
61
|
+
orig.each do |k,v|
|
62
|
+
case v
|
63
|
+
when Array
|
64
|
+
suffix=v.first.eql?('[]') ? v.shift : ''
|
65
|
+
v.each do |e|
|
66
|
+
params.push([k+suffix,e])
|
67
|
+
end
|
68
|
+
else
|
69
|
+
params.push([k,v])
|
67
70
|
end
|
68
|
-
else
|
69
|
-
params.push([k,v])
|
70
71
|
end
|
71
72
|
end
|
73
|
+
# CGI.unescape to transform back %5D into []
|
74
|
+
uri.query=CGI.unescape(URI.encode_www_form(params))
|
72
75
|
end
|
73
|
-
|
74
|
-
uri.query=CGI.unescape(URI.encode_www_form(params))
|
76
|
+
return uri
|
75
77
|
end
|
76
|
-
return uri
|
77
78
|
end
|
78
79
|
|
79
80
|
private
|
@@ -85,12 +86,10 @@ module Aspera
|
|
85
86
|
# this honors http_proxy env var
|
86
87
|
@http_session=Net::HTTP.new(uri.host, uri.port)
|
87
88
|
@http_session.use_ssl = uri.scheme.eql?('https')
|
88
|
-
|
89
|
-
@http_session.
|
90
|
-
|
91
|
-
|
92
|
-
@params[:session_cb].call(@http_session)
|
93
|
-
end
|
89
|
+
@http_session.verify_mode = OpenSSL::SSL::VERIFY_NONE if self.class.insecure
|
90
|
+
@http_session.set_debug_output($stdout) if self.class.debug
|
91
|
+
# set http options in callback, such as timeout and cert. verification
|
92
|
+
self.class.session_cb.call(@http_session) unless self.class.session_cb.nil?
|
94
93
|
# manually start session for keep alive (if supported by server, else, session is closed every time)
|
95
94
|
@http_session.start
|
96
95
|
end
|
@@ -100,33 +99,72 @@ module Aspera
|
|
100
99
|
public
|
101
100
|
|
102
101
|
attr_reader :params
|
103
|
-
attr_reader :oauth
|
104
102
|
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
103
|
+
def oauth
|
104
|
+
if @oauth.nil?
|
105
|
+
raise 'ERROR: no OAuth defined' unless @params[:auth][:type].eql?(:oauth2)
|
106
|
+
@oauth=Oauth.new(@params[:auth])
|
107
|
+
end
|
108
|
+
return @oauth
|
109
|
+
end
|
110
|
+
|
111
|
+
# @param a_rest_params [Hash] default call parameters (merged at call)
|
112
112
|
def initialize(a_rest_params)
|
113
|
-
raise
|
114
|
-
raise
|
113
|
+
raise 'ERROR: expecting Hash' unless a_rest_params.is_a?(Hash)
|
114
|
+
raise 'ERROR: expecting base_url' unless a_rest_params[:base_url].is_a?(String)
|
115
115
|
@params=a_rest_params.clone
|
116
116
|
Log.dump('REST params',@params)
|
117
117
|
# base url without trailing slashes (note: string may be frozen)
|
118
118
|
@params[:base_url]=@params[:base_url].gsub(/\/+$/,'')
|
119
119
|
@http_session=nil
|
120
120
|
# default is no auth
|
121
|
-
@params[:auth]||={:
|
121
|
+
@params[:auth]||={type: :none}
|
122
122
|
@params[:not_auth_codes]||=['401']
|
123
|
-
@oauth=
|
123
|
+
@oauth=nil
|
124
124
|
Log.dump('REST params(2)',@params)
|
125
125
|
end
|
126
126
|
|
127
|
-
def oauth_token(
|
128
|
-
raise "ERROR: expecting
|
129
|
-
return
|
127
|
+
def oauth_token(force_refresh: false)
|
128
|
+
raise "ERROR: expecting boolean, have #{force_refresh}" unless [true,false].include?(force_refresh)
|
129
|
+
return oauth.get_authorization(use_refresh_token: force_refresh)
|
130
|
+
end
|
131
|
+
|
132
|
+
def build_request(call_data)
|
133
|
+
# TODO: shall we percent encode subpath (spaces) test with access key delete with space in id
|
134
|
+
# URI.escape()
|
135
|
+
uri=self.class.build_uri("#{call_data[:base_url]}#{['','/'].include?(call_data[:subpath]) ? '' : '/'}#{call_data[:subpath]}",call_data[:url_params])
|
136
|
+
Log.log.debug("URI=#{uri}")
|
137
|
+
begin
|
138
|
+
# instanciate request object based on string name
|
139
|
+
req=Net::HTTP.const_get(call_data[:operation].capitalize).new(uri)
|
140
|
+
rescue NameError
|
141
|
+
raise "unsupported operation : #{call_data[:operation]}"
|
142
|
+
end
|
143
|
+
if call_data.has_key?(:json_params) && !call_data[:json_params].nil?
|
144
|
+
req.body=JSON.generate(call_data[:json_params])
|
145
|
+
Log.dump('body JSON data',call_data[:json_params])
|
146
|
+
#Log.log.debug("body JSON data=#{JSON.pretty_generate(call_data[:json_params])}")
|
147
|
+
req['Content-Type'] = 'application/json'
|
148
|
+
#call_data[:headers]['Accept']='application/json'
|
149
|
+
end
|
150
|
+
if call_data.has_key?(:www_body_params)
|
151
|
+
req.body=URI.encode_www_form(call_data[:www_body_params])
|
152
|
+
Log.log.debug("body www data=#{req.body.chomp}")
|
153
|
+
req['Content-Type'] = 'application/x-www-form-urlencoded'
|
154
|
+
end
|
155
|
+
if call_data.has_key?(:text_body_params)
|
156
|
+
req.body=call_data[:text_body_params]
|
157
|
+
Log.log.debug("body data=#{req.body.chomp}")
|
158
|
+
end
|
159
|
+
# set headers
|
160
|
+
if call_data.has_key?(:headers)
|
161
|
+
call_data[:headers].keys.each do |key|
|
162
|
+
req[key] = call_data[:headers][key]
|
163
|
+
end
|
164
|
+
end
|
165
|
+
# :type = :basic
|
166
|
+
req.basic_auth(call_data[:auth][:username],call_data[:auth][:password]) if call_data[:auth][:type].eql?(:basic)
|
167
|
+
return req
|
130
168
|
end
|
131
169
|
|
132
170
|
# HTTP/S REST call
|
@@ -139,21 +177,31 @@ module Aspera
|
|
139
177
|
# :url_params
|
140
178
|
# :www_body_params
|
141
179
|
# :text_body_params
|
142
|
-
# :save_to_file (filepath)
|
143
|
-
# :return_error (bool)
|
180
|
+
# :save_to_file (filepath) default: nil
|
181
|
+
# :return_error (bool) default: nil
|
182
|
+
# :redirect_max (int) default: 0
|
183
|
+
# :not_auth_codes (array)
|
184
|
+
# ----
|
185
|
+
# authentication (:auth) :
|
186
|
+
# :type (:none, :basic, :oauth2, :url)
|
187
|
+
# :username [:basic]
|
188
|
+
# :password [:basic]
|
189
|
+
# :url_creds [:url] a hash
|
190
|
+
# :* [:oauth2] see Oauth class
|
144
191
|
def call(call_data)
|
145
192
|
raise "Hash call parameter is required (#{call_data.class})" unless call_data.is_a?(Hash)
|
193
|
+
call_data[:subpath]='' if call_data[:subpath].nil?
|
146
194
|
Log.log.debug("accessing #{call_data[:subpath]}".red.bold.bg_green)
|
147
195
|
call_data[:headers]||={}
|
148
|
-
call_data[:headers]['User-Agent'] ||=
|
149
|
-
# defaults from @params are overriden by call
|
196
|
+
call_data[:headers]['User-Agent'] ||= self.class.user_agent
|
197
|
+
# defaults from @params are overriden by call data
|
150
198
|
call_data=@params.deep_merge(call_data)
|
151
199
|
case call_data[:auth][:type]
|
152
200
|
when :none
|
153
201
|
# no auth
|
154
202
|
when :basic
|
155
|
-
Log.log.debug(
|
156
|
-
|
203
|
+
Log.log.debug('using Basic auth')
|
204
|
+
# done in build_req
|
157
205
|
when :oauth2
|
158
206
|
call_data[:headers]['Authorization']=oauth_token unless call_data[:headers].has_key?('Authorization')
|
159
207
|
when :url
|
@@ -163,67 +211,33 @@ module Aspera
|
|
163
211
|
end
|
164
212
|
else raise "unsupported auth type: [#{call_data[:auth][:type]}]"
|
165
213
|
end
|
166
|
-
|
167
|
-
# URI.escape()
|
168
|
-
uri=self.class.build_uri("#{@params[:base_url]}#{call_data[:subpath].nil? ? '' : '/'}#{call_data[:subpath]}",call_data[:url_params])
|
169
|
-
Log.log.debug("URI=#{uri}")
|
170
|
-
begin
|
171
|
-
# instanciate request object based on string name
|
172
|
-
req=Object::const_get('Net::HTTP::'+call_data[:operation].capitalize).new(uri.request_uri)
|
173
|
-
rescue NameError => e
|
174
|
-
raise "unsupported operation : #{call_data[:operation]}"
|
175
|
-
end
|
176
|
-
if call_data.has_key?(:json_params) and !call_data[:json_params].nil? then
|
177
|
-
req.body=JSON.generate(call_data[:json_params])
|
178
|
-
Log.dump('body JSON data',call_data[:json_params])
|
179
|
-
#Log.log.debug("body JSON data=#{JSON.pretty_generate(call_data[:json_params])}")
|
180
|
-
req['Content-Type'] = 'application/json'
|
181
|
-
#call_data[:headers]['Accept']='application/json'
|
182
|
-
end
|
183
|
-
if call_data.has_key?(:www_body_params) then
|
184
|
-
req.body=URI.encode_www_form(call_data[:www_body_params])
|
185
|
-
Log.log.debug("body www data=#{req.body.chomp}")
|
186
|
-
req['Content-Type'] = 'application/x-www-form-urlencoded'
|
187
|
-
end
|
188
|
-
if call_data.has_key?(:text_body_params) then
|
189
|
-
req.body=call_data[:text_body_params]
|
190
|
-
Log.log.debug("body data=#{req.body.chomp}")
|
191
|
-
end
|
192
|
-
# set headers
|
193
|
-
if call_data.has_key?(:headers) then
|
194
|
-
call_data[:headers].keys.each do |key|
|
195
|
-
req[key] = call_data[:headers][key]
|
196
|
-
end
|
197
|
-
end
|
198
|
-
# :type = :basic
|
199
|
-
req.basic_auth(*basic_auth_data) unless basic_auth_data.nil?
|
200
|
-
|
214
|
+
req=build_request(call_data)
|
201
215
|
Log.log.debug("call_data = #{call_data}")
|
202
|
-
result={:
|
216
|
+
result={http: nil}
|
203
217
|
# start a block to be able to retry the actual HTTP request
|
204
218
|
begin
|
205
219
|
# we try the call, and will retry only if oauth, as we can, first with refresh, and then re-auth if refresh is bad
|
206
220
|
oauth_tries ||= 2
|
207
|
-
tries_remain_redirect||=
|
208
|
-
Log.log.debug(
|
221
|
+
tries_remain_redirect||=call_data[:redirect_max].nil? ? 0 : call_data[:redirect_max].to_i
|
222
|
+
Log.log.debug('send request')
|
209
223
|
# make http request (pipelined)
|
210
224
|
http_session.request(req) do |response|
|
211
225
|
result[:http] = response
|
212
|
-
if call_data.
|
226
|
+
if !call_data[:save_to_file].nil? && result[:http].code.to_s.start_with?('2')
|
213
227
|
total_size=result[:http]['Content-Length'].to_i
|
214
228
|
progress=ProgressBar.create(
|
215
|
-
:
|
216
|
-
:
|
217
|
-
:
|
218
|
-
:
|
219
|
-
Log.log.debug(
|
229
|
+
format: '%a %B %p%% %r KB/sec %e',
|
230
|
+
rate_scale: lambda{|rate|rate/1024},
|
231
|
+
title: 'progress',
|
232
|
+
total: total_size)
|
233
|
+
Log.log.debug('before write file')
|
220
234
|
target_file=call_data[:save_to_file]
|
221
235
|
# override user's path to path in header
|
222
|
-
if !response['Content-Disposition'].nil?
|
236
|
+
if !response['Content-Disposition'].nil? && (m=response['Content-Disposition'].match(/filename="([^"]+)"/))
|
223
237
|
target_file=File.join(File.dirname(target_file),m[1])
|
224
238
|
end
|
225
239
|
# download with temp filename
|
226
|
-
target_file_tmp="#{target_file}#{
|
240
|
+
target_file_tmp="#{target_file}#{self.class.download_partial_suffix}"
|
227
241
|
Log.log.debug("saving to: #{target_file}")
|
228
242
|
File.open(target_file_tmp, 'wb') do |file|
|
229
243
|
result[:http].read_body do |fragment|
|
@@ -238,27 +252,27 @@ module Aspera
|
|
238
252
|
progress=nil
|
239
253
|
end # save_to_file
|
240
254
|
end
|
241
|
-
# sometimes there is a
|
242
|
-
result[:http].body.force_encoding(
|
255
|
+
# sometimes there is a UTF8 char (e.g. (c) )
|
256
|
+
result[:http].body.force_encoding('UTF-8') if result[:http].body.is_a?(String)
|
243
257
|
Log.log.debug("result: body=#{result[:http].body}")
|
244
258
|
result_mime=(result[:http]['Content-Type']||'text/plain').split(';').first
|
245
|
-
case result_mime
|
259
|
+
result[:data]=case result_mime
|
246
260
|
when 'application/json','application/vnd.api+json'
|
247
|
-
|
261
|
+
JSON.parse(result[:http].body) rescue nil
|
248
262
|
else #when 'text/plain'
|
249
|
-
result[:
|
263
|
+
result[:http].body
|
250
264
|
end
|
251
265
|
Log.dump("result: parsed: #{result_mime}",result[:data])
|
252
266
|
Log.log.debug("result: code=#{result[:http].code}")
|
253
|
-
RestErrorAnalyzer.instance.
|
267
|
+
RestErrorAnalyzer.instance.raise_on_error(req,result)
|
254
268
|
rescue RestCallError => e
|
255
269
|
# not authorized: oauth token expired
|
256
|
-
if
|
270
|
+
if call_data[:not_auth_codes].include?(result[:http].code.to_s) && call_data[:auth][:type].eql?(:oauth2)
|
257
271
|
begin
|
258
272
|
# try to use refresh token
|
259
|
-
req['Authorization']=oauth_token(
|
273
|
+
req['Authorization']=oauth_token(force_refresh: true)
|
260
274
|
rescue RestCallError => e
|
261
|
-
Log.log.error(
|
275
|
+
Log.log.error('refresh failed'.bg_red)
|
262
276
|
# regenerate a brand new token
|
263
277
|
req['Authorization']=oauth_token
|
264
278
|
end
|
@@ -266,25 +280,30 @@ module Aspera
|
|
266
280
|
retry unless (oauth_tries -= 1).zero?
|
267
281
|
end # if oauth
|
268
282
|
# moved ?
|
269
|
-
|
270
|
-
|
271
|
-
|
272
|
-
|
273
|
-
|
274
|
-
|
275
|
-
|
283
|
+
raise e unless e.response.is_a?(Net::HTTPRedirection)
|
284
|
+
if tries_remain_redirect.positive?
|
285
|
+
tries_remain_redirect-=1
|
286
|
+
Log.log.info("URL is moved: #{e.response['location']}")
|
287
|
+
current_uri=URI.parse(call_data[:base_url])
|
288
|
+
redir_uri=URI.parse(e.response['location'])
|
289
|
+
call_data[:base_url]=e.response['location']
|
290
|
+
call_data[:subpath]=''
|
291
|
+
if current_uri.host.eql?(redir_uri.host) && current_uri.port.eql?(redir_uri.port)
|
292
|
+
req=build_request(call_data)
|
293
|
+
retry
|
276
294
|
else
|
277
|
-
|
295
|
+
# change host
|
296
|
+
Log.log.info("Redirect changes host: #{current_uri.host} -> #{redir_uri.host}")
|
297
|
+
return self.class.new(call_data).call(call_data)
|
278
298
|
end
|
279
299
|
else
|
280
|
-
raise e
|
300
|
+
raise e unless call_data[:return_error]
|
281
301
|
end
|
282
302
|
# raise exception if could not retry and not return error in result
|
283
303
|
raise e unless call_data[:return_error]
|
284
304
|
end # begin request
|
285
305
|
Log.log.debug("result=#{result}")
|
286
306
|
return result
|
287
|
-
|
288
307
|
end
|
289
308
|
|
290
309
|
#
|
@@ -293,23 +312,23 @@ module Aspera
|
|
293
312
|
|
294
313
|
# @param encoding : one of: :json_params, :url_params
|
295
314
|
def create(subpath,params,encoding=:json_params)
|
296
|
-
return call({:
|
315
|
+
return call({operation: 'POST',subpath: subpath,headers: {'Accept'=>'application/json'},encoding=>params})
|
297
316
|
end
|
298
317
|
|
299
318
|
def read(subpath,args=nil)
|
300
|
-
return call({:
|
319
|
+
return call({operation: 'GET',subpath: subpath,headers: {'Accept'=>'application/json'},url_params: args})
|
301
320
|
end
|
302
321
|
|
303
322
|
def update(subpath,params)
|
304
|
-
return call({:
|
323
|
+
return call({operation: 'PUT',subpath: subpath,headers: {'Accept'=>'application/json'},json_params: params})
|
305
324
|
end
|
306
325
|
|
307
326
|
def delete(subpath)
|
308
|
-
return call({:
|
327
|
+
return call({operation: 'DELETE',subpath: subpath,headers: {'Accept'=>'application/json'}})
|
309
328
|
end
|
310
329
|
|
311
330
|
def cancel(subpath)
|
312
|
-
return call({:
|
331
|
+
return call({operation: 'CANCEL',subpath: subpath,headers: {'Accept'=>'application/json'}})
|
313
332
|
end
|
314
333
|
end
|
315
334
|
end #module Aspera
|
@@ -1,8 +1,8 @@
|
|
1
|
+
# frozen_string_literal: true
|
1
2
|
module Aspera
|
2
3
|
# raised on error after REST call
|
3
4
|
class RestCallError < StandardError
|
4
|
-
attr_accessor :request
|
5
|
-
attr_accessor :response
|
5
|
+
attr_accessor :request, :response
|
6
6
|
# @param http response
|
7
7
|
def initialize(req,resp,msg)
|
8
8
|
@request = req
|