aspera-cli 4.2.0 → 4.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +749 -353
- data/docs/Makefile +4 -4
- data/docs/README.erb.md +743 -283
- data/docs/doc_tools.rb +58 -0
- data/docs/test_env.conf +9 -1
- data/examples/aoc.rb +14 -3
- data/examples/faspex4.rb +89 -0
- data/lib/aspera/aoc.rb +24 -22
- data/lib/aspera/cli/main.rb +48 -20
- data/lib/aspera/cli/plugin.rb +13 -6
- data/lib/aspera/cli/plugins/aoc.rb +117 -78
- data/lib/aspera/cli/plugins/config.rb +127 -80
- data/lib/aspera/cli/plugins/faspex.rb +112 -63
- data/lib/aspera/cli/plugins/faspex5.rb +29 -25
- data/lib/aspera/cli/plugins/node.rb +54 -25
- data/lib/aspera/cli/plugins/preview.rb +94 -68
- data/lib/aspera/cli/plugins/server.rb +16 -5
- data/lib/aspera/cli/transfer_agent.rb +92 -72
- data/lib/aspera/cli/version.rb +1 -1
- data/lib/aspera/command_line_builder.rb +48 -31
- data/lib/aspera/cos_node.rb +4 -3
- data/lib/aspera/fasp/http_gw.rb +47 -26
- data/lib/aspera/fasp/local.rb +31 -24
- data/lib/aspera/fasp/manager.rb +3 -0
- data/lib/aspera/fasp/node.rb +23 -1
- data/lib/aspera/fasp/parameters.rb +72 -89
- data/lib/aspera/fasp/parameters.yaml +531 -0
- data/lib/aspera/fasp/uri.rb +1 -1
- data/lib/aspera/faspex_gw.rb +10 -9
- data/lib/aspera/id_generator.rb +22 -0
- data/lib/aspera/node.rb +11 -3
- data/lib/aspera/oauth.rb +131 -135
- data/lib/aspera/persistency_action_once.rb +11 -7
- data/lib/aspera/persistency_folder.rb +6 -26
- data/lib/aspera/rest.rb +1 -1
- data/lib/aspera/sync.rb +40 -35
- data/lib/aspera/timer_limiter.rb +22 -0
- data/lib/aspera/web_auth.rb +105 -0
- metadata +22 -4
- data/docs/transfer_spec.html +0 -99
- data/lib/aspera/fasp/aoc.rb +0 -24
data/lib/aspera/oauth.rb
CHANGED
|
@@ -1,25 +1,35 @@
|
|
|
1
1
|
require 'aspera/open_application'
|
|
2
|
+
require 'aspera/web_auth'
|
|
3
|
+
require 'aspera/id_generator'
|
|
2
4
|
require 'base64'
|
|
3
5
|
require 'date'
|
|
4
6
|
require 'socket'
|
|
5
7
|
require 'securerandom'
|
|
6
8
|
|
|
7
9
|
module Aspera
|
|
8
|
-
#
|
|
10
|
+
# Implement OAuth 2 for the REST client and generate a bearer token
|
|
9
11
|
# call get_authorization() to get a token.
|
|
10
12
|
# bearer tokens are kept in memory and also in a file cache for later re-use
|
|
11
|
-
# if a token is expired (api returns 4xx), call again get_authorization({:
|
|
13
|
+
# if a token is expired (api returns 4xx), call again get_authorization({refresh: true})
|
|
12
14
|
class Oauth
|
|
15
|
+
# used for code exchange
|
|
16
|
+
DEFAULT_PATH_AUTHORIZE='authorize'
|
|
17
|
+
# to generate token
|
|
18
|
+
DEFAULT_PATH_TOKEN='token'
|
|
19
|
+
# field with token in result
|
|
20
|
+
DEFAULT_TOKEN_FIELD='access_token'
|
|
13
21
|
private
|
|
14
22
|
# remove 5 minutes to account for time offset (TODO: configurable?)
|
|
15
|
-
|
|
23
|
+
JWT_NOTBEFORE_OFFSET_SEC=300
|
|
16
24
|
# one hour validity (TODO: configurable?)
|
|
17
|
-
|
|
25
|
+
JWT_EXPIRY_OFFSET_SEC=3600
|
|
26
|
+
# tokens older than 30 minutes will be discarded from cache
|
|
27
|
+
TOKEN_CACHE_EXPIRY_SEC=1800
|
|
28
|
+
# a prefix for persistency of tokens (garbage collect)
|
|
18
29
|
PERSIST_CATEGORY_TOKEN='token'
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
private_constant :JWT_NOTBEFORE_OFFSET,:JWT_EXPIRY_OFFSET,:PERSIST_CATEGORY_TOKEN,:TOKEN_EXPIRY_SECONDS,:THANK_YOU_HTML
|
|
30
|
+
ONE_HOUR_AS_DAY_FRACTION=Rational(1,24)
|
|
31
|
+
|
|
32
|
+
private_constant :JWT_NOTBEFORE_OFFSET_SEC,:JWT_EXPIRY_OFFSET_SEC,:PERSIST_CATEGORY_TOKEN,:TOKEN_CACHE_EXPIRY_SEC,:ONE_HOUR_AS_DAY_FRACTION
|
|
23
33
|
class << self
|
|
24
34
|
# OAuth methods supported
|
|
25
35
|
def auth_types
|
|
@@ -44,8 +54,25 @@ module Aspera
|
|
|
44
54
|
def flush_tokens
|
|
45
55
|
persist_mgr.garbage_collect(PERSIST_CATEGORY_TOKEN,nil)
|
|
46
56
|
end
|
|
57
|
+
|
|
58
|
+
def register_decoder(method)
|
|
59
|
+
@decoders||=[]
|
|
60
|
+
@decoders.push(method)
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
def decode_token(token)
|
|
64
|
+
Log.log.debug(">>>> #{token} : #{@decoders.length}")
|
|
65
|
+
@decoders.each do |decoder|
|
|
66
|
+
result=decoder.call(token) rescue nil
|
|
67
|
+
return result unless result.nil?
|
|
68
|
+
end
|
|
69
|
+
return nil
|
|
70
|
+
end
|
|
47
71
|
end
|
|
48
72
|
|
|
73
|
+
# seems to be quite standard token encoding (RFC?)
|
|
74
|
+
self.register_decoder lambda { |token| parts=token.split('.'); raise "not aoc token" unless parts.length.eql?(3); JSON.parse(Base64.decode64(parts[1]))}
|
|
75
|
+
|
|
49
76
|
# for supported parameters, look in the code for @params
|
|
50
77
|
# parameters are provided all with oauth_ prefix :
|
|
51
78
|
# :base_url
|
|
@@ -55,8 +82,8 @@ module Aspera
|
|
|
55
82
|
# :jwt_audience
|
|
56
83
|
# :jwt_private_key_obj
|
|
57
84
|
# :jwt_subject
|
|
58
|
-
# :path_authorize (default:
|
|
59
|
-
# :path_token (default:
|
|
85
|
+
# :path_authorize (default: DEFAULT_PATH_AUTHORIZE)
|
|
86
|
+
# :path_token (default: DEFAULT_PATH_TOKEN)
|
|
60
87
|
# :scope (optional)
|
|
61
88
|
# :grant (one of returned by self.auth_types)
|
|
62
89
|
# :url_token
|
|
@@ -64,21 +91,21 @@ module Aspera
|
|
|
64
91
|
# :user_pass
|
|
65
92
|
# :token_type
|
|
66
93
|
def initialize(auth_params)
|
|
67
|
-
Log.log.debug
|
|
94
|
+
Log.log.debug("auth=#{auth_params}")
|
|
68
95
|
@params=auth_params.clone
|
|
69
96
|
# default values
|
|
70
97
|
# name of field to take as token from result of call to /token
|
|
71
|
-
@params[:token_field]||=
|
|
98
|
+
@params[:token_field]||=DEFAULT_TOKEN_FIELD
|
|
72
99
|
# default endpoint for /token
|
|
73
|
-
@params[:path_token]||=
|
|
100
|
+
@params[:path_token]||=DEFAULT_PATH_TOKEN
|
|
74
101
|
# default endpoint for /authorize
|
|
75
|
-
@params[:path_authorize]||=
|
|
76
|
-
rest_params={:
|
|
102
|
+
@params[:path_authorize]||=DEFAULT_PATH_AUTHORIZE
|
|
103
|
+
rest_params={base_url: @params[:base_url]}
|
|
77
104
|
if @params.has_key?(:client_id)
|
|
78
|
-
rest_params.merge!({:
|
|
79
|
-
:
|
|
80
|
-
:
|
|
81
|
-
:
|
|
105
|
+
rest_params.merge!({auth: {
|
|
106
|
+
type: :basic,
|
|
107
|
+
username: @params[:client_id],
|
|
108
|
+
password: @params[:client_secret]
|
|
82
109
|
}})
|
|
83
110
|
end
|
|
84
111
|
@token_auth_api=Rest.new(rest_params)
|
|
@@ -89,39 +116,37 @@ module Aspera
|
|
|
89
116
|
# we could check that host is localhost or local address
|
|
90
117
|
end
|
|
91
118
|
# cleanup expired tokens
|
|
92
|
-
self.class.persist_mgr.garbage_collect(PERSIST_CATEGORY_TOKEN,
|
|
119
|
+
self.class.persist_mgr.garbage_collect(PERSIST_CATEGORY_TOKEN,TOKEN_CACHE_EXPIRY_SEC)
|
|
93
120
|
end
|
|
94
121
|
|
|
95
122
|
# open the login page, wait for code and check_code, then return code
|
|
96
123
|
def goto_page_and_get_code(login_page_url,check_code)
|
|
97
|
-
|
|
124
|
+
Log.log.info("login_page_url=#{login_page_url}".bg_red.gray)
|
|
125
|
+
# start a web server to receive request code
|
|
126
|
+
webserver=WebAuth.new(@params[:redirect_uri])
|
|
127
|
+
# start browser on login page
|
|
128
|
+
OpenApplication.instance.uri(login_page_url)
|
|
129
|
+
# wait for code in request
|
|
130
|
+
request_params=webserver.get_request
|
|
98
131
|
Log.log.error("state does not match") if !check_code.eql?(request_params['state'])
|
|
99
132
|
code=request_params['code']
|
|
100
133
|
return code
|
|
101
134
|
end
|
|
102
135
|
|
|
103
|
-
def
|
|
136
|
+
def create_token(rest_params)
|
|
104
137
|
return @token_auth_api.call({
|
|
105
|
-
:
|
|
106
|
-
:
|
|
107
|
-
:
|
|
138
|
+
operation: 'POST',
|
|
139
|
+
subpath: @params[:path_token],
|
|
140
|
+
headers: {'Accept'=>'application/json'}}.merge(rest_params))
|
|
108
141
|
end
|
|
109
142
|
|
|
110
|
-
#
|
|
111
|
-
def
|
|
112
|
-
return create_token_advanced({:www_body_params=>creation_params})
|
|
113
|
-
end
|
|
114
|
-
|
|
115
|
-
# @return Array list of unique identifiers of token
|
|
116
|
-
def token_cache_ids(api_scope)
|
|
143
|
+
# @return unique identifier of token
|
|
144
|
+
def token_cache_id(api_scope)
|
|
117
145
|
oauth_uri=URI.parse(@params[:base_url])
|
|
118
|
-
parts=[PERSIST_CATEGORY_TOKEN,oauth_uri.host
|
|
119
|
-
|
|
120
|
-
parts.push(@params[
|
|
121
|
-
|
|
122
|
-
parts.push(@params[:url_token]) if @params.has_key?(:url_token)
|
|
123
|
-
parts.push(@params[:api_key]) if @params.has_key?(:api_key)
|
|
124
|
-
return parts
|
|
146
|
+
parts=[PERSIST_CATEGORY_TOKEN,api_scope,oauth_uri.host,oauth_uri.path]
|
|
147
|
+
# add some of the parameters that uniquely define the token
|
|
148
|
+
[:grant,:jwt_subject,:user_name,:url_token,:api_key].inject(parts){|p,i|p.push(@params[i])}
|
|
149
|
+
return IdGenerator.from_list(parts)
|
|
125
150
|
end
|
|
126
151
|
|
|
127
152
|
public
|
|
@@ -141,22 +166,23 @@ module Aspera
|
|
|
141
166
|
use_refresh_token=options[:refresh]
|
|
142
167
|
|
|
143
168
|
# generate token identifier to use with cache
|
|
144
|
-
|
|
169
|
+
token_id=token_cache_id(api_scope)
|
|
145
170
|
|
|
146
171
|
# get token_data from cache (or nil), token_data is what is returned by /token
|
|
147
|
-
token_data=self.class.persist_mgr.get(
|
|
172
|
+
token_data=self.class.persist_mgr.get(token_id)
|
|
148
173
|
token_data=JSON.parse(token_data) unless token_data.nil?
|
|
149
|
-
|
|
150
174
|
# Optional optimization: check if node token is expired, then force refresh
|
|
151
175
|
# in case the transfer agent cannot refresh himself
|
|
152
176
|
# else, anyway, faspmanager is equipped with refresh code
|
|
153
177
|
if !token_data.nil?
|
|
154
|
-
|
|
178
|
+
# TODO: use @params[:token_field] ?
|
|
179
|
+
decoded_node_token = self.class.decode_token(token_data['access_token'])
|
|
180
|
+
Log.dump('decoded_node_token',decoded_node_token) unless decoded_node_token.nil?
|
|
155
181
|
if decoded_node_token.is_a?(Hash) and decoded_node_token['expires_at'].is_a?(String)
|
|
156
|
-
Log.dump('decoded_node_token',decoded_node_token)
|
|
157
182
|
expires_at=DateTime.parse(decoded_node_token['expires_at'])
|
|
158
|
-
|
|
159
|
-
|
|
183
|
+
# Time.at(decoded_node_token['exp'])
|
|
184
|
+
# does it seem expired, with one hour of security
|
|
185
|
+
use_refresh_token=true if DateTime.now > (expires_at-ONE_HOUR_AS_DAY_FRACTION)
|
|
160
186
|
end
|
|
161
187
|
end
|
|
162
188
|
|
|
@@ -167,7 +193,7 @@ module Aspera
|
|
|
167
193
|
refresh_token=token_data['refresh_token']
|
|
168
194
|
end
|
|
169
195
|
# delete caches
|
|
170
|
-
self.class.persist_mgr.delete(
|
|
196
|
+
self.class.persist_mgr.delete(token_id)
|
|
171
197
|
token_data=nil
|
|
172
198
|
# lets try the existing refresh token
|
|
173
199
|
if !refresh_token.nil?
|
|
@@ -175,14 +201,14 @@ module Aspera
|
|
|
175
201
|
# try to refresh
|
|
176
202
|
# note: admin token has no refresh, and lives by default 1800secs
|
|
177
203
|
# Note: scope is mandatory in Files, and we can either provide basic auth, or client_Secret in data
|
|
178
|
-
resp=
|
|
179
|
-
:
|
|
180
|
-
:refresh_token
|
|
204
|
+
resp=create_token(www_body_params: p_client_id_and_scope.merge({
|
|
205
|
+
grant_type: 'refresh_token',
|
|
206
|
+
refresh_token: refresh_token}))
|
|
181
207
|
if resp[:http].code.start_with?('2') then
|
|
182
|
-
# save only if success
|
|
208
|
+
# save only if success
|
|
183
209
|
json_data=resp[:http].body
|
|
184
210
|
token_data=JSON.parse(json_data)
|
|
185
|
-
self.class.persist_mgr.put(
|
|
211
|
+
self.class.persist_mgr.put(token_id,json_data)
|
|
186
212
|
else
|
|
187
213
|
Log.log.debug("refresh failed: #{resp[:http].body}".bg_red)
|
|
188
214
|
end
|
|
@@ -197,19 +223,19 @@ module Aspera
|
|
|
197
223
|
# AoC Web based Auth
|
|
198
224
|
check_code=SecureRandom.uuid
|
|
199
225
|
auth_params=p_client_id_and_scope.merge({
|
|
200
|
-
:
|
|
201
|
-
:
|
|
202
|
-
:
|
|
226
|
+
response_type: 'code',
|
|
227
|
+
redirect_uri: @params[:redirect_uri],
|
|
228
|
+
state: check_code
|
|
203
229
|
})
|
|
204
230
|
auth_params[:client_secret]=@params[:client_secret] if @params.has_key?(:client_secret)
|
|
205
231
|
login_page_url=Rest.build_uri("#{@params[:base_url]}/#{@params[:path_authorize]}",auth_params)
|
|
206
232
|
# here, we need a human to authorize on a web page
|
|
207
233
|
code=goto_page_and_get_code(login_page_url,check_code)
|
|
208
234
|
# exchange code for token
|
|
209
|
-
resp=
|
|
210
|
-
:
|
|
211
|
-
:
|
|
212
|
-
:
|
|
235
|
+
resp=create_token(www_body_params: p_client_id_and_scope.merge({
|
|
236
|
+
grant_type: 'authorization_code',
|
|
237
|
+
code: code,
|
|
238
|
+
redirect_uri: @params[:redirect_uri]
|
|
213
239
|
}))
|
|
214
240
|
when :jwt
|
|
215
241
|
# https://tools.ietf.org/html/rfc7519
|
|
@@ -219,84 +245,81 @@ module Aspera
|
|
|
219
245
|
Log.log.info("seconds=#{seconds_since_epoch}")
|
|
220
246
|
|
|
221
247
|
payload = {
|
|
222
|
-
:
|
|
223
|
-
:
|
|
224
|
-
:
|
|
225
|
-
:
|
|
226
|
-
:
|
|
248
|
+
iss: @params[:client_id], # issuer
|
|
249
|
+
sub: @params[:jwt_subject], # subject
|
|
250
|
+
aud: @params[:jwt_audience], # audience
|
|
251
|
+
nbf: seconds_since_epoch-JWT_NOTBEFORE_OFFSET_SEC, # not before
|
|
252
|
+
exp: seconds_since_epoch+JWT_EXPIRY_OFFSET_SEC # expiration
|
|
227
253
|
}
|
|
228
254
|
# Hum.. compliant ? TODO: remove when Faspex5 API is clarified
|
|
229
|
-
if @params
|
|
230
|
-
payload[:jti] = SecureRandom.uuid
|
|
231
|
-
payload[:iat] = seconds_since_epoch
|
|
232
|
-
payload.delete(:nbf)
|
|
233
|
-
p_scope[:redirect_uri]="https://127.0.0.1:5000/token"
|
|
255
|
+
if @params.has_key?(:f5_username)
|
|
256
|
+
payload[:jti] = SecureRandom.uuid # JWT id
|
|
257
|
+
payload[:iat] = seconds_since_epoch # issued at
|
|
258
|
+
payload.delete(:nbf) # not used in f5
|
|
259
|
+
p_scope[:redirect_uri]="https://127.0.0.1:5000/token" # used ?
|
|
234
260
|
p_scope[:state]=SecureRandom.uuid
|
|
235
261
|
p_scope[:client_id]=@params[:client_id]
|
|
236
|
-
@token_auth_api.params[:auth]={:
|
|
262
|
+
@token_auth_api.params[:auth]={type: :basic,username: @params[:f5_username], password: @params[:f5_password]}
|
|
237
263
|
end
|
|
238
264
|
|
|
239
265
|
# non standard, only for global ids
|
|
240
266
|
payload.merge!(@params[:jwt_add]) if @params.has_key?(:jwt_add)
|
|
267
|
+
Log.log.debug("JWT payload=[#{payload}]")
|
|
241
268
|
|
|
242
269
|
rsa_private=@params[:jwt_private_key_obj] # type: OpenSSL::PKey::RSA
|
|
243
|
-
|
|
244
270
|
Log.log.debug("private=[#{rsa_private}]")
|
|
245
271
|
|
|
246
|
-
|
|
247
|
-
assertion = JWT.encode(payload, rsa_private, 'RS256',@params[:jwt_headers]||{})
|
|
248
|
-
|
|
272
|
+
assertion = JWT.encode(payload, rsa_private, 'RS256', @params[:jwt_headers]||{})
|
|
249
273
|
Log.log.debug("assertion=[#{assertion}]")
|
|
250
274
|
|
|
251
|
-
resp=
|
|
252
|
-
:
|
|
253
|
-
:
|
|
275
|
+
resp=create_token(www_body_params: p_scope.merge({
|
|
276
|
+
grant_type: 'urn:ietf:params:oauth:grant-type:jwt-bearer',
|
|
277
|
+
assertion: assertion
|
|
254
278
|
}))
|
|
255
279
|
when :url_token
|
|
256
280
|
# AoC Public Link
|
|
257
|
-
params={:
|
|
281
|
+
params={url_token: @params[:url_token]}
|
|
258
282
|
params[:password]=@params[:password] if @params.has_key?(:password)
|
|
259
|
-
resp=
|
|
260
|
-
:
|
|
261
|
-
:
|
|
262
|
-
|
|
263
|
-
})})
|
|
283
|
+
resp=create_token({
|
|
284
|
+
json_params: params,
|
|
285
|
+
url_params: p_scope.merge({grant_type: 'url_token'})
|
|
286
|
+
})
|
|
264
287
|
when :ibm_apikey
|
|
265
288
|
# ATS
|
|
266
|
-
resp=
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
289
|
+
resp=create_token(www_body_params: {
|
|
290
|
+
grant_type: 'urn:ibm:params:oauth:grant-type:apikey',
|
|
291
|
+
response_type: 'cloud_iam',
|
|
292
|
+
apikey: @params[:api_key]
|
|
270
293
|
})
|
|
271
294
|
when :delegated_refresh
|
|
272
295
|
# COS
|
|
273
|
-
resp=
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
296
|
+
resp=create_token(www_body_params: {
|
|
297
|
+
grant_type: 'urn:ibm:params:oauth:grant-type:apikey',
|
|
298
|
+
response_type: 'delegated_refresh_token',
|
|
299
|
+
apikey: @params[:api_key],
|
|
300
|
+
receiver_client_ids: 'aspera_ats'
|
|
278
301
|
})
|
|
279
302
|
when :header_userpass
|
|
280
303
|
# used in Faspex apiv4 and shares2
|
|
281
|
-
resp=
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
:
|
|
285
|
-
:
|
|
286
|
-
:
|
|
287
|
-
|
|
304
|
+
resp=create_token(
|
|
305
|
+
json_params: p_client_id_and_scope.merge({grant_type: 'password'}), #:www_body_params also works
|
|
306
|
+
auth: {
|
|
307
|
+
type: :basic,
|
|
308
|
+
username: @params[:user_name],
|
|
309
|
+
password: @params[:user_pass]}
|
|
310
|
+
)
|
|
288
311
|
when :body_userpass
|
|
289
312
|
# legacy, not used
|
|
290
|
-
resp=
|
|
291
|
-
:
|
|
292
|
-
:
|
|
293
|
-
:
|
|
313
|
+
resp=create_token(www_body_params: p_client_id_and_scope.merge({
|
|
314
|
+
grant_type: 'password',
|
|
315
|
+
username: @params[:user_name],
|
|
316
|
+
password: @params[:user_pass]
|
|
294
317
|
}))
|
|
295
318
|
when :body_data
|
|
296
319
|
# used in Faspex apiv5
|
|
297
|
-
resp=
|
|
298
|
-
:
|
|
299
|
-
:
|
|
320
|
+
resp=create_token({
|
|
321
|
+
auth: {type: :none},
|
|
322
|
+
json_params: @params[:userpass_body],
|
|
300
323
|
})
|
|
301
324
|
else
|
|
302
325
|
raise "auth grant type unknown: #{@params[:grant]}"
|
|
@@ -304,39 +327,12 @@ module Aspera
|
|
|
304
327
|
# TODO: test return code ?
|
|
305
328
|
json_data=resp[:http].body
|
|
306
329
|
token_data=JSON.parse(json_data)
|
|
307
|
-
self.class.persist_mgr.put(
|
|
330
|
+
self.class.persist_mgr.put(token_id,json_data)
|
|
308
331
|
end # if ! in_cache
|
|
309
|
-
|
|
332
|
+
raise "API error: No such field in answer: #{@params[:token_field]}" unless token_data.has_key?(@params[:token_field])
|
|
310
333
|
# ok we shall have a token here
|
|
311
334
|
return 'Bearer '+token_data[@params[:token_field]]
|
|
312
335
|
end
|
|
313
336
|
|
|
314
|
-
# open the login page, wait for code and return parameters
|
|
315
|
-
def self.goto_page_and_get_request(redirect_uri,login_page_url,html_page=THANK_YOU_HTML)
|
|
316
|
-
Log.log.info "login_page_url=#{login_page_url}".bg_red().gray()
|
|
317
|
-
# browser start is not blocking, we hope here that starting is slower than opening port
|
|
318
|
-
OpenApplication.instance.uri(login_page_url)
|
|
319
|
-
port=URI.parse(redirect_uri).port
|
|
320
|
-
Log.log.info "listening on port #{port}"
|
|
321
|
-
request_params=nil
|
|
322
|
-
TCPServer.open('127.0.0.1', port) { |webserver|
|
|
323
|
-
Log.log.info "server=#{webserver}"
|
|
324
|
-
websession = webserver.accept
|
|
325
|
-
sleep 1 # TODO: sometimes: returns nil ? use webrick ?
|
|
326
|
-
line = websession.gets.chomp
|
|
327
|
-
Log.log.info "line=#{line}"
|
|
328
|
-
if ! line.start_with?('GET /?') then
|
|
329
|
-
raise "unexpected request"
|
|
330
|
-
end
|
|
331
|
-
request = line.partition('?').last.partition(' ').first
|
|
332
|
-
data=URI.decode_www_form(request)
|
|
333
|
-
request_params=data.to_h
|
|
334
|
-
Log.log.debug "request_params=#{request_params}"
|
|
335
|
-
websession.print "HTTP/1.1 200/OK\r\nContent-type:text/html\r\n\r\n#{html_page}"
|
|
336
|
-
websession.close
|
|
337
|
-
}
|
|
338
|
-
return request_params
|
|
339
|
-
end
|
|
340
|
-
|
|
341
337
|
end # OAuth
|
|
342
338
|
end # Aspera
|
|
@@ -6,7 +6,7 @@ module Aspera
|
|
|
6
6
|
class PersistencyActionOnce
|
|
7
7
|
# @param :manager Mandatory Database
|
|
8
8
|
# @param :data Mandatory object to persist, must be same object from begin to end (assume array by default)
|
|
9
|
-
# @param :
|
|
9
|
+
# @param :id Mandatory identifiers
|
|
10
10
|
# @param :delete Optional delete persistency condition
|
|
11
11
|
# @param :parse Optional parse method (default to JSON)
|
|
12
12
|
# @param :format Optional dump method (default to JSON)
|
|
@@ -16,27 +16,31 @@ module Aspera
|
|
|
16
16
|
raise "options shall be Hash" unless options.is_a?(Hash)
|
|
17
17
|
raise "mandatory :manager" if options[:manager].nil?
|
|
18
18
|
raise "mandatory :data" if options[:data].nil?
|
|
19
|
-
raise "mandatory :
|
|
20
|
-
raise "mandatory 1 element in :
|
|
19
|
+
raise "mandatory :id (String)" unless options[:id].is_a?(String)
|
|
20
|
+
raise "mandatory 1 element in :id" unless options[:id].length >= 1
|
|
21
21
|
@manager=options[:manager]
|
|
22
22
|
@persisted_object=options[:data]
|
|
23
|
-
@
|
|
23
|
+
@object_id=options[:id]
|
|
24
24
|
# by default , at save time, file is deleted if data is nil
|
|
25
25
|
@delete_condition=options[:delete] || lambda{|d|d.empty?}
|
|
26
26
|
@persist_format=options[:format] || lambda {|h| JSON.generate(h)}
|
|
27
27
|
persist_parse=options[:parse] || lambda {|t| JSON.parse(t)}
|
|
28
28
|
persist_merge=options[:merge] || lambda {|current,file| current.concat(file).uniq rescue current}
|
|
29
|
-
value=@manager.get(@
|
|
29
|
+
value=@manager.get(@object_id)
|
|
30
30
|
persist_merge.call(@persisted_object,persist_parse.call(value)) unless value.nil?
|
|
31
31
|
end
|
|
32
32
|
|
|
33
33
|
def save
|
|
34
34
|
if @delete_condition.call(@persisted_object)
|
|
35
|
-
@manager.delete(@
|
|
35
|
+
@manager.delete(@object_id)
|
|
36
36
|
else
|
|
37
|
-
@manager.put(@
|
|
37
|
+
@manager.put(@object_id,@persist_format.call(@persisted_object))
|
|
38
38
|
end
|
|
39
39
|
end
|
|
40
40
|
|
|
41
|
+
def data
|
|
42
|
+
return @persisted_object
|
|
43
|
+
end
|
|
44
|
+
|
|
41
45
|
end
|
|
42
46
|
end
|
|
@@ -6,11 +6,8 @@ require 'aspera/log'
|
|
|
6
6
|
module Aspera
|
|
7
7
|
# Persist data on file system
|
|
8
8
|
class PersistencyFolder
|
|
9
|
-
WINDOWS_PROTECTED_CHAR=%r{[/:"<>\\\*\?]}
|
|
10
|
-
PROTECTED_CHAR_REPLACE='_'
|
|
11
|
-
ID_SEPARATOR='_'
|
|
12
9
|
FILE_SUFFIX='.txt'
|
|
13
|
-
private_constant :
|
|
10
|
+
private_constant :FILE_SUFFIX
|
|
14
11
|
def initialize(folder)
|
|
15
12
|
@cache={}
|
|
16
13
|
set_folder(folder)
|
|
@@ -23,7 +20,6 @@ module Aspera
|
|
|
23
20
|
|
|
24
21
|
# @return String or nil string on existing persist, else nil
|
|
25
22
|
def get(object_id)
|
|
26
|
-
object_id=marshalled_id(object_id)
|
|
27
23
|
Log.log.debug("persistency get: #{object_id}")
|
|
28
24
|
if @cache.has_key?(object_id)
|
|
29
25
|
Log.log.debug("got from memory cache")
|
|
@@ -39,18 +35,16 @@ module Aspera
|
|
|
39
35
|
end
|
|
40
36
|
|
|
41
37
|
def put(object_id,value)
|
|
42
|
-
raise "only String supported" unless value.is_a?(String)
|
|
43
|
-
object_id=marshalled_id(object_id)
|
|
38
|
+
raise "value: only String supported" unless value.is_a?(String)
|
|
44
39
|
persist_filepath=id_to_filepath(object_id)
|
|
45
|
-
Log.log.debug("saving: #{persist_filepath}")
|
|
40
|
+
Log.log.debug("persistency saving: #{persist_filepath}")
|
|
46
41
|
File.write(persist_filepath,value)
|
|
47
42
|
@cache[object_id]=value
|
|
48
43
|
end
|
|
49
44
|
|
|
50
45
|
def delete(object_id)
|
|
51
|
-
object_id=marshalled_id(object_id)
|
|
52
46
|
persist_filepath=id_to_filepath(object_id)
|
|
53
|
-
Log.log.debug("
|
|
47
|
+
Log.log.debug("persistency deleting: #{persist_filepath}")
|
|
54
48
|
File.delete(persist_filepath) if File.exist?(persist_filepath)
|
|
55
49
|
@cache.delete(object_id)
|
|
56
50
|
end
|
|
@@ -63,7 +57,7 @@ module Aspera
|
|
|
63
57
|
end
|
|
64
58
|
garbage_files.each do |filepath|
|
|
65
59
|
File.delete(filepath)
|
|
66
|
-
Log.log.debug("
|
|
60
|
+
Log.log.debug("persistency deleted expired: #{filepath}")
|
|
67
61
|
end
|
|
68
62
|
return garbage_files
|
|
69
63
|
end
|
|
@@ -72,25 +66,11 @@ module Aspera
|
|
|
72
66
|
|
|
73
67
|
# @param object_id String or Array
|
|
74
68
|
def id_to_filepath(object_id)
|
|
69
|
+
raise "object_id: only String supported" unless object_id.is_a?(String)
|
|
75
70
|
FileUtils.mkdir_p(@folder)
|
|
76
71
|
return File.join(@folder,"#{object_id}#{FILE_SUFFIX}")
|
|
77
72
|
#.gsub(/[^a-z]+/,FILE_FIELD_SEPARATOR)
|
|
78
73
|
end
|
|
79
74
|
|
|
80
|
-
def marshalled_id(object_id)
|
|
81
|
-
if object_id.is_a?(Array)
|
|
82
|
-
# special case, url in second position: TODO: check any position
|
|
83
|
-
if object_id[1].is_a?(String) and object_id[1] =~ URI::ABS_URI
|
|
84
|
-
object_id=object_id.clone
|
|
85
|
-
object_id[1]=URI.parse(object_id[1]).host
|
|
86
|
-
end
|
|
87
|
-
object_id=object_id.join(ID_SEPARATOR)
|
|
88
|
-
end
|
|
89
|
-
raise "id must be a String" unless object_id.is_a?(String)
|
|
90
|
-
return object_id.
|
|
91
|
-
gsub(WINDOWS_PROTECTED_CHAR,PROTECTED_CHAR_REPLACE). # remove windows forbidden chars
|
|
92
|
-
gsub('.',PROTECTED_CHAR_REPLACE). # keep dot for extension only (nicer)
|
|
93
|
-
downcase
|
|
94
|
-
end
|
|
95
75
|
end # PersistencyFolder
|
|
96
76
|
end # Aspera
|
data/lib/aspera/rest.rb
CHANGED
|
@@ -125,7 +125,7 @@ module Aspera
|
|
|
125
125
|
end
|
|
126
126
|
|
|
127
127
|
def oauth_token(options={})
|
|
128
|
-
raise "ERROR:
|
|
128
|
+
raise "ERROR: expecting Oauth, have #{@oauth.class}" unless @oauth.is_a?(Oauth)
|
|
129
129
|
return @oauth.get_authorization(options)
|
|
130
130
|
end
|
|
131
131
|
|
data/lib/aspera/sync.rb
CHANGED
|
@@ -1,48 +1,53 @@
|
|
|
1
|
+
require 'aspera/command_line_builder'
|
|
2
|
+
|
|
1
3
|
module Aspera
|
|
2
4
|
# builds command line arg for async
|
|
3
5
|
class Sync
|
|
4
|
-
private
|
|
5
6
|
INSTANCE_PARAMS=
|
|
6
7
|
{
|
|
7
|
-
'alt_logdir' => { :
|
|
8
|
-
'watchd' => { :
|
|
9
|
-
'apply_local_docroot' => { :
|
|
10
|
-
'quiet' => { :
|
|
8
|
+
'alt_logdir' => { :cltype => :opt_with_arg, :accepted_types=>:string},
|
|
9
|
+
'watchd' => { :cltype => :opt_with_arg, :accepted_types=>:string},
|
|
10
|
+
'apply_local_docroot' => { :cltype => :opt_without_arg},
|
|
11
|
+
'quiet' => { :cltype => :opt_without_arg},
|
|
11
12
|
}
|
|
12
13
|
SESSION_PARAMS=
|
|
13
14
|
{
|
|
14
|
-
'name' => { :
|
|
15
|
-
'local_dir' => { :
|
|
16
|
-
'remote_dir' => { :
|
|
17
|
-
'local_db_dir' => { :
|
|
18
|
-
'remote_db_dir' => { :
|
|
19
|
-
'host' => { :
|
|
20
|
-
'user' => { :
|
|
21
|
-
'private_key_path' => { :
|
|
22
|
-
'direction' => { :
|
|
23
|
-
'checksum' => { :
|
|
24
|
-
'tcp_port' => { :
|
|
25
|
-
'rate_policy' => { :
|
|
26
|
-
'target_rate' => { :
|
|
27
|
-
'cooloff' => { :
|
|
28
|
-
'pending_max' => { :
|
|
29
|
-
'scan_intensity' => { :
|
|
30
|
-
'cipher' => { :
|
|
31
|
-
'transfer_threads' => { :
|
|
32
|
-
'preserve_time' => { :
|
|
33
|
-
'preserve_access_time' => { :
|
|
34
|
-
'preserve_modification_time' => { :
|
|
35
|
-
'preserve_uid' => { :
|
|
36
|
-
'preserve_gid' => { :
|
|
37
|
-
'create_dir' => { :
|
|
38
|
-
'reset' => { :
|
|
15
|
+
'name' => { :cltype => :opt_with_arg, :accepted_types=>:string},
|
|
16
|
+
'local_dir' => { :cltype => :opt_with_arg, :accepted_types=>:string},
|
|
17
|
+
'remote_dir' => { :cltype => :opt_with_arg, :accepted_types=>:string},
|
|
18
|
+
'local_db_dir' => { :cltype => :opt_with_arg, :accepted_types=>:string},
|
|
19
|
+
'remote_db_dir' => { :cltype => :opt_with_arg, :accepted_types=>:string},
|
|
20
|
+
'host' => { :cltype => :opt_with_arg, :accepted_types=>:string},
|
|
21
|
+
'user' => { :cltype => :opt_with_arg, :accepted_types=>:string},
|
|
22
|
+
'private_key_path' => { :cltype => :opt_with_arg, :accepted_types=>:string},
|
|
23
|
+
'direction' => { :cltype => :opt_with_arg, :accepted_types=>:string},
|
|
24
|
+
'checksum' => { :cltype => :opt_with_arg, :accepted_types=>:string},
|
|
25
|
+
'tcp_port' => { :cltype => :opt_with_arg, :accepted_types=>:int},
|
|
26
|
+
'rate_policy' => { :cltype => :opt_with_arg, :accepted_types=>:string},
|
|
27
|
+
'target_rate' => { :cltype => :opt_with_arg, :accepted_types=>:string},
|
|
28
|
+
'cooloff' => { :cltype => :opt_with_arg, :accepted_types=>:int},
|
|
29
|
+
'pending_max' => { :cltype => :opt_with_arg, :accepted_types=>:int},
|
|
30
|
+
'scan_intensity' => { :cltype => :opt_with_arg, :accepted_types=>:string},
|
|
31
|
+
'cipher' => { :cltype => :opt_with_arg, :accepted_types=>:string},
|
|
32
|
+
'transfer_threads' => { :cltype => :opt_with_arg, :accepted_types=>:int},
|
|
33
|
+
'preserve_time' => { :cltype => :opt_without_arg},
|
|
34
|
+
'preserve_access_time' => { :cltype => :opt_without_arg},
|
|
35
|
+
'preserve_modification_time' => { :cltype => :opt_without_arg},
|
|
36
|
+
'preserve_uid' => { :cltype => :opt_without_arg},
|
|
37
|
+
'preserve_gid' => { :cltype => :opt_without_arg},
|
|
38
|
+
'create_dir' => { :cltype => :opt_without_arg},
|
|
39
|
+
'reset' => { :cltype => :opt_without_arg},
|
|
39
40
|
# note: only one env var, but multiple sessions... may be a problem
|
|
40
|
-
'remote_password' => { :
|
|
41
|
-
'cookie' => { :
|
|
42
|
-
'token' => { :
|
|
43
|
-
'license' => { :
|
|
41
|
+
'remote_password' => { :cltype => :envvar, :clvarname=>'ASPERA_SCP_PASS'},
|
|
42
|
+
'cookie' => { :cltype => :envvar, :clvarname=>'ASPERA_SCP_COOKIE'},
|
|
43
|
+
'token' => { :cltype => :envvar, :clvarname=>'ASPERA_SCP_TOKEN'},
|
|
44
|
+
'license' => { :cltype => :envvar, :clvarname=>'ASPERA_SCP_LICENSE'},
|
|
44
45
|
}
|
|
45
|
-
|
|
46
|
+
|
|
47
|
+
Aspera::CommandLineBuilder.normalize_description(INSTANCE_PARAMS)
|
|
48
|
+
Aspera::CommandLineBuilder.normalize_description(SESSION_PARAMS)
|
|
49
|
+
|
|
50
|
+
private_constant :INSTANCE_PARAMS,:SESSION_PARAMS
|
|
46
51
|
|
|
47
52
|
def initialize(sync_params)
|
|
48
53
|
@sync_params=sync_params
|