aspera-cli 4.10.0 → 4.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/BUGS.md +19 -0
- data/CHANGELOG.md +528 -0
- data/CONTRIBUTING.md +143 -0
- data/README.md +977 -589
- data/bin/ascli +4 -4
- data/bin/asession +12 -12
- data/docs/test_env.conf +29 -19
- data/examples/aoc.rb +6 -6
- data/examples/dascli +18 -16
- data/examples/faspex4.rb +15 -15
- data/examples/node.rb +12 -12
- data/examples/proxy.pac +2 -2
- data/examples/server.rb +12 -12
- data/lib/aspera/aoc.rb +344 -272
- data/lib/aspera/ascmd.rb +56 -54
- data/lib/aspera/ats_api.rb +4 -4
- data/lib/aspera/cli/basic_auth_plugin.rb +15 -12
- data/lib/aspera/cli/extended_value.rb +9 -9
- data/lib/aspera/cli/{formater.rb → formatter.rb} +69 -69
- data/lib/aspera/cli/listener/line_dump.rb +1 -1
- data/lib/aspera/cli/listener/logger.rb +1 -1
- data/lib/aspera/cli/listener/progress.rb +5 -6
- data/lib/aspera/cli/listener/progress_multi.rb +16 -21
- data/lib/aspera/cli/main.rb +72 -73
- data/lib/aspera/cli/manager.rb +112 -112
- data/lib/aspera/cli/plugin.rb +68 -48
- data/lib/aspera/cli/plugins/alee.rb +4 -4
- data/lib/aspera/cli/plugins/aoc.rb +322 -720
- data/lib/aspera/cli/plugins/ats.rb +50 -52
- data/lib/aspera/cli/plugins/bss.rb +10 -10
- data/lib/aspera/cli/plugins/config.rb +514 -410
- data/lib/aspera/cli/plugins/console.rb +12 -12
- data/lib/aspera/cli/plugins/cos.rb +18 -20
- data/lib/aspera/cli/plugins/faspex.rb +134 -136
- data/lib/aspera/cli/plugins/faspex5.rb +235 -70
- data/lib/aspera/cli/plugins/node.rb +378 -309
- data/lib/aspera/cli/plugins/orchestrator.rb +52 -49
- data/lib/aspera/cli/plugins/preview.rb +129 -120
- data/lib/aspera/cli/plugins/server.rb +137 -83
- data/lib/aspera/cli/plugins/shares.rb +77 -52
- data/lib/aspera/cli/plugins/sync.rb +13 -33
- data/lib/aspera/cli/transfer_agent.rb +61 -61
- data/lib/aspera/cli/version.rb +2 -1
- data/lib/aspera/colors.rb +3 -3
- data/lib/aspera/command_line_builder.rb +78 -74
- data/lib/aspera/cos_node.rb +31 -29
- data/lib/aspera/data_repository.rb +1 -1
- data/lib/aspera/environment.rb +30 -28
- data/lib/aspera/fasp/agent_base.rb +17 -15
- data/lib/aspera/fasp/agent_connect.rb +34 -32
- data/lib/aspera/fasp/agent_direct.rb +70 -73
- data/lib/aspera/fasp/agent_httpgw.rb +79 -74
- data/lib/aspera/fasp/agent_node.rb +26 -26
- data/lib/aspera/fasp/agent_trsdk.rb +20 -20
- data/lib/aspera/fasp/error.rb +3 -2
- data/lib/aspera/fasp/error_info.rb +11 -8
- data/lib/aspera/fasp/installation.rb +80 -80
- data/lib/aspera/fasp/listener.rb +2 -2
- data/lib/aspera/fasp/parameters.rb +103 -92
- data/lib/aspera/fasp/parameters.yaml +313 -214
- data/lib/aspera/fasp/resume_policy.rb +10 -10
- data/lib/aspera/fasp/transfer_spec.rb +22 -2
- data/lib/aspera/fasp/uri.rb +7 -7
- data/lib/aspera/faspex_gw.rb +80 -159
- data/lib/aspera/faspex_postproc.rb +77 -0
- data/lib/aspera/hash_ext.rb +3 -3
- data/lib/aspera/id_generator.rb +5 -5
- data/lib/aspera/keychain/encrypted_hash.rb +23 -28
- data/lib/aspera/keychain/macos_security.rb +21 -20
- data/lib/aspera/log.rb +13 -13
- data/lib/aspera/nagios.rb +24 -23
- data/lib/aspera/node.rb +217 -38
- data/lib/aspera/oauth.rb +78 -74
- data/lib/aspera/open_application.rb +19 -11
- data/lib/aspera/persistency_action_once.rb +4 -4
- data/lib/aspera/persistency_folder.rb +13 -13
- data/lib/aspera/preview/file_types.rb +8 -8
- data/lib/aspera/preview/generator.rb +67 -67
- data/lib/aspera/preview/utils.rb +27 -27
- data/lib/aspera/proxy_auto_config.js +63 -63
- data/lib/aspera/proxy_auto_config.rb +19 -19
- data/lib/aspera/rest.rb +65 -67
- data/lib/aspera/rest_call_error.rb +2 -1
- data/lib/aspera/rest_error_analyzer.rb +22 -21
- data/lib/aspera/rest_errors_aspera.rb +16 -16
- data/lib/aspera/secret_hider.rb +17 -14
- data/lib/aspera/ssh.rb +15 -14
- data/lib/aspera/sync.rb +177 -62
- data/lib/aspera/temp_file_manager.rb +2 -2
- data/lib/aspera/uri_reader.rb +4 -4
- data/lib/aspera/web_auth.rb +13 -64
- data/lib/aspera/web_server_simple.rb +76 -0
- data.tar.gz.sig +0 -0
- metadata +11 -6
- metadata.gz.sig +0 -0
data/lib/aspera/oauth.rb
CHANGED
@@ -24,8 +24,8 @@ module Aspera
|
|
24
24
|
# OAuth methods supported by default
|
25
25
|
STD_AUTH_TYPES = %i[web jwt].freeze
|
26
26
|
|
27
|
-
# remove 5 minutes to account for time offset (TODO: configurable?)
|
28
|
-
|
27
|
+
# remove 5 minutes to account for time offset between client and server (TODO: configurable?)
|
28
|
+
JWT_ACCEPTED_OFFSET_SEC = 300
|
29
29
|
# one hour validity (TODO: configurable?)
|
30
30
|
JWT_EXPIRY_OFFSET_SEC = 3600
|
31
31
|
# tokens older than 30 minutes will be discarded from cache
|
@@ -35,7 +35,7 @@ module Aspera
|
|
35
35
|
# a prefix for persistency of tokens (simplify garbage collect)
|
36
36
|
PERSIST_CATEGORY_TOKEN = 'token'
|
37
37
|
|
38
|
-
private_constant :
|
38
|
+
private_constant :JWT_ACCEPTED_OFFSET_SEC, :JWT_EXPIRY_OFFSET_SEC, :TOKEN_CACHE_EXPIRY_SEC, :PERSIST_CATEGORY_TOKEN, :TOKEN_EXPIRATION_GUARD_SEC
|
39
39
|
|
40
40
|
# persistency manager
|
41
41
|
@persist = nil
|
@@ -48,7 +48,7 @@ module Aspera
|
|
48
48
|
def persist_mgr=(manager)
|
49
49
|
@persist = manager
|
50
50
|
# cleanup expired tokens
|
51
|
-
@persist.garbage_collect(PERSIST_CATEGORY_TOKEN,TOKEN_CACHE_EXPIRY_SEC)
|
51
|
+
@persist.garbage_collect(PERSIST_CATEGORY_TOKEN, TOKEN_CACHE_EXPIRY_SEC)
|
52
52
|
end
|
53
53
|
|
54
54
|
def persist_mgr
|
@@ -56,7 +56,7 @@ module Aspera
|
|
56
56
|
Log.log.debug('Not using persistency') # (use Aspera::Oauth.persist_mgr=Aspera::PersistencyFolder.new)
|
57
57
|
# create NULL persistency class
|
58
58
|
@persist = Class.new do
|
59
|
-
def get(_x);nil;end;def delete(_x);nil;end;def put(_x,_y);nil;end;def garbage_collect(_x,_y);nil;end # rubocop:disable Layout/EmptyLineBetweenDefs
|
59
|
+
def get(_x); nil; end; def delete(_x); nil; end; def put(_x, _y); nil; end; def garbage_collect(_x, _y); nil; end # rubocop:disable Layout/EmptyLineBetweenDefs, Style/Semicolon, Layout/LineLength
|
60
60
|
end.new
|
61
61
|
end
|
62
62
|
return @persist
|
@@ -64,7 +64,7 @@ module Aspera
|
|
64
64
|
|
65
65
|
# delete all existing tokens
|
66
66
|
def flush_tokens
|
67
|
-
persist_mgr.garbage_collect(PERSIST_CATEGORY_TOKEN,nil)
|
67
|
+
persist_mgr.garbage_collect(PERSIST_CATEGORY_TOKEN, nil)
|
68
68
|
end
|
69
69
|
|
70
70
|
# register a bearer token decoder, mainly to inspect expiry date
|
@@ -83,7 +83,7 @@ module Aspera
|
|
83
83
|
end
|
84
84
|
|
85
85
|
# register a token creation method
|
86
|
-
# @param id creation type from field :
|
86
|
+
# @param id creation type from field :grant_method in constructor
|
87
87
|
# @param lambda_create called to create token
|
88
88
|
# @param id_create called to generate unique id for token, for cache
|
89
89
|
def register_token_creator(id, lambda_create, id_create)
|
@@ -94,86 +94,87 @@ module Aspera
|
|
94
94
|
|
95
95
|
# @return one of the registered creators for the given create type
|
96
96
|
def token_creator(id)
|
97
|
-
raise "token
|
97
|
+
raise "token grant method unknown: #{id}/#{id.class}" unless @create_handlers.key?(id)
|
98
98
|
@create_handlers[id]
|
99
99
|
end
|
100
100
|
|
101
|
-
# list of identifiers
|
101
|
+
# list of identifiers found in creation parameters that can be used to uniquely identify the token
|
102
102
|
def id_creator(id)
|
103
|
-
raise "id creator type unknown: #{id}/#{id.class}" unless @id_handlers.
|
103
|
+
raise "id creator type unknown: #{id}/#{id.class}" unless @id_handlers.key?(id)
|
104
104
|
@id_handlers[id]
|
105
105
|
end
|
106
106
|
end # self
|
107
107
|
|
108
108
|
# JSON Web Signature (JWS) compact serialization: https://datatracker.ietf.org/doc/html/rfc7515
|
109
|
-
register_decoder lambda { |token| parts = token.split('.'); raise 'not aoc token' unless parts.length.eql?(3); JSON.parse(Base64.decode64(parts[1]))}
|
109
|
+
register_decoder lambda { |token| parts = token.split('.'); raise 'not aoc token' unless parts.length.eql?(3); JSON.parse(Base64.decode64(parts[1]))} # rubocop:disable Style/Semicolon, Layout/LineLength
|
110
110
|
|
111
111
|
# generic token creation, parameters are provided in :generic
|
112
|
-
register_token_creator :generic,lambda { |oauth|
|
113
|
-
return oauth.create_token(oauth.
|
114
|
-
},lambda { |oauth|
|
112
|
+
register_token_creator :generic, lambda { |oauth|
|
113
|
+
return oauth.create_token(oauth.specific_parameters)
|
114
|
+
}, lambda { |oauth|
|
115
115
|
return [
|
116
|
-
oauth.
|
117
|
-
oauth.
|
118
|
-
oauth.
|
116
|
+
oauth.specific_parameters[:grant_type]&.split(':')&.last,
|
117
|
+
oauth.specific_parameters[:apikey],
|
118
|
+
oauth.specific_parameters[:response_type]
|
119
119
|
]
|
120
120
|
}
|
121
121
|
|
122
122
|
# Authentication using Web browser
|
123
|
-
register_token_creator :web,lambda { |oauth|
|
123
|
+
register_token_creator :web, lambda { |oauth|
|
124
124
|
random_state = SecureRandom.uuid # used to check later
|
125
|
-
login_page_url = Rest.build_uri(
|
126
|
-
oauth.
|
125
|
+
login_page_url = Rest.build_uri(
|
126
|
+
"#{oauth.api.params[:base_url]}/#{oauth.specific_parameters[:path_authorize]}",
|
127
|
+
oauth.optional_scope_client_id.merge(response_type: 'code', redirect_uri: oauth.specific_parameters[:redirect_uri], state: random_state))
|
127
128
|
# here, we need a human to authorize on a web page
|
128
|
-
Log.log.info
|
129
|
+
Log.log.info{"login_page_url=#{login_page_url}".bg_red.gray}
|
129
130
|
# start a web server to receive request code
|
130
|
-
|
131
|
+
web_server = WebAuth.new(oauth.specific_parameters[:redirect_uri])
|
131
132
|
# start browser on login page
|
132
133
|
OpenApplication.instance.uri(login_page_url)
|
133
134
|
# wait for code in request
|
134
|
-
received_params =
|
135
|
+
received_params = web_server.received_request
|
135
136
|
raise 'wrong received state' unless random_state.eql?(received_params['state'])
|
136
137
|
# exchange code for token
|
137
138
|
return oauth.create_token(oauth.optional_scope_client_id(add_secret: true).merge(
|
138
139
|
grant_type: 'authorization_code',
|
139
140
|
code: received_params['code'],
|
140
|
-
redirect_uri: oauth.
|
141
|
-
},lambda { |_oauth|
|
141
|
+
redirect_uri: oauth.specific_parameters[:redirect_uri]))
|
142
|
+
}, lambda { |_oauth|
|
142
143
|
return []
|
143
144
|
}
|
144
145
|
|
145
146
|
# Authentication using private key
|
146
|
-
register_token_creator :jwt,lambda { |oauth|
|
147
|
+
register_token_creator :jwt, lambda { |oauth|
|
147
148
|
# https://tools.ietf.org/html/rfc7523
|
148
149
|
# https://tools.ietf.org/html/rfc7519
|
149
150
|
require 'jwt'
|
150
151
|
seconds_since_epoch = Time.new.to_i
|
151
|
-
Log.log.info
|
152
|
-
raise 'missing JWT payload' unless oauth.
|
152
|
+
Log.log.info{"seconds=#{seconds_since_epoch}"}
|
153
|
+
raise 'missing JWT payload' unless oauth.specific_parameters[:payload].is_a?(Hash)
|
153
154
|
jwt_payload = {
|
154
155
|
exp: seconds_since_epoch + JWT_EXPIRY_OFFSET_SEC, # expiration time
|
155
|
-
nbf: seconds_since_epoch -
|
156
|
-
iat: seconds_since_epoch, # issued at
|
156
|
+
nbf: seconds_since_epoch - JWT_ACCEPTED_OFFSET_SEC, # not before
|
157
|
+
iat: seconds_since_epoch - JWT_ACCEPTED_OFFSET_SEC + 1, # issued at (we tell a little in the past so that server always accepts)
|
157
158
|
jti: SecureRandom.uuid # JWT id
|
158
|
-
}.merge(oauth.
|
159
|
-
Log.log.debug
|
160
|
-
rsa_private = oauth.
|
161
|
-
Log.log.debug
|
162
|
-
assertion = JWT.encode(jwt_payload, rsa_private, 'RS256', oauth.
|
163
|
-
Log.log.debug
|
159
|
+
}.merge(oauth.specific_parameters[:payload])
|
160
|
+
Log.log.debug{"JWT jwt_payload=[#{jwt_payload}]"}
|
161
|
+
rsa_private = oauth.specific_parameters[:private_key_obj] # type: OpenSSL::PKey::RSA
|
162
|
+
Log.log.debug{"private=[#{rsa_private}]"}
|
163
|
+
assertion = JWT.encode(jwt_payload, rsa_private, 'RS256', oauth.specific_parameters[:headers] || {})
|
164
|
+
Log.log.debug{"assertion=[#{assertion}]"}
|
164
165
|
return oauth.create_token(oauth.optional_scope_client_id.merge(grant_type: 'urn:ietf:params:oauth:grant-type:jwt-bearer', assertion: assertion))
|
165
|
-
},lambda { |oauth|
|
166
|
-
return [oauth.
|
166
|
+
}, lambda { |oauth|
|
167
|
+
return [oauth.specific_parameters.dig(:payload, :sub)]
|
167
168
|
}
|
168
169
|
|
169
|
-
attr_reader :
|
170
|
+
attr_reader :generic_parameters, :specific_parameters, :api
|
170
171
|
|
171
172
|
private
|
172
173
|
|
173
174
|
# [M]=mandatory [D]=has default value [0]=accept nil
|
174
175
|
# :base_url [M] URL of authentication API
|
175
176
|
# :auth
|
176
|
-
# :
|
177
|
+
# :grant_method [M] :generic, :web, :jwt, custom
|
177
178
|
# :client_id [0]
|
178
179
|
# :client_secret [0]
|
179
180
|
# :scope [0]
|
@@ -186,50 +187,53 @@ module Aspera
|
|
186
187
|
# :web:path_authorize [D] for type :web
|
187
188
|
# :generic [M] for type :generic
|
188
189
|
def initialize(a_params)
|
189
|
-
Log.log.debug
|
190
|
+
Log.log.debug{"auth=#{a_params}"}
|
190
191
|
# replace default values
|
191
|
-
@
|
192
|
+
@generic_parameters = DEFAULT_CREATE_PARAMS.deep_merge(a_params)
|
193
|
+
# legacy
|
194
|
+
@generic_parameters[:grant_method] ||= @generic_parameters.delete(:crtype) if @generic_parameters.key?(:crtype)
|
192
195
|
# check that type is known
|
193
|
-
self.class.token_creator(@
|
196
|
+
self.class.token_creator(@generic_parameters[:grant_method])
|
194
197
|
# specific parameters for the creation type
|
195
|
-
@
|
196
|
-
if @
|
197
|
-
uri = URI.parse(@
|
198
|
+
@specific_parameters = @generic_parameters[@generic_parameters[:grant_method]]
|
199
|
+
if @generic_parameters[:grant_method].eql?(:web) && @specific_parameters.key?(:redirect_uri)
|
200
|
+
uri = URI.parse(@specific_parameters[:redirect_uri])
|
198
201
|
raise 'redirect_uri scheme must be http or https' unless %w[http https].include?(uri.scheme)
|
199
202
|
raise 'redirect_uri must have a port' if uri.port.nil?
|
200
203
|
# TODO: we could check that host is localhost or local address
|
201
204
|
end
|
202
205
|
rest_params = {
|
203
|
-
base_url: @
|
206
|
+
base_url: @generic_parameters[:base_url],
|
204
207
|
redirect_max: 2
|
205
208
|
}
|
206
|
-
rest_params[:auth] = a_params[:auth] if a_params.
|
209
|
+
rest_params[:auth] = a_params[:auth] if a_params.key?(:auth)
|
207
210
|
@api = Rest.new(rest_params)
|
208
211
|
# if needed use from api
|
209
|
-
@
|
210
|
-
@
|
211
|
-
@
|
212
|
-
Log.dump(:
|
213
|
-
Log.dump(:
|
212
|
+
@generic_parameters.delete(:base_url)
|
213
|
+
@generic_parameters.delete(:auth)
|
214
|
+
@generic_parameters.delete(@generic_parameters[:grant_method])
|
215
|
+
Log.dump(:generic_parameters, @generic_parameters)
|
216
|
+
Log.dump(:specific_parameters, @specific_parameters)
|
214
217
|
end
|
215
218
|
|
216
219
|
public
|
217
220
|
|
218
221
|
# helper method to create token as per RFC
|
219
222
|
def create_token(www_params)
|
223
|
+
Log.log.debug{'Generating a new token'.bg_green}
|
220
224
|
return @api.call({
|
221
225
|
operation: 'POST',
|
222
|
-
subpath: @
|
226
|
+
subpath: @generic_parameters[:path_token],
|
223
227
|
headers: {'Accept' => 'application/json'},
|
224
228
|
www_body_params: www_params})
|
225
229
|
end
|
226
230
|
|
227
231
|
# @return Hash with optional general parameters
|
228
232
|
def optional_scope_client_id(add_secret: false)
|
229
|
-
call_params={}
|
230
|
-
call_params[:scope] = @
|
231
|
-
call_params[:client_id] = @
|
232
|
-
call_params[:client_secret] = @
|
233
|
+
call_params = {}
|
234
|
+
call_params[:scope] = @generic_parameters[:scope] unless @generic_parameters[:scope].nil?
|
235
|
+
call_params[:client_id] = @generic_parameters[:client_id] unless @generic_parameters[:client_id].nil?
|
236
|
+
call_params[:client_secret] = @generic_parameters[:client_secret] if add_secret && !@generic_parameters[:client_id].nil?
|
233
237
|
return call_params
|
234
238
|
end
|
235
239
|
|
@@ -240,21 +244,21 @@ module Aspera
|
|
240
244
|
token_id = IdGenerator.from_list([
|
241
245
|
PERSIST_CATEGORY_TOKEN,
|
242
246
|
@api.params[:base_url],
|
243
|
-
@
|
244
|
-
self.class.id_creator(@
|
245
|
-
@
|
247
|
+
@generic_parameters[:grant_method],
|
248
|
+
self.class.id_creator(@generic_parameters[:grant_method]).call(self), # array, so we flatten later
|
249
|
+
@generic_parameters[:scope],
|
246
250
|
@api.params.dig(%i[auth username])
|
247
251
|
].flatten)
|
248
252
|
|
249
253
|
# get token_data from cache (or nil), token_data is what is returned by /token
|
250
254
|
token_data = self.class.persist_mgr.get(token_id) if use_cache
|
251
255
|
token_data = JSON.parse(token_data) unless token_data.nil?
|
252
|
-
# Optional optimization: check if node token is expired
|
256
|
+
# Optional optimization: check if node token is expired based on decoded content then force refresh if close enough
|
253
257
|
# might help in case the transfer agent cannot refresh himself
|
254
258
|
# `direct` agent is equipped with refresh code
|
255
259
|
if !use_refresh_token && !token_data.nil?
|
256
|
-
decoded_token = self.class.decode_token(token_data[@
|
257
|
-
Log.dump('decoded_token',decoded_token) unless decoded_token.nil?
|
260
|
+
decoded_token = self.class.decode_token(token_data[@generic_parameters[:token_field]])
|
261
|
+
Log.dump('decoded_token', decoded_token) unless decoded_token.nil?
|
258
262
|
if decoded_token.is_a?(Hash)
|
259
263
|
expires_at_sec =
|
260
264
|
if decoded_token['expires_at'].is_a?(String) then DateTime.parse(decoded_token['expires_at']).to_time
|
@@ -262,13 +266,13 @@ module Aspera
|
|
262
266
|
end
|
263
267
|
# force refresh if we see a token too close from expiration
|
264
268
|
use_refresh_token = true if expires_at_sec.is_a?(Time) && (expires_at_sec - Time.now) < TOKEN_EXPIRATION_GUARD_SEC
|
265
|
-
Log.log.debug
|
269
|
+
Log.log.debug{"Expiration: #{expires_at_sec} / #{use_refresh_token}"}
|
266
270
|
end
|
267
271
|
end
|
268
272
|
|
269
273
|
# an API was already called, but failed, we need to regenerate or refresh
|
270
274
|
if use_refresh_token
|
271
|
-
if token_data.is_a?(Hash) && token_data.
|
275
|
+
if token_data.is_a?(Hash) && token_data.key?('refresh_token')
|
272
276
|
# save possible refresh token, before deleting the cache
|
273
277
|
refresh_token = token_data['refresh_token']
|
274
278
|
end
|
@@ -277,31 +281,31 @@ module Aspera
|
|
277
281
|
token_data = nil
|
278
282
|
# lets try the existing refresh token
|
279
283
|
if !refresh_token.nil?
|
280
|
-
Log.log.info
|
284
|
+
Log.log.info{"refresh=[#{refresh_token}]".bg_green}
|
281
285
|
# try to refresh
|
282
286
|
# note: AoC admin token has no refresh, and lives by default 1800secs
|
283
|
-
resp = create_token(optional_scope_client_id.merge(grant_type: 'refresh_token',refresh_token: refresh_token))
|
287
|
+
resp = create_token(optional_scope_client_id.merge(grant_type: 'refresh_token', refresh_token: refresh_token))
|
284
288
|
if resp[:http].code.start_with?('2')
|
285
289
|
# save only if success
|
286
290
|
json_data = resp[:http].body
|
287
291
|
token_data = JSON.parse(json_data)
|
288
|
-
self.class.persist_mgr.put(token_id,json_data)
|
292
|
+
self.class.persist_mgr.put(token_id, json_data)
|
289
293
|
else
|
290
|
-
Log.log.debug
|
294
|
+
Log.log.debug{"refresh failed: #{resp[:http].body}".bg_red}
|
291
295
|
end
|
292
296
|
end
|
293
297
|
end
|
294
298
|
|
295
299
|
# no cache, nor refresh: generate a token
|
296
300
|
if token_data.nil?
|
297
|
-
resp = self.class.token_creator(@
|
301
|
+
resp = self.class.token_creator(@generic_parameters[:grant_method]).call(self)
|
298
302
|
json_data = resp[:http].body
|
299
303
|
token_data = JSON.parse(json_data)
|
300
|
-
self.class.persist_mgr.put(token_id,json_data)
|
304
|
+
self.class.persist_mgr.put(token_id, json_data)
|
301
305
|
end # if ! in_cache
|
302
|
-
raise "API error: No such field in answer: #{@
|
306
|
+
raise "API error: No such field in answer: #{@generic_parameters[:token_field]}" unless token_data.key?(@generic_parameters[:token_field])
|
303
307
|
# ok we shall have a token here
|
304
|
-
return 'Bearer ' + token_data[@
|
308
|
+
return 'Bearer ' + token_data[@generic_parameters[:token_field]]
|
305
309
|
end
|
306
310
|
end # OAuth
|
307
311
|
end # Aspera
|
@@ -12,30 +12,38 @@ module Aspera
|
|
12
12
|
class OpenApplication
|
13
13
|
include Singleton
|
14
14
|
class << self
|
15
|
+
USER_INTERFACES = %i[text graphical].freeze
|
15
16
|
# User Interfaces
|
16
|
-
def user_interfaces;
|
17
|
+
def user_interfaces; USER_INTERFACES; end
|
17
18
|
|
18
19
|
def default_gui_mode
|
19
|
-
return :graphical if [Aspera::Environment::OS_WINDOWS,Aspera::Environment::OS_X].include?(Aspera::Environment.os)
|
20
|
+
return :graphical if [Aspera::Environment::OS_WINDOWS, Aspera::Environment::OS_X].include?(Aspera::Environment.os)
|
20
21
|
# unix family
|
21
|
-
return :graphical if ENV.
|
22
|
+
return :graphical if ENV.key?('DISPLAY') && !ENV['DISPLAY'].empty?
|
22
23
|
return :text
|
23
24
|
end
|
24
25
|
|
25
26
|
# command must be non blocking
|
26
27
|
def uri_graphical(uri)
|
27
28
|
case Aspera::Environment.os
|
28
|
-
when Aspera::Environment::OS_X
|
29
|
-
|
30
|
-
when Aspera::Environment::
|
31
|
-
return system('start explorer "' + uri.to_s + '"')
|
32
|
-
when Aspera::Environment::OS_LINUX
|
33
|
-
return system("xdg-open '#{uri}'")
|
29
|
+
when Aspera::Environment::OS_X then return system('open', uri.to_s)
|
30
|
+
when Aspera::Environment::OS_WINDOWS then return system('start', 'explorer', '"' + uri.to_s + '"')
|
31
|
+
when Aspera::Environment::OS_LINUX then return system('xdg-open', uri.to_s)
|
34
32
|
else
|
35
33
|
raise "no graphical open method for #{Aspera::Environment.os}"
|
36
34
|
end
|
37
35
|
end
|
38
|
-
|
36
|
+
|
37
|
+
def editor(file_path)
|
38
|
+
if ENV.key?('EDITOR')
|
39
|
+
system(ENV['EDITOR'], file_path.to_s)
|
40
|
+
elsif Aspera::Environment.os.eql?(Aspera::Environment::OS_WINDOWS)
|
41
|
+
system('notepad.exe', '"' + file_path.to_s + '"')
|
42
|
+
else
|
43
|
+
uri_graphical(file_path.to_s)
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end # self
|
39
47
|
|
40
48
|
attr_accessor :url_method
|
41
49
|
|
@@ -56,7 +64,7 @@ module Aspera
|
|
56
64
|
puts "USER ACTION: open this:\n" + the_url.to_s.red + "\n"
|
57
65
|
end
|
58
66
|
else
|
59
|
-
raise StandardError,"unsupported url open method: #{@url_method}"
|
67
|
+
raise StandardError, "unsupported url open method: #{@url_method}"
|
60
68
|
end
|
61
69
|
end
|
62
70
|
end # OpenApplication
|
@@ -14,7 +14,7 @@ module Aspera
|
|
14
14
|
# @param :format Optional dump method (default to JSON)
|
15
15
|
# @param :merge Optional merge data from file to current data
|
16
16
|
def initialize(options)
|
17
|
-
Log.log.debug
|
17
|
+
Log.log.debug{"persistency: #{options}"}
|
18
18
|
raise 'options shall be Hash' unless options.is_a?(Hash)
|
19
19
|
raise 'mandatory :manager' if options[:manager].nil?
|
20
20
|
raise 'mandatory :data' if options[:data].nil?
|
@@ -27,16 +27,16 @@ module Aspera
|
|
27
27
|
@delete_condition = options[:delete] || lambda{|d|d.empty?}
|
28
28
|
@persist_format = options[:format] || lambda {|h| JSON.generate(h)}
|
29
29
|
persist_parse = options[:parse] || lambda {|t| JSON.parse(t)}
|
30
|
-
persist_merge = options[:merge] || lambda {|current,file| current.concat(file).uniq rescue current}
|
30
|
+
persist_merge = options[:merge] || lambda {|current, file| current.concat(file).uniq rescue current}
|
31
31
|
value = @manager.get(@object_id)
|
32
|
-
persist_merge.call(@persisted_object,persist_parse.call(value)) unless value.nil?
|
32
|
+
persist_merge.call(@persisted_object, persist_parse.call(value)) unless value.nil?
|
33
33
|
end
|
34
34
|
|
35
35
|
def save
|
36
36
|
if @delete_condition.call(@persisted_object)
|
37
37
|
@manager.delete(@object_id)
|
38
38
|
else
|
39
|
-
@manager.put(@object_id
|
39
|
+
@manager.put(@object_id, @persist_format.call(@persisted_object))
|
40
40
|
end
|
41
41
|
end
|
42
42
|
|
@@ -14,17 +14,17 @@ module Aspera
|
|
14
14
|
def initialize(folder)
|
15
15
|
@cache = {}
|
16
16
|
@folder = folder
|
17
|
-
Log.log.debug
|
17
|
+
Log.log.debug{"persistency folder: #{@folder}"}
|
18
18
|
end
|
19
19
|
|
20
20
|
# @return String or nil string on existing persist, else nil
|
21
21
|
def get(object_id)
|
22
|
-
Log.log.debug
|
23
|
-
if @cache.
|
22
|
+
Log.log.debug{"persistency get: #{object_id}"}
|
23
|
+
if @cache.key?(object_id)
|
24
24
|
Log.log.debug('got from memory cache')
|
25
25
|
else
|
26
26
|
persist_filepath = id_to_filepath(object_id)
|
27
|
-
Log.log.debug
|
27
|
+
Log.log.debug{"persistency = #{persist_filepath}"}
|
28
28
|
if File.exist?(persist_filepath)
|
29
29
|
Log.log.debug('got from file cache')
|
30
30
|
@cache[object_id] = File.read(persist_filepath)
|
@@ -33,32 +33,32 @@ module Aspera
|
|
33
33
|
return @cache[object_id]
|
34
34
|
end
|
35
35
|
|
36
|
-
def put(object_id,value)
|
36
|
+
def put(object_id, value)
|
37
37
|
raise 'value: only String supported' unless value.is_a?(String)
|
38
38
|
persist_filepath = id_to_filepath(object_id)
|
39
|
-
Log.log.debug
|
39
|
+
Log.log.debug{"persistency saving: #{persist_filepath}"}
|
40
40
|
File.delete(persist_filepath) if File.exist?(persist_filepath)
|
41
|
-
File.write(persist_filepath,value)
|
41
|
+
File.write(persist_filepath, value)
|
42
42
|
Environment.restrict_file_access(persist_filepath)
|
43
43
|
@cache[object_id] = value
|
44
44
|
end
|
45
45
|
|
46
46
|
def delete(object_id)
|
47
47
|
persist_filepath = id_to_filepath(object_id)
|
48
|
-
Log.log.debug
|
48
|
+
Log.log.debug{"persistency deleting: #{persist_filepath}"}
|
49
49
|
File.delete(persist_filepath) if File.exist?(persist_filepath)
|
50
50
|
@cache.delete(object_id)
|
51
51
|
end
|
52
52
|
|
53
|
-
def garbage_collect(persist_category,max_age_seconds=nil)
|
54
|
-
garbage_files = Dir[File.join(@folder,persist_category + '*' + FILE_SUFFIX)]
|
53
|
+
def garbage_collect(persist_category, max_age_seconds=nil)
|
54
|
+
garbage_files = Dir[File.join(@folder, persist_category + '*' + FILE_SUFFIX)]
|
55
55
|
if !max_age_seconds.nil?
|
56
56
|
current_time = Time.now
|
57
57
|
garbage_files.select! { |filepath| (current_time - File.stat(filepath).mtime).to_i > max_age_seconds}
|
58
58
|
end
|
59
59
|
garbage_files.each do |filepath|
|
60
60
|
File.delete(filepath)
|
61
|
-
Log.log.debug
|
61
|
+
Log.log.debug{"persistency deleted expired: #{filepath}"}
|
62
62
|
end
|
63
63
|
return garbage_files
|
64
64
|
end
|
@@ -70,8 +70,8 @@ module Aspera
|
|
70
70
|
raise 'object_id: only String supported' unless object_id.is_a?(String)
|
71
71
|
FileUtils.mkdir_p(@folder)
|
72
72
|
Environment.restrict_file_access(@folder)
|
73
|
-
return File.join(@folder,"#{object_id}#{FILE_SUFFIX}")
|
74
|
-
|
73
|
+
return File.join(@folder, "#{object_id}#{FILE_SUFFIX}")
|
74
|
+
# .gsub(/[^a-z]+/,FILE_FIELD_SEPARATOR)
|
75
75
|
end
|
76
76
|
end # PersistencyFolder
|
77
77
|
end # Aspera
|
@@ -286,20 +286,20 @@ module Aspera
|
|
286
286
|
# check magic number inside file (empty string if not found)
|
287
287
|
detected_mime = MimeMagic.by_magic(File.open(filepath)).to_s
|
288
288
|
# check extension only
|
289
|
-
if !SUPPORTED_MIME_TYPES.
|
290
|
-
Log.log.debug
|
289
|
+
if !SUPPORTED_MIME_TYPES.key?(detected_mime)
|
290
|
+
Log.log.debug{"no conversion for #{detected_mime}, trying extension"}
|
291
291
|
detected_mime = MimeMagic.by_extension(File.extname(filepath)).to_s
|
292
292
|
end
|
293
293
|
detected_mime = nil if detected_mime.empty?
|
294
|
-
Log.log.debug
|
294
|
+
Log.log.debug{"mimemagic: #{detected_mime.class.name} [#{detected_mime}]"}
|
295
295
|
return detected_mime
|
296
296
|
end
|
297
297
|
|
298
298
|
# return file type, one of enum CONVERSION_TYPES
|
299
299
|
# @param filepath [String] full path to file
|
300
300
|
# @param mimetype [String] provided by node api
|
301
|
-
def conversion_type(filepath,mimetype)
|
302
|
-
Log.log.debug
|
301
|
+
def conversion_type(filepath, mimetype)
|
302
|
+
Log.log.debug{"conversion_type(#{filepath},m=#{mimetype},t=#{@use_mimemagic})"}
|
303
303
|
# 1- get type from provided mime type, using local mapping
|
304
304
|
conv_type = SUPPORTED_MIME_TYPES[mimetype] if !mimetype.nil?
|
305
305
|
# 2- else, from computed mime type (if available)
|
@@ -311,8 +311,8 @@ module Aspera
|
|
311
311
|
if mimetype.eql?(detected_mime)
|
312
312
|
Log.log.debug('matching mime type per magic number')
|
313
313
|
else
|
314
|
-
#
|
315
|
-
Log.log.debug
|
314
|
+
# NOTE: detected can be nil
|
315
|
+
Log.log.debug{"non matching mime types: node=[#{mimetype}], magic=[#{detected_mime}]"}
|
316
316
|
end
|
317
317
|
end
|
318
318
|
end
|
@@ -320,7 +320,7 @@ module Aspera
|
|
320
320
|
# 3- else, from extensions, using local mapping
|
321
321
|
extension = File.extname(filepath.downcase)[1..-1]
|
322
322
|
conv_type = SUPPORTED_EXTENSIONS[extension] if conv_type.nil?
|
323
|
-
Log.log.debug
|
323
|
+
Log.log.debug{"conversion_type(#{extension}): #{conv_type.class.name} [#{conv_type}]"}
|
324
324
|
return conv_type
|
325
325
|
end
|
326
326
|
end
|