aspera-cli 4.1.0 → 4.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +455 -229
  3. data/docs/Makefile +4 -4
  4. data/docs/README.erb.md +457 -126
  5. data/docs/test_env.conf +19 -2
  6. data/examples/aoc.rb +14 -3
  7. data/examples/faspex4.rb +89 -0
  8. data/lib/aspera/aoc.rb +38 -40
  9. data/lib/aspera/cli/main.rb +65 -33
  10. data/lib/aspera/cli/plugins/aoc.rb +54 -65
  11. data/lib/aspera/cli/plugins/ats.rb +2 -2
  12. data/lib/aspera/cli/plugins/config.rb +158 -137
  13. data/lib/aspera/cli/plugins/faspex.rb +111 -64
  14. data/lib/aspera/cli/plugins/faspex5.rb +35 -48
  15. data/lib/aspera/cli/plugins/node.rb +3 -2
  16. data/lib/aspera/cli/plugins/preview.rb +88 -55
  17. data/lib/aspera/cli/transfer_agent.rb +98 -62
  18. data/lib/aspera/cli/version.rb +1 -1
  19. data/lib/aspera/command_line_builder.rb +48 -31
  20. data/lib/aspera/cos_node.rb +34 -28
  21. data/lib/aspera/environment.rb +2 -2
  22. data/lib/aspera/fasp/aoc.rb +1 -1
  23. data/lib/aspera/fasp/installation.rb +68 -45
  24. data/lib/aspera/fasp/local.rb +89 -45
  25. data/lib/aspera/fasp/manager.rb +3 -0
  26. data/lib/aspera/fasp/node.rb +23 -1
  27. data/lib/aspera/fasp/parameters.rb +57 -86
  28. data/lib/aspera/fasp/parameters.yaml +531 -0
  29. data/lib/aspera/fasp/resume_policy.rb +13 -12
  30. data/lib/aspera/fasp/uri.rb +1 -1
  31. data/lib/aspera/id_generator.rb +22 -0
  32. data/lib/aspera/node.rb +14 -3
  33. data/lib/aspera/oauth.rb +135 -129
  34. data/lib/aspera/persistency_action_once.rb +11 -7
  35. data/lib/aspera/persistency_folder.rb +6 -26
  36. data/lib/aspera/rest.rb +3 -12
  37. data/lib/aspera/secrets.rb +20 -0
  38. data/lib/aspera/sync.rb +40 -35
  39. data/lib/aspera/timer_limiter.rb +22 -0
  40. data/lib/aspera/web_auth.rb +105 -0
  41. metadata +22 -3
  42. data/docs/transfer_spec.html +0 -99
data/lib/aspera/oauth.rb CHANGED
@@ -1,25 +1,35 @@
1
1
  require 'aspera/open_application'
2
+ require 'aspera/web_auth'
3
+ require 'aspera/id_generator'
2
4
  require 'base64'
3
5
  require 'date'
4
6
  require 'socket'
5
7
  require 'securerandom'
6
8
 
7
9
  module Aspera
8
- # implement OAuth 2 for the REST client and generate a bearer token
10
+ # Implement OAuth 2 for the REST client and generate a bearer token
9
11
  # call get_authorization() to get a token.
10
12
  # bearer tokens are kept in memory and also in a file cache for later re-use
11
- # if a token is expired (api returns 4xx), call again get_authorization({:refresh=>true})
13
+ # if a token is expired (api returns 4xx), call again get_authorization({refresh: true})
12
14
  class Oauth
15
+ # used for code exchange
16
+ DEFAULT_PATH_AUTHORIZE='authorize'
17
+ # to generate token
18
+ DEFAULT_PATH_TOKEN='token'
19
+ # field with token in result
20
+ DEFAULT_TOKEN_FIELD='access_token'
13
21
  private
14
22
  # remove 5 minutes to account for time offset (TODO: configurable?)
15
- JWT_NOTBEFORE_OFFSET=300
23
+ JWT_NOTBEFORE_OFFSET_SEC=300
16
24
  # one hour validity (TODO: configurable?)
17
- JWT_EXPIRY_OFFSET=3600
25
+ JWT_EXPIRY_OFFSET_SEC=3600
26
+ # tokens older than 30 minutes will be discarded from cache
27
+ TOKEN_CACHE_EXPIRY_SEC=1800
28
+ # a prefix for persistency of tokens (garbage collect)
18
29
  PERSIST_CATEGORY_TOKEN='token'
19
- # tokens older than 30 minutes will be discarded
20
- TOKEN_EXPIRY_SECONDS=1800
21
- THANK_YOU_HTML = "<html><head><title>Ok</title></head><body><h1>Thank you !</h1><p>You can close this window.</p></body></html>"
22
- private_constant :JWT_NOTBEFORE_OFFSET,:JWT_EXPIRY_OFFSET,:PERSIST_CATEGORY_TOKEN,:TOKEN_EXPIRY_SECONDS,:THANK_YOU_HTML
30
+ ONE_HOUR_AS_DAY_FRACTION=Rational(1,24)
31
+
32
+ private_constant :JWT_NOTBEFORE_OFFSET_SEC,:JWT_EXPIRY_OFFSET_SEC,:PERSIST_CATEGORY_TOKEN,:TOKEN_CACHE_EXPIRY_SEC,:ONE_HOUR_AS_DAY_FRACTION
23
33
  class << self
24
34
  # OAuth methods supported
25
35
  def auth_types
@@ -44,8 +54,25 @@ module Aspera
44
54
  def flush_tokens
45
55
  persist_mgr.garbage_collect(PERSIST_CATEGORY_TOKEN,nil)
46
56
  end
57
+
58
+ def register_decoder(method)
59
+ @decoders||=[]
60
+ @decoders.push(method)
61
+ end
62
+
63
+ def decode_token(token)
64
+ Log.log.debug(">>>> #{token} : #{@decoders.length}")
65
+ @decoders.each do |decoder|
66
+ result=decoder.call(token) rescue nil
67
+ return result unless result.nil?
68
+ end
69
+ return nil
70
+ end
47
71
  end
48
72
 
73
+ # seems to be quite standard token encoding (RFC?)
74
+ self.register_decoder lambda { |token| parts=token.split('.'); raise "not aoc token" unless parts.length.eql?(3); JSON.parse(Base64.decode64(parts[1]))}
75
+
49
76
  # for supported parameters, look in the code for @params
50
77
  # parameters are provided all with oauth_ prefix :
51
78
  # :base_url
@@ -55,8 +82,8 @@ module Aspera
55
82
  # :jwt_audience
56
83
  # :jwt_private_key_obj
57
84
  # :jwt_subject
58
- # :path_authorize (default: 'authorize')
59
- # :path_token (default: 'token')
85
+ # :path_authorize (default: DEFAULT_PATH_AUTHORIZE)
86
+ # :path_token (default: DEFAULT_PATH_TOKEN)
60
87
  # :scope (optional)
61
88
  # :grant (one of returned by self.auth_types)
62
89
  # :url_token
@@ -64,21 +91,21 @@ module Aspera
64
91
  # :user_pass
65
92
  # :token_type
66
93
  def initialize(auth_params)
67
- Log.log.debug "auth=#{auth_params}"
94
+ Log.log.debug("auth=#{auth_params}")
68
95
  @params=auth_params.clone
69
96
  # default values
70
97
  # name of field to take as token from result of call to /token
71
- @params[:token_field]||='access_token'
98
+ @params[:token_field]||=DEFAULT_TOKEN_FIELD
72
99
  # default endpoint for /token
73
- @params[:path_token]||='token'
100
+ @params[:path_token]||=DEFAULT_PATH_TOKEN
74
101
  # default endpoint for /authorize
75
- @params[:path_authorize]||='authorize'
76
- rest_params={:base_url => @params[:base_url]}
102
+ @params[:path_authorize]||=DEFAULT_PATH_AUTHORIZE
103
+ rest_params={base_url: @params[:base_url]}
77
104
  if @params.has_key?(:client_id)
78
- rest_params.merge!({:auth => {
79
- :type => :basic,
80
- :username => @params[:client_id],
81
- :password => @params[:client_secret]
105
+ rest_params.merge!({auth: {
106
+ type: :basic,
107
+ username: @params[:client_id],
108
+ password: @params[:client_secret]
82
109
  }})
83
110
  end
84
111
  @token_auth_api=Rest.new(rest_params)
@@ -89,39 +116,37 @@ module Aspera
89
116
  # we could check that host is localhost or local address
90
117
  end
91
118
  # cleanup expired tokens
92
- self.class.persist_mgr.garbage_collect(PERSIST_CATEGORY_TOKEN,TOKEN_EXPIRY_SECONDS)
119
+ self.class.persist_mgr.garbage_collect(PERSIST_CATEGORY_TOKEN,TOKEN_CACHE_EXPIRY_SEC)
93
120
  end
94
121
 
95
122
  # open the login page, wait for code and check_code, then return code
96
123
  def goto_page_and_get_code(login_page_url,check_code)
97
- request_params=self.class.goto_page_and_get_request(@params[:redirect_uri],login_page_url)
124
+ Log.log.info("login_page_url=#{login_page_url}".bg_red.gray)
125
+ # start a web server to receive request code
126
+ webserver=WebAuth.new(@params[:redirect_uri])
127
+ # start browser on login page
128
+ OpenApplication.instance.uri(login_page_url)
129
+ # wait for code in request
130
+ request_params=webserver.get_request
98
131
  Log.log.error("state does not match") if !check_code.eql?(request_params['state'])
99
132
  code=request_params['code']
100
133
  return code
101
134
  end
102
135
 
103
- def create_token_advanced(rest_params)
136
+ def create_token(rest_params)
104
137
  return @token_auth_api.call({
105
- :operation => 'POST',
106
- :subpath => @params[:path_token],
107
- :headers => {'Accept'=>'application/json'}}.merge(rest_params))
138
+ operation: 'POST',
139
+ subpath: @params[:path_token],
140
+ headers: {'Accept'=>'application/json'}}.merge(rest_params))
108
141
  end
109
142
 
110
- # shortcut for create_token_advanced
111
- def create_token_www_body(creation_params)
112
- return create_token_advanced({:www_body_params=>creation_params})
113
- end
114
-
115
- # @return Array list of unique identifiers of token
116
- def token_cache_ids(api_scope)
143
+ # @return unique identifier of token
144
+ def token_cache_id(api_scope)
117
145
  oauth_uri=URI.parse(@params[:base_url])
118
- parts=[PERSIST_CATEGORY_TOKEN,oauth_uri.host.downcase.gsub(/[^a-z]+/,'_'),oauth_uri.path.downcase.gsub(/[^a-z]+/,'_'),@params[:grant]]
119
- parts.push(api_scope) unless api_scope.nil?
120
- parts.push(@params[:jwt_subject]) if @params.has_key?(:jwt_subject)
121
- parts.push(@params[:user_name]) if @params.has_key?(:user_name)
122
- parts.push(@params[:url_token]) if @params.has_key?(:url_token)
123
- parts.push(@params[:api_key]) if @params.has_key?(:api_key)
124
- return parts
146
+ parts=[PERSIST_CATEGORY_TOKEN,api_scope,oauth_uri.host,oauth_uri.path]
147
+ # add some of the parameters that uniquely define the token
148
+ [:grant,:jwt_subject,:user_name,:url_token,:api_key].inject(parts){|p,i|p.push(@params[i])}
149
+ return IdGenerator.from_list(parts)
125
150
  end
126
151
 
127
152
  public
@@ -141,22 +166,23 @@ module Aspera
141
166
  use_refresh_token=options[:refresh]
142
167
 
143
168
  # generate token identifier to use with cache
144
- token_ids=token_cache_ids(api_scope)
169
+ token_id=token_cache_id(api_scope)
145
170
 
146
171
  # get token_data from cache (or nil), token_data is what is returned by /token
147
- token_data=self.class.persist_mgr.get(token_ids)
172
+ token_data=self.class.persist_mgr.get(token_id)
148
173
  token_data=JSON.parse(token_data) unless token_data.nil?
149
-
150
174
  # Optional optimization: check if node token is expired, then force refresh
151
175
  # in case the transfer agent cannot refresh himself
152
176
  # else, anyway, faspmanager is equipped with refresh code
153
177
  if !token_data.nil?
154
- decoded_node_token = Node.decode_bearer_token(token_data['access_token']) rescue nil
178
+ # TODO: use @params[:token_field] ?
179
+ decoded_node_token = self.class.decode_token(token_data['access_token'])
180
+ Log.dump('decoded_node_token',decoded_node_token) unless decoded_node_token.nil?
155
181
  if decoded_node_token.is_a?(Hash) and decoded_node_token['expires_at'].is_a?(String)
156
- Log.dump('decoded_node_token',decoded_node_token)
157
182
  expires_at=DateTime.parse(decoded_node_token['expires_at'])
158
- one_hour_as_day_fraction=Rational(1,24)
159
- use_refresh_token=true if DateTime.now > (expires_at-one_hour_as_day_fraction)
183
+ # Time.at(decoded_node_token['exp'])
184
+ # does it seem expired, with one hour of security
185
+ use_refresh_token=true if DateTime.now > (expires_at-ONE_HOUR_AS_DAY_FRACTION)
160
186
  end
161
187
  end
162
188
 
@@ -167,7 +193,7 @@ module Aspera
167
193
  refresh_token=token_data['refresh_token']
168
194
  end
169
195
  # delete caches
170
- self.class.persist_mgr.delete(token_ids)
196
+ self.class.persist_mgr.delete(token_id)
171
197
  token_data=nil
172
198
  # lets try the existing refresh token
173
199
  if !refresh_token.nil?
@@ -175,14 +201,14 @@ module Aspera
175
201
  # try to refresh
176
202
  # note: admin token has no refresh, and lives by default 1800secs
177
203
  # Note: scope is mandatory in Files, and we can either provide basic auth, or client_Secret in data
178
- resp=create_token_www_body(p_client_id_and_scope.merge({
179
- :grant_type =>'refresh_token',
180
- :refresh_token=>refresh_token}))
204
+ resp=create_token(www_body_params: p_client_id_and_scope.merge({
205
+ grant_type: 'refresh_token',
206
+ refresh_token: refresh_token}))
181
207
  if resp[:http].code.start_with?('2') then
182
- # save only if success ?
208
+ # save only if success
183
209
  json_data=resp[:http].body
184
210
  token_data=JSON.parse(json_data)
185
- self.class.persist_mgr.put(token_ids,json_data)
211
+ self.class.persist_mgr.put(token_id,json_data)
186
212
  else
187
213
  Log.log.debug("refresh failed: #{resp[:http].body}".bg_red)
188
214
  end
@@ -197,19 +223,19 @@ module Aspera
197
223
  # AoC Web based Auth
198
224
  check_code=SecureRandom.uuid
199
225
  auth_params=p_client_id_and_scope.merge({
200
- :response_type => 'code',
201
- :redirect_uri => @params[:redirect_uri],
202
- :state => check_code
226
+ response_type: 'code',
227
+ redirect_uri: @params[:redirect_uri],
228
+ state: check_code
203
229
  })
204
230
  auth_params[:client_secret]=@params[:client_secret] if @params.has_key?(:client_secret)
205
231
  login_page_url=Rest.build_uri("#{@params[:base_url]}/#{@params[:path_authorize]}",auth_params)
206
232
  # here, we need a human to authorize on a web page
207
233
  code=goto_page_and_get_code(login_page_url,check_code)
208
234
  # exchange code for token
209
- resp=create_token_www_body(p_client_id_and_scope.merge({
210
- :grant_type => 'authorization_code',
211
- :code => code,
212
- :redirect_uri => @params[:redirect_uri]
235
+ resp=create_token(www_body_params: p_client_id_and_scope.merge({
236
+ grant_type: 'authorization_code',
237
+ code: code,
238
+ redirect_uri: @params[:redirect_uri]
213
239
  }))
214
240
  when :jwt
215
241
  # https://tools.ietf.org/html/rfc7519
@@ -219,74 +245,81 @@ module Aspera
219
245
  Log.log.info("seconds=#{seconds_since_epoch}")
220
246
 
221
247
  payload = {
222
- :iss => @params[:client_id], # issuer
223
- :sub => @params[:jwt_subject], # subject
224
- :aud => @params[:jwt_audience], # audience
225
- :nbf => seconds_since_epoch-JWT_NOTBEFORE_OFFSET, # not before
226
- :exp => seconds_since_epoch+JWT_EXPIRY_OFFSET # expiration
248
+ iss: @params[:client_id], # issuer
249
+ sub: @params[:jwt_subject], # subject
250
+ aud: @params[:jwt_audience], # audience
251
+ nbf: seconds_since_epoch-JWT_NOTBEFORE_OFFSET_SEC, # not before
252
+ exp: seconds_since_epoch+JWT_EXPIRY_OFFSET_SEC # expiration
227
253
  }
254
+ # Hum.. compliant ? TODO: remove when Faspex5 API is clarified
255
+ if @params.has_key?(:f5_username)
256
+ payload[:jti] = SecureRandom.uuid # JWT id
257
+ payload[:iat] = seconds_since_epoch # issued at
258
+ payload.delete(:nbf) # not used in f5
259
+ p_scope[:redirect_uri]="https://127.0.0.1:5000/token" # used ?
260
+ p_scope[:state]=SecureRandom.uuid
261
+ p_scope[:client_id]=@params[:client_id]
262
+ @token_auth_api.params[:auth]={type: :basic,username: @params[:f5_username], password: @params[:f5_password]}
263
+ end
228
264
 
229
265
  # non standard, only for global ids
230
266
  payload.merge!(@params[:jwt_add]) if @params.has_key?(:jwt_add)
267
+ Log.log.debug("JWT payload=[#{payload}]")
231
268
 
232
269
  rsa_private=@params[:jwt_private_key_obj] # type: OpenSSL::PKey::RSA
233
-
234
270
  Log.log.debug("private=[#{rsa_private}]")
235
271
 
236
- Log.log.debug("JWT assertion=[#{payload}]")
237
- assertion = JWT.encode(payload, rsa_private, 'RS256')
238
-
272
+ assertion = JWT.encode(payload, rsa_private, 'RS256', @params[:jwt_headers]||{})
239
273
  Log.log.debug("assertion=[#{assertion}]")
240
274
 
241
- resp=create_token_www_body(p_scope.merge({
242
- :grant_type => 'urn:ietf:params:oauth:grant-type:jwt-bearer',
243
- :assertion => assertion
275
+ resp=create_token(www_body_params: p_scope.merge({
276
+ grant_type: 'urn:ietf:params:oauth:grant-type:jwt-bearer',
277
+ assertion: assertion
244
278
  }))
245
279
  when :url_token
246
280
  # AoC Public Link
247
- params={:url_token=>@params[:url_token]}
281
+ params={url_token: @params[:url_token]}
248
282
  params[:password]=@params[:password] if @params.has_key?(:password)
249
- resp=create_token_advanced({
250
- :json_params => params,
251
- :url_params => p_scope.merge({
252
- :grant_type => 'url_token'
253
- })})
283
+ resp=create_token({
284
+ json_params: params,
285
+ url_params: p_scope.merge({grant_type: 'url_token'})
286
+ })
254
287
  when :ibm_apikey
255
288
  # ATS
256
- resp=create_token_www_body({
257
- 'grant_type' => 'urn:ibm:params:oauth:grant-type:apikey',
258
- 'response_type' => 'cloud_iam',
259
- 'apikey' => @params[:api_key]
289
+ resp=create_token(www_body_params: {
290
+ grant_type: 'urn:ibm:params:oauth:grant-type:apikey',
291
+ response_type: 'cloud_iam',
292
+ apikey: @params[:api_key]
260
293
  })
261
294
  when :delegated_refresh
262
295
  # COS
263
- resp=create_token_www_body({
264
- 'grant_type' => 'urn:ibm:params:oauth:grant-type:apikey',
265
- 'response_type' => 'delegated_refresh_token',
266
- 'apikey' => @params[:api_key],
267
- 'receiver_client_ids' => 'aspera_ats'
296
+ resp=create_token(www_body_params: {
297
+ grant_type: 'urn:ibm:params:oauth:grant-type:apikey',
298
+ response_type: 'delegated_refresh_token',
299
+ apikey: @params[:api_key],
300
+ receiver_client_ids: 'aspera_ats'
268
301
  })
269
302
  when :header_userpass
270
303
  # used in Faspex apiv4 and shares2
271
- resp=create_token_advanced({
272
- :auth => {
273
- :type => :basic,
274
- :username => @params[:user_name],
275
- :password => @params[:user_pass]},
276
- :json_params => p_client_id_and_scope.merge({:grant_type => 'password'}), #:www_body_params also works
277
- })
304
+ resp=create_token(
305
+ json_params: p_client_id_and_scope.merge({grant_type: 'password'}), #:www_body_params also works
306
+ auth: {
307
+ type: :basic,
308
+ username: @params[:user_name],
309
+ password: @params[:user_pass]}
310
+ )
278
311
  when :body_userpass
279
312
  # legacy, not used
280
- resp=create_token_www_body(p_client_id_and_scope.merge({
281
- :grant_type => 'password',
282
- :username => @params[:user_name],
283
- :password => @params[:user_pass]
313
+ resp=create_token(www_body_params: p_client_id_and_scope.merge({
314
+ grant_type: 'password',
315
+ username: @params[:user_name],
316
+ password: @params[:user_pass]
284
317
  }))
285
318
  when :body_data
286
319
  # used in Faspex apiv5
287
- resp=create_token_advanced({
288
- :auth => {:type => :none},
289
- :json_params => @params[:userpass_body],
320
+ resp=create_token({
321
+ auth: {type: :none},
322
+ json_params: @params[:userpass_body],
290
323
  })
291
324
  else
292
325
  raise "auth grant type unknown: #{@params[:grant]}"
@@ -294,39 +327,12 @@ module Aspera
294
327
  # TODO: test return code ?
295
328
  json_data=resp[:http].body
296
329
  token_data=JSON.parse(json_data)
297
- self.class.persist_mgr.put(token_ids,json_data)
330
+ self.class.persist_mgr.put(token_id,json_data)
298
331
  end # if ! in_cache
299
-
332
+ raise "API error: No such field in answer: #{@params[:token_field]}" unless token_data.has_key?(@params[:token_field])
300
333
  # ok we shall have a token here
301
334
  return 'Bearer '+token_data[@params[:token_field]]
302
335
  end
303
336
 
304
- # open the login page, wait for code and return parameters
305
- def self.goto_page_and_get_request(redirect_uri,login_page_url,html_page=THANK_YOU_HTML)
306
- Log.log.info "login_page_url=#{login_page_url}".bg_red().gray()
307
- # browser start is not blocking, we hope here that starting is slower than opening port
308
- OpenApplication.instance.uri(login_page_url)
309
- port=URI.parse(redirect_uri).port
310
- Log.log.info "listening on port #{port}"
311
- request_params=nil
312
- TCPServer.open('127.0.0.1', port) { |webserver|
313
- Log.log.info "server=#{webserver}"
314
- websession = webserver.accept
315
- sleep 1 # TODO: sometimes: returns nil ? use webrick ?
316
- line = websession.gets.chomp
317
- Log.log.info "line=#{line}"
318
- if ! line.start_with?('GET /?') then
319
- raise "unexpected request"
320
- end
321
- request = line.partition('?').last.partition(' ').first
322
- data=URI.decode_www_form(request)
323
- request_params=data.to_h
324
- Log.log.debug "request_params=#{request_params}"
325
- websession.print "HTTP/1.1 200/OK\r\nContent-type:text/html\r\n\r\n#{html_page}"
326
- websession.close
327
- }
328
- return request_params
329
- end
330
-
331
337
  end # OAuth
332
338
  end # Aspera
data/lib/aspera/persistency_action_once.rb CHANGED
@@ -6,7 +6,7 @@ module Aspera
6
6
  class PersistencyActionOnce
7
7
  # @param :manager Mandatory Database
8
8
  # @param :data Mandatory object to persist, must be same object from begin to end (assume array by default)
9
- # @param :ids Mandatory identifiers
9
+ # @param :id Mandatory identifiers
10
10
  # @param :delete Optional delete persistency condition
11
11
  # @param :parse Optional parse method (default to JSON)
12
12
  # @param :format Optional dump method (default to JSON)
@@ -16,27 +16,31 @@ module Aspera
16
16
  raise "options shall be Hash" unless options.is_a?(Hash)
17
17
  raise "mandatory :manager" if options[:manager].nil?
18
18
  raise "mandatory :data" if options[:data].nil?
19
- raise "mandatory :ids (Array)" unless options[:ids].is_a?(Array)
20
- raise "mandatory 1 element in :ids" unless options[:ids].length >= 1
19
+ raise "mandatory :id (String)" unless options[:id].is_a?(String)
20
+ raise "mandatory 1 element in :id" unless options[:id].length >= 1
21
21
  @manager=options[:manager]
22
22
  @persisted_object=options[:data]
23
- @object_ids=options[:ids]
23
+ @object_id=options[:id]
24
24
  # by default , at save time, file is deleted if data is nil
25
25
  @delete_condition=options[:delete] || lambda{|d|d.empty?}
26
26
  @persist_format=options[:format] || lambda {|h| JSON.generate(h)}
27
27
  persist_parse=options[:parse] || lambda {|t| JSON.parse(t)}
28
28
  persist_merge=options[:merge] || lambda {|current,file| current.concat(file).uniq rescue current}
29
- value=@manager.get(@object_ids)
29
+ value=@manager.get(@object_id)
30
30
  persist_merge.call(@persisted_object,persist_parse.call(value)) unless value.nil?
31
31
  end
32
32
 
33
33
  def save
34
34
  if @delete_condition.call(@persisted_object)
35
- @manager.delete(@object_ids)
35
+ @manager.delete(@object_id)
36
36
  else
37
- @manager.put(@object_ids,@persist_format.call(@persisted_object))
37
+ @manager.put(@object_id,@persist_format.call(@persisted_object))
38
38
  end
39
39
  end
40
40
 
41
+ def data
42
+ return @persisted_object
43
+ end
44
+
41
45
  end
42
46
  end
data/lib/aspera/persistency_folder.rb CHANGED
@@ -6,11 +6,8 @@ require 'aspera/log'
6
6
  module Aspera
7
7
  # Persist data on file system
8
8
  class PersistencyFolder
9
- WINDOWS_PROTECTED_CHAR=%r{[/:"<>\\\*\?]}
10
- PROTECTED_CHAR_REPLACE='_'
11
- ID_SEPARATOR='_'
12
9
  FILE_SUFFIX='.txt'
13
- private_constant :PROTECTED_CHAR_REPLACE,:FILE_SUFFIX,:WINDOWS_PROTECTED_CHAR,:ID_SEPARATOR
10
+ private_constant :FILE_SUFFIX
14
11
  def initialize(folder)
15
12
  @cache={}
16
13
  set_folder(folder)
@@ -23,7 +20,6 @@ module Aspera
23
20
 
24
21
  # @return String or nil string on existing persist, else nil
25
22
  def get(object_id)
26
- object_id=marshalled_id(object_id)
27
23
  Log.log.debug("persistency get: #{object_id}")
28
24
  if @cache.has_key?(object_id)
29
25
  Log.log.debug("got from memory cache")
@@ -39,18 +35,16 @@ module Aspera
39
35
  end
40
36
 
41
37
  def put(object_id,value)
42
- raise "only String supported" unless value.is_a?(String)
43
- object_id=marshalled_id(object_id)
38
+ raise "value: only String supported" unless value.is_a?(String)
44
39
  persist_filepath=id_to_filepath(object_id)
45
- Log.log.debug("saving: #{persist_filepath}")
40
+ Log.log.debug("persistency saving: #{persist_filepath}")
46
41
  File.write(persist_filepath,value)
47
42
  @cache[object_id]=value
48
43
  end
49
44
 
50
45
  def delete(object_id)
51
- object_id=marshalled_id(object_id)
52
46
  persist_filepath=id_to_filepath(object_id)
53
- Log.log.debug("empty data, deleting: #{persist_filepath}")
47
+ Log.log.debug("persistency deleting: #{persist_filepath}")
54
48
  File.delete(persist_filepath) if File.exist?(persist_filepath)
55
49
  @cache.delete(object_id)
56
50
  end
@@ -63,7 +57,7 @@ module Aspera
63
57
  end
64
58
  garbage_files.each do |filepath|
65
59
  File.delete(filepath)
66
- Log.log.debug("Deleted expired: #{filepath}")
60
+ Log.log.debug("persistency deleted expired: #{filepath}")
67
61
  end
68
62
  return garbage_files
69
63
  end
@@ -72,25 +66,11 @@ module Aspera
72
66
 
73
67
  # @param object_id String or Array
74
68
  def id_to_filepath(object_id)
69
+ raise "object_id: only String supported" unless object_id.is_a?(String)
75
70
  FileUtils.mkdir_p(@folder)
76
71
  return File.join(@folder,"#{object_id}#{FILE_SUFFIX}")
77
72
  #.gsub(/[^a-z]+/,FILE_FIELD_SEPARATOR)
78
73
  end
79
74
 
80
- def marshalled_id(object_id)
81
- if object_id.is_a?(Array)
82
- # special case, url in second position: TODO: check any position
83
- if object_id[1].is_a?(String) and object_id[1] =~ URI::ABS_URI
84
- object_id=object_id.clone
85
- object_id[1]=URI.parse(object_id[1]).host
86
- end
87
- object_id=object_id.join(ID_SEPARATOR)
88
- end
89
- raise "id must be a String" unless object_id.is_a?(String)
90
- return object_id.
91
- gsub(WINDOWS_PROTECTED_CHAR,PROTECTED_CHAR_REPLACE). # remove windows forbidden chars
92
- gsub('.',PROTECTED_CHAR_REPLACE). # keep dot for extension only (nicer)
93
- downcase
94
- end
95
75
  end # PersistencyFolder
96
76
  end # Aspera
data/lib/aspera/rest.rb CHANGED
@@ -120,22 +120,12 @@ module Aspera
120
120
  # default is no auth
121
121
  @params[:auth]||={:type=>:none}
122
122
  @params[:not_auth_codes]||=['401']
123
- # translate old auth parameters, remove prefix, place in auth (TODO: delete this)
124
- # [:auth,:basic,:oauth].each do |p_sym|
125
- # p_str=p_sym.to_s+'_'
126
- # @params.keys.select{|k|k.to_s.start_with?(p_str)}.each do |k_sym|
127
- # name=k_sym.to_s[p_str.length..-1]
128
- # name='grant' if k_sym.eql?(:oauth_type)
129
- # @params[:auth][name.to_sym]=@params[k_sym]
130
- # @params.delete(k_sym)
131
- # end
132
- # end
133
123
  @oauth=Oauth.new(@params[:auth]) if @params[:auth][:type].eql?(:oauth2)
134
124
  Log.dump('REST params(2)',@params)
135
125
  end
136
126
 
137
127
  def oauth_token(options={})
138
- raise "ERROR: not Oauth" unless @oauth.is_a?(Oauth)
128
+ raise "ERROR: expecting Oauth, have #{@oauth.class}" unless @oauth.is_a?(Oauth)
139
129
  return @oauth.get_authorization(options)
140
130
  end
141
131
 
@@ -156,6 +146,7 @@ module Aspera
156
146
  Log.log.debug("accessing #{call_data[:subpath]}".red.bold.bg_green)
157
147
  call_data[:headers]||={}
158
148
  call_data[:headers]['User-Agent'] ||= @@user_agent
149
+ # defaults from @params are overriden by call dataz
159
150
  call_data=@params.deep_merge(call_data)
160
151
  case call_data[:auth][:type]
161
152
  when :none
@@ -278,7 +269,7 @@ module Aspera
278
269
  if e.response.is_a?(Net::HTTPRedirection)
279
270
  if tries_remain_redirect > 0
280
271
  tries_remain_redirect-=1
281
- Log.log.error("URL is moved, check your config: #{e.response['location']}")
272
+ Log.log.info("URL is moved: #{e.response['location']}")
282
273
  raise e
283
274
  # TODO: rebuild request with new location
284
275
  #retry
data/lib/aspera/secrets.rb ADDED
@@ -0,0 +1,20 @@
1
+ module Aspera
2
+ # Manage secrets in CLI using secure way (encryption, wallet, etc...)
3
+ class Secrets
4
+ attr_accessor :default_secret,:all_secrets
5
+ def initialize()
6
+ @default_secret=nil
7
+ @all_secrets={}
8
+ end
9
+
10
+ def get_secret(id=nil,mandatory=true)
11
+ secret=@default_secret || @all_secrets[id]
12
+ raise "please provide secret for #{id}" if secret.nil? and mandatory
13
+ return secret
14
+ end
15
+
16
+ def get_secrets
17
+ return @all_secrets
18
+ end
19
+ end
20
+ end