aspera-cli 4.20.0 → 4.21.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (73) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/CHANGELOG.md +41 -3
  4. data/CONTRIBUTING.md +69 -142
  5. data/README.md +687 -461
  6. data/bin/ascli +5 -14
  7. data/bin/asession +3 -5
  8. data/examples/get_proto_file.rb +4 -3
  9. data/examples/proxy.pac +20 -20
  10. data/lib/aspera/agent/base.rb +2 -0
  11. data/lib/aspera/agent/connect.rb +20 -2
  12. data/lib/aspera/agent/{alpha.rb → desktop.rb} +12 -18
  13. data/lib/aspera/agent/direct.rb +30 -31
  14. data/lib/aspera/agent/node.rb +1 -11
  15. data/lib/aspera/agent/{trsdk.rb → transferd.rb} +37 -51
  16. data/lib/aspera/api/alee.rb +1 -1
  17. data/lib/aspera/api/aoc.rb +13 -8
  18. data/lib/aspera/api/cos_node.rb +1 -1
  19. data/lib/aspera/api/node.rb +49 -32
  20. data/lib/aspera/ascp/installation.rb +98 -77
  21. data/lib/aspera/ascp/management.rb +27 -6
  22. data/lib/aspera/cli/extended_value.rb +9 -3
  23. data/lib/aspera/cli/formatter.rb +155 -154
  24. data/lib/aspera/cli/info.rb +2 -1
  25. data/lib/aspera/cli/main.rb +12 -0
  26. data/lib/aspera/cli/manager.rb +4 -4
  27. data/lib/aspera/cli/plugin.rb +2 -2
  28. data/lib/aspera/cli/plugins/aoc.rb +134 -73
  29. data/lib/aspera/cli/plugins/config.rb +114 -83
  30. data/lib/aspera/cli/plugins/cos.rb +1 -0
  31. data/lib/aspera/cli/plugins/faspex.rb +4 -2
  32. data/lib/aspera/cli/plugins/faspex5.rb +29 -14
  33. data/lib/aspera/cli/plugins/node.rb +51 -41
  34. data/lib/aspera/cli/transfer_progress.rb +2 -0
  35. data/lib/aspera/cli/version.rb +1 -1
  36. data/lib/aspera/command_line_builder.rb +1 -1
  37. data/lib/aspera/coverage.rb +5 -3
  38. data/lib/aspera/environment.rb +59 -16
  39. data/lib/aspera/faspex_postproc.rb +3 -5
  40. data/lib/aspera/hash_ext.rb +2 -12
  41. data/lib/aspera/node_simulator.rb +230 -112
  42. data/lib/aspera/oauth/base.rb +40 -48
  43. data/lib/aspera/oauth/factory.rb +41 -2
  44. data/lib/aspera/oauth/jwt.rb +4 -1
  45. data/lib/aspera/persistency_action_once.rb +1 -1
  46. data/lib/aspera/persistency_folder.rb +20 -2
  47. data/lib/aspera/preview/generator.rb +13 -10
  48. data/lib/aspera/preview/options.rb +2 -2
  49. data/lib/aspera/preview/terminal.rb +1 -1
  50. data/lib/aspera/preview/utils.rb +11 -6
  51. data/lib/aspera/products/connect.rb +82 -0
  52. data/lib/aspera/products/desktop.rb +30 -0
  53. data/lib/aspera/products/other.rb +82 -0
  54. data/lib/aspera/products/transferd.rb +61 -0
  55. data/lib/aspera/rest.rb +22 -17
  56. data/lib/aspera/secret_hider.rb +9 -2
  57. data/lib/aspera/ssh.rb +31 -24
  58. data/lib/aspera/temp_file_manager.rb +5 -4
  59. data/lib/aspera/transfer/parameters.rb +2 -1
  60. data/lib/aspera/transfer/spec.yaml +22 -20
  61. data/lib/aspera/transfer/sync.rb +1 -5
  62. data/lib/aspera/transfer/uri.rb +2 -2
  63. data/lib/aspera/uri_reader.rb +18 -1
  64. data/lib/transferd_pb.rb +86 -0
  65. data/lib/transferd_services_pb.rb +84 -0
  66. data.tar.gz.sig +0 -0
  67. metadata +13 -166
  68. metadata.gz.sig +0 -0
  69. data/examples/build_exec +0 -74
  70. data/examples/build_exec_rubyc +0 -40
  71. data/lib/aspera/ascp/products.rb +0 -168
  72. data/lib/transfer_pb.rb +0 -84
  73. data/lib/transfer_services_pb.rb +0 -82
@@ -7,28 +7,241 @@ require 'webrick'
7
7
  require 'json'
8
8
 
9
9
  module Aspera
10
+ class NodeSimulator
11
+ def initialize
12
+ @agent = Agent::Direct.new(management_cb: ->(event){process_event(event)})
13
+ @sessions = {}
14
+ end
15
+
16
+ def start(ts)
17
+ @agent.start_transfer(ts)
18
+ end
19
+
20
+ def all_sessions
21
+ @agent.sessions.map { |session| session[:job_id] }.uniq.each.map{|job_id|job_to_transfer(job_id)}
22
+ end
23
+
24
+ # status: ('waiting', 'partially_completed', 'unknown', 'waiting(read error)',] 'running', 'completed', 'failed'
25
+ def job_to_transfer(job_id)
26
+ jobs = @agent.sessions_by_job(job_id)
27
+ ts = nil
28
+ sessions = jobs.map do |j|
29
+ ts ||= j[:ts]
30
+ {
31
+ id: j[:id],
32
+ client_node_id: '',
33
+ server_node_id: '2bbdcc39-f789-4d47-8163-6767fc14f421',
34
+ client_ip_address: '192.168.0.100',
35
+ server_ip_address: '5.10.114.4',
36
+ status: 'running',
37
+ retry_timeout: 3600,
38
+ retry_count: 0,
39
+ start_time_usec: 1701094040000000,
40
+ end_time_usec: nil,
41
+ elapsed_usec: 405312,
42
+ bytes_transferred: 26,
43
+ bytes_written: 26,
44
+ bytes_lost: 0,
45
+ files_completed: 1,
46
+ directories_completed: 0,
47
+ target_rate_kbps: 500000,
48
+ min_rate_kbps: 0,
49
+ calc_rate_kbps: 9900,
50
+ network_delay_usec: 40000,
51
+ avg_rate_kbps: 0.51,
52
+ error_code: 0,
53
+ error_desc: '',
54
+ source_statistics: {
55
+ args_scan_attempted: 1,
56
+ args_scan_completed: 1,
57
+ paths_scan_attempted: 1,
58
+ paths_scan_failed: 0,
59
+ paths_scan_skipped: 0,
60
+ paths_scan_excluded: 0,
61
+ dirs_scan_completed: 0,
62
+ files_scan_completed: 1,
63
+ dirs_xfer_attempted: 0,
64
+ dirs_xfer_fail: 0,
65
+ files_xfer_attempted: 1,
66
+ files_xfer_fail: 0,
67
+ files_xfer_noxfer: 0
68
+ },
69
+ precalc: {
70
+ enabled: true,
71
+ status: 'ready',
72
+ bytes_expected: 0,
73
+ directories_expected: 0,
74
+ files_expected: 0,
75
+ files_excluded: 0,
76
+ files_special: 0,
77
+ files_failed: 1
78
+ }
79
+ }
80
+ end
81
+ ts ||= {}
82
+ result = {
83
+ id: job_id,
84
+ status: 'running',
85
+ start_spec: ts,
86
+ sessions: sessions,
87
+ bytes_transferred: 26,
88
+ bytes_written: 26,
89
+ bytes_lost: 0,
90
+ avg_rate_kbps: 0.51,
91
+ files_completed: 1,
92
+ files_skipped: 0,
93
+ directories_completed: 0,
94
+ start_time_usec: 1701094040000000,
95
+ end_time_usec: 1701094040405312,
96
+ elapsed_usec: 405312,
97
+ error_code: 0,
98
+ error_desc: '',
99
+ precalc: {
100
+ status: 'ready',
101
+ bytes_expected: 0,
102
+ files_expected: 0,
103
+ directories_expected: 0,
104
+ files_special: 0,
105
+ files_failed: 1
106
+ },
107
+ files: [{
108
+ id: 'd1b5c112-82b75425-860745fc-93851671-64541bdd',
109
+ path: '/workspaces/45071/packages/bYA_ilq73g.asp-package/contents/data_file.bin',
110
+ start_time_usec: 1701094040000000,
111
+ elapsed_usec: 105616,
112
+ end_time_usec: 1701094040001355,
113
+ status: 'completed',
114
+ error_code: 0,
115
+ error_desc: '',
116
+ size: 26,
117
+ type: 'file',
118
+ checksum_type: 'none',
119
+ checksum: nil,
120
+ start_byte: 0,
121
+ bytes_written: 26,
122
+ session_id: 'bafc72b8-366c-4501-8095-47208183d6b8'}]
123
+ }
124
+ Log.log.trace2{Log.dump(:job, result)}
125
+ return result
126
+ end
127
+
128
+ # Process event from management port
129
+ def process_event(event)
130
+ case event['Type']
131
+ when 'NOP' then Aspera.Log.debug{"event not managed: #{event['Type']}"}
132
+ # rubocop:disable Lint/DuplicateBranch
133
+ when 'START' then Aspera.Log.debug{"event not managed: #{event['Type']}"}
134
+ when 'QUERY' then Aspera.Log.debug{"event not managed: #{event['Type']}"}
135
+ when 'QUERYRSP' then Aspera.Log.debug{"event not managed: #{event['Type']}"}
136
+ when 'STATS' then Aspera.Log.debug{"event not managed: #{event['Type']}"}
137
+ when 'STOP' then Aspera.Log.debug{"event not managed: #{event['Type']}"}
138
+ when 'ERROR' then Aspera.Log.debug{"event not managed: #{event['Type']}"}
139
+ when 'CANCEL' then Aspera.Log.debug{"event not managed: #{event['Type']}"}
140
+ when 'DONE' then Aspera.Log.debug{"event not managed: #{event['Type']}"}
141
+ when 'RATE' then Aspera.Log.debug{"event not managed: #{event['Type']}"}
142
+ when 'FILEERROR' then Aspera.Log.debug{"event not managed: #{event['Type']}"}
143
+ when 'SESSION' then Aspera.Log.debug{"event not managed: #{event['Type']}"}
144
+ when 'NOTIFICATION' then Aspera.Log.debug{"event not managed: #{event['Type']}"}
145
+ when 'INIT' then Aspera.Log.debug{"event not managed: #{event['Type']}"}
146
+ when 'VLINK' then Aspera.Log.debug{"event not managed: #{event['Type']}"}
147
+ when 'PUT' then Aspera.Log.debug{"event not managed: #{event['Type']}"}
148
+ when 'WRITE' then Aspera.Log.debug{"event not managed: #{event['Type']}"}
149
+ when 'CLOSE' then Aspera.Log.debug{"event not managed: #{event['Type']}"}
150
+ when 'SKIP' then Aspera.Log.debug{"event not managed: #{event['Type']}"}
151
+ when 'ARGSTOP' then Aspera.Log.debug{"event not managed: #{event['Type']}"}
152
+ # rubocop:enable Lint/DuplicateBranch
153
+ else Aspera.error_unreachable_line
154
+ end
155
+ end
156
+ end
157
+
10
158
  # this class answers the Faspex /send API and creates a package on Aspera on Cloud
159
+ # a new instance is created for each request
11
160
  class NodeSimulatorServlet < WEBrick::HTTPServlet::AbstractServlet
12
161
  PATH_TRANSFERS = '/ops/transfers'
13
162
  PATH_ONE_TRANSFER = %r{/ops/transfers/(.+)$}
163
+ PATH_BROWSE = '/files/browse'
14
164
  # @param app_api [Api::AoC]
15
165
  # @param app_context [String]
16
- def initialize(server, credentials, transfer)
166
+ def initialize(server, credentials, simulator)
17
167
  super(server)
18
168
  @credentials = credentials
19
- @xfer_manager = Agent::Direct.new
169
+ @simulator = simulator
170
+ end
171
+
172
+ require 'json'
173
+ require 'time'
174
+
175
+ def folder_to_structure(folder_path)
176
+ raise "Path does not exist or is not a directory: #{folder_path}" unless Dir.exist?(folder_path)
177
+
178
+ # Build self structure
179
+ folder_stat = File.stat(folder_path)
180
+ structure = {
181
+ 'self' => {
182
+ 'path' => folder_path,
183
+ 'basename' => File.basename(folder_path),
184
+ 'type' => 'directory',
185
+ 'size' => folder_stat.size,
186
+ 'mtime' => folder_stat.mtime.utc.iso8601,
187
+ 'permissions' => [
188
+ { 'name' => 'view' },
189
+ { 'name' => 'edit' },
190
+ { 'name' => 'delete' }
191
+ ]
192
+ },
193
+ 'items' => []
194
+ }
195
+
196
+ # Iterate over folder contents
197
+ Dir.foreach(folder_path) do |entry|
198
+ next if entry == '.' || entry == '..' # Skip current and parent directory
199
+
200
+ item_path = File.join(folder_path, entry)
201
+ item_type = File.ftype(item_path) rescue 'unknown' # Get the type of file
202
+ item_stat = File.lstat(item_path) # Use lstat to handle symbolic links correctly
203
+
204
+ item = {
205
+ 'path' => item_path,
206
+ 'basename' => entry,
207
+ 'type' => item_type,
208
+ 'size' => item_stat.size,
209
+ 'mtime' => item_stat.mtime.utc.iso8601,
210
+ 'permissions' => [
211
+ { 'name' => 'view' },
212
+ { 'name' => 'edit' },
213
+ { 'name' => 'delete' }
214
+ ]
215
+ }
216
+
217
+ # Add additional details for specific types
218
+ case item_type
219
+ when 'file'
220
+ item['partial_file'] = false
221
+ when 'link'
222
+ item['target'] = File.readlink(item_path) rescue nil # Add the target of the symlink
223
+ when 'unknown'
224
+ item['note'] = 'File type could not be determined'
225
+ end
226
+
227
+ structure['items'] << item
228
+ end
229
+
230
+ structure
20
231
  end
21
232
 
22
233
  def do_POST(request, response)
23
234
  case request.path
24
235
  when PATH_TRANSFERS
25
- job_id = @xfer_manager.start_transfer(JSON.parse(request.body))
26
- session = @xfer_manager.sessions_by_job(job_id).first
27
- result = session[:ts].clone
28
- result['id'] = job_id
29
- set_json_response(response, result)
236
+ job_id = @simulator.start(JSON.parse(request.body))
237
+ sleep(0.5)
238
+ set_json_response(request, response, @simulator.job_to_transfer(job_id))
239
+ when PATH_BROWSE
240
+ req = JSON.parse(request.body)
241
+ # req['count']
242
+ set_json_response(request, response, folder_to_structure(req['path']))
30
243
  else
31
- set_json_response(response, [{error: 'Bad request'}], code: 400)
244
+ set_json_response(request, response, [{error: 'Bad request'}], code: 400)
32
245
  end
33
246
  end
34
247
 
@@ -36,7 +249,7 @@ module Aspera
36
249
  case request.path
37
250
  when '/info'
38
251
  info = Ascp::Installation.instance.ascp_info
39
- set_json_response(response, {
252
+ set_json_response(request, response, {
40
253
  application: 'node',
41
254
  current_time: Time.now.utc.iso8601(0),
42
255
  version: info['sdk_ascp_version'].gsub(/ .*$/, ''),
@@ -44,13 +257,13 @@ module Aspera
44
257
  license_max_rate: info['maximum_bandwidth'],
45
258
  os: %x(uname -srv).chomp,
46
259
  aej_status: 'disconnected',
47
- async_reporting: 'yes',
48
- transfer_activity_reporting: 'yes',
260
+ async_reporting: 'no',
261
+ transfer_activity_reporting: 'no',
49
262
  transfer_user: 'xfer',
50
263
  docroot: 'file:////data/aoc/eudemo-sedemo',
51
264
  node_id: '2bbdcc39-f789-4d47-8163-6767fc14f421',
52
265
  cluster_id: '6dae2844-d1a9-47a5-916d-9b3eac3ea466',
53
- acls: [],
266
+ acls: ['impersonation'],
54
267
  access_key_configuration_capabilities: {
55
268
  transfer: %w[
56
269
  cipher
@@ -99,115 +312,20 @@ module Aspera
99
312
  {name: 'wss_port', value: 443}
100
313
  ]})
101
314
  when PATH_TRANSFERS
102
- result = @xfer_manager.sessions.map { |session| job_to_transfer(session) }
103
- set_json_response(response, result)
315
+ set_json_response(request, response, @simulator.all_sessions)
104
316
  when PATH_ONE_TRANSFER
105
317
  job_id = request.path.match(PATH_ONE_TRANSFER)[1]
106
- set_json_response(response, job_to_transfer(@xfer_manager.sessions_by_job(job_id).first))
318
+ set_json_response(request, response, @simulator.job_to_transfer(job_id))
107
319
  else
108
- set_json_response(response, [{error: 'Unknown request'}], code: 400)
320
+ set_json_response(request, response, [{error: 'Unknown request'}], code: 400)
109
321
  end
110
322
  end
111
323
 
112
- def set_json_response(response, json, code: 200)
324
+ def set_json_response(request, response, json, code: 200)
113
325
  response.status = code
114
326
  response['Content-Type'] = 'application/json'
115
327
  response.body = json.to_json
116
- Log.log.trace1{Log.dump('response', json)}
117
- end
118
-
119
- def job_to_transfer(job)
120
- session = {
121
- id: 'bafc72b8-366c-4501-8095-47208183d6b8',
122
- client_node_id: '',
123
- server_node_id: '2bbdcc39-f789-4d47-8163-6767fc14f421',
124
- client_ip_address: '192.168.0.100',
125
- server_ip_address: '5.10.114.4',
126
- status: 'running',
127
- retry_timeout: 3600,
128
- retry_count: 0,
129
- start_time_usec: 1701094040000000,
130
- end_time_usec: nil,
131
- elapsed_usec: 405312,
132
- bytes_transferred: 26,
133
- bytes_written: 26,
134
- bytes_lost: 0,
135
- files_completed: 1,
136
- directories_completed: 0,
137
- target_rate_kbps: 500000,
138
- min_rate_kbps: 0,
139
- calc_rate_kbps: 9900,
140
- network_delay_usec: 40000,
141
- avg_rate_kbps: 0.51,
142
- error_code: 0,
143
- error_desc: '',
144
- source_statistics: {
145
- args_scan_attempted: 1,
146
- args_scan_completed: 1,
147
- paths_scan_attempted: 1,
148
- paths_scan_failed: 0,
149
- paths_scan_skipped: 0,
150
- paths_scan_excluded: 0,
151
- dirs_scan_completed: 0,
152
- files_scan_completed: 1,
153
- dirs_xfer_attempted: 0,
154
- dirs_xfer_fail: 0,
155
- files_xfer_attempted: 1,
156
- files_xfer_fail: 0,
157
- files_xfer_noxfer: 0
158
- },
159
- precalc: {
160
- enabled: true,
161
- status: 'ready',
162
- bytes_expected: 0,
163
- directories_expected: 0,
164
- files_expected: 0,
165
- files_excluded: 0,
166
- files_special: 0,
167
- files_failed: 1
168
- }}
169
- return {
170
- id: '609a667d-642e-4290-9312-b4d20d3c0159',
171
- status: 'running',
172
- start_spec: job[:ts],
173
- sessions: [session],
174
- bytes_transferred: 26,
175
- bytes_written: 26,
176
- bytes_lost: 0,
177
- avg_rate_kbps: 0.51,
178
- files_completed: 1,
179
- files_skipped: 0,
180
- directories_completed: 0,
181
- start_time_usec: 1701094040000000,
182
- end_time_usec: 1701094040405312,
183
- elapsed_usec: 405312,
184
- error_code: 0,
185
- error_desc: '',
186
- precalc: {
187
- status: 'ready',
188
- bytes_expected: 0,
189
- files_expected: 0,
190
- directories_expected: 0,
191
- files_special: 0,
192
- files_failed: 1
193
- },
194
- files: [{
195
- id: 'd1b5c112-82b75425-860745fc-93851671-64541bdd',
196
- path: '/workspaces/45071/packages/bYA_ilq73g.asp-package/contents/data_file.bin',
197
- start_time_usec: 1701094040000000,
198
- elapsed_usec: 105616,
199
- end_time_usec: 1701094040001355,
200
- status: 'completed',
201
- error_code: 0,
202
- error_desc: '',
203
- size: 26,
204
- type: 'file',
205
- checksum_type: 'none',
206
- checksum: nil,
207
- start_byte: 0,
208
- bytes_written: 26,
209
- session_id: 'bafc72b8-366c-4501-8095-47208183d6b8'}]
210
- }
328
+ Log.log.trace1{Log.dump("response for #{request.request_method} #{request.path}", json)}
211
329
  end
212
330
  end
213
331
  end
@@ -26,7 +26,7 @@ module Aspera
26
26
  scope: nil,
27
27
  use_query: false,
28
28
  path_token: 'token',
29
- token_field: 'access_token',
29
+ token_field: Factory::TOKEN_FIELD,
30
30
  cache_ids: nil,
31
31
  **rest_params
32
32
  )
@@ -38,12 +38,10 @@ module Aspera
38
38
  @client_id = client_id
39
39
  @client_secret = client_secret
40
40
  @use_query = use_query
41
- @base_cache_ids = cache_ids.clone
42
- @base_cache_ids = [] if @base_cache_ids.nil?
41
+ @base_cache_ids = cache_ids.nil? ? [] : cache_ids.clone
43
42
  Aspera.assert_type(@base_cache_ids, Array)
44
- if @api.auth_params.key?(:username)
45
- cache_ids.push(@api.auth_params[:username])
46
- end
43
+ @base_cache_ids.push(@api.auth_params[:username]) if @api.auth_params.key?(:username)
44
+ @base_cache_ids.compact!
47
45
  @base_cache_ids.freeze
48
46
  self.scope = scope
49
47
  end
@@ -81,54 +79,48 @@ module Aspera
81
79
  return call_params
82
80
  end
83
81
 
82
+ # @return value suitable for Authorization header
83
+ def authorization(**kwargs)
84
+ return OAuth::Factory.bearer_build(token(**kwargs))
85
+ end
86
+
84
87
  # get an OAuth v2 token (generated, cached, refreshed)
85
88
  # call token() to get a token.
86
89
  # if a token is expired (api returns 4xx), call again token(refresh: true)
87
90
  # @param cache set to false to disable cache
88
91
  # @param refresh set to true to force refresh or re-generation (if previous failed)
89
92
  def token(cache: true, refresh: false)
90
- # get token_data from cache (or nil), token_data is what is returned by /token
91
- token_data = Factory.instance.persist_mgr.get(@token_cache_id) if cache
92
- token_data = JSON.parse(token_data) unless token_data.nil?
93
- # Optional optimization: check if node token is expired based on decoded content then force refresh if close enough
94
- # might help in case the transfer agent cannot refresh himself
95
- # `direct` agent is equipped with refresh code
96
- if !refresh && !token_data.nil?
97
- decoded_token = OAuth::Factory.instance.decode_token(token_data[@token_field])
98
- Log.log.debug{Log.dump('decoded_token', decoded_token)} unless decoded_token.nil?
99
- if decoded_token.is_a?(Hash)
100
- expires_at_sec =
101
- if decoded_token['expires_at'].is_a?(String) then DateTime.parse(decoded_token['expires_at']).to_time
102
- elsif decoded_token['exp'].is_a?(Integer) then Time.at(decoded_token['exp'])
93
+ # get token info from cache (or nil), decoded with date and expiration status
94
+ token_info = Factory.instance.get_token_info(@token_cache_id) if cache
95
+ token_data = nil
96
+ unless token_info.nil?
97
+ token_data = token_info[:data]
98
+ # Optional optimization:
99
+ # check if node token is expired based on decoded content then force refresh if close enough
100
+ # might help in case the transfer agent cannot refresh himself
101
+ # `direct` agent is equipped with refresh code
102
+ # an API was already called, but failed, we need to regenerate or refresh
103
+ if refresh || token_info[:expired]
104
+ if token_data.key?('refresh_token') && token_data['refresh_token'].eql?('not_supported')
105
+ # save possible refresh token, before deleting the cache
106
+ refresh_token = token_data['refresh_token']
107
+ end
108
+ # delete cache
109
+ Factory.instance.persist_mgr.delete(@token_cache_id)
110
+ token_data = nil
111
+ # lets try the existing refresh token
112
+ if !refresh_token.nil?
113
+ Log.log.info{"refresh=[#{refresh_token}]".bg_green}
114
+ # NOTE: AoC admin token has no refresh, and lives by default 1800secs
115
+ resp = create_token_call(optional_scope_client_id.merge(grant_type: 'refresh_token', refresh_token: refresh_token))
116
+ if resp[:http].code.start_with?('2')
117
+ # save only if success
118
+ json_data = resp[:http].body
119
+ token_data = JSON.parse(json_data)
120
+ Factory.instance.persist_mgr.put(@token_cache_id, json_data)
121
+ else
122
+ Log.log.debug{"refresh failed: #{resp[:http].body}".bg_red}
103
123
  end
104
- # force refresh if we see a token too close from expiration
105
- refresh = true if expires_at_sec.is_a?(Time) && (expires_at_sec - Time.now) < OAuth::Factory.instance.parameters[:token_expiration_guard_sec]
106
- Log.log.debug{"Expiration: #{expires_at_sec} / #{refresh}"}
107
- end
108
- end
109
-
110
- # an API was already called, but failed, we need to regenerate or refresh
111
- if refresh
112
- if token_data.is_a?(Hash) && token_data.key?('refresh_token') && !token_data['refresh_token'].eql?('not_supported')
113
- # save possible refresh token, before deleting the cache
114
- refresh_token = token_data['refresh_token']
115
- end
116
- # delete cache
117
- Factory.instance.persist_mgr.delete(@token_cache_id)
118
- token_data = nil
119
- # lets try the existing refresh token
120
- if !refresh_token.nil?
121
- Log.log.info{"refresh=[#{refresh_token}]".bg_green}
122
- # try to refresh
123
- # note: AoC admin token has no refresh, and lives by default 1800secs
124
- resp = create_token_call(optional_scope_client_id.merge(grant_type: 'refresh_token', refresh_token: refresh_token))
125
- if resp[:http].code.start_with?('2')
126
- # save only if success
127
- json_data = resp[:http].body
128
- token_data = JSON.parse(json_data)
129
- Factory.instance.persist_mgr.put(@token_cache_id, json_data)
130
- else
131
- Log.log.debug{"refresh failed: #{resp[:http].body}".bg_red}
132
124
  end
133
125
  end
134
126
  end
@@ -142,7 +134,7 @@ module Aspera
142
134
  end
143
135
  Aspera.assert(token_data.key?(@token_field)){"API error: No such field in answer: #{@token_field}"}
144
136
  # ok we shall have a token here
145
- return OAuth::Factory.bearer_build(token_data[@token_field])
137
+ return token_data[@token_field]
146
138
  end
147
139
  end
148
140
  end
@@ -13,6 +13,7 @@ module Aspera
13
13
  PERSIST_CATEGORY_TOKEN = 'token'
14
14
  # prefix for bearer token when in header
15
15
  BEARER_PREFIX = 'Bearer '
16
+ TOKEN_FIELD = 'access_token'
16
17
 
17
18
  private_constant :PERSIST_CATEGORY_TOKEN, :BEARER_PREFIX
18
19
 
@@ -87,7 +88,45 @@ module Aspera
87
88
 
88
89
  # delete all existing tokens
89
90
  def flush_tokens
90
- persist_mgr.garbage_collect(PERSIST_CATEGORY_TOKEN, nil)
91
+ persist_mgr.garbage_collect(PERSIST_CATEGORY_TOKEN)
92
+ end
93
+
94
+ def persisted_tokens
95
+ data = persist_mgr.current_items(PERSIST_CATEGORY_TOKEN)
96
+ data.each.map do |k, v|
97
+ info = {id: k}
98
+ info.merge!(JSON.parse(v)) rescue nil
99
+ d = decode_token(info.delete(TOKEN_FIELD))
100
+ info.merge(d) if d
101
+ info
102
+ end
103
+ end
104
+
105
+ # get token information from cache
106
+ # @param id [String] identifier of token
107
+ # @return [Hash] token internal information , including Date object for `expiration_date`
108
+ def get_token_info(id)
109
+ token_raw_string = persist_mgr.get(id)
110
+ return nil if token_raw_string.nil?
111
+ token_data = JSON.parse(token_raw_string)
112
+ Aspera.assert_type(token_data, Hash)
113
+ decoded_token = decode_token(token_data[TOKEN_FIELD])
114
+ info = { data: token_data }
115
+ Log.log.debug{Log.dump('decoded_token', decoded_token)}
116
+ if decoded_token.is_a?(Hash)
117
+ info[:decoded] = decoded_token
118
+ # TODO: move date decoding to token decoder ?
119
+ expiration_date =
120
+ if decoded_token['expires_at'].is_a?(String) then DateTime.parse(decoded_token['expires_at']).to_time
121
+ elsif decoded_token['exp'].is_a?(Integer) then Time.at(decoded_token['exp'])
122
+ end
123
+ unless expiration_date.nil?
124
+ info[:expiration] = expiration_date
125
+ info[:ttl_sec] = expiration_date - Time.now
126
+ info[:expired] = info[:ttl_sec] < @parameters[:token_expiration_guard_sec]
127
+ end
128
+ end
129
+ return info
91
130
  end
92
131
 
93
132
  # register a bearer token decoder, mainly to inspect expiry date
@@ -125,6 +164,6 @@ module Aspera
125
164
  end
126
165
  end
127
166
  # JSON Web Signature (JWS) compact serialization: https://datatracker.ietf.org/doc/html/rfc7515
128
- Factory.instance.register_decoder(lambda { |token| parts = token.split('.'); Aspera.assert(parts.length.eql?(3)){'not aoc token'}; JSON.parse(Base64.decode64(parts[1]))}) # rubocop:disable Style/Semicolon, Layout/LineLength
167
+ Factory.instance.register_decoder(lambda { |token| parts = token.split('.'); Aspera.assert(parts.length.eql?(3)){'not JWS token'}; JSON.parse(Base64.decode64(parts[1]))}) # rubocop:disable Style/Semicolon, Layout/LineLength
129
168
  end
130
169
  end
@@ -21,12 +21,15 @@ module Aspera
21
21
  private_key_obj:,
22
22
  payload:,
23
23
  headers: {},
24
+ cache_ids: [],
24
25
  **base_params
25
26
  )
26
27
  Aspera.assert_type(private_key_obj, OpenSSL::PKey::RSA){'private_key_obj'}
27
28
  Aspera.assert_type(payload, Hash){'payload'}
28
29
  Aspera.assert_type(headers, Hash){'headers'}
29
- super(**base_params, cache_ids: [payload[:sub]])
30
+ Aspera.assert_type(cache_ids, Array){'cache ids'}
31
+ new_cache_ids = cache_ids.clone.push(payload[:sub])
32
+ super(**base_params, cache_ids: new_cache_ids)
30
33
  @private_key_obj = private_key_obj
31
34
  @additional_payload = payload
32
35
  @headers = headers
@@ -7,7 +7,7 @@ require 'aspera/assert'
7
7
  module Aspera
8
8
  # Persist data on file system
9
9
  class PersistencyActionOnce
10
- DELETE_DEFAULT = lambda{|d|d.empty?}
10
+ DELETE_DEFAULT = lambda(&:empty?)
11
11
  PARSE_DEFAULT = lambda {|t| JSON.parse(t)}
12
12
  FORMAT_DEFAULT = lambda {|h| JSON.generate(h)}
13
13
  MERGE_DEFAULT = lambda {|current, file| current.concat(file).uniq rescue current}
@@ -18,7 +18,8 @@ module Aspera
18
18
  Log.log.debug{"persistency folder: #{@folder}"}
19
19
  end
20
20
 
21
- # @return String or nil string on existing persist, else nil
21
+ # Get value of persisted item
22
+ # @return [String,nil] Value of persisted id
22
23
  def get(object_id)
23
24
  Log.log.debug{"persistency get: #{object_id}"}
24
25
  if @cache.key?(object_id)
@@ -34,6 +35,10 @@ module Aspera
34
35
  return @cache[object_id]
35
36
  end
36
37
 
38
+ # Set value of persisted item
39
+ # @param object_id [String] Identifier of persisted item
40
+ # @param value [String] Value of persisted item
41
+ # @return [nil]
37
42
  def put(object_id, value)
38
43
  Aspera.assert_type(value, String)
39
44
  persist_filepath = id_to_filepath(object_id)
@@ -42,8 +47,11 @@ module Aspera
42
47
  File.write(persist_filepath, value)
43
48
  Environment.restrict_file_access(persist_filepath)
44
49
  @cache[object_id] = value
50
+ nil
45
51
  end
46
52
 
53
+ # Delete persisted item
54
+ # @param object_id [String] Identifier of persisted item
47
55
  def delete(object_id)
48
56
  persist_filepath = id_to_filepath(object_id)
49
57
  Log.log.debug{"persistency deleting: #{persist_filepath}"}
@@ -51,8 +59,9 @@ module Aspera
51
59
  @cache.delete(object_id)
52
60
  end
53
61
 
62
+ # Delete persisted items
54
63
  def garbage_collect(persist_category, max_age_seconds=nil)
55
- garbage_files = Dir[File.join(@folder, persist_category + '*' + FILE_SUFFIX)]
64
+ garbage_files = current_files(persist_category)
56
65
  if !max_age_seconds.nil?
57
66
  current_time = Time.now
58
67
  garbage_files.select! { |filepath| (current_time - File.stat(filepath).mtime).to_i > max_age_seconds}
@@ -61,9 +70,18 @@ module Aspera
61
70
  File.delete(filepath)
62
71
  Log.log.debug{"persistency deleted expired: #{filepath}"}
63
72
  end
73
+ @cache.clear
64
74
  return garbage_files
65
75
  end
66
76
 
77
+ def current_files(persist_category)
78
+ Dir[File.join(@folder, persist_category + '*' + FILE_SUFFIX)]
79
+ end
80
+
81
+ def current_items(persist_category)
82
+ current_files(persist_category).each_with_object({}) {|i, h| h[File.basename(i, FILE_SUFFIX)] = File.read(i)}
83
+ end
84
+
67
85
  private
68
86
 
69
87
  # @param object_id String or Array