aspera-cli 4.20.0 → 4.21.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/CHANGELOG.md +41 -3
- data/CONTRIBUTING.md +69 -142
- data/README.md +687 -461
- data/bin/ascli +5 -14
- data/bin/asession +3 -5
- data/examples/get_proto_file.rb +4 -3
- data/examples/proxy.pac +20 -20
- data/lib/aspera/agent/base.rb +2 -0
- data/lib/aspera/agent/connect.rb +20 -2
- data/lib/aspera/agent/{alpha.rb → desktop.rb} +12 -18
- data/lib/aspera/agent/direct.rb +30 -31
- data/lib/aspera/agent/node.rb +1 -11
- data/lib/aspera/agent/{trsdk.rb → transferd.rb} +37 -51
- data/lib/aspera/api/alee.rb +1 -1
- data/lib/aspera/api/aoc.rb +13 -8
- data/lib/aspera/api/cos_node.rb +1 -1
- data/lib/aspera/api/node.rb +49 -32
- data/lib/aspera/ascp/installation.rb +98 -77
- data/lib/aspera/ascp/management.rb +27 -6
- data/lib/aspera/cli/extended_value.rb +9 -3
- data/lib/aspera/cli/formatter.rb +155 -154
- data/lib/aspera/cli/info.rb +2 -1
- data/lib/aspera/cli/main.rb +12 -0
- data/lib/aspera/cli/manager.rb +4 -4
- data/lib/aspera/cli/plugin.rb +2 -2
- data/lib/aspera/cli/plugins/aoc.rb +134 -73
- data/lib/aspera/cli/plugins/config.rb +114 -83
- data/lib/aspera/cli/plugins/cos.rb +1 -0
- data/lib/aspera/cli/plugins/faspex.rb +4 -2
- data/lib/aspera/cli/plugins/faspex5.rb +29 -14
- data/lib/aspera/cli/plugins/node.rb +51 -41
- data/lib/aspera/cli/transfer_progress.rb +2 -0
- data/lib/aspera/cli/version.rb +1 -1
- data/lib/aspera/command_line_builder.rb +1 -1
- data/lib/aspera/coverage.rb +5 -3
- data/lib/aspera/environment.rb +59 -16
- data/lib/aspera/faspex_postproc.rb +3 -5
- data/lib/aspera/hash_ext.rb +2 -12
- data/lib/aspera/node_simulator.rb +230 -112
- data/lib/aspera/oauth/base.rb +40 -48
- data/lib/aspera/oauth/factory.rb +41 -2
- data/lib/aspera/oauth/jwt.rb +4 -1
- data/lib/aspera/persistency_action_once.rb +1 -1
- data/lib/aspera/persistency_folder.rb +20 -2
- data/lib/aspera/preview/generator.rb +13 -10
- data/lib/aspera/preview/options.rb +2 -2
- data/lib/aspera/preview/terminal.rb +1 -1
- data/lib/aspera/preview/utils.rb +11 -6
- data/lib/aspera/products/connect.rb +82 -0
- data/lib/aspera/products/desktop.rb +30 -0
- data/lib/aspera/products/other.rb +82 -0
- data/lib/aspera/products/transferd.rb +61 -0
- data/lib/aspera/rest.rb +22 -17
- data/lib/aspera/secret_hider.rb +9 -2
- data/lib/aspera/ssh.rb +31 -24
- data/lib/aspera/temp_file_manager.rb +5 -4
- data/lib/aspera/transfer/parameters.rb +2 -1
- data/lib/aspera/transfer/spec.yaml +22 -20
- data/lib/aspera/transfer/sync.rb +1 -5
- data/lib/aspera/transfer/uri.rb +2 -2
- data/lib/aspera/uri_reader.rb +18 -1
- data/lib/transferd_pb.rb +86 -0
- data/lib/transferd_services_pb.rb +84 -0
- data.tar.gz.sig +0 -0
- metadata +13 -166
- metadata.gz.sig +0 -0
- data/examples/build_exec +0 -74
- data/examples/build_exec_rubyc +0 -40
- data/lib/aspera/ascp/products.rb +0 -168
- data/lib/transfer_pb.rb +0 -84
- data/lib/transfer_services_pb.rb +0 -82
@@ -7,28 +7,241 @@ require 'webrick'
|
|
7
7
|
require 'json'
|
8
8
|
|
9
9
|
module Aspera
|
10
|
+
class NodeSimulator
|
11
|
+
def initialize
|
12
|
+
@agent = Agent::Direct.new(management_cb: ->(event){process_event(event)})
|
13
|
+
@sessions = {}
|
14
|
+
end
|
15
|
+
|
16
|
+
def start(ts)
|
17
|
+
@agent.start_transfer(ts)
|
18
|
+
end
|
19
|
+
|
20
|
+
def all_sessions
|
21
|
+
@agent.sessions.map { |session| session[:job_id] }.uniq.each.map{|job_id|job_to_transfer(job_id)}
|
22
|
+
end
|
23
|
+
|
24
|
+
# status: ('waiting', 'partially_completed', 'unknown', 'waiting(read error)',] 'running', 'completed', 'failed'
|
25
|
+
def job_to_transfer(job_id)
|
26
|
+
jobs = @agent.sessions_by_job(job_id)
|
27
|
+
ts = nil
|
28
|
+
sessions = jobs.map do |j|
|
29
|
+
ts ||= j[:ts]
|
30
|
+
{
|
31
|
+
id: j[:id],
|
32
|
+
client_node_id: '',
|
33
|
+
server_node_id: '2bbdcc39-f789-4d47-8163-6767fc14f421',
|
34
|
+
client_ip_address: '192.168.0.100',
|
35
|
+
server_ip_address: '5.10.114.4',
|
36
|
+
status: 'running',
|
37
|
+
retry_timeout: 3600,
|
38
|
+
retry_count: 0,
|
39
|
+
start_time_usec: 1701094040000000,
|
40
|
+
end_time_usec: nil,
|
41
|
+
elapsed_usec: 405312,
|
42
|
+
bytes_transferred: 26,
|
43
|
+
bytes_written: 26,
|
44
|
+
bytes_lost: 0,
|
45
|
+
files_completed: 1,
|
46
|
+
directories_completed: 0,
|
47
|
+
target_rate_kbps: 500000,
|
48
|
+
min_rate_kbps: 0,
|
49
|
+
calc_rate_kbps: 9900,
|
50
|
+
network_delay_usec: 40000,
|
51
|
+
avg_rate_kbps: 0.51,
|
52
|
+
error_code: 0,
|
53
|
+
error_desc: '',
|
54
|
+
source_statistics: {
|
55
|
+
args_scan_attempted: 1,
|
56
|
+
args_scan_completed: 1,
|
57
|
+
paths_scan_attempted: 1,
|
58
|
+
paths_scan_failed: 0,
|
59
|
+
paths_scan_skipped: 0,
|
60
|
+
paths_scan_excluded: 0,
|
61
|
+
dirs_scan_completed: 0,
|
62
|
+
files_scan_completed: 1,
|
63
|
+
dirs_xfer_attempted: 0,
|
64
|
+
dirs_xfer_fail: 0,
|
65
|
+
files_xfer_attempted: 1,
|
66
|
+
files_xfer_fail: 0,
|
67
|
+
files_xfer_noxfer: 0
|
68
|
+
},
|
69
|
+
precalc: {
|
70
|
+
enabled: true,
|
71
|
+
status: 'ready',
|
72
|
+
bytes_expected: 0,
|
73
|
+
directories_expected: 0,
|
74
|
+
files_expected: 0,
|
75
|
+
files_excluded: 0,
|
76
|
+
files_special: 0,
|
77
|
+
files_failed: 1
|
78
|
+
}
|
79
|
+
}
|
80
|
+
end
|
81
|
+
ts ||= {}
|
82
|
+
result = {
|
83
|
+
id: job_id,
|
84
|
+
status: 'running',
|
85
|
+
start_spec: ts,
|
86
|
+
sessions: sessions,
|
87
|
+
bytes_transferred: 26,
|
88
|
+
bytes_written: 26,
|
89
|
+
bytes_lost: 0,
|
90
|
+
avg_rate_kbps: 0.51,
|
91
|
+
files_completed: 1,
|
92
|
+
files_skipped: 0,
|
93
|
+
directories_completed: 0,
|
94
|
+
start_time_usec: 1701094040000000,
|
95
|
+
end_time_usec: 1701094040405312,
|
96
|
+
elapsed_usec: 405312,
|
97
|
+
error_code: 0,
|
98
|
+
error_desc: '',
|
99
|
+
precalc: {
|
100
|
+
status: 'ready',
|
101
|
+
bytes_expected: 0,
|
102
|
+
files_expected: 0,
|
103
|
+
directories_expected: 0,
|
104
|
+
files_special: 0,
|
105
|
+
files_failed: 1
|
106
|
+
},
|
107
|
+
files: [{
|
108
|
+
id: 'd1b5c112-82b75425-860745fc-93851671-64541bdd',
|
109
|
+
path: '/workspaces/45071/packages/bYA_ilq73g.asp-package/contents/data_file.bin',
|
110
|
+
start_time_usec: 1701094040000000,
|
111
|
+
elapsed_usec: 105616,
|
112
|
+
end_time_usec: 1701094040001355,
|
113
|
+
status: 'completed',
|
114
|
+
error_code: 0,
|
115
|
+
error_desc: '',
|
116
|
+
size: 26,
|
117
|
+
type: 'file',
|
118
|
+
checksum_type: 'none',
|
119
|
+
checksum: nil,
|
120
|
+
start_byte: 0,
|
121
|
+
bytes_written: 26,
|
122
|
+
session_id: 'bafc72b8-366c-4501-8095-47208183d6b8'}]
|
123
|
+
}
|
124
|
+
Log.log.trace2{Log.dump(:job, result)}
|
125
|
+
return result
|
126
|
+
end
|
127
|
+
|
128
|
+
# Process event from management port
|
129
|
+
def process_event(event)
|
130
|
+
case event['Type']
|
131
|
+
when 'NOP' then Aspera.Log.debug{"event not managed: #{event['Type']}"}
|
132
|
+
# rubocop:disable Lint/DuplicateBranch
|
133
|
+
when 'START' then Aspera.Log.debug{"event not managed: #{event['Type']}"}
|
134
|
+
when 'QUERY' then Aspera.Log.debug{"event not managed: #{event['Type']}"}
|
135
|
+
when 'QUERYRSP' then Aspera.Log.debug{"event not managed: #{event['Type']}"}
|
136
|
+
when 'STATS' then Aspera.Log.debug{"event not managed: #{event['Type']}"}
|
137
|
+
when 'STOP' then Aspera.Log.debug{"event not managed: #{event['Type']}"}
|
138
|
+
when 'ERROR' then Aspera.Log.debug{"event not managed: #{event['Type']}"}
|
139
|
+
when 'CANCEL' then Aspera.Log.debug{"event not managed: #{event['Type']}"}
|
140
|
+
when 'DONE' then Aspera.Log.debug{"event not managed: #{event['Type']}"}
|
141
|
+
when 'RATE' then Aspera.Log.debug{"event not managed: #{event['Type']}"}
|
142
|
+
when 'FILEERROR' then Aspera.Log.debug{"event not managed: #{event['Type']}"}
|
143
|
+
when 'SESSION' then Aspera.Log.debug{"event not managed: #{event['Type']}"}
|
144
|
+
when 'NOTIFICATION' then Aspera.Log.debug{"event not managed: #{event['Type']}"}
|
145
|
+
when 'INIT' then Aspera.Log.debug{"event not managed: #{event['Type']}"}
|
146
|
+
when 'VLINK' then Aspera.Log.debug{"event not managed: #{event['Type']}"}
|
147
|
+
when 'PUT' then Aspera.Log.debug{"event not managed: #{event['Type']}"}
|
148
|
+
when 'WRITE' then Aspera.Log.debug{"event not managed: #{event['Type']}"}
|
149
|
+
when 'CLOSE' then Aspera.Log.debug{"event not managed: #{event['Type']}"}
|
150
|
+
when 'SKIP' then Aspera.Log.debug{"event not managed: #{event['Type']}"}
|
151
|
+
when 'ARGSTOP' then Aspera.Log.debug{"event not managed: #{event['Type']}"}
|
152
|
+
# rubocop:enable Lint/DuplicateBranch
|
153
|
+
else Aspera.error_unreachable_line
|
154
|
+
end
|
155
|
+
end
|
156
|
+
end
|
157
|
+
|
10
158
|
# this class answers the Faspex /send API and creates a package on Aspera on Cloud
|
159
|
+
# a new instance is created for each request
|
11
160
|
class NodeSimulatorServlet < WEBrick::HTTPServlet::AbstractServlet
|
12
161
|
PATH_TRANSFERS = '/ops/transfers'
|
13
162
|
PATH_ONE_TRANSFER = %r{/ops/transfers/(.+)$}
|
163
|
+
PATH_BROWSE = '/files/browse'
|
14
164
|
# @param app_api [Api::AoC]
|
15
165
|
# @param app_context [String]
|
16
|
-
def initialize(server, credentials,
|
166
|
+
def initialize(server, credentials, simulator)
|
17
167
|
super(server)
|
18
168
|
@credentials = credentials
|
19
|
-
@
|
169
|
+
@simulator = simulator
|
170
|
+
end
|
171
|
+
|
172
|
+
require 'json'
|
173
|
+
require 'time'
|
174
|
+
|
175
|
+
def folder_to_structure(folder_path)
|
176
|
+
raise "Path does not exist or is not a directory: #{folder_path}" unless Dir.exist?(folder_path)
|
177
|
+
|
178
|
+
# Build self structure
|
179
|
+
folder_stat = File.stat(folder_path)
|
180
|
+
structure = {
|
181
|
+
'self' => {
|
182
|
+
'path' => folder_path,
|
183
|
+
'basename' => File.basename(folder_path),
|
184
|
+
'type' => 'directory',
|
185
|
+
'size' => folder_stat.size,
|
186
|
+
'mtime' => folder_stat.mtime.utc.iso8601,
|
187
|
+
'permissions' => [
|
188
|
+
{ 'name' => 'view' },
|
189
|
+
{ 'name' => 'edit' },
|
190
|
+
{ 'name' => 'delete' }
|
191
|
+
]
|
192
|
+
},
|
193
|
+
'items' => []
|
194
|
+
}
|
195
|
+
|
196
|
+
# Iterate over folder contents
|
197
|
+
Dir.foreach(folder_path) do |entry|
|
198
|
+
next if entry == '.' || entry == '..' # Skip current and parent directory
|
199
|
+
|
200
|
+
item_path = File.join(folder_path, entry)
|
201
|
+
item_type = File.ftype(item_path) rescue 'unknown' # Get the type of file
|
202
|
+
item_stat = File.lstat(item_path) # Use lstat to handle symbolic links correctly
|
203
|
+
|
204
|
+
item = {
|
205
|
+
'path' => item_path,
|
206
|
+
'basename' => entry,
|
207
|
+
'type' => item_type,
|
208
|
+
'size' => item_stat.size,
|
209
|
+
'mtime' => item_stat.mtime.utc.iso8601,
|
210
|
+
'permissions' => [
|
211
|
+
{ 'name' => 'view' },
|
212
|
+
{ 'name' => 'edit' },
|
213
|
+
{ 'name' => 'delete' }
|
214
|
+
]
|
215
|
+
}
|
216
|
+
|
217
|
+
# Add additional details for specific types
|
218
|
+
case item_type
|
219
|
+
when 'file'
|
220
|
+
item['partial_file'] = false
|
221
|
+
when 'link'
|
222
|
+
item['target'] = File.readlink(item_path) rescue nil # Add the target of the symlink
|
223
|
+
when 'unknown'
|
224
|
+
item['note'] = 'File type could not be determined'
|
225
|
+
end
|
226
|
+
|
227
|
+
structure['items'] << item
|
228
|
+
end
|
229
|
+
|
230
|
+
structure
|
20
231
|
end
|
21
232
|
|
22
233
|
def do_POST(request, response)
|
23
234
|
case request.path
|
24
235
|
when PATH_TRANSFERS
|
25
|
-
job_id = @
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
236
|
+
job_id = @simulator.start(JSON.parse(request.body))
|
237
|
+
sleep(0.5)
|
238
|
+
set_json_response(request, response, @simulator.job_to_transfer(job_id))
|
239
|
+
when PATH_BROWSE
|
240
|
+
req = JSON.parse(request.body)
|
241
|
+
# req['count']
|
242
|
+
set_json_response(request, response, folder_to_structure(req['path']))
|
30
243
|
else
|
31
|
-
set_json_response(response, [{error: 'Bad request'}], code: 400)
|
244
|
+
set_json_response(request, response, [{error: 'Bad request'}], code: 400)
|
32
245
|
end
|
33
246
|
end
|
34
247
|
|
@@ -36,7 +249,7 @@ module Aspera
|
|
36
249
|
case request.path
|
37
250
|
when '/info'
|
38
251
|
info = Ascp::Installation.instance.ascp_info
|
39
|
-
set_json_response(response, {
|
252
|
+
set_json_response(request, response, {
|
40
253
|
application: 'node',
|
41
254
|
current_time: Time.now.utc.iso8601(0),
|
42
255
|
version: info['sdk_ascp_version'].gsub(/ .*$/, ''),
|
@@ -44,13 +257,13 @@ module Aspera
|
|
44
257
|
license_max_rate: info['maximum_bandwidth'],
|
45
258
|
os: %x(uname -srv).chomp,
|
46
259
|
aej_status: 'disconnected',
|
47
|
-
async_reporting: '
|
48
|
-
transfer_activity_reporting: '
|
260
|
+
async_reporting: 'no',
|
261
|
+
transfer_activity_reporting: 'no',
|
49
262
|
transfer_user: 'xfer',
|
50
263
|
docroot: 'file:////data/aoc/eudemo-sedemo',
|
51
264
|
node_id: '2bbdcc39-f789-4d47-8163-6767fc14f421',
|
52
265
|
cluster_id: '6dae2844-d1a9-47a5-916d-9b3eac3ea466',
|
53
|
-
acls: [],
|
266
|
+
acls: ['impersonation'],
|
54
267
|
access_key_configuration_capabilities: {
|
55
268
|
transfer: %w[
|
56
269
|
cipher
|
@@ -99,115 +312,20 @@ module Aspera
|
|
99
312
|
{name: 'wss_port', value: 443}
|
100
313
|
]})
|
101
314
|
when PATH_TRANSFERS
|
102
|
-
|
103
|
-
set_json_response(response, result)
|
315
|
+
set_json_response(request, response, @simulator.all_sessions)
|
104
316
|
when PATH_ONE_TRANSFER
|
105
317
|
job_id = request.path.match(PATH_ONE_TRANSFER)[1]
|
106
|
-
set_json_response(response,
|
318
|
+
set_json_response(request, response, @simulator.job_to_transfer(job_id))
|
107
319
|
else
|
108
|
-
set_json_response(response, [{error: 'Unknown request'}], code: 400)
|
320
|
+
set_json_response(request, response, [{error: 'Unknown request'}], code: 400)
|
109
321
|
end
|
110
322
|
end
|
111
323
|
|
112
|
-
def set_json_response(response, json, code: 200)
|
324
|
+
def set_json_response(request, response, json, code: 200)
|
113
325
|
response.status = code
|
114
326
|
response['Content-Type'] = 'application/json'
|
115
327
|
response.body = json.to_json
|
116
|
-
Log.log.trace1{Log.dump(
|
117
|
-
end
|
118
|
-
|
119
|
-
def job_to_transfer(job)
|
120
|
-
session = {
|
121
|
-
id: 'bafc72b8-366c-4501-8095-47208183d6b8',
|
122
|
-
client_node_id: '',
|
123
|
-
server_node_id: '2bbdcc39-f789-4d47-8163-6767fc14f421',
|
124
|
-
client_ip_address: '192.168.0.100',
|
125
|
-
server_ip_address: '5.10.114.4',
|
126
|
-
status: 'running',
|
127
|
-
retry_timeout: 3600,
|
128
|
-
retry_count: 0,
|
129
|
-
start_time_usec: 1701094040000000,
|
130
|
-
end_time_usec: nil,
|
131
|
-
elapsed_usec: 405312,
|
132
|
-
bytes_transferred: 26,
|
133
|
-
bytes_written: 26,
|
134
|
-
bytes_lost: 0,
|
135
|
-
files_completed: 1,
|
136
|
-
directories_completed: 0,
|
137
|
-
target_rate_kbps: 500000,
|
138
|
-
min_rate_kbps: 0,
|
139
|
-
calc_rate_kbps: 9900,
|
140
|
-
network_delay_usec: 40000,
|
141
|
-
avg_rate_kbps: 0.51,
|
142
|
-
error_code: 0,
|
143
|
-
error_desc: '',
|
144
|
-
source_statistics: {
|
145
|
-
args_scan_attempted: 1,
|
146
|
-
args_scan_completed: 1,
|
147
|
-
paths_scan_attempted: 1,
|
148
|
-
paths_scan_failed: 0,
|
149
|
-
paths_scan_skipped: 0,
|
150
|
-
paths_scan_excluded: 0,
|
151
|
-
dirs_scan_completed: 0,
|
152
|
-
files_scan_completed: 1,
|
153
|
-
dirs_xfer_attempted: 0,
|
154
|
-
dirs_xfer_fail: 0,
|
155
|
-
files_xfer_attempted: 1,
|
156
|
-
files_xfer_fail: 0,
|
157
|
-
files_xfer_noxfer: 0
|
158
|
-
},
|
159
|
-
precalc: {
|
160
|
-
enabled: true,
|
161
|
-
status: 'ready',
|
162
|
-
bytes_expected: 0,
|
163
|
-
directories_expected: 0,
|
164
|
-
files_expected: 0,
|
165
|
-
files_excluded: 0,
|
166
|
-
files_special: 0,
|
167
|
-
files_failed: 1
|
168
|
-
}}
|
169
|
-
return {
|
170
|
-
id: '609a667d-642e-4290-9312-b4d20d3c0159',
|
171
|
-
status: 'running',
|
172
|
-
start_spec: job[:ts],
|
173
|
-
sessions: [session],
|
174
|
-
bytes_transferred: 26,
|
175
|
-
bytes_written: 26,
|
176
|
-
bytes_lost: 0,
|
177
|
-
avg_rate_kbps: 0.51,
|
178
|
-
files_completed: 1,
|
179
|
-
files_skipped: 0,
|
180
|
-
directories_completed: 0,
|
181
|
-
start_time_usec: 1701094040000000,
|
182
|
-
end_time_usec: 1701094040405312,
|
183
|
-
elapsed_usec: 405312,
|
184
|
-
error_code: 0,
|
185
|
-
error_desc: '',
|
186
|
-
precalc: {
|
187
|
-
status: 'ready',
|
188
|
-
bytes_expected: 0,
|
189
|
-
files_expected: 0,
|
190
|
-
directories_expected: 0,
|
191
|
-
files_special: 0,
|
192
|
-
files_failed: 1
|
193
|
-
},
|
194
|
-
files: [{
|
195
|
-
id: 'd1b5c112-82b75425-860745fc-93851671-64541bdd',
|
196
|
-
path: '/workspaces/45071/packages/bYA_ilq73g.asp-package/contents/data_file.bin',
|
197
|
-
start_time_usec: 1701094040000000,
|
198
|
-
elapsed_usec: 105616,
|
199
|
-
end_time_usec: 1701094040001355,
|
200
|
-
status: 'completed',
|
201
|
-
error_code: 0,
|
202
|
-
error_desc: '',
|
203
|
-
size: 26,
|
204
|
-
type: 'file',
|
205
|
-
checksum_type: 'none',
|
206
|
-
checksum: nil,
|
207
|
-
start_byte: 0,
|
208
|
-
bytes_written: 26,
|
209
|
-
session_id: 'bafc72b8-366c-4501-8095-47208183d6b8'}]
|
210
|
-
}
|
328
|
+
Log.log.trace1{Log.dump("response for #{request.request_method} #{request.path}", json)}
|
211
329
|
end
|
212
330
|
end
|
213
331
|
end
|
data/lib/aspera/oauth/base.rb
CHANGED
@@ -26,7 +26,7 @@ module Aspera
|
|
26
26
|
scope: nil,
|
27
27
|
use_query: false,
|
28
28
|
path_token: 'token',
|
29
|
-
token_field:
|
29
|
+
token_field: Factory::TOKEN_FIELD,
|
30
30
|
cache_ids: nil,
|
31
31
|
**rest_params
|
32
32
|
)
|
@@ -38,12 +38,10 @@ module Aspera
|
|
38
38
|
@client_id = client_id
|
39
39
|
@client_secret = client_secret
|
40
40
|
@use_query = use_query
|
41
|
-
@base_cache_ids = cache_ids.clone
|
42
|
-
@base_cache_ids = [] if @base_cache_ids.nil?
|
41
|
+
@base_cache_ids = cache_ids.nil? ? [] : cache_ids.clone
|
43
42
|
Aspera.assert_type(@base_cache_ids, Array)
|
44
|
-
if @api.auth_params.key?(:username)
|
45
|
-
|
46
|
-
end
|
43
|
+
@base_cache_ids.push(@api.auth_params[:username]) if @api.auth_params.key?(:username)
|
44
|
+
@base_cache_ids.compact!
|
47
45
|
@base_cache_ids.freeze
|
48
46
|
self.scope = scope
|
49
47
|
end
|
@@ -81,54 +79,48 @@ module Aspera
|
|
81
79
|
return call_params
|
82
80
|
end
|
83
81
|
|
82
|
+
# @return value suitable for Authorization header
|
83
|
+
def authorization(**kwargs)
|
84
|
+
return OAuth::Factory.bearer_build(token(**kwargs))
|
85
|
+
end
|
86
|
+
|
84
87
|
# get an OAuth v2 token (generated, cached, refreshed)
|
85
88
|
# call token() to get a token.
|
86
89
|
# if a token is expired (api returns 4xx), call again token(refresh: true)
|
87
90
|
# @param cache set to false to disable cache
|
88
91
|
# @param refresh set to true to force refresh or re-generation (if previous failed)
|
89
92
|
def token(cache: true, refresh: false)
|
90
|
-
# get
|
91
|
-
|
92
|
-
token_data =
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
93
|
+
# get token info from cache (or nil), decoded with date and expiration status
|
94
|
+
token_info = Factory.instance.get_token_info(@token_cache_id) if cache
|
95
|
+
token_data = nil
|
96
|
+
unless token_info.nil?
|
97
|
+
token_data = token_info[:data]
|
98
|
+
# Optional optimization:
|
99
|
+
# check if node token is expired based on decoded content then force refresh if close enough
|
100
|
+
# might help in case the transfer agent cannot refresh himself
|
101
|
+
# `direct` agent is equipped with refresh code
|
102
|
+
# an API was already called, but failed, we need to regenerate or refresh
|
103
|
+
if refresh || token_info[:expired]
|
104
|
+
if token_data.key?('refresh_token') && token_data['refresh_token'].eql?('not_supported')
|
105
|
+
# save possible refresh token, before deleting the cache
|
106
|
+
refresh_token = token_data['refresh_token']
|
107
|
+
end
|
108
|
+
# delete cache
|
109
|
+
Factory.instance.persist_mgr.delete(@token_cache_id)
|
110
|
+
token_data = nil
|
111
|
+
# lets try the existing refresh token
|
112
|
+
if !refresh_token.nil?
|
113
|
+
Log.log.info{"refresh=[#{refresh_token}]".bg_green}
|
114
|
+
# NOTE: AoC admin token has no refresh, and lives by default 1800secs
|
115
|
+
resp = create_token_call(optional_scope_client_id.merge(grant_type: 'refresh_token', refresh_token: refresh_token))
|
116
|
+
if resp[:http].code.start_with?('2')
|
117
|
+
# save only if success
|
118
|
+
json_data = resp[:http].body
|
119
|
+
token_data = JSON.parse(json_data)
|
120
|
+
Factory.instance.persist_mgr.put(@token_cache_id, json_data)
|
121
|
+
else
|
122
|
+
Log.log.debug{"refresh failed: #{resp[:http].body}".bg_red}
|
103
123
|
end
|
104
|
-
# force refresh if we see a token too close from expiration
|
105
|
-
refresh = true if expires_at_sec.is_a?(Time) && (expires_at_sec - Time.now) < OAuth::Factory.instance.parameters[:token_expiration_guard_sec]
|
106
|
-
Log.log.debug{"Expiration: #{expires_at_sec} / #{refresh}"}
|
107
|
-
end
|
108
|
-
end
|
109
|
-
|
110
|
-
# an API was already called, but failed, we need to regenerate or refresh
|
111
|
-
if refresh
|
112
|
-
if token_data.is_a?(Hash) && token_data.key?('refresh_token') && !token_data['refresh_token'].eql?('not_supported')
|
113
|
-
# save possible refresh token, before deleting the cache
|
114
|
-
refresh_token = token_data['refresh_token']
|
115
|
-
end
|
116
|
-
# delete cache
|
117
|
-
Factory.instance.persist_mgr.delete(@token_cache_id)
|
118
|
-
token_data = nil
|
119
|
-
# lets try the existing refresh token
|
120
|
-
if !refresh_token.nil?
|
121
|
-
Log.log.info{"refresh=[#{refresh_token}]".bg_green}
|
122
|
-
# try to refresh
|
123
|
-
# note: AoC admin token has no refresh, and lives by default 1800secs
|
124
|
-
resp = create_token_call(optional_scope_client_id.merge(grant_type: 'refresh_token', refresh_token: refresh_token))
|
125
|
-
if resp[:http].code.start_with?('2')
|
126
|
-
# save only if success
|
127
|
-
json_data = resp[:http].body
|
128
|
-
token_data = JSON.parse(json_data)
|
129
|
-
Factory.instance.persist_mgr.put(@token_cache_id, json_data)
|
130
|
-
else
|
131
|
-
Log.log.debug{"refresh failed: #{resp[:http].body}".bg_red}
|
132
124
|
end
|
133
125
|
end
|
134
126
|
end
|
@@ -142,7 +134,7 @@ module Aspera
|
|
142
134
|
end
|
143
135
|
Aspera.assert(token_data.key?(@token_field)){"API error: No such field in answer: #{@token_field}"}
|
144
136
|
# ok we shall have a token here
|
145
|
-
return
|
137
|
+
return token_data[@token_field]
|
146
138
|
end
|
147
139
|
end
|
148
140
|
end
|
data/lib/aspera/oauth/factory.rb
CHANGED
@@ -13,6 +13,7 @@ module Aspera
|
|
13
13
|
PERSIST_CATEGORY_TOKEN = 'token'
|
14
14
|
# prefix for bearer token when in header
|
15
15
|
BEARER_PREFIX = 'Bearer '
|
16
|
+
TOKEN_FIELD = 'access_token'
|
16
17
|
|
17
18
|
private_constant :PERSIST_CATEGORY_TOKEN, :BEARER_PREFIX
|
18
19
|
|
@@ -87,7 +88,45 @@ module Aspera
|
|
87
88
|
|
88
89
|
# delete all existing tokens
|
89
90
|
def flush_tokens
|
90
|
-
persist_mgr.garbage_collect(PERSIST_CATEGORY_TOKEN
|
91
|
+
persist_mgr.garbage_collect(PERSIST_CATEGORY_TOKEN)
|
92
|
+
end
|
93
|
+
|
94
|
+
def persisted_tokens
|
95
|
+
data = persist_mgr.current_items(PERSIST_CATEGORY_TOKEN)
|
96
|
+
data.each.map do |k, v|
|
97
|
+
info = {id: k}
|
98
|
+
info.merge!(JSON.parse(v)) rescue nil
|
99
|
+
d = decode_token(info.delete(TOKEN_FIELD))
|
100
|
+
info.merge(d) if d
|
101
|
+
info
|
102
|
+
end
|
103
|
+
end
|
104
|
+
|
105
|
+
# get token information from cache
|
106
|
+
# @param id [String] identifier of token
|
107
|
+
# @return [Hash] token internal information , including Date object for `expiration_date`
|
108
|
+
def get_token_info(id)
|
109
|
+
token_raw_string = persist_mgr.get(id)
|
110
|
+
return nil if token_raw_string.nil?
|
111
|
+
token_data = JSON.parse(token_raw_string)
|
112
|
+
Aspera.assert_type(token_data, Hash)
|
113
|
+
decoded_token = decode_token(token_data[TOKEN_FIELD])
|
114
|
+
info = { data: token_data }
|
115
|
+
Log.log.debug{Log.dump('decoded_token', decoded_token)}
|
116
|
+
if decoded_token.is_a?(Hash)
|
117
|
+
info[:decoded] = decoded_token
|
118
|
+
# TODO: move date decoding to token decoder ?
|
119
|
+
expiration_date =
|
120
|
+
if decoded_token['expires_at'].is_a?(String) then DateTime.parse(decoded_token['expires_at']).to_time
|
121
|
+
elsif decoded_token['exp'].is_a?(Integer) then Time.at(decoded_token['exp'])
|
122
|
+
end
|
123
|
+
unless expiration_date.nil?
|
124
|
+
info[:expiration] = expiration_date
|
125
|
+
info[:ttl_sec] = expiration_date - Time.now
|
126
|
+
info[:expired] = info[:ttl_sec] < @parameters[:token_expiration_guard_sec]
|
127
|
+
end
|
128
|
+
end
|
129
|
+
return info
|
91
130
|
end
|
92
131
|
|
93
132
|
# register a bearer token decoder, mainly to inspect expiry date
|
@@ -125,6 +164,6 @@ module Aspera
|
|
125
164
|
end
|
126
165
|
end
|
127
166
|
# JSON Web Signature (JWS) compact serialization: https://datatracker.ietf.org/doc/html/rfc7515
|
128
|
-
Factory.instance.register_decoder(lambda { |token| parts = token.split('.'); Aspera.assert(parts.length.eql?(3)){'not
|
167
|
+
Factory.instance.register_decoder(lambda { |token| parts = token.split('.'); Aspera.assert(parts.length.eql?(3)){'not JWS token'}; JSON.parse(Base64.decode64(parts[1]))}) # rubocop:disable Style/Semicolon, Layout/LineLength
|
129
168
|
end
|
130
169
|
end
|
data/lib/aspera/oauth/jwt.rb
CHANGED
@@ -21,12 +21,15 @@ module Aspera
|
|
21
21
|
private_key_obj:,
|
22
22
|
payload:,
|
23
23
|
headers: {},
|
24
|
+
cache_ids: [],
|
24
25
|
**base_params
|
25
26
|
)
|
26
27
|
Aspera.assert_type(private_key_obj, OpenSSL::PKey::RSA){'private_key_obj'}
|
27
28
|
Aspera.assert_type(payload, Hash){'payload'}
|
28
29
|
Aspera.assert_type(headers, Hash){'headers'}
|
29
|
-
|
30
|
+
Aspera.assert_type(cache_ids, Array){'cache ids'}
|
31
|
+
new_cache_ids = cache_ids.clone.push(payload[:sub])
|
32
|
+
super(**base_params, cache_ids: new_cache_ids)
|
30
33
|
@private_key_obj = private_key_obj
|
31
34
|
@additional_payload = payload
|
32
35
|
@headers = headers
|
@@ -7,7 +7,7 @@ require 'aspera/assert'
|
|
7
7
|
module Aspera
|
8
8
|
# Persist data on file system
|
9
9
|
class PersistencyActionOnce
|
10
|
-
DELETE_DEFAULT = lambda
|
10
|
+
DELETE_DEFAULT = lambda(&:empty?)
|
11
11
|
PARSE_DEFAULT = lambda {|t| JSON.parse(t)}
|
12
12
|
FORMAT_DEFAULT = lambda {|h| JSON.generate(h)}
|
13
13
|
MERGE_DEFAULT = lambda {|current, file| current.concat(file).uniq rescue current}
|
@@ -18,7 +18,8 @@ module Aspera
|
|
18
18
|
Log.log.debug{"persistency folder: #{@folder}"}
|
19
19
|
end
|
20
20
|
|
21
|
-
#
|
21
|
+
# Get value of persisted item
|
22
|
+
# @return [String,nil] Value of persisted id
|
22
23
|
def get(object_id)
|
23
24
|
Log.log.debug{"persistency get: #{object_id}"}
|
24
25
|
if @cache.key?(object_id)
|
@@ -34,6 +35,10 @@ module Aspera
|
|
34
35
|
return @cache[object_id]
|
35
36
|
end
|
36
37
|
|
38
|
+
# Set value of persisted item
|
39
|
+
# @param object_id [String] Identifier of persisted item
|
40
|
+
# @param value [String] Value of persisted item
|
41
|
+
# @return [nil]
|
37
42
|
def put(object_id, value)
|
38
43
|
Aspera.assert_type(value, String)
|
39
44
|
persist_filepath = id_to_filepath(object_id)
|
@@ -42,8 +47,11 @@ module Aspera
|
|
42
47
|
File.write(persist_filepath, value)
|
43
48
|
Environment.restrict_file_access(persist_filepath)
|
44
49
|
@cache[object_id] = value
|
50
|
+
nil
|
45
51
|
end
|
46
52
|
|
53
|
+
# Delete persisted item
|
54
|
+
# @param object_id [String] Identifier of persisted item
|
47
55
|
def delete(object_id)
|
48
56
|
persist_filepath = id_to_filepath(object_id)
|
49
57
|
Log.log.debug{"persistency deleting: #{persist_filepath}"}
|
@@ -51,8 +59,9 @@ module Aspera
|
|
51
59
|
@cache.delete(object_id)
|
52
60
|
end
|
53
61
|
|
62
|
+
# Delete persisted items
|
54
63
|
def garbage_collect(persist_category, max_age_seconds=nil)
|
55
|
-
garbage_files =
|
64
|
+
garbage_files = current_files(persist_category)
|
56
65
|
if !max_age_seconds.nil?
|
57
66
|
current_time = Time.now
|
58
67
|
garbage_files.select! { |filepath| (current_time - File.stat(filepath).mtime).to_i > max_age_seconds}
|
@@ -61,9 +70,18 @@ module Aspera
|
|
61
70
|
File.delete(filepath)
|
62
71
|
Log.log.debug{"persistency deleted expired: #{filepath}"}
|
63
72
|
end
|
73
|
+
@cache.clear
|
64
74
|
return garbage_files
|
65
75
|
end
|
66
76
|
|
77
|
+
def current_files(persist_category)
|
78
|
+
Dir[File.join(@folder, persist_category + '*' + FILE_SUFFIX)]
|
79
|
+
end
|
80
|
+
|
81
|
+
def current_items(persist_category)
|
82
|
+
current_files(persist_category).each_with_object({}) {|i, h| h[File.basename(i, FILE_SUFFIX)] = File.read(i)}
|
83
|
+
end
|
84
|
+
|
67
85
|
private
|
68
86
|
|
69
87
|
# @param object_id String or Array
|