aspera-cli 4.20.0 → 4.21.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/CHANGELOG.md +29 -3
- data/CONTRIBUTING.md +2 -0
- data/README.md +571 -375
- data/bin/asession +2 -2
- data/examples/get_proto_file.rb +1 -1
- data/lib/aspera/agent/alpha.rb +10 -16
- data/lib/aspera/agent/connect.rb +20 -2
- data/lib/aspera/agent/direct.rb +21 -30
- data/lib/aspera/agent/node.rb +1 -11
- data/lib/aspera/agent/{trsdk.rb → transferd.rb} +13 -34
- data/lib/aspera/api/aoc.rb +13 -8
- data/lib/aspera/api/node.rb +45 -28
- data/lib/aspera/ascp/installation.rb +87 -48
- data/lib/aspera/ascp/management.rb +27 -6
- data/lib/aspera/cli/formatter.rb +148 -154
- data/lib/aspera/cli/info.rb +1 -1
- data/lib/aspera/cli/main.rb +12 -0
- data/lib/aspera/cli/manager.rb +2 -2
- data/lib/aspera/cli/plugin.rb +2 -2
- data/lib/aspera/cli/plugins/aoc.rb +28 -18
- data/lib/aspera/cli/plugins/config.rb +106 -54
- data/lib/aspera/cli/plugins/cos.rb +1 -0
- data/lib/aspera/cli/plugins/faspex.rb +4 -2
- data/lib/aspera/cli/plugins/faspex5.rb +21 -9
- data/lib/aspera/cli/plugins/node.rb +45 -38
- data/lib/aspera/cli/transfer_progress.rb +2 -0
- data/lib/aspera/cli/version.rb +1 -1
- data/lib/aspera/command_line_builder.rb +1 -1
- data/lib/aspera/environment.rb +48 -14
- data/lib/aspera/node_simulator.rb +230 -112
- data/lib/aspera/oauth/base.rb +34 -47
- data/lib/aspera/oauth/factory.rb +41 -2
- data/lib/aspera/oauth/jwt.rb +4 -1
- data/lib/aspera/persistency_action_once.rb +1 -1
- data/lib/aspera/persistency_folder.rb +20 -2
- data/lib/aspera/preview/generator.rb +1 -1
- data/lib/aspera/preview/utils.rb +8 -3
- data/lib/aspera/products/alpha.rb +30 -0
- data/lib/aspera/products/connect.rb +48 -0
- data/lib/aspera/products/other.rb +82 -0
- data/lib/aspera/products/transferd.rb +54 -0
- data/lib/aspera/rest.rb +18 -13
- data/lib/aspera/secret_hider.rb +2 -2
- data/lib/aspera/ssh.rb +31 -24
- data/lib/aspera/transfer/parameters.rb +2 -1
- data/lib/aspera/transfer/spec.yaml +22 -20
- data/lib/aspera/transfer/sync.rb +1 -5
- data/lib/aspera/transfer/uri.rb +2 -2
- data/lib/transferd_pb.rb +86 -0
- data/lib/transferd_services_pb.rb +84 -0
- data.tar.gz.sig +0 -0
- metadata +38 -21
- metadata.gz.sig +0 -0
- data/lib/aspera/ascp/products.rb +0 -168
- data/lib/transfer_pb.rb +0 -84
- data/lib/transfer_services_pb.rb +0 -82
data/lib/aspera/cli/version.rb
CHANGED
@@ -77,7 +77,7 @@ module Aspera
|
|
77
77
|
@param_hash.each_pair{|key, val|Log.log.warn{"unrecognized parameter: #{key} = \"#{val}\""} if !@used_param_names.include?(key)}
|
78
78
|
# set result
|
79
79
|
env_args[:env].merge!(@result[:env])
|
80
|
-
env_args[:args].
|
80
|
+
env_args[:args].concat(@result[:args])
|
81
81
|
return nil
|
82
82
|
end
|
83
83
|
|
data/lib/aspera/environment.rb
CHANGED
@@ -46,8 +46,7 @@ module Aspera
|
|
46
46
|
return OS_LINUX
|
47
47
|
when /aix/
|
48
48
|
return OS_AIX
|
49
|
-
else
|
50
|
-
raise "Unknown OS: #{RbConfig::CONFIG['host_os']}"
|
49
|
+
else Aspera.error_unexpected_value(RbConfig::CONFIG['host_os']){'host_os'}
|
51
50
|
end
|
52
51
|
end
|
53
52
|
|
@@ -62,8 +61,8 @@ module Aspera
|
|
62
61
|
return CPU_S390
|
63
62
|
when /arm/, /aarch64/
|
64
63
|
return CPU_ARM64
|
64
|
+
else Aspera.error_unexpected_value(RbConfig::CONFIG['host_cpu']){'host_cpu'}
|
65
65
|
end
|
66
|
-
raise "Unknown CPU: #{RbConfig::CONFIG['host_cpu']}"
|
67
66
|
end
|
68
67
|
|
69
68
|
# normalized architecture name
|
@@ -73,9 +72,9 @@ module Aspera
|
|
73
72
|
end
|
74
73
|
|
75
74
|
# executable file extension for current OS
|
76
|
-
def
|
77
|
-
return
|
78
|
-
return
|
75
|
+
def exe_file(name='')
|
76
|
+
return "#{name}.exe" if os.eql?(OS_WINDOWS)
|
77
|
+
return name
|
79
78
|
end
|
80
79
|
|
81
80
|
# on Windows, the env var %USERPROFILE% provides the path to user's home more reliably than %HOMEDRIVE%%HOMEPATH%
|
@@ -96,25 +95,60 @@ module Aspera
|
|
96
95
|
Kernel.send('lave'.reverse, code, empty_binding, file, line)
|
97
96
|
end
|
98
97
|
|
99
|
-
|
98
|
+
# Generate log line for external program with arguments
|
99
|
+
# @param env [Hash, nil] environment variables
|
100
|
+
# @param exec [String] path to executable
|
101
|
+
# @param args [Array, nil] arguments
|
102
|
+
# @return [String] log line with environment, program and arguments
|
103
|
+
def log_spawn(exec:, args: nil, env: nil)
|
100
104
|
[
|
101
105
|
'execute:'.red,
|
102
|
-
env
|
106
|
+
env&.map{|k, v| "#{k}=#{Shellwords.shellescape(v)}"},
|
103
107
|
Shellwords.shellescape(exec),
|
104
|
-
args
|
105
|
-
].flatten.join(' ')
|
108
|
+
args&.map{|a|Shellwords.shellescape(a)}
|
109
|
+
].compact.flatten.join(' ')
|
106
110
|
end
|
107
111
|
|
108
112
|
# start process in background, or raise exception
|
109
113
|
# caller can call Process.wait on returned value
|
110
|
-
|
111
|
-
|
114
|
+
# @param env [Hash, nil] environment variables
|
115
|
+
# @param exec [String] path to executable
|
116
|
+
# @param args [Array, nil] arguments
|
117
|
+
# @return [String] PID of process
|
118
|
+
def secure_spawn(exec:, args: nil, env: nil)
|
119
|
+
Aspera.assert_type(exec, String)
|
120
|
+
Aspera.assert_type(args, Array) unless args.nil?
|
121
|
+
Aspera.assert_type(env, Hash) unless env.nil?
|
122
|
+
Log.log.debug {log_spawn(exec: exec, args: args, env: env)}
|
112
123
|
# start ascp in separate process
|
113
|
-
|
124
|
+
spawn_args = []
|
125
|
+
spawn_args.push(env) unless env.nil?
|
126
|
+
spawn_args.push([exec, exec])
|
127
|
+
spawn_args.concat(args) unless args.nil?
|
128
|
+
ascp_pid = Process.spawn(*spawn_args, close_others: true)
|
114
129
|
Log.log.debug{"pid: #{ascp_pid}"}
|
115
130
|
return ascp_pid
|
116
131
|
end
|
117
132
|
|
133
|
+
# start process and wait for completion
|
134
|
+
# @param env [Hash, nil] environment variables
|
135
|
+
# @param exec [String] path to executable
|
136
|
+
# @param args [Array, nil] arguments
|
137
|
+
# @return [String] PID of process
|
138
|
+
def secure_execute(exec:, args: nil, env: nil)
|
139
|
+
Aspera.assert_type(exec, String)
|
140
|
+
Aspera.assert_type(args, Array) unless args.nil?
|
141
|
+
Aspera.assert_type(env, Hash) unless env.nil?
|
142
|
+
Log.log.debug {log_spawn(exec: exec, args: args, env: env)}
|
143
|
+
# start ascp in separate process
|
144
|
+
spawn_args = []
|
145
|
+
spawn_args.push(env) unless env.nil?
|
146
|
+
spawn_args.push([exec, exec])
|
147
|
+
spawn_args.concat(args) unless args.nil?
|
148
|
+
Kernel.system(*spawn_args, exception: true)
|
149
|
+
nil
|
150
|
+
end
|
151
|
+
|
118
152
|
# @param exec [String] path to executable
|
119
153
|
# @param args [Array] arguments to executable
|
120
154
|
# @param opts [Hash] options to capture3
|
@@ -123,7 +157,7 @@ module Aspera
|
|
123
157
|
Aspera.assert_type(exec, String)
|
124
158
|
Aspera.assert_type(args, Array)
|
125
159
|
Aspera.assert_type(opts, Hash)
|
126
|
-
Log.log.debug {log_spawn(
|
160
|
+
Log.log.debug {log_spawn(exec: exec, args: args)}
|
127
161
|
stdout, stderr, status = Open3.capture3(exec, *args, **opts)
|
128
162
|
Log.log.debug{"status=#{status}, stderr=#{stderr}"}
|
129
163
|
Log.log.trace1{"stdout=#{stdout}"}
|
@@ -7,28 +7,241 @@ require 'webrick'
|
|
7
7
|
require 'json'
|
8
8
|
|
9
9
|
module Aspera
|
10
|
+
class NodeSimulator
|
11
|
+
def initialize
|
12
|
+
@agent = Agent::Direct.new(management_cb: ->(event){process_event(event)})
|
13
|
+
@sessions = {}
|
14
|
+
end
|
15
|
+
|
16
|
+
def start(ts)
|
17
|
+
@agent.start_transfer(ts)
|
18
|
+
end
|
19
|
+
|
20
|
+
def all_sessions
|
21
|
+
@agent.sessions.map { |session| session[:job_id] }.uniq.each.map{|job_id|job_to_transfer(job_id)}
|
22
|
+
end
|
23
|
+
|
24
|
+
# status: ('waiting', 'partially_completed', 'unknown', 'waiting(read error)',] 'running', 'completed', 'failed'
|
25
|
+
def job_to_transfer(job_id)
|
26
|
+
jobs = @agent.sessions_by_job(job_id)
|
27
|
+
ts = nil
|
28
|
+
sessions = jobs.map do |j|
|
29
|
+
ts ||= j[:ts]
|
30
|
+
{
|
31
|
+
id: j[:id],
|
32
|
+
client_node_id: '',
|
33
|
+
server_node_id: '2bbdcc39-f789-4d47-8163-6767fc14f421',
|
34
|
+
client_ip_address: '192.168.0.100',
|
35
|
+
server_ip_address: '5.10.114.4',
|
36
|
+
status: 'running',
|
37
|
+
retry_timeout: 3600,
|
38
|
+
retry_count: 0,
|
39
|
+
start_time_usec: 1701094040000000,
|
40
|
+
end_time_usec: nil,
|
41
|
+
elapsed_usec: 405312,
|
42
|
+
bytes_transferred: 26,
|
43
|
+
bytes_written: 26,
|
44
|
+
bytes_lost: 0,
|
45
|
+
files_completed: 1,
|
46
|
+
directories_completed: 0,
|
47
|
+
target_rate_kbps: 500000,
|
48
|
+
min_rate_kbps: 0,
|
49
|
+
calc_rate_kbps: 9900,
|
50
|
+
network_delay_usec: 40000,
|
51
|
+
avg_rate_kbps: 0.51,
|
52
|
+
error_code: 0,
|
53
|
+
error_desc: '',
|
54
|
+
source_statistics: {
|
55
|
+
args_scan_attempted: 1,
|
56
|
+
args_scan_completed: 1,
|
57
|
+
paths_scan_attempted: 1,
|
58
|
+
paths_scan_failed: 0,
|
59
|
+
paths_scan_skipped: 0,
|
60
|
+
paths_scan_excluded: 0,
|
61
|
+
dirs_scan_completed: 0,
|
62
|
+
files_scan_completed: 1,
|
63
|
+
dirs_xfer_attempted: 0,
|
64
|
+
dirs_xfer_fail: 0,
|
65
|
+
files_xfer_attempted: 1,
|
66
|
+
files_xfer_fail: 0,
|
67
|
+
files_xfer_noxfer: 0
|
68
|
+
},
|
69
|
+
precalc: {
|
70
|
+
enabled: true,
|
71
|
+
status: 'ready',
|
72
|
+
bytes_expected: 0,
|
73
|
+
directories_expected: 0,
|
74
|
+
files_expected: 0,
|
75
|
+
files_excluded: 0,
|
76
|
+
files_special: 0,
|
77
|
+
files_failed: 1
|
78
|
+
}
|
79
|
+
}
|
80
|
+
end
|
81
|
+
ts ||= {}
|
82
|
+
result = {
|
83
|
+
id: job_id,
|
84
|
+
status: 'running',
|
85
|
+
start_spec: ts,
|
86
|
+
sessions: sessions,
|
87
|
+
bytes_transferred: 26,
|
88
|
+
bytes_written: 26,
|
89
|
+
bytes_lost: 0,
|
90
|
+
avg_rate_kbps: 0.51,
|
91
|
+
files_completed: 1,
|
92
|
+
files_skipped: 0,
|
93
|
+
directories_completed: 0,
|
94
|
+
start_time_usec: 1701094040000000,
|
95
|
+
end_time_usec: 1701094040405312,
|
96
|
+
elapsed_usec: 405312,
|
97
|
+
error_code: 0,
|
98
|
+
error_desc: '',
|
99
|
+
precalc: {
|
100
|
+
status: 'ready',
|
101
|
+
bytes_expected: 0,
|
102
|
+
files_expected: 0,
|
103
|
+
directories_expected: 0,
|
104
|
+
files_special: 0,
|
105
|
+
files_failed: 1
|
106
|
+
},
|
107
|
+
files: [{
|
108
|
+
id: 'd1b5c112-82b75425-860745fc-93851671-64541bdd',
|
109
|
+
path: '/workspaces/45071/packages/bYA_ilq73g.asp-package/contents/data_file.bin',
|
110
|
+
start_time_usec: 1701094040000000,
|
111
|
+
elapsed_usec: 105616,
|
112
|
+
end_time_usec: 1701094040001355,
|
113
|
+
status: 'completed',
|
114
|
+
error_code: 0,
|
115
|
+
error_desc: '',
|
116
|
+
size: 26,
|
117
|
+
type: 'file',
|
118
|
+
checksum_type: 'none',
|
119
|
+
checksum: nil,
|
120
|
+
start_byte: 0,
|
121
|
+
bytes_written: 26,
|
122
|
+
session_id: 'bafc72b8-366c-4501-8095-47208183d6b8'}]
|
123
|
+
}
|
124
|
+
Log.log.trace2{Log.dump(:job, result)}
|
125
|
+
return result
|
126
|
+
end
|
127
|
+
|
128
|
+
# Process event from management port
|
129
|
+
def process_event(event)
|
130
|
+
case event['Type']
|
131
|
+
when 'NOP' then Aspera.Log.debug{"event not managed: #{event['Type']}"}
|
132
|
+
# rubocop:disable Lint/DuplicateBranch
|
133
|
+
when 'START' then Aspera.Log.debug{"event not managed: #{event['Type']}"}
|
134
|
+
when 'QUERY' then Aspera.Log.debug{"event not managed: #{event['Type']}"}
|
135
|
+
when 'QUERYRSP' then Aspera.Log.debug{"event not managed: #{event['Type']}"}
|
136
|
+
when 'STATS' then Aspera.Log.debug{"event not managed: #{event['Type']}"}
|
137
|
+
when 'STOP' then Aspera.Log.debug{"event not managed: #{event['Type']}"}
|
138
|
+
when 'ERROR' then Aspera.Log.debug{"event not managed: #{event['Type']}"}
|
139
|
+
when 'CANCEL' then Aspera.Log.debug{"event not managed: #{event['Type']}"}
|
140
|
+
when 'DONE' then Aspera.Log.debug{"event not managed: #{event['Type']}"}
|
141
|
+
when 'RATE' then Aspera.Log.debug{"event not managed: #{event['Type']}"}
|
142
|
+
when 'FILEERROR' then Aspera.Log.debug{"event not managed: #{event['Type']}"}
|
143
|
+
when 'SESSION' then Aspera.Log.debug{"event not managed: #{event['Type']}"}
|
144
|
+
when 'NOTIFICATION' then Aspera.Log.debug{"event not managed: #{event['Type']}"}
|
145
|
+
when 'INIT' then Aspera.Log.debug{"event not managed: #{event['Type']}"}
|
146
|
+
when 'VLINK' then Aspera.Log.debug{"event not managed: #{event['Type']}"}
|
147
|
+
when 'PUT' then Aspera.Log.debug{"event not managed: #{event['Type']}"}
|
148
|
+
when 'WRITE' then Aspera.Log.debug{"event not managed: #{event['Type']}"}
|
149
|
+
when 'CLOSE' then Aspera.Log.debug{"event not managed: #{event['Type']}"}
|
150
|
+
when 'SKIP' then Aspera.Log.debug{"event not managed: #{event['Type']}"}
|
151
|
+
when 'ARGSTOP' then Aspera.Log.debug{"event not managed: #{event['Type']}"}
|
152
|
+
# rubocop:enable Lint/DuplicateBranch
|
153
|
+
else Aspera.error_unreachable_line
|
154
|
+
end
|
155
|
+
end
|
156
|
+
end
|
157
|
+
|
10
158
|
# this class answers the Faspex /send API and creates a package on Aspera on Cloud
|
159
|
+
# a new instance is created for each request
|
11
160
|
class NodeSimulatorServlet < WEBrick::HTTPServlet::AbstractServlet
|
12
161
|
PATH_TRANSFERS = '/ops/transfers'
|
13
162
|
PATH_ONE_TRANSFER = %r{/ops/transfers/(.+)$}
|
163
|
+
PATH_BROWSE = '/files/browse'
|
14
164
|
# @param app_api [Api::AoC]
|
15
165
|
# @param app_context [String]
|
16
|
-
def initialize(server, credentials,
|
166
|
+
def initialize(server, credentials, simulator)
|
17
167
|
super(server)
|
18
168
|
@credentials = credentials
|
19
|
-
@
|
169
|
+
@simulator = simulator
|
170
|
+
end
|
171
|
+
|
172
|
+
require 'json'
|
173
|
+
require 'time'
|
174
|
+
|
175
|
+
def folder_to_structure(folder_path)
|
176
|
+
raise "Path does not exist or is not a directory: #{folder_path}" unless Dir.exist?(folder_path)
|
177
|
+
|
178
|
+
# Build self structure
|
179
|
+
folder_stat = File.stat(folder_path)
|
180
|
+
structure = {
|
181
|
+
'self' => {
|
182
|
+
'path' => folder_path,
|
183
|
+
'basename' => File.basename(folder_path),
|
184
|
+
'type' => 'directory',
|
185
|
+
'size' => folder_stat.size,
|
186
|
+
'mtime' => folder_stat.mtime.utc.iso8601,
|
187
|
+
'permissions' => [
|
188
|
+
{ 'name' => 'view' },
|
189
|
+
{ 'name' => 'edit' },
|
190
|
+
{ 'name' => 'delete' }
|
191
|
+
]
|
192
|
+
},
|
193
|
+
'items' => []
|
194
|
+
}
|
195
|
+
|
196
|
+
# Iterate over folder contents
|
197
|
+
Dir.foreach(folder_path) do |entry|
|
198
|
+
next if entry == '.' || entry == '..' # Skip current and parent directory
|
199
|
+
|
200
|
+
item_path = File.join(folder_path, entry)
|
201
|
+
item_type = File.ftype(item_path) rescue 'unknown' # Get the type of file
|
202
|
+
item_stat = File.lstat(item_path) # Use lstat to handle symbolic links correctly
|
203
|
+
|
204
|
+
item = {
|
205
|
+
'path' => item_path,
|
206
|
+
'basename' => entry,
|
207
|
+
'type' => item_type,
|
208
|
+
'size' => item_stat.size,
|
209
|
+
'mtime' => item_stat.mtime.utc.iso8601,
|
210
|
+
'permissions' => [
|
211
|
+
{ 'name' => 'view' },
|
212
|
+
{ 'name' => 'edit' },
|
213
|
+
{ 'name' => 'delete' }
|
214
|
+
]
|
215
|
+
}
|
216
|
+
|
217
|
+
# Add additional details for specific types
|
218
|
+
case item_type
|
219
|
+
when 'file'
|
220
|
+
item['partial_file'] = false
|
221
|
+
when 'link'
|
222
|
+
item['target'] = File.readlink(item_path) rescue nil # Add the target of the symlink
|
223
|
+
when 'unknown'
|
224
|
+
item['note'] = 'File type could not be determined'
|
225
|
+
end
|
226
|
+
|
227
|
+
structure['items'] << item
|
228
|
+
end
|
229
|
+
|
230
|
+
structure
|
20
231
|
end
|
21
232
|
|
22
233
|
def do_POST(request, response)
|
23
234
|
case request.path
|
24
235
|
when PATH_TRANSFERS
|
25
|
-
job_id = @
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
236
|
+
job_id = @simulator.start(JSON.parse(request.body))
|
237
|
+
sleep(0.5)
|
238
|
+
set_json_response(request, response, @simulator.job_to_transfer(job_id))
|
239
|
+
when PATH_BROWSE
|
240
|
+
req = JSON.parse(request.body)
|
241
|
+
# req['count']
|
242
|
+
set_json_response(request, response, folder_to_structure(req['path']))
|
30
243
|
else
|
31
|
-
set_json_response(response, [{error: 'Bad request'}], code: 400)
|
244
|
+
set_json_response(request, response, [{error: 'Bad request'}], code: 400)
|
32
245
|
end
|
33
246
|
end
|
34
247
|
|
@@ -36,7 +249,7 @@ module Aspera
|
|
36
249
|
case request.path
|
37
250
|
when '/info'
|
38
251
|
info = Ascp::Installation.instance.ascp_info
|
39
|
-
set_json_response(response, {
|
252
|
+
set_json_response(request, response, {
|
40
253
|
application: 'node',
|
41
254
|
current_time: Time.now.utc.iso8601(0),
|
42
255
|
version: info['sdk_ascp_version'].gsub(/ .*$/, ''),
|
@@ -44,13 +257,13 @@ module Aspera
|
|
44
257
|
license_max_rate: info['maximum_bandwidth'],
|
45
258
|
os: %x(uname -srv).chomp,
|
46
259
|
aej_status: 'disconnected',
|
47
|
-
async_reporting: '
|
48
|
-
transfer_activity_reporting: '
|
260
|
+
async_reporting: 'no',
|
261
|
+
transfer_activity_reporting: 'no',
|
49
262
|
transfer_user: 'xfer',
|
50
263
|
docroot: 'file:////data/aoc/eudemo-sedemo',
|
51
264
|
node_id: '2bbdcc39-f789-4d47-8163-6767fc14f421',
|
52
265
|
cluster_id: '6dae2844-d1a9-47a5-916d-9b3eac3ea466',
|
53
|
-
acls: [],
|
266
|
+
acls: ['impersonation'],
|
54
267
|
access_key_configuration_capabilities: {
|
55
268
|
transfer: %w[
|
56
269
|
cipher
|
@@ -99,115 +312,20 @@ module Aspera
|
|
99
312
|
{name: 'wss_port', value: 443}
|
100
313
|
]})
|
101
314
|
when PATH_TRANSFERS
|
102
|
-
|
103
|
-
set_json_response(response, result)
|
315
|
+
set_json_response(request, response, @simulator.all_sessions)
|
104
316
|
when PATH_ONE_TRANSFER
|
105
317
|
job_id = request.path.match(PATH_ONE_TRANSFER)[1]
|
106
|
-
set_json_response(response,
|
318
|
+
set_json_response(request, response, @simulator.job_to_transfer(job_id))
|
107
319
|
else
|
108
|
-
set_json_response(response, [{error: 'Unknown request'}], code: 400)
|
320
|
+
set_json_response(request, response, [{error: 'Unknown request'}], code: 400)
|
109
321
|
end
|
110
322
|
end
|
111
323
|
|
112
|
-
def set_json_response(response, json, code: 200)
|
324
|
+
def set_json_response(request, response, json, code: 200)
|
113
325
|
response.status = code
|
114
326
|
response['Content-Type'] = 'application/json'
|
115
327
|
response.body = json.to_json
|
116
|
-
Log.log.trace1{Log.dump(
|
117
|
-
end
|
118
|
-
|
119
|
-
def job_to_transfer(job)
|
120
|
-
session = {
|
121
|
-
id: 'bafc72b8-366c-4501-8095-47208183d6b8',
|
122
|
-
client_node_id: '',
|
123
|
-
server_node_id: '2bbdcc39-f789-4d47-8163-6767fc14f421',
|
124
|
-
client_ip_address: '192.168.0.100',
|
125
|
-
server_ip_address: '5.10.114.4',
|
126
|
-
status: 'running',
|
127
|
-
retry_timeout: 3600,
|
128
|
-
retry_count: 0,
|
129
|
-
start_time_usec: 1701094040000000,
|
130
|
-
end_time_usec: nil,
|
131
|
-
elapsed_usec: 405312,
|
132
|
-
bytes_transferred: 26,
|
133
|
-
bytes_written: 26,
|
134
|
-
bytes_lost: 0,
|
135
|
-
files_completed: 1,
|
136
|
-
directories_completed: 0,
|
137
|
-
target_rate_kbps: 500000,
|
138
|
-
min_rate_kbps: 0,
|
139
|
-
calc_rate_kbps: 9900,
|
140
|
-
network_delay_usec: 40000,
|
141
|
-
avg_rate_kbps: 0.51,
|
142
|
-
error_code: 0,
|
143
|
-
error_desc: '',
|
144
|
-
source_statistics: {
|
145
|
-
args_scan_attempted: 1,
|
146
|
-
args_scan_completed: 1,
|
147
|
-
paths_scan_attempted: 1,
|
148
|
-
paths_scan_failed: 0,
|
149
|
-
paths_scan_skipped: 0,
|
150
|
-
paths_scan_excluded: 0,
|
151
|
-
dirs_scan_completed: 0,
|
152
|
-
files_scan_completed: 1,
|
153
|
-
dirs_xfer_attempted: 0,
|
154
|
-
dirs_xfer_fail: 0,
|
155
|
-
files_xfer_attempted: 1,
|
156
|
-
files_xfer_fail: 0,
|
157
|
-
files_xfer_noxfer: 0
|
158
|
-
},
|
159
|
-
precalc: {
|
160
|
-
enabled: true,
|
161
|
-
status: 'ready',
|
162
|
-
bytes_expected: 0,
|
163
|
-
directories_expected: 0,
|
164
|
-
files_expected: 0,
|
165
|
-
files_excluded: 0,
|
166
|
-
files_special: 0,
|
167
|
-
files_failed: 1
|
168
|
-
}}
|
169
|
-
return {
|
170
|
-
id: '609a667d-642e-4290-9312-b4d20d3c0159',
|
171
|
-
status: 'running',
|
172
|
-
start_spec: job[:ts],
|
173
|
-
sessions: [session],
|
174
|
-
bytes_transferred: 26,
|
175
|
-
bytes_written: 26,
|
176
|
-
bytes_lost: 0,
|
177
|
-
avg_rate_kbps: 0.51,
|
178
|
-
files_completed: 1,
|
179
|
-
files_skipped: 0,
|
180
|
-
directories_completed: 0,
|
181
|
-
start_time_usec: 1701094040000000,
|
182
|
-
end_time_usec: 1701094040405312,
|
183
|
-
elapsed_usec: 405312,
|
184
|
-
error_code: 0,
|
185
|
-
error_desc: '',
|
186
|
-
precalc: {
|
187
|
-
status: 'ready',
|
188
|
-
bytes_expected: 0,
|
189
|
-
files_expected: 0,
|
190
|
-
directories_expected: 0,
|
191
|
-
files_special: 0,
|
192
|
-
files_failed: 1
|
193
|
-
},
|
194
|
-
files: [{
|
195
|
-
id: 'd1b5c112-82b75425-860745fc-93851671-64541bdd',
|
196
|
-
path: '/workspaces/45071/packages/bYA_ilq73g.asp-package/contents/data_file.bin',
|
197
|
-
start_time_usec: 1701094040000000,
|
198
|
-
elapsed_usec: 105616,
|
199
|
-
end_time_usec: 1701094040001355,
|
200
|
-
status: 'completed',
|
201
|
-
error_code: 0,
|
202
|
-
error_desc: '',
|
203
|
-
size: 26,
|
204
|
-
type: 'file',
|
205
|
-
checksum_type: 'none',
|
206
|
-
checksum: nil,
|
207
|
-
start_byte: 0,
|
208
|
-
bytes_written: 26,
|
209
|
-
session_id: 'bafc72b8-366c-4501-8095-47208183d6b8'}]
|
210
|
-
}
|
328
|
+
Log.log.trace1{Log.dump("response for #{request.request_method} #{request.path}", json)}
|
211
329
|
end
|
212
330
|
end
|
213
331
|
end
|
data/lib/aspera/oauth/base.rb
CHANGED
@@ -26,7 +26,7 @@ module Aspera
|
|
26
26
|
scope: nil,
|
27
27
|
use_query: false,
|
28
28
|
path_token: 'token',
|
29
|
-
token_field:
|
29
|
+
token_field: Factory::TOKEN_FIELD,
|
30
30
|
cache_ids: nil,
|
31
31
|
**rest_params
|
32
32
|
)
|
@@ -38,12 +38,10 @@ module Aspera
|
|
38
38
|
@client_id = client_id
|
39
39
|
@client_secret = client_secret
|
40
40
|
@use_query = use_query
|
41
|
-
@base_cache_ids = cache_ids.clone
|
42
|
-
@base_cache_ids = [] if @base_cache_ids.nil?
|
41
|
+
@base_cache_ids = cache_ids.nil? ? [] : cache_ids.clone
|
43
42
|
Aspera.assert_type(@base_cache_ids, Array)
|
44
|
-
if @api.auth_params.key?(:username)
|
45
|
-
|
46
|
-
end
|
43
|
+
@base_cache_ids.push(@api.auth_params[:username]) if @api.auth_params.key?(:username)
|
44
|
+
@base_cache_ids.compact!
|
47
45
|
@base_cache_ids.freeze
|
48
46
|
self.scope = scope
|
49
47
|
end
|
@@ -87,48 +85,37 @@ module Aspera
|
|
87
85
|
# @param cache set to false to disable cache
|
88
86
|
# @param refresh set to true to force refresh or re-generation (if previous failed)
|
89
87
|
def token(cache: true, refresh: false)
|
90
|
-
# get
|
91
|
-
|
92
|
-
token_data =
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
88
|
+
# get token info from cache (or nil), decoded with date and expiration status
|
89
|
+
token_info = Factory.instance.get_token_info(@token_cache_id) if cache
|
90
|
+
token_data = nil
|
91
|
+
unless token_info.nil?
|
92
|
+
token_data = token_info[:data]
|
93
|
+
# Optional optimization:
|
94
|
+
# check if node token is expired based on decoded content then force refresh if close enough
|
95
|
+
# might help in case the transfer agent cannot refresh himself
|
96
|
+
# `direct` agent is equipped with refresh code
|
97
|
+
# an API was already called, but failed, we need to regenerate or refresh
|
98
|
+
if refresh || token_info[:expired]
|
99
|
+
if token_data.key?('refresh_token') && token_data['refresh_token'].eql?('not_supported')
|
100
|
+
# save possible refresh token, before deleting the cache
|
101
|
+
refresh_token = token_data['refresh_token']
|
102
|
+
end
|
103
|
+
# delete cache
|
104
|
+
Factory.instance.persist_mgr.delete(@token_cache_id)
|
105
|
+
token_data = nil
|
106
|
+
# lets try the existing refresh token
|
107
|
+
if !refresh_token.nil?
|
108
|
+
Log.log.info{"refresh=[#{refresh_token}]".bg_green}
|
109
|
+
# NOTE: AoC admin token has no refresh, and lives by default 1800secs
|
110
|
+
resp = create_token_call(optional_scope_client_id.merge(grant_type: 'refresh_token', refresh_token: refresh_token))
|
111
|
+
if resp[:http].code.start_with?('2')
|
112
|
+
# save only if success
|
113
|
+
json_data = resp[:http].body
|
114
|
+
token_data = JSON.parse(json_data)
|
115
|
+
Factory.instance.persist_mgr.put(@token_cache_id, json_data)
|
116
|
+
else
|
117
|
+
Log.log.debug{"refresh failed: #{resp[:http].body}".bg_red}
|
103
118
|
end
|
104
|
-
# force refresh if we see a token too close from expiration
|
105
|
-
refresh = true if expires_at_sec.is_a?(Time) && (expires_at_sec - Time.now) < OAuth::Factory.instance.parameters[:token_expiration_guard_sec]
|
106
|
-
Log.log.debug{"Expiration: #{expires_at_sec} / #{refresh}"}
|
107
|
-
end
|
108
|
-
end
|
109
|
-
|
110
|
-
# an API was already called, but failed, we need to regenerate or refresh
|
111
|
-
if refresh
|
112
|
-
if token_data.is_a?(Hash) && token_data.key?('refresh_token') && !token_data['refresh_token'].eql?('not_supported')
|
113
|
-
# save possible refresh token, before deleting the cache
|
114
|
-
refresh_token = token_data['refresh_token']
|
115
|
-
end
|
116
|
-
# delete cache
|
117
|
-
Factory.instance.persist_mgr.delete(@token_cache_id)
|
118
|
-
token_data = nil
|
119
|
-
# lets try the existing refresh token
|
120
|
-
if !refresh_token.nil?
|
121
|
-
Log.log.info{"refresh=[#{refresh_token}]".bg_green}
|
122
|
-
# try to refresh
|
123
|
-
# note: AoC admin token has no refresh, and lives by default 1800secs
|
124
|
-
resp = create_token_call(optional_scope_client_id.merge(grant_type: 'refresh_token', refresh_token: refresh_token))
|
125
|
-
if resp[:http].code.start_with?('2')
|
126
|
-
# save only if success
|
127
|
-
json_data = resp[:http].body
|
128
|
-
token_data = JSON.parse(json_data)
|
129
|
-
Factory.instance.persist_mgr.put(@token_cache_id, json_data)
|
130
|
-
else
|
131
|
-
Log.log.debug{"refresh failed: #{resp[:http].body}".bg_red}
|
132
119
|
end
|
133
120
|
end
|
134
121
|
end
|