aspera-cli 4.27.2 → 4.27.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/CHANGELOG.md +133 -0
- data/CONTRIBUTING.md +5 -2
- data/bin/ascli +3 -1
- data/docs/README.md +1006 -766
- data/lib/aspera/agent/base.rb +7 -2
- data/lib/aspera/agent/connect.rb +6 -8
- data/lib/aspera/agent/desktop.rb +2 -6
- data/lib/aspera/agent/direct.rb +52 -22
- data/lib/aspera/agent/node.rb +9 -8
- data/lib/aspera/agent/transferd.rb +2 -2
- data/lib/aspera/api/alee.rb +1 -1
- data/lib/aspera/api/aoc.rb +14 -12
- data/lib/aspera/api/ats.rb +1 -1
- data/lib/aspera/api/cos_node.rb +2 -2
- data/lib/aspera/api/faspex.rb +11 -7
- data/lib/aspera/api/httpgw.rb +38 -35
- data/lib/aspera/api/node.rb +39 -34
- data/lib/aspera/ascmd.rb +3 -1
- data/lib/aspera/ascp/installation.rb +63 -28
- data/lib/aspera/ascp/management.rb +1 -0
- data/lib/aspera/assert.rb +4 -0
- data/lib/aspera/cli/ascp_actions.rb +20 -41
- data/lib/aspera/cli/async_transfer_store.rb +12 -11
- data/lib/aspera/cli/bootstrapper.rb +14 -16
- data/lib/aspera/cli/command_line.rb +252 -0
- data/lib/aspera/cli/command_registry.rb +215 -37
- data/lib/aspera/cli/command_spec.rb +104 -15
- data/lib/aspera/cli/completion/ascli.bash +12 -0
- data/lib/aspera/cli/completion/ascli.fish +16 -0
- data/lib/aspera/cli/completion/ascli.zsh +19 -0
- data/lib/aspera/cli/context.rb +3 -0
- data/lib/aspera/cli/deprecation.rb +37 -0
- data/lib/aspera/cli/extended_value.rb +6 -3
- data/lib/aspera/cli/formatter.rb +94 -80
- data/lib/aspera/cli/gem_checker.rb +1 -1
- data/lib/aspera/cli/hints.rb +7 -6
- data/lib/aspera/cli/http.rb +22 -34
- data/lib/aspera/cli/info.rb +3 -0
- data/lib/aspera/cli/mcp_tool.rb +47 -83
- data/lib/aspera/cli/option_declarator.rb +33 -42
- data/lib/aspera/cli/option_registry.rb +69 -0
- data/lib/aspera/cli/option_types.rb +105 -0
- data/lib/aspera/cli/option_value.rb +278 -0
- data/lib/aspera/cli/options.schema.yaml +124 -15
- data/lib/aspera/cli/parser.rb +333 -862
- data/lib/aspera/cli/plugins/alee.rb +7 -4
- data/lib/aspera/cli/plugins/aoc.rb +545 -518
- data/lib/aspera/cli/plugins/ats.rb +59 -80
- data/lib/aspera/cli/plugins/base.rb +221 -265
- data/lib/aspera/cli/plugins/basic_auth.rb +2 -10
- data/lib/aspera/cli/plugins/config.rb +263 -184
- data/lib/aspera/cli/plugins/console.rb +103 -39
- data/lib/aspera/cli/plugins/cos.rb +6 -23
- data/lib/aspera/cli/plugins/factory.rb +3 -0
- data/lib/aspera/cli/plugins/faspex5.rb +204 -182
- data/lib/aspera/cli/plugins/faspio.rb +6 -11
- data/lib/aspera/cli/plugins/httpgw.rb +8 -11
- data/lib/aspera/cli/plugins/mcp.rb +20 -55
- data/lib/aspera/cli/plugins/node.rb +300 -327
- data/lib/aspera/cli/plugins/orchestrator.rb +152 -110
- data/lib/aspera/cli/plugins/preview.rb +96 -105
- data/lib/aspera/cli/plugins/server.rb +78 -53
- data/lib/aspera/cli/plugins/shares.rb +80 -131
- data/lib/aspera/cli/preset_actions.rb +44 -27
- data/lib/aspera/cli/preset_manager.rb +44 -19
- data/lib/aspera/cli/prompt.rb +36 -0
- data/lib/aspera/cli/result.rb +42 -36
- data/lib/aspera/cli/runner.rb +32 -59
- data/lib/aspera/cli/special_values.rb +5 -0
- data/lib/aspera/cli/sync_actions.rb +51 -46
- data/lib/aspera/cli/terminal_formatter.rb +9 -3
- data/lib/aspera/cli/transfer_actions.rb +14 -9
- data/lib/aspera/cli/transfer_agent.rb +34 -38
- data/lib/aspera/cli/transfer_progress.rb +290 -55
- data/lib/aspera/cli/vault_manager.rb +0 -17
- data/lib/aspera/cli/version.rb +1 -1
- data/lib/aspera/cli/wizard.rb +5 -3
- data/lib/aspera/coverage.rb +1 -1
- data/lib/aspera/environment.rb +35 -5
- data/lib/aspera/faspex_gw.rb +2 -1
- data/lib/aspera/faspex_postproc.rb +1 -0
- data/lib/aspera/graphql.rb +5 -5
- data/lib/aspera/json_rpc/client.rb +5 -5
- data/lib/aspera/keychain/encrypted_hash.rb +2 -2
- data/lib/aspera/keychain/factory.rb +2 -1
- data/lib/aspera/keychain/one_password_api.rb +1 -1
- data/lib/aspera/link_header.rb +2 -2
- data/lib/aspera/log.rb +47 -27
- data/lib/aspera/markdown.rb +2 -0
- data/lib/aspera/mime.rb +25 -0
- data/lib/aspera/node_emulator.rb +759 -0
- data/lib/aspera/oauth/base.rb +37 -26
- data/lib/aspera/oauth/factory.rb +7 -3
- data/lib/aspera/oauth/generic.rb +1 -1
- data/lib/aspera/oauth/json_credentials.rb +34 -0
- data/lib/aspera/oauth/jwt.rb +4 -5
- data/lib/aspera/oauth/web.rb +9 -8
- data/lib/aspera/oauth.rb +1 -0
- data/lib/aspera/persistency_folder.rb +1 -3
- data/lib/aspera/preview/file_types.rb +4 -4
- data/lib/aspera/preview/generator.rb +11 -1
- data/lib/aspera/preview/options.schema.yaml +119 -0
- data/lib/aspera/preview/terminal.rb +4 -3
- data/lib/aspera/preview/utils.rb +9 -6
- data/lib/aspera/products/connect.rb +1 -1
- data/lib/aspera/rainbow.rb +7 -0
- data/lib/aspera/rest/aspera_errors.rb +72 -0
- data/lib/aspera/rest/call_error.rb +27 -0
- data/lib/aspera/rest/client.rb +523 -0
- data/lib/aspera/rest/error_analyzer.rb +113 -0
- data/lib/aspera/rest/list.rb +149 -0
- data/lib/aspera/rest/parameters.rb +55 -0
- data/lib/aspera/rest/util.rb +176 -0
- data/lib/aspera/rest.rb +7 -621
- data/lib/aspera/schema/IBM Aspera Console-enhanced.yaml +1125 -0
- data/lib/aspera/schema/IBM Aspera Faspex API-5.0-enhanced.yaml +0 -20
- data/lib/aspera/schema/IBM Aspera Orchestrator API-v1.yaml +1784 -0
- data/lib/aspera/schema/IBM Aspera on Cloud API-0.2.6-enhanced.yaml +1230 -137
- data/lib/aspera/schema/IBM Aspera on Cloud Automation API-1.0.5-enhanced.yaml +2395 -0
- data/lib/aspera/schema/IBM_Aspera_Shares.yaml +14 -13
- data/lib/aspera/schema/documentation.rb +13 -3
- data/lib/aspera/schema/reader.rb +12 -18
- data/lib/aspera/schema/registry.rb +23 -1
- data/lib/aspera/schema/validator.rb +92 -0
- data/lib/aspera/secret_hider.rb +36 -25
- data/lib/aspera/string_ext.rb +15 -0
- data/lib/aspera/temp_file_manager.rb +6 -5
- data/lib/aspera/transfer/parameters.rb +2 -0
- data/lib/aspera/transfer/spec.rb +1 -0
- data/lib/aspera/uri_reader.rb +11 -11
- data/lib/aspera/web_auth/index.html +147 -0
- data/lib/aspera/web_auth/server.rb +81 -0
- data.tar.gz.sig +0 -0
- metadata +43 -9
- metadata.gz.sig +0 -0
- data/lib/aspera/colors.rb +0 -79
- data/lib/aspera/node_simulator.rb +0 -344
- data/lib/aspera/preview/options.rb +0 -45
- data/lib/aspera/rest_call_error.rb +0 -25
- data/lib/aspera/rest_error_analyzer.rb +0 -111
- data/lib/aspera/rest_errors_aspera.rb +0 -58
- data/lib/aspera/rest_list.rb +0 -136
- data/lib/aspera/web_auth.rb +0 -211
data/lib/aspera/agent/base.rb
CHANGED
|
@@ -23,14 +23,19 @@ module Aspera
|
|
|
23
23
|
end
|
|
24
24
|
|
|
25
25
|
# Wait for all sessions to terminate and return a typed Transfer::Result.
|
|
26
|
+
# Notifies the end of transfer to the progress bar (agents do not).
|
|
26
27
|
# @return [Transfer::Result::Success, Transfer::Result::Error]
|
|
27
28
|
def wait_for_completion
|
|
29
|
+
success = false
|
|
28
30
|
statuses = wait_for_transfers_completion
|
|
29
|
-
@progress&.reset
|
|
30
31
|
Aspera.assert_type(statuses, Array)
|
|
31
32
|
Aspera.assert(statuses.none? { |i| !i.eql?(:success) && !i.is_a?(StandardError) }) { "bad statuses content: #{statuses}" }
|
|
32
33
|
errors = statuses.reject { |i| i.eql?(:success) }
|
|
33
|
-
|
|
34
|
+
success = errors.empty?
|
|
35
|
+
return success ? Transfer::Result.success : Transfer::Result.error(errors.first)
|
|
36
|
+
ensure
|
|
37
|
+
# Also when the agent raises an exception
|
|
38
|
+
notify_progress(:end, info: success)
|
|
34
39
|
end
|
|
35
40
|
|
|
36
41
|
# Return the job_id of the last transfer submitted by this agent.
|
data/lib/aspera/agent/connect.rb
CHANGED
|
@@ -6,6 +6,8 @@ require 'aspera/products/other'
|
|
|
6
6
|
require 'aspera/rest'
|
|
7
7
|
require 'aspera/environment'
|
|
8
8
|
require 'securerandom'
|
|
9
|
+
require 'aspera/rainbow'
|
|
10
|
+
using Rainbow
|
|
9
11
|
|
|
10
12
|
module Aspera
|
|
11
13
|
module Agent
|
|
@@ -21,9 +23,9 @@ module Aspera
|
|
|
21
23
|
# agent_params must contain 'app_id'; the Connect URL is auto-discovered.
|
|
22
24
|
# @return [Hash] normalized status hash
|
|
23
25
|
def transfer_status(transfer_id, agent_params)
|
|
24
|
-
connect_api = Rest.new(
|
|
26
|
+
connect_api = Rest::Client.new(
|
|
25
27
|
base_url: "#{connect_api_url}/v5/connect",
|
|
26
|
-
headers: {'Origin' =>
|
|
28
|
+
headers: {'Origin' => Rest::Parameters.instance.user_agent}
|
|
27
29
|
)
|
|
28
30
|
tr_info = connect_api.create("transfers/info/#{transfer_id}", {'aspera_connect_settings' => {'app_id' => agent_params['app_id']}})
|
|
29
31
|
transfer = tr_info['transfer_info']
|
|
@@ -72,9 +74,9 @@ module Aspera
|
|
|
72
74
|
# raise exception if connect not started and file does not exist
|
|
73
75
|
connect_url = self.class.connect_api_url
|
|
74
76
|
Log.log.debug { "found: #{connect_url}" }
|
|
75
|
-
@connect_api = Rest.new(
|
|
77
|
+
@connect_api = Rest::Client.new(
|
|
76
78
|
base_url: "#{connect_url}/v5/connect", # could use v6 also now
|
|
77
|
-
headers: {'Origin' =>
|
|
79
|
+
headers: {'Origin' => Rest::Parameters.instance.user_agent}
|
|
78
80
|
)
|
|
79
81
|
connect_info = @connect_api.read('info/version')
|
|
80
82
|
Log.log.debug('Connect was reached') if method_index > 0
|
|
@@ -158,19 +160,15 @@ module Aspera
|
|
|
158
160
|
end
|
|
159
161
|
when 'completed'
|
|
160
162
|
notify_progress(:session_end, session_id: @transfer_id)
|
|
161
|
-
notify_progress(:end)
|
|
162
163
|
break
|
|
163
164
|
when 'failed'
|
|
164
165
|
notify_progress(:session_end, session_id: @transfer_id)
|
|
165
|
-
notify_progress(:end)
|
|
166
166
|
raise Transfer::Error, transfer['error_desc']
|
|
167
167
|
when 'cancelled'
|
|
168
168
|
notify_progress(:session_end, session_id: @transfer_id)
|
|
169
|
-
notify_progress(:end)
|
|
170
169
|
raise Transfer::Error, 'Transfer cancelled by user'
|
|
171
170
|
else
|
|
172
171
|
notify_progress(:session_end, session_id: @transfer_id)
|
|
173
|
-
notify_progress(:end)
|
|
174
172
|
raise Transfer::Error, "unknown status: #{transfer['status']}: #{transfer['error_desc']}"
|
|
175
173
|
end
|
|
176
174
|
end
|
data/lib/aspera/agent/desktop.rb
CHANGED
|
@@ -24,7 +24,7 @@ module Aspera
|
|
|
24
24
|
# @return [Hash] normalized status hash
|
|
25
25
|
def transfer_status(transfer_id, agent_params)
|
|
26
26
|
app_id = agent_params['application_id']
|
|
27
|
-
client = Aspera::JsonRpc::Client.new(Aspera::Rest.new(base_url: desktop_api_url))
|
|
27
|
+
client = Aspera::JsonRpc::Client.new(Aspera::Rest::Client.new(base_url: desktop_api_url))
|
|
28
28
|
raw = client.get_transfer(app_id: app_id, transfer_id: transfer_id)
|
|
29
29
|
normalize_status(raw['status'], bytes: raw['bytes_written'].to_i, error: raw['error_desc'])
|
|
30
30
|
end
|
|
@@ -70,7 +70,7 @@ module Aspera
|
|
|
70
70
|
begin
|
|
71
71
|
# curl 'http://127.0.0.1:33024/' -X POST -H 'content-type: application/json' --data-raw '{"jsonrpc":"2.0","params":[],"id":999999,"method":"rpc.discover"}'
|
|
72
72
|
# https://playground.open-rpc.org/?schemaUrl=http://127.0.0.1:33024
|
|
73
|
-
@client_app_api = Aspera::JsonRpc::Client.new(Aspera::Rest.new(base_url: aspera_client_api_url))
|
|
73
|
+
@client_app_api = Aspera::JsonRpc::Client.new(Aspera::Rest::Client.new(base_url: aspera_client_api_url))
|
|
74
74
|
client_info = @client_app_api.get_info
|
|
75
75
|
Log.dump(:client_version, client_info)
|
|
76
76
|
Log.log.debug('Client was reached') if method_index > 0
|
|
@@ -121,19 +121,15 @@ module Aspera
|
|
|
121
121
|
end
|
|
122
122
|
when 'completed'
|
|
123
123
|
notify_progress(:session_end, session_id: @transfer_id)
|
|
124
|
-
notify_progress(:end)
|
|
125
124
|
break
|
|
126
125
|
when 'failed'
|
|
127
126
|
notify_progress(:session_end, session_id: @transfer_id)
|
|
128
|
-
notify_progress(:end)
|
|
129
127
|
raise Transfer::Error, transfer['error_desc']
|
|
130
128
|
when 'cancelled'
|
|
131
129
|
notify_progress(:session_end, session_id: @transfer_id)
|
|
132
|
-
notify_progress(:end)
|
|
133
130
|
raise Transfer::Error, 'Transfer cancelled by user'
|
|
134
131
|
else
|
|
135
132
|
notify_progress(:session_end, session_id: @transfer_id)
|
|
136
|
-
notify_progress(:end)
|
|
137
133
|
raise Transfer::Error, "unknown status: #{transfer['status']}: #{transfer['error_desc']}"
|
|
138
134
|
end
|
|
139
135
|
sleep(1)
|
data/lib/aspera/agent/direct.rb
CHANGED
|
@@ -14,6 +14,8 @@ require 'socket'
|
|
|
14
14
|
require 'securerandom'
|
|
15
15
|
require 'shellwords'
|
|
16
16
|
require 'English'
|
|
17
|
+
require 'aspera/rainbow'
|
|
18
|
+
using Rainbow
|
|
17
19
|
|
|
18
20
|
module Aspera
|
|
19
21
|
module Agent
|
|
@@ -105,8 +107,6 @@ module Aspera
|
|
|
105
107
|
@sessions = []
|
|
106
108
|
# mutex protects global data accessed by threads
|
|
107
109
|
@mutex = Mutex.new
|
|
108
|
-
@pre_calc_sent = false
|
|
109
|
-
@pre_calc_last_size = nil
|
|
110
110
|
# Check on all management messages if that file exists, and if so, read commands from it
|
|
111
111
|
@command_file = File.join(config_dir || '.', "send_#{$PROCESS_ID}")
|
|
112
112
|
end
|
|
@@ -162,6 +162,11 @@ module Aspera
|
|
|
162
162
|
error: nil, # exception if failed
|
|
163
163
|
io: nil, # management port server socket
|
|
164
164
|
token_regenerator: token_regenerator, # regenerate bearer token with oauth
|
|
165
|
+
progress_id: nil, # session identifier for progress, one per `ascp` process, same on resume
|
|
166
|
+
size_sent: false, # `true` when total size was notified (PreTransferBytes)
|
|
167
|
+
last_size: nil, # last notified transferred size
|
|
168
|
+
skipped: 0, # size of whole files already at destination
|
|
169
|
+
last_skipped: nil, # last notified size not transferred (already at destination)
|
|
165
170
|
# env vars and args for ascp (from transfer spec)
|
|
166
171
|
exec_spec: Transfer::Parameters.new(transfer_spec, **@tr_opts).ascp_args
|
|
167
172
|
}
|
|
@@ -208,7 +213,6 @@ module Aspera
|
|
|
208
213
|
session[:thread].join
|
|
209
214
|
result.push(session[:error] || :success)
|
|
210
215
|
end
|
|
211
|
-
notify_progress(:end)
|
|
212
216
|
Log.log.debug('all transfers joined')
|
|
213
217
|
# since all are finished and we return the result, clear statuses
|
|
214
218
|
@sessions.clear
|
|
@@ -258,7 +262,7 @@ module Aspera
|
|
|
258
262
|
@resume_policy.execute_with_resume do
|
|
259
263
|
start_and_monitor_process(session: session, exec_spec: session[:exec_spec])
|
|
260
264
|
end
|
|
261
|
-
Log.log.debug('transfer ok'.
|
|
265
|
+
Log.log.debug('transfer ok'.bg(:green))
|
|
262
266
|
rescue => e
|
|
263
267
|
session[:error] = e
|
|
264
268
|
raise if Log.log.debug? || !e.is_a?(Transfer::Error)
|
|
@@ -345,7 +349,7 @@ module Aspera
|
|
|
345
349
|
# store session identifier
|
|
346
350
|
session[:id] = event['SessionId'] if event['Type'].eql?('INIT')
|
|
347
351
|
@management_cb&.call(event)
|
|
348
|
-
process_progress(event)
|
|
352
|
+
process_progress(session, event)
|
|
349
353
|
next unless File.exist?(@command_file)
|
|
350
354
|
begin
|
|
351
355
|
commands = JSON.parse(File.read(@command_file))
|
|
@@ -371,7 +375,9 @@ module Aspera
|
|
|
371
375
|
env['ASPERA_SCP_TOKEN'] = session[:token_regenerator].refreshed_transfer_token
|
|
372
376
|
end
|
|
373
377
|
raise Transfer::Error.new(last_event['Description'], code: last_event['Code'].to_i)
|
|
374
|
-
else
|
|
378
|
+
else
|
|
379
|
+
# e.g. process was killed: like a lost connection (code 16), so that transfer is resumed
|
|
380
|
+
raise Transfer::Error.new("#{exec} ended without final status (last event: #{last_event['Type']})", code: 16)
|
|
375
381
|
end
|
|
376
382
|
rescue SystemCallError => e
|
|
377
383
|
# Process.spawn failed, or socket error
|
|
@@ -416,39 +422,63 @@ module Aspera
|
|
|
416
422
|
attr_reader :sessions
|
|
417
423
|
|
|
418
424
|
# Notify progress to callback
|
|
419
|
-
# @param
|
|
420
|
-
|
|
421
|
-
|
|
425
|
+
# @param session [Hash] This session information, progress state is stored in it
|
|
426
|
+
# @param event [Hash] management port event
|
|
427
|
+
def process_progress(session, event)
|
|
428
|
+
# Not the `ascp` session id: must be the same when session is resumed
|
|
429
|
+
progress_id = session[:progress_id] ||= SecureRandom.uuid
|
|
422
430
|
case event['Type']
|
|
423
431
|
when 'INIT'
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
432
|
+
session[:size_sent] = false
|
|
433
|
+
session[:last_size] = nil
|
|
434
|
+
session[:skipped] = 0
|
|
435
|
+
session[:last_skipped] = nil
|
|
436
|
+
notify_progress(:session_start, session_id: progress_id)
|
|
427
437
|
when 'NOTIFICATION' # sent from remote
|
|
428
438
|
if event.key?('PreTransferBytes')
|
|
429
|
-
|
|
430
|
-
notify_progress(:session_size, session_id:
|
|
439
|
+
session[:size_sent] = true
|
|
440
|
+
notify_progress(:session_size, session_id: progress_id, info: event['PreTransferBytes'])
|
|
431
441
|
end
|
|
432
442
|
when 'STATS' # during transfer
|
|
433
|
-
|
|
434
|
-
|
|
443
|
+
notify_transferred(session, event)
|
|
444
|
+
when 'STOP' # one file is completed
|
|
445
|
+
size = event['Size'].to_i
|
|
446
|
+
# Whole file already at destination (resume): not transferred, and not in `FileBytes`.
|
|
447
|
+
# Note: in multi-session, each session notifies all such files: progress bar limits progress to size of session.
|
|
448
|
+
session[:skipped] += size if size.positive? && event['StartByte'].to_i.eql?(size)
|
|
449
|
+
notify_transferred(session, event)
|
|
435
450
|
when 'DONE', 'ERROR' # end of session
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
notify_progress(:
|
|
439
|
-
notify_progress(:session_end, session_id:
|
|
451
|
+
notify_transferred(session, event)
|
|
452
|
+
total_size = session[:last_size] + session[:last_skipped]
|
|
453
|
+
notify_progress(:session_size, session_id: progress_id, info: total_size) if !session[:size_sent] && !total_size.zero?
|
|
454
|
+
notify_progress(:session_end, session_id: progress_id)
|
|
440
455
|
# cspell:disable
|
|
441
456
|
when 'SESSION'
|
|
442
457
|
when 'ARGSTOP'
|
|
443
458
|
when 'FILEERROR'
|
|
444
459
|
Log.log.error { "#{event['Type']} #{event['Description']}" }
|
|
445
|
-
when 'STOP'
|
|
446
460
|
# cspell:enable
|
|
447
|
-
# stop event when one file is completed
|
|
448
461
|
else
|
|
449
462
|
Log.log.debug { "Unknown event type for progress: #{event['Type']}" }
|
|
450
463
|
end
|
|
451
464
|
end
|
|
465
|
+
|
|
466
|
+
# Notify transferred size, and size not transferred because already at destination (resume), if changed
|
|
467
|
+
# @param session [Hash] This session information, progress state is stored in it
|
|
468
|
+
# @param event [Hash] management port event, with `TransferBytes` and `FileBytes`
|
|
469
|
+
def notify_transferred(session, event)
|
|
470
|
+
transferred = event['TransferBytes'].to_i
|
|
471
|
+
# `FileBytes` also includes the part of files already at destination.
|
|
472
|
+
# Note: `StartByte` is not used: in multi-session, it is the offset of the part of file transferred by the session.
|
|
473
|
+
skipped = session[:skipped] + [event['FileBytes'].to_i - transferred, 0].max
|
|
474
|
+
if session[:last_skipped] != skipped
|
|
475
|
+
session[:last_skipped] = skipped
|
|
476
|
+
notify_progress(:skip, session_id: session[:progress_id], info: skipped)
|
|
477
|
+
end
|
|
478
|
+
return if session[:last_size].eql?(transferred)
|
|
479
|
+
session[:last_size] = transferred
|
|
480
|
+
notify_progress(:transfer, session_id: session[:progress_id], info: transferred)
|
|
481
|
+
end
|
|
452
482
|
end
|
|
453
483
|
end
|
|
454
484
|
end
|
data/lib/aspera/agent/node.rb
CHANGED
|
@@ -24,7 +24,7 @@ module Aspera
|
|
|
24
24
|
else
|
|
25
25
|
rest_params[:auth] = {type: :basic, username: agent_params['username'], password: agent_params['password']}
|
|
26
26
|
end
|
|
27
|
-
node_api = Rest.new(**rest_params)
|
|
27
|
+
node_api = Rest::Client.new(**rest_params)
|
|
28
28
|
data = node_api.read("ops/transfers/#{transfer_id}") || {'status' => 'unknown'}
|
|
29
29
|
normalize_status(data)
|
|
30
30
|
end
|
|
@@ -38,6 +38,9 @@ module Aspera
|
|
|
38
38
|
result['ended_at'] = Time.now.utc.iso8601
|
|
39
39
|
when 'failed'
|
|
40
40
|
result.merge!('status' => 'failed', 'ended_at' => Time.now.utc.iso8601, 'error' => data['error_desc'].to_s)
|
|
41
|
+
when 'canceled'
|
|
42
|
+
result['status'] = 'cancelled'
|
|
43
|
+
result['ended_at'] = Time.now.utc.iso8601
|
|
41
44
|
else
|
|
42
45
|
result['status'] = 'running'
|
|
43
46
|
end
|
|
@@ -71,7 +74,7 @@ module Aspera
|
|
|
71
74
|
password: password
|
|
72
75
|
}
|
|
73
76
|
end
|
|
74
|
-
@node_api = Rest.new(**rest_params)
|
|
77
|
+
@node_api = Rest::Client.new(**rest_params)
|
|
75
78
|
# TODO: currently only supports one transfer. This is bad shortcut. but ok for CLI.
|
|
76
79
|
@transfer_id = nil
|
|
77
80
|
end
|
|
@@ -109,7 +112,7 @@ module Aspera
|
|
|
109
112
|
# status is empty sometimes with status 200...
|
|
110
113
|
transfer_data = node_api_.read("ops/transfers/#{@transfer_id}") || {'status' => 'unknown'} rescue {'status' => 'waiting(api error)'}
|
|
111
114
|
case transfer_data['status']
|
|
112
|
-
when 'waiting', 'partially_completed', 'unknown', 'waiting(read error)', 'waiting(api error)'
|
|
115
|
+
when 'waiting', 'partially_completed', 'paused', 'unknown', 'waiting(read error)', 'waiting(api error)'
|
|
113
116
|
notify_progress(:sessions_init, info: transfer_data['status'])
|
|
114
117
|
when 'running'
|
|
115
118
|
if !session_started
|
|
@@ -129,13 +132,11 @@ module Aspera
|
|
|
129
132
|
when 'completed'
|
|
130
133
|
notify_progress(:transfer, session_id: @transfer_id, info: bytes_expected) if bytes_expected
|
|
131
134
|
notify_progress(:session_end, session_id: @transfer_id)
|
|
132
|
-
notify_progress(:end)
|
|
133
135
|
break
|
|
134
|
-
when 'failed'
|
|
136
|
+
when 'failed', 'canceled'
|
|
135
137
|
notify_progress(:session_end, session_id: @transfer_id)
|
|
136
|
-
notify_progress(:end)
|
|
137
138
|
# Bug in HSTS ? transfer is marked failed, but there is no reason
|
|
138
|
-
break if transfer_data['error_code'].eql?(0) && transfer_data['error_desc'].empty?
|
|
139
|
+
break if transfer_data['status'].eql?('failed') && transfer_data['error_code'].eql?(0) && transfer_data['error_desc'].empty?
|
|
139
140
|
raise Transfer::Error, "status: #{transfer_data['status']}. code: #{transfer_data['error_code']}. description: #{transfer_data['error_desc']}"
|
|
140
141
|
else Aspera.error_unexpected_value(transfer_data['status']) { "transfer_data -> #{transfer_data}" }
|
|
141
142
|
end
|
|
@@ -149,7 +150,7 @@ module Aspera
|
|
|
149
150
|
|
|
150
151
|
# used internally to ensure node api is set before using.
|
|
151
152
|
def node_api_
|
|
152
|
-
Aspera.assert(!@node_api.nil?, 'Before using this object, set the node_api attribute to a Aspera::Rest object')
|
|
153
|
+
Aspera.assert(!@node_api.nil?, 'Before using this object, set the node_api attribute to a Aspera::Rest::Client object')
|
|
153
154
|
return @node_api
|
|
154
155
|
end
|
|
155
156
|
end
|
|
@@ -140,6 +140,8 @@ module Aspera
|
|
|
140
140
|
|
|
141
141
|
# Actual endpoint the daemon is listening on (resolved after connect, e.g. when port 0 was used)
|
|
142
142
|
attr_reader :daemon_endpoint
|
|
143
|
+
# gRPC client of the daemon
|
|
144
|
+
attr_reader :transfer_client
|
|
143
145
|
|
|
144
146
|
# :reek:UnusedParameters token_regenerator
|
|
145
147
|
def start_transfer(transfer_spec, token_regenerator: nil)
|
|
@@ -181,11 +183,9 @@ module Aspera
|
|
|
181
183
|
when :COMPLETED
|
|
182
184
|
notify_progress(:transfer, session_id: @transfer_id, info: bytes_expected) if bytes_expected
|
|
183
185
|
notify_progress(:session_end, session_id: @transfer_id)
|
|
184
|
-
notify_progress(:end)
|
|
185
186
|
break
|
|
186
187
|
when :FAILED, :CANCELED
|
|
187
188
|
notify_progress(:session_end, session_id: @transfer_id)
|
|
188
|
-
notify_progress(:end)
|
|
189
189
|
raise Transfer::Error, JSON.parse(response.message)['Description']
|
|
190
190
|
when :QUEUED, :UNKNOWN_STATUS, :PAUSED, :ORPHANED
|
|
191
191
|
notify_progress(:sessions_init, info: response.status.to_s.downcase)
|
data/lib/aspera/api/alee.rb
CHANGED
|
@@ -4,7 +4,7 @@ require 'aspera/api/aoc'
|
|
|
4
4
|
module Aspera
|
|
5
5
|
module Api
|
|
6
6
|
# Aspera License Entitlement Engine API client
|
|
7
|
-
class Alee < Aspera::Rest
|
|
7
|
+
class Alee < Aspera::Rest::Client
|
|
8
8
|
def initialize(entitlement_id, customer_id, api_domain: AoC::SAAS_DOMAIN_PROD, version: 'v1')
|
|
9
9
|
super(
|
|
10
10
|
base_url: "https://api.#{api_domain}/metering/#{version}",
|
data/lib/aspera/api/aoc.rb
CHANGED
|
@@ -7,16 +7,18 @@ require 'aspera/hash_ext'
|
|
|
7
7
|
require 'aspera/data_repository'
|
|
8
8
|
require 'aspera/transfer/spec'
|
|
9
9
|
require 'aspera/api/node'
|
|
10
|
-
require 'aspera/
|
|
10
|
+
require 'aspera/rest/list'
|
|
11
11
|
require 'base64'
|
|
12
|
+
require 'aspera/rainbow'
|
|
13
|
+
using Rainbow
|
|
12
14
|
|
|
13
15
|
module Aspera
|
|
14
16
|
module Api
|
|
15
17
|
# information relative to Files or Packages App
|
|
16
18
|
|
|
17
19
|
# Aspera on Cloud API client
|
|
18
|
-
class AoC < Rest
|
|
19
|
-
include
|
|
20
|
+
class AoC < Rest::Client
|
|
21
|
+
include Rest::List
|
|
20
22
|
|
|
21
23
|
module AppType
|
|
22
24
|
FILES = 'files'
|
|
@@ -140,7 +142,7 @@ module Aspera
|
|
|
140
142
|
# @param url [String] URL of AoC public link
|
|
141
143
|
# @return [Hash{Symbol => String, Hash}] Information about public link, or nil if not a public link
|
|
142
144
|
def link_info(url)
|
|
143
|
-
final_uri = Rest.new(base_url: url, redirect_max: MAX_AOC_URL_REDIRECT).call(operation: 'GET', ret: :resp).uri
|
|
145
|
+
final_uri = Rest::Client.new(base_url: url, redirect_max: MAX_AOC_URL_REDIRECT).call(operation: 'GET', ret: :resp).uri
|
|
144
146
|
Log.dump(:final_uri, final_uri, level: :trace1)
|
|
145
147
|
org_domain = split_org_domain(final_uri)
|
|
146
148
|
if (m = final_uri.path.match(%r{/oauth2/([^/]+)/login$}))
|
|
@@ -202,8 +204,8 @@ module Aspera
|
|
|
202
204
|
Aspera.assert(block_given?, 'block required for call_paging')
|
|
203
205
|
# set default large page if user does not specify own parameters. AoC Caps to 1000 anyway
|
|
204
206
|
query['per_page'] = 1000 unless query.key?('per_page')
|
|
205
|
-
max_items = query.delete(
|
|
206
|
-
max_pages = query.delete(
|
|
207
|
+
max_items = query.delete(Rest::List::MAX_ITEMS)
|
|
208
|
+
max_pages = query.delete(Rest::List::MAX_PAGES)
|
|
207
209
|
item_list = []
|
|
208
210
|
total_count = nil
|
|
209
211
|
current_page = query['page']
|
|
@@ -225,9 +227,9 @@ module Aspera
|
|
|
225
227
|
break if !max_items.nil? && item_list.count >= max_items
|
|
226
228
|
break if !max_pages.nil? && page_count >= max_pages
|
|
227
229
|
break if total_count&.<=(item_list.count)
|
|
228
|
-
|
|
230
|
+
Rest::Parameters.instance.spinner_cb.call("#{item_list.count} / #{total_count}") unless total_count.eql?(item_list.count.to_s)
|
|
229
231
|
end
|
|
230
|
-
|
|
232
|
+
Rest::Parameters.instance.spinner_cb.call(action: :success)
|
|
231
233
|
item_list = item_list[0..max_items - 1] if !max_items.nil? && item_list.count > max_items
|
|
232
234
|
return {items: item_list, total: total_count}
|
|
233
235
|
end
|
|
@@ -394,7 +396,7 @@ module Aspera
|
|
|
394
396
|
@cache_user_info =
|
|
395
397
|
begin
|
|
396
398
|
read('self')
|
|
397
|
-
rescue Aspera::
|
|
399
|
+
rescue Aspera::Rest::CallError => e
|
|
398
400
|
raise if exception || e.message.include?('invalid_grant')
|
|
399
401
|
Log.log.debug { "Ignoring error: (#{e.class}) #{e}" }
|
|
400
402
|
{}
|
|
@@ -496,9 +498,9 @@ module Aspera
|
|
|
496
498
|
}
|
|
497
499
|
else
|
|
498
500
|
# OAuth bearer token
|
|
499
|
-
|
|
500
|
-
node_params[:auth][:params]
|
|
501
|
-
node_params[:auth][:params][:scope] = Node.
|
|
501
|
+
# copy nested params, so that auth params of this API are not modified
|
|
502
|
+
node_params[:auth] = auth_params.merge(params: auth_params[:params]&.dup || {})
|
|
503
|
+
node_params[:auth][:params][:scope] = Node::Scope.join(access_key: node_info['access_key'], scope: scope)
|
|
502
504
|
node_params[:auth][:params][:owner_access] = true if scope.eql?(Node::Scope::ADMIN)
|
|
503
505
|
# Special header required for bearer token only
|
|
504
506
|
node_params[:headers] = {Node::HEADER_X_ASPERA_ACCESS_KEY => node_info['access_key']}
|
data/lib/aspera/api/ats.rb
CHANGED
|
@@ -6,7 +6,7 @@ require 'aspera/rest'
|
|
|
6
6
|
module Aspera
|
|
7
7
|
module Api
|
|
8
8
|
# Aspera Transfer Service API client without authentication
|
|
9
|
-
class Ats < Aspera::Rest
|
|
9
|
+
class Ats < Aspera::Rest::Client
|
|
10
10
|
SERVICE_BASE_URL = 'https://ats.aspera.io'
|
|
11
11
|
# currently supported clouds
|
|
12
12
|
# Note to Aspera: shall be an API call
|
data/lib/aspera/api/cos_node.rb
CHANGED
|
@@ -22,7 +22,7 @@ module Aspera
|
|
|
22
22
|
Aspera.assert(service_credentials.key?(field)) { "service_credentials must have a field: #{field}" }
|
|
23
23
|
end
|
|
24
24
|
# read endpoints from service provided in service credentials
|
|
25
|
-
endpoints = Aspera::Rest.new(base_url: service_credentials['endpoints']).read('')
|
|
25
|
+
endpoints = Aspera::Rest::Client.new(base_url: service_credentials['endpoints']).read('')
|
|
26
26
|
Log.dump(:endpoints, endpoints)
|
|
27
27
|
endpoint = endpoints.dig('service-endpoints', 'regional', bucket_region, 'public', bucket_region)
|
|
28
28
|
Aspera.assert(!endpoint.nil?) { "no such region: #{bucket_region}" }
|
|
@@ -40,7 +40,7 @@ module Aspera
|
|
|
40
40
|
endpoint = "https://#{endpoint}" unless endpoint.start_with?('http')
|
|
41
41
|
@auth_url = auth_url
|
|
42
42
|
@api_key = api_key
|
|
43
|
-
s3_api = Aspera::Rest.new(
|
|
43
|
+
s3_api = Aspera::Rest::Client.new(
|
|
44
44
|
base_url: endpoint,
|
|
45
45
|
not_auth_codes: %w[401 403], # error codes when not authorized
|
|
46
46
|
headers: {'ibm-service-instance-id' => instance_id},
|
data/lib/aspera/api/faspex.rb
CHANGED
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
require 'aspera/assert'
|
|
4
4
|
require 'aspera/rest'
|
|
5
5
|
require 'aspera/oauth/base'
|
|
6
|
-
require 'aspera/
|
|
6
|
+
require 'aspera/rest/list'
|
|
7
7
|
require 'digest'
|
|
8
8
|
|
|
9
9
|
module Aspera
|
|
@@ -15,6 +15,7 @@ module Aspera
|
|
|
15
15
|
# @param context [String] The `context` query parameter in public link
|
|
16
16
|
# @param redirect_uri [String] URI of web UI login
|
|
17
17
|
# @param path_authorize [String] Path to provide passcode
|
|
18
|
+
# @param base_params [Hash] Parameters for base class
|
|
18
19
|
def initialize(
|
|
19
20
|
context:,
|
|
20
21
|
redirect_uri:,
|
|
@@ -48,18 +49,18 @@ module Aspera
|
|
|
48
49
|
raise Error, info['action_message'] if info['action_message']
|
|
49
50
|
Aspera.assert(info['code'], 'Missing code in answer')
|
|
50
51
|
# Exchange code for token
|
|
51
|
-
|
|
52
|
+
create_token_base(
|
|
52
53
|
grant_type: 'authorization_code',
|
|
53
54
|
code: info['code'],
|
|
54
55
|
redirect_uri: @redirect_uri
|
|
55
|
-
)
|
|
56
|
+
)
|
|
56
57
|
end
|
|
57
58
|
end
|
|
58
59
|
OAuth::Factory.instance.register_token_creator(FaspexPubLink)
|
|
59
60
|
module Api
|
|
60
61
|
# Aspera Faspex 5 API Client
|
|
61
|
-
class Faspex < Aspera::Rest
|
|
62
|
-
include
|
|
62
|
+
class Faspex < Aspera::Rest::Client
|
|
63
|
+
include Rest::List
|
|
63
64
|
|
|
64
65
|
# endpoint for authentication API
|
|
65
66
|
PATH_AUTH = 'auth'
|
|
@@ -137,10 +138,12 @@ module Aspera
|
|
|
137
138
|
SENT_MAILBOX_TYPES.include?(box) || box == 'ALL' ? :sent : :received
|
|
138
139
|
end
|
|
139
140
|
end
|
|
141
|
+
# @return [Hash, nil] public link context decoded from URL (keys: resource, type, id, passcode, package_id, email), or nil if not a public link
|
|
140
142
|
attr_reader :pub_link_context
|
|
141
143
|
|
|
142
144
|
# @param url [String] Faspex URL, can be a public link
|
|
143
145
|
# @param auth [Symbol] Authentication method: :boot (token in header), :web (open browser), :jwt (client_id + private key), :public_link (context in URL)
|
|
146
|
+
# @param root [String] Root path for API
|
|
144
147
|
# @param password [String, nil] For :boot auth, the token copied directly from browser in developer mode
|
|
145
148
|
# @param client_id [String, nil] For :web and :jwt auth, the client_id of web UI application
|
|
146
149
|
# @param client_secret [String, nil] For :web auth, the client_secret of web UI application (not needed for :jwt)
|
|
@@ -161,12 +164,13 @@ module Aspera
|
|
|
161
164
|
passphrase: nil
|
|
162
165
|
)
|
|
163
166
|
auth = :public_link if self.class.public_link?(url)
|
|
167
|
+
# @type [Hash, nil]
|
|
164
168
|
@pub_link_context = nil
|
|
165
169
|
super(**
|
|
166
170
|
case auth
|
|
167
171
|
when :public_link
|
|
168
172
|
# Get URL of final redirect of provided public link
|
|
169
|
-
final_url = Rest.new(base_url: url, redirect_max: 3).call(operation: 'GET', ret: :resp).uri.to_s
|
|
173
|
+
final_url = Rest::Client.new(base_url: url, redirect_max: 3).call(operation: 'GET', ret: :resp).uri.to_s
|
|
170
174
|
Log.dump(:final_url, final_url, level: :trace1)
|
|
171
175
|
# Get context from query
|
|
172
176
|
encoded_context = Rest.query_to_h(URI.parse(final_url).query)['context']
|
|
@@ -178,7 +182,7 @@ module Aspera
|
|
|
178
182
|
base_url = final_url.gsub(%r{/public/.*}, '').gsub(/\?.*/, '')
|
|
179
183
|
# Get web UI client_id and redirect_uri
|
|
180
184
|
# TODO: change this for something more reliable
|
|
181
|
-
config = JSON.parse(Rest.new(base_url: "#{base_url}/config.js", redirect_max: 3).call(operation: 'GET').sub(/^[^=]+=/, '').gsub(/([a-z_]+):/, '"\1":').delete("\n ").tr("'", '"')).symbolize_keys
|
|
185
|
+
config = JSON.parse(Rest::Client.new(base_url: "#{base_url}/config.js", redirect_max: 3).call(operation: 'GET').sub(/^[^=]+=/, '').gsub(/([a-z_]+):/, '"\1":').delete("\n ").tr("'", '"')).symbolize_keys
|
|
182
186
|
Log.dump(:config_js, config)
|
|
183
187
|
{
|
|
184
188
|
base_url: "#{base_url}/#{root}",
|