aspera-cli 4.27.2 → 4.27.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/CHANGELOG.md +133 -0
- data/CONTRIBUTING.md +5 -2
- data/bin/ascli +3 -1
- data/docs/README.md +1006 -766
- data/lib/aspera/agent/base.rb +7 -2
- data/lib/aspera/agent/connect.rb +6 -8
- data/lib/aspera/agent/desktop.rb +2 -6
- data/lib/aspera/agent/direct.rb +52 -22
- data/lib/aspera/agent/node.rb +9 -8
- data/lib/aspera/agent/transferd.rb +2 -2
- data/lib/aspera/api/alee.rb +1 -1
- data/lib/aspera/api/aoc.rb +14 -12
- data/lib/aspera/api/ats.rb +1 -1
- data/lib/aspera/api/cos_node.rb +2 -2
- data/lib/aspera/api/faspex.rb +11 -7
- data/lib/aspera/api/httpgw.rb +38 -35
- data/lib/aspera/api/node.rb +39 -34
- data/lib/aspera/ascmd.rb +3 -1
- data/lib/aspera/ascp/installation.rb +63 -28
- data/lib/aspera/ascp/management.rb +1 -0
- data/lib/aspera/assert.rb +4 -0
- data/lib/aspera/cli/ascp_actions.rb +20 -41
- data/lib/aspera/cli/async_transfer_store.rb +12 -11
- data/lib/aspera/cli/bootstrapper.rb +14 -16
- data/lib/aspera/cli/command_line.rb +252 -0
- data/lib/aspera/cli/command_registry.rb +215 -37
- data/lib/aspera/cli/command_spec.rb +104 -15
- data/lib/aspera/cli/completion/ascli.bash +12 -0
- data/lib/aspera/cli/completion/ascli.fish +16 -0
- data/lib/aspera/cli/completion/ascli.zsh +19 -0
- data/lib/aspera/cli/context.rb +3 -0
- data/lib/aspera/cli/deprecation.rb +37 -0
- data/lib/aspera/cli/extended_value.rb +6 -3
- data/lib/aspera/cli/formatter.rb +94 -80
- data/lib/aspera/cli/gem_checker.rb +1 -1
- data/lib/aspera/cli/hints.rb +7 -6
- data/lib/aspera/cli/http.rb +22 -34
- data/lib/aspera/cli/info.rb +3 -0
- data/lib/aspera/cli/mcp_tool.rb +47 -83
- data/lib/aspera/cli/option_declarator.rb +33 -42
- data/lib/aspera/cli/option_registry.rb +69 -0
- data/lib/aspera/cli/option_types.rb +105 -0
- data/lib/aspera/cli/option_value.rb +278 -0
- data/lib/aspera/cli/options.schema.yaml +124 -15
- data/lib/aspera/cli/parser.rb +333 -862
- data/lib/aspera/cli/plugins/alee.rb +7 -4
- data/lib/aspera/cli/plugins/aoc.rb +545 -518
- data/lib/aspera/cli/plugins/ats.rb +59 -80
- data/lib/aspera/cli/plugins/base.rb +221 -265
- data/lib/aspera/cli/plugins/basic_auth.rb +2 -10
- data/lib/aspera/cli/plugins/config.rb +263 -184
- data/lib/aspera/cli/plugins/console.rb +103 -39
- data/lib/aspera/cli/plugins/cos.rb +6 -23
- data/lib/aspera/cli/plugins/factory.rb +3 -0
- data/lib/aspera/cli/plugins/faspex5.rb +204 -182
- data/lib/aspera/cli/plugins/faspio.rb +6 -11
- data/lib/aspera/cli/plugins/httpgw.rb +8 -11
- data/lib/aspera/cli/plugins/mcp.rb +20 -55
- data/lib/aspera/cli/plugins/node.rb +300 -327
- data/lib/aspera/cli/plugins/orchestrator.rb +152 -110
- data/lib/aspera/cli/plugins/preview.rb +96 -105
- data/lib/aspera/cli/plugins/server.rb +78 -53
- data/lib/aspera/cli/plugins/shares.rb +80 -131
- data/lib/aspera/cli/preset_actions.rb +44 -27
- data/lib/aspera/cli/preset_manager.rb +44 -19
- data/lib/aspera/cli/prompt.rb +36 -0
- data/lib/aspera/cli/result.rb +42 -36
- data/lib/aspera/cli/runner.rb +32 -59
- data/lib/aspera/cli/special_values.rb +5 -0
- data/lib/aspera/cli/sync_actions.rb +51 -46
- data/lib/aspera/cli/terminal_formatter.rb +9 -3
- data/lib/aspera/cli/transfer_actions.rb +14 -9
- data/lib/aspera/cli/transfer_agent.rb +34 -38
- data/lib/aspera/cli/transfer_progress.rb +290 -55
- data/lib/aspera/cli/vault_manager.rb +0 -17
- data/lib/aspera/cli/version.rb +1 -1
- data/lib/aspera/cli/wizard.rb +5 -3
- data/lib/aspera/coverage.rb +1 -1
- data/lib/aspera/environment.rb +35 -5
- data/lib/aspera/faspex_gw.rb +2 -1
- data/lib/aspera/faspex_postproc.rb +1 -0
- data/lib/aspera/graphql.rb +5 -5
- data/lib/aspera/json_rpc/client.rb +5 -5
- data/lib/aspera/keychain/encrypted_hash.rb +2 -2
- data/lib/aspera/keychain/factory.rb +2 -1
- data/lib/aspera/keychain/one_password_api.rb +1 -1
- data/lib/aspera/link_header.rb +2 -2
- data/lib/aspera/log.rb +47 -27
- data/lib/aspera/markdown.rb +2 -0
- data/lib/aspera/mime.rb +25 -0
- data/lib/aspera/node_emulator.rb +759 -0
- data/lib/aspera/oauth/base.rb +37 -26
- data/lib/aspera/oauth/factory.rb +7 -3
- data/lib/aspera/oauth/generic.rb +1 -1
- data/lib/aspera/oauth/json_credentials.rb +34 -0
- data/lib/aspera/oauth/jwt.rb +4 -5
- data/lib/aspera/oauth/web.rb +9 -8
- data/lib/aspera/oauth.rb +1 -0
- data/lib/aspera/persistency_folder.rb +1 -3
- data/lib/aspera/preview/file_types.rb +4 -4
- data/lib/aspera/preview/generator.rb +11 -1
- data/lib/aspera/preview/options.schema.yaml +119 -0
- data/lib/aspera/preview/terminal.rb +4 -3
- data/lib/aspera/preview/utils.rb +9 -6
- data/lib/aspera/products/connect.rb +1 -1
- data/lib/aspera/rainbow.rb +7 -0
- data/lib/aspera/rest/aspera_errors.rb +72 -0
- data/lib/aspera/rest/call_error.rb +27 -0
- data/lib/aspera/rest/client.rb +523 -0
- data/lib/aspera/rest/error_analyzer.rb +113 -0
- data/lib/aspera/rest/list.rb +149 -0
- data/lib/aspera/rest/parameters.rb +55 -0
- data/lib/aspera/rest/util.rb +176 -0
- data/lib/aspera/rest.rb +7 -621
- data/lib/aspera/schema/IBM Aspera Console-enhanced.yaml +1125 -0
- data/lib/aspera/schema/IBM Aspera Faspex API-5.0-enhanced.yaml +0 -20
- data/lib/aspera/schema/IBM Aspera Orchestrator API-v1.yaml +1784 -0
- data/lib/aspera/schema/IBM Aspera on Cloud API-0.2.6-enhanced.yaml +1230 -137
- data/lib/aspera/schema/IBM Aspera on Cloud Automation API-1.0.5-enhanced.yaml +2395 -0
- data/lib/aspera/schema/IBM_Aspera_Shares.yaml +14 -13
- data/lib/aspera/schema/documentation.rb +13 -3
- data/lib/aspera/schema/reader.rb +12 -18
- data/lib/aspera/schema/registry.rb +23 -1
- data/lib/aspera/schema/validator.rb +92 -0
- data/lib/aspera/secret_hider.rb +36 -25
- data/lib/aspera/string_ext.rb +15 -0
- data/lib/aspera/temp_file_manager.rb +6 -5
- data/lib/aspera/transfer/parameters.rb +2 -0
- data/lib/aspera/transfer/spec.rb +1 -0
- data/lib/aspera/uri_reader.rb +11 -11
- data/lib/aspera/web_auth/index.html +147 -0
- data/lib/aspera/web_auth/server.rb +81 -0
- data.tar.gz.sig +0 -0
- metadata +43 -9
- metadata.gz.sig +0 -0
- data/lib/aspera/colors.rb +0 -79
- data/lib/aspera/node_simulator.rb +0 -344
- data/lib/aspera/preview/options.rb +0 -45
- data/lib/aspera/rest_call_error.rb +0 -25
- data/lib/aspera/rest_error_analyzer.rb +0 -111
- data/lib/aspera/rest_errors_aspera.rb +0 -58
- data/lib/aspera/rest_list.rb +0 -136
- data/lib/aspera/web_auth.rb +0 -211
|
@@ -15,7 +15,8 @@ module Aspera
|
|
|
15
15
|
#
|
|
16
16
|
# Architecture:
|
|
17
17
|
# - remote-daemon agents (desktop, node, connect, transferd): transfer_id + agent_params
|
|
18
|
-
# are persisted in AsyncTransferStore
|
|
18
|
+
# are persisted in AsyncTransferStore (without secrets, taken from current options);
|
|
19
|
+
# status is re-queried via Agent::Xxx.transfer_status
|
|
19
20
|
# - direct agent: transfers live in Ruby threads - store is the agent's @sessions;
|
|
20
21
|
# status is not persistable across process restarts (returns an informational message)
|
|
21
22
|
module TransferActions
|
|
@@ -25,10 +26,12 @@ module Aspera
|
|
|
25
26
|
store = async_transfer_store
|
|
26
27
|
entry = store.read(job_id)
|
|
27
28
|
Aspera.assert(!entry.nil?, type: Cli::BadArgument) { "Unknown job_id: #{job_id}" }
|
|
29
|
+
# Secrets are not persisted: take them from current transfer options (e.g. `--transfer`), for the same agent type
|
|
30
|
+
agent_params = current_agent_params(entry['agent_type']).merge(entry['agent_params'] || {})
|
|
28
31
|
# Inject the in-process agent reference (direct/httpgw) if still alive in this process.
|
|
29
32
|
ref = store.agent_ref(job_id)
|
|
30
|
-
|
|
31
|
-
live = query_live_status(entry)
|
|
33
|
+
agent_params['_agent_ref'] = ref if ref
|
|
34
|
+
live = query_live_status(entry.merge('agent_params' => agent_params))
|
|
32
35
|
if live
|
|
33
36
|
entry.merge!(live)
|
|
34
37
|
store.write(job_id, entry)
|
|
@@ -36,12 +39,6 @@ module Aspera
|
|
|
36
39
|
Result::SingleObject.new(entry)
|
|
37
40
|
end
|
|
38
41
|
|
|
39
|
-
# List all persisted async transfer entries.
|
|
40
|
-
def action_transfer_list(**)
|
|
41
|
-
rows = async_transfer_store.list
|
|
42
|
-
Result::ObjectList.new(rows, fields: %w[job_id agent_type status started_at ended_at bytes_transferred transfer_id])
|
|
43
|
-
end
|
|
44
|
-
|
|
45
42
|
TERMINAL_STATUSES = %w[completed failed cancelled].freeze
|
|
46
43
|
private_constant :TERMINAL_STATUSES
|
|
47
44
|
|
|
@@ -59,6 +56,14 @@ module Aspera
|
|
|
59
56
|
|
|
60
57
|
private
|
|
61
58
|
|
|
59
|
+
# @param agent_type [String] agent of the async transfer
|
|
60
|
+
# @return [Hash] agent parameters from current transfer options (e.g. `--transfer`), if for the same agent type
|
|
61
|
+
def current_agent_params(agent_type)
|
|
62
|
+
current = context.transfer.transfer_options
|
|
63
|
+
return {} unless (current['agent'] || 'direct').to_s.eql?(agent_type)
|
|
64
|
+
current.except('agent', 'asynchronous')
|
|
65
|
+
end
|
|
66
|
+
|
|
62
67
|
# Lazy accessor - requires context.persistency (available in all Base sub-classes).
|
|
63
68
|
# Delegate to the TransferAgent's store so that the in-memory agent-ref
|
|
64
69
|
# registry is shared between the two components in the same process.
|
|
@@ -32,18 +32,24 @@ module Aspera
|
|
|
32
32
|
<%=ts.to_yaml%>
|
|
33
33
|
END_OF_TEMPLATE
|
|
34
34
|
CP4I_REMOTE_HOST_LB = 'N/A'
|
|
35
|
+
# Transfer spec parameters overridden by option `ts`
|
|
36
|
+
TS_DEFAULTS = {
|
|
37
|
+
'create_dir' => true,
|
|
38
|
+
'resume_policy' => 'sparse_csum'
|
|
39
|
+
}.freeze
|
|
35
40
|
private_constant :FILE_LIST_FROM_TRANSFER_SPEC,
|
|
36
41
|
:FILE_LIST_OPTIONS,
|
|
37
|
-
:DEFAULT_TRANSFER_NOTIFY_TEMPLATE
|
|
42
|
+
:DEFAULT_TRANSFER_NOTIFY_TEMPLATE,
|
|
43
|
+
:TS_DEFAULTS
|
|
38
44
|
|
|
39
45
|
extend OptionDeclarator
|
|
40
46
|
|
|
41
|
-
option :ts, description: 'Override transfer spec values', schema: Schema::Registry::TRANSFER_SPEC
|
|
47
|
+
option :ts, description: 'Override transfer spec values', schema: Schema::Registry::TRANSFER_SPEC, on_set: :user_transfer_spec=
|
|
42
48
|
option :to_folder, description: 'Destination folder for transferred files'
|
|
43
49
|
option :sources, description: "How list of transferred files is provided (#{FILE_LIST_OPTIONS.join(',')})", default: FILE_LIST_FROM_ARGS
|
|
44
50
|
option :src_type, description: 'Type of file list', allowed: %i[list pair], default: :list
|
|
45
|
-
option :transfer, description: 'Transfer agent type, or agent parameters with optional agent key', allowed: [Hash, String], schema: Schema::Registry::TRANSFER_AGENT_OPTIONS
|
|
46
|
-
option :transfer_info, description: 'Parameters for transfer agent', deprecation: 'use --transfer instead', schema: Schema::Registry::TRANSFER_AGENT_OPTIONS
|
|
51
|
+
option :transfer, description: 'Transfer agent type, or agent parameters with optional agent key', allowed: [Hash, String], shorthand: 'agent', schema: Schema::Registry::TRANSFER_AGENT_OPTIONS
|
|
52
|
+
option :transfer_info, description: 'Parameters for transfer agent', deprecation: {last: '4.26.2', message: 'use --transfer instead'}, schema: Schema::Registry::TRANSFER_AGENT_OPTIONS
|
|
47
53
|
|
|
48
54
|
# @param context [Context] Application context
|
|
49
55
|
def initialize(context)
|
|
@@ -51,22 +57,15 @@ module Aspera
|
|
|
51
57
|
Aspera.assert_type(context.options, Parser) { 'context.options' }
|
|
52
58
|
@context = context
|
|
53
59
|
# Command line can override transfer spec
|
|
54
|
-
@user_transfer_spec =
|
|
55
|
-
'create_dir' => true,
|
|
56
|
-
'resume_policy' => 'sparse_csum'
|
|
57
|
-
}
|
|
58
|
-
# options for transfer agent (agent type + agent-specific parameters)
|
|
59
|
-
@transfer_options = {}
|
|
60
|
+
@user_transfer_spec = TS_DEFAULTS.dup
|
|
60
61
|
# the currently selected transfer agent
|
|
61
62
|
@agent = nil
|
|
62
63
|
# source/destination pair, like "paths" of transfer spec
|
|
63
64
|
@transfer_paths = nil
|
|
64
65
|
# HTTPGW URL provided by webapp
|
|
65
66
|
@httpgw_url_lambda = nil
|
|
66
|
-
|
|
67
|
-
@context.options
|
|
68
|
-
@context.options.set_handler(:transfer, object: self, method: :option_transfer)
|
|
69
|
-
@context.options.set_handler(:transfer_info, object: self, method: :transfer_options)
|
|
67
|
+
# Instance variables used by `on_set` callbacks are set above
|
|
68
|
+
self.class.declare_options(@context.options, target: self)
|
|
70
69
|
@context.options.parse_options!
|
|
71
70
|
@notification_cb = nil
|
|
72
71
|
if !@context.options.get_option(:notify_to).nil?
|
|
@@ -80,22 +79,17 @@ module Aspera
|
|
|
80
79
|
end
|
|
81
80
|
end
|
|
82
81
|
|
|
83
|
-
|
|
82
|
+
attr_reader :user_transfer_spec
|
|
84
83
|
|
|
85
|
-
#
|
|
86
|
-
#
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
@transfer_options = @transfer_options.deep_merge(value)
|
|
95
|
-
when :get
|
|
96
|
-
return @transfer_options
|
|
97
|
-
end
|
|
98
|
-
nil
|
|
84
|
+
# `on_set` callback of option `ts`: values override defaults
|
|
85
|
+
# @param value [Hash] transfer spec parameters
|
|
86
|
+
def user_transfer_spec=(value)
|
|
87
|
+
@user_transfer_spec = TS_DEFAULTS.merge(value)
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
# @return [Hash] agent type (key `agent`) and agent parameters, from options `transfer` and deprecated `transfer_info`
|
|
91
|
+
def transfer_options
|
|
92
|
+
@context.options.get_option(:transfer_info).deep_merge(@context.options.get_option(:transfer))
|
|
99
93
|
end
|
|
100
94
|
|
|
101
95
|
def agent_instance=(instance)
|
|
@@ -106,10 +100,10 @@ module Aspera
|
|
|
106
100
|
def agent_instance
|
|
107
101
|
return @agent unless @agent.nil?
|
|
108
102
|
# agent type: from composite option 'agent' key, default :direct
|
|
109
|
-
raw_type =
|
|
103
|
+
raw_type = transfer_options['agent'] || :direct
|
|
110
104
|
agent_type = Parser.get_from_list(raw_type.to_s, 'transfer agent', Agent::Factory::ALL.keys)
|
|
111
105
|
# set keys as symbols, strip internal keys not forwarded to the agent constructor
|
|
112
|
-
agent_options =
|
|
106
|
+
agent_options = transfer_options.except('agent', 'asynchronous').symbolize_keys
|
|
113
107
|
agent_options[:progress] = @context.progress_bar
|
|
114
108
|
agent_options[:config_dir] = @context.main_folder
|
|
115
109
|
# special cases
|
|
@@ -121,8 +115,10 @@ module Aspera
|
|
|
121
115
|
agent_options.merge!(@context.presets.by_name(param_set_name).symbolize_keys)
|
|
122
116
|
end
|
|
123
117
|
when :direct
|
|
124
|
-
#
|
|
118
|
+
# By default, no native `ascp` progress bar
|
|
125
119
|
agent_options[:quiet] = true unless agent_options.key?(:quiet)
|
|
120
|
+
# Native `ascp` progress bar requested: no other progress bar, unless option `progress_bar` is set
|
|
121
|
+
agent_options[:progress] = nil if !agent_options[:quiet] && @context.options.option_def(:progress_bar).source.eql?(:default)
|
|
126
122
|
agent_options[:check_ignore_cb] = ->(host, port) { @context.http_config.ignore_cert?(host, port) }
|
|
127
123
|
# JRuby
|
|
128
124
|
agent_options[:trusted_certs] = @context.http_config.trusted_cert_locations unless agent_options.key?(:trusted_certs)
|
|
@@ -210,8 +206,8 @@ module Aspera
|
|
|
210
206
|
when FILE_LIST_FROM_TRANSFER_SPEC
|
|
211
207
|
Log.log.debug('assume list provided in transfer spec')
|
|
212
208
|
special_case_direct_with_list =
|
|
213
|
-
(
|
|
214
|
-
Transfer::Parameters.ascp_args_file_list?(
|
|
209
|
+
(transfer_options['agent'] || :direct).to_sym.eql?(:direct) &&
|
|
210
|
+
Transfer::Parameters.ascp_args_file_list?(transfer_options['ascp_args'])
|
|
215
211
|
Aspera.assert(!@transfer_paths.nil? || special_case_direct_with_list, type: Cli::BadArgument) { 'transfer spec on command line must have sources' }
|
|
216
212
|
# can be nil
|
|
217
213
|
@transfer_paths
|
|
@@ -227,7 +223,7 @@ module Aspera
|
|
|
227
223
|
|
|
228
224
|
# Start a transfer and wait for completion, plugins shall use this method
|
|
229
225
|
# @param transfer_spec [Hash] transfer specification
|
|
230
|
-
# @param rest_token [Rest, nil] if oauth token regeneration supported
|
|
226
|
+
# @param rest_token [Rest::Client, nil] if oauth token regeneration supported
|
|
231
227
|
def start(transfer_spec, rest_token: nil)
|
|
232
228
|
# check parameters
|
|
233
229
|
Aspera.assert_type(transfer_spec, Hash) { 'transfer_spec' }
|
|
@@ -264,13 +260,13 @@ module Aspera
|
|
|
264
260
|
# create transfer agent
|
|
265
261
|
agent_instance.start_transfer(transfer_spec, token_regenerator: rest_token)
|
|
266
262
|
# --- async mode ---
|
|
267
|
-
if
|
|
268
|
-
agent_type = (
|
|
263
|
+
if transfer_options['asynchronous']
|
|
264
|
+
agent_type = (transfer_options['agent'] || 'direct').to_s
|
|
269
265
|
# In-process agents (direct, httpgw) expose last_job_id directly.
|
|
270
266
|
# Remote-daemon agents do not override last_job_id (returns nil): generate a UUID.
|
|
271
267
|
job_id = agent_instance.last_job_id || SecureRandom.uuid
|
|
272
268
|
# Base agent_params from transfer options (strip internal keys)
|
|
273
|
-
agent_params =
|
|
269
|
+
agent_params = transfer_options.except('agent', 'asynchronous')
|
|
274
270
|
# For daemon agents: capture any runtime-resolved connection detail
|
|
275
271
|
case agent_type
|
|
276
272
|
when 'desktop'
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require 'monitor'
|
|
4
|
+
require 'aspera/environment'
|
|
3
5
|
require 'aspera/log'
|
|
4
6
|
require 'aspera/assert'
|
|
5
7
|
require 'ruby-progressbar'
|
|
@@ -7,95 +9,328 @@ require 'ruby-progressbar'
|
|
|
7
9
|
module Aspera
|
|
8
10
|
module Cli
|
|
9
11
|
# Progress bar for transfers.
|
|
10
|
-
# Supports multi-session.
|
|
12
|
+
# Supports multi-session: events may come from several threads.
|
|
11
13
|
# Note that we can have this case:
|
|
12
|
-
# 2 sessions (-C x:2), but one session fails and restarts
|
|
14
|
+
# 2 sessions (-C x:2), but one session fails and restarts (resume): it restarts with the same session id.
|
|
15
|
+
# Errors are logged, never raised: the progress bar shall not fail a transfer.
|
|
13
16
|
class TransferProgress
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
17
|
+
# Rate is computed on this sliding window
|
|
18
|
+
RATE_WINDOW_SEC = 5.0
|
|
19
|
+
# Rate is not displayed on a shorter period
|
|
20
|
+
RATE_MIN_SEC = 1.0
|
|
21
|
+
# Decimal units, like `ascp`
|
|
22
|
+
KILO = 1000
|
|
23
|
+
BITS_PER_MEGABIT = KILO * KILO
|
|
24
|
+
BYTE_UNITS = %w[B KB MB GB TB PB].freeze
|
|
25
|
+
private_constant :RATE_WINDOW_SEC, :RATE_MIN_SEC, :KILO, :BITS_PER_MEGABIT, :BYTE_UNITS
|
|
26
|
+
|
|
27
|
+
class << self
|
|
28
|
+
# @param bytes [Integer] Size in bytes
|
|
29
|
+
# @return [String] Size with decimal unit, e.g. `12.3 MB`
|
|
30
|
+
def format_bytes(bytes)
|
|
31
|
+
value = bytes.to_f
|
|
32
|
+
index = 0
|
|
33
|
+
while value >= KILO && index < BYTE_UNITS.length - 1
|
|
34
|
+
value /= KILO
|
|
35
|
+
index += 1
|
|
36
|
+
end
|
|
37
|
+
return "#{bytes} B" if index.zero?
|
|
38
|
+
format('%.1f %s', value, BYTE_UNITS[index])
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
# @param bytes_per_second [Float, nil] Rate, `nil` if unknown
|
|
42
|
+
# @return [String] Rate in megabits per second (decimal, like `ascp`)
|
|
43
|
+
def format_rate(bytes_per_second)
|
|
44
|
+
return '-- Mbps' if bytes_per_second.nil?
|
|
45
|
+
mbps = bytes_per_second * Environment::BITS_PER_BYTE / BITS_PER_MEGABIT
|
|
46
|
+
mbps < 10 ? format('%.2f Mbps', mbps) : format('%.0f Mbps', mbps)
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
# @param seconds [Numeric, nil] Duration, `nil` if unknown
|
|
50
|
+
# @return [String] Duration as `hh:mm:ss`
|
|
51
|
+
def format_duration(seconds)
|
|
52
|
+
return '--:--:--' if seconds.nil?
|
|
53
|
+
seconds = seconds.round
|
|
54
|
+
format('%02d:%02d:%02d', seconds / 3600, (seconds / 60) % 60, seconds % 60)
|
|
55
|
+
end
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
# @param output [IO] Where the progress bar is displayed: not on `$stdout`, which may be redirected for command output
|
|
59
|
+
def initialize(output: $stderr)
|
|
60
|
+
@output = output
|
|
61
|
+
# Re-entrant, see `suspend`
|
|
62
|
+
@mutex = Monitor.new
|
|
63
|
+
reset_state
|
|
20
64
|
end
|
|
21
65
|
|
|
22
66
|
# Reset progress bar, to re-use it.
|
|
23
67
|
def reset
|
|
24
|
-
|
|
68
|
+
@mutex.synchronize { reset_state }
|
|
25
69
|
end
|
|
26
70
|
|
|
27
|
-
# Called by user of progress bar with a status on a transfer session
|
|
28
|
-
#
|
|
29
|
-
#
|
|
30
|
-
#
|
|
71
|
+
# Called by user of progress bar with a status on a transfer session.
|
|
72
|
+
# Event types:
|
|
73
|
+
# - `sessions_init` : `info` is a status message, displayed until a session starts
|
|
74
|
+
# - `session_start` : a session starts, or restarts (same `session_id`)
|
|
75
|
+
# - `session_size` : `info` is the total size of the session
|
|
76
|
+
# - `transfer` : `info` is the number of bytes transferred by the session
|
|
77
|
+
# - `skip` : `info` is the number of bytes of the session already at destination (not transferred)
|
|
78
|
+
# - `session_end` : a session ends
|
|
79
|
+
# - `end` : all sessions are finished, `info` is `false` if the transfer failed
|
|
80
|
+
# @param type [Symbol] Event type
|
|
81
|
+
# @param session_id [String, nil] Unique identifier of a transfer session, same when the session is restarted
|
|
82
|
+
# @param info [Object, nil] Optional specific additional info for the given event type
|
|
31
83
|
def event(type, session_id: nil, info: nil)
|
|
32
84
|
Log.log.trace1 { "progress: #{type} #{session_id} #{info}" }
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
85
|
+
@mutex.synchronize { process_event(type, session_id, info) }
|
|
86
|
+
rescue StandardError => e
|
|
87
|
+
Log.log.warn { "Progress bar: #{e.class}: #{e.message}" } unless @error_logged
|
|
88
|
+
@error_logged = true
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
# Execute the block with the progress bar cleared, and then display it again.
|
|
92
|
+
# Used to write log lines on the terminal.
|
|
93
|
+
# Does not wait if the progress bar is being updated by another thread (no deadlock with a logger lock).
|
|
94
|
+
def suspend
|
|
95
|
+
return yield unless @mutex.try_enter
|
|
96
|
+
begin
|
|
97
|
+
return yield if @progress_bar.nil? || !@output.tty?
|
|
98
|
+
@progress_bar.clear
|
|
99
|
+
@output.flush
|
|
100
|
+
begin
|
|
101
|
+
yield
|
|
102
|
+
ensure
|
|
103
|
+
@progress_bar.refresh(force: true)
|
|
104
|
+
end
|
|
105
|
+
ensure
|
|
106
|
+
@mutex.exit
|
|
41
107
|
end
|
|
42
|
-
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
private
|
|
111
|
+
|
|
112
|
+
def reset_state
|
|
113
|
+
@progress_bar = nil
|
|
114
|
+
# Key is session id
|
|
115
|
+
@sessions = {}
|
|
116
|
+
# Status displayed before session start
|
|
117
|
+
@title = nil
|
|
118
|
+
# Format currently set on the progress bar
|
|
119
|
+
@format = nil
|
|
120
|
+
# [Integer] Bytes actually transferred, by all sessions, including previous runs of restarted sessions
|
|
121
|
+
@transferred = 0
|
|
122
|
+
# [Array<Array(Float, Integer)>] (time, transferred bytes) on the rate window
|
|
123
|
+
@samples = []
|
|
124
|
+
# [Array(Float, Integer), nil] First sample (data starts to flow, not session start), for average rate
|
|
125
|
+
@start = nil
|
|
126
|
+
@error_logged = false
|
|
127
|
+
end
|
|
128
|
+
|
|
129
|
+
# @param type [Symbol] Event type
|
|
130
|
+
# @param session_id [String, nil] Session identifier
|
|
131
|
+
# @param info [Object, nil] Event information
|
|
132
|
+
def process_event(type, session_id, info)
|
|
43
133
|
case type
|
|
44
134
|
when :sessions_init
|
|
45
135
|
# Give opportunity to show progress of initialization with multiple status
|
|
46
136
|
Aspera.assert(session_id.nil?, 'session_id must be nil for :sessions_init event')
|
|
47
137
|
Aspera.assert_type(info, String)
|
|
48
|
-
# Initialization of progress bar
|
|
49
138
|
@title = info
|
|
50
139
|
when :session_start
|
|
51
|
-
Aspera.assert_type(session_id, String)
|
|
52
140
|
Aspera.assert(info.nil?, 'info must be nil for :session_start event')
|
|
53
|
-
|
|
54
|
-
@sessions[session_id] = {
|
|
55
|
-
job_size: 0, # Total size of transfer (pre-calc)
|
|
56
|
-
current: 0,
|
|
57
|
-
running: true
|
|
58
|
-
}
|
|
141
|
+
start_session(session_id)
|
|
59
142
|
# Remove last pre-start message if any
|
|
60
143
|
@title = nil
|
|
61
144
|
when :session_size
|
|
62
|
-
|
|
63
|
-
Aspera.assert(!info.nil?, 'info must not be nil for :session_size event')
|
|
64
|
-
Aspera.assert_type(@sessions[session_id], Hash)
|
|
65
|
-
@sessions[session_id][:job_size] = info.to_i
|
|
66
|
-
sessions_total = total(:job_size)
|
|
67
|
-
@progress_bar.total = sessions_total unless sessions_total.eql?(@progress_bar.total) || sessions_total < @progress_bar.progress
|
|
145
|
+
session(session_id)[:size] = info.to_i unless info.nil?
|
|
68
146
|
when :transfer
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
@progress_bar.progress = sessions_current unless @progress_bar.progress.eql?(sessions_current) || sessions_current > total(:job_size)
|
|
147
|
+
unless info.nil?
|
|
148
|
+
current_session = session(session_id)
|
|
149
|
+
transferred = info.to_i
|
|
150
|
+
@transferred += transferred - current_session[:transferred] if transferred > current_session[:transferred]
|
|
151
|
+
current_session[:transferred] = transferred
|
|
152
|
+
add_sample
|
|
76
153
|
end
|
|
154
|
+
when :skip
|
|
155
|
+
session(session_id)[:skipped] = info.to_i unless info.nil?
|
|
77
156
|
when :session_end
|
|
78
|
-
Aspera.assert_type(session_id, String)
|
|
79
157
|
Aspera.assert(info.nil?, 'info must be nil for :session_end event')
|
|
80
158
|
# A session may be too short and finish before it has been started
|
|
81
|
-
@sessions[session_id][:running] = false if @sessions
|
|
159
|
+
@sessions[session_id][:running] = false if @sessions.key?(session_id)
|
|
82
160
|
when :end
|
|
83
161
|
Aspera.assert(session_id.nil?, 'session_id must be nil for :end event')
|
|
84
|
-
Aspera.
|
|
85
|
-
|
|
162
|
+
Aspera.assert_values(info, [nil, true, false]) { ':end event info' }
|
|
163
|
+
begin
|
|
164
|
+
finish(success: !info.eql?(false))
|
|
165
|
+
ensure
|
|
166
|
+
reset_state
|
|
167
|
+
end
|
|
168
|
+
return
|
|
86
169
|
else Aspera.error_unexpected_value(type) { 'event type' }
|
|
87
170
|
end
|
|
88
|
-
|
|
89
|
-
@progress_bar&.title = new_title unless @progress_bar&.title.eql?(new_title)
|
|
90
|
-
@progress_bar&.increment if !progress_provided && @progress_bar.progress.nil?
|
|
91
|
-
rescue ProgressBar::InvalidProgressError => e
|
|
92
|
-
Log.log.error { "Progress error: #{e}" }
|
|
171
|
+
display
|
|
93
172
|
end
|
|
94
173
|
|
|
95
|
-
|
|
174
|
+
# @param session_id [String] Session identifier
|
|
175
|
+
# @return [Hash] Session information, created if not already started
|
|
176
|
+
def session(session_id)
|
|
177
|
+
Aspera.assert_type(session_id, String)
|
|
178
|
+
@sessions[session_id] ||= {
|
|
179
|
+
size: nil, # Total size of session, `nil` if unknown
|
|
180
|
+
transferred: 0, # Bytes transferred
|
|
181
|
+
skipped: 0, # Bytes already at destination
|
|
182
|
+
floor: 0, # Progress reached before restart of session
|
|
183
|
+
running: true
|
|
184
|
+
}
|
|
185
|
+
end
|
|
186
|
+
|
|
187
|
+
# Start a session, or restart it: a restarted session notifies again its progress from zero
|
|
188
|
+
# @param session_id [String] Session identifier
|
|
189
|
+
def start_session(session_id)
|
|
190
|
+
restarted = @sessions.key?(session_id)
|
|
191
|
+
current_session = session(session_id)
|
|
192
|
+
if restarted
|
|
193
|
+
current_session[:floor] = done_bytes(current_session)
|
|
194
|
+
current_session[:transferred] = 0
|
|
195
|
+
current_session[:skipped] = 0
|
|
196
|
+
end
|
|
197
|
+
current_session[:running] = true
|
|
198
|
+
end
|
|
199
|
+
|
|
200
|
+
# @param session [Hash] Session information
|
|
201
|
+
# @return [Integer] Bytes processed by the session (transferred or skipped), not more than its size
|
|
202
|
+
def done_bytes(session)
|
|
203
|
+
done = [session[:floor], session[:transferred] + session[:skipped]].max
|
|
204
|
+
session[:size].nil? ? done : [done, session[:size]].min
|
|
205
|
+
end
|
|
206
|
+
|
|
207
|
+
# @return [Integer] Bytes processed by all sessions
|
|
208
|
+
def current_bytes
|
|
209
|
+
@sessions.each_value.sum { |s| done_bytes(s) }
|
|
210
|
+
end
|
|
211
|
+
|
|
212
|
+
# @return [Integer, nil] Total size of all sessions, `nil` if unknown for one of them
|
|
213
|
+
def total_bytes
|
|
214
|
+
return if @sessions.empty? || @sessions.each_value.any? { |s| s[:size].nil? }
|
|
215
|
+
@sessions.each_value.sum { |s| s[:size] }
|
|
216
|
+
end
|
|
217
|
+
|
|
218
|
+
# @return [Float] Monotonic time in seconds
|
|
219
|
+
def now
|
|
220
|
+
Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
221
|
+
end
|
|
222
|
+
|
|
223
|
+
# Record transferred bytes for rate computation
|
|
224
|
+
# @return [Array(Float, Integer)] The new sample
|
|
225
|
+
def add_sample
|
|
226
|
+
sample = [now, @transferred]
|
|
227
|
+
@start ||= sample
|
|
228
|
+
@samples.push(sample)
|
|
229
|
+
# Keep one sample older than the window
|
|
230
|
+
@samples.shift while @samples.length > 2 && @samples[1].first <= sample.first - RATE_WINDOW_SEC
|
|
231
|
+
sample
|
|
232
|
+
end
|
|
233
|
+
|
|
234
|
+
# @return [Float, nil] Rate on the sliding window, in bytes per second, `nil` if unknown
|
|
235
|
+
def rate
|
|
236
|
+
return if @samples.length < 2
|
|
237
|
+
elapsed = @samples.last.first - @samples.first.first
|
|
238
|
+
return if elapsed < RATE_MIN_SEC
|
|
239
|
+
(@samples.last.last - @samples.first.last) / elapsed
|
|
240
|
+
end
|
|
241
|
+
|
|
242
|
+
# @return [Float, nil] Average rate between first and last sample, in bytes per second, `nil` if unknown
|
|
243
|
+
def average_rate
|
|
244
|
+
return if @start.nil?
|
|
245
|
+
last = @samples.last
|
|
246
|
+
elapsed = last.first - @start.first
|
|
247
|
+
return unless elapsed.positive?
|
|
248
|
+
(last.last - @start.last) / elapsed
|
|
249
|
+
end
|
|
250
|
+
|
|
251
|
+
# Update the progress bar with current state
|
|
252
|
+
def display
|
|
253
|
+
if @progress_bar.nil?
|
|
254
|
+
@progress_bar = ProgressBar.create(output: @output, format: '%a %B', total: nil, autofinish: false, throttle_rate: 0)
|
|
255
|
+
@format = '%a %B'
|
|
256
|
+
end
|
|
257
|
+
current = current_bytes
|
|
258
|
+
total = total_bytes
|
|
259
|
+
update_bar(total, current)
|
|
260
|
+
fields = ['%a', '%B']
|
|
261
|
+
unless @sessions.empty?
|
|
262
|
+
bytes_per_second = rate
|
|
263
|
+
# Fixed width, so that bar does not change size
|
|
264
|
+
rate_text = self.class.format_rate(bytes_per_second).rjust(11)
|
|
265
|
+
if total.nil?
|
|
266
|
+
fields.push(self.class.format_bytes(current).rjust(8), rate_text)
|
|
267
|
+
else
|
|
268
|
+
eta = bytes_per_second&.positive? ? (total - current) / bytes_per_second : nil
|
|
269
|
+
fields.push('%j%%', rate_text, "ETA: #{self.class.format_duration(eta)}")
|
|
270
|
+
end
|
|
271
|
+
end
|
|
272
|
+
update_format(title_text(final: false), fields)
|
|
273
|
+
end
|
|
274
|
+
|
|
275
|
+
# Display final state of the progress bar
|
|
276
|
+
# @param success [Boolean] `false` if the transfer failed: progress is not completed
|
|
277
|
+
def finish(success:)
|
|
278
|
+
return if @progress_bar.nil?
|
|
279
|
+
current = current_bytes
|
|
280
|
+
total = total_bytes
|
|
281
|
+
if success
|
|
282
|
+
total ||= current
|
|
283
|
+
current = total
|
|
284
|
+
end
|
|
285
|
+
fields = ['%a']
|
|
286
|
+
unless total.nil?
|
|
287
|
+
update_bar(total, current)
|
|
288
|
+
fields.push('%B', '%j%%')
|
|
289
|
+
end
|
|
290
|
+
fields.push(self.class.format_bytes(current), self.class.format_rate(average_rate)) unless @sessions.empty?
|
|
291
|
+
# Last status (e.g. `running`) is no more relevant
|
|
292
|
+
@title = success ? nil : 'failed'
|
|
293
|
+
# Final line may be shorter (no bar)
|
|
294
|
+
@progress_bar.clear
|
|
295
|
+
update_format(title_text(final: true), fields)
|
|
296
|
+
success ? @progress_bar.finish : @progress_bar.stop
|
|
297
|
+
end
|
|
298
|
+
|
|
299
|
+
# Set total and progress of the bar, in an order accepted by `ruby-progressbar`
|
|
300
|
+
# @param total [Integer, nil] Total size, `nil` if unknown
|
|
301
|
+
# @param current [Integer] Processed bytes
|
|
302
|
+
def update_bar(total, current)
|
|
303
|
+
if total.nil?
|
|
304
|
+
# Unknown total: progress of the bar is used for animation only
|
|
305
|
+
@progress_bar.total = nil unless @progress_bar.total.nil?
|
|
306
|
+
@progress_bar.increment
|
|
307
|
+
elsif total >= @progress_bar.progress
|
|
308
|
+
@progress_bar.total = total unless @progress_bar.total.eql?(total)
|
|
309
|
+
@progress_bar.progress = current unless @progress_bar.progress.eql?(current)
|
|
310
|
+
else
|
|
311
|
+
@progress_bar.progress = current
|
|
312
|
+
@progress_bar.total = total
|
|
313
|
+
end
|
|
314
|
+
end
|
|
315
|
+
|
|
316
|
+
# @param final [Boolean] `true` for the final display
|
|
317
|
+
# @return [String] Title of the progress bar, with number of running sessions if several sessions
|
|
318
|
+
def title_text(final:)
|
|
319
|
+
parts = []
|
|
320
|
+
running = @sessions.count { |_, s| s[:running] }
|
|
321
|
+
parts.push("[#{running}]") if !final && running.positive? && @sessions.length > 1
|
|
322
|
+
parts.push(@title) unless @title.to_s.empty?
|
|
323
|
+
parts.join(' ')
|
|
324
|
+
end
|
|
96
325
|
|
|
97
|
-
|
|
98
|
-
|
|
326
|
+
# Change format of the bar, if changed
|
|
327
|
+
# @param title [String] Literal text
|
|
328
|
+
# @param fields [Array<String>] `ruby-progressbar` format elements
|
|
329
|
+
def update_format(title, fields)
|
|
330
|
+
new_format = (title.empty? ? fields : [title.gsub('%', '%%'), *fields]).join(' ')
|
|
331
|
+
return if new_format.eql?(@format)
|
|
332
|
+
@progress_bar.format = new_format
|
|
333
|
+
@format = new_format
|
|
99
334
|
end
|
|
100
335
|
end
|
|
101
336
|
end
|
|
@@ -10,17 +10,6 @@ module Aspera
|
|
|
10
10
|
# Mixin providing vault/keychain functionality to Plugin::Config.
|
|
11
11
|
# Depends on `options` and `context.main_folder` being available in the including class.
|
|
12
12
|
module VaultManager
|
|
13
|
-
def action_vault_show(label:, id: nil, **)
|
|
14
|
-
v = vault_required
|
|
15
|
-
kwargs = id && v.method(:get).parameters.any? { |_t, n| n == :id } ? {id: id} : {}
|
|
16
|
-
Result::SingleObject.new(v.get(label: label, **kwargs))
|
|
17
|
-
end
|
|
18
|
-
|
|
19
|
-
def action_vault_create(info:, **)
|
|
20
|
-
vault_required.set(info.symbolize_keys)
|
|
21
|
-
Result::Status.new('Secret added')
|
|
22
|
-
end
|
|
23
|
-
|
|
24
13
|
# Import secrets from a JSON array; skips entries missing :label
|
|
25
14
|
def action_vault_import(secrets:, **)
|
|
26
15
|
bulk_result(secrets, command: :import, id_result: 'label') do |entry|
|
|
@@ -36,12 +25,6 @@ module Aspera
|
|
|
36
25
|
Result::Status.new("Secret deleted: #{label}")
|
|
37
26
|
end
|
|
38
27
|
|
|
39
|
-
def action_vault_password(new_password:, **)
|
|
40
|
-
Aspera.assert(vault_required.respond_to?(:change_password), 'Vault does not support password change')
|
|
41
|
-
vault_required.change_password(new_password)
|
|
42
|
-
Result::Status.new('Vault password updated')
|
|
43
|
-
end
|
|
44
|
-
|
|
45
28
|
# @return [Keychain::Base] vault instance, raises if not configured
|
|
46
29
|
def vault_required
|
|
47
30
|
Aspera.assert(!vault.nil?, type: Cli::BadArgument) { 'Missing mandatory option: vault' }
|