aspera-cli 4.27.2 → 4.27.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (145) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/CHANGELOG.md +133 -0
  4. data/CONTRIBUTING.md +5 -2
  5. data/bin/ascli +3 -1
  6. data/docs/README.md +1006 -766
  7. data/lib/aspera/agent/base.rb +7 -2
  8. data/lib/aspera/agent/connect.rb +6 -8
  9. data/lib/aspera/agent/desktop.rb +2 -6
  10. data/lib/aspera/agent/direct.rb +52 -22
  11. data/lib/aspera/agent/node.rb +9 -8
  12. data/lib/aspera/agent/transferd.rb +2 -2
  13. data/lib/aspera/api/alee.rb +1 -1
  14. data/lib/aspera/api/aoc.rb +14 -12
  15. data/lib/aspera/api/ats.rb +1 -1
  16. data/lib/aspera/api/cos_node.rb +2 -2
  17. data/lib/aspera/api/faspex.rb +11 -7
  18. data/lib/aspera/api/httpgw.rb +38 -35
  19. data/lib/aspera/api/node.rb +39 -34
  20. data/lib/aspera/ascmd.rb +3 -1
  21. data/lib/aspera/ascp/installation.rb +63 -28
  22. data/lib/aspera/ascp/management.rb +1 -0
  23. data/lib/aspera/assert.rb +4 -0
  24. data/lib/aspera/cli/ascp_actions.rb +20 -41
  25. data/lib/aspera/cli/async_transfer_store.rb +12 -11
  26. data/lib/aspera/cli/bootstrapper.rb +14 -16
  27. data/lib/aspera/cli/command_line.rb +252 -0
  28. data/lib/aspera/cli/command_registry.rb +215 -37
  29. data/lib/aspera/cli/command_spec.rb +104 -15
  30. data/lib/aspera/cli/completion/ascli.bash +12 -0
  31. data/lib/aspera/cli/completion/ascli.fish +16 -0
  32. data/lib/aspera/cli/completion/ascli.zsh +19 -0
  33. data/lib/aspera/cli/context.rb +3 -0
  34. data/lib/aspera/cli/deprecation.rb +37 -0
  35. data/lib/aspera/cli/extended_value.rb +6 -3
  36. data/lib/aspera/cli/formatter.rb +94 -80
  37. data/lib/aspera/cli/gem_checker.rb +1 -1
  38. data/lib/aspera/cli/hints.rb +7 -6
  39. data/lib/aspera/cli/http.rb +22 -34
  40. data/lib/aspera/cli/info.rb +3 -0
  41. data/lib/aspera/cli/mcp_tool.rb +47 -83
  42. data/lib/aspera/cli/option_declarator.rb +33 -42
  43. data/lib/aspera/cli/option_registry.rb +69 -0
  44. data/lib/aspera/cli/option_types.rb +105 -0
  45. data/lib/aspera/cli/option_value.rb +278 -0
  46. data/lib/aspera/cli/options.schema.yaml +124 -15
  47. data/lib/aspera/cli/parser.rb +333 -862
  48. data/lib/aspera/cli/plugins/alee.rb +7 -4
  49. data/lib/aspera/cli/plugins/aoc.rb +545 -518
  50. data/lib/aspera/cli/plugins/ats.rb +59 -80
  51. data/lib/aspera/cli/plugins/base.rb +221 -265
  52. data/lib/aspera/cli/plugins/basic_auth.rb +2 -10
  53. data/lib/aspera/cli/plugins/config.rb +263 -184
  54. data/lib/aspera/cli/plugins/console.rb +103 -39
  55. data/lib/aspera/cli/plugins/cos.rb +6 -23
  56. data/lib/aspera/cli/plugins/factory.rb +3 -0
  57. data/lib/aspera/cli/plugins/faspex5.rb +204 -182
  58. data/lib/aspera/cli/plugins/faspio.rb +6 -11
  59. data/lib/aspera/cli/plugins/httpgw.rb +8 -11
  60. data/lib/aspera/cli/plugins/mcp.rb +20 -55
  61. data/lib/aspera/cli/plugins/node.rb +300 -327
  62. data/lib/aspera/cli/plugins/orchestrator.rb +152 -110
  63. data/lib/aspera/cli/plugins/preview.rb +96 -105
  64. data/lib/aspera/cli/plugins/server.rb +78 -53
  65. data/lib/aspera/cli/plugins/shares.rb +80 -131
  66. data/lib/aspera/cli/preset_actions.rb +44 -27
  67. data/lib/aspera/cli/preset_manager.rb +44 -19
  68. data/lib/aspera/cli/prompt.rb +36 -0
  69. data/lib/aspera/cli/result.rb +42 -36
  70. data/lib/aspera/cli/runner.rb +32 -59
  71. data/lib/aspera/cli/special_values.rb +5 -0
  72. data/lib/aspera/cli/sync_actions.rb +51 -46
  73. data/lib/aspera/cli/terminal_formatter.rb +9 -3
  74. data/lib/aspera/cli/transfer_actions.rb +14 -9
  75. data/lib/aspera/cli/transfer_agent.rb +34 -38
  76. data/lib/aspera/cli/transfer_progress.rb +290 -55
  77. data/lib/aspera/cli/vault_manager.rb +0 -17
  78. data/lib/aspera/cli/version.rb +1 -1
  79. data/lib/aspera/cli/wizard.rb +5 -3
  80. data/lib/aspera/coverage.rb +1 -1
  81. data/lib/aspera/environment.rb +35 -5
  82. data/lib/aspera/faspex_gw.rb +2 -1
  83. data/lib/aspera/faspex_postproc.rb +1 -0
  84. data/lib/aspera/graphql.rb +5 -5
  85. data/lib/aspera/json_rpc/client.rb +5 -5
  86. data/lib/aspera/keychain/encrypted_hash.rb +2 -2
  87. data/lib/aspera/keychain/factory.rb +2 -1
  88. data/lib/aspera/keychain/one_password_api.rb +1 -1
  89. data/lib/aspera/link_header.rb +2 -2
  90. data/lib/aspera/log.rb +47 -27
  91. data/lib/aspera/markdown.rb +2 -0
  92. data/lib/aspera/mime.rb +25 -0
  93. data/lib/aspera/node_emulator.rb +759 -0
  94. data/lib/aspera/oauth/base.rb +37 -26
  95. data/lib/aspera/oauth/factory.rb +7 -3
  96. data/lib/aspera/oauth/generic.rb +1 -1
  97. data/lib/aspera/oauth/json_credentials.rb +34 -0
  98. data/lib/aspera/oauth/jwt.rb +4 -5
  99. data/lib/aspera/oauth/web.rb +9 -8
  100. data/lib/aspera/oauth.rb +1 -0
  101. data/lib/aspera/persistency_folder.rb +1 -3
  102. data/lib/aspera/preview/file_types.rb +4 -4
  103. data/lib/aspera/preview/generator.rb +11 -1
  104. data/lib/aspera/preview/options.schema.yaml +119 -0
  105. data/lib/aspera/preview/terminal.rb +4 -3
  106. data/lib/aspera/preview/utils.rb +9 -6
  107. data/lib/aspera/products/connect.rb +1 -1
  108. data/lib/aspera/rainbow.rb +7 -0
  109. data/lib/aspera/rest/aspera_errors.rb +72 -0
  110. data/lib/aspera/rest/call_error.rb +27 -0
  111. data/lib/aspera/rest/client.rb +523 -0
  112. data/lib/aspera/rest/error_analyzer.rb +113 -0
  113. data/lib/aspera/rest/list.rb +149 -0
  114. data/lib/aspera/rest/parameters.rb +55 -0
  115. data/lib/aspera/rest/util.rb +176 -0
  116. data/lib/aspera/rest.rb +7 -621
  117. data/lib/aspera/schema/IBM Aspera Console-enhanced.yaml +1125 -0
  118. data/lib/aspera/schema/IBM Aspera Faspex API-5.0-enhanced.yaml +0 -20
  119. data/lib/aspera/schema/IBM Aspera Orchestrator API-v1.yaml +1784 -0
  120. data/lib/aspera/schema/IBM Aspera on Cloud API-0.2.6-enhanced.yaml +1230 -137
  121. data/lib/aspera/schema/IBM Aspera on Cloud Automation API-1.0.5-enhanced.yaml +2395 -0
  122. data/lib/aspera/schema/IBM_Aspera_Shares.yaml +14 -13
  123. data/lib/aspera/schema/documentation.rb +13 -3
  124. data/lib/aspera/schema/reader.rb +12 -18
  125. data/lib/aspera/schema/registry.rb +23 -1
  126. data/lib/aspera/schema/validator.rb +92 -0
  127. data/lib/aspera/secret_hider.rb +36 -25
  128. data/lib/aspera/string_ext.rb +15 -0
  129. data/lib/aspera/temp_file_manager.rb +6 -5
  130. data/lib/aspera/transfer/parameters.rb +2 -0
  131. data/lib/aspera/transfer/spec.rb +1 -0
  132. data/lib/aspera/uri_reader.rb +11 -11
  133. data/lib/aspera/web_auth/index.html +147 -0
  134. data/lib/aspera/web_auth/server.rb +81 -0
  135. data.tar.gz.sig +0 -0
  136. metadata +43 -9
  137. metadata.gz.sig +0 -0
  138. data/lib/aspera/colors.rb +0 -79
  139. data/lib/aspera/node_simulator.rb +0 -344
  140. data/lib/aspera/preview/options.rb +0 -45
  141. data/lib/aspera/rest_call_error.rb +0 -25
  142. data/lib/aspera/rest_error_analyzer.rb +0 -111
  143. data/lib/aspera/rest_errors_aspera.rb +0 -58
  144. data/lib/aspera/rest_list.rb +0 -136
  145. data/lib/aspera/web_auth.rb +0 -211
@@ -15,7 +15,8 @@ module Aspera
15
15
  #
16
16
  # Architecture:
17
17
  # - remote-daemon agents (desktop, node, connect, transferd): transfer_id + agent_params
18
- # are persisted in AsyncTransferStore; status is re-queried via Agent::Xxx.transfer_status
18
+ # are persisted in AsyncTransferStore (without secrets, taken from current options);
19
+ # status is re-queried via Agent::Xxx.transfer_status
19
20
  # - direct agent: transfers live in Ruby threads - store is the agent's @sessions;
20
21
  # status is not persistable across process restarts (returns an informational message)
21
22
  module TransferActions
@@ -25,10 +26,12 @@ module Aspera
25
26
  store = async_transfer_store
26
27
  entry = store.read(job_id)
27
28
  Aspera.assert(!entry.nil?, type: Cli::BadArgument) { "Unknown job_id: #{job_id}" }
29
+ # Secrets are not persisted: take them from current transfer options (e.g. `--transfer`), for the same agent type
30
+ agent_params = current_agent_params(entry['agent_type']).merge(entry['agent_params'] || {})
28
31
  # Inject the in-process agent reference (direct/httpgw) if still alive in this process.
29
32
  ref = store.agent_ref(job_id)
30
- entry['agent_params']['_agent_ref'] = ref if ref
31
- live = query_live_status(entry)
33
+ agent_params['_agent_ref'] = ref if ref
34
+ live = query_live_status(entry.merge('agent_params' => agent_params))
32
35
  if live
33
36
  entry.merge!(live)
34
37
  store.write(job_id, entry)
@@ -36,12 +39,6 @@ module Aspera
36
39
  Result::SingleObject.new(entry)
37
40
  end
38
41
 
39
- # List all persisted async transfer entries.
40
- def action_transfer_list(**)
41
- rows = async_transfer_store.list
42
- Result::ObjectList.new(rows, fields: %w[job_id agent_type status started_at ended_at bytes_transferred transfer_id])
43
- end
44
-
45
42
  TERMINAL_STATUSES = %w[completed failed cancelled].freeze
46
43
  private_constant :TERMINAL_STATUSES
47
44
 
@@ -59,6 +56,14 @@ module Aspera
59
56
 
60
57
  private
61
58
 
59
+ # @param agent_type [String] agent of the async transfer
60
+ # @return [Hash] agent parameters from current transfer options (e.g. `--transfer`), if for the same agent type
61
+ def current_agent_params(agent_type)
62
+ current = context.transfer.transfer_options
63
+ return {} unless (current['agent'] || 'direct').to_s.eql?(agent_type)
64
+ current.except('agent', 'asynchronous')
65
+ end
66
+
62
67
  # Lazy accessor - requires context.persistency (available in all Base sub-classes).
63
68
  # Delegate to the TransferAgent's store so that the in-memory agent-ref
64
69
  # registry is shared between the two components in the same process.
@@ -32,18 +32,24 @@ module Aspera
32
32
  <%=ts.to_yaml%>
33
33
  END_OF_TEMPLATE
34
34
  CP4I_REMOTE_HOST_LB = 'N/A'
35
+ # Transfer spec parameters overridden by option `ts`
36
+ TS_DEFAULTS = {
37
+ 'create_dir' => true,
38
+ 'resume_policy' => 'sparse_csum'
39
+ }.freeze
35
40
  private_constant :FILE_LIST_FROM_TRANSFER_SPEC,
36
41
  :FILE_LIST_OPTIONS,
37
- :DEFAULT_TRANSFER_NOTIFY_TEMPLATE
42
+ :DEFAULT_TRANSFER_NOTIFY_TEMPLATE,
43
+ :TS_DEFAULTS
38
44
 
39
45
  extend OptionDeclarator
40
46
 
41
- option :ts, description: 'Override transfer spec values', schema: Schema::Registry::TRANSFER_SPEC
47
+ option :ts, description: 'Override transfer spec values', schema: Schema::Registry::TRANSFER_SPEC, on_set: :user_transfer_spec=
42
48
  option :to_folder, description: 'Destination folder for transferred files'
43
49
  option :sources, description: "How list of transferred files is provided (#{FILE_LIST_OPTIONS.join(',')})", default: FILE_LIST_FROM_ARGS
44
50
  option :src_type, description: 'Type of file list', allowed: %i[list pair], default: :list
45
- option :transfer, description: 'Transfer agent type, or agent parameters with optional agent key', allowed: [Hash, String], schema: Schema::Registry::TRANSFER_AGENT_OPTIONS
46
- option :transfer_info, description: 'Parameters for transfer agent', deprecation: 'use --transfer instead', schema: Schema::Registry::TRANSFER_AGENT_OPTIONS
51
+ option :transfer, description: 'Transfer agent type, or agent parameters with optional agent key', allowed: [Hash, String], shorthand: 'agent', schema: Schema::Registry::TRANSFER_AGENT_OPTIONS
52
+ option :transfer_info, description: 'Parameters for transfer agent', deprecation: {last: '4.26.2', message: 'use --transfer instead'}, schema: Schema::Registry::TRANSFER_AGENT_OPTIONS
47
53
 
48
54
  # @param context [Context] Application context
49
55
  def initialize(context)
@@ -51,22 +57,15 @@ module Aspera
51
57
  Aspera.assert_type(context.options, Parser) { 'context.options' }
52
58
  @context = context
53
59
  # Command line can override transfer spec
54
- @user_transfer_spec = {
55
- 'create_dir' => true,
56
- 'resume_policy' => 'sparse_csum'
57
- }
58
- # options for transfer agent (agent type + agent-specific parameters)
59
- @transfer_options = {}
60
+ @user_transfer_spec = TS_DEFAULTS.dup
60
61
  # the currently selected transfer agent
61
62
  @agent = nil
62
63
  # source/destination pair, like "paths" of transfer spec
63
64
  @transfer_paths = nil
64
65
  # HTTPGW URL provided by webapp
65
66
  @httpgw_url_lambda = nil
66
- self.class.declare_options(@context.options)
67
- @context.options.set_handler(:ts, object: self, method: :user_transfer_spec)
68
- @context.options.set_handler(:transfer, object: self, method: :option_transfer)
69
- @context.options.set_handler(:transfer_info, object: self, method: :transfer_options)
67
+ # Instance variables used by `on_set` callbacks are set above
68
+ self.class.declare_options(@context.options, target: self)
70
69
  @context.options.parse_options!
71
70
  @notification_cb = nil
72
71
  if !@context.options.get_option(:notify_to).nil?
@@ -80,22 +79,17 @@ module Aspera
80
79
  end
81
80
  end
82
81
 
83
- attr_accessor :user_transfer_spec, :transfer_options
82
+ attr_reader :user_transfer_spec
84
83
 
85
- # Composite option handler for :transfer
86
- # String value: shorthand for agent type, stored as {'agent' => value}
87
- # Hash value: merged into @transfer_options (may include 'agent' key)
88
- def option_transfer(_option_sym, operation, value = nil)
89
- Aspera.assert_values(operation, %i[set get])
90
- case operation
91
- when :set
92
- value = {'agent' => value} if value.is_a?(String)
93
- Aspera.assert_type(value, Hash)
94
- @transfer_options = @transfer_options.deep_merge(value)
95
- when :get
96
- return @transfer_options
97
- end
98
- nil
84
+ # `on_set` callback of option `ts`: values override defaults
85
+ # @param value [Hash] transfer spec parameters
86
+ def user_transfer_spec=(value)
87
+ @user_transfer_spec = TS_DEFAULTS.merge(value)
88
+ end
89
+
90
+ # @return [Hash] agent type (key `agent`) and agent parameters, from options `transfer` and deprecated `transfer_info`
91
+ def transfer_options
92
+ @context.options.get_option(:transfer_info).deep_merge(@context.options.get_option(:transfer))
99
93
  end
100
94
 
101
95
  def agent_instance=(instance)
@@ -106,10 +100,10 @@ module Aspera
106
100
  def agent_instance
107
101
  return @agent unless @agent.nil?
108
102
  # agent type: from composite option 'agent' key, default :direct
109
- raw_type = @transfer_options['agent'] || :direct
103
+ raw_type = transfer_options['agent'] || :direct
110
104
  agent_type = Parser.get_from_list(raw_type.to_s, 'transfer agent', Agent::Factory::ALL.keys)
111
105
  # set keys as symbols, strip internal keys not forwarded to the agent constructor
112
- agent_options = @transfer_options.except('agent', 'asynchronous').symbolize_keys
106
+ agent_options = transfer_options.except('agent', 'asynchronous').symbolize_keys
113
107
  agent_options[:progress] = @context.progress_bar
114
108
  agent_options[:config_dir] = @context.main_folder
115
109
  # special cases
@@ -121,8 +115,10 @@ module Aspera
121
115
  agent_options.merge!(@context.presets.by_name(param_set_name).symbolize_keys)
122
116
  end
123
117
  when :direct
124
- # by default do not display ascp native progress bar
118
+ # By default, no native `ascp` progress bar
125
119
  agent_options[:quiet] = true unless agent_options.key?(:quiet)
120
+ # Native `ascp` progress bar requested: no other progress bar, unless option `progress_bar` is set
121
+ agent_options[:progress] = nil if !agent_options[:quiet] && @context.options.option_def(:progress_bar).source.eql?(:default)
126
122
  agent_options[:check_ignore_cb] = ->(host, port) { @context.http_config.ignore_cert?(host, port) }
127
123
  # JRuby
128
124
  agent_options[:trusted_certs] = @context.http_config.trusted_cert_locations unless agent_options.key?(:trusted_certs)
@@ -210,8 +206,8 @@ module Aspera
210
206
  when FILE_LIST_FROM_TRANSFER_SPEC
211
207
  Log.log.debug('assume list provided in transfer spec')
212
208
  special_case_direct_with_list =
213
- (@transfer_options['agent'] || :direct).to_sym.eql?(:direct) &&
214
- Transfer::Parameters.ascp_args_file_list?(@transfer_options['ascp_args'])
209
+ (transfer_options['agent'] || :direct).to_sym.eql?(:direct) &&
210
+ Transfer::Parameters.ascp_args_file_list?(transfer_options['ascp_args'])
215
211
  Aspera.assert(!@transfer_paths.nil? || special_case_direct_with_list, type: Cli::BadArgument) { 'transfer spec on command line must have sources' }
216
212
  # can be nil
217
213
  @transfer_paths
@@ -227,7 +223,7 @@ module Aspera
227
223
 
228
224
  # Start a transfer and wait for completion, plugins shall use this method
229
225
  # @param transfer_spec [Hash] transfer specification
230
- # @param rest_token [Rest, nil] if oauth token regeneration supported
226
+ # @param rest_token [Rest::Client, nil] if oauth token regeneration supported
231
227
  def start(transfer_spec, rest_token: nil)
232
228
  # check parameters
233
229
  Aspera.assert_type(transfer_spec, Hash) { 'transfer_spec' }
@@ -264,13 +260,13 @@ module Aspera
264
260
  # create transfer agent
265
261
  agent_instance.start_transfer(transfer_spec, token_regenerator: rest_token)
266
262
  # --- async mode ---
267
- if @transfer_options['asynchronous']
268
- agent_type = (@transfer_options['agent'] || 'direct').to_s
263
+ if transfer_options['asynchronous']
264
+ agent_type = (transfer_options['agent'] || 'direct').to_s
269
265
  # In-process agents (direct, httpgw) expose last_job_id directly.
270
266
  # Remote-daemon agents do not override last_job_id (returns nil): generate a UUID.
271
267
  job_id = agent_instance.last_job_id || SecureRandom.uuid
272
268
  # Base agent_params from transfer options (strip internal keys)
273
- agent_params = @transfer_options.except('agent', 'asynchronous')
269
+ agent_params = transfer_options.except('agent', 'asynchronous')
274
270
  # For daemon agents: capture any runtime-resolved connection detail
275
271
  case agent_type
276
272
  when 'desktop'
@@ -1,5 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ require 'monitor'
4
+ require 'aspera/environment'
3
5
  require 'aspera/log'
4
6
  require 'aspera/assert'
5
7
  require 'ruby-progressbar'
@@ -7,95 +9,328 @@ require 'ruby-progressbar'
7
9
  module Aspera
8
10
  module Cli
9
11
  # Progress bar for transfers.
10
- # Supports multi-session.
12
+ # Supports multi-session: events may come from several threads.
11
13
  # Note that we can have this case:
12
- # 2 sessions (-C x:2), but one session fails and restarts...
14
+ # 2 sessions (-C x:2), but one session fails and restarts (resume): it restarts with the same session id.
15
+ # Errors are logged, never raised: the progress bar shall not fail a transfer.
13
16
  class TransferProgress
14
- def initialize
15
- @progress_bar = nil
16
- # Key is session id
17
- @sessions = {}
18
- @completed = false
19
- @title = nil
17
+ # Rate is computed on this sliding window
18
+ RATE_WINDOW_SEC = 5.0
19
+ # Rate is not displayed on a shorter period
20
+ RATE_MIN_SEC = 1.0
21
+ # Decimal units, like `ascp`
22
+ KILO = 1000
23
+ BITS_PER_MEGABIT = KILO * KILO
24
+ BYTE_UNITS = %w[B KB MB GB TB PB].freeze
25
+ private_constant :RATE_WINDOW_SEC, :RATE_MIN_SEC, :KILO, :BITS_PER_MEGABIT, :BYTE_UNITS
26
+
27
+ class << self
28
+ # @param bytes [Integer] Size in bytes
29
+ # @return [String] Size with decimal unit, e.g. `12.3 MB`
30
+ def format_bytes(bytes)
31
+ value = bytes.to_f
32
+ index = 0
33
+ while value >= KILO && index < BYTE_UNITS.length - 1
34
+ value /= KILO
35
+ index += 1
36
+ end
37
+ return "#{bytes} B" if index.zero?
38
+ format('%.1f %s', value, BYTE_UNITS[index])
39
+ end
40
+
41
+ # @param bytes_per_second [Float, nil] Rate, `nil` if unknown
42
+ # @return [String] Rate in megabits per second (decimal, like `ascp`)
43
+ def format_rate(bytes_per_second)
44
+ return '-- Mbps' if bytes_per_second.nil?
45
+ mbps = bytes_per_second * Environment::BITS_PER_BYTE / BITS_PER_MEGABIT
46
+ mbps < 10 ? format('%.2f Mbps', mbps) : format('%.0f Mbps', mbps)
47
+ end
48
+
49
+ # @param seconds [Numeric, nil] Duration, `nil` if unknown
50
+ # @return [String] Duration as `hh:mm:ss`
51
+ def format_duration(seconds)
52
+ return '--:--:--' if seconds.nil?
53
+ seconds = seconds.round
54
+ format('%02d:%02d:%02d', seconds / 3600, (seconds / 60) % 60, seconds % 60)
55
+ end
56
+ end
57
+
58
+ # @param output [IO] Where the progress bar is displayed: not on `$stdout`, which may be redirected for command output
59
+ def initialize(output: $stderr)
60
+ @output = output
61
+ # Re-entrant, see `suspend`
62
+ @mutex = Monitor.new
63
+ reset_state
20
64
  end
21
65
 
22
66
  # Reset progress bar, to re-use it.
23
67
  def reset
24
- send(:initialize)
68
+ @mutex.synchronize { reset_state }
25
69
  end
26
70
 
27
- # Called by user of progress bar with a status on a transfer session
28
- # @param session_id [String] the unique identifier of a transfer session
29
- # @param type [Symbol] one of: sessions_init, session_start, session_size, transfer, session_end and end
30
- # @param info [Object, nil] optional specific additional info for the given event type
71
+ # Called by user of progress bar with a status on a transfer session.
72
+ # Event types:
73
+ # - `sessions_init` : `info` is a status message, displayed until a session starts
74
+ # - `session_start` : a session starts, or restarts (same `session_id`)
75
+ # - `session_size` : `info` is the total size of the session
76
+ # - `transfer` : `info` is the number of bytes transferred by the session
77
+ # - `skip` : `info` is the number of bytes of the session already at destination (not transferred)
78
+ # - `session_end` : a session ends
79
+ # - `end` : all sessions are finished, `info` is `false` if the transfer failed
80
+ # @param type [Symbol] Event type
81
+ # @param session_id [String, nil] Unique identifier of a transfer session, same when the session is restarted
82
+ # @param info [Object, nil] Optional specific additional info for the given event type
31
83
  def event(type, session_id: nil, info: nil)
32
84
  Log.log.trace1 { "progress: #{type} #{session_id} #{info}" }
33
- return if @completed
34
- if @progress_bar.nil?
35
- @progress_bar = ProgressBar.create(
36
- format: '%t %a %B %p%% %r Mbps %E',
37
- rate_scale: lambda { |rate| rate / Environment::BYTES_PER_MEBIBIT },
38
- title: '',
39
- total: nil
40
- )
85
+ @mutex.synchronize { process_event(type, session_id, info) }
86
+ rescue StandardError => e
87
+ Log.log.warn { "Progress bar: #{e.class}: #{e.message}" } unless @error_logged
88
+ @error_logged = true
89
+ end
90
+
91
+ # Execute the block with the progress bar cleared, and then display it again.
92
+ # Used to write log lines on the terminal.
93
+ # Does not wait if the progress bar is being updated by another thread (no deadlock with a logger lock).
94
+ def suspend
95
+ return yield unless @mutex.try_enter
96
+ begin
97
+ return yield if @progress_bar.nil? || !@output.tty?
98
+ @progress_bar.clear
99
+ @output.flush
100
+ begin
101
+ yield
102
+ ensure
103
+ @progress_bar.refresh(force: true)
104
+ end
105
+ ensure
106
+ @mutex.exit
41
107
  end
42
- progress_provided = false
108
+ end
109
+
110
+ private
111
+
112
+ def reset_state
113
+ @progress_bar = nil
114
+ # Key is session id
115
+ @sessions = {}
116
+ # Status displayed before session start
117
+ @title = nil
118
+ # Format currently set on the progress bar
119
+ @format = nil
120
+ # [Integer] Bytes actually transferred, by all sessions, including previous runs of restarted sessions
121
+ @transferred = 0
122
+ # [Array<Array(Float, Integer)>] (time, transferred bytes) on the rate window
123
+ @samples = []
124
+ # [Array(Float, Integer), nil] First sample (data starts to flow, not session start), for average rate
125
+ @start = nil
126
+ @error_logged = false
127
+ end
128
+
129
+ # @param type [Symbol] Event type
130
+ # @param session_id [String, nil] Session identifier
131
+ # @param info [Object, nil] Event information
132
+ def process_event(type, session_id, info)
43
133
  case type
44
134
  when :sessions_init
45
135
  # Give opportunity to show progress of initialization with multiple status
46
136
  Aspera.assert(session_id.nil?, 'session_id must be nil for :sessions_init event')
47
137
  Aspera.assert_type(info, String)
48
- # Initialization of progress bar
49
138
  @title = info
50
139
  when :session_start
51
- Aspera.assert_type(session_id, String)
52
140
  Aspera.assert(info.nil?, 'info must be nil for :session_start event')
53
- Aspera.assert(!@sessions[session_id]) { "Session #{session_id} already started" }
54
- @sessions[session_id] = {
55
- job_size: 0, # Total size of transfer (pre-calc)
56
- current: 0,
57
- running: true
58
- }
141
+ start_session(session_id)
59
142
  # Remove last pre-start message if any
60
143
  @title = nil
61
144
  when :session_size
62
- Aspera.assert_type(session_id, String)
63
- Aspera.assert(!info.nil?, 'info must not be nil for :session_size event')
64
- Aspera.assert_type(@sessions[session_id], Hash)
65
- @sessions[session_id][:job_size] = info.to_i
66
- sessions_total = total(:job_size)
67
- @progress_bar.total = sessions_total unless sessions_total.eql?(@progress_bar.total) || sessions_total < @progress_bar.progress
145
+ session(session_id)[:size] = info.to_i unless info.nil?
68
146
  when :transfer
69
- Aspera.assert_type(session_id, String)
70
- Aspera.assert_type(@sessions[session_id], Hash)
71
- if !@progress_bar.total.nil? && !info.nil?
72
- progress_provided = true
73
- @sessions[session_id][:current] = info.to_i
74
- sessions_current = total(:current)
75
- @progress_bar.progress = sessions_current unless @progress_bar.progress.eql?(sessions_current) || sessions_current > total(:job_size)
147
+ unless info.nil?
148
+ current_session = session(session_id)
149
+ transferred = info.to_i
150
+ @transferred += transferred - current_session[:transferred] if transferred > current_session[:transferred]
151
+ current_session[:transferred] = transferred
152
+ add_sample
76
153
  end
154
+ when :skip
155
+ session(session_id)[:skipped] = info.to_i unless info.nil?
77
156
  when :session_end
78
- Aspera.assert_type(session_id, String)
79
157
  Aspera.assert(info.nil?, 'info must be nil for :session_end event')
80
158
  # A session may be too short and finish before it has been started
81
- @sessions[session_id][:running] = false if @sessions[session_id].is_a?(Hash)
159
+ @sessions[session_id][:running] = false if @sessions.key?(session_id)
82
160
  when :end
83
161
  Aspera.assert(session_id.nil?, 'session_id must be nil for :end event')
84
- Aspera.assert(info.nil?, 'info must be nil for :end event')
85
- @progress_bar.finish
162
+ Aspera.assert_values(info, [nil, true, false]) { ':end event info' }
163
+ begin
164
+ finish(success: !info.eql?(false))
165
+ ensure
166
+ reset_state
167
+ end
168
+ return
86
169
  else Aspera.error_unexpected_value(type) { 'event type' }
87
170
  end
88
- new_title = @sessions.length < 2 ? @title.to_s : "[#{@sessions.count { |_i, d| d[:running] }}] #{@title}"
89
- @progress_bar&.title = new_title unless @progress_bar&.title.eql?(new_title)
90
- @progress_bar&.increment if !progress_provided && @progress_bar.progress.nil?
91
- rescue ProgressBar::InvalidProgressError => e
92
- Log.log.error { "Progress error: #{e}" }
171
+ display
93
172
  end
94
173
 
95
- private
174
+ # @param session_id [String] Session identifier
175
+ # @return [Hash] Session information, created if not already started
176
+ def session(session_id)
177
+ Aspera.assert_type(session_id, String)
178
+ @sessions[session_id] ||= {
179
+ size: nil, # Total size of session, `nil` if unknown
180
+ transferred: 0, # Bytes transferred
181
+ skipped: 0, # Bytes already at destination
182
+ floor: 0, # Progress reached before restart of session
183
+ running: true
184
+ }
185
+ end
186
+
187
+ # Start a session, or restart it: a restarted session notifies again its progress from zero
188
+ # @param session_id [String] Session identifier
189
+ def start_session(session_id)
190
+ restarted = @sessions.key?(session_id)
191
+ current_session = session(session_id)
192
+ if restarted
193
+ current_session[:floor] = done_bytes(current_session)
194
+ current_session[:transferred] = 0
195
+ current_session[:skipped] = 0
196
+ end
197
+ current_session[:running] = true
198
+ end
199
+
200
+ # @param session [Hash] Session information
201
+ # @return [Integer] Bytes processed by the session (transferred or skipped), not more than its size
202
+ def done_bytes(session)
203
+ done = [session[:floor], session[:transferred] + session[:skipped]].max
204
+ session[:size].nil? ? done : [done, session[:size]].min
205
+ end
206
+
207
+ # @return [Integer] Bytes processed by all sessions
208
+ def current_bytes
209
+ @sessions.each_value.sum { |s| done_bytes(s) }
210
+ end
211
+
212
+ # @return [Integer, nil] Total size of all sessions, `nil` if unknown for one of them
213
+ def total_bytes
214
+ return if @sessions.empty? || @sessions.each_value.any? { |s| s[:size].nil? }
215
+ @sessions.each_value.sum { |s| s[:size] }
216
+ end
217
+
218
+ # @return [Float] Monotonic time in seconds
219
+ def now
220
+ Process.clock_gettime(Process::CLOCK_MONOTONIC)
221
+ end
222
+
223
+ # Record transferred bytes for rate computation
224
+ # @return [Array(Float, Integer)] The new sample
225
+ def add_sample
226
+ sample = [now, @transferred]
227
+ @start ||= sample
228
+ @samples.push(sample)
229
+ # Keep one sample older than the window
230
+ @samples.shift while @samples.length > 2 && @samples[1].first <= sample.first - RATE_WINDOW_SEC
231
+ sample
232
+ end
233
+
234
+ # @return [Float, nil] Rate on the sliding window, in bytes per second, `nil` if unknown
235
+ def rate
236
+ return if @samples.length < 2
237
+ elapsed = @samples.last.first - @samples.first.first
238
+ return if elapsed < RATE_MIN_SEC
239
+ (@samples.last.last - @samples.first.last) / elapsed
240
+ end
241
+
242
+ # @return [Float, nil] Average rate between first and last sample, in bytes per second, `nil` if unknown
243
+ def average_rate
244
+ return if @start.nil?
245
+ last = @samples.last
246
+ elapsed = last.first - @start.first
247
+ return unless elapsed.positive?
248
+ (last.last - @start.last) / elapsed
249
+ end
250
+
251
+ # Update the progress bar with current state
252
+ def display
253
+ if @progress_bar.nil?
254
+ @progress_bar = ProgressBar.create(output: @output, format: '%a %B', total: nil, autofinish: false, throttle_rate: 0)
255
+ @format = '%a %B'
256
+ end
257
+ current = current_bytes
258
+ total = total_bytes
259
+ update_bar(total, current)
260
+ fields = ['%a', '%B']
261
+ unless @sessions.empty?
262
+ bytes_per_second = rate
263
+ # Fixed width, so that bar does not change size
264
+ rate_text = self.class.format_rate(bytes_per_second).rjust(11)
265
+ if total.nil?
266
+ fields.push(self.class.format_bytes(current).rjust(8), rate_text)
267
+ else
268
+ eta = bytes_per_second&.positive? ? (total - current) / bytes_per_second : nil
269
+ fields.push('%j%%', rate_text, "ETA: #{self.class.format_duration(eta)}")
270
+ end
271
+ end
272
+ update_format(title_text(final: false), fields)
273
+ end
274
+
275
+ # Display final state of the progress bar
276
+ # @param success [Boolean] `false` if the transfer failed: progress is not completed
277
+ def finish(success:)
278
+ return if @progress_bar.nil?
279
+ current = current_bytes
280
+ total = total_bytes
281
+ if success
282
+ total ||= current
283
+ current = total
284
+ end
285
+ fields = ['%a']
286
+ unless total.nil?
287
+ update_bar(total, current)
288
+ fields.push('%B', '%j%%')
289
+ end
290
+ fields.push(self.class.format_bytes(current), self.class.format_rate(average_rate)) unless @sessions.empty?
291
+ # Last status (e.g. `running`) is no more relevant
292
+ @title = success ? nil : 'failed'
293
+ # Final line may be shorter (no bar)
294
+ @progress_bar.clear
295
+ update_format(title_text(final: true), fields)
296
+ success ? @progress_bar.finish : @progress_bar.stop
297
+ end
298
+
299
+ # Set total and progress of the bar, in an order accepted by `ruby-progressbar`
300
+ # @param total [Integer, nil] Total size, `nil` if unknown
301
+ # @param current [Integer] Processed bytes
302
+ def update_bar(total, current)
303
+ if total.nil?
304
+ # Unknown total: progress of the bar is used for animation only
305
+ @progress_bar.total = nil unless @progress_bar.total.nil?
306
+ @progress_bar.increment
307
+ elsif total >= @progress_bar.progress
308
+ @progress_bar.total = total unless @progress_bar.total.eql?(total)
309
+ @progress_bar.progress = current unless @progress_bar.progress.eql?(current)
310
+ else
311
+ @progress_bar.progress = current
312
+ @progress_bar.total = total
313
+ end
314
+ end
315
+
316
+ # @param final [Boolean] `true` for the final display
317
+ # @return [String] Title of the progress bar, with number of running sessions if several sessions
318
+ def title_text(final:)
319
+ parts = []
320
+ running = @sessions.count { |_, s| s[:running] }
321
+ parts.push("[#{running}]") if !final && running.positive? && @sessions.length > 1
322
+ parts.push(@title) unless @title.to_s.empty?
323
+ parts.join(' ')
324
+ end
96
325
 
97
- def total(key)
98
- @sessions.values.inject(0) { |m, s| m + s[key] }
326
+ # Change format of the bar, if changed
327
+ # @param title [String] Literal text
328
+ # @param fields [Array<String>] `ruby-progressbar` format elements
329
+ def update_format(title, fields)
330
+ new_format = (title.empty? ? fields : [title.gsub('%', '%%'), *fields]).join(' ')
331
+ return if new_format.eql?(@format)
332
+ @progress_bar.format = new_format
333
+ @format = new_format
99
334
  end
100
335
  end
101
336
  end
@@ -10,17 +10,6 @@ module Aspera
10
10
  # Mixin providing vault/keychain functionality to Plugin::Config.
11
11
  # Depends on `options` and `context.main_folder` being available in the including class.
12
12
  module VaultManager
13
- def action_vault_show(label:, id: nil, **)
14
- v = vault_required
15
- kwargs = id && v.method(:get).parameters.any? { |_t, n| n == :id } ? {id: id} : {}
16
- Result::SingleObject.new(v.get(label: label, **kwargs))
17
- end
18
-
19
- def action_vault_create(info:, **)
20
- vault_required.set(info.symbolize_keys)
21
- Result::Status.new('Secret added')
22
- end
23
-
24
13
  # Import secrets from a JSON array; skips entries missing :label
25
14
  def action_vault_import(secrets:, **)
26
15
  bulk_result(secrets, command: :import, id_result: 'label') do |entry|
@@ -36,12 +25,6 @@ module Aspera
36
25
  Result::Status.new("Secret deleted: #{label}")
37
26
  end
38
27
 
39
- def action_vault_password(new_password:, **)
40
- Aspera.assert(vault_required.respond_to?(:change_password), 'Vault does not support password change')
41
- vault_required.change_password(new_password)
42
- Result::Status.new('Vault password updated')
43
- end
44
-
45
28
  # @return [Keychain::Base] vault instance, raises if not configured
46
29
  def vault_required
47
30
  Aspera.assert(!vault.nil?, type: Cli::BadArgument) { 'Missing mandatory option: vault' }
@@ -4,6 +4,6 @@ module Aspera
4
4
  module Cli
5
5
  # For beta add extension : .beta1
6
6
  # For dev version add extension : .pre
7
- VERSION = '4.27.2'
7
+ VERSION = '4.27.4'
8
8
  end
9
9
  end