aspera-cli 4.27.3 → 4.27.5

data/lib/aspera/cli/transfer_progress.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require 'monitor'
+require 'aspera/environment'
 require 'aspera/log'
 require 'aspera/assert'
 require 'ruby-progressbar'
@@ -7,95 +9,328 @@ require 'ruby-progressbar'
 module Aspera
   module Cli
     # Progress bar for transfers.
-    # Supports multi-session.
+    # Supports multi-session: events may come from several threads.
     # Note that we can have this case:
-    # 2 sessions (-C x:2), but one session fails and restarts...
+    # 2 sessions (-C x:2), but one session fails and restarts (resume): it restarts with the same session id.
+    # Errors are logged, never raised: the progress bar shall not fail a transfer.
     class TransferProgress
-      def initialize
-        @progress_bar = nil
-        # Key is session id
-        @sessions = {}
-        @completed = false
-        @title = nil
+      # Rate is computed on this sliding window
+      RATE_WINDOW_SEC = 5.0
+      # Rate is not displayed on a shorter period
+      RATE_MIN_SEC = 1.0
+      # Decimal units, like `ascp`
+      KILO = 1000
+      BITS_PER_MEGABIT = KILO * KILO
+      BYTE_UNITS = %w[B KB MB GB TB PB].freeze
+      private_constant :RATE_WINDOW_SEC, :RATE_MIN_SEC, :KILO, :BITS_PER_MEGABIT, :BYTE_UNITS
+
+      class << self
+        # @param bytes [Integer] Size in bytes
+        # @return [String] Size with decimal unit, e.g. `12.3 MB`
+        def format_bytes(bytes)
+          value = bytes.to_f
+          index = 0
+          while value >= KILO && index < BYTE_UNITS.length - 1
+            value /= KILO
+            index += 1
+          end
+          return "#{bytes} B" if index.zero?
+          format('%.1f %s', value, BYTE_UNITS[index])
+        end
+
+        # @param bytes_per_second [Float, nil] Rate, `nil` if unknown
+        # @return [String] Rate in megabits per second (decimal, like `ascp`)
+        def format_rate(bytes_per_second)
+          return '-- Mbps' if bytes_per_second.nil?
+          mbps = bytes_per_second * Environment::BITS_PER_BYTE / BITS_PER_MEGABIT
+          mbps < 10 ? format('%.2f Mbps', mbps) : format('%.0f Mbps', mbps)
+        end
+
+        # @param seconds [Numeric, nil] Duration, `nil` if unknown
+        # @return [String] Duration as `hh:mm:ss`
+        def format_duration(seconds)
+          return '--:--:--' if seconds.nil?
+          seconds = seconds.round
+          format('%02d:%02d:%02d', seconds / 3600, (seconds / 60) % 60, seconds % 60)
+        end
+      end
+
+      # @param output [IO] Where the progress bar is displayed: not on `$stdout`, which may be redirected for command output
+      def initialize(output: $stderr)
+        @output = output
+        # Re-entrant, see `suspend`
+        @mutex = Monitor.new
+        reset_state
       end
 
       # Reset progress bar, to re-use it.
       def reset
-        send(:initialize)
+        @mutex.synchronize { reset_state }
       end
 
-      # Called by user of progress bar with a status on a transfer session
-      # @param session_id [String] the unique identifier of a transfer session
-      # @param type [Symbol] one of: sessions_init, session_start, session_size, transfer, session_end and end
-      # @param info [Object, nil] optional specific additional info for the given event type
+      # Called by user of progress bar with a status on a transfer session.
+      # Event types:
+      # - `sessions_init` : `info` is a status message, displayed until a session starts
+      # - `session_start` : a session starts, or restarts (same `session_id`)
+      # - `session_size`  : `info` is the total size of the session
+      # - `transfer`      : `info` is the number of bytes transferred by the session
+      # - `skip`          : `info` is the number of bytes of the session already at destination (not transferred)
+      # - `session_end`   : a session ends
+      # - `end`           : all sessions are finished, `info` is `false` if the transfer failed
+      # @param type       [Symbol]      Event type
+      # @param session_id [String, nil] Unique identifier of a transfer session, same when the session is restarted
+      # @param info       [Object, nil] Optional specific additional info for the given event type
       def event(type, session_id: nil, info: nil)
         Log.log.trace1 { "progress: #{type} #{session_id} #{info}" }
-        return if @completed
-        if @progress_bar.nil?
-          @progress_bar = ProgressBar.create(
-            format:      '%t %a %B %p%% %r Mbps %E',
-            rate_scale:  lambda { |rate| rate / Environment::BYTES_PER_MEBIBIT },
-            title:       '',
-            total:       nil
-          )
+        @mutex.synchronize { process_event(type, session_id, info) }
+      rescue StandardError => e
+        Log.log.warn { "Progress bar: #{e.class}: #{e.message}" } unless @error_logged
+        @error_logged = true
+      end
+
+      # Execute the block with the progress bar cleared, and then display it again.
+      # Used to write log lines on the terminal.
+      # Does not wait if the progress bar is being updated by another thread (no deadlock with a logger lock).
+      def suspend
+        return yield unless @mutex.try_enter
+        begin
+          return yield if @progress_bar.nil? || !@output.tty?
+          @progress_bar.clear
+          @output.flush
+          begin
+            yield
+          ensure
+            @progress_bar.refresh(force: true)
+          end
+        ensure
+          @mutex.exit
         end
-        progress_provided = false
+      end
+
+      private
+
+      def reset_state
+        @progress_bar = nil
+        # Key is session id
+        @sessions = {}
+        # Status displayed before session start
+        @title = nil
+        # Format currently set on the progress bar
+        @format = nil
+        # [Integer] Bytes actually transferred, by all sessions, including previous runs of restarted sessions
+        @transferred = 0
+        # [Array<Array(Float, Integer)>] (time, transferred bytes) on the rate window
+        @samples = []
+        # [Array(Float, Integer), nil] First sample (data starts to flow, not session start), for average rate
+        @start = nil
+        @error_logged = false
+      end
+
+      # @param type       [Symbol]      Event type
+      # @param session_id [String, nil] Session identifier
+      # @param info       [Object, nil] Event information
+      def process_event(type, session_id, info)
         case type
         when :sessions_init
           # Give opportunity to show progress of initialization with multiple status
           Aspera.assert(session_id.nil?, 'session_id must be nil for :sessions_init event')
           Aspera.assert_type(info, String)
-          # Initialization of progress bar
           @title = info
         when :session_start
-          Aspera.assert_type(session_id, String)
           Aspera.assert(info.nil?, 'info must be nil for :session_start event')
-          Aspera.assert(!@sessions[session_id]) { "Session #{session_id} already started" }
-          @sessions[session_id] = {
-            job_size: 0, # Total size of transfer (pre-calc)
-            current:  0,
-            running:  true
-          }
+          start_session(session_id)
           # Remove last pre-start message if any
           @title = nil
         when :session_size
-          Aspera.assert_type(session_id, String)
-          Aspera.assert(!info.nil?, 'info must not be nil for :session_size event')
-          Aspera.assert_type(@sessions[session_id], Hash)
-          @sessions[session_id][:job_size] = info.to_i
-          sessions_total = total(:job_size)
-          @progress_bar.total = sessions_total unless sessions_total.eql?(@progress_bar.total) || sessions_total < @progress_bar.progress
+          session(session_id)[:size] = info.to_i unless info.nil?
         when :transfer
-          Aspera.assert_type(session_id, String)
-          Aspera.assert_type(@sessions[session_id], Hash)
-          if !@progress_bar.total.nil? && !info.nil?
-            progress_provided = true
-            @sessions[session_id][:current] = info.to_i
-            sessions_current = total(:current)
-            @progress_bar.progress = sessions_current unless @progress_bar.progress.eql?(sessions_current) || sessions_current > total(:job_size)
+          unless info.nil?
+            current_session = session(session_id)
+            transferred = info.to_i
+            @transferred += transferred - current_session[:transferred] if transferred > current_session[:transferred]
+            current_session[:transferred] = transferred
+            add_sample
           end
+        when :skip
+          session(session_id)[:skipped] = info.to_i unless info.nil?
         when :session_end
-          Aspera.assert_type(session_id, String)
           Aspera.assert(info.nil?, 'info must be nil for :session_end event')
           # A session may be too short and finish before it has been started
-          @sessions[session_id][:running] = false if @sessions[session_id].is_a?(Hash)
+          @sessions[session_id][:running] = false if @sessions.key?(session_id)
         when :end
           Aspera.assert(session_id.nil?, 'session_id must be nil for :end event')
-          Aspera.assert(info.nil?, 'info must be nil for :end event')
-          @progress_bar.finish
+          Aspera.assert_values(info, [nil, true, false]) { ':end event info' }
+          begin
+            finish(success: !info.eql?(false))
+          ensure
+            reset_state
+          end
+          return
         else Aspera.error_unexpected_value(type) { 'event type' }
         end
-        new_title = @sessions.length < 2 ? @title.to_s : "[#{@sessions.count { |_i, d| d[:running] }}] #{@title}"
-        @progress_bar&.title = new_title unless @progress_bar&.title.eql?(new_title)
-        @progress_bar&.increment if !progress_provided && @progress_bar.progress.nil?
-      rescue ProgressBar::InvalidProgressError => e
-        Log.log.error { "Progress error: #{e}" }
+        display
       end
 
-      private
+      # @param session_id [String] Session identifier
+      # @return [Hash] Session information, created if not already started
+      def session(session_id)
+        Aspera.assert_type(session_id, String)
+        @sessions[session_id] ||= {
+          size:        nil, # Total size of session, `nil` if unknown
+          transferred: 0,   # Bytes transferred
+          skipped:     0,   # Bytes already at destination
+          floor:       0,   # Progress reached before restart of session
+          running:     true
+        }
+      end
+
+      # Start a session, or restart it: a restarted session notifies again its progress from zero
+      # @param session_id [String] Session identifier
+      def start_session(session_id)
+        restarted = @sessions.key?(session_id)
+        current_session = session(session_id)
+        if restarted
+          current_session[:floor] = done_bytes(current_session)
+          current_session[:transferred] = 0
+          current_session[:skipped] = 0
+        end
+        current_session[:running] = true
+      end
+
+      # @param session [Hash] Session information
+      # @return [Integer] Bytes processed by the session (transferred or skipped), not more than its size
+      def done_bytes(session)
+        done = [session[:floor], session[:transferred] + session[:skipped]].max
+        session[:size].nil? ? done : [done, session[:size]].min
+      end
+
+      # @return [Integer] Bytes processed by all sessions
+      def current_bytes
+        @sessions.each_value.sum { |s| done_bytes(s) }
+      end
+
+      # @return [Integer, nil] Total size of all sessions, `nil` if unknown for one of them
+      def total_bytes
+        return if @sessions.empty? || @sessions.each_value.any? { |s| s[:size].nil? }
+        @sessions.each_value.sum { |s| s[:size] }
+      end
+
+      # @return [Float] Monotonic time in seconds
+      def now
+        Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      end
+
+      # Record transferred bytes for rate computation
+      # @return [Array(Float, Integer)] The new sample
+      def add_sample
+        sample = [now, @transferred]
+        @start ||= sample
+        @samples.push(sample)
+        # Keep one sample older than the window
+        @samples.shift while @samples.length > 2 && @samples[1].first <= sample.first - RATE_WINDOW_SEC
+        sample
+      end
+
+      # @return [Float, nil] Rate on the sliding window, in bytes per second, `nil` if unknown
+      def rate
+        return if @samples.length < 2
+        elapsed = @samples.last.first - @samples.first.first
+        return if elapsed < RATE_MIN_SEC
+        (@samples.last.last - @samples.first.last) / elapsed
+      end
+
+      # @return [Float, nil] Average rate between first and last sample, in bytes per second, `nil` if unknown
+      def average_rate
+        return if @start.nil?
+        last = @samples.last
+        elapsed = last.first - @start.first
+        return unless elapsed.positive?
+        (last.last - @start.last) / elapsed
+      end
+
+      # Update the progress bar with current state
+      def display
+        if @progress_bar.nil?
+          @progress_bar = ProgressBar.create(output: @output, format: '%a %B', total: nil, autofinish: false, throttle_rate: 0)
+          @format = '%a %B'
+        end
+        current = current_bytes
+        total = total_bytes
+        update_bar(total, current)
+        fields = ['%a', '%B']
+        unless @sessions.empty?
+          bytes_per_second = rate
+          # Fixed width, so that bar does not change size
+          rate_text = self.class.format_rate(bytes_per_second).rjust(11)
+          if total.nil?
+            fields.push(self.class.format_bytes(current).rjust(8), rate_text)
+          else
+            eta = bytes_per_second&.positive? ? (total - current) / bytes_per_second : nil
+            fields.push('%j%%', rate_text, "ETA: #{self.class.format_duration(eta)}")
+          end
+        end
+        update_format(title_text(final: false), fields)
+      end
+
+      # Display final state of the progress bar
+      # @param success [Boolean] `false` if the transfer failed: progress is not completed
+      def finish(success:)
+        return if @progress_bar.nil?
+        current = current_bytes
+        total = total_bytes
+        if success
+          total ||= current
+          current = total
+        end
+        fields = ['%a']
+        unless total.nil?
+          update_bar(total, current)
+          fields.push('%B', '%j%%')
+        end
+        fields.push(self.class.format_bytes(current), self.class.format_rate(average_rate)) unless @sessions.empty?
+        # Last status (e.g. `running`) is no more relevant
+        @title = success ? nil : 'failed'
+        # Final line may be shorter (no bar)
+        @progress_bar.clear
+        update_format(title_text(final: true), fields)
+        success ? @progress_bar.finish : @progress_bar.stop
+      end
+
+      # Set total and progress of the bar, in an order accepted by `ruby-progressbar`
+      # @param total   [Integer, nil] Total size, `nil` if unknown
+      # @param current [Integer]      Processed bytes
+      def update_bar(total, current)
+        if total.nil?
+          # Unknown total: progress of the bar is used for animation only
+          @progress_bar.total = nil unless @progress_bar.total.nil?
+          @progress_bar.increment
+        elsif total >= @progress_bar.progress
+          @progress_bar.total = total unless @progress_bar.total.eql?(total)
+          @progress_bar.progress = current unless @progress_bar.progress.eql?(current)
+        else
+          @progress_bar.progress = current
+          @progress_bar.total = total
+        end
+      end
+
+      # @param final [Boolean] `true` for the final display
+      # @return [String] Title of the progress bar, with number of running sessions if several sessions
+      def title_text(final:)
+        parts = []
+        running = @sessions.count { |_, s| s[:running] }
+        parts.push("[#{running}]") if !final && running.positive? && @sessions.length > 1
+        parts.push(@title) unless @title.to_s.empty?
+        parts.join(' ')
+      end
 
-      def total(key)
-        @sessions.values.inject(0) { |m, s| m + s[key] }
+      # Change format of the bar, if changed
+      # @param title  [String]        Literal text
+      # @param fields [Array<String>] `ruby-progressbar` format elements
+      def update_format(title, fields)
+        new_format = (title.empty? ? fields : [title.gsub('%', '%%'), *fields]).join(' ')
+        return if new_format.eql?(@format)
+        @progress_bar.format = new_format
+        @format = new_format
       end
     end
   end

data/lib/aspera/cli/version.rb

@@ -4,6 +4,6 @@ module Aspera
   module Cli
     # For beta add extension : .beta1
     # For dev version add extension : .pre
-    VERSION = '4.27.3'
+    VERSION = '4.27.5'
   end
 end

data/lib/aspera/cli/wizard.rb

@@ -17,6 +17,7 @@ module Aspera
       DEFAULT_PRIV_KEY_FILENAME = 'my_private_key.pem' # pragma: allowlist secret
       private_constant :WIZARD_RESULT_KEYS, :DEFAULT_PRIV_KEY_FILENAME
 
+      # Declared by the `config` plugin (`use_options Wizard`)
       option :override, description: 'Wizard: override existing value',                                          allowed: Type::BOOLEAN, default: false
       option :default,  description: 'Wizard: set as default configuration for specified plugin (also: update)', allowed: Type::BOOLEAN, default: true
       option :key_path, description: 'Wizard: path to private key for JWT'
@@ -24,7 +25,6 @@ module Aspera
       def initialize(parent, main_folder)
         @parent = parent
         @main_folder = main_folder
-        self.class.declare_options(options)
       end
 
       # @return [Boolean] false if in test mode to avoid interactive input

data/lib/aspera/coverage.rb

@@ -33,7 +33,6 @@ if ENV.key?('ENABLE_COVERAGE')
       false
     end
     # no coverage in these files
-    add_filter 'lib/aspera/node_simulator.rb'
     add_filter 'lib/aspera/keychain/macos_security.rb'
     add_filter 'lib/aspera/assert.rb'
   end

data/lib/aspera/environment.rb

@@ -5,13 +5,14 @@ require 'aspera/log'
 require 'aspera/assert'
 require 'rbconfig'
 require 'singleton'
+require 'openssl'
 require 'open3'
 require 'English'
 require 'shellwords'
 require 'aspera/rainbow'
 using Rainbow
 
-# cspell:words MEBI mswin bccwin
+# cspell:words mswin bccwin
 
 module Aspera
   # detect OS, architecture, and specific stuff
@@ -36,8 +37,6 @@ module Aspera
     CPU_LIST = [CPU_X86_64, CPU_ARM64, CPU_PPC64, CPU_PPC64LE, CPU_S390].freeze
 
     BITS_PER_BYTE = 8
-    MEBI = 1024 * 1024
-    BYTES_PER_MEBIBIT = MEBI / BITS_PER_BYTE
 
     I18N_VARS = %w(LC_ALL LC_CTYPE LANG).freeze
 
@@ -50,6 +49,14 @@ module Aspera
 
     PROCESS_MODES = %i[execute background capture].freeze
 
+    # System CA certificate bundles, used when the default locations of Ruby's OpenSSL do not exist
+    CA_BUNDLE_FILES = [
+      '/etc/ssl/certs/ca-certificates.crt', # Debian, Ubuntu, Alpine, Arch
+      '/etc/pki/tls/certs/ca-bundle.crt',   # RHEL, Fedora
+      '/etc/ssl/ca-bundle.pem',             # SUSE
+      '/etc/ssl/cert.pem'                   # macOS, BSD
+    ].freeze
+
     class << self
       def ruby_version
         return RbConfig::CONFIG['RUBY_PROGRAM_VERSION']
@@ -165,8 +172,8 @@ module Aspera
         if force || !File.exist?(path)
           # Windows may give error
           File.unlink(path) rescue nil
-          # content provided by block
-          File.write(path, yield)
+          # content provided by block, file created with restricted access: never readable by others, even briefly
+          File.write(path, yield, perm: mode || 0o600)
           restrict_file_access(path, mode: mode)
         end
         return path
@@ -299,6 +306,23 @@ module Aspera
       nil
     end
 
+    # When Ruby's OpenSSL was built on another system (e.g. single executable built on RHEL, used on Ubuntu),
+    # its default CA locations may not exist: use the system CA bundle instead.
+    # @return [String, nil] CA bundle file used, or nil if no change
+    def fix_ca_certificates
+      return if defined?(JRUBY_VERSION)
+      return if [OpenSSL::X509::DEFAULT_CERT_FILE_ENV, OpenSSL::X509::DEFAULT_CERT_DIR_ENV].any? { |var| ENV.key?(var) }
+      return if File.exist?(OpenSSL::X509::DEFAULT_CERT_FILE) || Dir.exist?(OpenSSL::X509::DEFAULT_CERT_DIR)
+      ca_file = CA_BUNDLE_FILES.find { |file| File.file?(file) }
+      return if ca_file.nil?
+      # Used by OpenSSL for new default stores, and by child processes
+      ENV[OpenSSL::X509::DEFAULT_CERT_FILE_ENV] = ca_file
+      # Default store of SSL contexts is initialized when openssl is loaded
+      OpenSSL::SSL::SSLContext::DEFAULT_CERT_STORE.add_file(ca_file)
+      Log.log.debug { "Using system CA certificates: #{ca_file}" }
+      ca_file
+    end
+
     def graphical?
       @default_gui_mode == :graphical
     end

data/lib/aspera/keychain/encrypted_hash.rb

@@ -181,7 +181,7 @@ module Aspera
           'kdf'     => @kdf_params,
           'data'    => @cipher.encrypt(YAML.dump(@all_secrets))
         }
-        File.write(@path, YAML.dump(vault_info))
+        Environment.write_file_restricted(@path, force: true) { YAML.dump(vault_info) }
       end
     end
   end

data/lib/aspera/keychain/factory.rb

@@ -18,7 +18,8 @@ module Aspera
           Aspera.assert_values(vault_type, LIST.map(&:to_s)) { 'vault.type' }
           case vault_type
           when 'file'
-            info[:file] = info.delete(:name) || 'vault.bin' if info.key?(:name)
+            # `name` (or `file`), default: `vault.bin`
+            info[:file] = info.delete(:name) || info[:file] || 'vault.bin'
             info[:file] = File.join(folder, info[:file]) unless File.absolute_path?(info[:file])
             Aspera.assert(!password.nil?, 'please provide password')
             info[:password] = password

data/lib/aspera/log.rb

@@ -68,6 +68,26 @@ module Aspera
     # Where logs are sent to
     LOG_TYPES = %i[stderr stdout syslog].freeze
 
+    # Logger device for a terminal: the status line (e.g. progress bar) is suspended while a log line is written
+    class TerminalDevice
+      # @param io [IO] Output stream
+      def initialize(io)
+        @io = io
+      end
+
+      # Called by `Logger`
+      # @param message [String] Formatted log line
+      def write(message)
+        status_line = Log.instance.status_line if @io.tty?
+        return @io.write(message) if status_line.nil?
+        status_line.suspend { @io.write(message) }
+      end
+
+      # Standard streams are not closed
+      def close; end
+    end
+    private_constant :TerminalDevice
+
     # Levels are :trace2,:trace1,:debug,:info,:warn,:error,fatal,:unknown
     LEVELS = Logger::Severity.constants.sort { |a, b| Logger::Severity.const_get(a) <=> Logger::Severity.const_get(b) }.map { |c| c.downcase.to_sym }.freeze
 
@@ -146,6 +166,8 @@ module Aspera
 
     attr_reader :logger_type, :logger
     attr_accessor :dump_format
+    # [#suspend, nil] Object displaying a status line on the terminal (e.g. progress bar), see TerminalDevice
+    attr_accessor :status_line
 
     # Set the program name used in log output
     # @param value [String] Program name
@@ -197,9 +219,9 @@ module Aspera
       current_severity_integer = @logger&.level || ENV['AS_LOG_LEVEL']&.to_i || Logger::Severity::INFO
       case new_log_type
       when :stderr
-        @logger = Logger.new($stderr, progname: @program_name, formatter: DEFAULT_FORMATTER)
+        @logger = Logger.new(TerminalDevice.new($stderr), progname: @program_name, formatter: DEFAULT_FORMATTER)
       when :stdout
-        @logger = Logger.new($stdout, progname: @program_name, formatter: DEFAULT_FORMATTER)
+        @logger = Logger.new(TerminalDevice.new($stdout), progname: @program_name, formatter: DEFAULT_FORMATTER)
       when :syslog
         require 'syslog/logger'
         # The syslog class automatically creates methods from the severity names.
@@ -225,6 +247,7 @@ module Aspera
 
     def initialize
       @logger = nil
+      @status_line = nil
       @program_name = 'aspera'
       @dump_format = :json
       @logger_type = :stderr