aspera-cli 4.27.5 → 4.28.0

data/lib/aspera/schema/documentation.rb

@@ -1,11 +1,9 @@
 # frozen_string_literal: true
 
-require 'aspera/agent/factory'
-require 'aspera/markdown'
-
 module Aspera
   module Schema
-    # Generate documentation from Schema, for Transfer Spec, or async Conf spec
+    # Generate documentation from a JSON Schema: in terminal and in manual.
+    # Information from `x-` extensions is provided by the caller (`annotate`).
     class Documentation
       # Map JSON Schema type names to user-friendly display names
       JSON_TYPE_TO_DOC = {
@@ -16,19 +14,21 @@ module Aspera
         'array'   => 'Array',
         'object'  => 'Hash'
       }.freeze
-      # @param formatter [Cli::Formatter] Formatter instance with methods: markdown_text, tick, check_row
-      # @param schema [Reader]
-      # @param include_option [Boolean] `true`: include CLI options (switches, env vars) in descriptions
-      # @param agent_columns  [Boolean] `true`: add separate columns for each transfer agent compatibility
+      COLUMNS = %w[name type description].freeze
+      # @param formatter      [FormatterInterface] Formatter with methods: markdown_text, special_format
+      # @param schema         [Reader]
       # @param code_highlight [Boolean] `true`: format name and type as code
-      def initialize(formatter, schema, include_option: false, agent_columns: false, code_highlight: false)
+      # @param annotate       [Proc, nil] Called with the schema of a property (`Hash`) and the formatter.
+      #   Returns lines added to the description (`Array<String>`), or `nil` to hide the property.
+      # @param extra_columns  [Hash{String => Proc}] Columns inserted before the description: name => Proc
+      #   called with the schema of a property (`Hash`) and the formatter, returns the value of the cell.
+      def initialize(formatter, schema, code_highlight: false, annotate: nil, extra_columns: {})
         @formatter = formatter
         @schema = schema
-        @include_option = include_option
-        @agent_columns = agent_columns
         @code_highlight = code_highlight
-        @columns = %w[name type description]
-        @columns.insert(-2, *Agent::Factory::ALL.values.map { |i| i[:short].to_s }.sort) if @agent_columns
+        @annotate = annotate
+        @extra_columns = extra_columns
+        @columns = COLUMNS.dup.insert(-2, *extra_columns.keys).freeze
         # Sections: each entry is {header: row_or_nil, rows: []}
         # A flat schema produces a single section with no header.
         @sections = [{header: nil, rows: []}]
@@ -50,7 +50,7 @@ module Aspera
         [@columns] + rows.map { |row| @columns.map { |field| row[field] } }
       end
 
-      # Generate a documentation table from a JSON schema for transfer specifications
+      # Generate a documentation table from a JSON schema
       #
       # Recursively processes a JSON schema to create a formatted table for manual documentation.
       # Handles nested objects, arrays, and extracts metadata (descriptions, types, enums, deprecations).
@@ -60,88 +60,60 @@ module Aspera
       def build(schema = nil)
         code = @code_highlight ? ->(c) { "`#{c}`" } : ->(c) { c }
         schema ||= @schema
-        render_title = ->(title) { title.gsub(Markdown::FORMATS) { @formatter.markdown_text(Regexp.last_match) } }
         on_variant = ->(variant_reader, discriminant_property, discriminant_value) do
           title = variant_reader.current['title'] || variant_reader.current['description']
           header =
             if discriminant_property && discriminant_value
-              desc = render_title.call("`#{discriminant_value}`")
-              desc += ": #{render_title.call(title)}" if title
-              @formatter.check_row({
-                'name'        => render_title.call("**#{discriminant_property}**"),
+              desc = @formatter.markdown_text("`#{discriminant_value}`")
+              desc += ": #{@formatter.markdown_text(title)}" if title
+              {
+                'name'        => @formatter.markdown_text("**#{discriminant_property}**"),
                 'type'        => code.call(JSON_TYPE_TO_DOC['string']),
                 'description' => desc
-              })
+              }
             elsif title
-              @formatter.check_row({'name' => "**#{render_title.call(title)}**", 'type' => '&nbsp;', 'description' => '&nbsp;'})
+              {'name' => @formatter.markdown_text("**#{title}**")}
             end
           @sections.push({header: header, rows: []})
         end
-        schema.each_property(on_variant: on_variant) do |property_schema, _name, property_full_name|
+        schema.each_property(on_variant: on_variant) do |property_schema, _name, property_full_name, required|
           node = property_schema.current
-          # Manual table
-          item_type =
-            if node['type'].is_a?(Array)
-              node['type'].map { |t| JSON_TYPE_TO_DOC.fetch(t, t) }.join(', ')
-            elsif node['type'].eql?('array') && node.dig('items', 'type').is_a?(String)
-              "#{JSON_TYPE_TO_DOC.fetch(node['type'], node['type'])}[#{JSON_TYPE_TO_DOC.fetch(node.dig('items', 'type'), node.dig('items', 'type'))}]"
-            else
-              JSON_TYPE_TO_DOC.fetch(node['type'], node['type'])
-            end
-          item = {
-            'name'        => code.call(property_full_name),
-            'type'        => code.call(item_type),
-            'description' => []
-          }
+          annotations = @annotate ? @annotate.call(node, @formatter) : []
+          next if annotations.nil?
+          item = {'name' => code.call(property_full_name)}
+          item_type = type_text(node)
+          item['type'] = code.call(item_type) unless item_type.nil?
+          @extra_columns.each { |column, value| item[column] = value.call(node, @formatter) }
+          description = []
+          description.push("DEPRECATED: #{node['x-deprecation']}") if node.key?('x-deprecation')
           # Render Markdown formatting and split lines
-          item['description'] =
-            node['description']
-              .gsub(Markdown::FORMATS) { @formatter.markdown_text(Regexp.last_match) }
-              .split("\n") if node.key?('description')
-          item['description'].unshift("DEPRECATED: #{node['x-deprecation']}") if node.key?('x-deprecation')
-          # Add flags for supported agents in doc
-          agents = []
-          Agent::Factory::ALL.each_key do |sym|
-            agents.push(sym) if node['x-agents'].nil? || node['x-agents'].include?(sym.to_s)
-          end
-          Aspera.assert(agents.include?(:direct)) { "#{property_full_name}: x-cli-option requires agent direct (or nil)" } if node['x-cli-option']
-          if @agent_columns
-            Agent::Factory::ALL.each do |sym, names|
-              item[names[:short].to_s] = @formatter.tick(agents.include?(sym))
-            end
-          else
-            item['description'].push("(#{agents.map { |i| Agent::Factory::ALL[i][:short].to_s.upcase }.sort.join(', ')})") unless agents.length.eql?(Agent::Factory::ALL.length)
-          end
-          # Only keep lines that are usable in supported agents
-          next if agents.empty?
-          item['description'].push("Allowed values: #{node['enum'].map { |v| value_text(v) }.join(', ')}.") if node.key?('enum')
-          item['description'].push("Default: #{value_text(node['default'])}.") if node.key?('default')
-          item['description'].push("Example: #{value_text(node['example'])}.") if node.key?('example')
-          if @include_option
-            envvar_prefix = ''
-            cli_option =
-              if node.key?('x-cli-envvar')
-                envvar_prefix = 'env:'
-                node['x-cli-envvar']
-              elsif node['x-cli-switch']
-                false_part = node.key?('x-cli-false') ? " / #{node['x-cli-false']}" : ''
-                "#{node['x-cli-option']}#{false_part}"
-              elsif node['x-cli-option']
-                arg_type = node.key?('enum') ? '{enum}' : "{#{[node['type']].flatten.join('|')}}"
-                conversion_tag = node.key?('x-cli-convert') ? 'conversion' : nil
-                sep = node['x-cli-option'].start_with?('--') ? '=' : ' '
-                "#{node['x-cli-option']}#{sep}#{"(#{conversion_tag})" if conversion_tag}#{arg_type}"
-              end
-            short = node.key?('x-cli-short') ? "(#{node['x-cli-short']})" : nil
-            item['description'].push("(#{'special:' if node['x-cli-special']}#{envvar_prefix}#{@formatter.markdown_text("`#{cli_option}`")})#{short}") if cli_option
-          end
-          @sections.last[:rows].push(@formatter.check_row(item))
+          description.concat(@formatter.markdown_text(node['description']).split("\n")) if node.key?('description')
+          description.push('Required.') if required
+          description.push("Allowed values: #{node['enum'].map { |v| value_text(v) }.join(', ')}.") if node.key?('enum')
+          description.push("Default: #{value_text(node['default'])}.") if node.key?('default')
+          description.push("Example: #{value_text(node['example'])}.") if node.key?('example')
+          description.concat(annotations)
+          item['description'] = description unless description.empty?
+          @sections.last[:rows].push(item)
         end
         self
       end
 
       private
 
+      # @param node [Hash] Schema of a property
+      # @return [String, nil] Readable type, from `type`, or else from the types of `oneOf` alternatives, `nil` if none
+      def type_text(node)
+        if node['type'].nil?
+          types = Array(node['oneOf']).filter_map { |variant| type_text(variant) }.uniq
+          return types.empty? ? nil : types.join(', ')
+        end
+        return node['type'].map { |t| JSON_TYPE_TO_DOC.fetch(t, t) }.join(', ') if node['type'].is_a?(Array)
+        result = JSON_TYPE_TO_DOC.fetch(node['type'], node['type'])
+        result = "#{result}[#{type_text(node['items'])}]" if node['type'].eql?('array') && node.dig('items', 'type').is_a?(String)
+        result
+      end
+
       # Format a schema value (enum, default, example) as code
       # @param value [Object] value from schema, `nil` for JSON `null`
       # @return [String]

data/lib/aspera/schema/reader.rb

@@ -5,6 +5,10 @@ module Aspera
   module Schema
     # JSON schema reader
     class Reader
+      # Keys of a schema node that provide properties
+      PROPERTY_SOURCES = %w[properties allOf oneOf].freeze
+      private_constant :PROPERTY_SOURCES
+
       attr_reader :current
 
       class << self
@@ -72,14 +76,17 @@ module Aspera
       # Recursively traverse schema properties with a block.
       # If the current node has `oneOf`, each variant is traversed in turn and
       # `on_variant` is called (if given) before each variant's properties.
-      # @param prefix     [String] Prefix for property names (e.g., 'parent.child.')
-      # @param on_variant [Proc, nil] Called with the variant Reader before its properties
-      # @yield [property_schema, name, full_name] Yields property info to block
+      # @param prefix          [String] Prefix for property names (e.g., 'parent.child.')
+      # @param on_variant      [Proc, nil] Called with the variant Reader before its properties
+      # @param parent_required [Array<String>] Names required by the enclosing `oneOf` or `allOf` node
+      # @yield [property_schema, name, full_name, required] Yields property info to block
       # @yieldparam property_schema [Reader] Schema reader for this property
       # @yieldparam name [String] Property name
       # @yieldparam full_name [String] Full property name with prefix
+      # @yieldparam required [Boolean] `true` if listed in `required` of its object, or of the `oneOf` or `allOf` node that includes it
       # @return [nil]
-      def each_property(prefix = '', on_variant: nil, &block)
+      def each_property(prefix = '', on_variant: nil, parent_required: [], &block)
+        required_names = parent_required + Array(@current['required'])
         if @current.key?('oneOf')
           # Build reverse map: $ref -> discriminant value, from discriminator.mapping if present
           discriminant_by_ref = {}
@@ -94,16 +101,24 @@ module Aspera
             variant_reader = ref ? resolve_ref(ref) : Reader.new(@root, variant_node)
             discriminant_value = ref ? discriminant_by_ref[ref] : nil
             on_variant&.call(variant_reader, discriminant_property, discriminant_value)
-            variant_reader.each_property(prefix, on_variant: on_variant, &block)
+            variant_reader.each_property(prefix, on_variant: on_variant, parent_required: required_names, &block)
           end
           return
         end
         if @current.key?('allOf')
-          # Merge all branches: each branch contributes its properties (no variants)
-          @current['allOf'].each do |branch_node|
-            ref = branch_node['$ref']
-            branch_reader = ref ? resolve_ref(ref) : Reader.new(@root, branch_node)
-            branch_reader.each_property(prefix, on_variant: on_variant, &block)
+          # Merge all branches: each branch contributes its properties (no variants).
+          # A property is listed once, with its first declaration: a later one may be empty,
+          # only to allow the property under `additionalProperties: false`.
+          # A property is required if any branch requires it.
+          branches = @current['allOf'].map { |branch_node| branch_reader(branch_node) }
+          required_names += branches.flat_map { |branch| Array(branch.current['required']) }
+          listed = Set.new
+          branches.each do |branch|
+            # A branch may only constrain properties of other branches, e.g. with `required`
+            next unless PROPERTY_SOURCES.any? { |key| branch.current.key?(key) }
+            branch.each_property(prefix, on_variant: on_variant, parent_required: required_names) do |property_schema, name, full_name, required|
+              yield(property_schema, name, full_name, required) if listed.add?(full_name)
+            end
           end
           return
         end
@@ -114,7 +129,7 @@ module Aspera
           node = property_schema.current
 
           # Yield current property to block
-          yield(property_schema, name, property_full_name)
+          yield(property_schema, name, property_full_name, required_names.include?(name))
 
           # Recursively process nested structures (`type` is a list for a nullable value, e.g. `[array, 'null']`)
           types = Array(node['type'])
@@ -150,9 +165,22 @@ module Aspera
 
       private
 
+      # @param node [Hash] Branch of `allOf` or `oneOf`
+      # @return [Reader] Reader on the branch, with `$ref` resolved
+      def branch_reader(node)
+        node['$ref'] ? resolve_ref(node['$ref']) : Reader.new(@root, node)
+      end
+
       # Recursively collect rows from a schema node, passing each property's own
       # parent `required` array so that only direct-parent membership is checked.
+      # Branches of `allOf` are merged: a property is listed once, with its first declaration.
       def collect_rows(rows, reader, prefix)
+        reader.current['allOf']&.each do |branch_node|
+          branch_rows = []
+          collect_rows(branch_rows, branch_reader(branch_node), prefix)
+          listed = rows.to_set { |row| row['name'] }
+          rows.concat(branch_rows.reject { |row| listed.include?(row['name']) })
+        end
         return unless reader.current.key?('properties')
         parent_required = Set.new(Array(reader.current['required']))
         props_reader = reader.dig('properties')

data/lib/aspera/schema/registry.rb

@@ -62,11 +62,11 @@ module Aspera
         aoc:          'aspera/schema/IBM Aspera on Cloud API-0.2.6-enhanced.yaml',
         automation:   'aspera/schema/IBM Aspera on Cloud Automation API-1.0.5-enhanced.yaml',
         faspex:       'aspera/schema/IBM Aspera Faspex API-5.0-enhanced.yaml',
-        faspio:       'aspera/schema/IBM Aspera faspio Gateway API-1.0.0.yaml',
+        faspio:       'aspera/schema/IBM Aspera faspio Gateway API-1.0.0-enhanced.yaml',
         console:      'aspera/schema/IBM Aspera Console-enhanced.yaml',
         node:         'aspera/schema/IBM Aspera Node API-4.4.6.yaml',
         shares:       'aspera/schema/IBM_Aspera_Shares.yaml',
-        orchestrator: 'aspera/schema/IBM Aspera Orchestrator API-v1.yaml',
+        orchestrator: 'aspera/schema/IBM Aspera Orchestrator API-v1-enhanced.yaml',
         async_tables: 'aspera/schema/async_tables.yaml'
       }
 
@@ -88,6 +88,9 @@ module Aspera
       SHARES = 'shares+/api/v1'
       ASYNC_TABLES = 'async_tables'
       LOG_OPTIONS             = "#{OPTIONS}:components.schemas.LogOptions"
+      UI_OPTIONS              = "#{OPTIONS}:components.schemas.UiOptions"
+      SECURITY_OPTIONS        = "#{OPTIONS}:components.schemas.SecurityOptions"
+      TEMP_OPTIONS            = "#{OPTIONS}:components.schemas.TempOptions"
       DIRECT_AGENT_OPTIONS    = "#{OPTIONS}:components.schemas.DirectAgentOptions"
       NODE_AGENT_OPTIONS      = "#{OPTIONS}:components.schemas.NodeAgentOptions"
       NODE_EMULATOR_OPTIONS   = "#{OPTIONS}:components.schemas.NodeEmulatorOptions"
@@ -101,6 +104,9 @@ module Aspera
       IMAGE_OPTIONS           = "#{OPTIONS}:components.schemas.ImageOptions"
       OUT_OPTIONS             = "#{OPTIONS}:components.schemas.OutOptions"
       PACKAGE_FOLDER_OPTIONS  = "#{OPTIONS}:components.schemas.PackageFolderOptions"
+      WEB_SERVER_OPTIONS      = "#{OPTIONS}:components.schemas.WebServerOptions"
+      GATEWAY_OPTIONS         = "#{OPTIONS}:components.schemas.Faspex4GatewayOptions"
+      POSTPROC_OPTIONS        = "#{OPTIONS}:components.schemas.Faspex4PostProcOptions"
 
       REQ_BODY = '.requestBody.content.application/json.schema'
       # Suffix appended to a dotted path to signal query-param extraction in reader()

data/lib/aspera/secret_hider.rb

@@ -4,17 +4,23 @@
 require 'logger'
 require 'set'
 require 'singleton'
+require 'json'
+require 'uri'
 
 module Aspera
   # remove secret from logs and output
   #
+  # Secrets are identified:
+  # - by key name, in structures (`secret?`, `hidden_copy`) and in strings with known patterns (`hide_secrets_in_string`)
+  # - by value, when declared where they come from (`add_secret_values`, `add_secrets_in`): option values, tokens, ...
+  #
   # @!method self.instance
   #   Returns the singleton instance of SecretHider
   #   @return [SecretHider] the singleton instance
   class SecretHider
     include Singleton
 
-    # display string for hidden secrets
+    # display string for hidden secrets (see `Environment::UNICODE_TO_ASCII`)
     HIDDEN_PASSWORD = '🔑'
     # env vars for ascp with secrets
     ASCP_ENV_SECRETS = %w[ASPERA_SCP_PASS ASPERA_SCP_KEY ASPERA_SCP_FILEPASS ASPERA_PROXY_PASS ASPERA_SCP_TOKEN].freeze
@@ -26,6 +32,9 @@ module Aspera
     KEY_FALSE_POSITIVES = [/^access_key$/, /^fallback_private_key$/, /public_key$/, /^token_type$/].freeze
     # min length of hidden secrets, use `+` or `{n,}`
     SECRET_LENGTH = '{5,}'
+    # min length of a declared secret value to be hidden inside a string: shorter ones are hidden only when they are the whole string,
+    # else common words would be hidden (e.g. password `aspera` in URLs), or optionally as a word (`short_in_text`)
+    VALUE_IN_STRING_LENGTH = 8
     # regex that define named captures :begin and :end
     REGEX_LOG_REPLACES = [
       # private key values (place first)
@@ -38,12 +47,16 @@ module Aspera
       /(?<begin>(?:(?<quote>["'])|:)[^"':=]*(?:#{ALL_SECRETS.join('|')})[^"':=]*\k<quote>? *(?:=>|:) *")(?:[^"\\]|\\.)+(?<end>")/i,
       # rendered Ruby 3.4+ symbol key: `k: "v"`
       /(?<begin>\b\w*(?:#{ALL_SECRETS.join('|')})\w*: *")(?:[^"\\]|\\.)+(?<end>")/i,
+      # rendered Ruby Struct or object: `k="v"`, `@k="v"`
+      /(?<begin>@?\b\w*(?:#{ALL_SECRETS.join('|')})\w*=")(?:[^"\\]|\\.)+(?<end>")/i,
+      # query parameter of URL: `?token=v` or `&client_secret=v`
+      /(?<begin>[?&][^=&\s"]*(?:#{KEY_SECRETS.join('|')})[^=&\s"]*=)[^&\s"#]+(?<end>)/i,
       # logged data
       /(?<begin>(?:#{NON_ENV_SECRETS.join('|')})[ =:]+)[^ "\n]#{SECRET_LENGTH}(?<end>$)/i,
       # cred in http dump
       /(?<begin>(?:#{HTTP_SECRETS.join('|')}): )[^\\]+(?<end>\\)/i
     ].freeze
-    private_constant :HIDDEN_PASSWORD, :ASCP_ENV_SECRETS, :KEY_SECRETS, :HTTP_SECRETS, :ALL_SECRETS, :NON_ENV_SECRETS, :KEY_FALSE_POSITIVES, :SECRET_LENGTH, :REGEX_LOG_REPLACES
+    private_constant :HIDDEN_PASSWORD, :ASCP_ENV_SECRETS, :KEY_SECRETS, :HTTP_SECRETS, :ALL_SECRETS, :NON_ENV_SECRETS, :KEY_FALSE_POSITIVES, :SECRET_LENGTH, :VALUE_IN_STRING_LENGTH, :REGEX_LOG_REPLACES
     attr_accessor :log_secrets
 
     # @return [Proc] new log formatter that hides secrets
@@ -60,11 +73,13 @@ module Aspera
 
     # Replace secrets in a string with the hidden password placeholder
     # @param value [String] Input string possibly containing secrets
-    # @param all   [Boolean] `false`: only private keys, `true`: all known secret patterns
+    # @param all   [Boolean] `false`: only declared values and private keys, `true`: also all known secret patterns
     # @return [String] String with secrets replaced by placeholder
     def hide_secrets_in_string(value, all: false)
+      marker = hidden_marker
+      value = hide_values_in_string(value, marker)
       (all ? REGEX_LOG_REPLACES : REGEX_LOG_REPLACES.first(1)).each do |reg_ex|
-        value = value.gsub(reg_ex) { "#{Regexp.last_match(:begin)}#{HIDDEN_PASSWORD}#{Regexp.last_match(:end)}" }
+        value = value.gsub(reg_ex) { "#{Regexp.last_match(:begin)}#{marker}#{Regexp.last_match(:end)}" }
       end
       return value
     end
@@ -84,21 +99,57 @@ module Aspera
       ALL_SECRETS.any? { |kw| keyword.include?(kw.downcase) }
     end
 
-    # Hides recursively secrets in Hash or Array of Hash
-    def deep_remove_secret(obj)
+    # Hides secrets in a structure: by key name, and declared values, also in key names (e.g. a secret given as name of a parameter).
+    # Containers are copied, not modified: the structure may be shared (e.g. configuration, transfer spec).
+    # @param obj           [Object]  Hash, Array, Struct, String, or other value (returned as is)
+    # @param short_in_text [Boolean] `true`: a declared value shorter than 8 characters is also hidden inside a text, as a word
+    #   (not next to a letter or digit), e.g. a short password in an error message of `mcp server`
+    # @return [Object] Copy of obj with secrets replaced by placeholder
+    def hidden_copy(obj, short_in_text: false)
       case obj
-      when Array
-        obj.each { |i| deep_remove_secret(i) }
       when Hash
-        obj.each do |k, v|
-          if secret?(k, v)
-            obj[k] = HIDDEN_PASSWORD
-          elsif v.is_a?(Hash) || v.is_a?(Array)
-            deep_remove_secret(v)
-          end
+        obj.to_h do |k, v|
+          [k.is_a?(String) ? hidden_copy(k, short_in_text: short_in_text) : k, secret?(k, v) ? hidden_marker : hidden_copy(v, short_in_text: short_in_text)]
         end
+      when Array then obj.map { |i| hidden_copy(i, short_in_text: short_in_text) }
+      when Struct then obj.dup.tap { |s| s.each_pair { |k, v| s[k] = secret?(k, v) ? hidden_marker : hidden_copy(v, short_in_text: short_in_text) } }
+      when String then hide_values_in_string(obj, hidden_marker, short_in_text: short_in_text)
+      else obj
+      end
+    end
+
+    # @param obj [Object] Hash, Array, Struct, String, or other value
+    # @return [Boolean] `true` if obj contains a secret: by key name, or declared value
+    def contains_secret?(obj)
+      !hidden_copy(obj).eql?(obj)
+    end
+
+    # Declare secret values: hidden wherever they appear (logs, results), also when escaped in JSON, Ruby or URL
+    # @param values [Array<Object>] Secret values: only non-empty Strings are retained
+    def add_secret_values(*values)
+      forms = values.select { |v| v.is_a?(String) && !v.empty? }.flat_map do |v|
+        [v, (JSON.generate(v)[1..-2] rescue nil), v.inspect[1..-2], URI.encode_www_form_component(v)]
+      end.compact
+      return if forms.empty?
+      @mutex.synchronize do
+        values = @values | forms
+        next if values.size.eql?(@values.size)
+        long, short = values.partition { |v| v.length >= VALUE_IN_STRING_LENGTH }
+        # Longest first, so that a secret is not partly hidden by another one that it contains
+        @values_regex = long.empty? ? nil : Regexp.union(long.sort_by { |v| -v.length })
+        @short_values_regex = short.empty? ? nil : /(?<![[:alnum:]])#{Regexp.union(short.sort_by { |v| -v.length })}(?![[:alnum:]])/
+        # Replaced, not modified, for readers in other threads
+        @values = values.freeze
+      end
+    end
+
+    # Declare the secret values found by key name in a structure
+    # @param obj [Object] Hash or Array, e.g. a transfer spec
+    def add_secrets_in(obj)
+      case obj
+      when Array then obj.each { |i| add_secrets_in(i) }
+      when Hash then obj.each { |k, v| secret?(k, v) ? add_secret_values(v) : add_secrets_in(v) }
       end
-      return obj
     end
 
     # Declare additional exact key names whose values are secrets
@@ -112,6 +163,34 @@ module Aspera
     def initialize
       @log_secrets = false
       @additional_keys = Set.new
+      # Declared secret values, and their escaped forms
+      @values = Set.new.freeze
+      # Matches declared values long enough to be hidden inside a string
+      @values_regex = nil
+      # Matches shorter declared values as a word
+      @short_values_regex = nil
+      @mutex = Mutex.new
+    end
+
+    # @param value         [String]  Input string
+    # @param marker        [String]  Placeholder
+    # @param short_in_text [Boolean] `true`: also hide short values as a word
+    # @return [String] value with declared secret values replaced by placeholder
+    def hide_values_in_string(value, marker, short_in_text: false)
+      return marker if @values.include?(value)
+      long_regex = @values_regex
+      short_regex = @short_values_regex if short_in_text
+      # Long values first, as they may contain short ones
+      value = value.gsub(long_regex, marker) unless long_regex.nil?
+      value = value.gsub(short_regex, marker) unless short_regex.nil?
+      value
+    end
+
+    # @return [String] Placeholder for hidden secrets, in ASCII when Unicode is not supported
+    def hidden_marker
+      # Required here, as `aspera/environment` requires `aspera/log`, which requires this file
+      require 'aspera/environment'
+      Environment.char(HIDDEN_PASSWORD)
     end
   end
 end

data/lib/aspera/secret_strength.rb

@@ -0,0 +1,114 @@
+# frozen_string_literal: true
+
+require 'aspera/log'
+require 'aspera/assert'
+
+module Aspera
+  # Estimate of the strength of a secret (e.g. a token), in bits: log2 of the number of guesses to find it
+  #
+  # Uses gem `zxcvbn` if installed (dictionaries, l33t, dates...), else a built-in estimate, the lowest of:
+  # - alphabet: length x log2(size of character classes used)
+  # - Shannon: empirical entropy of characters (catches repeats)
+  # - patterns: repeats, sequences and keyboard rows cost 1 bit per character, words and years 2 bits per character
+  # Both under-estimate random secrets: a random secret of 32 bytes (e.g. `openssl rand -hex 32`) is far above any usual minimum.
+  module SecretStrength
+    # Default minimum strength of tokens, in bits: random tokens of 32 bytes are far above
+    DEFAULT_MIN_BITS = 80
+    # Classes of characters, with their size
+    CHARACTER_CLASSES = [[/[a-z]/, 26], [/[A-Z]/, 26], [/\d/, 10], [/[^a-zA-Z\d]/, 33]].freeze
+    # Adjacent characters in those rows form a sequence
+    KEYBOARD_ROWS = %w[1234567890 qwertyuiop asdfghjkl zxcvbnm azertyuiop qsdfghjklm wxcvbn].freeze
+    # Words (3+ letters, optionally capitalized) and years
+    LOW_ENTROPY_REGEXES = [/[A-Z]?[a-z]{3,}/, /(?:19|20)\d\d/].freeze
+    # Entropy of English text is 1 to 1.5 bit per letter: keep a margin
+    LOW_ENTROPY_BITS = 2.0
+    # zxcvbn is slow on long secrets: the beginning is enough to assess strength
+    ZXCVBN_MAX_LENGTH = 100
+    private_constant :CHARACTER_CLASSES, :KEYBOARD_ROWS, :LOW_ENTROPY_REGEXES, :LOW_ENTROPY_BITS, :ZXCVBN_MAX_LENGTH
+
+    class << self
+      # @param secret [String] Secret to assess
+      # @return [Float] Estimated strength in bits
+      def bits(secret)
+        secret = secret.to_s
+        return 0.0 if secret.empty?
+        zxcvbn_bits(secret) || estimate_bits(secret)
+      end
+
+      # Refuse an empty or weak secret: an empty one would often accept requests without secret
+      # @param secret   [String]    Secret to check
+      # @param name     [String]    Name of the parameter holding the secret, its minimum is in parameter `<name>_min_bits`
+      # @param min_bits [Integer]   Minimum estimated strength in bits, `0` checks only that the secret is not empty
+      # @param type     [Exception] Exception to raise
+      # @raise [Exception] if the secret is empty or weaker than `min_bits`
+      def assert_strong(secret, name:, min_bits: DEFAULT_MIN_BITS, type: ParameterError)
+        Aspera.assert(!secret.to_s.empty?, type: type) { "#{name} must not be empty" }
+        strength = bits(secret)
+        Aspera.assert(strength >= min_bits, type: type) do
+          "#{name} is too weak: about #{strength.round} bits, minimum #{min_bits} (#{name}_min_bits). Generate one with: openssl rand -hex 32"
+        end
+      end
+
+      # Built-in estimate, without dictionary
+      # @param secret [String] Secret to assess
+      # @return [Float] Estimated strength in bits
+      def estimate_bits(secret)
+        return 0.0 if secret.empty?
+        bits_per_char = Math.log2(CHARACTER_CLASSES.sum { |regex, size| secret.match?(regex) ? size : 0 })
+        [secret.size * bits_per_char, shannon_bits(secret), pattern_bits(secret, bits_per_char)].min
+      end
+
+      private
+
+      # @return [Float, nil] Strength estimated by gem `zxcvbn`, or `nil` if not installed
+      def zxcvbn_bits(secret)
+        return unless zxcvbn?
+        Math.log2(Zxcvbn.test(secret[0, ZXCVBN_MAX_LENGTH]).guesses)
+      rescue StandardError => e
+        Log.log.debug { "zxcvbn failed: #{e.message}" }
+        nil
+      end
+
+      # @return [Boolean] `true` if gem `zxcvbn` (or `zxcvbn-ruby`) is installed
+      def zxcvbn?
+        return @zxcvbn unless @zxcvbn.nil?
+        @zxcvbn =
+          begin
+            require 'zxcvbn'
+            true
+          rescue LoadError
+            false
+          end
+      end
+
+      # Empirical Shannon entropy of characters of the secret
+      def shannon_bits(secret)
+        secret.chars.tally.values.sum do |count|
+          probability = count.fdiv(secret.size)
+          -probability * Math.log2(probability)
+        end * secret.size
+      end
+
+      # Sum of bits of each character, depending on patterns
+      def pattern_bits(secret, bits_per_char)
+        # Bits of characters in words and years
+        low = Array.new(secret.size)
+        LOW_ENTROPY_REGEXES.each do |regex|
+          secret.scan(regex) { low.fill(LOW_ENTROPY_BITS, Regexp.last_match.begin(0)...Regexp.last_match.end(0)) }
+        end
+        secret.chars.each_with_index.sum do |char, index|
+          next 1.0 if index.positive? && sequence?(secret[index - 1], char)
+          low[index] || bits_per_char
+        end
+      end
+
+      # @return [Boolean] `true` if `char` repeats `previous`, or follows it in alphabet, digits or keyboard row
+      def sequence?(previous, char)
+        previous = previous.downcase
+        char = char.downcase
+        return true if (char.ord - previous.ord).abs <= 1
+        KEYBOARD_ROWS.any? { |row| (one = row.index(previous)) && (two = row.index(char)) && (one - two).abs.eql?(1) }
+      end
+    end
+  end
+end