aspera-cli 4.21.1 → 4.22.0

data/lib/aspera/secret_hider.rb

@@ -15,8 +15,8 @@ module Aspera
     # keys in hash that contain secrets
     KEY_SECRETS = %w[password secret passphrase _key apikey crn token].freeze
     HTTP_SECRETS = %w[Authorization].freeze
-    ALL_SECRETS = [ASCP_ENV_SECRETS, KEY_SECRETS, HTTP_SECRETS].flatten.freeze
-    ALL_SECRETS2 = [KEY_SECRETS, HTTP_SECRETS].flatten.freeze
+    ALL_SECRETS = (ASCP_ENV_SECRETS + KEY_SECRETS + HTTP_SECRETS).freeze
+    ALL_SECRETS2 = (KEY_SECRETS + HTTP_SECRETS).freeze
     KEY_FALSE_POSITIVES = [/^access_key$/, /^fallback_private_key$/].freeze
     # regex that define named captures :begin and :end
     REGEX_LOG_REPLACES = [
@@ -38,6 +38,7 @@ module Aspera
     class << self
       attr_accessor :log_secrets
 
+      # @return new log formatter that hides secrets
       def log_formatter(original_formatter)
         original_formatter ||= Logger::Formatter.new
         # NOTE: that @log_secrets may be set AFTER this init is done, so it's done at runtime
@@ -51,21 +52,27 @@ module Aspera
         end
       end
 
+      def hide_secrets_in_string(value)
+        return value.gsub(REGEX_LOG_REPLACES.first){"#{Regexp.last_match(:begin)}#{HIDDEN_PASSWORD}#{Regexp.last_match(:end)}"}
+      end
+
+      # @return true if the key denotes a secret
       def secret?(keyword, value)
         keyword = keyword.to_s if keyword.is_a?(Symbol)
         # only Strings can be secrets, not booleans, or hash, arrays
         return false unless keyword.is_a?(String) && value.is_a?(String)
         # those are not secrets
-        return false if KEY_FALSE_POSITIVES.any?{|f|f.match?(keyword)}
+        return false if KEY_FALSE_POSITIVES.any?{ |f| f.match?(keyword)}
         return true if ADDITIONAL_KEYS_TO_HIDE.include?(keyword)
         # check if keyword (name) contains an element that designate it as a secret
-        ALL_SECRETS.any?{|kw|keyword.include?(kw)}
+        ALL_SECRETS.any?{ |kw| keyword.include?(kw)}
       end
 
+      # Hides recursively secrets in Hash or Array of Hash
       def deep_remove_secret(obj)
         case obj
         when Array
-          obj.each{|i|deep_remove_secret(i)}
+          obj.each{ |i| deep_remove_secret(i)}
         when Hash
           obj.each do |k, v|
             if secret?(k, v)

data/lib/aspera/ssh.rb

@@ -24,8 +24,8 @@ module Aspera
 
       def disable_ecd_sha2_algorithms
         Log.log.debug('Disabling SSH ecdsa')
-        Net::SSH::Transport::Algorithms::ALGORITHMS.each_value { |a| a.reject! { |a| a =~ /^ecd(sa|h)-sha2/ } }
-        Net::SSH::KnownHosts::SUPPORTED_TYPE.reject! { |t| t =~ /^ecd(sa|h)-sha2/ }
+        Net::SSH::Transport::Algorithms::ALGORITHMS.each_value{ |a| a.reject!{ |a| a =~ /^ecd(sa|h)-sha2/}}
+        Net::SSH::KnownHosts::SUPPORTED_TYPE.reject!{ |t| t =~ /^ecd(sa|h)-sha2/}
       end
     end
     # ssh_options: same as Net::SSH.start
@@ -49,7 +49,7 @@ module Aspera
       Net::SSH.start(@host, @username, @ssh_options) do |session|
         ssh_channel = session.open_channel do |channel|
           # prepare stdout processing
-          channel.on_data{|_chan, data|response.push(data)}
+          channel.on_data{ |_chan, data| response.push(data)}
           # prepare stderr processing, stderr if type = 1
           channel.on_extended_data do |_chan, _type, data|
             error_message = "#{cmd}: [#{data.chomp}]"
@@ -58,7 +58,7 @@ module Aspera
             raise error_message
           end
           # send command to SSH channel (execute) cspell: disable-next-line
-          channel.send('cexe'.reverse, cmd){|_ch, _success|channel.send_data(input) unless input.nil?}
+          channel.send('cexe'.reverse, cmd){ |_ch, _success| channel.send_data(input) unless input.nil?}
         end
         # wait for channel to finish (command exit)
         ssh_channel.wait

data/lib/aspera/temp_file_manager.rb

@@ -35,22 +35,23 @@ module Aspera
 
     # ensure that provided folder exists, or create it, generate a unique filename
     # @return path to that unique file
-    def new_file_path_in_folder(temp_folder, add_base = '')
+    def new_file_path_in_folder(temp_folder, prefix: nil, suffix: nil)
       FileUtils.mkdir_p(temp_folder)
-      new_file = File.join(temp_folder, add_base + SecureRandom.uuid)
+      new_file = File.join(temp_folder, [prefix, SecureRandom.uuid, suffix].compact.join('-'))
       @created_files.push(new_file)
       new_file
     end
 
     # same as above but in global temp folder, with user's name
-    def new_file_path_global(base_name)
+    def new_file_path_global(prefix=nil, suffix: nil)
       username =
         begin
           Etc.getlogin || Etc.getpwuid(Process.uid).name || 'unknown_user'
         rescue StandardError
           'unknown_user'
         end
-      new_file_path_in_folder(Etc.systmpdir, base_name + '_' + username + '_')
+      prefix = [prefix, username].compact.join('-')
+      new_file_path_in_folder(Etc.systmpdir, prefix: prefix, suffix: suffix)
     end
 
     def cleanup_expired(temp_folder)

data/lib/aspera/transfer/convert.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+require 'aspera/assert'
+module Aspera
+  module Transfer
+    # concertion class for transfert spec values to CLI values (ascp)
+    class Convert
+      class << self
+        # special encoding methods used in YAML (key: convert)
+        def remove_hyphen(value); value.tr('-', ''); end
+
+        # special encoding methods used in YAML (key: convert)
+        def json64(value); Base64.strict_encode64(JSON.generate(value)); end
+
+        # special encoding methods used in YAML (key: convert)
+        def base64(value); Base64.strict_encode64(value); end
+
+        # transform yes/no to true/false
+        def yes_to_true(value)
+          case value
+          when 'yes' then return true
+          when 'no' then return false
+          else Aspera.error_unexpected_value(value){'only: yes or no: '}
+          end
+        end
+      end
+    end
+  end
+end

data/lib/aspera/transfer/error_info.rb

@@ -10,81 +10,81 @@ module Aspera
     # rubocop:disable Layout/FirstHashElementLineBreak
     ERROR_INFO = {
       # id   retry-able     mnemo                       message                                              additional info
-      1  => { r: false, c: 'FASP_PROTO',                  m: 'Generic fasp(tm) protocol error',                a: 'fasp(tm) error'},
-      2  => { r: false, c: 'ASCP',                        m: 'Generic SCP error',                              a: 'ASCP error'},
-      3  => { r: false, c: 'AMBIGUOUS_TARGET',            m: 'Target incorrectly specified',                   a: 'Ambiguous target'},
-      4  => { r: false, c: 'NO_SUCH_FILE',                m: 'No such file or directory',                      a: 'No such file or directory'},
-      5  => { r: false, c: 'NO_PERMS',                    m: 'Insufficient permission to read or write',       a: 'Insufficient permissions'},
-      6  => { r: false, c: 'NOT_DIR',                     m: 'Target is not a directory',                      a: 'Target must be a directory'},
-      7  => { r: false, c: 'IS_DIR',                      m: 'File is a directory - expected regular file',    a: 'Expected regular file'},
-      8  => { r: false, c: 'USAGE',                       m: 'Incorrect usage of scp command',                 a: 'Incorrect usage of Aspera scp command'},
-      9  => { r: false, c: 'LIC_DUP',                     m: 'Duplicate license',                              a: 'Duplicate license'},
-      10 => { r: false, c: 'LIC_RATE_EXCEEDED',           m: 'Rate exceeds the cap imposed by license',        a: 'Rate exceeds cap imposed by license'},
-      11 => { r: false, c: 'INTERNAL_ERROR',              m: 'Internal error (unexpected error)',              a: 'Internal error'},
-      12 => { r: true,  c: 'TRANSFER_ERROR',              m: 'Error establishing control connection',
+      1  => {r: false, c: 'FASP_PROTO',                  m: 'Generic fasp(tm) protocol error',                a: 'fasp(tm) error'},
+      2  => {r: false, c: 'ASCP',                        m: 'Generic SCP error',                              a: 'ASCP error'},
+      3  => {r: false, c: 'AMBIGUOUS_TARGET',            m: 'Target incorrectly specified',                   a: 'Ambiguous target'},
+      4  => {r: false, c: 'NO_SUCH_FILE',                m: 'No such file or directory',                      a: 'No such file or directory'},
+      5  => {r: false, c: 'NO_PERMS',                    m: 'Insufficient permission to read or write',       a: 'Insufficient permissions'},
+      6  => {r: false, c: 'NOT_DIR',                     m: 'Target is not a directory',                      a: 'Target must be a directory'},
+      7  => {r: false, c: 'IS_DIR',                      m: 'File is a directory - expected regular file',    a: 'Expected regular file'},
+      8  => {r: false, c: 'USAGE',                       m: 'Incorrect usage of scp command',                 a: 'Incorrect usage of Aspera scp command'},
+      9  => {r: false, c: 'LIC_DUP',                     m: 'Duplicate license',                              a: 'Duplicate license'},
+      10 => {r: false, c: 'LIC_RATE_EXCEEDED',           m: 'Rate exceeds the cap imposed by license',        a: 'Rate exceeds cap imposed by license'},
+      11 => {r: false, c: 'INTERNAL_ERROR',              m: 'Internal error (unexpected error)',              a: 'Internal error'},
+      12 => {r: true,  c: 'TRANSFER_ERROR',              m: 'Error establishing control connection',
                                                               a: 'Error establishing SSH connection (check SSH port and firewall)'},
-      13 => { r: true,  c: 'TRANSFER_TIMEOUT',            m: 'Timeout establishing control connection',
+      13 => {r: true,  c: 'TRANSFER_TIMEOUT',            m: 'Timeout establishing control connection',
                                                               a: 'Timeout establishing SSH connection (check SSH port and firewall)'},
-      14 => { r: true,  c: 'CONNECTION_ERROR',            m: 'Error establishing data connection',
+      14 => {r: true,  c: 'CONNECTION_ERROR',            m: 'Error establishing data connection',
                                                               a: 'Error establishing UDP connection (check UDP port and firewall)'},
-      15 => { r: true,  c: 'CONNECTION_TIMEOUT',          m: 'Timeout establishing data connection',
+      15 => {r: true,  c: 'CONNECTION_TIMEOUT',          m: 'Timeout establishing data connection',
                                                               a: 'Timeout establishing UDP connection (check UDP port and firewall)'},
-      16 => { r: true,  c: 'CONNECTION_LOST',             m: 'Connection lost',                                a: 'Connection lost'},
-      17 => { r: true,  c: 'RCVR_SEND_ERROR',             m: 'Receiver fails to send feedback',                a: 'Network failure (receiver can\'t send feedback)'},
-      18 => { r: true,  c: 'RCVR_RECV_ERROR',             m: 'Receiver fails to receive data packets',         a: 'Network failure (receiver can\'t receive UDP data)'},
-      19 => { r: false, c: 'AUTH',                        m: 'Authentication failure',                         a: 'Authentication failure'},
-      20 => { r: false, c: 'NOTHING',                     m: 'Nothing to transfer',                            a: 'Nothing to transfer'},
-      21 => { r: false, c: 'NOT_REGULAR',                 m: 'Not a regular file (special file)',              a: 'Not a regular file'},
-      22 => { r: false, c: 'FILE_TABLE_OVR',              m: 'File table overflow',                            a: 'File table overflow'},
-      23 => { r: true,  c: 'TOO_MANY_FILES',              m: 'Too many files open',                            a: 'Too many files open'},
-      24 => { r: false, c: 'FILE_TOO_BIG',                m: 'File too big for file system',                   a: 'File too big for filesystem'},
-      25 => { r: false, c: 'NO_SPACE_LEFT',               m: 'No space left on disk',                          a: 'No space left on disk'},
-      26 => { r: false, c: 'READ_ONLY_FS',                m: 'Read only file system',                          a: 'Read only filesystem'},
-      27 => { r: false, c: 'SOME_FILE_ERRS',              m: 'Some individual files failed',                   a: 'One or more files failed'},
-      28 => { r: false, c: 'USER_CANCEL',                 m: 'Cancelled by user',                              a: 'Cancelled by user'},
-      29 => { r: false, c: 'LIC_NOLIC',                   m: 'License not found or unable to access',          a: 'Unable to access license info'},
-      30 => { r: false, c: 'LIC_EXPIRED',                 m: 'License expired',                                a: 'License expired'},
-      31 => { r: false, c: 'SOCK_SETUP',                  m: 'Unable to setup socket (create, bind, etc ...)', a: 'Unable to set up socket'},
-      32 => { r: true,  c: 'OUT_OF_MEMORY',               m: 'Out of memory, unable to allocate',              a: 'Out of memory'},
-      33 => { r: true,  c: 'THREAD_SPAWN',                m: 'Can\'t spawn thread',                            a: 'Unable to spawn thread'},
-      34 => { r: false, c: 'UNAUTHORIZED',                m: 'Unauthorized by external auth server',           a: 'Unauthorized'},
-      35 => { r: true,  c: 'DISK_READ',                   m: 'Error reading source file from disk',            a: 'Disk read error'},
-      36 => { r: true,  c: 'DISK_WRITE',                  m: 'Error writing to disk',                          a: 'Disk write error'},
-      37 => { r: true,  c: 'AUTHORIZATION',               m: 'Used interchangeably with ERR_UNAUTHORIZED',     a: 'Authorization failure'},
-      38 => { r: false, c: 'LIC_ILLEGAL',                 m: 'Operation not permitted by license',             a: 'Operation not permitted by license'},
-      39 => { r: true,  c: 'PEER_ABORTED_SESSION',        m: 'Remote peer terminated session',                 a: 'Peer aborted session'},
-      40 => { r: true,  c: 'DATA_TRANSFER_TIMEOUT',       m: 'Transfer stalled, timed out',                    a: 'Data transfer stalled, timed out'},
-      41 => { r: false, c: 'BAD_PATH',                    m: 'Path violates docroot containment',              a: 'File location is outside \'docroot\' hierarchy'},
-      42 => { r: false, c: 'ALREADY_EXISTS',              m: 'File or directory already exists',               a: 'File or directory already exists'},
-      43 => { r: false, c: 'STAT_FAILS',                  m: 'Cannot stat file',                               a: 'Cannot collect details about file or directory'},
-      44 => { r: true,  c: 'PMTU_BRTT_ERROR',             m: 'UDP session initiation fatal error',             a: 'UDP session initiation fatal error'},
-      45 => { r: true,  c: 'BWMEAS_ERROR',                m: 'Bandwidth measurement fatal error',              a: 'Bandwidth measurement fatal error'},
-      46 => { r: false, c: 'VLINK_ERROR',                 m: 'Virtual link error',                             a: 'Virtual link error'},
-      47 => { r: false, c: 'CONNECTION_ERROR_HTTP',       m: 'Error establishing HTTP connection',
+      16 => {r: true,  c: 'CONNECTION_LOST',             m: 'Connection lost',                                a: 'Connection lost'},
+      17 => {r: true,  c: 'RCVR_SEND_ERROR',             m: 'Receiver fails to send feedback',                a: 'Network failure (receiver can\'t send feedback)'},
+      18 => {r: true,  c: 'RCVR_RECV_ERROR',             m: 'Receiver fails to receive data packets',         a: 'Network failure (receiver can\'t receive UDP data)'},
+      19 => {r: false, c: 'AUTH',                        m: 'Authentication failure',                         a: 'Authentication failure'},
+      20 => {r: false, c: 'NOTHING',                     m: 'Nothing to transfer',                            a: 'Nothing to transfer'},
+      21 => {r: false, c: 'NOT_REGULAR',                 m: 'Not a regular file (special file)',              a: 'Not a regular file'},
+      22 => {r: false, c: 'FILE_TABLE_OVR',              m: 'File table overflow',                            a: 'File table overflow'},
+      23 => {r: true,  c: 'TOO_MANY_FILES',              m: 'Too many files open',                            a: 'Too many files open'},
+      24 => {r: false, c: 'FILE_TOO_BIG',                m: 'File too big for file system',                   a: 'File too big for filesystem'},
+      25 => {r: false, c: 'NO_SPACE_LEFT',               m: 'No space left on disk',                          a: 'No space left on disk'},
+      26 => {r: false, c: 'READ_ONLY_FS',                m: 'Read only file system',                          a: 'Read only filesystem'},
+      27 => {r: false, c: 'SOME_FILE_ERRS',              m: 'Some individual files failed',                   a: 'One or more files failed'},
+      28 => {r: false, c: 'USER_CANCEL',                 m: 'Cancelled by user',                              a: 'Cancelled by user'},
+      29 => {r: false, c: 'LIC_NOLIC',                   m: 'License not found or unable to access',          a: 'Unable to access license info'},
+      30 => {r: false, c: 'LIC_EXPIRED',                 m: 'License expired',                                a: 'License expired'},
+      31 => {r: false, c: 'SOCK_SETUP',                  m: 'Unable to setup socket (create, bind, etc ...)', a: 'Unable to set up socket'},
+      32 => {r: true,  c: 'OUT_OF_MEMORY',               m: 'Out of memory, unable to allocate',              a: 'Out of memory'},
+      33 => {r: true,  c: 'THREAD_SPAWN',                m: 'Can\'t spawn thread',                            a: 'Unable to spawn thread'},
+      34 => {r: false, c: 'UNAUTHORIZED',                m: 'Unauthorized by external auth server',           a: 'Unauthorized'},
+      35 => {r: true,  c: 'DISK_READ',                   m: 'Error reading source file from disk',            a: 'Disk read error'},
+      36 => {r: true,  c: 'DISK_WRITE',                  m: 'Error writing to disk',                          a: 'Disk write error'},
+      37 => {r: true,  c: 'AUTHORIZATION',               m: 'Used interchangeably with ERR_UNAUTHORIZED',     a: 'Authorization failure'},
+      38 => {r: false, c: 'LIC_ILLEGAL',                 m: 'Operation not permitted by license',             a: 'Operation not permitted by license'},
+      39 => {r: true,  c: 'PEER_ABORTED_SESSION',        m: 'Remote peer terminated session',                 a: 'Peer aborted session'},
+      40 => {r: true,  c: 'DATA_TRANSFER_TIMEOUT',       m: 'Transfer stalled, timed out',                    a: 'Data transfer stalled, timed out'},
+      41 => {r: false, c: 'BAD_PATH',                    m: 'Path violates docroot containment',              a: 'File location is outside \'docroot\' hierarchy'},
+      42 => {r: false, c: 'ALREADY_EXISTS',              m: 'File or directory already exists',               a: 'File or directory already exists'},
+      43 => {r: false, c: 'STAT_FAILS',                  m: 'Cannot stat file',                               a: 'Cannot collect details about file or directory'},
+      44 => {r: true,  c: 'PMTU_BRTT_ERROR',             m: 'UDP session initiation fatal error',             a: 'UDP session initiation fatal error'},
+      45 => {r: true,  c: 'BWMEAS_ERROR',                m: 'Bandwidth measurement fatal error',              a: 'Bandwidth measurement fatal error'},
+      46 => {r: false, c: 'VLINK_ERROR',                 m: 'Virtual link error',                             a: 'Virtual link error'},
+      47 => {r: false, c: 'CONNECTION_ERROR_HTTP',       m: 'Error establishing HTTP connection',
                                                           a: 'Error establishing HTTP connection (check HTTP port and firewall)'},
-      48 => { r: false, c: 'FILE_ENCRYPTION_ERROR',       m: 'File encryption error, e.g. corrupt file',
+      48 => {r: false, c: 'FILE_ENCRYPTION_ERROR',       m: 'File encryption error, e.g. corrupt file',
                                                           a: 'File encryption/decryption error, e.g. corrupt file'},
-      49 => { r: false, c: 'FILE_DECRYPTION_PASS',        m: 'File encryption/decryption error, e.g. corrupt file', a: 'File decryption error, bad passphrase'},
-      50 => { r: false, c: 'BAD_CONFIGURATION',           m: 'Aspera.conf contains invalid data and was rejected',  a: 'Invalid configuration'},
-      51 => { r: false, c: 'INSECURE_CONNECTION',         m: 'Remote-host key check failure',                  a: 'Remote host is not who we expected'},
-      52 => { r: false, c: 'START_VALIDATION_FAILED',     m: 'File start validation failed',                   a: 'File start validation failed'},
-      53 => { r: false, c: 'STOP_VALIDATION_FAILED',      m: 'File stop validation failed',                    a: 'File stop validation failed'},
-      54 => { r: false, c: 'THRESHOLD_VALIDATION_FAILED', m: 'File threshold validation failed',               a: 'File threshold validation failed'},
-      55 => { r: false, c: 'FILEPATH_TOO_LONG',           m: 'File path/name too long for underlying file system', a: 'File path exceeds underlying file system limit'},
-      56 => { r: false, c: 'ILLEGAL_CHARS_IN_PATH',       m: 'Windows path contains illegal characters',
+      49 => {r: false, c: 'FILE_DECRYPTION_PASS',        m: 'File encryption/decryption error, e.g. corrupt file', a: 'File decryption error, bad passphrase'},
+      50 => {r: false, c: 'BAD_CONFIGURATION',           m: 'Aspera.conf contains invalid data and was rejected',  a: 'Invalid configuration'},
+      51 => {r: false, c: 'INSECURE_CONNECTION',         m: 'Remote-host key check failure',                  a: 'Remote host is not who we expected'},
+      52 => {r: false, c: 'START_VALIDATION_FAILED',     m: 'File start validation failed',                   a: 'File start validation failed'},
+      53 => {r: false, c: 'STOP_VALIDATION_FAILED',      m: 'File stop validation failed',                    a: 'File stop validation failed'},
+      54 => {r: false, c: 'THRESHOLD_VALIDATION_FAILED', m: 'File threshold validation failed',               a: 'File threshold validation failed'},
+      55 => {r: false, c: 'FILEPATH_TOO_LONG',           m: 'File path/name too long for underlying file system', a: 'File path exceeds underlying file system limit'},
+      56 => {r: false, c: 'ILLEGAL_CHARS_IN_PATH',       m: 'Windows path contains illegal characters',
                                                               a: 'Path being written to Windows file system contains illegal characters'},
-      57 => { r: false, c: 'CHUNK_MUST_MATCH_ALIGNMENT',  m: 'Chunk size/start must be aligned with storage',  a: 'Chunk size/start must be aligned with storage'},
-      58 => { r: false, c: 'VALIDATION_SESSION_ABORT',    m: 'Session aborted to due to validation error',     a: 'Session aborted to due validation error'},
-      59 => { r: false, c: 'REMOTE_STORAGE_ERROR',        m: 'Remote storage errored',                         a: 'Remote storage errored'},
-      60 => { r: false, c: 'LUA_SCRIPT_ABORTED_SESSION',  m: 'Session aborted due to Lua script abort',        a: 'Session aborted due to Lua script abort'},
-      61 => { r: true,  c: 'SSEAR_RETRYABLE',             m: 'Transfer failed because of a retryable Encryption at Rest error',
+      57 => {r: false, c: 'CHUNK_MUST_MATCH_ALIGNMENT',  m: 'Chunk size/start must be aligned with storage',  a: 'Chunk size/start must be aligned with storage'},
+      58 => {r: false, c: 'VALIDATION_SESSION_ABORT',    m: 'Session aborted to due to validation error',     a: 'Session aborted to due validation error'},
+      59 => {r: false, c: 'REMOTE_STORAGE_ERROR',        m: 'Remote storage errored',                         a: 'Remote storage errored'},
+      60 => {r: false, c: 'LUA_SCRIPT_ABORTED_SESSION',  m: 'Session aborted due to Lua script abort',        a: 'Session aborted due to Lua script abort'},
+      61 => {r: true,  c: 'SSEAR_RETRYABLE',             m: 'Transfer failed because of a retryable Encryption at Rest error',
                                                               a: 'Transfer failed because of a retryable Encryption at Rest error'},
-      62 => { r: false, c: 'SSEAR_FATAL',                 m: 'Transfer failed because of a fatal Encryption at Rest error',
+      62 => {r: false, c: 'SSEAR_FATAL',                 m: 'Transfer failed because of a fatal Encryption at Rest error',
                                                               a: 'Transfer failed because of a fatal Encryption at Rest error'},
-      63 => { r: false, c: 'LINK_LOOP',                   m: 'Path refers to a symbolic link loop',            a: 'Path refers to a symbolic link loop'},
-      64 => { r: false, c: 'CANNOT_RENAME_PARTIAL_FILES', m: 'Can\'t rename a partial file',                   a: 'Can\'t rename a partial file.'},
-      65 => { r: false, c: 'CIPHER_NON_COMPAT_FIPS',      m: 'Can\'t use this cipher with FIPS mode enabled',  a: 'Can\'t use this cipher with FIPS mode enabled'},
-      66 => { r: false, c: 'PEER_REQUIRES_FIPS',          m: 'Peer rejects cipher due to FIPS mode enabled on peer',
+      63 => {r: false, c: 'LINK_LOOP',                   m: 'Path refers to a symbolic link loop',            a: 'Path refers to a symbolic link loop'},
+      64 => {r: false, c: 'CANNOT_RENAME_PARTIAL_FILES', m: 'Can\'t rename a partial file',                   a: 'Can\'t rename a partial file.'},
+      65 => {r: false, c: 'CIPHER_NON_COMPAT_FIPS',      m: 'Can\'t use this cipher with FIPS mode enabled',  a: 'Can\'t use this cipher with FIPS mode enabled'},
+      66 => {r: false, c: 'PEER_REQUIRES_FIPS',          m: 'Peer rejects cipher due to FIPS mode enabled on peer',
                                                               a: 'Peer rejects cipher due to FIPS mode enabled on peer'}
     }.freeze
     # rubocop:enable Layout/MultilineHashKeyLineBreaks

data/lib/aspera/transfer/parameters.rb

@@ -6,9 +6,9 @@ require 'aspera/command_line_builder'
 require 'aspera/temp_file_manager'
 require 'aspera/transfer/error'
 require 'aspera/transfer/spec'
+require 'aspera/transfer/convert'
 require 'aspera/ascp/installation'
 require 'aspera/cli/formatter'
-require 'aspera/agent/base'
 require 'aspera/rest'
 require 'securerandom'
 require 'base64'
@@ -20,15 +20,10 @@ module Aspera
   module Transfer
     # translate transfer specification to ascp parameter list
     class Parameters
-      # Agents shown in manual for parameters (sub list)
-      SUPPORTED_AGENTS = Agent::Base.agent_list.freeze
       FILE_LIST_OPTIONS = ['--file-list', '--file-pair-list'].freeze
-      # Short names of columns in manual
-      SUPPORTED_AGENTS_SHORT = SUPPORTED_AGENTS.map{|agent_sym|agent_sym.to_s[0].to_sym}
+      private_constant :FILE_LIST_OPTIONS
       HTTP_FALLBACK_ACTIVATION_VALUES = ['1', 1, true, 'force'].freeze
 
-      private_constant :SUPPORTED_AGENTS, :FILE_LIST_OPTIONS
-
       class << self
         # temp file list files are created here
         def file_list_folder=(value)
@@ -46,64 +41,14 @@ module Aspera
           @file_list_folder ||= TempFileManager.instance.new_file_path_global('asession_filelists')
         end
 
-        # @param formatter [Cli::Formatter] formatter to use
-        # @return a table suitable to display in manual
-        def man_table(formatter)
-          result = []
-          Spec::DESCRIPTION.each do |name, options|
-            param = {name: name, type: [options[:accepted_types]].flatten.join(','), description: options[:desc]}
-            # add flags for supported agents in doc
-            SUPPORTED_AGENTS.each do |agent_sym|
-              param[agent_sym.to_s[0].to_sym] = Cli::Formatter.tick(options[:agents].nil? || options[:agents].include?(agent_sym))
-            end
-            # only keep lines that are usable in supported agents
-            next if SUPPORTED_AGENTS_SHORT.inject(true){|memory, agent_short_sym|memory && param[agent_short_sym].empty?}
-            param[:cli] =
-              case options[:cli][:type]
-              when :envvar then 'env:' + options[:cli][:variable]
-              when :opt_without_arg then options[:cli][:switch]
-              when :opt_with_arg
-                values = if options.key?(:enum)
-                  ['enum']
-                elsif options[:accepted_types].is_a?(Array)
-                  options[:accepted_types]
-                elsif !options[:accepted_types].nil?
-                  [options[:accepted_types]]
-                else
-                  raise "error: #{param}"
-                end.map{|n|"{#{n}}"}.join('|')
-                conversion_tag = options[:cli].key?(:convert) ? '(conversion)' : ''
-                "#{options[:cli][:switch]} #{conversion_tag}#{values}"
-              when :special then formatter.special_format('special')
-              when :ignore then formatter.special_format('ignored')
-              else
-                param[:d].eql?(tick_yes) ? '' : 'n/a'
-              end
-            param[:description] += "\nAllowed values: #{options[:enum].join(', ')}" if options.key?(:enum)
-            # replace "solidus" HTML entity with its text value
-            param[:description] = param[:description].gsub('&sol;', '\\')
-            result.push(param)
-          end
-          return result.sort_by { |parameter_info| parameter_info[:name] }
-        end
-
-        # special encoding methods used in YAML (key: :convert)
-        def convert_remove_hyphen(value); value.tr('-', ''); end
-
-        # special encoding methods used in YAML (key: :convert)
-        def convert_json64(value); Base64.strict_encode64(JSON.generate(value)); end
-
-        # special encoding methods used in YAML (key: :convert)
-        def convert_base64(value); Base64.strict_encode64(value); end
-
         # file list is provided directly with ascp arguments
-        # @param ascp_args [Array,NilClass] ascp arguments
+        # @columns ascp_args [Array,NilClass] ascp arguments
         def ascp_args_file_list?(ascp_args)
-          ascp_args&.any?{|i|FILE_LIST_OPTIONS.include?(i)}
+          ascp_args&.any?{ |i| FILE_LIST_OPTIONS.include?(i)}
         end
       end
 
-      # @param options [Hash] key: :wss: bool, :ascp_args: array of strings
+      # @columns options [Hash] key: :wss: bool, :ascp_args: array of strings
       def initialize(
         job_spec,
         ascp_args:       nil,
@@ -125,7 +70,7 @@ module Aspera
         Aspera.assert(@ascp_args.all?(String)){'all ascp arguments must be String'}
         Aspera.assert_type(@trusted_certs, Array){'trusted_certs'}
         Aspera.assert_values(@client_ssh_key, Ascp::Installation::CLIENT_SSH_KEY_OPTIONS)
-        @builder = CommandLineBuilder.new(@job_spec, Spec::DESCRIPTION)
+        @builder = CommandLineBuilder.new(@job_spec, Spec::SCHEMA, Convert)
       end
 
       # either place source files on command line, or add file list file
@@ -133,29 +78,29 @@ module Aspera
         # is the file list provided through ascp parameters?
         ascp_file_list_provided = self.class.ascp_args_file_list?(@ascp_args)
         # set if paths is mandatory in ts
-        @builder.params_definition['paths'][:mandatory] = !@job_spec.key?('keepalive') && !ascp_file_list_provided # cspell:words keepalive
+        @builder.required('paths', !@job_spec.key?('keepalive') && !ascp_file_list_provided) # cspell:words keepalive
         # get paths in transfer spec (after setting if it is mandatory)
         ts_paths_array = @builder.read_param('paths')
         file_list_option = nil
         # transfer spec contains paths ?
         if !ts_paths_array.nil?
           Aspera.assert(!ascp_file_list_provided){'file list provided both in transfer spec and ascp file list. Remove one of them.'}
-          Aspera.assert(ts_paths_array.all?{|i|i.key?('source')}){"All elements of paths must have a 'source' key"}
-          is_pair_list = ts_paths_array.any?{|i|i.key?('destination')}
-          raise "All elements of paths must be consistent with 'destination' key" if is_pair_list && !ts_paths_array.all?{|i|i.key?('destination')}
+          Aspera.assert(ts_paths_array.all?{ |i| i.key?('source')}){"All elements of paths must have a 'source' key"}
+          is_pair_list = ts_paths_array.any?{ |i| i.key?('destination')}
+          raise "All elements of paths must be consistent with 'destination' key" if is_pair_list && !ts_paths_array.all?{ |i| i.key?('destination')}
           if self.class.file_list_folder.nil?
             Aspera.assert(!is_pair_list){'file pair list is not supported when file list folder is not set'}
             # not safe for special characters ? (maybe not, depends on OS)
             Log.log.debug('placing source file list on command line (no file list file)')
-            @builder.add_command_line_options(ts_paths_array.map{|i|i['source']})
+            @builder.add_command_line_options(ts_paths_array.map{ |i| i['source']})
           else
             # safer option: generate a file list file if there is storage defined for it
             if is_pair_list
               file_list_option = '--file-pair-list'
-              lines = ts_paths_array.each_with_object([]){|e, m|m.push(e['source'], e['destination']) }
+              lines = ts_paths_array.each_with_object([]){ |e, m| m.push(e['source'], e['destination'])}
             else
               file_list_option = '--file-list'
-              lines = ts_paths_array.map{|i|i['source']}
+              lines = ts_paths_array.map{ |i| i['source']}
             end
             file_list_file = TempFileManager.instance.new_file_path_in_folder(self.class.file_list_folder)
             Log.log.debug{Log.dump(:file_list, lines)}

data/lib/aspera/transfer/spec.rb

@@ -2,7 +2,6 @@
 
 require 'aspera/command_line_builder'
 require 'aspera/assert'
-require 'yaml'
 
 module Aspera
   module Transfer
@@ -36,12 +35,12 @@ module Aspera
           XFER_DIR_TO_TYPE.fetch(direction)
         end
       end
-      DESCRIPTION = CommandLineBuilder.normalize_description(YAML.load_file("#{__FILE__[0..-3]}yaml"))
+      SCHEMA = CommandLineBuilder.read_schema(__FILE__)
       # define constants for enums of parameters: <parameter>_<enum>, e.g. CIPHER_AES_128, DIRECTION_SEND, ...
-      DESCRIPTION.each do |name, description|
-        next unless description[:enum].is_a?(Array)
-        const_set(:"#{name.to_s.upcase}_ENUM_VALUES", description[:enum])
-        description[:enum].each do |enum|
+      SCHEMA['properties'].each do |name, description|
+        next unless description['enum'].is_a?(Array)
+        const_set(:"#{name.to_s.upcase}_ENUM_VALUES", description['enum'])
+        description['enum'].each do |enum|
           const_set("#{name.to_s.upcase}_#{enum.upcase.gsub(/[^A-Z0-9]/, '_')}", enum.freeze)
         end
       end