aspera-cli 4.13.0 → 4.15.0

data/lib/aspera/fasp/management.rb

@@ -0,0 +1,244 @@
+# frozen_string_literal: true
+
+module Aspera
+  module Fasp
+    # executes a local "ascp", connects mgt port, equivalent of "Fasp Manager"
+    class Management
+      # cspell: disable
+      OPERATIONS = %w[
+        NOP
+        START
+        QUERY
+        QUERYRSP
+        STATS
+        STOP
+        ERROR
+        CANCEL
+        DONE
+        RATE
+        FILEERROR
+        SESSION
+        NOTIFICATION
+        INIT
+        VLINK
+        NOTIFICATION
+        PUT
+        WRITE
+        CLOSE
+        SKIP
+        ARGSTOP
+      ]
+
+      PARAMETERS = %w[
+        Type
+        File
+        Size
+        Written
+        Bytescont
+        Rate
+        Loss
+        Query
+        Code
+        Password
+        Progress
+        Remaining
+        Elapsed
+        RexInfo
+        BlockInfo
+        DiskInfo
+        RateInfo
+        MinRate
+        Description
+        Elapsedusec
+        ServiceLevel
+        SessionId
+        User
+        Host
+        Encryption
+        Adaptive
+        Direction
+        Remote
+        Port
+        UserStr
+        CommandId
+        StartByte
+        EndByte
+        Token
+        Cookie
+        QueryResponse
+        Source
+        Destination
+        BWMeasurement
+        BWInfo
+        PMTU
+        TransferBytes
+        FileBytes
+        Operation
+        Delay
+        PreTransferFiles
+        PreTransferDirs
+        PreTransferSpecial
+        PreTransferFailed
+        PartialPreTransferBytes
+        PreTransferBytes
+        Priority
+        Transport
+        VlinkID
+        VlinkOn
+        VlinkCapIn
+        VlinkCapOut
+        ManifestFile
+        ArgScansAttempted
+        ArgScansCompleted
+        PathScansAttempted
+        PathScansFailed
+        PathScansIrregular
+        PathScansExcluded
+        DirScansCompleted
+        FileScansCompleted
+        DirCreatesAttempted
+        DirCreatesFailed
+        DirCreatesPassed
+        TransfersAttempted
+        TransfersFailed
+        TransfersPassed
+        TransfersSkipped
+        FallbackProtocol
+        RetryTimeout
+        PreTransferExcluded
+        XferId
+        XferRetry
+        Tags
+        FaspFileArgIndex
+        ArgTransfersStatus
+        ArgTransfersAttempted
+        ArgTransfersFailed
+        ArgTransfersPassed
+        ArgTransfersSkipped
+        FaspFileID
+        RateCap
+        MinRateCap
+        PolicyCap
+        PriorityCap
+        RateLock
+        MinRateLock
+        PolicyLock
+        FileChecksum
+        ServerHostname
+        ServerNodeId
+        ClientNodeId
+        ServerClusterId
+        ClientClusterId
+        FileChecksumType
+        ServerDocroot
+        ClientDocroot
+        NodeUser
+        ClientUser
+        SourcePrefix
+        RemoteAddress
+        TCPPort
+        Cipher
+        ResumePolicy
+        CreatePolicy
+        ManifestPolicy
+        Precalc
+        OverwritePolicy
+        RTTAutocorrect
+        TimePolicy
+        ManifestPath
+        ManifestInprogress
+        PartialFiles
+        FilesEncrypt
+        FilesDecrypt
+        DatagramSize
+        PrepostCommand
+        XoptFlags
+        VLinkVersion
+        PeerVLinkVersion
+        VLinkLocalEnabled
+        VLinkLocalId
+        VLinkLocalCL
+        VLinkRemoteEnabled
+        VLinkRemoteId
+        VLRemoteCL
+        DSPipelineDepth
+        PeerDSPipelineDepth
+        LocalIP
+        SourceBase
+        ReadBlockSize
+        WriteBlockSize
+        ClusterNumNodes
+        ClusterNodeId
+        MoveRange
+        MoveRangeLow
+        MoveRangeHigh
+        Keepalive
+        TestLogin
+        UseProxy
+        ProxyIP
+        RateControlAlgorithm
+        ClientMacAddress
+        Offset
+        ChunkSize
+        PostTransferValidation
+        OverwritePolicyCap
+        ExtraCreatePolicy]
+      # Management port start message
+      MGT_HEADER = 'FASPMGR 2'
+      # fields description for JSON generation
+      # spellchecker: disable
+      INTEGER_FIELDS = %w[Bytescont FaspFileArgIndex StartByte Rate MinRate Port Priority RateCap MinRateCap TCPPort CreatePolicy TimePolicy
+                          DatagramSize XoptFlags VLinkVersion PeerVLinkVersion DSPipelineDepth PeerDSPipelineDepth ReadBlockSize WriteBlockSize
+                          ClusterNumNodes ClusterNodeId Size Written Loss FileBytes PreTransferBytes TransferBytes PMTU Elapsedusec ArgScansAttempted
+                          ArgScansCompleted PathScansAttempted FileScansCompleted TransfersAttempted TransfersPassed Delay].freeze
+      BOOLEAN_FIELDS = %w[Encryption Remote RateLock MinRateLock PolicyLock FilesEncrypt FilesDecrypt VLinkLocalEnabled VLinkRemoteEnabled
+                          MoveRange Keepalive TestLogin UseProxy Precalc RTTAutocorrect].freeze
+      # cspell: enable
+
+      class << self
+        # translates legacy event into enhanced (JSON) event
+        def enhanced_event_format(event)
+          return event.keys.each_with_object({}) do |e, h|
+            # capital_to_snake_case
+            new_name = e
+                .gsub(/([a-z\d])([A-Z])/, '\1_\2')
+                .gsub(/([a-z\d])(usec)$/, '\1_\2')
+                .gsub(/([A-Z]+)([A-Z][a-z])/, '\1_\2')
+                .downcase
+            value = event[e]
+            value = value.to_i if INTEGER_FIELDS.include?(e)
+            value = value.eql?('Yes') if BOOLEAN_FIELDS.include?(e)
+            h[new_name] = value
+          end
+        end
+      end # class << self
+
+      def initialize
+        @event_build = nil
+        @last_event = nil
+      end
+      attr_reader :last_event
+
+      def process_line(line)
+        # Log.log.debug{"line=[#{line}]"}
+        case line
+        when MGT_HEADER
+          # begin event
+          @event_build = {}
+        when /^([^:]+): (.*)$/
+          # event field
+          @event_build[Regexp.last_match(1)] = Regexp.last_match(2)
+        when ''
+          # empty line is separator to end event information
+          raise 'unexpected empty line' if @event_build.nil?
+          @last_event = @event_build
+          @event_build = nil
+          return @last_event
+        else
+          raise "unexpected line:[#{line}]"
+        end # case
+        return nil
+      end
+    end
+  end
+end

data/lib/aspera/fasp/parameters.rb

@@ -3,6 +3,10 @@
 require 'aspera/log'
 require 'aspera/command_line_builder'
 require 'aspera/temp_file_manager'
+require 'aspera/fasp/error'
+require 'aspera/fasp/installation'
+require 'aspera/cli/formatter'
+require 'aspera/rest'
 require 'securerandom'
 require 'base64'
 require 'json'
@@ -15,10 +19,11 @@ module Aspera
     # translate transfer specification to ascp parameter list
     class Parameters
       # Agents shown in manual for parameters (sub list)
-      SUPPORTED_AGENTS = %i[direct node connect].freeze
+      SUPPORTED_AGENTS = %i[direct node connect trsdk httpgw].freeze
       # Short names of columns in manual
       SUPPORTED_AGENTS_SHORT = SUPPORTED_AGENTS.map{|a|a.to_s[0].to_sym}
       FILE_LIST_OPTIONS = ['--file-list', '--file-pair-list'].freeze
+      SUPPORTED_OPTIONS = %i[ascp_args wss check_ignore quiet].freeze
 
       private_constant :SUPPORTED_AGENTS, :FILE_LIST_OPTIONS
 
@@ -26,7 +31,7 @@ module Aspera
         # Temp folder for file lists, must contain only file lists
         # because of garbage collection takes any file there
         # this could be refined, as , for instance, on macos, temp folder is already user specific
-        @file_list_folder = TempFileManager.instance.new_file_path_global('asession_filelists')
+        @file_list_folder = TempFileManager.instance.new_file_path_global('asession_filelists') # cspell:disable-line
         @param_description_cache = nil
         # @return normalized description of transfer spec parameters, direct from yaml
         def description
@@ -46,7 +51,7 @@ module Aspera
             param = {name: name, type: [options[:accepted_types]].flatten.join(','), description: options[:desc]}
             # add flags for supported agents in doc
             SUPPORTED_AGENTS.each do |a|
-              param[a.to_s[0].to_sym] = options[:agents].nil? || options[:agents].include?(a) ? 'Y' : ''
+              param[a.to_s[0].to_sym] = Cli::Formatter.tick(options[:agents].nil? || options[:agents].include?(a))
             end
             # only keep lines that are usable in supported agents
             next if SUPPORTED_AGENTS_SHORT.inject(true){|m, j|m && param[j].empty?}
@@ -64,9 +69,12 @@ module Aspera
                 else
                   raise "error: #{param}"
                 end.map{|n|"{#{n}}"}.join('|')
-                conv = options[:cli].key?(:convert) ? '(conversion)' : ''
-                "#{options[:cli][:switch]} #{conv}#{values}"
-              else ''
+                conversion_tag = options[:cli].key?(:convert) ? '(conversion)' : ''
+                "#{options[:cli][:switch]} #{conversion_tag}#{values}"
+              when :special then Cli::Formatter.special('special')
+              when :ignore then Cli::Formatter.special('ignored')
+              else
+                param[:d].eql?(tick_yes) ? '' : 'n/a'
               end
             if options.key?(:enum)
               param[:description] += "\nAllowed values: #{options[:enum].join(', ')}"
@@ -103,10 +111,6 @@ module Aspera
             ts.key?('EX_file_pair_list')
         end
 
-        def ts_to_env_args(transfer_spec, wss:, ascp_args:)
-          return Parameters.new(transfer_spec, wss: wss, ascp_args: ascp_args).ascp_args
-        end
-
         # temp file list files are created here
         def file_list_folder=(v)
           @file_list_folder = v
@@ -120,21 +124,23 @@ module Aspera
       end # self
 
       # @param options [Hash] key: :wss: bool, :ascp_args: array of strings
-      def initialize(job_spec, wss:, ascp_args:)
+      def initialize(job_spec, options)
         @job_spec = job_spec
-        @opt_wss = wss
-        @opt_args = ascp_args
-        Log.log.debug{"agent options: #{@opt_wss} #{@opt_args}"}
-        raise 'ascp args must be an Array' unless @opt_args.is_a?(Array)
-        raise 'ascp args must be an Array of String' if @opt_args.any?{|i|!i.is_a?(String)}
+        # check necessary options
+        missing_options = SUPPORTED_OPTIONS - options.keys
+        raise "Internal: missing options: #{missing_options.join(', ')}" unless missing_options.empty?
+        @options = SUPPORTED_OPTIONS.each_with_object({}){|o, h| h[o] = options[o]}
+        Log.log.debug{Log.dump(:parameters_options, @options)}
+        raise 'ascp arguments must be an Array' unless @options[:ascp_args].is_a?(Array)
+        raise 'ascp arguments must be an Array of String' if @options[:ascp_args].any?{|i|!i.is_a?(String)}
         @builder = Aspera::CommandLineBuilder.new(@job_spec, self.class.description)
       end
 
       def process_file_list
         # is the file list provided through EX_ parameters?
-        ascp_file_list_provided = self.class.ts_has_ascp_file_list(@job_spec, @opt_args)
+        ascp_file_list_provided = self.class.ts_has_ascp_file_list(@job_spec, @options[:ascp_args])
         # set if paths is mandatory in ts
-        @builder.params_definition['paths'][:mandatory] = !@job_spec.key?('keepalive') && !ascp_file_list_provided
+        @builder.params_definition['paths'][:mandatory] = !@job_spec.key?('keepalive') && !ascp_file_list_provided # cspell:words keepalive
         # get paths in transfer spec (after setting if it is mandatory)
         ts_paths_array = @builder.read_param('paths')
         if ascp_file_list_provided && !ts_paths_array.nil?
@@ -154,18 +160,21 @@ module Aspera
               Log.log.debug('placing source file list on command line (no file list file)')
               @builder.add_command_line_options(ts_paths_array.map{|i|i['source']})
             else
+              raise "All elements of paths must have a 'source' key" unless ts_paths_array.all?{|i|i.key?('source')}
+              is_pair_list = ts_paths_array.any?{|i|i.key?('destination')}
+              raise "All elements of paths must be consistent with 'destination' key" if is_pair_list && !ts_paths_array.all?{|i|i.key?('destination')}
               # safer option: generate a file list file if there is storage defined for it
-              # if there is destination in paths, then use file-pair-list
-              # TODO: well, we test only the first one, but anyway it shall be consistent
-              if ts_paths_array.first.key?('destination')
+              # if there is one destination in paths, then use file-pair-list
+              if is_pair_list
                 option = '--file-pair-list'
-                lines = ts_paths_array.each_with_object([]){|e, m|m.push(e['source'], e['destination']); }
+                lines = ts_paths_array.each_with_object([]){|e, m|m.push(e['source'], e['destination'] || e['source']) }
               else
                 option = '--file-list'
                 lines = ts_paths_array.map{|i|i['source']}
               end
               file_list_file = Aspera::TempFileManager.instance.new_file_path_in_folder(self.class.file_list_folder)
-              File.write(file_list_file, lines.join("\n"))
+              Log.log.debug{Log.dump(:file_list, lines)}
+              File.write(file_list_file, lines.join("\n"), encoding: 'UTF-8')
               Log.log.debug{"#{option}=\n#{File.read(file_list_file)}".red}
             end
           end
@@ -183,18 +192,15 @@ module Aspera
           env:          {},
           ascp_version: :ascp
         }
-        # some ssh credentials are required to avoid interactive password input
-        if !@job_spec.key?('remote_password') &&
-            !@job_spec.key?('ssh_private_key') &&
-            !@job_spec.key?('EX_ssh_key_paths')
-          raise Fasp::Error, 'required: password or ssh key (value or path)'
-        end
 
         # special cases
         @job_spec.delete('source_root') if @job_spec.key?('source_root') && @job_spec['source_root'].empty?
 
-        # use web socket session initiation ?
-        if @builder.read_param('wss_enabled') && (@opt_wss || !@job_spec.key?('fasp_port'))
+        # notify multi-session was already used, anyway it was deleted by agent direct
+        raise 'internal error' if @builder.read_param('multi_session')
+
+        # use web socket secure for session ?
+        if @builder.read_param('wss_enabled') && (@options[:wss] || !@job_spec.key?('fasp_port'))
           # by default use web socket session if available, unless removed by user
           @builder.add_command_line_options(['--ws-connect'])
           # TODO: option to give order ssh,ws (legacy http is implied bu ssh)
@@ -203,12 +209,30 @@ module Aspera
           @job_spec.delete('fasp_port')
           @job_spec.delete('EX_ssh_key_paths')
           @job_spec.delete('sshfp')
-          # set location for CA bundle to be the one of Ruby, see env var SSL_CERT_FILE / SSL_CERT_DIR
-          @job_spec['EX_ssh_key_paths'] = [OpenSSL::X509::DEFAULT_CERT_FILE]
-          Log.log.debug('CA certs: EX_ssh_key_paths <- DEFAULT_CERT_FILE from openssl')
+          if @options[:check_ignore]&.call(@job_spec['remote_host'], @job_spec['wss_port'])
+            http_session = Rest.start_http_session("https://#{@job_spec['remote_host']}:#{@job_spec['wss_port']}")
+            # wss_api = Rest.new(base_url: "/v1/transfer")
+            # wss_api.read('start') rescue nil
+            wss_cert_file = TempFileManager.instance.new_file_path_global('wss_cert')
+            File.write(wss_cert_file, http_session.peer_cert.to_pem)
+            http_session.finish
+            env_args[:args].unshift('-i', wss_cert_file)
+            Log.log.debug{"CA certs for wss: remote cert: #{wss_cert_file}"}
+          else
+            # set location for CA bundle to be the one of Ruby, see env var SSL_CERT_FILE / SSL_CERT_DIR
+            @options[:trusted_certs].each do |file|
+              env_args[:args].unshift('-i', file)
+              Log.log.debug{"trusted certs for wss: #{file}"}
+            end
+          end
         else
           # remove unused parameter (avoid warning)
           @job_spec.delete('wss_port')
+          # add SSH bypass keys when authentication is token and no auth is provided
+          if @job_spec.key?('token') && !@job_spec.key?('remote_password')
+            # @job_spec['remote_password'] = Installation.instance.ssh_cert_uuid # not used: no passphrase
+            Installation.instance.aspera_token_ssh_key_paths.each { |key| env_args[:args].unshift('-i', key) }
+          end
         end
 
         # process parameters as specified in table
@@ -224,9 +248,9 @@ module Aspera
         end
         # get list of files to transfer and build arg for ascp
         process_file_list
-        # optional args, at the end to override previous ones (to allow override)
+        # optional arguments, at the end to override previous ones (to allow override)
         @builder.add_command_line_options(@builder.read_param('EX_ascp_args'))
-        @builder.add_command_line_options(@opt_args)
+        @builder.add_command_line_options(@options[:ascp_args])
         # process destination folder
         destination_folder = @builder.read_param('destination_root') || '/'
         # ascp4 does not support base64 encoding of destination
@@ -234,7 +258,17 @@ module Aspera
         # destination MUST be last command line argument to ascp
         @builder.add_command_line_options([destination_folder])
 
-        @builder.add_env_args(env_args[:env], env_args[:args])
+        Log.log.debug{"ascp args: #{env_args}"}
+
+        @builder.add_env_args(env_args)
+
+        env_args[:args].unshift('-q') if @options[:quiet]
+
+        # add fallback cert and key as arguments if needed
+        if ['1', 1, true, 'force'].include?(@job_spec['http_fallback'])
+          env_args[:args].unshift('-Y', Installation.instance.path(:fallback_private_key))
+          env_args[:args].unshift('-I', Installation.instance.path(:fallback_certificate))
+        end
 
         return env_args
       end