aspera-cli 4.15.0 → 4.16.0

data/lib/aspera/fasp/installation.rb

@@ -5,6 +5,7 @@ require 'aspera/environment'
 require 'aspera/data_repository'
 require 'aspera/fasp/products'
 require 'aspera/log'
+require 'aspera/assert'
 require 'aspera/web_server_simple'
 require 'English'
 require 'singleton'
@@ -107,6 +108,7 @@ module Aspera
       # get path of one resource file of currently activated product
       # keys and certs are generated locally... (they are well known values, arch. independent)
       def path(k)
+        file_is_optional = false
         case k
         when :ascp, :ascp4
           use_ascp_from_product(FIRST_FOUND) if @path_to_ascp.nil?
@@ -115,17 +117,13 @@ module Aspera
           file = file.gsub('ascp', 'ascp4') if k.eql?(:ascp4)
         when :transferd
           file = transferd_filepath
+          file_is_optional = true
         when :ssh_private_dsa, :ssh_private_rsa
           # assume last 3 letters are type
-          type = k.to_s[-3..-1]
-          file = check_or_create_sdk_file("aspera_bypass_#{type}.pem") do
-            # generate PEM from DER
-            OpenSSL::PKey.const_get(type.upcase).new(DataRepository.instance.data(type.eql?('dsa') ? 1 : 2)).to_pem
-          end
+          type = k.to_s[-3..-1].to_sym
+          file = check_or_create_sdk_file("aspera_bypass_#{type}.pem") {DataRepository.instance.item(type)}
         when :aspera_license
-          file = check_or_create_sdk_file('aspera-license') do
-            Zlib::Inflate.inflate(DataRepository.instance.data(6))
-          end
+          file = check_or_create_sdk_file('aspera-license') {DataRepository.instance.item(:license)}
         when :aspera_conf
           file = check_or_create_sdk_file('aspera.conf') {DEFAULT_ASPERA_CONF}
         when :fallback_certificate, :fallback_private_key
@@ -141,16 +139,16 @@ module Aspera
             check_or_create_sdk_file('aspera_fallback_cert.pem', force: true) {cert.to_pem}
           end
           file = k.eql?(:fallback_certificate) ? file_cert : file_key
-        else
-          raise "INTERNAL ERROR: #{k}"
+        else error_unexpected_value(k)
         end
-        raise "no such file: #{file}" unless File.exist?(file)
+        return nil if file_is_optional && !File.exist?(file)
+        assert(File.exist?(file)){"no such file: #{file}"}
         return file
       end
 
       # default bypass key phrase
       def ssh_cert_uuid
-        return format('%08x-%04x-%04x-%04x-%04x%08x', *DataRepository.instance.data(3).unpack('NnnnnN'))
+        return DataRepository.instance.item(:uuid)
       end
 
       def aspera_token_ssh_key_paths
@@ -175,6 +173,47 @@ module Aspera
         return exe_version
       end
 
+      def ascp_info
+        data = file_paths
+        # read PATHs from ascp directly, and pvcl modules as well
+        Open3.popen3(data['ascp'], '-DDL-') do |_stdin, _stdout, stderr, thread|
+          last_line = ''
+          while (line = stderr.gets)
+            line.chomp!
+            last_line = line
+            case line
+            when /^DBG Path ([^ ]+) (dir|file) +: (.*)$/
+              data[Regexp.last_match(1)] = Regexp.last_match(3)
+            when /^DBG Added module group:"(?<module>[^"]+)" name:"(?<scheme>[^"]+)", version:"(?<version>[^"]+)" interface:"(?<interface>[^"]+)"$/
+              c = Regexp.last_match.named_captures.symbolize_keys
+              data[c[:interface]] ||= {}
+              data[c[:interface]][c[:module]] ||= []
+              data[c[:interface]][c[:module]].push("#{c[:scheme]} v#{c[:version]}")
+            when %r{^DBG License result \(/license/(\S+)\): (.+)$}
+              data[Regexp.last_match(1)] = Regexp.last_match(2)
+            when /^LOG (.+) version ([0-9.]+)$/
+              data['product_name'] = Regexp.last_match(1)
+              data['product_version'] = Regexp.last_match(2)
+            when /^LOG Initializing FASP version ([^,]+),/
+              data['ascp_version'] = Regexp.last_match(1)
+            end
+          end
+          if !thread.value.exitstatus.eql?(1) && !data.key?('root')
+            raise last_line
+          end
+        end
+        # ascp's openssl directory
+        ascp_file = data['ascp']
+        File.binread(ascp_file).scan(/[\x20-\x7E]{4,}/) do |match|
+          if (m = match.match(/OPENSSLDIR.*"(.*)"/))
+            data['openssldir'] = m[1]
+          end
+        end if File.file?(ascp_file)
+        # log is "-" no need to display
+        data.delete('log')
+        return data
+      end
+
       # download aspera SDK or use local file
       # extracts ascp binary for current system architecture
       # @return ascp version (from execution)

data/lib/aspera/fasp/management.rb

@@ -185,6 +185,8 @@ module Aspera
         ExtraCreatePolicy]
       # Management port start message
       MGT_HEADER = 'FASPMGR 2'
+      # empty line is separator to end event information
+      MGT_FRAME_SEPARATOR = ''
       # fields description for JSON generation
       # spellchecker: disable
       INTEGER_FIELDS = %w[Bytescont FaspFileArgIndex StartByte Rate MinRate Port Priority RateCap MinRateCap TCPPort CreatePolicy TimePolicy
@@ -193,10 +195,11 @@ module Aspera
                           ArgScansCompleted PathScansAttempted FileScansCompleted TransfersAttempted TransfersPassed Delay].freeze
       BOOLEAN_FIELDS = %w[Encryption Remote RateLock MinRateLock PolicyLock FilesEncrypt FilesDecrypt VLinkLocalEnabled VLinkRemoteEnabled
                           MoveRange Keepalive TestLogin UseProxy Precalc RTTAutocorrect].freeze
+      BOOLEAN_TRUE = 'Yes'
       # cspell: enable
 
       class << self
-        # translates legacy event into enhanced (JSON) event
+        # translates mgt port event into (enhanced) typed event
         def enhanced_event_format(event)
           return event.keys.each_with_object({}) do |e, h|
             # capital_to_snake_case
@@ -207,14 +210,16 @@ module Aspera
                 .downcase
             value = event[e]
             value = value.to_i if INTEGER_FIELDS.include?(e)
-            value = value.eql?('Yes') if BOOLEAN_FIELDS.include?(e)
+            value = value.eql?(BOOLEAN_TRUE) if BOOLEAN_FIELDS.include?(e)
             h[new_name] = value
           end
         end
       end # class << self
 
       def initialize
+        # current event being parsed line by line
         @event_build = nil
+        # last fully built event
         @last_event = nil
       end
       attr_reader :last_event
@@ -226,16 +231,16 @@ module Aspera
           # begin event
           @event_build = {}
         when /^([^:]+): (.*)$/
+          raise 'mgt port: unexpected line: data without header' if @event_build.nil?
           # event field
           @event_build[Regexp.last_match(1)] = Regexp.last_match(2)
-        when ''
-          # empty line is separator to end event information
-          raise 'unexpected empty line' if @event_build.nil?
+        when MGT_FRAME_SEPARATOR
+          raise 'mgt port: unexpected line: end frame without header' if @event_build.nil?
           @last_event = @event_build
           @event_build = nil
           return @last_event
         else
-          raise "unexpected line:[#{line}]"
+          raise "mgt port: unexpected line: [#{line}]"
         end # case
         return nil
       end

data/lib/aspera/fasp/parameters.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require 'aspera/log'
+require 'aspera/assert'
 require 'aspera/command_line_builder'
 require 'aspera/temp_file_manager'
 require 'aspera/fasp/error'
@@ -23,9 +24,10 @@ module Aspera
       # Short names of columns in manual
       SUPPORTED_AGENTS_SHORT = SUPPORTED_AGENTS.map{|a|a.to_s[0].to_sym}
       FILE_LIST_OPTIONS = ['--file-list', '--file-pair-list'].freeze
-      SUPPORTED_OPTIONS = %i[ascp_args wss check_ignore quiet].freeze
+      # options that can be provided to the constructor, and then in @options
+      SUPPORTED_OPTIONS = %i[ascp_args wss check_ignore quiet trusted_certs].freeze
 
-      private_constant :SUPPORTED_AGENTS, :FILE_LIST_OPTIONS
+      private_constant :SUPPORTED_AGENTS, :FILE_LIST_OPTIONS, :SUPPORTED_OPTIONS
 
       class << self
         # Temp folder for file lists, must contain only file lists
@@ -125,17 +127,21 @@ module Aspera
 
       # @param options [Hash] key: :wss: bool, :ascp_args: array of strings
       def initialize(job_spec, options)
+        assert_type(job_spec, Hash)
+        assert_type(options, Hash)
         @job_spec = job_spec
         # check necessary options
         missing_options = SUPPORTED_OPTIONS - options.keys
-        raise "Internal: missing options: #{missing_options.join(', ')}" unless missing_options.empty?
+        assert(missing_options.empty?){"missing options: #{missing_options.join(', ')}"}
         @options = SUPPORTED_OPTIONS.each_with_object({}){|o, h| h[o] = options[o]}
         Log.log.debug{Log.dump(:parameters_options, @options)}
-        raise 'ascp arguments must be an Array' unless @options[:ascp_args].is_a?(Array)
-        raise 'ascp arguments must be an Array of String' if @options[:ascp_args].any?{|i|!i.is_a?(String)}
+        Log.log.debug{Log.dump(:dismiss_options, options.keys - SUPPORTED_OPTIONS)}
+        assert_type(@options[:ascp_args], Array){'ascp_args'}
+        assert(@options[:ascp_args].all?(String)){'ascp arguments must Strings'}
         @builder = Aspera::CommandLineBuilder.new(@job_spec, self.class.description)
       end
 
+      # either place source files on command line, or add file list file
       def process_file_list
         # is the file list provided through EX_ parameters?
         ascp_file_list_provided = self.class.ts_has_ascp_file_list(@job_spec, @options[:ascp_args])
@@ -160,7 +166,7 @@ module Aspera
               Log.log.debug('placing source file list on command line (no file list file)')
               @builder.add_command_line_options(ts_paths_array.map{|i|i['source']})
             else
-              raise "All elements of paths must have a 'source' key" unless ts_paths_array.all?{|i|i.key?('source')}
+              assert(ts_paths_array.all?{|i|i.key?('source')}){"All elements of paths must have a 'source' key"}
               is_pair_list = ts_paths_array.any?{|i|i.key?('destination')}
               raise "All elements of paths must be consistent with 'destination' key" if is_pair_list && !ts_paths_array.all?{|i|i.key?('destination')}
               # safer option: generate a file list file if there is storage defined for it
@@ -184,92 +190,92 @@ module Aspera
         @builder.add_command_line_options(["#{option}=#{file_list_file}"]) unless option.nil?
       end
 
-      # translate transfer spec to env vars and command line arguments for ascp
-      # NOTE: parameters starting with "EX_" (extended) are not standard
-      def ascp_args
-        env_args = {
-          args:         [],
-          env:          {},
-          ascp_version: :ascp
-        }
-
-        # special cases
-        @job_spec.delete('source_root') if @job_spec.key?('source_root') && @job_spec['source_root'].empty?
-
-        # notify multi-session was already used, anyway it was deleted by agent direct
-        raise 'internal error' if @builder.read_param('multi_session')
-
+      def remote_certificates
+        certificates_to_use = []
         # use web socket secure for session ?
         if @builder.read_param('wss_enabled') && (@options[:wss] || !@job_spec.key?('fasp_port'))
           # by default use web socket session if available, unless removed by user
           @builder.add_command_line_options(['--ws-connect'])
-          # TODO: option to give order ssh,ws (legacy http is implied bu ssh)
+          # TODO: option to give order ssh,ws (legacy http is implied by ssh)
           # This will need to be cleaned up in aspera core
           @job_spec['ssh_port'] = @builder.read_param('wss_port')
           @job_spec.delete('fasp_port')
           @job_spec.delete('EX_ssh_key_paths')
           @job_spec.delete('sshfp')
+          # ignore cert for wss ?
           if @options[:check_ignore]&.call(@job_spec['remote_host'], @job_spec['wss_port'])
-            http_session = Rest.start_http_session("https://#{@job_spec['remote_host']}:#{@job_spec['wss_port']}")
-            # wss_api = Rest.new(base_url: "/v1/transfer")
-            # wss_api.read('start') rescue nil
             wss_cert_file = TempFileManager.instance.new_file_path_global('wss_cert')
-            File.write(wss_cert_file, http_session.peer_cert.to_pem)
-            http_session.finish
-            env_args[:args].unshift('-i', wss_cert_file)
-            Log.log.debug{"CA certs for wss: remote cert: #{wss_cert_file}"}
-          else
-            # set location for CA bundle to be the one of Ruby, see env var SSL_CERT_FILE / SSL_CERT_DIR
-            @options[:trusted_certs].each do |file|
-              env_args[:args].unshift('-i', file)
-              Log.log.debug{"trusted certs for wss: #{file}"}
-            end
+            wss_url = "https://#{@job_spec['remote_host']}:#{@job_spec['wss_port']}"
+            File.write(wss_cert_file, Rest.remote_certificates(wss_url))
+            certificates_to_use.push(wss_cert_file)
           end
+          # set location for CA bundle to be the one of Ruby, see env var SSL_CERT_FILE / SSL_CERT_DIR
+          certificates_to_use.concat(@options[:trusted_certs]) if @options[:trusted_certs]
         else
           # remove unused parameter (avoid warning)
           @job_spec.delete('wss_port')
           # add SSH bypass keys when authentication is token and no auth is provided
           if @job_spec.key?('token') && !@job_spec.key?('remote_password')
             # @job_spec['remote_password'] = Installation.instance.ssh_cert_uuid # not used: no passphrase
-            Installation.instance.aspera_token_ssh_key_paths.each { |key| env_args[:args].unshift('-i', key) }
+            certificates_to_use.concat(Installation.instance.aspera_token_ssh_key_paths)
           end
         end
+        return certificates_to_use
+      end
+
+      # translate transfer spec to env vars and command line arguments for ascp
+      # NOTE: parameters starting with "EX_" (extended) are not standard
+      def ascp_args
+        env_args = {
+          args:         [],
+          env:          {},
+          ascp_version: :ascp
+        }
+
+        # special cases
+        @job_spec.delete('source_root') if @job_spec.key?('source_root') && @job_spec['source_root'].empty?
+
+        # notify multi-session was already used, anyway it was deleted by agent direct
+        assert(!@builder.read_param('multi_session'))
+
+        # add ssh or wss certificates
+        remote_certificates.each do |cert|
+          Log.log.trace1{"adding certificate: #{cert}"}
+          env_args[:args].unshift('-i', cert)
+        end
 
         # process parameters as specified in table
         @builder.process_params
 
+        base64_destination = false
         # symbol must be index of Installation.paths
         if @builder.read_param('use_ascp4')
           env_args[:ascp_version] = :ascp4
         else
           env_args[:ascp_version] = :ascp
-          # destination will be base64 encoded, put before path arguments
-          @builder.add_command_line_options(['--dest64'])
+          base64_destination = true
         end
-        # get list of files to transfer and build arg for ascp
-        process_file_list
+        # destination will be base64 encoded, put this before source path arguments
+        @builder.add_command_line_options(['--dest64']) if base64_destination
         # optional arguments, at the end to override previous ones (to allow override)
         @builder.add_command_line_options(@builder.read_param('EX_ascp_args'))
         @builder.add_command_line_options(@options[:ascp_args])
+        # get list of source files to transfer and build arg for ascp
+        process_file_list
         # process destination folder
         destination_folder = @builder.read_param('destination_root') || '/'
         # ascp4 does not support base64 encoding of destination
-        destination_folder = Base64.strict_encode64(destination_folder) unless env_args[:ascp_version].eql?(:ascp4)
+        destination_folder = Base64.strict_encode64(destination_folder) if base64_destination
         # destination MUST be last command line argument to ascp
         @builder.add_command_line_options([destination_folder])
-
-        Log.log.debug{"ascp args: #{env_args}"}
-
         @builder.add_env_args(env_args)
-
         env_args[:args].unshift('-q') if @options[:quiet]
-
         # add fallback cert and key as arguments if needed
         if ['1', 1, true, 'force'].include?(@job_spec['http_fallback'])
           env_args[:args].unshift('-Y', Installation.instance.path(:fallback_private_key))
           env_args[:args].unshift('-I', Installation.instance.path(:fallback_certificate))
         end
-
+        Log.log.debug{"ascp args: #{env_args}"}
         return env_args
       end
     end # Parameters

data/lib/aspera/fasp/resume_policy.rb

@@ -2,6 +2,7 @@
 
 require 'singleton'
 require 'aspera/log'
+require 'aspera/assert'
 
 module Aspera
   module Fasp
@@ -19,10 +20,10 @@ module Aspera
       def initialize(params=nil)
         @parameters = DEFAULTS.dup
         if !params.nil?
-          raise "expecting Hash (or nil), but have #{params.class}" unless params.is_a?(Hash)
+          assert_type(params, Hash)
           params.each do |k, v|
-            raise "unknown resume parameter: #{k}, expect one of #{DEFAULTS.keys.map(&:to_s).join(',')}" unless DEFAULTS.key?(k)
-            raise "#{k} must be Integer" unless v.is_a?(Integer)
+            assert_values(k, DEFAULTS.keys){'resume parameter'}
+            assert_type(v, Integer){k}
             @parameters[k] = v
           end
         end
@@ -32,7 +33,7 @@ module Aspera
       # calls block a number of times (resumes) until success or limit reached
       # this is re-entrant, one resumer can handle multiple transfers in //
       def execute_with_resume
-        raise 'block mandatory' unless block_given?
+        assert(block_given?)
         # maximum of retry
         remaining_resumes = @parameters[:iter_max]
         sleep_seconds = @parameters[:sleep_initial]
@@ -43,9 +44,10 @@ module Aspera
           begin
             # call provided block
             yield
+            # exit retry loop if success
             break
           rescue Fasp::Error => e
-            Log.log.warn{"An error occurred: #{e.message}"}
+            Log.log.warn{"An error occurred during transfer: #{e.message}"}
             # failure in ascp
             if e.retryable?
               # exit if we exceed the max number of retry