aspera-cli 4.14.0 → 4.15.0

data/lib/aspera/fasp/agent_base.rb

@@ -2,93 +2,47 @@
 
 module Aspera
   module Fasp
-    # Base class for FASP transfer agents
-    # sub classes shall implement start_transfer and shutdown
+    # Base class for transfer agents
     class AgentBase
-      # fields description for JSON generation
-      # spellchecker: disable
-      INTEGER_FIELDS = %w[Bytescont FaspFileArgIndex StartByte Rate MinRate Port Priority RateCap MinRateCap TCPPort CreatePolicy TimePolicy
-                          DatagramSize XoptFlags VLinkVersion PeerVLinkVersion DSPipelineDepth PeerDSPipelineDepth ReadBlockSize WriteBlockSize
-                          ClusterNumNodes ClusterNodeId Size Written Loss FileBytes PreTransferBytes TransferBytes PMTU Elapsedusec ArgScansAttempted
-                          ArgScansCompleted PathScansAttempted FileScansCompleted TransfersAttempted TransfersPassed Delay].freeze
-      BOOLEAN_FIELDS = %w[Encryption Remote RateLock MinRateLock PolicyLock FilesEncrypt FilesDecrypt VLinkLocalEnabled VLinkRemoteEnabled
-                          MoveRange Keepalive TestLogin UseProxy Precalc RTTAutocorrect].freeze
-      EXPECTED_METHODS = %i[text struct enhanced].freeze
-      private_constant :INTEGER_FIELDS, :BOOLEAN_FIELDS, :EXPECTED_METHODS
-      # spellchecker: enable
-
       class << self
-        # This checks the validity of the value returned by wait_for_transfers_completion
-        # it must be a list of :success or exception
-        def validate_status_list(statuses)
-          raise "internal error: bad statuses type: #{statuses.class}" unless statuses.is_a?(Array)
-          raise "internal error: bad statuses content: #{statuses}" unless statuses.select{|i|!i.eql?(:success) && !i.is_a?(StandardError)}.empty?
-        end
-      end
-
-      private
-
-      # translates legacy event into enhanced (JSON) event
-      def enhanced_event_format(event)
-        return event.keys.each_with_object({}) do |e, h|
-          # capital_to_snake_case
-          new_name = e
-              .gsub(/([a-z\d])([A-Z])/, '\1_\2')
-              .gsub(/([a-z\d])(usec)$/, '\1_\2')
-              .gsub(/([A-Z]+)([A-Z][a-z])/, '\1_\2')
-              .downcase
-          value = event[e]
-          value = value.to_i if INTEGER_FIELDS.include?(e)
-          value = value.eql?('Yes') if BOOLEAN_FIELDS.include?(e)
-          h[new_name] = value
-        end
-      end
-
-      def initialize
-        @listeners = []
-      end
-
-      def notify_listeners(current_event_text, current_event_data)
-        Log.log.debug('send event to listeners')
-        enhanced_event = nil
-        @listeners.each do |listener|
-          listener.event_text(current_event_text) if listener.respond_to?(:event_text)
-          listener.event_struct(current_event_data) if listener.respond_to?(:event_struct)
-          if listener.respond_to?(:event_enhanced)
-            enhanced_event = enhanced_event_format(current_event_data) if enhanced_event.nil?
-            listener.event_enhanced(enhanced_event)
+        # compute options from user provided and default options
+        def options(default:, options:)
+          result = options.symbolize_keys
+          available = default.map{|k, v|"#{k}(#{v})"}.join(', ')
+          result.each do |k, _v|
+            raise "Unknown transfer agent parameter: #{k}, expect one of #{available}" unless default.key?(k)
+          end
+          default.each do |k, v|
+            raise "Missing required agent parameter: #{k}. Parameters: #{available}" if v.eql?(:required) && !result.key?(k)
+            result[k] = v unless result.key?(k)
           end
+          return result
         end
-      end # notify_listeners
-
-      def notify_begin(id, size)
-        notify_listeners('emulated', {LISTENER_SESSION_ID_B => id, 'Type' => 'NOTIFICATION', 'PreTransferBytes' => size})
       end
-
-      def notify_progress(id, size)
-        notify_listeners('emulated', {LISTENER_SESSION_ID_B => id, 'Type' => 'STATS', 'Bytescont' => size})
+      def wait_for_completion
+        # list of: :success or "error message string"
+        statuses = wait_for_transfers_completion
+        @progress&.reset
+        raise "internal error: bad statuses type: #{statuses.class}" unless statuses.is_a?(Array)
+        raise "internal error: bad statuses content: #{statuses}" unless statuses.select{|i|!i.eql?(:success) && !i.is_a?(StandardError)}.empty?
+        return statuses
       end
 
-      def notify_end(id)
-        notify_listeners('emulated', {LISTENER_SESSION_ID_B => id, 'Type' => 'DONE'})
-      end
-
-      public
-
-      LISTENER_SESSION_ID_B = 'ListenerSessionId'
-      LISTENER_SESSION_ID_S = 'listener_session_id'
+      private
 
-      # listener receives events
-      def add_listener(listener)
-        raise "expect one of #{EXPECTED_METHODS}" if EXPECTED_METHODS.inject(0){|m, e|m + (listener.respond_to?("event_#{e}") ? 1 : 0)}.eql?(0)
-        @listeners.push(listener)
-        self
+      def initialize(options)
+        raise 'internal error' unless respond_to?(:start_transfer)
+        raise 'internal error' unless respond_to?(:wait_for_transfers_completion)
+        # method `shutdown` is optional
+        Log.log.debug{Log.dump(:agent_options, options)}
+        raise "transfer agent options expecting Hash, but have #{options.class}" unless options.is_a?(Hash)
+        @progress = options[:progress]
+        options.delete(:progress)
       end
 
-      # the following methods must be implemented by subclass:
-      # start_transfer(transfer_spec, token_regenerator: nil) : start transfer
-      # wait_for_transfers_completion : wait for termination of all transfers, @return list of : :success or error message
-      # optional: shutdown
+      def notify_progress(**parameters)
+        @progress&.event(**parameters)
+      end
     end
   end
 end

data/lib/aspera/fasp/agent_connect.rb

@@ -4,7 +4,6 @@ require 'aspera/fasp/agent_base'
 require 'aspera/rest'
 require 'aspera/open_application'
 require 'securerandom'
-require 'tty-spinner'
 
 module Aspera
   module Fasp
@@ -14,20 +13,20 @@ module Aspera
       # delay between each try to start connect
       SLEEP_SEC_BETWEEN_RETRY = 3
       private_constant :CONNECT_START_URIS, :SLEEP_SEC_BETWEEN_RETRY
-      def initialize(_options)
-        super()
+      def initialize(options)
+        super(options)
         @connect_settings = {
           'app_id' => SecureRandom.uuid
         }
         raise 'Using connect requires a graphical environment' if !OpenApplication.default_gui_mode.eql?(:graphical)
         method_index = 0
         begin
-          connect_url = Installation.instance.connect_uri
+          connect_url = Products.connect_uri
           Log.log.debug{"found: #{connect_url}"}
           @connect_api = Rest.new({base_url: "#{connect_url}/v5/connect", headers: {'Origin' => Rest.user_agent}}) # could use v6 also now
           connect_info = @connect_api.read('info/version')[:data]
           Log.log.info('Connect was reached') if method_index > 0
-          Log.dump(:connect_version, connect_info)
+          Log.log.debug{Log.dump(:connect_version, connect_info)}
         rescue StandardError => e # Errno::ECONNREFUSED
           start_url = CONNECT_START_URIS[method_index]
           method_index += 1
@@ -74,10 +73,12 @@ module Aspera
       def wait_for_transfers_completion
         connect_activity_args = {'aspera_connect_settings' => @connect_settings}
         started = false
-        spinner = nil
+        pre_calc = false
+        session_id = @xfer_id
         begin
           loop do
             tr_info = @connect_api.create("transfers/info/#{@xfer_id}", connect_activity_args)[:data]
+            Log.log.trace1{Log.dump(:tr_info, tr_info)}
             if tr_info['transfer_info'].is_a?(Hash)
               transfer = tr_info['transfer_info']
               if transfer.nil?
@@ -86,32 +87,30 @@ module Aspera
               end
               # TODO: get session id
               case transfer['status']
-              when 'completed'
-                notify_end(@connect_settings['app_id'])
-                break
               when 'initiating', 'queued'
-                if spinner.nil?
-                  spinner = TTY::Spinner.new('[:spinner] :title', format: :classic)
-                  spinner.start
-                end
-                spinner.update(title: transfer['status'])
-                spinner.spin
+                notify_progress(session_id: nil, type: :pre_start, info: transfer['status'])
               when 'running'
-                # puts "running: sessions:#{transfer['sessions'].length}, #{transfer['sessions'].map{|i| i['bytes_transferred']}.join(',')}"
-                if !started && (transfer['bytes_expected'] != 0)
-                  spinner&.success
-                  notify_begin(@connect_settings['app_id'], transfer['bytes_expected'])
+                if !started
+                  notify_progress(session_id: session_id, type: :session_start)
                   started = true
+                end
+                if !pre_calc && (transfer['bytes_expected'] != 0)
+                  notify_progress(type: :session_size, session_id: session_id, info: transfer['bytes_expected'])
+                  pre_calc = true
                 else
-                  notify_progress(@connect_settings['app_id'], transfer['bytes_written'])
+                  notify_progress(type: :transfer, session_id: session_id, info: transfer['bytes_written'])
                 end
+              when 'completed'
+                notify_progress(type: :end, session_id: session_id)
+                break
               when 'failed'
-                spinner&.error
+                notify_progress(type: :end, session_id: session_id)
                 raise Fasp::Error, transfer['error_desc']
               when 'cancelled'
-                spinner&.error
+                notify_progress(type: :end, session_id: session_id)
                 raise Fasp::Error, 'Transfer cancelled by user'
               else
+                notify_progress(type: :end, session_id: session_id)
                 raise Fasp::Error, "unknown status: #{transfer['status']}: #{transfer['error_desc']}"
               end
             end

data/lib/aspera/fasp/agent_direct.rb

@@ -1,17 +1,18 @@
 # frozen_string_literal: true
 
-require 'English'
 require 'aspera/fasp/agent_base'
 require 'aspera/fasp/error'
 require 'aspera/fasp/parameters'
 require 'aspera/fasp/installation'
 require 'aspera/fasp/resume_policy'
 require 'aspera/fasp/transfer_spec'
+require 'aspera/fasp/management'
 require 'aspera/log'
 require 'socket'
 require 'timeout'
 require 'securerandom'
 require 'shellwords'
+require 'English'
 
 module Aspera
   module Fasp
@@ -25,8 +26,11 @@ module Aspera
         multi_incr_udp:    true,
         resume:            {},
         ascp_args:         [],
-        quiet:             true # by default no native ascp progress bar
+        check_ignore:      nil, # callback with host,port
+        quiet:             true, # by default no native ascp progress bar
+        trusted_certs:     [] # list of files with trusted certificates (stores)
       }.freeze
+      # spellchecker: enable
       private_constant :DEFAULT_OPTIONS
 
       # start ascp transfer (non blocking), single or multi-session
@@ -47,15 +51,7 @@ module Aspera
           # TODO: useful ? node only ?
           transfer_spec['tags'][Fasp::TransferSpec::TAG_RESERVED]['xfer_retry'] ||= 3600
         end
-        Log.dump('ts', transfer_spec)
-
-        # add bypass keys when authentication is token and no auth is provided
-        if transfer_spec.key?('token') &&
-            !transfer_spec.key?('remote_password') &&
-            !transfer_spec.key?('EX_ssh_key_paths')
-          # transfer_spec['remote_password'] = Installation.instance.bypass_pass # not used: no passphrase
-          transfer_spec['EX_ssh_key_paths'] = Installation.instance.bypass_keys
-        end
+        Log.log.debug{Log.dump('ts', transfer_spec)}
 
         # Compute this before using transfer spec because it potentially modifies the transfer spec
         # (even if the var is not used in single session)
@@ -81,16 +77,8 @@ module Aspera
           end
         end
 
-        # compute known args
-        env_args =  Parameters.new(transfer_spec, @options).ascp_args
-
-        # add fallback cert and key as arguments if needed
-        if ['1', 1, true, 'force'].include?(transfer_spec['http_fallback'])
-          env_args[:args].unshift('-Y', Installation.instance.path(:fallback_cert_privkey))
-          env_args[:args].unshift('-I', Installation.instance.path(:fallback_certificate))
-        end
-
-        env_args[:args].unshift('-q') if @options[:quiet]
+        # compute known arguments and environment variables
+        env_args = Parameters.new(transfer_spec, @options).ascp_args
 
         # transfer job can be multi session
         xfer_job = {
@@ -162,6 +150,49 @@ module Aspera
         Log.log.debug('fasp local shutdown')
       end
 
+      # cspell:disable
+      # begin 'Type' => 'NOTIFICATION', 'PreTransferBytes' => size
+      # progress 'Type' => 'STATS', 'Bytescont' => size
+      # end 'Type' => 'DONE'
+      # cspell:enable
+
+      # @param event management port event
+      def process_progress(event)
+        session_id = event['SessionId']
+        case event['Type']
+        when 'INIT'
+          @pre_calc_sent = false
+          @pre_calc_last_size = nil
+          notify_progress(session_id: session_id, type: :session_start)
+        when 'NOTIFICATION' # sent from remote
+          if event.key?('PreTransferBytes')
+            @pre_calc_sent = true
+            notify_progress(session_id: session_id, type: :session_size, info: event['PreTransferBytes'])
+          end
+        when 'STATS' # during transfer
+          @pre_calc_last_size = event['TransferBytes'].to_i + event['StartByte'].to_i
+          notify_progress(session_id: session_id, type: :transfer, info: @pre_calc_last_size)
+        when 'DONE', 'ERROR' # end of session
+          total_size = event['TransferBytes'].to_i + event['StartByte'].to_i
+          if !@pre_calc_sent && !total_size.zero?
+            notify_progress(session_id: session_id, type: :session_size, info: total_size)
+          end
+          if @pre_calc_last_size != total_size
+            notify_progress(session_id: session_id, type: :transfer, info: total_size)
+          end
+          notify_progress(session_id: session_id, type: :end)
+          # cspell:disable
+        when 'SESSION'
+        when 'ARGSTOP'
+        when 'FILEERROR'
+        when 'STOP'
+          # cspell:enable
+          # stop event when one file is completed
+        else
+          Log.log.debug{"unknown event type #{event['Type']}"}
+        end
+      end
+
       # This is the low level method to start the "ascp" process
       # currently, relies on command line arguments
       # start ascp with management port.
@@ -173,8 +204,6 @@ module Aspera
       def start_transfer_with_args_env(env_args, session)
         raise 'env_args must be Hash' unless env_args.is_a?(Hash)
         raise 'session must be Hash' unless session.is_a?(Hash)
-        # by default we assume an exception will be raised (for ensure block)
-        exception_raised = true
         begin
           Log.log.debug{"env_args=#{env_args.inspect}"}
           # get location of ascp executable
@@ -183,13 +212,14 @@ module Aspera
           end
           # (optional) check it exists
           raise Fasp::Error, "no such file: #{ascp_path}" unless File.exist?(ascp_path)
+          notify_progress(session_id: nil, type: :pre_start, info: 'starting ascp')
           # open an available (0) local TCP port as ascp management
           mgt_sock = TCPServer.new('127.0.0.1', 0)
           # clone arguments as we eed to modify with mgt port
           ascp_arguments = env_args[:args].clone
           # add management port on the selected local port
           ascp_arguments.unshift('-M', mgt_sock.addr[1].to_s)
-          # start ascp in sub process
+          # display ascp command line
           Log.log.debug do
             [
               'execute:',
@@ -198,12 +228,13 @@ module Aspera
               ascp_arguments.map{|a|Shellwords.shellescape(a)}
             ].flatten.join(' ')
           end
-          # start process
+          # start ascp in separate process
           ascp_pid = Process.spawn(env_args[:env], [ascp_path, ascp_path], *ascp_arguments)
           # in parent, wait for connection to socket max 3 seconds
           Log.log.debug{"before accept for pid (#{ascp_pid})"}
           # init management socket
           ascp_mgt_io = nil
+          notify_progress(session_id: nil, type: :pre_start, info: 'waiting for ascp')
           Timeout.timeout(@options[:spawn_timeout_sec]) do
             ascp_mgt_io = mgt_sock.accept
             # management messages include file names which may be utf8
@@ -213,73 +244,35 @@ module Aspera
           end
           Log.log.debug{"after accept (#{ascp_mgt_io})"}
           session[:io] = ascp_mgt_io
-          # exact text for event, with \n
-          current_event_text = ''
-          # parsed event (hash)
-          current_event_data = nil
-          # this is the last full status
-          last_status_event = nil
-          # read management port
-          loop do
-            # TODO: timeout here ?
-            line = ascp_mgt_io.gets
-            # nil when ascp process exits
-            break if line.nil?
-            current_event_text += line
-            line.chomp!
-            Log.log.debug{"line=[#{line}]"}
-            case line
-            when 'FASPMGR 2'
-              # begin event
-              current_event_data = {}
-              current_event_text = ''
-            when /^([^:]+): (.*)$/
-              # event field
-              current_event_data[Regexp.last_match(1)] = Regexp.last_match(2)
-            when ''
-              # empty line is separator to end event information
-              raise 'unexpected empty line' if current_event_data.nil?
-              current_event_data[AgentBase::LISTENER_SESSION_ID_B] = ascp_pid
-              notify_listeners(current_event_text, current_event_data)
-              case current_event_data['Type']
-              when 'INIT'
-                session[:id] = current_event_data['SessionId']
-                Log.log.debug{"session id: #{session[:id]}"}
-              when 'DONE', 'ERROR'
-                # TODO: check if this is always the last event
-                last_status_event = current_event_data
-              end # event type
-            else
-              raise "unexpected line:[#{line}]"
-            end # case
-          end # loop (process mgt port lines)
+          processor = Management.new
+          # read management port, until socket is closed (gets returns nil)
+          while (line = ascp_mgt_io.gets)
+            event = processor.process_line(line.chomp)
+            next unless event
+            # event is ready
+            Log.log.debug{Log.dump(:management_port, event)}
+            # Log.log.trace1{"event: #{JSON.generate(Management.enhanced_event_format(event))}"}
+            process_progress(event)
+            Log.log.error((event['Description']).to_s) if event['Type'].eql?('FILEERROR') # cspell:disable-line
+          end
+          last_event = processor.last_event
           # check that last status was received before process exit
-          if last_status_event.is_a?(Hash)
-            case last_status_event['Type']
-            when 'DONE'
-              # all went well
-              exception_raised = false
+          if last_event.is_a?(Hash)
+            case last_event['Type']
             when 'ERROR'
-              Log.log.error{"code: #{last_status_event['Code']}"}
-              if /bearer token/i.match?(last_status_event['Description'])
-                Log.log.error('need to regenerate token'.red)
-                if session[:token_regenerator].respond_to?(:refreshed_transfer_token)
-                  # regenerate token here, expired, or error on it
-                  # Note: in multi-session, each session will have a different one.
-                  env_args[:env]['ASPERA_SCP_TOKEN'] = session[:token_regenerator].refreshed_transfer_token
-                end
+              if /bearer token/i.match?(last_event['Description']) &&
+                  session[:token_regenerator].respond_to?(:refreshed_transfer_token)
+                # regenerate token here, expired, or error on it
+                # Note: in multi-session, each session will have a different one.
+                Log.log.warn('Regenerating bearer token')
+                env_args[:env]['ASPERA_SCP_TOKEN'] = session[:token_regenerator].refreshed_transfer_token
               end
-              # cannot resolve address
-              # if last_status_event['Code'].to_i.eql?(14)
-              #  Log.log.warn{"host: #{}"}
-              # end
-              raise Fasp::Error.new(last_status_event['Description'], last_status_event['Code'].to_i)
-            else # case
-              raise "unexpected last event type: #{last_status_event['Type']}"
-            end
-          else
-            exception_raised = false
-            Log.log.debug('no status read from ascp mgt port')
+              raise Fasp::Error.new(last_event['Description'], last_event['Code'].to_i)
+            when 'DONE'
+              nil
+            else
+              raise "unexpected last event type: #{last_event['Type']}"
+            end # case
           end
         rescue SystemCallError => e
           # Process.spawn
@@ -299,7 +292,7 @@ module Aspera
             if !status.success?
               message = "ascp failed with code #{status.exitstatus}"
               # raise error only if there was not already an exception
-              raise Fasp::Error, message unless exception_raised
+              raise Fasp::Error, message unless $ERROR_INFO
               # else just debug, as main exception is already here
               Log.log.debug(message)
             end
@@ -323,7 +316,7 @@ module Aspera
         command = data
           .keys
           .map{|k|"#{k.capitalize}: #{data[k]}"}
-          .unshift('FASPMGR 2')
+          .unshift(MGT_HEADER)
           .push('', '')
           .join("\n")
         session[:io].puts(command)
@@ -332,23 +325,16 @@ module Aspera
       private
 
       # @param options : keys(symbol): see DEFAULT_OPTIONS
-      def initialize(options=nil)
-        super()
+      def initialize(options={})
+        super(options)
         # all transfer jobs, key = SecureRandom.uuid, protected by mutex, cond var on change
         @jobs = {}
         # mutex protects global data accessed by threads
         @mutex = Mutex.new
         # set default options and override if specified
-        @options = DEFAULT_OPTIONS.dup
-        if !options.nil?
-          raise "expecting Hash (or nil), but have #{options.class}" unless options.is_a?(Hash)
-          options.each do |k, v|
-            raise "Unknown local agent parameter: #{k}, expect one of #{DEFAULT_OPTIONS.keys.map(&:to_s).join(',')}" unless DEFAULT_OPTIONS.key?(k)
-            @options[k] = v
-          end
-        end
-        Log.log.debug{"local options= #{options}"}
+        @options = AgentBase.options(default: DEFAULT_OPTIONS, options: options)
         @resume_policy = ResumePolicy.new(@options[:resume].symbolize_keys)
+        Log.log.debug{Log.dump(:agent_options, @options)}
       end
 
       # transfer thread entry