ruby_llm 2.1.1 → 2.2.0

data/lib/ruby_llm/transport/connection.rb

@@ -6,6 +6,7 @@ require 'faraday/retry'
 require 'ruby_llm/transport/error_middleware'
 require 'ruby_llm/transport/signing_middleware'
 require 'ruby_llm/transport/usage_middleware'
+require 'time'
 require 'timeout'
 
 module RubyLLM
@@ -15,23 +16,37 @@ module RubyLLM
 
       IDEMPOTENT_KEY = :ruby_llm_idempotent
       STREAM_PROGRESS_KEY = :ruby_llm_stream_progress
+      FALLBACK_KEY = :ruby_llm_fallback_ready
+
+      # The longest wait, in seconds, a rate-limited or overloaded model's
+      # retry hint may ask for before a fallback takes the request instead.
+      # Providers often ask for a second or two, which costs less than
+      # moving to another model; waits of ten seconds or more are left to
+      # a fallback when one is configured.
+      FALLBACK_WAIT = 2
 
       # The key a shared Faraday connection is cached under: the provider,
       # whether it streams, and everything ::build reads.
       Settings = Struct.new(
-        :provider, :stream, :api_base, :adapter, :timeout, :proxy, :logger, :log_bodies, :log_regexp_timeout,
-        :max_retries, :retry_interval, :retry_max_interval, :retry_interval_randomness, :retry_backoff_factor,
+        :provider, :stream, :api_base, :adapter, :timeout, :connect_timeout, :idle_timeout, :proxy, :ssl, :logger,
+        :log_bodies, :log_regexp_timeout, :max_retries, :retry_interval, :retry_max_interval,
+        :retry_interval_randomness, :retry_backoff_factor,
         keyword_init: true
       )
 
       CACHE = Support::ProcessCache.new
+      REDIRECTS = [301, 302, 303, 307, 308].freeze
+      REDIRECT_LIMIT = 5
 
       attr_reader :provider, :config
 
       def self.basic(config = RubyLLM.config, &)
         Faraday.new do |f|
+          f.use ReleaseRequest
           f.options.timeout = config.request_timeout
+          f.options.open_timeout = config.connect_timeout
           f.proxy = config.http_proxy if config.http_proxy
+          f.ssl.update(ssl_snapshot(config.ssl)) if config.ssl
           f.response :logger,
                      RubyLLM.logger,
                      bodies: false,
@@ -43,10 +58,56 @@ module RubyLLM
         end
       end
 
+      # Fetches the body at +url+ with no credentials, so a redirect to
+      # another host carries none.
+      def self.download(url, config = RubyLLM.config)
+        connection = basic(config)
+        uri = URI(url.to_s)
+        response = connection.get(uri)
+        REDIRECT_LIMIT.times do
+          break unless REDIRECTS.include?(response.status) && response.headers['location']
+
+          uri = redirect_target(uri, response.headers['location'])
+          response = connection.get(uri)
+        end
+        return response.body if response.success?
+
+        raise Error.new("Download from #{uri.host} failed with status #{response.status}", response:)
+      end
+
+      def self.redirect_target(from, location)
+        target = URI.join(from, location)
+        return target if target.scheme == 'https' || (target.scheme == 'http' && from.scheme == 'http')
+
+        raise Error, "Download from #{from.host} redirected to an unsafe #{target.scheme} URL"
+      rescue URI::InvalidURIError
+        raise Error, "Download from #{from.host} redirected to an invalid URL"
+      end
+      private_class_method :redirect_target
+
       def self.cache
         CACHE
       end
 
+      # Runs the block with a fallback model ready to take over its
+      # requests: +ready+ answers whether an error moves on to it. Until the
+      # block returns, a timeout or a failed connection is not retried on
+      # the same model, nor an error whose retry hint asks for a wait
+      # longer than FALLBACK_WAIT, when +ready+ accepts the error.
+      def self.with_fallback_ready(ready)
+        previous = Fiber[FALLBACK_KEY]
+        Fiber[FALLBACK_KEY] = ready
+        yield
+      ensure
+        Fiber[FALLBACK_KEY] = previous
+      end
+
+      # A frozen copy of +ssl+, so a value changed in place afterwards
+      # reaches neither a cache key nor a connection already built.
+      def self.ssl_snapshot(ssl)
+        ssl.to_h.transform_values { |value| value.is_a?(String) ? value.dup.freeze : value }.freeze
+      end
+
       # The result is shared across threads and contexts, so the middleware
       # stack is built now rather than lazily, without a lock, on the first
       # request, and the defaults every request copies are frozen.
@@ -57,6 +118,7 @@ module RubyLLM
           setup_retry(faraday, settings)
           setup_middleware(faraday, settings)
           setup_http_proxy(faraday, settings)
+          setup_ssl(faraday, settings)
         end
         [connection.headers, connection.params, connection.options].each(&:freeze)
         connection.app
@@ -68,36 +130,37 @@ module RubyLLM
         @config = config
         @headers = headers
         @signer = signer
-        @settings = settings_for(api_base || provider.api_base)
-        @stream_settings = @settings.dup.tap { |settings| settings.stream = true }.freeze
+        @api_base = api_base
         connection
       end
 
       # The Faraday connection shared by every request with these settings in
       # this process. Looked up per request, so an object that outlives a fork
-      # never reaches the parent's sockets. Streaming requests get their own:
-      # some adapters, such as httpx, only stream if their first request did.
+      # never reaches the parent's sockets, and a request goes to the endpoint
+      # configured for the credentials it carries. Streaming requests get
+      # their own: some adapters, such as httpx, only stream if their first
+      # request did.
       def connection(stream: false)
-        settings = stream ? @stream_settings : @settings
+        settings = settings_for(stream)
         CACHE.fetch(settings) { self.class.build(settings) }
       end
 
       def post(url, payload, usage: nil, idempotent: true, stream: false, &)
         instrument_request(:post, url) do
-          response = connection(stream:).post url, payload do |req|
+          connection(stream:).post url, payload do |req|
             prepare(req)
             set_usage_tracker(req, usage) if usage
             mark_non_idempotent(req) unless idempotent
             yield req if block_given?
           end
-          release_request(response)
         end
       end
 
-      def get(url, &)
+      def get(url, raw: false, &)
         instrument_request(:get, url) do
           connection.get url do |req|
             prepare(req)
+            keep_raw_body(req) if raw
             yield req if block_given?
           end
         end
@@ -105,11 +168,10 @@ module RubyLLM
 
       def patch(url, payload, &)
         instrument_request(:patch, url) do
-          response = connection.patch url, payload do |req|
+          connection.patch url, payload do |req|
             prepare(req)
             yield req if block_given?
           end
-          release_request(response)
         end
       end
 
@@ -132,32 +194,29 @@ module RubyLLM
         }
 
         RubyLLM.instrument('request.ruby_llm', payload, config: @config) do |event|
-          response = yield
+          response = ReleaseRequest.release(yield)
           event[:status] = response.status if response.respond_to?(:status)
           response
+        rescue RubyLLM::Error => e
+          ReleaseRequest.release(e.response)
+          raise
         end
       end
 
-      # The response outlives the call as Message#raw and other results. A chat
-      # request body is the whole serialized conversation, and a streaming
-      # callback closes over the payload it was built from.
-      def release_request(response)
-        response.env.request_body = nil
-        response.env.request.on_data = nil
-        response.env.request.context&.delete(ErrorMiddleware::STREAM_RESET_KEY)
-        response
-      end
-
-      def settings_for(api_base)
+      def settings_for(stream)
+        logger = @config.logger || RubyLLM.logger
         Settings.new(
           provider: @provider.class,
-          stream: false,
-          api_base: api_base,
+          stream: stream,
+          api_base: @api_base || @provider.api_base,
           adapter: @config.faraday_adapter,
           timeout: @config.request_timeout,
+          connect_timeout: @config.connect_timeout,
+          idle_timeout: (@config.stream_idle_timeout if stream),
           proxy: @config.http_proxy,
-          logger: RubyLLM.logger,
-          log_bodies: RubyLLM.logger.debug?,
+          ssl: (self.class.ssl_snapshot(@config.ssl) if @config.ssl),
+          logger: logger,
+          log_bodies: logger.debug?,
           log_regexp_timeout: @config.log_regexp_timeout,
           max_retries: @config.max_retries,
           retry_interval: @config.retry_interval,
@@ -177,8 +236,12 @@ module RubyLLM
         context[SigningMiddleware::CONTEXT_KEY] = @signer if @signer
       end
 
+      # Adapters time each read of a response with +read_timeout+, so on a
+      # streaming connection it is the longest wait for the next bytes.
       def self.setup_timeout(faraday, settings)
         faraday.options.timeout = settings.timeout
+        faraday.options.open_timeout = settings.connect_timeout
+        faraday.options.read_timeout = settings.idle_timeout if settings.idle_timeout
       end
 
       def self.setup_logging(faraday, settings)
@@ -187,7 +250,8 @@ module RubyLLM
                          bodies: settings.log_bodies,
                          errors: true,
                          headers: false,
-                         log_level: :debug do |logger|
+                         log_level: :debug,
+                         formatter: LogFormatter do |logger|
           logger.filter(logging_regexp('[A-Za-z0-9+/=]{100,}', settings), '[BASE64 DATA]')
           logger.filter(logging_regexp('[-\\d.e,\\s]{100,}', settings), '[EMBEDDINGS ARRAY]')
         end
@@ -207,8 +271,10 @@ module RubyLLM
           interval_randomness: settings.retry_interval_randomness,
           backoff_factor: settings.retry_backoff_factor,
           methods: Faraday::Retry::Middleware::IDEMPOTENT_METHODS,
-          retry_if: lambda { |env, _exception|
-            env[:method] == :post && idempotent?(env) && !stream_delivered?(env)
+          retry_if: lambda { |env, exception|
+            Support::Cancellation.check
+            env[:method] == :post && idempotent?(env) && !stream_delivered?(env) &&
+              !fall_back_instead?(env, exception)
           },
           exceptions: retry_exceptions
         }
@@ -223,21 +289,61 @@ module RubyLLM
         env[:request]&.context&.dig(IDEMPOTENT_KEY) != false
       end
 
+      # Whether a fallback set up by ::with_fallback_ready takes this
+      # failure instead of a retry on the same model.
+      def self.fall_back_instead?(env, exception)
+        ready = Fiber[FALLBACK_KEY]
+        return false unless ready && exception
+
+        error = exception.is_a?(RubyLLM::Error) ? exception : RubyLLM::ConnectionError.new(exception.message)
+        return false unless ready.call(error)
+
+        error.is_a?(RubyLLM::ConnectionError) || retry_wait(env, error) > FALLBACK_WAIT
+      end
+
+      # The wait in seconds the response's retry hints ask for, read as the
+      # retry middleware reads them, or 0 without one.
+      def self.retry_wait(env, error)
+        headers = env[:response_headers] || error.response&.headers || {}
+        %w[Retry-After RateLimit-Reset].filter_map { |name| header_seconds(headers[name]) }.max || 0
+      end
+
+      def self.header_seconds(value)
+        text = value.to_s.strip
+        Float(text, exception: false) || (Time.httpdate(text) - Time.now)
+      rescue ArgumentError
+        nil
+      end
+
       def self.setup_middleware(faraday, settings)
         faraday.request :multipart
         faraday.request :json
         faraday.use :llm_signing
         faraday.use JsonResponse
-        faraday.adapter(settings.adapter)
+        faraday.adapter(settings.adapter, **adapter_options(settings))
         faraday.use :llm_errors
       end
 
+      # async-http-faraday reads its timeout for each read from its own
+      # options rather than the request's.
+      def self.adapter_options(settings)
+        return {} unless settings.adapter == :async_http && settings.idle_timeout
+
+        { read_timeout: settings.idle_timeout }
+      end
+
       def self.setup_http_proxy(faraday, settings)
         return unless settings.proxy
 
         faraday.proxy = settings.proxy
       end
 
+      def self.setup_ssl(faraday, settings)
+        return unless settings.ssl
+
+        faraday.ssl.update(settings.ssl)
+      end
+
       def self.retry_exceptions
         [
           Errno::ETIMEDOUT,
@@ -246,6 +352,7 @@ module RubyLLM
           Faraday::ConnectionFailed,
           Faraday::SSLError,
           Faraday::RetriableResponse,
+          RubyLLM::ConnectionError,
           RubyLLM::RateLimitError,
           RubyLLM::ServerError,
           RubyLLM::ServiceUnavailableError,
@@ -254,7 +361,8 @@ module RubyLLM
       end
 
       private_class_method :setup_timeout, :setup_logging, :logging_regexp, :setup_retry, :stream_delivered?,
-                           :idempotent?, :setup_middleware, :setup_http_proxy, :retry_exceptions
+                           :idempotent?, :fall_back_instead?, :retry_wait, :header_seconds, :setup_middleware,
+                           :adapter_options, :setup_http_proxy, :setup_ssl, :retry_exceptions
 
       def set_usage_tracker(request, tracker)
         context = request.options.context ||= {}
@@ -268,6 +376,11 @@ module RubyLLM
         context[IDEMPOTENT_KEY] = false
       end
 
+      def keep_raw_body(request)
+        context = request.options.context ||= {}
+        context[JsonResponse::RAW_KEY] = true
+      end
+
       def inspect_attributes # :nodoc:
         { provider: @provider.slug }
       end

data/lib/ruby_llm/transport/error_middleware.rb

@@ -8,6 +8,8 @@ module RubyLLM
     class ErrorMiddleware < Faraday::Middleware # :nodoc: all
       PROVIDER_KEY = :ruby_llm_provider
       STREAM_RESET_KEY = :ruby_llm_stream_reset
+      STREAM_FAILURE_KEY = :ruby_llm_stream_failure
+      NETWORK_FAILURES = [Faraday::ConnectionFailed, Faraday::TimeoutError, Faraday::SSLError].freeze
 
       def initialize(app, options = {})
         super(app)
@@ -16,25 +18,59 @@ module RubyLLM
 
       # Sits directly above the adapter, inside the retry middleware, so this
       # runs once per attempt: streaming state stored on the env by a previous
-      # attempt must not leak into the next one.
+      # attempt must not leak into the next one, and a body the previous
+      # attempt read to its end is sent again from its start.
       def call(env)
+        context = env[:request]&.context
+        provider = context&.[](PROVIDER_KEY) || @provider
+        attempt(env, context, provider)
+      rescue *NETWORK_FAILURES => e
+        raise if context&.dig(Connection::STREAM_PROGRESS_KEY, :callback_error).equal?(e)
+
+        raise ConnectionError.new(e.message, provider: (provider.slug if provider.respond_to?(:slug)))
+      end
+
+      private
+
+      def attempt(env, context, provider)
         env[:streaming_error_response] = nil
         env[:streaming_state] = nil
-        context = env[:request]&.context
+        env[:body].rewind if env[:body].respond_to?(:rewind)
         context&.[](STREAM_RESET_KEY)&.call
-        provider = context&.[](PROVIDER_KEY) || @provider
         @app.call(env).on_complete do |response|
+          context&.[](STREAM_FAILURE_KEY)&.call(response)
           apply_retry_delay(response, provider)
           self.class.parse_error(provider:, response: streaming_error_response(response))
         end
+      rescue RubyLLM::Error => e
+        apply_raised_retry_delay(env, provider, e)
+        raise
       end
 
-      private
+      # A rate limit reported inside a stream raises with no failed HTTP
+      # status, and without an env on Faraday 1, so its hint is only on the
+      # error's own response.
+      def apply_raised_retry_delay(env, provider, error)
+        return apply_retry_delay(env, provider) unless rate_limit_report?(error)
+
+        env[:streaming_error_response] ||= error.response
+        env[:response_headers] ||= Faraday::Utils::Headers.new
+        apply_retry_delay(env, provider, status: 429)
+      end
+
+      def rate_limit_report?(error)
+        return true if error.is_a?(RateLimitError)
+
+        [error.response&.body].flatten.grep(Hash).any? do |part|
+          part['error'].is_a?(Hash) && part['error']['code'] == 429
+        end
+      end
 
       # The retry middleware only reads the standard Retry-After header, so
-      # other retry hints are normalized into seconds here.
-      def apply_retry_delay(response, provider)
-        status = response.respond_to?(:status) ? response.status : response[:status]
+      # other retry hints are normalized into seconds here. A failed stream
+      # usually raises while it is read, before on_complete, and keeps its
+      # body in the stored streaming error rather than on the response.
+      def apply_retry_delay(response, provider, status: response_status(response))
         return unless status && status >= 400
 
         headers = response[:response_headers]
@@ -44,8 +80,12 @@ module RubyLLM
         headers['Retry-After'] = delay.to_s if delay
       end
 
+      def response_status(response)
+        response.respond_to?(:status) ? response.status : response[:status]
+      end
+
       def provider_retry_delay(provider, response, status)
-        provider&.retry_delay(response) if status == 429
+        provider&.retry_delay(streaming_error_response(response)) if status == 429
       end
 
       def millisecond_retry_delay(headers)
@@ -78,7 +118,11 @@ module RubyLLM
           /input or output tokens? must be reduced/i,
           /reduce the length of messages/i,
           /prompt is too long/i,
-          /context limit/i
+          /context limit/i,
+          /exceeded model token limit/i,
+          /prompt exceeds max length/i,
+          /input is too long/i,
+          /longer than the maximum model length/i
         ].freeze
 
         RATE_LIMIT_PATTERNS = [
@@ -105,59 +149,60 @@ module RubyLLM
         ].freeze
 
         def parse_error(provider:, response:)
-          return if (200..399).cover?(response.status)
+          status = failure_status(provider, response)
+          return if (200..399).cover?(status)
 
           message = provider&.parse_error(response)
-
-          case response.status
-          when 400
-            raise_bad_request(message, response)
-          when 401
-            raise UnauthorizedError.new(message, response:)
-          when 402
-            raise PaymentRequiredError.new(message, response:)
-          when 403
-            raise_forbidden(message, response)
-          when 413
-            raise_too_large(message, response)
-          when 429
-            raise RateLimitError.new(message, response:) if rate_limited?(message)
-            raise ContextLengthExceededError.new(message, response:) if context_length_exceeded?(message)
-
-            raise RateLimitError.new(message, response:)
-          when 500
-            raise ServerError.new(message, response:)
-          when 502..504
-            raise ServiceUnavailableError.new(message, response:)
-          when 529
-            raise OverloadedError.new(message, response:)
-          else
-            raise Error.new(message, response:)
-          end
+          raise error_class(message, status).new(response:, provider_message: message,
+                                                 **provider_details(provider, response))
         end
 
         private
 
-        def raise_bad_request(message, response)
-          raise UnauthorizedError.new(message, response:) if authentication_failed?(message)
-          raise ContextLengthExceededError.new(message, response:) if context_length_exceeded?(message)
-          raise RateLimitError.new(message, response:) if rate_limited?(message)
-          raise OverloadedError.new(message, response:) if overloaded?(message)
-          raise PaymentRequiredError.new(message, response:) if payment_required?(message)
+        def failure_status(provider, response)
+          return response.status unless (200..299).cover?(response.status)
+
+          provider&.error_status(response) || response.status
+        end
 
-          raise BadRequestError.new(message, response:)
+        def provider_details(provider, response)
+          {
+            provider: (provider.slug if provider.respond_to?(:slug)),
+            error_code: (provider.parse_error_code(response) if provider.respond_to?(:parse_error_code)),
+            redact: (provider.config&.redact_error_messages if provider.respond_to?(:config))
+          }
         end
 
-        def raise_forbidden(message, response)
-          raise UnauthorizedError.new(message, response:) if authentication_failed?(message)
+        def error_class(message, status)
+          case status
+          when 400 then bad_request_class(message)
+          when 401 then UnauthorizedError
+          when 402 then PaymentRequiredError
+          when 403 then authentication_failed?(message) ? UnauthorizedError : ForbiddenError
+          when 413 then context_length_exceeded?(message) ? ContextLengthExceededError : Error
+          when 429 then rate_limit_class(message)
+          when 500 then ServerError
+          when 502..504 then ServiceUnavailableError
+          when 529 then OverloadedError
+          else Error
+          end
+        end
 
-          raise ForbiddenError.new(message, response:)
+        def bad_request_class(message)
+          if authentication_failed?(message) then UnauthorizedError
+          elsif context_length_exceeded?(message) then ContextLengthExceededError
+          elsif rate_limited?(message) then RateLimitError
+          elsif overloaded?(message) then OverloadedError
+          elsif payment_required?(message) then PaymentRequiredError
+          else BadRequestError
+          end
         end
 
-        def raise_too_large(message, response)
-          raise ContextLengthExceededError.new(message, response:) if context_length_exceeded?(message)
+        def rate_limit_class(message)
+          return RateLimitError if rate_limited?(message)
+          return ContextLengthExceededError if context_length_exceeded?(message)
 
-          raise Error.new(message, response:)
+          RateLimitError
         end
 
         def context_length_exceeded?(message)

data/lib/ruby_llm/transport/event_stream_parser.rb

@@ -6,6 +6,8 @@ module RubyLLM
     # whatever pieces the network delivers. Each dispatched event is yielded
     # as its type, data, and last event ID. The last event ID and the
     # reconnection time the server asked for stay readable between events.
+    # The standard ignores a line that names no known field; +on_unknown_line+
+    # receives each one, undecoded, for a caller that wants to look at it.
     class EventStreamParser # :nodoc:
       BOM = "\xEF\xBB\xBF".b.freeze
       LINE_BREAK = /[\r\n]/
@@ -16,7 +18,8 @@ module RubyLLM
 
       attr_reader :last_event_id, :reconnection_time
 
-      def initialize
+      def initialize(on_unknown_line: nil)
+        @on_unknown_line = on_unknown_line
         @buffer = String.new(encoding: Encoding::BINARY)
         @position = 0
         @searched = 0
@@ -43,6 +46,16 @@ module RubyLLM
         @searched = @buffer.bytesize
       end
 
+      # Reads a last line the stream ended without a line break. It can't
+      # complete an event, which needs a blank line after it.
+      def finish
+        return if @position >= @buffer.bytesize
+
+        line = @buffer.byteslice(@position, @buffer.bytesize - @position)
+        @position = @searched = @buffer.bytesize
+        process_line(line)
+      end
+
       private
 
       def compact
@@ -85,18 +98,19 @@ module RubyLLM
         return if line.start_with?(':')
 
         colon = line.index(':')
-        return process_field(line, +'') unless colon
+        return process_field(line, +'', line) unless colon
 
         offset = line.getbyte(colon + 1) == SPACE ? colon + 2 : colon + 1
-        process_field(line.byteslice(0, colon), line.byteslice(offset, line.bytesize - offset))
+        process_field(line.byteslice(0, colon), line.byteslice(offset, line.bytesize - offset), line)
       end
 
-      def process_field(field, value)
+      def process_field(field, value, line)
         case field
         when 'event' then @event_type = decode(value)
         when 'data' then @data << decode(value) << "\n"
         when 'id' then @id_buffer = decode(value) unless value.include?("\0")
         when 'retry' then @reconnection_time = value.to_i if value.match?(DIGITS)
+        else @on_unknown_line&.call(line)
         end
       end
 

data/lib/ruby_llm/transport/json_response.rb

@@ -6,6 +6,8 @@ require 'json'
 module RubyLLM
   module Transport
     class JsonResponse < Faraday::Middleware # :nodoc:
+      RAW_KEY = :ruby_llm_raw_body
+
       def initialize(app, parser_options: {}, preserve_raw: false)
         super(app)
         @parser_options = parser_options || {}
@@ -14,16 +16,20 @@ module RubyLLM
 
       def on_complete(env)
         body = env[:body]
-        return unless body.respond_to?(:to_str) && json_response?(env)
+        return unless body.respond_to?(:to_str) && json_response?(env) && !raw?(env)
 
         env[:raw_body] = body if @preserve_raw
         env[:body] = body.strip.empty? ? nil : JSON.parse(body, **@parser_options)
-      rescue StandardError, SyntaxError => e
-        raise Faraday::ParsingError.new(e, env[:response])
+      rescue StandardError, SyntaxError
+        raise Error.new('The response body is not valid JSON', response: env[:response])
       end
 
       private
 
+      def raw?(env)
+        env[:request]&.context&.dig(RAW_KEY)
+      end
+
       def json_response?(env)
         content_type = env[:response_headers]['content-type'].to_s.split(';', 2).first
         content_type&.match?(/\bjson$/)

data/lib/ruby_llm/transport/log_formatter.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+require 'faraday'
+require 'faraday/logging/formatter'
+
+module RubyLLM
+  module Transport
+    # A request that succeeded and was billed must not fail because a debug
+    # log filter ran out of time on a large body.
+    class LogFormatter < Faraday::Logging::Formatter # :nodoc:
+      private
+
+      def apply_filters(output)
+        super
+      rescue Regexp::TimeoutError
+        '[NOT LOGGED: FILTER TIMED OUT]'
+      end
+    end
+  end
+end