patient_http-sidekiq 1.2.0 → 1.4.0

data/lib/patient_http/sidekiq/processor_observer.rb

@@ -2,47 +2,188 @@
 
 module PatientHttp
   module Sidekiq
-    # Procesor Observer that collect stats in Redis for the WebUI and
-    # monitors for crashed processes in order to re-enqueue workers.
+    # Processor observer that records stats and maintains the crash-recovery
+    # registry for one processor. The stats aggregator and task monitor are
+    # shared across all processors in the process; the module owns them and
+    # the monitor thread.
+    #
+    # Tasks are registered in the crash-recovery registry when the processor
+    # accepts them, before Processor#enqueue returns, so a request always has
+    # a durable record from the moment the caller hands it off. The entry is
+    # removed when the request completes or when a Sidekiq job owns the
+    # request again (the task was rejected or re-enqueued). When result
+    # delivery fails (completion_failed), the entry is kept so the orphan
+    # collector re-enqueues the request instead of losing it, unless the
+    # failure is one that trying again cannot fix; see
+    # +UNDELIVERABLE_RESULT_ERRORS+.
     class ProcessorObserver < PatientHttp::ProcessorObserver
+      # Delivery failures that mean the result can never be delivered: the
+      # payload cannot be serialized, so every re-enqueue would end the same
+      # way. A request that fails with one of these is moved to the Sidekiq
+      # dead set instead of being kept for crash recovery. Every other failure
+      # is treated as temporary (Redis unavailable, for example) and keeps its
+      # crash-recovery record.
+      UNDELIVERABLE_RESULT_ERRORS = [
+        JSON::GeneratorError,
+        Encoding::UndefinedConversionError,
+        Encoding::InvalidByteSequenceError,
+        Encoding::CompatibilityError
+      ].freeze
+
       attr_reader :task_monitor
 
-      def initialize(processor)
+      def initialize(processor, stats:, task_monitor:)
         @processor = processor
-        @stats = Stats.new(processor.config)
-        @task_monitor = TaskMonitor.new(processor.config)
-        @monitor_thread = TaskMonitorThread.new(
-          processor.config,
-          @task_monitor,
-          -> { @processor.inflight_request_ids }
-        )
+        @stats = stats
+        @task_monitor = task_monitor
+        @processor_name = processor.name
+        @requeued_task_ids = Set.new
+        @requeued_mutex = Mutex.new
       end
 
-      def start
-        @monitor_thread.start
+      def capacity_exceeded
+        @stats.record_capacity_exceeded(processor_name: @processor_name)
       end
 
-      def stop
-        @monitor_thread.stop
-        task_monitor.remove_process
+      def request_enqueued(request_task)
+        task_monitor.register(request_task, processor_name: @processor_name)
+        @stats.record_inflight_peak(inflight_after_enqueue, processor_name: @processor_name)
       end
 
-      def capacity_exceeded
-        @stats.record_capacity_exceeded
+      def request_rejected(request_task)
+        task_monitor.unregister(request_task)
       end
 
-      def request_start(request_task)
-        task_monitor.register(request_task)
+      def request_requeued(request_task)
+        task_monitor.unregister(request_task)
+        # The re-enqueue path fires request_end after request_requeued, but
+        # only for tasks that already started. Remember those tasks so that
+        # request_end does not unregister a second time or record a completion
+        # stat for a request that never completed. A task that never started
+        # gets no request_end, so remembering it would leak the id forever.
+        return unless request_task.started?
+
+        @requeued_mutex.synchronize { @requeued_task_ids << request_task.id }
       end
 
       def request_end(request_task)
+        requeued = @requeued_mutex.synchronize { @requeued_task_ids.delete?(request_task.id) }
+        return if requeued
+
         task_monitor.unregister(request_task)
-        @stats.record_request(request_task.response&.status, request_task.duration)
+        @stats.record_request(request_task.response&.status, request_task.duration, processor_name: @processor_name)
       end
 
       def request_error(error)
         error_type = error.is_a?(PatientHttp::Error) ? error.error_type : :exception
-        @stats.record_error(error_type)
+        @stats.record_error(error_type, processor_name: @processor_name)
+      end
+
+      def completion_failed(request_task, error)
+        if undeliverable_result?(error) && kill_job(request_task, error)
+          # The request itself finished and nothing will deliver its result,
+          # so record it and drop its crash-recovery entry.
+          task_monitor.unregister(request_task)
+          @stats.record_request(request_task.response&.status, request_task.duration, processor_name: @processor_name)
+          @stats.record_error(:undeliverable_result, processor_name: @processor_name)
+          PatientHttp::Sidekiq.configuration.logger&.error(
+            "[PatientHttp::Sidekiq] Result for request #{request_task.id} can never be delivered; " \
+            "moved its job to the dead set: #{error.class} - #{error_message(error)}"
+          )
+          return
+        end
+
+        # Keep the crash-recovery registry entry: the orphan collector will
+        # re-enqueue the request once its heartbeat goes stale.
+        @stats.record_error(:completion_failed, processor_name: @processor_name)
+        PatientHttp::Sidekiq.configuration.logger&.error(
+          "[PatientHttp::Sidekiq] Result delivery failed for request #{request_task.id}; " \
+          "leaving crash-recovery record for re-enqueue: #{error.class} - #{error_message(error)}"
+        )
+      end
+
+      private
+
+      # The number of requests the processor holds once it accepts the request
+      # being announced. The count only rises when a request is accepted, so
+      # sampling it here catches every high-water mark.
+      #
+      # The announcement is made before the task is counted, so the task itself
+      # is added. A task announced when the processor is already full is
+      # rejected right after, so the count is held to the processor's capacity.
+      #
+      # @return [Integer]
+      def inflight_after_enqueue
+        [@processor.total_count + 1, @processor.config.max_connections].min
+      end
+
+      # Whether an error means the result can never be delivered. The cause
+      # chain is examined as well, because the failure is usually raised while
+      # the result is being written to Redis.
+      #
+      # @param error [Exception] the delivery failure
+      # @return [Boolean]
+      def undeliverable_result?(error)
+        while error
+          return true if UNDELIVERABLE_RESULT_ERRORS.any? { |error_class| error.is_a?(error_class) }
+
+          error = error.cause
+        end
+
+        false
+      end
+
+      # Move a request's job to the Sidekiq dead set, where it can be
+      # inspected and retried by hand.
+      #
+      # Sidekiq's API is loaded on demand because loading it eagerly fails on
+      # some supported Sidekiq versions. A job that cannot be moved reports
+      # false, so the caller keeps the crash-recovery record rather than
+      # dropping the request.
+      #
+      # @param request_task [RequestTask] the request task
+      # @param error [Exception] the delivery failure
+      # @return [Boolean] whether the job was moved
+      def kill_job(request_task, error)
+        require "sidekiq/api"
+
+        job = dead_job(request_task.task_handler.sidekiq_job, error)
+        ::Sidekiq::DeadSet.new.kill(JSON.generate(job), notify_failure: true, ex: error)
+        true
+      rescue LoadError, StandardError => e
+        PatientHttp::Sidekiq.configuration.logger&.error(
+          "[PatientHttp::Sidekiq] Failed to move request #{request_task.id} to the dead set: #{e.class} - #{error_message(e)}"
+        )
+        false
+      end
+
+      # Build the job record for the dead set. The failure fields are the ones
+      # Sidekiq writes when a job dies on its own, so the entry reads the same
+      # in the Web UI. A directly executed request has no job id yet, so it is
+      # given one; the Web UI identifies dead entries by it.
+      #
+      # @param job [Hash] the Sidekiq job hash
+      # @param error [Exception] the delivery failure
+      # @return [Hash] the job record to store
+      def dead_job(job, error)
+        job.merge(
+          "jid" => job["jid"] || SecureRandom.hex(12),
+          "failed_at" => Time.now.to_f,
+          "error_class" => error.class.name,
+          "error_message" => error_message(error)
+        )
+      end
+
+      # An error message that is safe to serialize and to log. A message that
+      # reports a byte the result could not be serialized with holds that byte
+      # itself, which would fail the same way the result did.
+      #
+      # @param error [Exception] the error
+      # @return [String] the message with any invalid bytes replaced
+      def error_message(error)
+        message = error.message.to_s
+        message = message.dup.force_encoding(Encoding::UTF_8) if message.encoding == Encoding::BINARY
+        message.scrub
       end
     end
   end

data/lib/patient_http/sidekiq/redis_pool.rb

@@ -0,0 +1,88 @@
+# frozen_string_literal: true
+
+module PatientHttp
+  module Sidekiq
+    # Dedicated Redis connection pool for the gem's own threads.
+    #
+    # The processor's completion worker threads and the task monitor thread
+    # carry no Sidekiq capsule state, so plain `Sidekiq.redis` calls from them
+    # fall through to Sidekiq's small internal pool (10 connections, 1 second
+    # checkout timeout). Under load that pool becomes a serialization point
+    # and checkout timeouts can lose work. This pool is built from the
+    # application's own Sidekiq Redis configuration and is used for all
+    # registry, stats, and job pushes made from gem-owned threads.
+    class RedisPool
+      DEFAULT_MINIMUM_SIZE = 10
+
+      # @param config [Configuration] the gem configuration
+      def initialize(config)
+        @config = config
+        @pid = nil
+        @pool = nil
+        @mutex = Mutex.new
+      end
+
+      # The underlying ConnectionPool, created lazily and rebuilt after a
+      # process fork so child processes never share parent connections.
+      #
+      # @return [ConnectionPool]
+      def pool
+        @mutex.synchronize do
+          if @pool.nil? || @pid != ::Process.pid
+            @pool = ::Sidekiq.default_configuration.new_redis_pool(size, "patient_http")
+            @pid = ::Process.pid
+          end
+          @pool
+        end
+      end
+
+      # Check out a connection with the gem's checkout timeout and yield it.
+      # A connection-level failure is retried once on a fresh checkout,
+      # mirroring the retry Sidekiq itself performs.
+      #
+      # The retry replays the whole block, so callers whose block is not
+      # idempotent (counter increments, anything the server may already have
+      # applied before the connection dropped) must pass
+      # +retry_on_connection_error: false+ and handle the failure themselves.
+      #
+      # @param retry_on_connection_error [Boolean] whether to replay the block
+      #   once after a connection-level failure
+      # @yield [conn] the Redis connection
+      # @return [Object] the block's return value
+      def with(retry_on_connection_error: true, &block)
+        retryable = retry_on_connection_error
+        begin
+          pool.with(timeout: @config.redis_pool_timeout) do |conn|
+            yield conn
+          end
+        rescue RedisClient::ConnectionError
+          raise unless retryable
+          retryable = false
+          retry
+        end
+      end
+
+      # Close all connections and drop the pool.
+      #
+      # @return [void]
+      def shutdown
+        @mutex.synchronize do
+          @pool&.shutdown { |conn| conn.close }
+          @pool = nil
+          @pid = nil
+        end
+      end
+
+      private
+
+      # Pool size: explicit configuration wins; otherwise size it to cover the
+      # completion worker threads plus the monitor thread and request
+      # registration, with a sane floor.
+      #
+      # @return [Integer]
+      def size
+        @config.redis_pool_size || [DEFAULT_MINIMUM_SIZE, @config.completion_threads + 3].max
+      end
+    end
+  end
+end

data/lib/patient_http/sidekiq/request_executor.rb

@@ -7,9 +7,8 @@ module PatientHttp
       class << self
         # Execute the request directly on the async processor.
         #
-        # This method enqueues the request directly to the async processor. It must be
-        # called from within a Sidekiq job context (the sidekiq_job parameter is required).
-        # Used internally by RequestWorker.
+        # This method enqueues the request directly to the async processor. Used internally
+        # by RequestWorker.
         #
         # When the request completes, the callback's +on_complete+ method is called with
         # a Response object. If an error occurs (network error, timeout, or non-2xx response
@@ -22,6 +21,9 @@ module PatientHttp
         #   If not provided, uses PatientHttp::Sidekiq::Context.current_job.
         #   This requires the PatientHttp::Sidekiq::Context::Middleware to be added
         #   to the Sidekiq server middleware chain.
+        # @param task_handler [PatientHttp::TaskHandler, nil] A prebuilt task handler.
+        #   When provided, the sidekiq_job parameter is ignored and no handler is
+        #   built from it. Used for direct execution on the local processor.
         # @param synchronous [Boolean] If true, runs the request inline (for testing).
         # @param callback_args [#to_h, nil] Arguments to pass to callback via the
         #   Response/Error object. Must respond to +to_h+ and contain only JSON-native types
@@ -32,20 +34,31 @@ module PatientHttp
         #   and calls +on_error+ instead of +on_complete+. Defaults to false.
         # @param request_id [String, nil] Unique request ID for tracking. If nil, a new UUID
         #   will be generated.
+        # @param processor_name [Symbol, String, nil] Name of the processor profile to run
+        #   the request on. Defaults to the request's own processor name or :default.
         # @return [String] the request ID
         # @api private
         def execute(
           request,
           callback:,
           sidekiq_job: nil,
+          task_handler: nil,
           synchronous: false,
           callback_args: nil,
           raise_error_responses: false,
-          request_id: nil
+          request_id: nil,
+          processor_name: nil
         )
-          sidekiq_job = validate_sidekiq_job(sidekiq_job)
+          task_handler ||= TaskHandler.new(validate_sidekiq_job(sidekiq_job))
           config = PatientHttp::Sidekiq.configuration
-          task_handler = TaskHandler.new(sidekiq_job)
+
+          # Look up the named processor and the effective configuration for its
+          # profile, so per-processor overrides apply to the request itself.
+          # A running processor already holds the built profile configuration.
+          name = (processor_name || request.processor || :default).to_sym
+          processor = PatientHttp::Sidekiq.processor(name)
+          profile_declared = config.processor_profiles.key?(name)
+          task_config = processor&.config || (profile_declared ? config.processor_config(name) : config)
 
           task = PatientHttp::RequestTask.new(
             request: request,
@@ -54,26 +67,43 @@ module PatientHttp
             callback_args: callback_args,
             raise_error_responses: raise_error_responses,
             id: request_id,
-            default_max_redirects: config.max_redirects
+            default_max_redirects: task_config.max_redirects
           )
 
           # Run the request inline if Sidekiq::Testing.inline! is enabled
           if synchronous || async_disabled?
             PatientHttp::SynchronousExecutor.new(
               task,
-              config: config,
+              config: task_config,
               on_complete: ->(response) { PatientHttp::Sidekiq.invoke_completion_callbacks(response) },
               on_error: ->(error) { PatientHttp::Sidekiq.invoke_error_callbacks(error) }
             ).call
             return task.id
           end
 
-          # Check if processor is running
-          processor = PatientHttp::Sidekiq.processor
+          # An unknown name raises so the job lands in Sidekiq's retry
+          # mechanism instead of being dropped; this covers rolling deploys
+          # where an old process has not configured a new profile yet.
+          if processor.nil? && !profile_declared
+            raise PatientHttp::UnknownProcessorError.new("No processor profile configured for #{name.inspect}")
+          end
+
           unless processor&.running?
             raise PatientHttp::NotRunningError.new("Cannot enqueue request: processor is not running")
           end
 
+          # Advisory capacity check before enqueueing. A real enqueue pays for
+          # durable registration before the authoritative capacity check, so a
+          # full processor would cost several Redis round trips just to be
+          # rejected. This peek rejects for free; the race where capacity fills
+          # after the peek falls through to the normal rejection path.
+          unless processor.capacity_available?
+            PatientHttp::Sidekiq.stats.record_capacity_exceeded(processor_name: name)
+            raise PatientHttp::MaxCapacityError.new(
+              "Cannot enqueue request: processor #{name} is at max capacity (#{processor.config.max_connections} connections)"
+            )
+          end
+
           processor.enqueue(task)
 
           task.id

data/lib/patient_http/sidekiq/request_worker.rb

@@ -40,8 +40,10 @@ module PatientHttp
       #   nil is treated as false
       # @param callback_args [Hash, nil] Arguments to pass to the callback
       # @param request_id [String, nil] Unique request ID for tracking
+      # @param processor_name [String, nil] Name of the processor profile to run the request
+      #   on; nil (jobs enqueued by older versions) runs on the default processor
       # @return [void]
-      def perform(data, callback_service_name, raise_error_responses, callback_args, request_id)
+      def perform(data, callback_service_name, raise_error_responses, callback_args, request_id, processor_name = nil)
         # Fetch from external storage if needed
         actual_data = PatientHttp::ExternalStorage.storage_ref?(data) ? Sidekiq.external_storage.fetch(data) : data
         actual_data = Sidekiq.decrypt(actual_data)
@@ -59,7 +61,8 @@ module PatientHttp
           raise_error_responses: raise_error_responses,
           callback_args: callback_args,
           sidekiq_job: sidekiq_job,
-          request_id: request_id
+          request_id: request_id,
+          processor_name: processor_name || "default"
         )
       end
     end