patient_http-sidekiq 1.0.0

data/lib/patient_http/sidekiq/callback_worker.rb

@@ -0,0 +1,96 @@
+# frozen_string_literal: true
+
+module PatientHttp
+  module Sidekiq
+    # Sidekiq worker that invokes callback services for HTTP request results.
+    #
+    # This worker receives serialized Response or Error data and invokes the
+    # appropriate callback service method (+on_complete+ or +on_error+).
+    #
+    # Callback services are plain Ruby classes that define +on_complete+ and +on_error+
+    # instance methods:
+    #
+    # @example Callback service
+    #   class MyCallback
+    #     def on_complete(response)
+    #       # Handle successful response
+    #       User.find(response.callback_args[:user_id]).update!(data: response.json)
+    #     end
+    #
+    #     def on_error(error)
+    #       # Handle request error
+    #       Rails.logger.error("Request failed: #{error.message}")
+    #     end
+    #   end
+    #
+    # @api private
+    class CallbackWorker
+      include ::Sidekiq::Job
+
+      # Clean up externally stored payloads when job exhausts all retries.
+      # This prevents orphaned payload files when callbacks fail permanently.
+      # Also invokes the on_retries_exhausted handler if configured.
+      sidekiq_retries_exhausted do |job, _exception|
+        data = job["args"][0]
+        result_type = job["args"][1]
+
+        begin
+          handler = PatientHttp::Sidekiq.configuration.on_retries_exhausted
+          if handler && result_type == "error"
+            actual_data = if Sidekiq.external_storage.storage_ref?(data)
+              Sidekiq.external_storage.fetch(data)
+            else
+              data
+            end
+            actual_data = Sidekiq.decrypt(actual_data)
+            error = PatientHttp::Error.load(actual_data)
+            handler.call(error)
+          end
+        rescue => e
+          PatientHttp::Sidekiq.configuration.logger&.warn(
+            "[PatientHttp::Sidekiq] on_retries_exhausted handler failed: #{e.class.name} #{e.message}".strip
+          )
+        end
+
+        begin
+          ExternalStorage.delete(data)
+        rescue => e
+          PatientHttp::Sidekiq.configuration.logger&.warn(
+            "[PatientHttp::Sidekiq] Failed to delete stored payload for dead job: #{e.class.name} #{e.message}".strip
+          )
+        end
+      end
+
+      # Perform the callback invocation.
+      #
+      # @param data [Hash] Response or Error data (possibly a storage reference)
+      # @param result_type [String] "response" or "error" indicating the type of result
+      # @param callback_service_name [String] Fully qualified callback service class name
+      def perform(data, result_type, callback_service_name)
+        callback_service_class = PatientHttp::ClassHelper.resolve_class_name(callback_service_name)
+        callback_service = callback_service_class.new
+
+        # Fetch from external storage if needed
+        ref_data = Sidekiq.external_storage.storage_ref?(data) ? data : nil
+        actual_data = ref_data ? Sidekiq.external_storage.fetch(data) : data
+        actual_data = Sidekiq.decrypt(actual_data)
+
+        begin
+          if result_type == "response"
+            response = PatientHttp::Response.load(actual_data)
+            PatientHttp::Sidekiq.invoke_completion_callbacks(response)
+            callback_service.on_complete(response)
+          elsif result_type == "error"
+            error = PatientHttp::Error.load(actual_data)
+            PatientHttp::Sidekiq.invoke_error_callbacks(error)
+            callback_service.on_error(error)
+          else
+            raise ArgumentError, "Unknown result_type: #{result_type}"
+          end
+        ensure
+          Sidekiq.external_storage.delete(ref_data) if ref_data
+        end
+      end
+    end
+  end
+end

data/lib/patient_http/sidekiq/configuration.rb

@@ -0,0 +1,175 @@
+# frozen_string_literal: true
+
+module PatientHttp
+  module Sidekiq
+    # Configuration for the Sidekiq Async HTTP gem.
+    #
+    # Wraps PatientHttp::Configuration with Sidekiq-aware defaults and adds
+    # Sidekiq-specific options like worker queue/retry settings.
+    #
+    # Access the underlying pool configuration via the +http_pool+ attribute.
+    class Configuration < PatientHttp::Configuration
+      # Default threshold in bytes above which payloads are stored externally
+      DEFAULT_PAYLOAD_STORE_THRESHOLD = 64 * 1024 # 64KB
+
+      # @return [Integer] Size threshold in bytes for external payload storage
+      attr_reader :payload_store_threshold
+
+      # @return [Numeric] Orphan detection threshold in seconds
+      attr_reader :orphan_threshold
+
+      # @return [Numeric] Heartbeat update interval in seconds
+      attr_reader :heartbeat_interval
+
+      # @return [Hash, nil] Sidekiq options to apply to RequestWorker and CallbackWorker
+      attr_reader :sidekiq_options
+
+      # @return [#call, nil] Handler invoked when a CallbackWorker job exhausts all retries.
+      # @overload on_retries_exhausted
+      #   Returns the current handler.
+      #   @return [#call, nil]
+      # @overload on_retries_exhausted(&block)
+      #   Sets a block as the handler.
+      #   @yield [error] block to execute when retries are exhausted
+      #   @yieldparam error [PatientHttp::Error] information about the error
+      def on_retries_exhausted(&block)
+        if block
+          @on_retries_exhausted = block
+        else
+          @on_retries_exhausted
+        end
+      end
+
+      # @return [Array<PatientHttp::ProcessorObserver>] Registered processor observers
+      #   Observers will be registered with the processor when it is started, allowing them to
+      #   receive lifecycle callbacks for PatientHttp requests.
+      attr_reader :observers
+
+      # Initializes a new Configuration with the specified options.
+      #
+      # @param heartbeat_interval [Integer] Interval for updating inflight request heartbeats in seconds
+      # @param orphan_threshold [Integer] Age threshold for detecting orphaned requests in seconds
+      # @param sidekiq_options [Hash, nil] Sidekiq options to apply to RequestWorker and CallbackWorker
+      # @param on_retries_exhausted [#call, nil] Handler called when a CallbackWorker job exhausts retries
+      # @param pool_options [Hash] Options passed through to PatientHttp::Configuration.
+      #   Sidekiq-aware defaults are applied for shutdown_timeout and logger
+      #   if not explicitly provided.
+      def initialize(
+        heartbeat_interval: 60,
+        orphan_threshold: 300,
+        sidekiq_options: nil,
+        payload_store_threshold: DEFAULT_PAYLOAD_STORE_THRESHOLD,
+        on_retries_exhausted: nil,
+        **pool_options
+      )
+        pool_options[:shutdown_timeout] ||= (::Sidekiq.default_configuration[:timeout] || 25) - 2
+        pool_options[:logger] ||= ::Sidekiq.logger
+
+        super(**pool_options)
+
+        @observers = []
+        self.sidekiq_options = sidekiq_options
+        self.heartbeat_interval = heartbeat_interval
+        self.orphan_threshold = orphan_threshold
+        self.payload_store_threshold = payload_store_threshold || DEFAULT_PAYLOAD_STORE_THRESHOLD
+        self.on_retries_exhausted = on_retries_exhausted
+      end
+
+      # Set the on_retries_exhausted handler.
+      #
+      # This handler is called when a CallbackWorker job exhausts all retries.
+      # It receives the same arguments as the on_error callback.
+      #
+      # @param value [#call, nil] A callable object or nil to clear the handler
+      # @raise [ArgumentError] If value is not callable and not nil
+      def on_retries_exhausted=(value)
+        if value && !value.respond_to?(:call)
+          raise ArgumentError.new("on_retries_exhausted must respond to #call, got: #{value.class}")
+        end
+
+        @on_retries_exhausted = value
+      end
+
+      # Set the threshold size for external payload storage.
+      #
+      # Payloads larger than this size (in bytes) will be stored externally
+      # when a payload store is configured.
+      #
+      # @param value [Integer] Threshold in bytes
+      # @raise [ArgumentError] If value is not a positive integer
+      def payload_store_threshold=(value)
+        validate_positive_integer(:payload_store_threshold, value)
+        @payload_store_threshold = value
+      end
+
+      # Set the heartbeat interval for crash recovery.
+      #
+      # @param value [Numeric] interval in seconds (must be positive)
+      # @raise [ArgumentError] if value is not positive or not less than orphan_threshold
+      def heartbeat_interval=(value)
+        raise ArgumentError.new("heartbeat_interval must be positive, got: #{value.inspect}") unless value.positive?
+
+        @heartbeat_interval = value
+        validate_heartbeat_and_threshold
+      end
+
+      # Set the orphan detection threshold for crash recovery.
+      #
+      # @param value [Numeric] threshold in seconds (must be positive and greater than heartbeat_interval)
+      # @raise [ArgumentError] if value is not positive or not greater than heartbeat_interval
+      def orphan_threshold=(value)
+        raise ArgumentError.new("orphan_threshold must be positive, got: #{value.inspect}") unless value.positive?
+
+        @orphan_threshold = value
+        validate_heartbeat_and_threshold
+      end
+
+      # Set Sidekiq worker options and apply them to RequestWorker and CallbackWorker.
+      # The options will be applied to both workers. If you want to customize just
+      # one of them, set the options directly on that worker class.
+      #
+      # @param options [Hash, nil] Sidekiq options hash
+      # @return [void]
+      def sidekiq_options=(options)
+        if options.nil?
+          @sidekiq_options = nil
+          return
+        end
+
+        unless options.is_a?(Hash)
+          raise ArgumentError.new("sidekiq_options must be a Hash, got: #{options.class}")
+        end
+
+        @sidekiq_options = options
+        apply_sidekiq_options(options)
+      end
+
+      # Convert to hash for inspection
+      # @return [Hash] hash representation with string keys
+      def to_h
+        super.merge(
+          "payload_store_threshold" => payload_store_threshold,
+          "heartbeat_interval" => heartbeat_interval,
+          "orphan_threshold" => orphan_threshold,
+          "sidekiq_options" => sidekiq_options,
+          "on_retries_exhausted" => on_retries_exhausted ? "defined" : nil
+        )
+      end
+
+      private
+
+      def apply_sidekiq_options(options)
+        PatientHttp::Sidekiq::RequestWorker.sidekiq_options(options)
+        PatientHttp::Sidekiq::CallbackWorker.sidekiq_options(options)
+      end
+
+      def validate_heartbeat_and_threshold
+        return unless @heartbeat_interval && @orphan_threshold
+
+        return unless @heartbeat_interval >= @orphan_threshold
+
+        raise ArgumentError.new("heartbeat_interval (#{@heartbeat_interval}) must be less than orphan_threshold (#{@orphan_threshold})")
+      end
+    end
+  end
+end

data/lib/patient_http/sidekiq/context.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+module PatientHttp
+  module Sidekiq
+    # Provides thread-safe context for Sidekiq jobs.
+    #
+    # This class manages the current Sidekiq job context using a thread-id keyed hash,
+    # allowing async HTTP requests to access job information without it being passed explicitly.
+    # Only RequestWorker needs this context for re-enqueueing jobs.
+    class Context
+      # Thread-safe hash keyed by thread object_id
+      @jobs = Concurrent::Map.new
+
+      # Sidekiq server middleware that sets the current job context.
+      #
+      # This middleware only activates for RequestWorker, which is the only
+      # worker that needs access to the job context for re-enqueueing.
+      class Middleware
+        include ::Sidekiq::ServerMiddleware
+
+        def call(worker, job, queue)
+          # Only set context for RequestWorker (the only worker that needs it)
+          if job["class"] == PatientHttp::Sidekiq::RequestWorker.name
+            PatientHttp::Sidekiq::Context.with_job(job) do
+              yield
+            end
+          else
+            yield
+          end
+        end
+      end
+
+      class << self
+        # Returns the current Sidekiq job hash from context.
+        #
+        # @return [Hash, nil] the current job hash or nil if no job context is set
+        def current_job
+          @jobs[Thread.current.object_id]
+        end
+
+        # Sets the current job context for the duration of the block.
+        #
+        # @param job [Hash] the Sidekiq job hash
+        # @yield executes the block with the job context set
+        # @return [Object] the return value of the block
+        def with_job(job)
+          thread_id = Thread.current.object_id
+          previous_job = @jobs[thread_id]
+          @jobs[thread_id] = job
+          yield
+        ensure
+          if previous_job
+            @jobs[thread_id] = previous_job
+          else
+            @jobs.delete(thread_id)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/patient_http/sidekiq/lifecycle_hooks.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+# Sidekiq server lifecycle hooks for automatic Sidekiq processor management.
+#
+# This class registers lifecycle hooks with Sidekiq to automatically start, drain,
+# and stop the Sidekiq processor along with the Sidekiq server.
+##
+# The hooks will:
+# - Start the processor when Sidekiq server starts (:startup event)
+# - Drain the processor when Sidekiq receives TSTP signal (:quiet event)
+# - Stop the processor when Sidekiq shuts down (:shutdown event)
+module PatientHttp
+  module Sidekiq
+    class LifecycleHooks
+      @registered = false
+
+      class << self
+        def register
+          return if @registered
+
+          PatientHttp::Sidekiq.append_middleware
+
+          ::Sidekiq.configure_server do |config|
+            config.on(:startup) do
+              PatientHttp::Sidekiq.start
+            end
+
+            config.on(:quiet) do
+              PatientHttp::Sidekiq.quiet
+            end
+
+            config.on(:shutdown) do
+              PatientHttp::Sidekiq.stop
+            end
+
+            @registered = true
+          end
+        end
+      end
+    end
+  end
+end

data/lib/patient_http/sidekiq/processor_observer.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+module PatientHttp
+  module Sidekiq
+    # Procesor Observer that collect stats in Redis for the WebUI and
+    # monitors for crashed processes in order to re-enqueue workers.
+    class ProcessorObserver < PatientHttp::ProcessorObserver
+      attr_reader :task_monitor
+
+      def initialize(processor)
+        @processor = processor
+        @stats = Stats.new(processor.config)
+        @task_monitor = TaskMonitor.new(processor.config)
+        @monitor_thread = TaskMonitorThread.new(
+          processor.config,
+          @task_monitor,
+          -> { @processor.inflight_request_ids }
+        )
+      end
+
+      def start
+        @monitor_thread.start
+      end
+
+      def stop
+        @monitor_thread.stop
+        task_monitor.remove_process
+      end
+
+      def capacity_exceeded
+        @stats.record_capacity_exceeded
+      end
+
+      def request_start(request_task)
+        task_monitor.register(request_task)
+      end
+
+      def request_end(request_task)
+        task_monitor.unregister(request_task)
+        @stats.record_request(request_task.response&.status, request_task.duration)
+      end
+
+      def request_error(error)
+        error_type = error.is_a?(PatientHttp::Error) ? error.error_type : :exception
+        @stats.record_error(error_type)
+      end
+    end
+  end
+end

data/lib/patient_http/sidekiq/request_executor.rb

@@ -0,0 +1,104 @@
+# frozen_string_literal: true
+
+module PatientHttp
+  module Sidekiq
+    # Helper methods for executing HTTP requests asynchronously.
+    class RequestExecutor
+      class << self
+        # Execute the request directly on the async processor.
+        #
+        # This method enqueues the request directly to the async processor. It must be
+        # called from within a Sidekiq job context (the sidekiq_job parameter is required).
+        # Used internally by RequestWorker.
+        #
+        # When the request completes, the callback's +on_complete+ method is called with
+        # a Response object. If an error occurs (network error, timeout, or non-2xx response
+        # if raise_error_responses is true), the +on_error+ method is called with an Error object.
+        #
+        # @param request [Request] the HTTP request to execute
+        # @param callback [Class, String] Callback service class with +on_complete+ and +on_error+
+        #   instance methods, or its fully qualified class name.
+        # @param sidekiq_job [Hash, nil] Sidekiq job hash with "class" and "args" keys.
+        #   If not provided, uses PatientHttp::Sidekiq::Context.current_job.
+        #   This requires the PatientHttp::Sidekiq::Context::Middleware to be added
+        #   to the Sidekiq server middleware chain.
+        # @param synchronous [Boolean] If true, runs the request inline (for testing).
+        # @param callback_args [#to_h, nil] Arguments to pass to callback via the
+        #   Response/Error object. Must respond to +to_h+ and contain only JSON-native types
+        #   (nil, true, false, String, Integer, Float, Array, Hash). All hash keys will be
+        #   converted to strings for serialization. Access via +response.callback_args+ or
+        #   +error.callback_args+ using symbol or string keys.
+        # @param raise_error_responses [Boolean] If true, treats non-2xx responses as errors
+        #   and calls +on_error+ instead of +on_complete+. Defaults to false.
+        # @param request_id [String, nil] Unique request ID for tracking. If nil, a new UUID
+        #   will be generated.
+        # @return [String] the request ID
+        # @api private
+        def execute(
+          request,
+          callback:,
+          sidekiq_job: nil,
+          synchronous: false,
+          callback_args: nil,
+          raise_error_responses: false,
+          request_id: nil
+        )
+          sidekiq_job = validate_sidekiq_job(sidekiq_job)
+          config = PatientHttp::Sidekiq.configuration
+          task_handler = TaskHandler.new(sidekiq_job)
+
+          task = PatientHttp::RequestTask.new(
+            request: request,
+            task_handler: task_handler,
+            callback: callback,
+            callback_args: callback_args,
+            raise_error_responses: raise_error_responses,
+            id: request_id,
+            default_max_redirects: config.max_redirects
+          )
+
+          # Run the request inline if Sidekiq::Testing.inline! is enabled
+          if synchronous || async_disabled?
+            PatientHttp::SynchronousExecutor.new(
+              task,
+              config: config,
+              on_complete: ->(response) { PatientHttp::Sidekiq.invoke_completion_callbacks(response) },
+              on_error: ->(error) { PatientHttp::Sidekiq.invoke_error_callbacks(error) }
+            ).call
+            return task.id
+          end
+
+          # Check if processor is running
+          processor = PatientHttp::Sidekiq.processor
+          unless processor&.running?
+            raise PatientHttp::NotRunningError.new("Cannot enqueue request: processor is not running")
+          end
+
+          processor.enqueue(task)
+
+          task.id
+        end
+
+        private
+
+        def validate_sidekiq_job(sidekiq_job)
+          sidekiq_job ||= PatientHttp::Sidekiq::Context.current_job
+
+          raise ArgumentError.new("sidekiq_job is required") if sidekiq_job.nil?
+
+          raise ArgumentError.new("sidekiq_job must be a Hash, got: #{sidekiq_job.class}") unless sidekiq_job.is_a?(Hash)
+
+          raise ArgumentError.new("sidekiq_job must have 'class' key") unless sidekiq_job.key?("class")
+
+          raise ArgumentError.new("sidekiq_job must have 'args' array") unless sidekiq_job["args"].is_a?(Array)
+
+          sidekiq_job
+        end
+
+        def async_disabled?
+          defined?(::Sidekiq::Testing) && ::Sidekiq::Testing.inline?
+        end
+      end
+    end
+  end
+end

data/lib/patient_http/sidekiq/request_worker.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+module PatientHttp
+  module Sidekiq
+    # Sidekiq worker for executing HTTP requests asynchronously.
+    #
+    # This worker is enqueued when calling +PatientHttp::Sidekiq.get+, +PatientHttp::Sidekiq.post+,
+    # etc. It allows HTTP requests to be made from anywhere in your code (not just Sidekiq jobs)
+    # while still processing them through the async HTTP processor.
+    #
+    # When the request completes, the specified callback service's +on_complete+ or +on_error+
+    # method is invoked via CallbackWorker.
+    #
+    # @api private
+    class RequestWorker
+      include ::Sidekiq::Job
+
+      # Perform the HTTP request.
+      #
+      # @param data [Hash] Request data (possibly a storage reference) with keys:
+      #   - "http_method" [String] HTTP method (get, post, put, patch, delete)
+      #   - "url" [String] The request URL
+      #   - "headers" [Hash] Request headers
+      #   - "body" [String, nil] Request body
+      #   - "timeout" [Numeric, nil] Request timeout
+      #   - "max_redirects" [Integer, nil] Maximum redirects to follow
+      # @param callback_service_name [String] Fully qualified callback service class name
+      # @param raise_error_responses [Boolean, nil] Whether to treat non-2xx responses as errors;
+      #   defaults to the global config if nil
+      # @param callback_args [Hash, nil] Arguments to pass to the callback
+      # @param request_id [String, nil] Unique request ID for tracking
+      # @return [void]
+      def perform(data, callback_service_name, raise_error_responses, callback_args, request_id)
+        # Fetch from external storage if needed
+        ref_data = PatientHttp::ExternalStorage.storage_ref?(data) ? data : nil
+        actual_data = ref_data ? Sidekiq.external_storage.fetch(data) : data
+        actual_data = Sidekiq.decrypt(actual_data)
+
+        request = PatientHttp::Request.load(actual_data)
+        sidekiq_job = Sidekiq::Context.current_job
+
+        begin
+          RequestExecutor.execute(
+            request,
+            callback: callback_service_name,
+            raise_error_responses: raise_error_responses,
+            callback_args: callback_args,
+            sidekiq_job: sidekiq_job,
+            request_id: request_id
+          )
+        ensure
+          Sidekiq.external_storage.delete(ref_data) if ref_data
+        end
+      end
+    end
+  end
+end