patient_http-solid_queue 1.0.0

data/lib/patient_http/solid_queue/callback_job.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+module PatientHttp
+  module SolidQueue
+    # Active Job that invokes callback services for HTTP request results.
+    #
+    # Receives serialized Response or Error data and invokes the appropriate
+    # callback service method (+on_complete+ or +on_error+).
+    #
+    # @api private
+    class CallbackJob < ActiveJob::Base
+      # Clean up externally stored payloads when job exhausts all retries.
+      after_discard do |job, _exception|
+        data = job.arguments[0]
+        result_type = job.arguments[1]
+
+        begin
+          handler = PatientHttp::SolidQueue.configuration.on_retries_exhausted
+          if handler && result_type == "error"
+            actual_data = if PatientHttp::SolidQueue.external_storage.storage_ref?(data)
+              PatientHttp::SolidQueue.external_storage.fetch(data)
+            else
+              data
+            end
+            actual_data = PatientHttp::SolidQueue.decrypt(actual_data)
+            error = PatientHttp::Error.load(actual_data)
+            handler.call(error)
+          end
+        rescue => e
+          PatientHttp::SolidQueue.configuration.logger&.warn(
+            "[PatientHttp::SolidQueue] on_retries_exhausted handler failed: #{e.class.name} #{e.message}".strip
+          )
+        end
+
+        begin
+          PatientHttp::SolidQueue.external_storage.delete(data)
+        rescue => e
+          PatientHttp::SolidQueue.configuration.logger&.warn(
+            "[PatientHttp::SolidQueue] Failed to delete stored payload for dead job: #{e.class.name} #{e.message}".strip
+          )
+        end
+      end
+
+      # @param data [Hash] Response or Error data (possibly a storage reference)
+      # @param result_type [String] "response" or "error" indicating the type of result
+      # @param callback_service_name [String] Fully qualified callback service class name
+      def perform(data, result_type, callback_service_name)
+        callback_service_class = PatientHttp::ClassHelper.resolve_class_name(callback_service_name)
+        callback_service = callback_service_class.new
+
+        ref_data = PatientHttp::ExternalStorage.storage_ref?(data) ? data : nil
+        actual_data = ref_data ? PatientHttp::SolidQueue.external_storage.fetch(data) : data
+        actual_data = PatientHttp::SolidQueue.decrypt(actual_data)
+
+        begin
+          if result_type == "response"
+            response = PatientHttp::Response.load(actual_data)
+            PatientHttp::SolidQueue.invoke_completion_callbacks(response)
+            callback_service.on_complete(response)
+          elsif result_type == "error"
+            error = PatientHttp::Error.load(actual_data)
+            PatientHttp::SolidQueue.invoke_error_callbacks(error)
+            callback_service.on_error(error)
+          else
+            raise ArgumentError, "Unknown result_type: #{result_type}"
+          end
+        ensure
+          PatientHttp::SolidQueue.external_storage.delete(ref_data) if ref_data
+        end
+      end
+    end
+  end
+end

data/lib/patient_http/solid_queue/configuration.rb

@@ -0,0 +1,142 @@
+# frozen_string_literal: true
+
+module PatientHttp
+  module SolidQueue
+    # Configuration for the Solid Queue Async HTTP gem.
+    #
+    # Wraps PatientHttp::Configuration with Solid Queue-aware defaults and adds
+    # Solid Queue-specific options like queue name settings.
+    class Configuration < PatientHttp::Configuration
+      # Default threshold in bytes above which payloads are stored externally.
+      DEFAULT_PAYLOAD_STORE_THRESHOLD = 64 * 1024 # 64KB
+
+      # @return [Integer] Size threshold in bytes for external payload storage
+      attr_reader :payload_store_threshold
+
+      # @return [Numeric] Orphan detection threshold in seconds
+      attr_reader :orphan_threshold
+
+      # @return [Numeric] Heartbeat update interval in seconds
+      attr_reader :heartbeat_interval
+
+      # @return [String, nil] Queue name for RequestJob and CallbackJob
+      attr_reader :queue_name
+
+      # @return [#call, nil] Handler invoked when a CallbackWorker job exhausts all retries.
+      # @overload on_retries_exhausted
+      #   Returns the current handler.
+      #   @return [#call, nil]
+      # @overload on_retries_exhausted(&block)
+      #   Sets a block as the handler.
+      #   @yield [error] block to execute when retries are exhausted
+      #   @yieldparam error [PatientHttp::Error] information about the error
+      def on_retries_exhausted(&block)
+        if block
+          @on_retries_exhausted = block
+        else
+          @on_retries_exhausted
+        end
+      end
+
+      # Buffer in seconds subtracted from SolidQueue.shutdown_timeout to derive
+      # the default shutdown_timeout for this gem's connection pool.
+      SHUTDOWN_TIMEOUT_BUFFER = 2
+
+      # @param heartbeat_interval [Numeric] Interval in seconds for heartbeat updates (default: 60)
+      # @param orphan_threshold [Numeric] Time in seconds to consider a job orphaned (default: 300)
+      # @param queue_name [String, nil] Optional queue name for RequestJob and CallbackJob (default: nil)
+      # @param payload_store_threshold [Integer] Size threshold in bytes for external payload storage (default: 64KB)
+      # @param on_retries_exhausted [#call, nil] Handler called when a CallbackWorker job exhausts retries
+      # @param pool_options [Hash] Additional options passed to the SolidQueue connection pool
+      def initialize(
+        heartbeat_interval: 60,
+        orphan_threshold: 300,
+        queue_name: nil,
+        payload_store_threshold: DEFAULT_PAYLOAD_STORE_THRESHOLD,
+        on_retries_exhausted: nil,
+        **pool_options
+      )
+        if ::SolidQueue.shutdown_timeout
+          pool_options[:shutdown_timeout] ||= [::SolidQueue.shutdown_timeout - SHUTDOWN_TIMEOUT_BUFFER, 1].max
+        end
+        pool_options[:user_agent] ||= "SolidQueue-AsyncHttp"
+        pool_options[:logger] ||= (defined?(SolidQueue.logger) ? SolidQueue.logger : nil)
+
+        super(**pool_options)
+
+        self.queue_name = queue_name
+        self.heartbeat_interval = heartbeat_interval
+        self.orphan_threshold = orphan_threshold
+        self.payload_store_threshold = payload_store_threshold || DEFAULT_PAYLOAD_STORE_THRESHOLD
+        self.on_retries_exhausted = on_retries_exhausted
+      end
+
+      def payload_store_threshold=(value)
+        validate_positive_integer(:payload_store_threshold, value)
+        @payload_store_threshold = value
+      end
+
+      def heartbeat_interval=(value)
+        raise ArgumentError, "heartbeat_interval must be positive, got: #{value.inspect}" unless value.positive?
+        @heartbeat_interval = value
+        validate_heartbeat_and_threshold
+      end
+
+      def orphan_threshold=(value)
+        raise ArgumentError, "orphan_threshold must be positive, got: #{value.inspect}" unless value.positive?
+        @orphan_threshold = value
+        validate_heartbeat_and_threshold
+      end
+
+      def queue_name=(name)
+        if name.nil?
+          @queue_name = nil
+          return
+        end
+
+        raise ArgumentError, "queue_name must be a String, got: #{name.class}" unless name.is_a?(String)
+        @queue_name = name
+        apply_queue_name(name)
+      end
+
+      # Set the on_retries_exhausted handler.
+      #
+      # This handler is called when a CallbackWorker job exhausts all retries.
+      # It receives the same arguments as the on_error callback.
+      #
+      # @param value [#call, nil] A callable object or nil to clear the handler
+      # @raise [ArgumentError] If value is not callable and not nil
+      def on_retries_exhausted=(value)
+        if value && !value.respond_to?(:call)
+          raise ArgumentError.new("on_retries_exhausted must respond to #call, got: #{value.class}")
+        end
+
+        @on_retries_exhausted = value
+      end
+
+      # @return [Hash] configuration as a hash for logging/inspection
+      def to_h
+        super.merge(
+          "payload_store_threshold" => payload_store_threshold,
+          "heartbeat_interval" => heartbeat_interval,
+          "orphan_threshold" => orphan_threshold,
+          "queue_name" => queue_name,
+          "on_retries_exhausted" => on_retries_exhausted ? "defined" : nil
+        )
+      end
+
+      private
+
+      def apply_queue_name(name)
+        PatientHttp::SolidQueue::RequestJob.queue_as(name)
+        PatientHttp::SolidQueue::CallbackJob.queue_as(name)
+      end
+
+      def validate_heartbeat_and_threshold
+        return unless @heartbeat_interval && @orphan_threshold
+        return unless @heartbeat_interval >= @orphan_threshold
+        raise ArgumentError, "heartbeat_interval (#{@heartbeat_interval}) must be less than orphan_threshold (#{@orphan_threshold})"
+      end
+    end
+  end
+end

data/lib/patient_http/solid_queue/context.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module PatientHttp
+  module SolidQueue
+    # Provides thread-safe context for Active Jobs.
+    #
+    # Manages the current Active Job context using a thread-id keyed hash,
+    # allowing async HTTP requests to access job information without it being
+    # passed explicitly. Only RequestJob needs this context for re-enqueueing jobs.
+    class Context
+      @jobs = Concurrent::Map.new
+
+      class << self
+        # Returns the current job data hash for the running thread.
+        #
+        # @return [Hash, nil]
+        def current_job
+          @jobs[Thread.current.object_id]
+        end
+
+        # Set the current job context for the duration of a block.
+        #
+        # @param job_data [Hash] Active Job serialized hash
+        # @yield
+        def with_job(job_data)
+          thread_id = Thread.current.object_id
+          previous_job = @jobs[thread_id]
+          @jobs[thread_id] = job_data
+          yield
+        ensure
+          previous_job ? @jobs[thread_id] = previous_job : @jobs.delete(thread_id)
+        end
+      end
+    end
+  end
+end

data/lib/patient_http/solid_queue/engine.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module PatientHttp
+  module SolidQueue
+    # Rails Engine that makes gem migrations discoverable by the host application.
+    class Engine < ::Rails::Engine
+      engine_name "patient_http_solid_queue"
+
+      initializer "patient_http_solid_queue.migrations" do
+        config.paths["db/migrate"] << File.expand_path("../../../db/migrate", __dir__)
+      end
+    end
+  end
+end

data/lib/patient_http/solid_queue/gc_lock.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module PatientHttp
+  module SolidQueue
+    # Active Record model for distributed garbage collection locking.
+    #
+    # Ensures only one process runs orphan detection at a time. The last_gc_at
+    # column records when GC was last successfully completed, allowing processes
+    # to skip GC attempts if another process ran GC recently.
+    class GcLock < Record
+      self.table_name = "patient_http_solid_queue_gc_locks"
+    end
+  end
+end

data/lib/patient_http/solid_queue/inflight_request.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module PatientHttp
+  module SolidQueue
+    # Active Record model tracking inflight HTTP requests for crash recovery.
+    #
+    # Each record represents a single in-flight HTTP request. The heartbeat_at
+    # timestamp is updated periodically; stale records from dead processes are
+    # detected and re-enqueued by the GC mechanism.
+    class InflightRequest < Record
+      self.table_name = "patient_http_solid_queue_inflight_requests"
+    end
+  end
+end

data/lib/patient_http/solid_queue/lifecycle_hooks.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module PatientHttp
+  module SolidQueue
+    # Registers lifecycle hooks with SolidQueue to start/stop the async HTTP processor.
+    class LifecycleHooks
+      @registered = false
+
+      class << self
+        def register
+          return if @registered
+
+          ::SolidQueue.on_worker_start { PatientHttp::SolidQueue.start }
+          ::SolidQueue.on_worker_stop { PatientHttp::SolidQueue.stop }
+
+          @registered = true
+        end
+      end
+    end
+  end
+end

data/lib/patient_http/solid_queue/process_registration.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module PatientHttp
+  module SolidQueue
+    # Active Record model tracking registered async HTTP processor processes.
+    #
+    # Each record represents a running processor process. The last_seen_at
+    # timestamp is updated via heartbeats. Records for processes that are not
+    # in this table are considered orphaned during GC.
+    class ProcessRegistration < Record
+      self.table_name = "patient_http_solid_queue_processes"
+    end
+  end
+end

data/lib/patient_http/solid_queue/processor_observer.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module PatientHttp
+  module SolidQueue
+    # Processor Observer that monitors for crashed processes in order to
+    # re-enqueue workers, and manages the TaskMonitor lifecycle.
+    class ProcessorObserver < PatientHttp::ProcessorObserver
+      attr_reader :task_monitor
+
+      def initialize(processor)
+        @processor = processor
+        @task_monitor = TaskMonitor.new(processor.config)
+        @monitor_thread = TaskMonitorThread.new(
+          processor.config,
+          @task_monitor,
+          -> { @processor.inflight_request_ids }
+        )
+      end
+
+      def start
+        @monitor_thread.start
+      end
+
+      def stop
+        @monitor_thread.stop
+        task_monitor.remove_process
+      end
+
+      def request_start(request_task)
+        task_monitor.register(request_task)
+      end
+
+      def request_end(request_task)
+        task_monitor.unregister(request_task)
+      end
+    end
+  end
+end

data/lib/patient_http/solid_queue/record.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+module PatientHttp
+  module SolidQueue
+    # Base Active Record class for patient_http-solid_queue models.
+    class Record < ::SolidQueue::Record
+      self.abstract_class = true
+    end
+  end
+end

data/lib/patient_http/solid_queue/request_executor.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+module PatientHttp
+  module SolidQueue
+    # Helper methods for executing HTTP requests asynchronously.
+    class RequestExecutor
+      class << self
+        # Execute the request directly on the async processor.
+        #
+        # @param request [PatientHttp::Request] the HTTP request to execute
+        # @param callback [Class, String] Callback service class or its fully qualified class name
+        # @param active_job_data [Hash, nil] Active Job serialized hash with "job_class" and "arguments" keys
+        # @param synchronous [Boolean] If true, runs the request inline (for testing)
+        # @param callback_args [#to_h, nil] Arguments to pass to callback
+        # @param raise_error_responses [Boolean] If true, treats non-2xx responses as errors
+        # @param request_id [String, nil] Unique request ID for tracking
+        # @return [String] the request ID
+        # @api private
+        def execute(
+          request,
+          callback:,
+          active_job_data: nil,
+          synchronous: false,
+          callback_args: nil,
+          raise_error_responses: false,
+          request_id: nil
+        )
+          active_job_data = validate_active_job_data(active_job_data)
+          task_handler = TaskHandler.new(active_job_data)
+          config = PatientHttp::SolidQueue.configuration
+
+          task = PatientHttp::RequestTask.new(
+            request: request,
+            task_handler: task_handler,
+            callback: callback,
+            callback_args: callback_args,
+            raise_error_responses: raise_error_responses,
+            id: request_id,
+            default_max_redirects: config.max_redirects
+          )
+
+          if synchronous || async_disabled?
+            PatientHttp::SynchronousExecutor.new(
+              task,
+              config: config,
+              on_complete: ->(response) { PatientHttp::SolidQueue.invoke_completion_callbacks(response) },
+              on_error: ->(error) { PatientHttp::SolidQueue.invoke_error_callbacks(error) }
+            ).call
+            return task.id
+          end
+
+          processor = PatientHttp::SolidQueue.processor
+          unless processor&.running?
+            raise PatientHttp::NotRunningError, "Cannot enqueue request: processor is not running"
+          end
+
+          processor.enqueue(task)
+          task.id
+        end
+
+        private
+
+        def validate_active_job_data(active_job_data)
+          active_job_data ||= PatientHttp::SolidQueue::Context.current_job
+          raise ArgumentError, "active_job_data is required" if active_job_data.nil?
+          raise ArgumentError, "active_job_data must be a Hash, got: #{active_job_data.class}" unless active_job_data.is_a?(Hash)
+          raise ArgumentError, "active_job_data must have 'job_class' key" unless active_job_data.key?("job_class")
+          raise ArgumentError, "active_job_data must have 'arguments' array" unless active_job_data["arguments"].is_a?(Array)
+          active_job_data
+        end
+
+        def async_disabled?
+          PatientHttp.testing?
+        end
+      end
+    end
+  end
+end

data/lib/patient_http/solid_queue/request_job.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module PatientHttp
+  module SolidQueue
+    # Active Job that executes HTTP requests asynchronously.
+    #
+    # Enqueued when calling PatientHttp::SolidQueue.get, .post, etc.
+    # On completion, the specified callback service's on_complete or on_error is
+    # invoked via CallbackJob.
+    #
+    # @api private
+    class RequestJob < ActiveJob::Base
+      # Capture the Active Job serialized hash into Context so RequestExecutor can use it.
+      around_perform do |job, block|
+        PatientHttp::SolidQueue::Context.with_job(job.serialize) { block.call }
+      end
+
+      # @param data [Hash] Request data (possibly a storage reference)
+      # @param callback_service_name [String] Fully qualified callback service class name
+      # @param raise_error_responses [Boolean, nil] Whether to treat non-2xx responses as errors
+      # @param callback_args [Hash, nil] Arguments to pass to the callback
+      # @param request_id [String, nil] Unique request ID for tracking
+      def perform(data, callback_service_name, raise_error_responses, callback_args, request_id)
+        ref_data = PatientHttp::ExternalStorage.storage_ref?(data) ? data : nil
+        actual_data = ref_data ? PatientHttp::SolidQueue.external_storage.fetch(data) : data
+        actual_data = PatientHttp::SolidQueue.decrypt(actual_data)
+
+        request = PatientHttp::Request.load(actual_data)
+        active_job_data = PatientHttp::SolidQueue::Context.current_job
+
+        begin
+          RequestExecutor.execute(
+            request,
+            callback: callback_service_name,
+            raise_error_responses: raise_error_responses,
+            callback_args: callback_args,
+            active_job_data: active_job_data,
+            request_id: request_id
+          )
+        ensure
+          PatientHttp::SolidQueue.external_storage.delete(ref_data) if ref_data
+        end
+      end
+    end
+  end
+end

data/lib/patient_http/solid_queue/task_handler.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module PatientHttp
+  module SolidQueue
+    # Active Job implementation of TaskHandler.
+    #
+    # Handles task lifecycle operations using Active Job for job management:
+    # - Completion and error callbacks are triggered via CallbackJob
+    # - Large payloads are stored via ExternalStorage before enqueuing
+    # - Job retry uses ActiveJob::Base.deserialize
+    class TaskHandler < PatientHttp::TaskHandler
+      attr_reader :active_job_data
+
+      def initialize(active_job_data)
+        @active_job_data = active_job_data
+      end
+
+      def on_complete(response, callback)
+        data = store_if_needed(response.as_json)
+        CallbackJob.perform_later(data, "response", callback)
+      end
+
+      def on_error(error, callback)
+        data = store_if_needed(error.as_json)
+        CallbackJob.perform_later(data, "error", callback)
+      end
+
+      def retry
+        ActiveJob::Base.deserialize(@active_job_data).tap { |j| j.executions = 0 }.enqueue
+      end
+
+      def job_id
+        @active_job_data["job_id"]
+      end
+
+      def worker_class
+        PatientHttp::ClassHelper.resolve_class_name(@active_job_data["job_class"])
+      end
+
+      private
+
+      def store_if_needed(data)
+        encrypted = PatientHttp::SolidQueue.encrypt(data)
+        external_storage = PatientHttp::SolidQueue.external_storage
+        if external_storage.enabled?
+          external_storage.store(encrypted, max_size: PatientHttp::SolidQueue.configuration.payload_store_threshold)
+        else
+          encrypted
+        end
+      end
+    end
+  end
+end