patient_http-solid_queue 1.0.0

data/lib/patient_http/solid_queue/task_monitor.rb

@@ -0,0 +1,260 @@
+# frozen_string_literal: true
+
+module PatientHttp
+  module SolidQueue
+    # Manages inflight request tracking in the database for crash recovery.
+    #
+    # This class maintains Active Record records for each in-flight request.
+    # It provides distributed locking for orphan detection and automatic
+    # re-enqueueing of requests interrupted by process crashes.
+    #
+    # Task ID format: "hostname:pid:hex/request-uuid"
+    # - hostname: sanitized hostname (colons and slashes replaced with dashes)
+    # - pid: process ID
+    # - hex: 8-character random hex for uniqueness
+    # - request-uuid: unique identifier for the request
+    class TaskMonitor
+      GC_LOCK_NAME = "gc"
+
+      # @return [Configuration] the configuration object
+      attr_reader :config
+
+      def initialize(config)
+        @config = config
+        hostname = ::Socket.gethostname.force_encoding("UTF-8").tr(":/", "-")
+        pid = ::Process.pid
+        @lock_identifier = "#{hostname}:#{pid}:#{SecureRandom.hex(8)}".freeze
+      end
+
+      # Register a request as inflight in the database.
+      #
+      # @param task [PatientHttp::RequestTask] the request task to register
+      # @return [void]
+      def register(task)
+        job_payload = task.task_handler.active_job_data.to_json
+        task_id = full_task_id(task.id)
+        now = Time.current
+
+        InflightRequest.create!(
+          task_id: task_id,
+          process_id: @lock_identifier,
+          job_payload: job_payload,
+          heartbeat_at: now,
+          created_at: now
+        )
+      rescue => e
+        @config.logger&.error("[PatientHttp::SolidQueue] Failed to register task #{task_id}: #{e.message}")
+        raise if PatientHttp.testing?
+      end
+
+      # Unregister a request from the database (called when request completes).
+      #
+      # @param task [PatientHttp::RequestTask] the request task to unregister
+      # @return [void]
+      def unregister(task)
+        task_id = full_task_id(task.id)
+        InflightRequest.where(task_id: task_id).delete_all
+      rescue => e
+        @config.logger&.error("[PatientHttp::SolidQueue] Failed to unregister task #{task_id}: #{e.message}")
+        raise if PatientHttp.testing?
+      end
+
+      # Update heartbeat timestamps for multiple requests in a single operation.
+      #
+      # @param task_ids [Array<String>] the request IDs to update
+      # @return [void]
+      def update_heartbeats(task_ids)
+        return if task_ids.empty?
+
+        full_ids = task_ids.map { |id| full_task_id(id) }
+        InflightRequest.where(task_id: full_ids).update_all(heartbeat_at: Time.current)
+      rescue => e
+        @config.logger&.error("[PatientHttp::SolidQueue] Failed to update heartbeats: #{e.message}")
+        raise if PatientHttp.testing?
+      end
+
+      # Record or refresh this process's registration.
+      #
+      # @return [void]
+      def ping_process
+        ProcessRegistration.upsert(
+          {process_id: @lock_identifier, max_connections: @config.max_connections, last_seen_at: Time.current},
+          unique_by: :process_id
+        )
+      rescue => e
+        @config.logger&.error("[PatientHttp::SolidQueue] Failed to ping process: #{e.message}")
+        raise if PatientHttp.testing?
+      end
+
+      # Remove this process's registration.
+      #
+      # @return [void]
+      def remove_process
+        ProcessRegistration.where(process_id: @lock_identifier).delete_all
+      rescue => e
+        @config.logger&.error("[PatientHttp::SolidQueue] Failed to remove process: #{e.message}")
+        raise if PatientHttp.testing?
+      end
+
+      # Try to acquire the distributed garbage collection lock.
+      #
+      # Uses a single semaphore row and pessimistic locking to ensure only one
+      # process can claim the lock at a time.
+      # Returns false if another process holds a non-expired lock, or if GC was
+      # run recently (within heartbeat_interval).
+      #
+      # @return [Boolean] true if lock acquired, false otherwise
+      def acquire_gc_lock
+        now = Time.current
+        expires_at = now + gc_lock_ttl.seconds
+        acquired = false
+
+        ensure_gc_lock_row!
+
+        GcLock.transaction do
+          lock = GcLock.lock.find_by!(lock_name: GC_LOCK_NAME)
+
+          recent_gc = lock.last_gc_at && lock.last_gc_at > (now - @config.heartbeat_interval)
+          next if recent_gc
+
+          lock_held = lock.lock_holder.present? && lock.expires_at.present? && lock.expires_at > now
+          next if lock_held
+
+          lock.update!(
+            lock_holder: @lock_identifier,
+            acquired_at: now,
+            expires_at: expires_at
+          )
+          acquired = true
+        end
+
+        acquired
+      rescue => e
+        @config.logger&.error("[PatientHttp::SolidQueue] Failed to acquire GC lock: #{e.message}")
+        raise if PatientHttp.testing?
+        false
+      end
+
+      # Release the garbage collection lock if held by this process, and record last_gc_at.
+      #
+      # @return [void]
+      def release_gc_lock
+        GcLock.where(lock_name: GC_LOCK_NAME, lock_holder: @lock_identifier)
+          .update_all(last_gc_at: Time.current, lock_holder: nil, acquired_at: nil, expires_at: nil)
+      rescue => e
+        @config.logger&.error("[PatientHttp::SolidQueue] Failed to release GC lock: #{e.message}")
+        raise if PatientHttp.testing?
+      end
+
+      # Find and re-enqueue orphaned requests.
+      #
+      # @param orphan_threshold_seconds [Numeric] age threshold for considering a request orphaned
+      # @param logger [Logger] logger for output
+      # @return [Integer] number of orphaned requests re-enqueued
+      def cleanup_orphaned_requests(orphan_threshold_seconds, logger)
+        threshold = Time.current - orphan_threshold_seconds.seconds
+
+        prune_stale_process_registrations(threshold)
+
+        # Get process IDs with a recent heartbeat
+        active_process_ids = ProcessRegistration.where("last_seen_at >= ?", threshold).pluck(:process_id)
+
+        # Find stale requests from processes not in the active set
+        orphaned = InflightRequest
+          .where("heartbeat_at < ?", threshold)
+          .where.not(process_id: active_process_ids)
+          .to_a
+
+        return 0 if orphaned.empty?
+
+        reenqueued_count = 0
+
+        orphaned.each do |record|
+          reenqueued_count += 1 if reenqueue_orphaned_record(record, threshold, logger)
+        end
+
+        reenqueued_count
+      end
+
+      # Build unique task ID for a request task that includes process identifier.
+      #
+      # @param task_id [String] the request task ID
+      # @return [String] the unique task ID
+      def full_task_id(task_id)
+        "#{@lock_identifier}/#{task_id}"
+      end
+
+      # Check if a task is registered in the inflight table.
+      #
+      # @param task [PatientHttp::RequestTask] the request task
+      # @return [Boolean]
+      # @api private
+      def registered?(task)
+        InflightRequest.where(task_id: full_task_id(task.id)).exists?
+      end
+
+      # Clear all records. Only allowed in test environment.
+      #
+      # @raise [RuntimeError] if called outside of test environment
+      # @return [void]
+      # @api private
+      def self.clear_all!
+        unless PatientHttp.testing?
+          raise "clear_all! is only allowed in test environment"
+        end
+
+        InflightRequest.delete_all
+        ProcessRegistration.delete_all
+        GcLock.delete_all
+      end
+
+      private
+
+      def ensure_gc_lock_row!
+        GcLock.insert_all([{lock_name: GC_LOCK_NAME}], unique_by: :lock_name)
+      end
+
+      # Re-enqueue a single orphaned record atomically.
+      #
+      # Uses a delete-by-exact-heartbeat to handle race conditions: if the
+      # heartbeat was updated between our read and the delete, the delete
+      # returns 0 rows and we skip re-enqueueing.
+      #
+      # @param record [InflightRequest] the orphaned record
+      # @param threshold [Float] heartbeat threshold (only records below this are orphaned)
+      # @param logger [Logger] logger for output
+      # @return [Boolean] true if successfully re-enqueued
+      def reenqueue_orphaned_record(record, threshold, logger)
+        # Atomically remove only if still orphaned (heartbeat unchanged)
+        deleted = InflightRequest
+          .where(task_id: record.task_id, heartbeat_at: record.heartbeat_at)
+          .where("heartbeat_at < ?", threshold)
+          .delete_all
+
+        return false if deleted == 0
+
+        job_data = JSON.parse(record.job_payload)
+        ActiveJob::Base.deserialize(job_data).tap { |j| j.executions = 0 }.enqueue
+
+        logger&.info(
+          "[PatientHttp::SolidQueue] Re-enqueued orphaned request #{record.task_id} to #{job_data["job_class"]}"
+        )
+
+        true
+      rescue => e
+        logger&.error(
+          "[PatientHttp::SolidQueue] Failed to re-enqueue orphaned request #{record.task_id}: #{e.class} - #{e.message}"
+        )
+        false
+      end
+
+      def gc_lock_ttl
+        [@config.heartbeat_interval * 2, 120].max
+      end
+
+      def prune_stale_process_registrations(threshold)
+        ProcessRegistration.where("last_seen_at < ?", threshold).delete_all
+      end
+    end
+  end
+end

data/lib/patient_http/solid_queue/task_monitor_thread.rb

@@ -0,0 +1,133 @@
+# frozen_string_literal: true
+
+module PatientHttp
+  module SolidQueue
+    # Background thread that maintains heartbeats and performs garbage collection
+    # for in-flight HTTP requests.
+    class TaskMonitorThread
+      include PatientHttp::TimeHelper
+
+      # Maximum seconds to sleep between monitor thread checks.
+      MAX_MONITOR_SLEEP = 5.0
+
+      # @return [Configuration] the configuration object
+      attr_reader :config
+
+      # @return [TaskMonitor] the inflight request registry
+      attr_reader :task_monitor
+
+      # Initialize the monitor thread.
+      #
+      # @param config [Configuration] the configuration object
+      # @param task_monitor [TaskMonitor] the inflight request registry
+      # @param inflight_ids_callback [Proc] callback to get current inflight request IDs
+      def initialize(config, task_monitor, inflight_ids_callback)
+        @config = config
+        @task_monitor = task_monitor
+        @inflight_ids_callback = inflight_ids_callback
+        @thread = nil
+        @running = Concurrent::AtomicBoolean.new(false)
+        @stop_signal = Concurrent::Event.new
+      end
+
+      # Start the monitor thread.
+      #
+      # @return [void]
+      def start
+        return if @running.true?
+        @running.make_true
+        @stop_signal.reset
+
+        @task_monitor.ping_process
+
+        @thread = Thread.new do
+          run
+        rescue => e
+          @config.logger&.error("[PatientHttp::SolidQueue] Monitor error: #{e.message}\n#{e.backtrace.join("\n")}")
+          raise if PatientHttp.testing?
+        end
+
+        @thread.name = "async-http-monitor"
+      end
+
+      # Stop the monitor thread.
+      #
+      # @return [void]
+      def stop
+        @running.make_false
+        @stop_signal.set
+        @thread&.join(1)
+        @thread&.kill if @thread&.alive?
+        @thread = nil
+      end
+
+      # Check if monitor thread is running.
+      #
+      # @return [Boolean]
+      def running?
+        @running.true?
+      end
+
+      private
+
+      def run
+        @config.logger&.info("[PatientHttp::SolidQueue] Monitor thread started")
+
+        last_heartbeat_update = monotonic_time - @config.heartbeat_interval
+        last_gc_attempt = monotonic_time - @config.heartbeat_interval
+
+        loop do
+          break unless @running.true?
+
+          current_time = monotonic_time
+
+          if current_time - last_heartbeat_update >= @config.heartbeat_interval
+            @task_monitor.ping_process
+            update_heartbeats
+            last_heartbeat_update = current_time
+          end
+
+          if current_time - last_gc_attempt >= @config.heartbeat_interval
+            attempt_garbage_collection
+            last_gc_attempt = current_time
+          end
+
+          wait_time = @config.heartbeat_interval / 2.0
+          wait_time = MAX_MONITOR_SLEEP if wait_time > MAX_MONITOR_SLEEP
+          @stop_signal.wait(wait_time)
+        end
+
+        @config.logger&.info("[PatientHttp::SolidQueue] Monitor thread stopped")
+      end
+
+      def update_heartbeats
+        request_ids = @inflight_ids_callback.call
+        return if request_ids.empty?
+
+        @task_monitor.update_heartbeats(request_ids)
+
+        @config.logger&.debug("[PatientHttp::SolidQueue] Updated heartbeats for #{request_ids.size} inflight requests")
+      rescue => e
+        @config.logger&.error("[PatientHttp::SolidQueue] Failed to update heartbeats: #{e.class} - #{e.message}")
+        raise if PatientHttp.testing?
+      end
+
+      def attempt_garbage_collection
+        return unless @task_monitor.acquire_gc_lock
+
+        begin
+          count = @task_monitor.cleanup_orphaned_requests(@config.orphan_threshold, @config.logger)
+
+          if count > 0
+            @config.logger&.info("[PatientHttp::SolidQueue] Garbage collection: re-enqueued #{count} orphaned requests")
+          end
+        ensure
+          @task_monitor.release_gc_lock
+        end
+      rescue => e
+        @config.logger&.error("[PatientHttp::SolidQueue] Garbage collection failed: #{e.class} - #{e.message}")
+        raise if PatientHttp.testing?
+      end
+    end
+  end
+end

data/lib/patient_http/solid_queue.rb

@@ -0,0 +1,284 @@
+# frozen_string_literal: true
+
+require "patient_http"
+require "solid_queue"
+
+# Main module for the Solid Queue Async HTTP gem.
+#
+# This gem provides a mechanism to offload long-running HTTP requests from Solid Queue workers
+# to a dedicated async I/O processor running in the same process, freeing worker threads
+# immediately while HTTP requests are in flight.
+#
+# == Usage
+#
+#   request = PatientHttp::Request.new(:get, "https://api.example.com/users/123")
+#   PatientHttp::SolidQueue.execute(
+#     request,
+#     callback: MyCallback,
+#     callback_args: {user_id: 123}
+#   )
+#
+# Define a callback service class with +on_complete+ and +on_error+ methods:
+#
+#   class MyCallback
+#     def on_complete(response)
+#       user_id = response.callback_args[:user_id]
+#       User.find(user_id).update!(data: response.json)
+#     end
+#
+#     def on_error(error)
+#       Rails.logger.error("Request failed: #{error.message}")
+#     end
+#   end
+module PatientHttp
+  module SolidQueue
+    VERSION = File.read(File.join(__dir__, "../../VERSION")).strip
+
+    autoload :CallbackJob, File.join(__dir__, "solid_queue/callback_job")
+    autoload :Configuration, File.join(__dir__, "solid_queue/configuration")
+    autoload :Context, File.join(__dir__, "solid_queue/context")
+    autoload :GcLock, File.join(__dir__, "solid_queue/gc_lock")
+    autoload :InflightRequest, File.join(__dir__, "solid_queue/inflight_request")
+    autoload :ProcessorObserver, File.join(__dir__, "solid_queue/processor_observer")
+    autoload :ProcessRegistration, File.join(__dir__, "solid_queue/process_registration")
+    autoload :Record, File.join(__dir__, "solid_queue/record")
+    autoload :RequestExecutor, File.join(__dir__, "solid_queue/request_executor")
+    autoload :RequestJob, File.join(__dir__, "solid_queue/request_job")
+    autoload :LifecycleHooks, File.join(__dir__, "solid_queue/lifecycle_hooks")
+    autoload :TaskHandler, File.join(__dir__, "solid_queue/task_handler")
+    autoload :TaskMonitor, File.join(__dir__, "solid_queue/task_monitor")
+    autoload :TaskMonitorThread, File.join(__dir__, "solid_queue/task_monitor_thread")
+
+    @processor = nil
+    @configuration = nil
+    @after_completion_callbacks = []
+    @after_error_callbacks = []
+    @external_storage = nil
+    @request_handler = nil
+
+    class << self
+      attr_writer :configuration
+
+      # Configure the gem with a block.
+      #
+      # @yield [Configuration] the configuration object
+      # @return [Configuration]
+      def configure
+        configuration = Configuration.new
+        yield(configuration) if block_given?
+        @configuration = configuration
+      end
+
+      # Return the current configuration, initializing with defaults if necessary.
+      #
+      # @return [Configuration]
+      def configuration
+        @configuration ||= Configuration.new
+      end
+
+      # Reset configuration to defaults (useful for testing).
+      #
+      # @return [Configuration]
+      def reset_configuration!
+        @configuration = nil
+        configuration
+      end
+
+      # Add a callback to be executed after a successful request completion.
+      #
+      # @yield [response] block to execute after an HTTP request completes
+      # @yieldparam response [PatientHttp::Response] the HTTP response
+      def after_completion(&block)
+        @after_completion_callbacks << block
+      end
+
+      # Add a callback to be executed after a request error.
+      #
+      # @yield [error] block to execute after an HTTP request errors
+      # @yieldparam error [PatientHttp::Error] information about the error
+      def after_error(&block)
+        @after_error_callbacks << block
+      end
+
+      # Check if the processor is running.
+      #
+      # @return [Boolean]
+      def running?
+        !!@processor&.running?
+      end
+
+      # Check if the processor is draining (not accepting new requests).
+      #
+      # @return [Boolean]
+      def draining?
+        !!@processor&.draining?
+      end
+
+      # Check if the processor is stopping.
+      #
+      # @return [Boolean]
+      def stopping?
+        !!@processor&.stopping?
+      end
+
+      # Check if the processor is stopped.
+      #
+      # @return [Boolean]
+      def stopped?
+        @processor.nil? || @processor.stopped?
+      end
+
+      # Get an ExternalStorage instance for storing and fetching payloads.
+      #
+      # @return [PatientHttp::ExternalStorage]
+      # @api private
+      def external_storage
+        @external_storage ||= PatientHttp::ExternalStorage.new(configuration)
+      end
+
+      # Execute an async HTTP request.
+      #
+      # @param request [PatientHttp::Request] the HTTP request to execute
+      # @param callback [Class, String] Callback service class with +on_complete+ and +on_error+
+      #   instance methods, or its fully qualified class name.
+      # @param callback_args [#to_h, nil] Arguments to pass to callback
+      # @param raise_error_responses [Boolean] If true, treats non-2xx responses as errors
+      # @return [String] the request ID
+      def execute(request, callback:, callback_args: nil, raise_error_responses: false)
+        PatientHttp::CallbackValidator.validate!(callback)
+        callback_name = callback.is_a?(Class) ? callback.name : callback.to_s
+        callback_args = PatientHttp::CallbackValidator.validate_callback_args(callback_args)
+        request_id = SecureRandom.uuid
+
+        encrypted = encrypt(request.as_json)
+
+        data = if external_storage.enabled?
+          external_storage.store(encrypted, max_size: configuration.payload_store_threshold)
+        else
+          encrypted
+        end
+
+        RequestJob.perform_later(data, callback_name, raise_error_responses, callback_args, request_id)
+
+        request_id
+      end
+
+      # Start the processor.
+      #
+      # @return [void]
+      def start
+        return if running?
+
+        @processor = PatientHttp::Processor.new(configuration)
+        @processor.observe(ProcessorObserver.new(@processor))
+        @processor.start
+
+        @request_handler ||= lambda do |request:, callback:, raise_error_responses:, callback_args:|
+          execute(
+            request,
+            callback: callback,
+            raise_error_responses: raise_error_responses,
+            callback_args: callback_args
+          )
+        end
+
+        PatientHttp.register_handler(@request_handler)
+      end
+
+      # Signal the processor to drain (stop accepting new requests).
+      #
+      # @return [void]
+      def quiet
+        return unless running?
+
+        @processor.drain
+      end
+
+      # Stop the processor gracefully.
+      #
+      # @param timeout [Float, nil] maximum time to wait for in-flight requests to complete
+      # @return [void]
+      def stop(timeout: nil)
+        return unless @processor
+
+        if @request_handler
+          PatientHttp.unregister_handler(@request_handler)
+        end
+
+        @processor.stop(timeout: timeout)
+        @processor = nil
+      end
+
+      # Reset all state (useful for testing).
+      #
+      # @return [void]
+      # @api private
+      def reset!
+        if @request_handler
+          PatientHttp.unregister_handler(@request_handler)
+          @request_handler = nil
+        end
+        @processor&.stop(timeout: 0)
+        @processor = nil
+        @configuration = nil
+        @external_storage = nil
+        @after_completion_callbacks = []
+        @after_error_callbacks = []
+      end
+
+      # Invoke the registered completion callbacks.
+      #
+      # @param response [PatientHttp::Response] the HTTP response
+      # @return [void]
+      # @api private
+      def invoke_completion_callbacks(response)
+        @after_completion_callbacks.each do |callback|
+          callback.call(response)
+        rescue => e
+          configuration.logger&.error("[PatientHttp::SolidQueue] after_completion callback error: #{e.class} - #{e.message}")
+        end
+      end
+
+      # Invoke the registered error callbacks.
+      #
+      # @param error [PatientHttp::Error] information about the error
+      # @return [void]
+      # @api private
+      def invoke_error_callbacks(error)
+        @after_error_callbacks.each do |callback|
+          callback.call(error)
+        rescue => e
+          configuration.logger&.error("[PatientHttp::SolidQueue] after_error callback error: #{e.class} - #{e.message}")
+        end
+      end
+
+      # Encrypt a value using the configured encryptor.
+      #
+      # @param value [Object] the value to encrypt
+      # @return [String] the encrypted value
+      def encrypt(value)
+        configuration.encryptor.encrypt(value)
+      end
+
+      # Decrypt a value using the configured encryptor.
+      #
+      # @param value [String] the encrypted value to decrypt
+      # @return [Object] the decrypted value
+      def decrypt(value)
+        configuration.encryptor.decrypt(value)
+      end
+
+      # Returns the processor instance.
+      #
+      # @return [PatientHttp::Processor, nil]
+      # @api private
+      attr_accessor :processor
+    end
+  end
+end
+
+if defined?(::Rails::Engine)
+  require_relative "solid_queue/engine"
+end
+
+PatientHttp::SolidQueue::LifecycleHooks.register

data/lib/patient_http-solid_queue.rb

@@ -0,0 +1,3 @@
+# frozen_string_literal: true
+
+require_relative "patient_http/solid_queue"

data/lib/tasks/patient_http_solid_queue.rake

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+if Rake::Task.task_defined?("patient_http_solid_queue:install:migrations")
+  Rake::Task["patient_http_solid_queue:install:migrations"].clear
+end
+
+namespace :patient_http_solid_queue do
+  namespace :install do
+    desc "Copy migrations from patient_http_solid_queue to the queue database migration path"
+    task migrations: :"db:load_config" do
+      ENV["FROM"] = "patient_http_solid_queue"
+      ENV["DATABASE"] = "queue" if ENV["DATABASE"].nil? || ENV["DATABASE"].empty?
+
+      if Rake::Task.task_defined?("railties:install:migrations")
+        Rake::Task["railties:install:migrations"].invoke
+      else
+        Rake::Task["app:railties:install:migrations"].invoke
+      end
+    end
+  end
+end