patient_http-solid_queue 1.2.1 → 1.3.0

data/lib/patient_http/solid_queue.rb

@@ -3,40 +3,43 @@
 require "patient_http"
 require "solid_queue"
 
-# Main module for the Solid Queue Async HTTP gem.
-#
-# This gem provides a mechanism to offload long-running HTTP requests from Solid Queue workers
-# to a dedicated async I/O processor running in the same process, freeing worker threads
-# immediately while HTTP requests are in flight.
-#
-# == Usage
-#
-#   request = PatientHttp::Request.new(:get, "https://api.example.com/users/123")
-#   PatientHttp::SolidQueue.execute(
-#     request,
-#     callback: MyCallback,
-#     callback_args: {user_id: 123}
-#   )
-#
-# Define a callback service class with +on_complete+ and +on_error+ methods:
-#
-#   class MyCallback
-#     def on_complete(response)
-#       user_id = response.callback_args[:user_id]
-#       User.find(user_id).update!(data: response.json)
-#     end
-#
-#     def on_error(error)
-#       Rails.logger.error("Request failed: #{error.message}")
-#     end
-#   end
 module PatientHttp
+  # Runs HTTP requests from Solid Queue jobs on an async I/O processor.
+  #
+  # The processor runs in the Solid Queue worker process. Worker threads hand
+  # off long-running HTTP requests to it and are free to run other jobs while
+  # the requests are in flight.
+  #
+  # This module manages the processors for the current process. It starts one
+  # processor for each configured processor profile when a Solid Queue worker
+  # starts, and stops them when the worker stops. All processors in a process
+  # share one crash-recovery monitor.
+  #
+  # @example Make a request
+  #   PatientHttp.get(
+  #     "https://api.example.com/users/123",
+  #     callback: MyCallback,
+  #     callback_args: {user_id: 123}
+  #   )
+  #
+  # @example Define a callback service
+  #   class MyCallback
+  #     def on_complete(response)
+  #       user_id = response.callback_args[:user_id]
+  #       User.find(user_id).update!(data: response.json)
+  #     end
+  #
+  #     def on_error(error)
+  #       Rails.logger.error("Request failed: #{error.message}")
+  #     end
+  #   end
   module SolidQueue
+    # The gem version.
     VERSION = File.read(File.join(__dir__, "../../VERSION")).strip
 
-    # Raised when the crash-recovery registry entry for a request cannot be
-    # written. The request is rejected rather than accepted without a durable
-    # record, and the job retries.
+    # Raised when the crash recovery registry entry for a request can't be
+    # written. The processor rejects the request instead of accepting it
+    # without a durable record, and the job retries.
     class RegistrationError < StandardError; end
 
     autoload :CallbackJob, File.join(__dir__, "solid_queue/callback_job")
@@ -55,7 +58,6 @@ module PatientHttp
     autoload :TaskMonitorThread, File.join(__dir__, "solid_queue/task_monitor_thread")
 
     @processors = {}
-    @configuration = nil
     @after_completion_callbacks = []
     @after_error_callbacks = []
     @external_storage = nil
@@ -65,105 +67,185 @@ module PatientHttp
     @monitor_thread = nil
 
     class << self
-      attr_writer :configuration
+      # Sets the configuration. Intended for tests.
+      #
+      # `PatientHttp` stores the configuration, so this method assigns it there.
+      #
+      # @param config [Configuration, nil] The configuration, or `nil` to build a
+      #   new one on next use.
+      # @return [void]
+      def configuration=(config)
+        PatientHttp.default_configuration = config
+      end
 
-      # Configure the gem with a block. The built configuration is also set as the
-      # `PatientHttp.default_configuration` so that secrets registered at the module
-      # level with `PatientHttp.register_secret` are applied to the configuration the
-      # processor runs with, regardless of boot order.
+      # Yields the configuration to a block.
+      #
+      # Every call yields the same configuration object, so options accumulate.
+      # Several initializers can each set options without overwriting one
+      # another. `PatientHttp.configure` calls this method, so application code
+      # can use either one.
       #
-      # @yield [Configuration] the configuration object
-      # @return [Configuration]
+      # Configure the gem before the processors start. Running processors use
+      # this same configuration object, so an option changed while they run
+      # takes effect partway through the requests they're handling, and a
+      # processor profile declared while they run isn't started until the
+      # next restart. Changing the configuration while processors run logs a
+      # warning.
+      #
+      # @example
+      #   PatientHttp.configure do |config|
+      #     config.max_connections = 512
+      #   end
+      #
+      # @yield [config] The block that sets configuration options.
+      # @yieldparam config [Configuration] The configuration.
+      # @return [Configuration] The configuration.
       def configure
-        configuration = Configuration.new
-        yield(configuration) if block_given?
-        @configuration = configuration
-        @external_storage = nil
-        register_handler
-        PatientHttp.default_configuration = configuration
-        configuration
+        config = configuration
+        if block_given?
+          if running?
+            config.logger&.warn(
+              "[PatientHttp::SolidQueue] Configuration changed while processors are running; " \
+              "configure the gem before the Solid Queue worker starts."
+            )
+          end
+          yield(config)
+        end
+        config
       end
 
-      # Return the current configuration, initializing with defaults if necessary.
+      # Returns the configuration for this process, and creates it on first use.
       #
-      # @return [Configuration]
+      # `PatientHttp` stores the configuration, so this method and
+      # `PatientHttp.configuration` return the same object. As a result, secrets
+      # registered with `PatientHttp.register_secret` reach the configuration
+      # that the processors use, regardless of load order.
+      #
+      # @return [Configuration] The configuration.
       def configuration
-        @configuration ||= Configuration.new
+        PatientHttp.configuration
+      end
+
+      # Builds a new configuration. `PatientHttp` calls this method when it
+      # creates the configuration for this process.
+      #
+      # @return [Configuration] The new configuration.
+      # @api private
+      def new_configuration
+        Configuration.new
       end
 
-      # Reset configuration to defaults (useful for testing).
+      # Resets the configuration to the defaults. Intended for tests.
       #
-      # @return [Configuration]
+      # @return [Configuration] The new configuration.
       def reset_configuration!
-        @configuration = nil
-        @external_storage = nil
+        PatientHttp.default_configuration = nil
         configuration
       end
 
-      # Add a callback to be executed after a successful request completion.
+      # Registers a block to run after each request completes. Use it for
+      # monitoring. Blocks run in the order they're registered.
+      #
+      # @example
+      #   PatientHttp::SolidQueue.after_completion do |response|
+      #     StatsD.timing("patient_http.duration", response.duration * 1000)
+      #   end
       #
-      # @yield [response] block to execute after an HTTP request completes
-      # @yieldparam response [PatientHttp::Response] the HTTP response
+      # @yield [response] The block to run.
+      # @yieldparam response [PatientHttp::Response] The HTTP response.
+      # @return [void]
       def after_completion(&block)
         @after_completion_callbacks << block
       end
 
-      # Add a callback to be executed after a request error.
+      # Registers a block to run after each request error. Use it for
+      # monitoring. Blocks run in the order they're registered.
+      #
+      # @example
+      #   PatientHttp::SolidQueue.after_error do |error|
+      #     StatsD.increment("patient_http.error.#{error.error_type}")
+      #   end
       #
-      # @yield [error] block to execute after an HTTP request errors
-      # @yieldparam error [PatientHttp::Error] information about the error
+      # @yield [error] The block to run.
+      # @yieldparam error [PatientHttp::Error] The error.
+      # @return [void]
       def after_error(&block)
         @after_error_callbacks << block
       end
 
-      # Check if any processor is running.
+      # Returns whether any processor is running.
       #
-      # @return [Boolean]
+      # @return [Boolean] `true` if any processor is running.
       def running?
         @processors.values.any?(&:running?)
       end
 
-      # Check if any processor is draining (not accepting new requests).
+      # Returns whether any processor is draining. A draining processor doesn't
+      # accept new requests but continues to run in-flight requests.
       #
-      # @return [Boolean]
+      # @return [Boolean] `true` if any processor is draining.
       def draining?
         @processors.values.any?(&:draining?)
       end
 
-      # Check if any processor is stopping.
+      # Returns whether any processor is stopping.
       #
-      # @return [Boolean]
+      # @return [Boolean] `true` if any processor is stopping.
       def stopping?
         @processors.values.any?(&:stopping?)
       end
 
-      # Check if all processors are stopped or none have been started.
+      # Returns whether all processors are stopped.
       #
-      # @return [Boolean]
+      # @return [Boolean] `true` if all processors are stopped or none has
+      #   started.
       def stopped?
         @processors.values.all?(&:stopped?)
       end
 
-      # Get an ExternalStorage instance for storing and fetching payloads.
+      # Returns the external storage for request and result payloads. The
+      # storage is rebuilt when the configuration is replaced.
       #
-      # @return [PatientHttp::ExternalStorage]
+      # @return [PatientHttp::ExternalStorage] The external storage.
       # @api private
       def external_storage
-        @external_storage ||= PatientHttp::ExternalStorage.new(configuration)
+        config = configuration
+        storage = @external_storage
+        unless storage&.config.equal?(config)
+          storage = PatientHttp::ExternalStorage.new(config)
+          @external_storage = storage
+        end
+        storage
       end
 
-      # Execute an async HTTP request.
-      #
-      # @param request [PatientHttp::Request] the HTTP request to execute
-      # @param callback [Class, String] Callback service class with +on_complete+ and +on_error+
-      #   instance methods, or its fully qualified class name.
-      # @param callback_args [#to_h, nil] Arguments to pass to callback
-      # @param raise_error_responses [Boolean] If true, treats non-2xx responses as errors
-      # @param processor [Symbol, String, nil] Name of the processor profile that should
-      #   execute the request. Defaults to the request's own processor name or :default.
-      # @return [String] the request ID
-      # @raise [PatientHttp::UnknownProcessorError] if the processor profile is not configured
-      def execute(request, callback:, callback_args: nil, raise_error_responses: false, processor: nil)
+      # Runs an HTTP request asynchronously and calls the callback service with
+      # the result.
+      #
+      # Application code normally uses the `PatientHttp` module methods instead,
+      # such as `PatientHttp.get`, `PatientHttp.post`, or the
+      # PatientHttp::RequestHelper mixin. Those methods take the same options and
+      # keep application code independent of the job system. They call this
+      # method through the registered request handler.
+      #
+      # @param request [PatientHttp::Request] The HTTP request.
+      # @param callback [Class, String] The callback service class, or its fully
+      #   qualified name. The class must define `on_complete` and `on_error`
+      #   instance methods.
+      # @param callback_args [#to_h, nil] The arguments to pass to the callback.
+      #   Values must be JSON-native types: `nil`, `true`, `false`, String,
+      #   Integer, Float, Array, or Hash. Hash keys are converted to strings. The
+      #   callback reads the arguments from `response.callback_args` or
+      #   `error.callback_args` with symbol or string keys.
+      # @param raise_error_responses [Boolean, nil] Whether to treat non-2xx
+      #   responses as errors and call `on_error` instead of `on_complete`. If
+      #   `nil`, uses the `raise_error_responses` configuration option.
+      # @param processor [Symbol, String, nil] The name of the processor profile
+      #   that runs the request. If `nil`, uses the processor set on the request,
+      #   then `:default`.
+      # @return [String] The request ID.
+      # @raise [PatientHttp::UnknownProcessorError] If the processor profile
+      #   isn't configured.
+      def execute(request, callback:, callback_args: nil, raise_error_responses: nil, processor: nil)
         PatientHttp::CallbackValidator.validate!(callback)
         callback_name = callback.is_a?(Class) ? callback.name : callback.to_s
         callback_args = PatientHttp::CallbackValidator.validate_callback_args(callback_args)
@@ -173,14 +255,19 @@ module PatientHttp
         # Catch a misspelled profile name at the call site. A job that names an
         # unconfigured profile is retried instead, which covers rolling deploys
         # where the executing process is older than the enqueueing one.
-        unless configuration.processor_profiles.key?(processor_name.to_sym)
+        profile_config = processor_config_for(processor_name)
+        unless profile_config
           raise PatientHttp::UnknownProcessorError.new("No processor profile configured for #{processor_name.inspect}")
         end
 
+        # The PatientHttp module methods pass nil when the caller did not ask for a
+        # specific behavior, so fall back to the processor profile's setting.
+        raise_error_responses = profile_config.raise_error_responses if raise_error_responses.nil?
+
         encrypted = encrypt(request.as_json)
 
         data = if external_storage.enabled?
-          external_storage.store(encrypted, max_size: configuration.payload_store_threshold)
+          external_storage.store(encrypted, max_size: profile_config.payload_store_threshold)
         else
           encrypted
         end
@@ -190,8 +277,9 @@ module PatientHttp
         request_id
       end
 
-      # Start a processor for each configured processor profile, along with
-      # the shared crash-recovery monitor.
+      # Starts a processor for each configured processor profile. Also starts
+      # the crash-recovery monitor that the processors share. The Solid Queue
+      # lifecycle hooks call this method when a Solid Queue worker starts.
       #
       # @return [void]
       def start
@@ -226,7 +314,8 @@ module PatientHttp
         register_handler
       end
 
-      # Signal all processors to drain (stop accepting new requests).
+      # Drains all processors. A draining processor doesn't accept new requests
+      # but continues to run in-flight requests.
       #
       # @return [void]
       def quiet
@@ -237,14 +326,12 @@ module PatientHttp
         end
       end
 
-      # Stop all processors gracefully.
+      # Stops all processors and the services they share. The Solid Queue
+      # lifecycle hooks call this method when a Solid Queue worker stops.
       #
-      # The request handler stays registered. SolidQueue runs its worker stop hooks
-      # before the execution pool is drained, and a job that submits a request after
-      # the processors have stopped must have it enqueued as a job for the next
-      # process rather than raise. Only {.reset!} removes the handler.
-      #
-      # @param timeout [Float, nil] maximum time to wait for in-flight requests to complete
+      # @param timeout [Float, nil] The maximum number of seconds to wait for
+      #   in-flight requests to finish. If `nil`, uses the `shutdown_timeout`
+      #   configuration option.
       # @return [void]
       def stop(timeout: nil)
         @lifecycle_mutex.synchronize do
@@ -256,27 +343,32 @@ module PatientHttp
         end
       end
 
-      # Reset all state (useful for testing).
+      # Stops all processors and resets all state. Intended for tests.
       #
       # @return [void]
       # @api private
       def reset!
-        if @request_handler
-          PatientHttp.unregister_handler(@request_handler)
-        end
         @lifecycle_mutex.synchronize do
           stop_processors(timeout: 0)
           @processors = {}
           shutdown_shared_services
         end
-        @configuration = nil
         @external_storage = nil
         @after_completion_callbacks = []
         @after_error_callbacks = []
+        PatientHttp.default_configuration = nil
+        # Restore the state a freshly loaded process is in: the handler is
+        # registered, the configuration is not built yet.
+        register_handler
       end
 
-      # Register SolidQueue as the request handler for processing HTTP requests. This is called
-      # automatically when the processor starts or you call PatientHttp::SolidQueue.configure.
+      # Registers this gem as the request handler for `PatientHttp`.
+      #
+      # The gem calls this method when it loads. As a result, the `PatientHttp`
+      # module methods work in every process that loads the gem, whether or not
+      # the process runs a processor. The handler stays registered for the life
+      # of the process. After the processors stop, requests are enqueued in the
+      # queue database for another process to run.
       #
       # @return [void]
       def register_handler
@@ -292,9 +384,9 @@ module PatientHttp
         PatientHttp.register_handler(@request_handler)
       end
 
-      # Invoke the registered completion callbacks.
+      # Calls the blocks registered with {after_completion}.
       #
-      # @param response [PatientHttp::Response] the HTTP response
+      # @param response [PatientHttp::Response] The HTTP response.
       # @return [void]
       # @api private
       def invoke_completion_callbacks(response)
@@ -305,9 +397,9 @@ module PatientHttp
         end
       end
 
-      # Invoke the registered error callbacks.
+      # Calls the blocks registered with {after_error}.
       #
-      # @param error [PatientHttp::Error] information about the error
+      # @param error [PatientHttp::Error] The error.
       # @return [void]
       # @api private
       def invoke_error_callbacks(error)
@@ -318,34 +410,41 @@ module PatientHttp
         end
       end
 
-      # Encrypt a value using the configured encryptor.
+      # Encrypts data with the configured encryptor.
       #
-      # @param value [Object] the value to encrypt
-      # @return [String] the encrypted value
+      # @param value [Hash] The data to encrypt.
+      # @return [Hash] The encrypted data, or the original data if encryption
+      #   isn't configured.
+      # @api private
       def encrypt(value)
         configuration.encryptor.encrypt(value)
       end
 
-      # Decrypt a value using the configured encryptor.
+      # Decrypts data with the configured encryptor.
       #
-      # @param value [String] the encrypted value to decrypt
-      # @return [Object] the decrypted value
+      # @param value [Hash] The data to decrypt.
+      # @return [Hash] The decrypted data, or the original data if it isn't
+      #   encrypted.
+      # @api private
       def decrypt(value)
         configuration.encryptor.decrypt(value)
       end
 
-      # Returns a processor instance by name (internal accessor).
+      # Returns the processor with the given name.
       #
-      # @param name [Symbol, String] the processor name
-      # @return [PatientHttp::Processor, nil]
+      # @param name [Symbol, String] The processor name.
+      # @return [PatientHttp::Processor, nil] The processor, or `nil` if no
+      #   processor has that name.
       # @api private
       def processor(name = :default)
         @processors[name.to_sym]
       end
 
-      # Set the default processor (internal, for testing).
+      # Sets the default processor. Intended for tests.
       #
-      # @param value [PatientHttp::Processor, nil]
+      # @param value [PatientHttp::Processor, nil] The processor, or `nil` to
+      #   remove it.
+      # @return [void]
       # @api private
       def processor=(value)
         if value.nil?
@@ -355,24 +454,67 @@ module PatientHttp
         end
       end
 
+      # Returns the configuration for a processor profile. Uses the running
+      # processor's configuration if there is one.
+      #
+      # @param name [Symbol, String] The processor name.
+      # @return [PatientHttp::Configuration, nil] The configuration for the
+      #   profile, or `nil` if no processor has that name and the profile isn't
+      #   declared.
+      # @api private
+      def processor_config_for(name)
+        key = name.to_s
+        return nil if key.empty?
+
+        running = @processors[key.to_sym]
+        return running.config if running
+
+        config = configuration
+        config.processor_config(key) if config.processor_options(key)
+      end
+
       private
 
-      # Stop every processor, draining them at the same time so the timeout
-      # bounds the whole shutdown instead of each processor in turn.
+      # Stops every processor.
+      #
+      # Each processor waits up to the full timeout for its in-flight requests,
+      # so the processors stop in parallel. Stopping them one at a time would
+      # multiply the shutdown time by the number of processors. An error from
+      # one processor is logged so that the other processors and the shared
+      # services still shut down.
+      #
+      # @param timeout [Float, nil] The maximum number of seconds to wait for
+      #   in-flight requests.
+      # @return [void]
       def stop_processors(timeout:)
         processors = @processors.values
         return if processors.empty?
 
         if processors.one?
-          processors.first.stop(timeout: timeout)
+          stop_processor(processors.first, timeout)
         else
-          processors.map { |processor| Thread.new { processor.stop(timeout: timeout) } }.each(&:join)
+          processors.map { |processor| Thread.new { stop_processor(processor, timeout) } }.each(&:join)
         end
       end
 
-      # Stop the shared monitor thread and remove this process from the
-      # registry. Called with the lifecycle mutex held after all processors
-      # have stopped.
+      # Stops a processor and logs any error instead of raising it.
+      #
+      # @param processor [PatientHttp::Processor] The processor.
+      # @param timeout [Float, nil] The maximum number of seconds to wait for
+      #   in-flight requests.
+      # @return [void]
+      def stop_processor(processor, timeout)
+        processor.stop(timeout: timeout)
+      rescue => e
+        configuration.logger&.error(
+          "[PatientHttp::SolidQueue] Failed to stop processor #{processor.name}: #{e.inspect}"
+        )
+      end
+
+      # Stops the monitor thread and removes this process from the registry. The
+      # caller must hold the lifecycle mutex and must stop all processors first.
+      #
+      # @return [void]
       def shutdown_shared_services
         @monitor_thread&.stop
         @monitor_thread = nil
@@ -391,4 +533,14 @@ if defined?(::Rails::Engine)
   require_relative "solid_queue/engine"
 end
 
+# Wire the gem up as soon as it is loaded so that no setup step is required to
+# start making requests:
+#
+# - the request handler is registered, so PatientHttp.get and friends work in
+#   every process that requires the gem, configured or not;
+# - PatientHttp.configure and PatientHttp.configuration resolve to this gem's
+#   configuration, so applications never have to name the integration;
+# - the Solid Queue lifecycle hooks start and stop the processor with the worker.
+PatientHttp::SolidQueue.register_handler
+PatientHttp.register_configuration_provider(PatientHttp::SolidQueue)
 PatientHttp::SolidQueue::LifecycleHooks.register

data/patient_http-solid_queue.gemspec

@@ -35,8 +35,8 @@ Gem::Specification.new do |spec|
 
   spec.require_paths = ["lib"]
 
-  spec.required_ruby_version = ">= 3.2"
+  spec.required_ruby_version = ">= 3.3"
 
-  spec.add_dependency "patient_http", ">= 1.5.0"
+  spec.add_dependency "patient_http", ">= 1.7.0"
   spec.add_dependency "solid_queue", ">= 1.0.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: patient_http-solid_queue
 version: !ruby/object:Gem::Version
-  version: 1.2.1
+  version: 1.3.0
 platform: ruby
 authors:
 - Brian Durand
@@ -15,14 +15,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.5.0
+        version: 1.7.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.5.0
+        version: 1.7.0
 - !ruby/object:Gem::Dependency
   name: solid_queue
   requirement: !ruby/object:Gem::Requirement
@@ -48,7 +48,9 @@ files:
 - MIT-LICENSE
 - README.md
 - VERSION
-- db/migrate/20260216000000_create_solid_queue_async_http_tables.rb
+- db/migrate/20260216000000_create_patient_http_solid_queue_tables.rb
+- lib/generators/patient_http/solid_queue/install_generator.rb
+- lib/generators/patient_http/solid_queue/templates/initializer.rb
 - lib/patient_http-solid_queue.rb
 - lib/patient_http/solid_queue.rb
 - lib/patient_http/solid_queue/callback_job.rb
@@ -82,14 +84,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '3.2'
+      version: '3.3'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.6.9
+rubygems_version: 4.0.3
 specification_version: 4
 summary: Offload async HTTP requests from Solid Queue workers to a dedicated async
   I/O processor