patient_http-sidekiq 1.0.0

data/lib/patient_http/sidekiq.rb

@@ -0,0 +1,328 @@
+# frozen_string_literal: true
+
+require "sidekiq"
+require "patient_http"
+
+# This gem provides a mechanism to offload long-running HTTP requests from Sidekiq workers
+# to a dedicated async I/O processor running in the same process, freeing worker threads
+# immediately while HTTP requests are in flight.
+#
+# == Usage
+#
+# Make HTTP requests from anywhere in your code:
+#
+#   request = PatientHttp::Request.new(:get, "https://api.example.com/users/123")
+#   PatientHttp::Sidekiq.execute(
+#     request,
+#     callback: MyCallback,
+#     callback_args: {user_id: 123}
+#   )
+#
+# Define a callback service class with +on_complete+ and +on_error+ methods:
+#
+#   class MyCallback
+#     def on_complete(response)
+#       user_id = response.callback_args[:user_id]
+#       User.find(user_id).update!(data: response.json)
+#     end
+#
+#     def on_error(error)
+#       Rails.logger.error("Request failed: #{error.message}")
+#     end
+#   end
+#
+# == Key Features
+#
+# - Asynchronous HTTP processing using Ruby's Fiber scheduler
+# - Non-blocking worker threads
+# - Automatic connection pooling and HTTP/2 support
+# - Comprehensive error handling and retry logic
+# - Integration with Sidekiq's job lifecycle
+# - Optional Web UI for monitoring
+#
+# = Singleton Processor Pattern
+#
+# This module maintains a single Processor instance at the module level (@processor).
+# This is an intentional design decision driven by integration requirements with Sidekiq's
+# lifecycle and practical operational considerations:
+#
+# == Rationale:
+#
+# 1. **Sidekiq Integration**: The processor lifecycle (start/quiet/stop) must align with
+#    Sidekiq's own lifecycle hooks. A single processor instance integrates cleanly with
+#    Sidekiq's startup and shutdown signals.
+#
+# 2. **Resource Management**: Running multiple async I/O reactors in a single process would
+#    create resource contention and complexity. A single reactor efficiently handles all
+#    HTTP requests using connection pooling and fiber-based concurrency.
+#
+# 3. **Configuration Simplicity**: A singleton processor means one configuration, one set
+#    of metrics, and one connection pool. Multiple processors would require complex
+#    coordination and resource allocation.
+#
+# 4. **Process Model**: Sidekiq's process model (multiple workers, single process) maps
+#    naturally to a single async processor per process. Each Sidekiq process gets one
+#    processor, workers within that process share it.
+module PatientHttp
+  module Sidekiq
+    VERSION = File.read(File.expand_path("../../../VERSION", __FILE__)).strip
+
+    # Sidekiq-specific autoloads
+    autoload :CallbackWorker, File.join(__dir__, "sidekiq/callback_worker")
+    autoload :Configuration, File.join(__dir__, "sidekiq/configuration")
+    autoload :Context, File.join(__dir__, "sidekiq/context")
+    autoload :ProcessorObserver, File.join(__dir__, "sidekiq/processor_observer")
+    autoload :RequestExecutor, File.join(__dir__, "sidekiq/request_executor")
+    autoload :RequestWorker, File.join(__dir__, "sidekiq/request_worker")
+    autoload :LifecycleHooks, File.join(__dir__, "sidekiq/lifecycle_hooks")
+    autoload :TaskHandler, File.join(__dir__, "sidekiq/task_handler")
+    autoload :Stats, File.join(__dir__, "sidekiq/stats")
+    autoload :TaskMonitor, File.join(__dir__, "sidekiq/task_monitor")
+    autoload :TaskMonitorThread, File.join(__dir__, "sidekiq/task_monitor_thread")
+    autoload :WebUI, File.join(__dir__, "sidekiq/web_ui")
+
+    @processor = nil
+    @configuration = nil
+    @after_completion_callbacks = []
+    @after_error_callbacks = []
+    @external_storage = nil
+    @request_handler = nil
+
+    class << self
+      attr_writer :configuration
+
+      # Configure the gem with a block
+      # @yield [Configuration] the configuration object
+      # @return [Configuration]
+      def configure
+        configuration = Configuration.new
+        yield(configuration) if block_given?
+        @configuration = configuration
+      end
+
+      # Ensure configuration is initialized
+      # @return [Configuration]
+      def configuration
+        @configuration ||= Configuration.new
+      end
+
+      # Reset configuration to defaults (useful for testing)
+      # @return [Configuration]
+      def reset_configuration!
+        @configuration = nil
+        configuration
+      end
+
+      # Add a callback to be executed after a successful request completion.
+      #
+      # @yield [response] block to execute after an HTTP request completes
+      # @yieldparam response [PatientHttp::Response] the HTTP response
+      def after_completion(&block)
+        @after_completion_callbacks << block
+      end
+
+      # Add a callback to be executed after a request error.
+      #
+      # @yield [error] block to execute after an HTTP request errors
+      # @yieldparam error [PatientHttp::Error] information about the error that was raised
+      def after_error(&block)
+        @after_error_callbacks << block
+      end
+
+      # Add Sidekiq middleware for context handling. The middleware
+      # is already added during initialization. You can call this method again to
+      # append the middleware if needed to insert it after other middleware. If you need
+      # further control, you can manually add the `PatientHttp::Sidekiq::Context::Middleware`
+      # middleware yourself.
+      #
+      # @return [void]
+      def append_middleware
+        ::Sidekiq.configure_server do |config|
+          config.server_middleware do |chain|
+            chain.add PatientHttp::Sidekiq::Context::Middleware
+          end
+        end
+      end
+
+      # Check if the processor is running.
+      #
+      # @return [Boolean]
+      def running?
+        !!@processor&.running?
+      end
+
+      # Check if the processor is draining (not accepting new requests
+      # but still processing in-flight ones).
+      #
+      # @return [Boolean]
+      def draining?
+        !!@processor&.draining?
+      end
+
+      # Check if the processor is in the process of stopping.
+      #
+      # @return [Boolean]
+      def stopping?
+        !!@processor&.stopping?
+      end
+
+      # Check if the processor is stopped or has not been started.
+      #
+      # @return [Boolean]
+      def stopped?
+        @processor.nil? || @processor.stopped?
+      end
+
+      # Get an ExternalStorage instance for storing and fetching payloads.
+      #
+      # @return [PatientHttp::ExternalStorage]
+      # @api private
+      def external_storage
+        @external_storage ||= PatientHttp::ExternalStorage.new(configuration)
+      end
+
+      # Encrypt data using the configured encryptor.
+      #
+      # @param data [Hash] the data to encrypt
+      # @return [Hash] the encrypted data (or original if no encryption configured)
+      # @api private
+      def encrypt(data)
+        configuration.encryptor.encrypt(data)
+      end
+
+      # Decrypt data using the configured encryptor.
+      #
+      # @param data [Hash] the data to decrypt
+      # @return [Hash] the decrypted data (or original if not encrypted)
+      # @api private
+      def decrypt(data)
+        configuration.encryptor.decrypt(data)
+      end
+
+      # Execute an async HTTP request.
+      #
+      # @param request [PatientHttp::Request] the HTTP request to execute
+      # @param callback [Class, String] Callback service class with +on_complete+ and +on_error+
+      #   instance methods, or its fully qualified class name.
+      # @param callback_args [#to_h, nil] Arguments to pass to callback via the
+      #   PatientHttp::Response/PatientHttp::Error object. Must respond to +to_h+ and contain only JSON-native types
+      #   (nil, true, false, String, Integer, Float, Array, Hash). All hash keys will be
+      #   converted to strings for serialization. Access via +response.callback_args+ or
+      #   +error.callback_args+ using symbol or string keys.
+      # @param raise_error_responses [Boolean] If true, treats non-2xx responses as errors
+      #   and calls +on_error+ instead of +on_complete+. Defaults to false.
+      # @return [String] the request ID
+      def execute(request, callback:, callback_args: nil, raise_error_responses: false)
+        PatientHttp::CallbackValidator.validate!(callback)
+        callback_name = callback.is_a?(Class) ? callback.name : callback.to_s
+        callback_args = PatientHttp::CallbackValidator.validate_callback_args(callback_args)
+        request_id = SecureRandom.uuid
+
+        encrypted = encrypt(request.as_json)
+
+        data = if external_storage.enabled?
+          external_storage.store(encrypted, max_size: configuration.payload_store_threshold)
+        else
+          encrypted
+        end
+
+        RequestWorker.perform_async(data, callback_name, raise_error_responses, callback_args, request_id)
+
+        request_id
+      end
+
+      # Start the processor
+      #
+      # @return [void]
+      def start
+        return if running?
+
+        @processor = PatientHttp::Processor.new(configuration)
+        @processor.observe(ProcessorObserver.new(@processor))
+        configuration.observers.each do |observer|
+          @processor.observe(observer)
+        end
+        @processor.start
+
+        @request_handler ||= lambda do |request:, callback:, raise_error_responses:, callback_args:|
+          execute(
+            request,
+            callback: callback,
+            raise_error_responses: raise_error_responses,
+            callback_args: callback_args
+          )
+        end
+
+        PatientHttp.register_handler(@request_handler)
+      end
+
+      # Signal the processor to drain (stop accepting new requests)
+      #
+      # @return [void]
+      def quiet
+        return unless running?
+
+        @processor.drain
+      end
+
+      # Stop the processor gracefully
+      #
+      # @param timeout [Float, nil] maximum time to wait for in-flight requests to complete
+      # @return [void]
+      def stop(timeout: nil)
+        if @request_handler
+          PatientHttp.unregister_handler(@request_handler)
+        end
+
+        return unless @processor
+
+        @processor.stop(timeout: timeout)
+        @processor = nil
+      end
+
+      # Reset all state (useful for testing)
+      #
+      # @return [void]
+      # @api private
+      def reset!
+        @processor&.stop(timeout: 0)
+        @processor = nil
+        @configuration = nil
+        @external_storage = nil
+        @after_completion_callbacks = []
+        @after_error_callbacks = []
+        PatientHttp.unregister_handler(@request_handler) if @request_handler
+      end
+
+      # Invoke the registered completion callbacks
+      #
+      # @param response [PatientHttp::Response] the HTTP response
+      # @return [void]
+      # @api private
+      def invoke_completion_callbacks(response)
+        @after_completion_callbacks.each do |callback|
+          callback.call(response)
+        end
+      end
+
+      # Invoke the registered error callbacks
+      #
+      # @param error [PatientHttp::Error] information about the error that was raised
+      # @return [void]
+      # @api private
+      def invoke_error_callbacks(error)
+        @after_error_callbacks.each do |callback|
+          callback.call(error)
+        end
+      end
+
+      # Returns the processor instance (internal accessor)
+      #
+      # @return [PatientHttp::Processor, nil]
+      # @api private
+      attr_accessor :processor
+    end
+  end
+
+  PatientHttp::Sidekiq::LifecycleHooks.register
+end

data/lib/patient_http-sidekiq.rb

@@ -0,0 +1,3 @@
+# frozen_string_literal: true
+
+require_relative "patient_http/sidekiq"

data/patient_http-sidekiq.gemspec

@@ -0,0 +1,46 @@
+Gem::Specification.new do |spec|
+  spec.name = "patient_http-sidekiq"
+  spec.version = File.read(File.expand_path("../VERSION", __FILE__)).strip
+  spec.authors = ["Brian Durand"]
+  spec.email = ["bbdurand@gmail.com"]
+
+  spec.summary = "Offload long-running HTTP requests from Sidekiq workers to a dedicated async I/O processor"
+  spec.description = "This gem provides a mechanism to offload long-running HTTP requests from Sidekiq workers to a dedicated async I/O processor running in the same process, freeing the worker thread immediately while the HTTP request is in flight."
+
+  spec.homepage = "https://github.com/bdurand/patient_http-sidekiq"
+  spec.license = "MIT"
+
+  spec.metadata = {
+    "homepage_uri" => spec.homepage,
+    "source_code_uri" => spec.homepage,
+    "changelog_uri" => "#{spec.homepage}/blob/main/CHANGELOG.md"
+  }
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  ignore_files = %w[
+    .
+    AGENTS.md
+    Appraisals
+    Gemfile
+    Gemfile.lock
+    Rakefile
+    docker-compose.yml
+    bin/
+    gemfiles/
+    spec/
+    test_app/
+  ]
+  spec.files = Dir.chdir(File.expand_path("..", __FILE__)) do
+    `git ls-files -z`.split("\x0").reject { |f| ignore_files.any? { |path| f.start_with?(path) } }
+  end
+
+  spec.require_paths = ["lib"]
+
+  spec.required_ruby_version = ">= 3.2"
+
+  spec.add_dependency "sidekiq", ">= 7.0"
+  spec.add_dependency "patient_http"
+
+  spec.add_development_dependency "bundler"
+end

metadata

@@ -0,0 +1,140 @@
+--- !ruby/object:Gem::Specification
+name: patient_http-sidekiq
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- Brian Durand
+bindir: bin
+cert_chain: []
+date: 1980-01-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: sidekiq
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '7.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '7.0'
+- !ruby/object:Gem::Dependency
+  name: patient_http
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: This gem provides a mechanism to offload long-running HTTP requests from
+  Sidekiq workers to a dedicated async I/O processor running in the same process,
+  freeing the worker thread immediately while the HTTP request is in flight.
+email:
+- bbdurand@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ARCHITECTURE.md
+- CHANGELOG.md
+- MIT-LICENSE
+- README.md
+- VERSION
+- lib/patient_http-sidekiq.rb
+- lib/patient_http/sidekiq.rb
+- lib/patient_http/sidekiq/callback_worker.rb
+- lib/patient_http/sidekiq/configuration.rb
+- lib/patient_http/sidekiq/context.rb
+- lib/patient_http/sidekiq/lifecycle_hooks.rb
+- lib/patient_http/sidekiq/processor_observer.rb
+- lib/patient_http/sidekiq/request_executor.rb
+- lib/patient_http/sidekiq/request_worker.rb
+- lib/patient_http/sidekiq/stats.rb
+- lib/patient_http/sidekiq/task_handler.rb
+- lib/patient_http/sidekiq/task_monitor.rb
+- lib/patient_http/sidekiq/task_monitor_thread.rb
+- lib/patient_http/sidekiq/web_ui.rb
+- lib/patient_http/sidekiq/web_ui/assets/patient-http/css/patient_http.css
+- lib/patient_http/sidekiq/web_ui/locales/ar.yml
+- lib/patient_http/sidekiq/web_ui/locales/cs.yml
+- lib/patient_http/sidekiq/web_ui/locales/da.yml
+- lib/patient_http/sidekiq/web_ui/locales/de.yml
+- lib/patient_http/sidekiq/web_ui/locales/el.yml
+- lib/patient_http/sidekiq/web_ui/locales/en.yml
+- lib/patient_http/sidekiq/web_ui/locales/es.yml
+- lib/patient_http/sidekiq/web_ui/locales/fa.yml
+- lib/patient_http/sidekiq/web_ui/locales/fr.yml
+- lib/patient_http/sidekiq/web_ui/locales/gd.yml
+- lib/patient_http/sidekiq/web_ui/locales/he.yml
+- lib/patient_http/sidekiq/web_ui/locales/hi.yml
+- lib/patient_http/sidekiq/web_ui/locales/it.yml
+- lib/patient_http/sidekiq/web_ui/locales/ja.yml
+- lib/patient_http/sidekiq/web_ui/locales/ko.yml
+- lib/patient_http/sidekiq/web_ui/locales/lt.yml
+- lib/patient_http/sidekiq/web_ui/locales/nb.yml
+- lib/patient_http/sidekiq/web_ui/locales/nl.yml
+- lib/patient_http/sidekiq/web_ui/locales/pl.yml
+- lib/patient_http/sidekiq/web_ui/locales/pt-BR.yml
+- lib/patient_http/sidekiq/web_ui/locales/pt.yml
+- lib/patient_http/sidekiq/web_ui/locales/ru.yml
+- lib/patient_http/sidekiq/web_ui/locales/sv.yml
+- lib/patient_http/sidekiq/web_ui/locales/ta.yml
+- lib/patient_http/sidekiq/web_ui/locales/tr.yml
+- lib/patient_http/sidekiq/web_ui/locales/uk.yml
+- lib/patient_http/sidekiq/web_ui/locales/ur.yml
+- lib/patient_http/sidekiq/web_ui/locales/vi.yml
+- lib/patient_http/sidekiq/web_ui/locales/zh-CN.yml
+- lib/patient_http/sidekiq/web_ui/locales/zh-TW.yml
+- lib/patient_http/sidekiq/web_ui/views/patient_http.html.erb
+- patient_http-sidekiq.gemspec
+homepage: https://github.com/bdurand/patient_http-sidekiq
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://github.com/bdurand/patient_http-sidekiq
+  source_code_uri: https://github.com/bdurand/patient_http-sidekiq
+  changelog_uri: https://github.com/bdurand/patient_http-sidekiq/blob/main/CHANGELOG.md
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '3.2'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 4.0.3
+specification_version: 4
+summary: Offload long-running HTTP requests from Sidekiq workers to a dedicated async
+  I/O processor
+test_files: []