patient_http-sidekiq 1.0.1 → 1.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bcb233e85930e0c27c9d950b3fee8621cb78d72534f128e2d13bb5072da6a95b
-  data.tar.gz: 86f1b667bb854573f4123d1d35895aea0d6159a21f8026c1313877a61ae93b7a
+  metadata.gz: f244dd52326602169e67b8471527b939c3e451fd11dcbcfe424305d5ad0929dd
+  data.tar.gz: 28b502c3aca097c48e364195e338e475d67f4f584209797054e2cc8ed75dda90
 SHA512:
-  metadata.gz: 69fc0eece8eb9075ee02ee3a8ef5422017f08c7bb06206b8309cb0174f5c28943ac489c0b9b58004524b301007f2e1ed6355e2dcd0c5a0e4d855eaf027e6d1cf
-  data.tar.gz: 2a24b218cd8a33687b141a0ed1a05e08e00e01da7d3fce9fe4b94bdb560c39a5d9d931caa8f3116ad63b03dc94089cac30a73e00ba34ac31443903ca115c25b2
+  metadata.gz: 12bb283a157d40689ffadfb5fdb58b74260374017409f99185a650f989d54da2ff858bbccac294fd6fe7095c0150cc024a0e15649280d0d1aaf5da04d2211b12
+  data.tar.gz: a27fbd9f52d110b05bb4177dd675642dde873c3a519078ea8c95f295c96f8be499e8fff119e79a8c0f446d1c6d608e537c374dceda993d98bcdf2081ef9e0a00

data/CHANGELOG.md

@@ -4,6 +4,19 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## 1.0.2
+
+### Fixed
+
+- Crash recovery now recovers requests from crashed processes. Previously a crashed process's ID remained in the process registry indefinitely, permanently excluding its in-flight requests from orphan detection unless the Sidekiq Web UI happened to prune it.
+- Fixed calls to nonexistent `Sidekiq.testing?` in error handlers, which would raise `NoMethodError` and permanently kill the task monitor thread on the first transient Redis error.
+- Externally stored request payloads are no longer deleted as soon as the request is submitted to the processor. They are now retained until the request completes so that Sidekiq retries, shutdown re-enqueues, and crash recovery can still fetch them. Dead `RequestWorker` jobs clean up their stored payloads when retries are exhausted.
+- `CallbackWorker` no longer deletes externally stored payloads when the callback raises, so Sidekiq retries can fetch the payload and re-run the callback.
+- Fixed dead job payload cleanup in `CallbackWorker`, which silently failed by calling a nonexistent method.
+- `PatientHttp::Sidekiq.configure` and `reset_configuration!` now reset the memoized external storage so it picks up the new configuration.
+- Heartbeat updates now refresh the TTL on the inflight tracking keys so they cannot expire while long-running requests are still in flight.
+- Fixed race conditions that could create duplicate processors or task monitor threads from concurrent lifecycle calls; starting while the processor is draining no longer replaces it.
+
 ## 1.0.1
 
 ### Fixed

data/VERSION

@@ -1 +1 @@
-1.0.1
+1.0.2

data/lib/patient_http/sidekiq/callback_worker.rb

@@ -53,7 +53,7 @@ module PatientHttp
         end
 
         begin
-          ExternalStorage.delete(data)
+          Sidekiq.external_storage.delete(data)
         rescue => e
           PatientHttp::Sidekiq.configuration.logger&.warn(
             "[PatientHttp::Sidekiq] Failed to delete stored payload for dead job: #{e.class.name} #{e.message}".strip
@@ -75,21 +75,22 @@ module PatientHttp
         actual_data = ref_data ? Sidekiq.external_storage.fetch(data) : data
         actual_data = Sidekiq.decrypt(actual_data)
 
-        begin
-          if result_type == "response"
-            response = PatientHttp::Response.load(actual_data)
-            PatientHttp::Sidekiq.invoke_completion_callbacks(response)
-            callback_service.on_complete(response)
-          elsif result_type == "error"
-            error = PatientHttp::Error.load(actual_data)
-            PatientHttp::Sidekiq.invoke_error_callbacks(error)
-            callback_service.on_error(error)
-          else
-            raise ArgumentError, "Unknown result_type: #{result_type}"
-          end
-        ensure
-          Sidekiq.external_storage.delete(ref_data) if ref_data
+        if result_type == "response"
+          response = PatientHttp::Response.load(actual_data)
+          PatientHttp::Sidekiq.invoke_completion_callbacks(response)
+          callback_service.on_complete(response)
+        elsif result_type == "error"
+          error = PatientHttp::Error.load(actual_data)
+          PatientHttp::Sidekiq.invoke_error_callbacks(error)
+          callback_service.on_error(error)
+        else
+          raise ArgumentError, "Unknown result_type: #{result_type}"
         end
+
+        # Only delete the stored payload after the callback succeeds so that
+        # Sidekiq retries can still fetch it. Failed jobs are cleaned up by
+        # the sidekiq_retries_exhausted hook.
+        Sidekiq.external_storage.delete(ref_data) if ref_data
       end
     end
   end

data/lib/patient_http/sidekiq/request_worker.rb

@@ -15,6 +15,17 @@ module PatientHttp
     class RequestWorker
       include ::Sidekiq::Job
 
+      # Clean up the externally stored request payload when the job exhausts all
+      # retries. The payload is normally deleted by TaskHandler when the request
+      # completes, so this only fires for requests that never made it that far.
+      sidekiq_retries_exhausted do |job, _exception|
+        Sidekiq.external_storage.delete(job["args"][0])
+      rescue => e
+        PatientHttp::Sidekiq.configuration.logger&.warn(
+          "[PatientHttp::Sidekiq] Failed to delete stored payload for dead job: #{e.class.name} #{e.message}".strip
+        )
+      end
+
       # Perform the HTTP request.
       #
       # @param data [Hash] Request data (possibly a storage reference) with keys:
@@ -26,31 +37,30 @@ module PatientHttp
       #   - "max_redirects" [Integer, nil] Maximum redirects to follow
       # @param callback_service_name [String] Fully qualified callback service class name
       # @param raise_error_responses [Boolean, nil] Whether to treat non-2xx responses as errors;
-      #   defaults to the global config if nil
+      #   nil is treated as false
       # @param callback_args [Hash, nil] Arguments to pass to the callback
       # @param request_id [String, nil] Unique request ID for tracking
       # @return [void]
       def perform(data, callback_service_name, raise_error_responses, callback_args, request_id)
         # Fetch from external storage if needed
-        ref_data = PatientHttp::ExternalStorage.storage_ref?(data) ? data : nil
-        actual_data = ref_data ? Sidekiq.external_storage.fetch(data) : data
+        actual_data = PatientHttp::ExternalStorage.storage_ref?(data) ? Sidekiq.external_storage.fetch(data) : data
         actual_data = Sidekiq.decrypt(actual_data)
 
         request = PatientHttp::Request.load(actual_data)
         sidekiq_job = Sidekiq::Context.current_job
 
-        begin
-          RequestExecutor.execute(
-            request,
-            callback: callback_service_name,
-            raise_error_responses: raise_error_responses,
-            callback_args: callback_args,
-            sidekiq_job: sidekiq_job,
-            request_id: request_id
-          )
-        ensure
-          Sidekiq.external_storage.delete(ref_data) if ref_data
-        end
+        # The stored payload must not be deleted here: this job hash is re-pushed
+        # for Sidekiq retries (e.g. MaxCapacityError), processor shutdown retries,
+        # and crash recovery, all of which need to fetch the payload again.
+        # TaskHandler deletes it when the request completes.
+        RequestExecutor.execute(
+          request,
+          callback: callback_service_name,
+          raise_error_responses: raise_error_responses,
+          callback_args: callback_args,
+          sidekiq_job: sidekiq_job,
+          request_id: request_id
+        )
       end
     end
   end

data/lib/patient_http/sidekiq/stats.rb

@@ -112,7 +112,7 @@ module PatientHttp
 
       def handle_error(error)
         @config&.logger&.error("[PatientHttp::Sidekiq] Stats error: #{error.inspect}")
-        raise error if ::Sidekiq.testing?
+        raise error if PatientHttp.testing?
       end
     end
   end

data/lib/patient_http/sidekiq/task_handler.rb

@@ -29,6 +29,7 @@ module PatientHttp
       def on_complete(response, callback)
         data = store_if_needed(response.as_json)
         CallbackWorker.perform_async(data, "response", callback)
+        delete_stored_request_payload
       end
 
       # Trigger the error callback with the error.
@@ -42,6 +43,7 @@ module PatientHttp
       def on_error(error, callback)
         data = store_if_needed(error.as_json)
         CallbackWorker.perform_async(data, "error", callback)
+        delete_stored_request_payload
       end
 
       # Re-enqueue the original Sidekiq job for retry.
@@ -67,6 +69,24 @@ module PatientHttp
 
       private
 
+      # Delete the externally stored request payload once the request has
+      # completed. Until then the payload must remain fetchable because the
+      # Sidekiq job hash referencing it can be re-pushed by Sidekiq retries,
+      # processor shutdown retries, and crash recovery. Only applies to
+      # RequestWorker jobs; other job types own their own arguments.
+      def delete_stored_request_payload
+        return unless @sidekiq_job["class"] == RequestWorker.name
+
+        data = @sidekiq_job["args"]&.first
+        return unless PatientHttp::ExternalStorage.storage_ref?(data)
+
+        PatientHttp::Sidekiq.external_storage.delete(data)
+      rescue => e
+        PatientHttp::Sidekiq.configuration.logger&.warn(
+          "[PatientHttp::Sidekiq] Failed to delete stored request payload: #{e.class.name} #{e.message}".strip
+        )
+      end
+
       def store_if_needed(data)
         encrypted = Sidekiq.encrypt(data)
         external_storage = PatientHttp::Sidekiq.external_storage

data/lib/patient_http/sidekiq/task_monitor.rb

@@ -226,6 +226,11 @@ module PatientHttp
             task_ids.each do |task_id|
               pipeline.call("ZADD", INFLIGHT_INDEX_KEY, "XX", timestamp_ms, full_task_id(task_id))
             end
+            # Keep the inflight keys alive while requests are still in flight;
+            # otherwise they only get their TTL refreshed when new requests
+            # are registered.
+            pipeline.call("EXPIRE", INFLIGHT_INDEX_KEY, inflight_ttl)
+            pipeline.call("EXPIRE", INFLIGHT_JOBS_KEY, inflight_ttl)
           end
         end
       end
@@ -401,10 +406,8 @@ module PatientHttp
         orphaned_request_ids_by_process = all_orphaned_request_ids.group_by do |request_id|
           request_id.split("/", 2).first
         end
-        all_process_ids = ::Sidekiq.redis do |redis|
-          redis.smembers(PROCESS_SET_KEY)
-        end
-        orphaned_request_ids = orphaned_request_ids_by_process.except(*all_process_ids).values.flatten
+        live_process_ids = prune_stale_processes(orphaned_request_ids_by_process.keys)
+        orphaned_request_ids = orphaned_request_ids_by_process.except(*live_process_ids).values.flatten
 
         return [] if orphaned_request_ids.empty?
 
@@ -416,6 +419,42 @@ module PatientHttp
         orphaned_request_ids.zip(job_payloads).reject { |_id, payload| payload.nil? }
       end
 
+      # Determine which of the given process IDs belong to live processes,
+      # removing dead ones from the process set.
+      #
+      # Membership in the process set alone doesn't prove liveness: a crashed
+      # process never removes itself from the set. A process is only considered
+      # live if its max_connections key (refreshed on every heartbeat with a
+      # short TTL) still exists. Stale members are removed from the set so
+      # their inflight requests can be recovered.
+      #
+      # @param process_ids [Array<String>] candidate process IDs
+      #
+      # @return [Array<String>] the subset of process IDs that are live
+      def prune_stale_processes(process_ids)
+        registered_ids = ::Sidekiq.redis do |redis|
+          redis.smembers(PROCESS_SET_KEY)
+        end
+        candidates = process_ids & registered_ids
+        return [] if candidates.empty?
+
+        max_connection_values = ::Sidekiq.redis do |redis|
+          redis.mget(*candidates.map { |process_id| max_connections_key_for(process_id) })
+        end
+
+        stale_process_ids, live_process_ids = candidates.zip(max_connection_values)
+          .partition { |_process_id, max_conn| max_conn.nil? }
+          .map { |pairs| pairs.map(&:first) }
+
+        unless stale_process_ids.empty?
+          ::Sidekiq.redis do |redis|
+            redis.srem(PROCESS_SET_KEY, stale_process_ids)
+          end
+        end
+
+        live_process_ids
+      end
+
       # Re-enqueue all orphaned jobs.
       #
       # @param orphaned_requests [Array<Array(String, String)>] array of [request_id, job_payload] pairs
@@ -503,7 +542,7 @@ module PatientHttp
       # @return [Integer] TTL in seconds
       def inflight_ttl
         # Set to 3x the orphan threshold, with a minimum of 1 hour
-        [config.orphan_threshold * 3, 3600].max
+        [config.orphan_threshold * 3, 3600].max.round
       end
 
       # Calculate the TTL for the garbage collection lock.
@@ -535,7 +574,11 @@ module PatientHttp
       end
 
       def max_connections_key
-        "#{PROCESS_SET_KEY}:#{@lock_identifier}:max_connections"
+        max_connections_key_for(@lock_identifier)
+      end
+
+      def max_connections_key_for(process_id)
+        "#{PROCESS_SET_KEY}:#{process_id}:max_connections"
       end
     end
   end

data/lib/patient_http/sidekiq/task_monitor_thread.rb

@@ -35,8 +35,7 @@ module PatientHttp
       #
       # @return [void]
       def start
-        return if @running.true?
-        @running.make_true
+        return unless @running.make_true
         @stop_signal.reset
 
         @task_monitor.ping_process
@@ -46,7 +45,7 @@ module PatientHttp
         rescue => e
           # Log error but don't crash
           @config.logger&.error("[PatientHttp::Sidekiq] Monitor error: #{e.message}\n#{e.backtrace.join("\n")}")
-          raise if ::Sidekiq.testing?
+          raise if PatientHttp.testing?
         end
 
         @thread.name = "patient-http-monitor"
@@ -120,7 +119,7 @@ module PatientHttp
         @config.logger&.debug("[PatientHttp::Sidekiq] Updated heartbeats for #{request_ids.size} inflight requests")
       rescue => e
         @config.logger&.error("[PatientHttp::Sidekiq] Failed to update heartbeats: #{e.class} - #{e.message}")
-        raise if ::Sidekiq.testing?
+        raise if PatientHttp.testing?
       end
 
       # Attempt to acquire GC lock and clean up orphaned requests.
@@ -147,7 +146,7 @@ module PatientHttp
         end
       rescue => e
         @config.logger&.error("[PatientHttp::Sidekiq] Garbage collection failed: #{e.class} - #{e.message}")
-        raise if ::Sidekiq.testing?
+        raise if PatientHttp.testing?
       end
     end
   end

data/lib/patient_http/sidekiq.rb

@@ -87,6 +87,7 @@ module PatientHttp
     @after_error_callbacks = []
     @external_storage = nil
     @request_handler = nil
+    @lifecycle_mutex = Mutex.new
 
     class << self
       attr_writer :configuration
@@ -97,8 +98,10 @@ module PatientHttp
       def configure
         configuration = Configuration.new
         yield(configuration) if block_given?
-        register_handler
         @configuration = configuration
+        @external_storage = nil
+        register_handler
+        configuration
       end
 
       # Ensure configuration is initialized
@@ -111,6 +114,7 @@ module PatientHttp
       # @return [Configuration]
       def reset_configuration!
         @configuration = nil
+        @external_storage = nil
         configuration
       end
 
@@ -234,6 +238,8 @@ module PatientHttp
 
       # Register Sidekiq as the request handler for processing HTTP requests. This is called
       # automatically when the processor starts or you call PatientHttp::Sidekiq.configure.
+      #
+      # @return [void]
       def register_handler
         @request_handler ||= lambda do |request:, callback:, raise_error_responses:, callback_args:|
           execute(
@@ -251,14 +257,16 @@ module PatientHttp
       #
       # @return [void]
       def start
-        return if running?
+        @lifecycle_mutex.synchronize do
+          return if @processor && !@processor.stopped?
 
-        @processor = PatientHttp::Processor.new(configuration)
-        @processor.observe(ProcessorObserver.new(@processor))
-        configuration.observers.each do |observer|
-          @processor.observe(observer)
+          @processor = PatientHttp::Processor.new(configuration)
+          @processor.observe(ProcessorObserver.new(@processor))
+          configuration.observers.each do |observer|
+            @processor.observe(observer)
+          end
+          @processor.start
         end
-        @processor.start
 
         register_handler
       end
@@ -267,9 +275,11 @@ module PatientHttp
       #
       # @return [void]
       def quiet
-        return unless running?
+        @lifecycle_mutex.synchronize do
+          return unless running?
 
-        @processor.drain
+          @processor.drain
+        end
       end
 
       # Stop the processor gracefully
@@ -281,10 +291,12 @@ module PatientHttp
           PatientHttp.unregister_handler(@request_handler)
         end
 
-        return unless @processor
+        @lifecycle_mutex.synchronize do
+          return unless @processor
 
-        @processor.stop(timeout: timeout)
-        @processor = nil
+          @processor.stop(timeout: timeout)
+          @processor = nil
+        end
       end
 
       # Reset all state (useful for testing)
@@ -292,8 +304,10 @@ module PatientHttp
       # @return [void]
       # @api private
       def reset!
-        @processor&.stop(timeout: 0)
-        @processor = nil
+        @lifecycle_mutex.synchronize do
+          @processor&.stop(timeout: 0)
+          @processor = nil
+        end
         @configuration = nil
         @external_storage = nil
         @after_completion_callbacks = []

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: patient_http-sidekiq
 version: !ruby/object:Gem::Version
-  version: 1.0.1
+  version: 1.0.2
 platform: ruby
 authors:
 - Brian Durand