dispatch-rails 0.8.1 → 0.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: eb09ad358ca9afa12624a37603693b0892b3816a1733852a6e5bddec3f7e0ad8
-  data.tar.gz: 6d877756e23368e5bc033fb4696709b464eb957a1b76a3106e37e1c876cb91db
+  metadata.gz: 797a73d62a39b0ab3640364d8ef71186b61f3233da2788f7733b72c85e77cc75
+  data.tar.gz: 235fa20b827e25a6f0537e20d3cdd51221c0bebbed18013046901c9bdabf51ed
 SHA512:
-  metadata.gz: 02e6876ac14a61b0215ae79eb52bf47fe632e3dd3ed676c9706921552e1fc73d3a208f21b07047484b9ee8c3e9a5aa9eb58d018b83f17bf80ce6c0609ac95ebe
-  data.tar.gz: c22f0667ea5fa682f27f2273f1c3630cc54ffa657694830342dea94b7f5e6c693492b0e82cb006438c7373a2cf6be1a7be3e16203f5927440a9ebb5f54963a24
+  metadata.gz: f3cbf75d09473580ec5f66ffaaf7d7623631c1a453bbe9b67b86e091e4fbcb56cd7da1ea141080b6375b714802915c00c4fd9f64d7c649bb2de7e6d6b46d1c0b
+  data.tar.gz: f24f36ca5d2769053faa65cb5b77da6e09a28aea5b4e3166bcbb64e2473729cd9b54df4d6936572fe3ecb41937cb5e43e32f0b9de24f2c27b1879532da0232de

data/README.md

@@ -98,6 +98,40 @@ c.capture_at_exit  = true  # default; set false to skip the crash-at-exit report
 c.shutdown_timeout = 3     # seconds to wait for the send queue to drain at exit; 0 skips
 ```
 
+### SolidQueue infrastructure failures
+
+Errors raised **inside** a job's `perform` are reported automatically (ActiveJob
+sends them through `Rails.error`, which the SDK subscribes to). But SolidQueue's
+own *infrastructure* failures never touch that path — SolidQueue writes them
+straight into its own tables, so they don't reach `Rails.error` and stay
+invisible in the dashboard. The classic case: a container runs out of memory,
+Heroku SIGKILLs the worker, and the supervisor later prunes the dead process and
+force-fails the jobs it had claimed with `SolidQueue::Processes::ProcessPrunedError`
+— a failure you'd only find by opening the queue.
+
+When SolidQueue is present, the SDK subscribes to its `ActiveSupport::Notifications`
+and turns these into first-class events (all tagged `source: solid_queue`):
+
+- **Lost jobs** (`fail_many_claimed`) — claimed jobs force-failed because their
+  worker died (pruned after a missed heartbeat, or orphaned by a dead process).
+  Reported as `Dispatch::Rails::SolidQueueJobsLost` with `tags.jobs_lost`,
+  `tags.process_ids`, and `tags.job_ids`. **This is the OOM-restart case.**
+- **Thread crashes** (`thread_error`) — an exception that escaped a SolidQueue
+  thread (supervisor/dispatcher/worker/scheduler), reported with its real class
+  and backtrace. Captured even if your app overrides `SolidQueue.on_thread_error`
+  away from the default `Rails.error.report` (deduped against it when it isn't).
+- **Recurring-enqueue misses** (`enqueue_recurring_task`) — a cron task that
+  failed to enqueue, as `Dispatch::Rails::SolidQueueEnqueueError`.
+
+```ruby
+c.capture_solid_queue = true  # default; no-op unless SolidQueue is running
+```
+
+It only does anything when SolidQueue is actually running, so apps that don't use
+it pay nothing. Memory pressure itself (Heroku R14/R15) is a platform signal, not
+an app exception — pair this with a Heroku memory alert to catch the root cause,
+not just the orphaned jobs.
+
 ### Traffic heartbeats (confound signal)
 
 On by default in enabled environments, the SDK ships lightweight per-`transaction`

data/lib/dispatch/rails/configuration.rb

@@ -20,6 +20,9 @@ module Dispatch
       attr_accessor :capture_csp_violations, :capture_browser_reports, :reporting_endpoint_path
       # Process lifecycle (crash-at-exit capture, rake failures, shutdown flush)
       attr_accessor :capture_at_exit, :shutdown_timeout
+      # SolidQueue infrastructure failures (pruned/orphaned jobs, thread crashes,
+      # recurring-enqueue misses) that never flow through Rails.error/ActiveJob.
+      attr_accessor :capture_solid_queue
       # Structured error responses (API-only)
       attr_accessor :structured_error_responses, :annotate_error_body, :report_base_url
       # Traffic heartbeats — per-transaction success counts that let the Dispatch
@@ -67,6 +70,13 @@ module Dispatch
         @capture_at_exit = true
         @shutdown_timeout = 3 # seconds to wait for the queue to drain at exit; 0 skips the flush
 
+        # SolidQueue infrastructure capture. On by default in enabled environments
+        # (no-op unless SolidQueue is actually running, so apps that don't use it
+        # pay nothing). These failures — a worker pruned after a missed heartbeat,
+        # jobs orphaned by a dead process, a recurring task that didn't enqueue —
+        # bypass Rails.error entirely, so without this they never reach Dispatch.
+        @capture_solid_queue = true
+
         # Structured error responses (off by default — opt-in so we never alter a
         # host app's error contract without being asked).
         @structured_error_responses = false
@@ -145,6 +155,14 @@ module Dispatch
         @capture_traffic && error_tracking_enabled? && environment_enabled?
       end
 
+      # SolidQueue capture shares error capture's gating (credentials + the
+      # capture_exceptions master switch + enabled environments), plus its own
+      # toggle. The subscriber is wired unconditionally at boot and checks this
+      # at event time, so toggling it in an initializer always takes effect.
+      def solid_queue_tracking_enabled?
+        @capture_solid_queue && error_tracking_enabled? && environment_enabled?
+      end
+
       def environment_enabled?
         list = Array(@enabled_environments).map(&:to_s)
         list.empty? || list.include?(effective_environment)

data/lib/dispatch/rails/engine.rb

@@ -45,6 +45,18 @@ module Dispatch
         end
       end
 
+      # SolidQueue infrastructure failures (a worker pruned after a missed
+      # heartbeat, jobs orphaned by a dead process, a recurring task that failed
+      # to enqueue) are recorded straight into SolidQueue's own tables and never
+      # flow through Rails.error/ActiveJob — so the subscriber above never sees
+      # them. Bridge SolidQueue's ActiveSupport::Notifications into capture.
+      # Subscribed unconditionally and idempotently; the subscriber no-ops unless
+      # config.solid_queue_tracking_enabled?, and the events only ever fire when
+      # SolidQueue is actually running.
+      initializer "dispatch-rails.solid_queue" do
+        Dispatch::Rails::SolidQueueSubscriber.install!
+      end
+
       # Per-transaction traffic heartbeats. Mounted unconditionally; the middleware
       # no-ops at request time unless config.traffic_tracking_enabled? (false in
       # dev/test), so it never phones home outside enabled environments.

data/lib/dispatch/rails/solid_queue_subscriber.rb

@@ -0,0 +1,168 @@
+require "active_support/notifications"
+
+module Dispatch
+  module Rails
+    # Synthetic exceptions for SolidQueue infrastructure failures that have no
+    # Ruby exception of their own (the real cause is written onto SolidQueue's
+    # own tables, not raised). thread_error reports the genuine exception instead.
+    class SolidQueueError < StandardError; end
+    # A worker process died (pruned after a missed heartbeat, or its process went
+    # missing) and SolidQueue force-failed the jobs it had claimed. This is the
+    # OOM-restart case: Heroku SIGKILLs the dyno, the supervisor later prunes the
+    # dead process and fails its claimed jobs straight into solid_queue_failed_executions.
+    class SolidQueueJobsLost < SolidQueueError; end
+    # A recurring (cron) task could not be enqueued — a silent scheduling miss.
+    class SolidQueueEnqueueError < SolidQueueError; end
+
+    # Bridges SolidQueue's ActiveSupport::Notifications into Dispatch error
+    # capture.
+    #
+    # SolidQueue records its *infrastructure* failures — a worker pruned after a
+    # missed heartbeat, claimed jobs orphaned by a dead process, a recurring task
+    # that failed to enqueue — straight into its own tables. They are NOT raised
+    # through ActiveJob's perform path, so they never reach Rails.error and never
+    # reach the Dispatch::Rails::ErrorSubscriber. The result is an invisible class
+    # of failure: jobs that died are only discoverable by opening the queue.
+    #
+    # This subscriber listens for those events and turns each into a first-class
+    # Dispatch event. Captured events (all gated by config.capture_solid_queue and
+    # the usual environment gating):
+    #
+    #   fail_many_claimed       -> SolidQueueJobsLost (pruned / orphaned / dead-worker jobs)
+    #   thread_error            -> the real exception  (a crash in a SolidQueue thread)
+    #   enqueue_recurring_task  -> SolidQueueEnqueueError (only when enqueue_error is set)
+    #
+    # On thread_error: SolidQueue fires this event BEFORE calling on_thread_error
+    # (see SolidQueue::AppExecutor#handle_thread_error), and Reporter marks the
+    # exception object on capture. So when the default on_thread_error
+    # (-> Rails.error.report) runs next, the Dispatch error subscriber dedups it
+    # via that marker — no double report. Capturing here means we still see these
+    # crashes even when the host app overrides on_thread_error away from
+    # Rails.error (a common customization that would otherwise hide them).
+    #
+    # Idempotent: install! subscribes once per process; repeat calls no-op. The
+    # handlers never raise back into SolidQueue's instrumentation.
+    module SolidQueueSubscriber
+      EVENTS = %w[
+        fail_many_claimed.solid_queue
+        thread_error.solid_queue
+        enqueue_recurring_task.solid_queue
+      ].freeze
+
+      # How many job/process ids to inline into tags before truncating. The
+      # payload batches are bounded (prune runs in batches of 50), but we keep
+      # tag values short regardless.
+      MAX_IDS = 20
+
+      class << self
+        def install!
+          return if @installed
+
+          @installed = true
+          @subscriptions = EVENTS.map do |event|
+            ActiveSupport::Notifications.subscribe(event) do |name, _start, _finish, _id, payload|
+              handle(name, payload)
+            end
+          end
+          true
+        end
+
+        # Test/reset seam — unsubscribe so a fresh install! re-registers.
+        def uninstall!
+          Array(@subscriptions).each { |sub| ActiveSupport::Notifications.unsubscribe(sub) }
+          @subscriptions = nil
+          @installed = false
+        end
+
+        def installed?
+          @installed == true
+        end
+
+        private
+
+        # A subscriber that raises surfaces as an InstrumentationSubscriberError
+        # inside SolidQueue's prune/dispatch loop, so every path is guarded.
+        # Reporter.capture already never raises; this guards the tag-building too.
+        def handle(name, payload)
+          return unless Dispatch::Rails.configuration.solid_queue_tracking_enabled?
+
+          case name
+          when "fail_many_claimed.solid_queue"      then on_jobs_lost(payload)
+          when "thread_error.solid_queue"           then on_thread_error(payload)
+          when "enqueue_recurring_task.solid_queue" then on_enqueue_recurring_task(payload)
+          end
+        rescue StandardError => e
+          warn "[dispatch-rails] solid_queue subscriber failed: #{e.class}: #{e.message}"
+          nil
+        end
+
+        # The underlying ProcessPrunedError / ProcessMissingError isn't in the
+        # payload — SolidQueue writes it onto the failed_executions rows — so we
+        # synthesize a stable error (constant message for clean grouping) and
+        # carry the volatile ids/count as tags.
+        def on_jobs_lost(payload)
+          size = payload[:size].to_i
+          return if size.zero?
+
+          Reporter.capture(
+            SolidQueueJobsLost.new(
+              "SolidQueue force-failed claimed job(s) after a worker process died"
+            ),
+            handled: false,
+            level: "error",
+            context: { tags: {
+              source: "solid_queue",
+              solid_queue_event: "fail_many_claimed",
+              jobs_lost: size.to_s,
+              process_ids: truncate_ids(payload[:process_ids]),
+              job_ids: truncate_ids(payload[:job_ids])
+            }.compact }
+          )
+        end
+
+        # The real exception that escaped a SolidQueue thread (supervisor,
+        # dispatcher, worker, scheduler) — report it as-is for an accurate class
+        # and backtrace. Dedups against the default on_thread_error via the
+        # Reporter capture marker (see class comment).
+        def on_thread_error(payload)
+          error = payload[:error]
+          return unless error.is_a?(Exception)
+
+          Reporter.capture(
+            error,
+            handled: false,
+            level: "error",
+            context: { tags: { source: "solid_queue", solid_queue_event: "thread_error" } }
+          )
+        end
+
+        # A recurring task failed to enqueue. payload[:enqueue_error] is the
+        # message string (no exception object), present only on failure.
+        def on_enqueue_recurring_task(payload)
+          message = payload[:enqueue_error].to_s
+          return if message.empty?
+
+          task = payload[:task].to_s
+          Reporter.capture(
+            SolidQueueEnqueueError.new("SolidQueue failed to enqueue recurring task #{task}: #{message}"),
+            handled: false,
+            level: "error",
+            context: { tags: {
+              source: "solid_queue",
+              solid_queue_event: "enqueue_recurring_task",
+              task: task
+            }.reject { |_, v| v.nil? || v == "" } }
+          )
+        end
+
+        def truncate_ids(ids)
+          list = Array(ids)
+          return nil if list.empty?
+
+          shown = list.first(MAX_IDS).join(",")
+          list.size > MAX_IDS ? "#{shown},+#{list.size - MAX_IDS} more" : shown
+        end
+      end
+    end
+  end
+end

data/lib/dispatch/rails/version.rb

@@ -1,5 +1,5 @@
 module Dispatch
   module Rails
-    VERSION = "0.8.1".freeze
+    VERSION = "0.9.0".freeze
   end
 end

data/lib/dispatch-rails.rb

@@ -9,6 +9,7 @@ require "dispatch/rails/response_annotator"
 require "dispatch/rails/heartbeat_aggregator"
 require "dispatch/rails/heartbeat_middleware"
 require "dispatch/rails/error_subscriber"
+require "dispatch/rails/solid_queue_subscriber"
 require "dispatch/rails/rake_handler"
 require "dispatch/rails/engine"
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: dispatch-rails
 version: !ruby/object:Gem::Version
-  version: 0.8.1
+  version: 0.9.0
 platform: ruby
 authors:
 - Dispatch Team
@@ -57,6 +57,7 @@ files:
 - lib/dispatch/rails/reporter.rb
 - lib/dispatch/rails/reporting_endpoint_middleware.rb
 - lib/dispatch/rails/response_annotator.rb
+- lib/dispatch/rails/solid_queue_subscriber.rb
 - lib/dispatch/rails/transport.rb
 - lib/dispatch/rails/version.rb
 homepage: https://dispatchit.app