solid_queue_web 1.3.0 → 1.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f84e63b803df1ce7a322b564eeb78262d7ec76c00d34d087076c842606960e45
-  data.tar.gz: 3cc8a8e1dd074bf9770cea28053b7f7d947f40588f6ac20bc0ccf3b1936adcd0
+  metadata.gz: c0d9757682265e639856df718934858d99390e8ca06ff14cb575b117db105932
+  data.tar.gz: a6c0ca7e495f76467cde1aabb0f54674c1439a5489913b4fe9d95837df3acd03
 SHA512:
-  metadata.gz: d7ecc0d4f79f041cca6f6d1aff56953336c63579eae4802e9fdf72603d19523189cf81f27409560f26d1c9df8ecb165c7a0c5bb1302a91a681759ecf5840f8bb
-  data.tar.gz: 5ae9f1035b8443dcbaa66fc02a5171cd91d65b0c09bcb7d66f365350c86aa87a682a5a10c0235227315818fdc7ad8b7d6a3bdb72807c232d429f2a0d046f4a16
+  metadata.gz: 41ff634f54cb990c14c57870835f62e091a36130c26e0ecdca0a9fbe07f6ed70b9f128a41d22b59177f03cd21497c3bda9fc17e3044e3fcb843a6031ad9ece3d
+  data.tar.gz: 15ed48fde349d58b1f2778cd9b0cf5a8e502ccd74f610a23368665f938fe36c3d66041400eeb96b3aa196d03a5dd56c7becfbd6b743b055cba60ee01e4cb6c3b

data/README.md

@@ -53,7 +53,8 @@ SolidQueueWeb surfaces all of this in a browser UI available at any route you ch
 - **Dashboard quick actions** — "Retry All Failed" and "Discard All Blocked" cards appear on the dashboard only when the respective count is non-zero; one-click bulk operations with confirm dialogs, keeping the dashboard clean when everything is healthy
 - **CSV export** — "Export CSV" button on the jobs, failed jobs, and history pages downloads all records matching the current filters; columns are tailored per view
 - **Slow job detection** — when `slow_job_threshold` is configured, claimed jobs running longer than the threshold are flagged with an orange row, a "slow" badge, and a "Running For" duration column on the Running tab; a "Slow Jobs" warning card appears on the dashboard with a link to the Running tab
-- **Webhook alerts** — set `alert_webhook_url` and `alert_failure_threshold` to receive a POST request whenever the failed job count meets or exceeds the threshold; fires asynchronously so dashboard performance is unaffected; a configurable cooldown (default 1 h) prevents repeated alerts while the count stays elevated
+- **Job wait time** — the Running tab shows a "Wait Time" column with how long each job waited in the queue from enqueue to pickup; also exported as `wait_time_seconds` in the claimed-status CSV
+- **Webhook alerts** — set `alert_webhook_url` and `alert_failure_threshold` to receive a POST request whenever the failed job count meets or exceeds the threshold; set `alert_queue_thresholds` for per-queue depth alerts; set `alert_slow_job_count_threshold` (requires `slow_job_threshold`) for slow-job count alerts; set `alert_stale_process_threshold` for stale-worker alerts; all fire asynchronously with a configurable cooldown (default 1 h) to prevent repeated alerts
 - **Performance analytics** — per-job-class statistics at `/jobs/performance` showing run count, average, p50, p95, p99, standard deviation, min, and max duration; sorted by p95 descending so the slowest classes surface first; high std dev surfaces inconsistent jobs worth investigating; period filter scopes to 1h / 24h / 7d or all time; each class name links to the filtered History view
 - **Failed job trend chart** — a "Failures — Last 12 Hours" bar chart on the dashboard shows failures per hour over the last 12 hours; bars are red, making failure spikes visible before clicking into the failed jobs list
 - **Error frequency report** — `GET /jobs/failed_jobs/errors` groups all failed jobs by error class and message prefix, shows a count per group, and surfaces a sample backtrace in an expandable row; sorted by count descending so the most common errors appear first; accessible via the "Error Summary" button on the Failed Jobs page
@@ -106,8 +107,10 @@ SolidQueueWeb.configure do |config|
   config.slow_job_threshold         = 5.minutes # flag claimed jobs running longer than this (default: nil = disabled)
   config.alert_webhook_url          = "https://hooks.example.com/solid-queue" # POST target — string or array (default: nil = disabled)
   config.alert_failure_threshold    = 10         # fire when failed count >= this (default: nil = disabled)
-  config.alert_queue_thresholds     = { "critical" => 50, "default" => 200 } # fire when queue depth >= threshold (default: {})
-  config.alert_webhook_cooldown     = 1800       # seconds between repeated alerts per alert type (default: 3600)
+  config.alert_queue_thresholds          = { "critical" => 50, "default" => 200 } # fire when queue depth >= threshold (default: {})
+  config.alert_slow_job_count_threshold  = 5          # fire when slow job count >= this (default: nil = disabled)
+  config.alert_stale_process_threshold  = 1          # fire when stale process count >= this (default: nil = disabled)
+  config.alert_webhook_cooldown         = 1800       # seconds between repeated alerts per alert type (default: 3600)
   config.connects_to                = { reading: :reading, writing: :writing } # read replica (default: nil)
   config.time_zone                  = "America/New_York" # display timezone for all timestamps (default: nil = UTC)
 end
@@ -182,6 +185,57 @@ The same `alert_webhook_url` endpoint(s) receive the payload, with a distinct ev
 
 Cooldown is tracked independently per queue, so a persistently deep "critical" queue does not suppress alerts for "default". The shared `alert_webhook_cooldown` setting applies to each queue separately.
 
+## Slow job alerts
+
+Set `alert_slow_job_count_threshold` to fire a webhook when the number of currently-running slow jobs meets or exceeds a count. This requires `slow_job_threshold` to also be configured — it defines what "slow" means.
+
+```ruby
+SolidQueueWeb.configure do |config|
+  config.slow_job_threshold             = 5.minutes  # a job is "slow" if it has been claimed longer than this
+  config.alert_slow_job_count_threshold = 3           # fire when >= 3 jobs are slow
+  config.alert_webhook_url              = "https://hooks.example.com/solid-queue"
+  config.alert_webhook_cooldown         = 1800        # don't re-fire for 30 minutes (default: 3600)
+end
+```
+
+The same `alert_webhook_url` endpoint(s) receive the payload with a distinct event type:
+
+```json
+{
+  "event": "slow_job_threshold_exceeded",
+  "slow_job_count": 5,
+  "threshold": 3,
+  "fired_at": "2026-05-28T08:00:00Z"
+}
+```
+
+The alert fires on every dashboard page load while the condition persists, subject to the cooldown window.
+
+## Stale process alerts
+
+Set `alert_stale_process_threshold` to fire a webhook when the number of stale workers meets or exceeds a count. A process is considered stale when its `last_heartbeat_at` has not been updated within `SolidQueue.process_alive_threshold` (default 5 minutes). A stale worker means jobs in its queues have silently stopped processing.
+
+```ruby
+SolidQueueWeb.configure do |config|
+  config.alert_stale_process_threshold = 1    # fire when any process goes stale
+  config.alert_webhook_url             = "https://hooks.example.com/solid-queue"
+  config.alert_webhook_cooldown        = 1800  # don't re-fire for 30 minutes (default: 3600)
+end
+```
+
+The same `alert_webhook_url` endpoint(s) receive the payload with a distinct event type:
+
+```json
+{
+  "event": "stale_process_detected",
+  "stale_process_count": 2,
+  "threshold": 1,
+  "fired_at": "2026-05-28T08:00:00Z"
+}
+```
+
+The alert fires on every dashboard page load while the condition persists, subject to the cooldown window.
+
 ## Metrics endpoint
 
 `GET /jobs/metrics.json` returns a machine-readable JSON document suitable for Prometheus scraping, uptime monitors, or external dashboards. No configuration is required — the endpoint is available as soon as the engine is mounted.

data/app/controllers/solid_queue_web/dashboard_controller.rb

@@ -4,6 +4,8 @@ module SolidQueueWeb
       @stats = DashboardStats.new
       AlertWebhook.call(failure_count: @stats.counts[:failed])
       QueueDepthAlert.call
+      SlowJobAlert.call
+      StaleProcessAlert.call
     end
   end
 end

data/app/controllers/solid_queue_web/jobs_controller.rb

@@ -82,10 +82,14 @@ module SolidQueueWeb
 
     def jobs_csv(scope)
       CSV.generate(headers: true) do |csv|
-        csv << %w[id class_name queue_name status priority enqueued_at]
+        headers = %w[id class_name queue_name status priority enqueued_at]
+        headers << "wait_time_seconds" if @status == "claimed"
+        csv << headers
         scope.each do |execution|
           job = execution.job
-          csv << [job.id, job.class_name, job.queue_name, @status, job.priority, job.created_at.iso8601]
+          row = [job.id, job.class_name, job.queue_name, @status, job.priority, job.created_at.iso8601]
+          row << (execution.created_at - job.created_at).to_i if @status == "claimed"
+          csv << row
         end
       end
     end

data/app/controllers/solid_queue_web/queues_controller.rb

@@ -7,6 +7,11 @@ module SolidQueueWeb
       @failed_24h         = stats.failed_24h
       @oldest_ready       = stats.oldest_ready
       @failure_sparklines = stats.failure_sparklines
+      @queue_sizes        = SolidQueue::ReadyExecution
+        .joins(:job)
+        .group("solid_queue_jobs.queue_name")
+        .count
+      @paused_queue_names = SolidQueue::Pause.pluck(:queue_name).to_set
     end
   end
 end

data/app/services/solid_queue_web/slow_job_alert.rb

@@ -0,0 +1,70 @@
+require "net/http"
+require "json"
+require "uri"
+
+module SolidQueueWeb
+  class SlowJobAlert
+    MUTEX = Mutex.new
+
+    class << self
+      def call
+        return unless configured?
+
+        slow_count = SolidQueue::ClaimedExecution
+          .where("created_at <= ?", SolidQueueWeb.slow_job_threshold.ago)
+          .count
+
+        return if slow_count < SolidQueueWeb.alert_slow_job_count_threshold
+        return unless should_fire?
+
+        urls = webhook_urls
+        Thread.new { urls.each { |url| post(url, slow_count) } }
+      end
+
+      def reset!
+        MUTEX.synchronize { @last_fired_at = nil }
+      end
+
+      private
+
+      def configured?
+        SolidQueueWeb.slow_job_threshold.present? &&
+          SolidQueueWeb.alert_slow_job_count_threshold.present? &&
+          webhook_urls.any?
+      end
+
+      def webhook_urls
+        Array(SolidQueueWeb.alert_webhook_url).flatten.compact.select(&:present?)
+      end
+
+      def should_fire?
+        MUTEX.synchronize do
+          cooldown = SolidQueueWeb.alert_webhook_cooldown
+          return false if @last_fired_at && Time.current - @last_fired_at < cooldown
+
+          @last_fired_at = Time.current
+          true
+        end
+      end
+
+      def post(url_string, slow_count)
+        uri = URI.parse(url_string)
+        payload = JSON.generate(
+          event:          "slow_job_threshold_exceeded",
+          slow_job_count: slow_count,
+          threshold:      SolidQueueWeb.alert_slow_job_count_threshold,
+          fired_at:       Time.current.iso8601
+        )
+        http = Net::HTTP.new(uri.host, uri.port)
+        http.use_ssl = uri.scheme == "https"
+        http.open_timeout = 5
+        http.read_timeout = 10
+        request = Net::HTTP::Post.new(uri.path.presence || "/", "Content-Type" => "application/json")
+        request.body = payload
+        http.request(request)
+      rescue => e
+        Rails.logger.error("[SolidQueueWeb] Slow job alert webhook failed: #{e.message}")
+      end
+    end
+  end
+end

data/app/services/solid_queue_web/stale_process_alert.rb

@@ -0,0 +1,68 @@
+require "net/http"
+require "json"
+require "uri"
+
+module SolidQueueWeb
+  class StaleProcessAlert
+    MUTEX = Mutex.new
+
+    class << self
+      def call
+        return unless configured?
+
+        stale_count = SolidQueue::Process
+          .where("last_heartbeat_at < ?", SolidQueue.process_alive_threshold.ago)
+          .count
+
+        return if stale_count < SolidQueueWeb.alert_stale_process_threshold
+        return unless should_fire?
+
+        urls = webhook_urls
+        Thread.new { urls.each { |url| post(url, stale_count) } }
+      end
+
+      def reset!
+        MUTEX.synchronize { @last_fired_at = nil }
+      end
+
+      private
+
+      def configured?
+        SolidQueueWeb.alert_stale_process_threshold.present? && webhook_urls.any?
+      end
+
+      def webhook_urls
+        Array(SolidQueueWeb.alert_webhook_url).flatten.compact.select(&:present?)
+      end
+
+      def should_fire?
+        MUTEX.synchronize do
+          cooldown = SolidQueueWeb.alert_webhook_cooldown
+          return false if @last_fired_at && Time.current - @last_fired_at < cooldown
+
+          @last_fired_at = Time.current
+          true
+        end
+      end
+
+      def post(url_string, stale_count)
+        uri = URI.parse(url_string)
+        payload = JSON.generate(
+          event:               "stale_process_detected",
+          stale_process_count: stale_count,
+          threshold:           SolidQueueWeb.alert_stale_process_threshold,
+          fired_at:            Time.current.iso8601
+        )
+        http = Net::HTTP.new(uri.host, uri.port)
+        http.use_ssl = uri.scheme == "https"
+        http.open_timeout = 5
+        http.read_timeout = 10
+        request = Net::HTTP::Post.new(uri.path.presence || "/", "Content-Type" => "application/json")
+        request.body = payload
+        http.request(request)
+      rescue => e
+        Rails.logger.error("[SolidQueueWeb] Stale process alert webhook failed: #{e.message}")
+      end
+    end
+  end
+end

data/app/views/solid_queue_web/jobs/index.html.erb

@@ -164,6 +164,7 @@
             <%= sort_header_th("Enqueued At", "created_at", sort_url, current_sort: @sort, current_dir: @direction) %>
             <% if @status == "claimed" %>
               <th scope="col">Running For</th>
+              <th scope="col">Wait Time</th>
             <% end %>
           </tr>
         </thead>
@@ -192,6 +193,9 @@
                 <td class="sqd-mono<%= slow ? " sqd-slow-duration" : "" %>">
                   <%= time_ago_in_words(execution.created_at) %>
                 </td>
+                <td class="sqd-mono">
+                  <%= format_duration(execution.created_at - job.created_at) %>
+                </td>
               <% end %>
             </tr>
           <% end %>

data/app/views/solid_queue_web/queues/index.html.erb

@@ -21,7 +21,7 @@
         <% @queues.each do |queue| %>
           <tr>
             <td class="sqd-mono"><%= queue.name %></td>
-            <td><%= queue.size %></td>
+            <td><%= @queue_sizes[queue.name] || 0 %></td>
             <td>
               <% if (oldest = @oldest_ready[queue.name]) %>
                 <% age = Time.current - oldest %>
@@ -52,14 +52,14 @@
               <% end %>
             </td>
             <td>
-              <% if queue.paused? %>
+              <% if @paused_queue_names.include?(queue.name) %>
                 <span class="sqd-badge sqd-badge--paused">Paused</span>
               <% else %>
                 <span class="sqd-badge sqd-badge--running">Running</span>
               <% end %>
             </td>
             <td class="sqd-row-actions">
-              <% if queue.paused? %>
+              <% if @paused_queue_names.include?(queue.name) %>
                 <%= button_to "Resume", queue_pause_path(queue.name), method: :delete,
                       class: "sqd-btn sqd-btn--primary sqd-btn--sm" %>
               <% else %>

data/lib/solid_queue_web/version.rb

@@ -1,3 +1,3 @@
 module SolidQueueWeb
-  VERSION = "1.3.0"
+  VERSION = "1.4.0"
 end

data/lib/solid_queue_web.rb

@@ -6,7 +6,8 @@ module SolidQueueWeb
   class << self
     attr_writer :page_size, :dashboard_refresh_interval, :default_refresh_interval, :search_results_limit,
                 :slow_job_threshold, :alert_webhook_url, :alert_failure_threshold, :alert_webhook_cooldown,
-                :alert_queue_thresholds, :connects_to, :time_zone
+                :alert_queue_thresholds, :alert_slow_job_count_threshold, :alert_stale_process_threshold,
+                :connects_to, :time_zone
 
     def page_size
       @page_size || 25
@@ -44,6 +45,14 @@ module SolidQueueWeb
       @alert_queue_thresholds || {}
     end
 
+    def alert_slow_job_count_threshold
+      @alert_slow_job_count_threshold
+    end
+
+    def alert_stale_process_threshold
+      @alert_stale_process_threshold
+    end
+
     def connects_to
       @connects_to
     end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: solid_queue_web
 version: !ruby/object:Gem::Version
-  version: 1.3.0
+  version: 1.4.0
 platform: ruby
 authors:
 - Chuck Smith
@@ -159,6 +159,8 @@ files:
 - app/services/solid_queue_web/metrics_payload.rb
 - app/services/solid_queue_web/queue_depth_alert.rb
 - app/services/solid_queue_web/queue_stats.rb
+- app/services/solid_queue_web/slow_job_alert.rb
+- app/services/solid_queue_web/stale_process_alert.rb
 - app/views/layouts/solid_queue_web/application.html.erb
 - app/views/solid_queue_web/dashboard/index.html.erb
 - app/views/solid_queue_web/failed_jobs/errors/index.html.erb