solid_queue_web 1.3.0 → 1.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f84e63b803df1ce7a322b564eeb78262d7ec76c00d34d087076c842606960e45
-  data.tar.gz: 3cc8a8e1dd074bf9770cea28053b7f7d947f40588f6ac20bc0ccf3b1936adcd0
+  metadata.gz: 0aea2023ccc5983daeb0781f0b56105492179d638822d9d6c4a853e7296f3258
+  data.tar.gz: 0b430cccb3b56335a562451f0d598390a4a3f81d2abb1ad5122ce3233fa5663c
 SHA512:
-  metadata.gz: d7ecc0d4f79f041cca6f6d1aff56953336c63579eae4802e9fdf72603d19523189cf81f27409560f26d1c9df8ecb165c7a0c5bb1302a91a681759ecf5840f8bb
-  data.tar.gz: 5ae9f1035b8443dcbaa66fc02a5171cd91d65b0c09bcb7d66f365350c86aa87a682a5a10c0235227315818fdc7ad8b7d6a3bdb72807c232d429f2a0d046f4a16
+  metadata.gz: f654b581b1bddc2c2559d39fa09cfc83c0bc664268ba085770179476fbed468e672341025f1d34f5e64c8cb9fc9127914d0d1601a073cc7fb19df03ed890c369
+  data.tar.gz: 5c4794c4ab5e1f5bd5e33aaa4b6630f5de7c4289c55b6506d2f21493c9b63e4e24680b03c2cd1554812426b3b285c5cf640adc079f4b5a9fe93916bcbbd9b3c3

data/README.md

@@ -53,7 +53,9 @@ SolidQueueWeb surfaces all of this in a browser UI available at any route you ch
 - **Dashboard quick actions** — "Retry All Failed" and "Discard All Blocked" cards appear on the dashboard only when the respective count is non-zero; one-click bulk operations with confirm dialogs, keeping the dashboard clean when everything is healthy
 - **CSV export** — "Export CSV" button on the jobs, failed jobs, and history pages downloads all records matching the current filters; columns are tailored per view
 - **Slow job detection** — when `slow_job_threshold` is configured, claimed jobs running longer than the threshold are flagged with an orange row, a "slow" badge, and a "Running For" duration column on the Running tab; a "Slow Jobs" warning card appears on the dashboard with a link to the Running tab
-- **Webhook alerts** — set `alert_webhook_url` and `alert_failure_threshold` to receive a POST request whenever the failed job count meets or exceeds the threshold; fires asynchronously so dashboard performance is unaffected; a configurable cooldown (default 1 h) prevents repeated alerts while the count stays elevated
+- **Job wait time** — the Running tab shows a "Wait Time" column with how long each job waited in the queue from enqueue to pickup; also exported as `wait_time_seconds` in the claimed-status CSV
+- **Admin audit log** — every discard, retry, queue pause, and resume is recorded to a `solid_queue_web_audit_events` table and viewable at `/jobs/audit` with action/actor/queue filters and CSV export; actor identity captured via the optional `current_actor` config block; requires running the install generator to create the table
+- **Webhook alerts** — set `alert_webhook_url` and `alert_failure_threshold` to receive a POST request whenever the failed job count meets or exceeds the threshold; set `alert_queue_thresholds` for per-queue depth alerts; set `alert_slow_job_count_threshold` (requires `slow_job_threshold`) for slow-job count alerts; set `alert_stale_process_threshold` for stale-worker alerts; all fire asynchronously with a configurable cooldown (default 1 h) to prevent repeated alerts
 - **Performance analytics** — per-job-class statistics at `/jobs/performance` showing run count, average, p50, p95, p99, standard deviation, min, and max duration; sorted by p95 descending so the slowest classes surface first; high std dev surfaces inconsistent jobs worth investigating; period filter scopes to 1h / 24h / 7d or all time; each class name links to the filtered History view
 - **Failed job trend chart** — a "Failures — Last 12 Hours" bar chart on the dashboard shows failures per hour over the last 12 hours; bars are red, making failure spikes visible before clicking into the failed jobs list
 - **Error frequency report** — `GET /jobs/failed_jobs/errors` groups all failed jobs by error class and message prefix, shows a count per group, and surfaces a sample backtrace in an expandable row; sorted by count descending so the most common errors appear first; accessible via the "Error Summary" button on the Failed Jobs page
@@ -106,8 +108,11 @@ SolidQueueWeb.configure do |config|
   config.slow_job_threshold         = 5.minutes # flag claimed jobs running longer than this (default: nil = disabled)
   config.alert_webhook_url          = "https://hooks.example.com/solid-queue" # POST target — string or array (default: nil = disabled)
   config.alert_failure_threshold    = 10         # fire when failed count >= this (default: nil = disabled)
-  config.alert_queue_thresholds     = { "critical" => 50, "default" => 200 } # fire when queue depth >= threshold (default: {})
-  config.alert_webhook_cooldown     = 1800       # seconds between repeated alerts per alert type (default: 3600)
+  config.alert_queue_thresholds          = { "critical" => 50, "default" => 200 } # fire when queue depth >= threshold (default: {})
+  config.alert_slow_job_count_threshold  = 5          # fire when slow job count >= this (default: nil = disabled)
+  config.alert_stale_process_threshold  = 1          # fire when stale process count >= this (default: nil = disabled)
+  config.alert_webhook_cooldown         = 1800       # seconds between repeated alerts per alert type (default: 3600)
+  config.current_actor                  = -> { current_user&.email } # identity for audit log (default: nil)
   config.connects_to                = { reading: :reading, writing: :writing } # read replica (default: nil)
   config.time_zone                  = "America/New_York" # display timezone for all timestamps (default: nil = UTC)
 end
@@ -182,6 +187,97 @@ The same `alert_webhook_url` endpoint(s) receive the payload, with a distinct ev
 
 Cooldown is tracked independently per queue, so a persistently deep "critical" queue does not suppress alerts for "default". The shared `alert_webhook_cooldown` setting applies to each queue separately.
 
+## Slow job alerts
+
+Set `alert_slow_job_count_threshold` to fire a webhook when the number of currently-running slow jobs meets or exceeds a count. This requires `slow_job_threshold` to also be configured — it defines what "slow" means.
+
+```ruby
+SolidQueueWeb.configure do |config|
+  config.slow_job_threshold             = 5.minutes  # a job is "slow" if it has been claimed longer than this
+  config.alert_slow_job_count_threshold = 3           # fire when >= 3 jobs are slow
+  config.alert_webhook_url              = "https://hooks.example.com/solid-queue"
+  config.alert_webhook_cooldown         = 1800        # don't re-fire for 30 minutes (default: 3600)
+end
+```
+
+The same `alert_webhook_url` endpoint(s) receive the payload with a distinct event type:
+
+```json
+{
+  "event": "slow_job_threshold_exceeded",
+  "slow_job_count": 5,
+  "threshold": 3,
+  "fired_at": "2026-05-28T08:00:00Z"
+}
+```
+
+The alert fires on every dashboard page load while the condition persists, subject to the cooldown window.
+
+## Stale process alerts
+
+Set `alert_stale_process_threshold` to fire a webhook when the number of stale workers meets or exceeds a count. A process is considered stale when its `last_heartbeat_at` has not been updated within `SolidQueue.process_alive_threshold` (default 5 minutes). A stale worker means jobs in its queues have silently stopped processing.
+
+```ruby
+SolidQueueWeb.configure do |config|
+  config.alert_stale_process_threshold = 1    # fire when any process goes stale
+  config.alert_webhook_url             = "https://hooks.example.com/solid-queue"
+  config.alert_webhook_cooldown        = 1800  # don't re-fire for 30 minutes (default: 3600)
+end
+```
+
+The same `alert_webhook_url` endpoint(s) receive the payload with a distinct event type:
+
+```json
+{
+  "event": "stale_process_detected",
+  "stale_process_count": 2,
+  "threshold": 1,
+  "fired_at": "2026-05-28T08:00:00Z"
+}
+```
+
+The alert fires on every dashboard page load while the condition persists, subject to the cooldown window.
+
+## Admin audit log
+
+Every discard, retry, queue pause, and resume action is recorded to a `solid_queue_web_audit_events` table and viewable at `/jobs/audit`.
+
+### Installation
+
+The audit log requires an opt-in migration. Run the install generator to copy it to your application:
+
+```bash
+rails generate solid_queue_web:install:migrations
+rails db:migrate
+```
+
+### Identity
+
+Set `SolidQueueWeb.current_actor` to a block that returns the current user's identity as a string. The block is evaluated in controller context, so you have access to helpers like `current_user`:
+
+```ruby
+SolidQueueWeb.configure do |config|
+  config.current_actor = -> { current_user&.email }
+end
+```
+
+If not configured, the actor column is left `nil`.
+
+### Audited actions
+
+| Action | Trigger |
+|---|---|
+| `job_discarded` | Single job discarded from the jobs list |
+| `jobs_discarded` | Bulk or selection discard from the jobs list |
+| `failed_job_retried` | Single failed job retried |
+| `failed_jobs_retried` | Bulk or selection retry of failed jobs |
+| `failed_job_discarded` | Single failed job discarded |
+| `failed_jobs_discarded` | Bulk or selection discard of failed jobs |
+| `queue_paused` | Queue paused |
+| `queue_resumed` | Queue resumed |
+
+The audit log page at `/jobs/audit` supports filtering by action, actor, and queue name. All records can be exported as CSV.
+
 ## Metrics endpoint
 
 `GET /jobs/metrics.json` returns a machine-readable JSON document suitable for Prometheus scraping, uptime monitors, or external dashboards. No configuration is required — the endpoint is available as soon as the engine is mounted.

data/app/controllers/solid_queue_web/application_controller.rb

@@ -37,5 +37,25 @@ module SolidQueueWeb
     def request_basic_auth
       request_http_basic_authentication("Solid Queue Dashboard")
     end
+
+    def record_audit(action, job_class: nil, queue_name: nil, item_count: 1)
+      AuditEvent.create!(
+        action:     action,
+        actor:      resolve_current_actor,
+        job_class:  job_class,
+        queue_name: queue_name,
+        item_count: item_count
+      )
+    rescue => e
+      Rails.logger.error("[SolidQueueWeb] Audit log failed: #{e.message}")
+    end
+
+    def resolve_current_actor
+      block = SolidQueueWeb.current_actor
+      instance_exec(&block) if block
+    rescue => e
+      Rails.logger.error("[SolidQueueWeb] current_actor block failed: #{e.message}")
+      nil
+    end
   end
 end

data/app/controllers/solid_queue_web/audit_controller.rb

@@ -0,0 +1,43 @@
+module SolidQueueWeb
+  class AuditController < ApplicationController
+    before_action :set_filters
+
+    def index
+      scope = audit_scope
+      respond_to do |format|
+        format.html { @pagy, @audit_events = pagy(scope) }
+        format.csv do
+          send_data audit_csv(scope),
+                    filename: "audit-log-#{Date.today}.csv",
+                    type: "text/csv", disposition: "attachment"
+        end
+      end
+    end
+
+    private
+
+    def set_filters
+      @action_filter = params[:action_filter].presence_in(AuditEvent::ACTIONS)
+      @actor_filter  = params[:actor].presence
+      @queue_filter  = params[:queue].presence
+    end
+
+    def audit_scope
+      scope = AuditEvent.recent
+      scope = scope.where(action:     @action_filter) if @action_filter
+      scope = scope.where(actor:      @actor_filter)  if @actor_filter
+      scope = scope.where(queue_name: @queue_filter)  if @queue_filter
+      scope
+    end
+
+    def audit_csv(scope)
+      CSV.generate(headers: true) do |csv|
+        csv << %w[id action actor job_class queue_name item_count created_at]
+        scope.each do |event|
+          csv << [event.id, event.action, event.actor, event.job_class,
+                  event.queue_name, event.item_count, event.created_at.iso8601]
+        end
+      end
+    end
+  end
+end

data/app/controllers/solid_queue_web/dashboard_controller.rb

@@ -4,6 +4,8 @@ module SolidQueueWeb
       @stats = DashboardStats.new
       AlertWebhook.call(failure_count: @stats.counts[:failed])
       QueueDepthAlert.call
+      SlowJobAlert.call
+      StaleProcessAlert.call
     end
   end
 end

data/app/controllers/solid_queue_web/failed_jobs/selections_controller.rb

@@ -6,6 +6,7 @@ module SolidQueueWeb
         executions = SolidQueue::FailedExecution.where(id: ids)
         jobs = executions.includes(:job).map(&:job)
         SolidQueue::FailedExecution.retry_all(jobs)
+        record_audit("failed_jobs_retried", item_count: jobs.size)
         redirect_to failed_jobs_path,
           notice: "#{jobs.size} #{"job".pluralize(jobs.size)} queued for retry."
       rescue => e
@@ -17,6 +18,7 @@ module SolidQueueWeb
         executions = SolidQueue::FailedExecution.where(id: ids)
         jobs = executions.includes(:job).map(&:job)
         SolidQueue::FailedExecution.discard_all_from_jobs(jobs)
+        record_audit("failed_jobs_discarded", item_count: jobs.size)
         redirect_to failed_jobs_path,
           notice: "#{jobs.size} #{"job".pluralize(jobs.size)} discarded."
       rescue => e

data/app/controllers/solid_queue_web/failed_jobs_controller.rb

@@ -37,7 +37,9 @@ module SolidQueueWeb
 
     def perform_discard(executions)
       jobs = executions.map(&:job)
+      action = params[:id] ? "failed_job_discarded" : "failed_jobs_discarded"
       SolidQueue::FailedExecution.discard_all_from_jobs(jobs)
+      record_audit(action, job_class: jobs.first&.class_name, queue_name: jobs.first&.queue_name, item_count: jobs.size)
       redirect_to failed_jobs_path(queue: @queue, q: @search, period: @period),
         notice: "#{jobs.size} #{"job".pluralize(jobs.size)} discarded."
     end

data/app/controllers/solid_queue_web/jobs/selections_controller.rb

@@ -9,6 +9,7 @@ module SolidQueueWeb
         ids = Array(params[:ids]).map(&:to_i).reject(&:zero?)
         jobs = model.where(id: ids).includes(:job).map(&:job)
         model.discard_all_from_jobs(jobs)
+        record_audit("jobs_discarded", item_count: jobs.size)
         redirect_to jobs_path(status: status, period: period),
           notice: "#{jobs.size} #{"job".pluralize(jobs.size)} discarded."
       rescue ArgumentError => e

data/app/controllers/solid_queue_web/jobs_controller.rb

@@ -30,7 +30,9 @@ module SolidQueueWeb
       model = Job.execution_model_for!(@status)
       if params[:id]
         @execution = model.find(params[:id])
+        discarded_job = @execution.job
         @execution.discard
+        record_audit("job_discarded", job_class: discarded_job&.class_name, queue_name: discarded_job&.queue_name)
         @remaining_count = filtered_scope(model).count
         respond_to do |format|
           format.turbo_stream
@@ -39,6 +41,7 @@ module SolidQueueWeb
       else
         jobs = filtered_scope(model).map(&:job)
         model.discard_all_from_jobs(jobs)
+        record_audit("jobs_discarded", item_count: jobs.size)
         redirect_to jobs_return_path, notice: "#{jobs.size} #{"job".pluralize(jobs.size)} discarded."
       end
     rescue ArgumentError => e
@@ -82,10 +85,14 @@ module SolidQueueWeb
 
     def jobs_csv(scope)
       CSV.generate(headers: true) do |csv|
-        csv << %w[id class_name queue_name status priority enqueued_at]
+        headers = %w[id class_name queue_name status priority enqueued_at]
+        headers << "wait_time_seconds" if @status == "claimed"
+        csv << headers
         scope.each do |execution|
           job = execution.job
-          csv << [job.id, job.class_name, job.queue_name, @status, job.priority, job.created_at.iso8601]
+          row = [job.id, job.class_name, job.queue_name, @status, job.priority, job.created_at.iso8601]
+          row << (execution.created_at - job.created_at).to_i if @status == "claimed"
+          csv << row
         end
       end
     end

data/app/controllers/solid_queue_web/queues/pauses_controller.rb

@@ -4,6 +4,7 @@ module SolidQueueWeb
       def create
         queue = SolidQueue::Queue.find_by_name(params[:queue_name])
         queue.pause
+        record_audit("queue_paused", queue_name: queue.name)
         redirect_to queues_path, notice: "Queue \"#{queue.name}\" paused."
       rescue => e
         redirect_to queues_path, alert: "Could not pause queue: #{e.message}"
@@ -12,6 +13,7 @@ module SolidQueueWeb
       def destroy
         queue = SolidQueue::Queue.find_by_name(params[:queue_name])
         queue.resume
+        record_audit("queue_resumed", queue_name: queue.name)
         redirect_to queues_path, notice: "Queue \"#{queue.name}\" resumed."
       rescue => e
         redirect_to queues_path, alert: "Could not resume queue: #{e.message}"

data/app/controllers/solid_queue_web/queues_controller.rb

@@ -7,6 +7,11 @@ module SolidQueueWeb
       @failed_24h         = stats.failed_24h
       @oldest_ready       = stats.oldest_ready
       @failure_sparklines = stats.failure_sparklines
+      @queue_sizes        = SolidQueue::ReadyExecution
+        .joins(:job)
+        .group("solid_queue_jobs.queue_name")
+        .count
+      @paused_queue_names = SolidQueue::Pause.pluck(:queue_name).to_set
     end
   end
 end

data/app/controllers/solid_queue_web/retry_failed_jobs_controller.rb

@@ -16,6 +16,8 @@ module SolidQueueWeb
       else
         SolidQueue::FailedExecution.retry_all(jobs)
       end
+      action = params[:id] ? "failed_job_retried" : "failed_jobs_retried"
+      record_audit(action, job_class: jobs.first&.class_name, queue_name: jobs.first&.queue_name, item_count: jobs.size)
       redirect_to failed_jobs_path(queue: @queue, q: @search, period: @period),
         notice: retry_notice(jobs.size)
     rescue ArgumentError => e

data/app/models/solid_queue_web/audit_event.rb

@@ -0,0 +1,17 @@
+module SolidQueueWeb
+  class AuditEvent < ApplicationRecord
+    self.table_name = "solid_queue_web_audit_events"
+
+    ACTIONS = %w[
+      job_discarded jobs_discarded
+      failed_job_retried failed_jobs_retried
+      failed_job_discarded failed_jobs_discarded
+      queue_paused queue_resumed
+    ].freeze
+
+    validates :action, presence: true, inclusion: { in: ACTIONS }
+    validates :item_count, numericality: { greater_than: 0 }
+
+    scope :recent, -> { order(created_at: :desc) }
+  end
+end

data/app/services/solid_queue_web/slow_job_alert.rb

@@ -0,0 +1,70 @@
+require "net/http"
+require "json"
+require "uri"
+
+module SolidQueueWeb
+  class SlowJobAlert
+    MUTEX = Mutex.new
+
+    class << self
+      def call
+        return unless configured?
+
+        slow_count = SolidQueue::ClaimedExecution
+          .where("created_at <= ?", SolidQueueWeb.slow_job_threshold.ago)
+          .count
+
+        return if slow_count < SolidQueueWeb.alert_slow_job_count_threshold
+        return unless should_fire?
+
+        urls = webhook_urls
+        Thread.new { urls.each { |url| post(url, slow_count) } }
+      end
+
+      def reset!
+        MUTEX.synchronize { @last_fired_at = nil }
+      end
+
+      private
+
+      def configured?
+        SolidQueueWeb.slow_job_threshold.present? &&
+          SolidQueueWeb.alert_slow_job_count_threshold.present? &&
+          webhook_urls.any?
+      end
+
+      def webhook_urls
+        Array(SolidQueueWeb.alert_webhook_url).flatten.compact.select(&:present?)
+      end
+
+      def should_fire?
+        MUTEX.synchronize do
+          cooldown = SolidQueueWeb.alert_webhook_cooldown
+          return false if @last_fired_at && Time.current - @last_fired_at < cooldown
+
+          @last_fired_at = Time.current
+          true
+        end
+      end
+
+      def post(url_string, slow_count)
+        uri = URI.parse(url_string)
+        payload = JSON.generate(
+          event:          "slow_job_threshold_exceeded",
+          slow_job_count: slow_count,
+          threshold:      SolidQueueWeb.alert_slow_job_count_threshold,
+          fired_at:       Time.current.iso8601
+        )
+        http = Net::HTTP.new(uri.host, uri.port)
+        http.use_ssl = uri.scheme == "https"
+        http.open_timeout = 5
+        http.read_timeout = 10
+        request = Net::HTTP::Post.new(uri.path.presence || "/", "Content-Type" => "application/json")
+        request.body = payload
+        http.request(request)
+      rescue => e
+        Rails.logger.error("[SolidQueueWeb] Slow job alert webhook failed: #{e.message}")
+      end
+    end
+  end
+end

data/app/services/solid_queue_web/stale_process_alert.rb

@@ -0,0 +1,68 @@
+require "net/http"
+require "json"
+require "uri"
+
+module SolidQueueWeb
+  class StaleProcessAlert
+    MUTEX = Mutex.new
+
+    class << self
+      def call
+        return unless configured?
+
+        stale_count = SolidQueue::Process
+          .where("last_heartbeat_at < ?", SolidQueue.process_alive_threshold.ago)
+          .count
+
+        return if stale_count < SolidQueueWeb.alert_stale_process_threshold
+        return unless should_fire?
+
+        urls = webhook_urls
+        Thread.new { urls.each { |url| post(url, stale_count) } }
+      end
+
+      def reset!
+        MUTEX.synchronize { @last_fired_at = nil }
+      end
+
+      private
+
+      def configured?
+        SolidQueueWeb.alert_stale_process_threshold.present? && webhook_urls.any?
+      end
+
+      def webhook_urls
+        Array(SolidQueueWeb.alert_webhook_url).flatten.compact.select(&:present?)
+      end
+
+      def should_fire?
+        MUTEX.synchronize do
+          cooldown = SolidQueueWeb.alert_webhook_cooldown
+          return false if @last_fired_at && Time.current - @last_fired_at < cooldown
+
+          @last_fired_at = Time.current
+          true
+        end
+      end
+
+      def post(url_string, stale_count)
+        uri = URI.parse(url_string)
+        payload = JSON.generate(
+          event:               "stale_process_detected",
+          stale_process_count: stale_count,
+          threshold:           SolidQueueWeb.alert_stale_process_threshold,
+          fired_at:            Time.current.iso8601
+        )
+        http = Net::HTTP.new(uri.host, uri.port)
+        http.use_ssl = uri.scheme == "https"
+        http.open_timeout = 5
+        http.read_timeout = 10
+        request = Net::HTTP::Post.new(uri.path.presence || "/", "Content-Type" => "application/json")
+        request.body = payload
+        http.request(request)
+      rescue => e
+        Rails.logger.error("[SolidQueueWeb] Stale process alert webhook failed: #{e.message}")
+      end
+    end
+  end
+end

data/app/views/layouts/solid_queue_web/application.html.erb

@@ -26,6 +26,7 @@
           <li><%= link_to "Recurring", recurring_tasks_path, class: current_page?(recurring_tasks_path) ? "active" : "", aria: { current: current_page?(recurring_tasks_path) ? "page" : nil } %></li>
           <li><%= link_to "Processes", processes_path, class: current_page?(processes_path) ? "active" : "", aria: { current: current_page?(processes_path) ? "page" : nil } %></li>
           <li><%= link_to "Search", search_path, class: current_page?(search_path) ? "active" : "", aria: { current: current_page?(search_path) ? "page" : nil } %></li>
+          <li><%= link_to "Audit", audit_path, class: current_page?(audit_path) ? "active" : "", aria: { current: current_page?(audit_path) ? "page" : nil } %></li>
         </ul>
       </nav>
     </div>

data/app/views/solid_queue_web/audit/index.html.erb

@@ -0,0 +1,78 @@
+<h1 class="sqd-page-title">Audit Log</h1>
+
+<div class="sqd-page-header">
+  <div class="sqd-filters">
+    <form action="<%= audit_path %>" method="get" style="display: flex; gap: 0.5rem; align-items: center; flex-wrap: wrap;">
+      <select name="action_filter" class="sqd-select" aria-label="Filter by action" onchange="this.form.submit()">
+        <option value="">All actions</option>
+        <% SolidQueueWeb::AuditEvent::ACTIONS.each do |a| %>
+          <option value="<%= a %>" <%= @action_filter == a ? "selected" : "" %>><%= a.tr("_", " ") %></option>
+        <% end %>
+      </select>
+      <% if @actor_filter.present? %>
+        <span class="sqd-badge sqd-badge--muted">Actor: <%= @actor_filter %></span>
+        <%= link_to "×", audit_path(action_filter: @action_filter, queue: @queue_filter), class: "sqd-btn sqd-btn--muted sqd-btn--sm" %>
+      <% end %>
+      <% if @queue_filter.present? %>
+        <span class="sqd-badge sqd-badge--muted">Queue: <%= @queue_filter %></span>
+        <%= link_to "×", audit_path(action_filter: @action_filter, actor: @actor_filter), class: "sqd-btn sqd-btn--muted sqd-btn--sm" %>
+      <% end %>
+      <% if @action_filter.present? || @actor_filter.present? || @queue_filter.present? %>
+        <%= link_to "Clear", audit_path, class: "sqd-btn sqd-btn--muted sqd-btn--sm" %>
+      <% end %>
+    </form>
+  </div>
+  <% if @audit_events.any? %>
+    <div class="sqd-actions">
+      <%= link_to "Export CSV", audit_path(format: :csv, action_filter: @action_filter, actor: @actor_filter, queue: @queue_filter),
+            class: "sqd-btn sqd-btn--muted", data: { turbo: false } %>
+    </div>
+  <% end %>
+</div>
+
+<div class="sqd-card">
+  <% if @audit_events.empty? %>
+    <div class="sqd-empty">No audit events recorded.</div>
+  <% else %>
+    <table>
+      <thead>
+        <tr>
+          <th scope="col">Time</th>
+          <th scope="col">Action</th>
+          <th scope="col">Actor</th>
+          <th scope="col">Job Class</th>
+          <th scope="col">Queue</th>
+          <th scope="col">Count</th>
+        </tr>
+      </thead>
+      <tbody>
+        <% @audit_events.each do |event| %>
+          <tr>
+            <td class="sqd-mono"><%= format_timestamp(event.created_at) %></td>
+            <td><span class="sqd-badge sqd-badge--<%= event.action.include?("discard") ? "failed" : event.action.include?("paused") || event.action.include?("resumed") ? "paused" : "ready" %>"><%= event.action.tr("_", " ") %></span></td>
+            <td class="sqd-mono sqd-muted-text">
+              <% if event.actor.present? %>
+                <%= link_to event.actor, audit_path(action_filter: @action_filter, queue: @queue_filter, actor: event.actor), style: "color: inherit;" %>
+              <% else %>
+                <span style="color: var(--muted)">—</span>
+              <% end %>
+            </td>
+            <td class="sqd-mono"><%= event.job_class || "—" %></td>
+            <td class="sqd-mono">
+              <% if event.queue_name.present? %>
+                <%= link_to event.queue_name, audit_path(action_filter: @action_filter, actor: @actor_filter, queue: event.queue_name), style: "color: inherit;" %>
+              <% else %>
+                <span style="color: var(--muted)">—</span>
+              <% end %>
+            </td>
+            <td><%= event.item_count %></td>
+          </tr>
+        <% end %>
+      </tbody>
+    </table>
+  <% end %>
+</div>
+
+<% if @pagy.last > 1 %>
+  <%= @pagy.series_nav.html_safe %>
+<% end %>

data/app/views/solid_queue_web/jobs/index.html.erb

@@ -164,6 +164,7 @@
             <%= sort_header_th("Enqueued At", "created_at", sort_url, current_sort: @sort, current_dir: @direction) %>
             <% if @status == "claimed" %>
               <th scope="col">Running For</th>
+              <th scope="col">Wait Time</th>
             <% end %>
           </tr>
         </thead>
@@ -192,6 +193,9 @@
                 <td class="sqd-mono<%= slow ? " sqd-slow-duration" : "" %>">
                   <%= time_ago_in_words(execution.created_at) %>
                 </td>
+                <td class="sqd-mono">
+                  <%= format_duration(execution.created_at - job.created_at) %>
+                </td>
               <% end %>
             </tr>
           <% end %>

data/app/views/solid_queue_web/queues/index.html.erb

@@ -21,7 +21,7 @@
         <% @queues.each do |queue| %>
           <tr>
             <td class="sqd-mono"><%= queue.name %></td>
-            <td><%= queue.size %></td>
+            <td><%= @queue_sizes[queue.name] || 0 %></td>
             <td>
               <% if (oldest = @oldest_ready[queue.name]) %>
                 <% age = Time.current - oldest %>
@@ -52,14 +52,14 @@
               <% end %>
             </td>
             <td>
-              <% if queue.paused? %>
+              <% if @paused_queue_names.include?(queue.name) %>
                 <span class="sqd-badge sqd-badge--paused">Paused</span>
               <% else %>
                 <span class="sqd-badge sqd-badge--running">Running</span>
               <% end %>
             </td>
             <td class="sqd-row-actions">
-              <% if queue.paused? %>
+              <% if @paused_queue_names.include?(queue.name) %>
                 <%= button_to "Resume", queue_pause_path(queue.name), method: :delete,
                       class: "sqd-btn sqd-btn--primary sqd-btn--sm" %>
               <% else %>

data/config/routes.rb

@@ -3,6 +3,7 @@ SolidQueueWeb::Engine.routes.draw do
   resource :blocked_jobs, only: [:destroy]
 
   get "metrics", to: "metrics#index", as: :metrics, defaults: { format: :json }
+  get "audit",       to: "audit#index",       as: :audit
   get "search",      to: "search#index",      as: :search
   get "history",     to: "history#index",     as: :history
   get "performance", to: "performance#index", as: :performance

data/db/migrate/01_create_solid_queue_web_audit_events.rb

@@ -0,0 +1,16 @@
+class CreateSolidQueueWebAuditEvents < ActiveRecord::Migration[7.1]
+  def change
+    create_table :solid_queue_web_audit_events do |t|
+      t.string  :action,     null: false
+      t.string  :actor
+      t.string  :job_class
+      t.string  :queue_name
+      t.integer :item_count, null: false, default: 1
+      t.datetime :created_at, null: false
+    end
+
+    add_index :solid_queue_web_audit_events, :created_at
+    add_index :solid_queue_web_audit_events, :action
+    add_index :solid_queue_web_audit_events, :actor
+  end
+end

data/lib/generators/solid_queue_web/install/migrations_generator.rb

@@ -0,0 +1,24 @@
+require "rails/generators"
+require "rails/generators/active_record"
+
+module SolidQueueWeb
+  module Install
+    class MigrationsGenerator < Rails::Generators::Base
+      include Rails::Generators::Migration
+
+      source_root File.expand_path("templates", __dir__)
+      desc "Copy SolidQueueWeb migrations to your application."
+
+      def self.next_migration_number(path)
+        ActiveRecord::Generators::Base.next_migration_number(path)
+      end
+
+      def create_migration_file
+        migration_template(
+          "create_solid_queue_web_audit_events.rb.tt",
+          "db/migrate/create_solid_queue_web_audit_events.rb"
+        )
+      end
+    end
+  end
+end

data/lib/generators/solid_queue_web/install/templates/create_solid_queue_web_audit_events.rb.tt

@@ -0,0 +1,16 @@
+class CreateSolidQueueWebAuditEvents < ActiveRecord::Migration[<%= ActiveRecord::Migration.current_version %>]
+  def change
+    create_table :solid_queue_web_audit_events do |t|
+      t.string  :action,     null: false
+      t.string  :actor
+      t.string  :job_class
+      t.string  :queue_name
+      t.integer :item_count, null: false, default: 1
+      t.datetime :created_at, null: false
+    end
+
+    add_index :solid_queue_web_audit_events, :created_at
+    add_index :solid_queue_web_audit_events, :action
+    add_index :solid_queue_web_audit_events, :actor
+  end
+end

data/lib/solid_queue_web/version.rb

@@ -1,3 +1,3 @@
 module SolidQueueWeb
-  VERSION = "1.3.0"
+  VERSION = "1.5.0"
 end

data/lib/solid_queue_web.rb

@@ -6,7 +6,8 @@ module SolidQueueWeb
   class << self
     attr_writer :page_size, :dashboard_refresh_interval, :default_refresh_interval, :search_results_limit,
                 :slow_job_threshold, :alert_webhook_url, :alert_failure_threshold, :alert_webhook_cooldown,
-                :alert_queue_thresholds, :connects_to, :time_zone
+                :alert_queue_thresholds, :alert_slow_job_count_threshold, :alert_stale_process_threshold,
+                :connects_to, :time_zone
 
     def page_size
       @page_size || 25
@@ -44,6 +45,14 @@ module SolidQueueWeb
       @alert_queue_thresholds || {}
     end
 
+    def alert_slow_job_count_threshold
+      @alert_slow_job_count_threshold
+    end
+
+    def alert_stale_process_threshold
+      @alert_stale_process_threshold
+    end
+
     def connects_to
       @connects_to
     end
@@ -60,5 +69,10 @@ module SolidQueueWeb
       @authenticate = block if block_given?
       @authenticate
     end
+
+    def current_actor(&block)
+      @current_actor = block if block_given?
+      @current_actor
+    end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: solid_queue_web
 version: !ruby/object:Gem::Version
-  version: 1.3.0
+  version: 1.5.0
 platform: ruby
 authors:
 - Chuck Smith
@@ -122,6 +122,7 @@ files:
 - app/assets/stylesheets/solid_queue_web/_12_dark_mode.css
 - app/assets/stylesheets/solid_queue_web/application.css
 - app/controllers/solid_queue_web/application_controller.rb
+- app/controllers/solid_queue_web/audit_controller.rb
 - app/controllers/solid_queue_web/blocked_jobs_controller.rb
 - app/controllers/solid_queue_web/dashboard_controller.rb
 - app/controllers/solid_queue_web/failed_jobs/arguments_controller.rb
@@ -151,6 +152,7 @@ files:
 - app/javascript/solid_queue_web/theme_controller.js
 - app/jobs/solid_queue_web/application_job.rb
 - app/models/solid_queue_web/application_record.rb
+- app/models/solid_queue_web/audit_event.rb
 - app/models/solid_queue_web/job.rb
 - app/services/solid_queue_web/alert_webhook.rb
 - app/services/solid_queue_web/dashboard_stats.rb
@@ -159,7 +161,10 @@ files:
 - app/services/solid_queue_web/metrics_payload.rb
 - app/services/solid_queue_web/queue_depth_alert.rb
 - app/services/solid_queue_web/queue_stats.rb
+- app/services/solid_queue_web/slow_job_alert.rb
+- app/services/solid_queue_web/stale_process_alert.rb
 - app/views/layouts/solid_queue_web/application.html.erb
+- app/views/solid_queue_web/audit/index.html.erb
 - app/views/solid_queue_web/dashboard/index.html.erb
 - app/views/solid_queue_web/failed_jobs/errors/index.html.erb
 - app/views/solid_queue_web/failed_jobs/index.html.erb
@@ -177,6 +182,9 @@ files:
 - app/views/solid_queue_web/search/index.html.erb
 - config/importmap.rb
 - config/routes.rb
+- db/migrate/01_create_solid_queue_web_audit_events.rb
+- lib/generators/solid_queue_web/install/migrations_generator.rb
+- lib/generators/solid_queue_web/install/templates/create_solid_queue_web_audit_events.rb.tt
 - lib/solid_queue_web.rb
 - lib/solid_queue_web/engine.rb
 - lib/solid_queue_web/version.rb