pgbus 0.9.4 → 0.9.6

data/lib/pgbus/mcp/tools/health_tool.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Pgbus
+  module MCP
+    module Tools
+      # Top-level health verdict: OK / DEGRADED / STALLED, with the reasons
+      # that drove it. Delegates the actual analysis to HealthAnalyzer, which
+      # reads the same DataSource surface the dashboard uses. This is the
+      # single tool that catches the silent-worker-wedge class of incident.
+      class HealthTool < BaseTool
+        tool_name "pgbus_health"
+        title "Pgbus Health"
+        description <<~DESC
+          Top-level pgbus health verdict. Returns OK, DEGRADED, or STALLED with
+          the specific reasons. STALLED means the silent-wedge condition:
+          queues have visible backlog while workers are heart-beating but not
+          claiming (or a worker's claim loop has stalled). DEGRADED covers
+          stale processes, paused queues holding a backlog, dead-letter
+          buildup, or a long-running transaction pinning the MVCC horizon. Use
+          this first when triaging — it summarizes the whole system in one call
+          and is suitable for automated alerting.
+        DESC
+
+        input_schema(properties: {}, required: [])
+
+        def self.call(server_context: nil)
+          data_source = data_source_from(server_context)
+          json_response(HealthAnalyzer.new(data_source).verdict)
+        end
+      end
+    end
+  end
+end

data/lib/pgbus/mcp/tools/job_detail_tool.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module Pgbus
+  module MCP
+    module Tools
+      # Inspect a single enqueued message by queue + msg_id. Returns its
+      # metadata (read_ct, vt, enqueued_at, last_read_at); the body and headers
+      # are redacted unless payloads are explicitly allowed. Maps to
+      # DataSource#job_detail.
+      class JobDetailTool < BaseTool
+        tool_name "pgbus_job_detail"
+        title "Pgbus Job Detail"
+        description <<~DESC
+          Inspect one enqueued message: read_ct, visibility timeout (vt),
+          enqueued_at, last_read_at. Pass the full physical queue name and the
+          numeric msg_id. The message body and headers are redacted unless the
+          server allows payloads and include_payloads is set.
+        DESC
+
+        input_schema(
+          properties: {
+            queue: { type: "string", description: "Full physical queue name (e.g. \"pgbus_default\")." },
+            msg_id: { type: "integer", description: "Numeric PGMQ message id." },
+            include_payloads: {
+              type: "boolean",
+              description: "Include the raw message body/headers. Only honored when the server allows payloads."
+            }
+          },
+          required: %w[queue msg_id]
+        )
+
+        def self.call(queue:, msg_id:, include_payloads: false, server_context: nil)
+          data_source = data_source_from(server_context)
+          detail = data_source.job_detail(queue, msg_id)
+          return error_response("Message #{msg_id} not found in #{queue}") unless detail
+
+          json_response({ job: detail }, server_context: server_context, include_payloads: include_payloads)
+        end
+      end
+    end
+  end
+end

data/lib/pgbus/mcp/tools/jobs_tool.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+module Pgbus
+  module MCP
+    module Tools
+      # Paginated list of enqueued jobs/messages for a queue (or across all
+      # non-DLQ queues). Returns metadata — msg_id, read_ct, vt, enqueued_at,
+      # queue_name — with message bodies and headers redacted unless payloads
+      # are explicitly allowed. Maps to DataSource#jobs.
+      class JobsTool < BaseTool
+        tool_name "pgbus_jobs"
+        title "Pgbus Jobs"
+        description <<~DESC
+          List enqueued jobs/messages with read_ct, visibility timeout (vt) and
+          enqueued_at. Pass a full physical queue name to scope to one queue,
+          or omit it to span all non-DLQ queues. Paginated (default 25 per
+          page, capped at 100). Message bodies and headers are redacted unless
+          the server was started with payloads allowed and include_payloads is
+          set.
+        DESC
+
+        MAX_PER_PAGE = 100
+        MAX_PAGE = 1_000
+
+        input_schema(
+          properties: {
+            queue: {
+              type: "string",
+              description: "Full physical queue name (e.g. \"pgbus_default\"). Omit to span all non-DLQ queues."
+            },
+            page: { type: "integer", description: "1-based page number (default 1).", minimum: 1 },
+            per_page: { type: "integer", description: "Rows per page (default 25, max 100).", minimum: 1 },
+            include_payloads: {
+              type: "boolean",
+              description: "Include raw message bodies/headers. Only honored when the server allows payloads."
+            }
+          },
+          required: []
+        )
+
+        def self.call(queue: nil, page: 1, per_page: 25, include_payloads: false, server_context: nil)
+          data_source = data_source_from(server_context)
+          per_page = per_page.to_i.clamp(1, MAX_PER_PAGE)
+          page = page.to_i.clamp(1, MAX_PAGE)
+          rows = data_source.jobs(queue_name: queue, page: page, per_page: per_page)
+
+          json_response(
+            { queue: queue, page: page, per_page: per_page, jobs: rows },
+            server_context: server_context,
+            include_payloads: include_payloads
+          )
+        end
+      end
+    end
+  end
+end

data/lib/pgbus/mcp/tools/locks_tool.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Pgbus
+  module MCP
+    module Tools
+      # Lists active job uniqueness locks (the leaked-uniqueness-lock
+      # diagnostic). Maps to DataSource#job_locks — lock_key, queue_name,
+      # msg_id, created_at, age_seconds. No payloads involved.
+      class LocksTool < BaseTool
+        tool_name "pgbus_locks"
+        title "Pgbus Locks"
+        description <<~DESC
+          List active job uniqueness locks with their lock_key, queue_name,
+          msg_id, and age in seconds. A lock that outlives its message
+          indicates a leaked uniqueness key blocking re-enqueues. Returns the
+          100 most recent locks.
+        DESC
+
+        input_schema(properties: {}, required: [])
+
+        def self.call(server_context: nil)
+          data_source = data_source_from(server_context)
+          json_response({ locks: data_source.job_locks })
+        end
+      end
+    end
+  end
+end

data/lib/pgbus/mcp/tools/processes_tool.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Pgbus
+  module MCP
+    module Tools
+      # Lists every pgbus process (worker/consumer/dispatcher/scheduler/
+      # supervisor/outbox/streamer) with kind, pid, heartbeat age, derived
+      # status and the claim-loop "stalled" verdict. Maps to
+      # DataSource#processes — which already computes :status as :healthy,
+      # :stale, or :stalled (the silent-wedge signal from #179/#181).
+      class ProcessesTool < BaseTool
+        tool_name "pgbus_processes"
+        title "Pgbus Processes"
+        description <<~DESC
+          List every pgbus process with its kind (worker, consumer,
+          dispatcher, scheduler, supervisor, outbox, streamer), hostname, pid,
+          last heartbeat time, and derived status. Status is "healthy",
+          "stale" (no recent heartbeat), or "stalled" (worker heart-beating but
+          its claim loop has stopped advancing — the silent-wedge condition).
+          Use this to answer "are workers alive but not claiming?".
+        DESC
+
+        input_schema(properties: {}, required: [])
+
+        def self.call(server_context: nil)
+          data_source = data_source_from(server_context)
+          json_response({ processes: data_source.processes })
+        end
+      end
+    end
+  end
+end

data/lib/pgbus/mcp/tools/queue_detail_tool.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module Pgbus
+  module MCP
+    module Tools
+      # Per-queue detail: metrics, paused state, and table health (dead tuples,
+      # bloat, vacuum age). Maps to DataSource#queue_detail + #queue_paused? +
+      # #queue_health_detail. The +name+ is the full physical queue name as
+      # returned by pgbus_queues (e.g. "pgbus_default"), not the logical name.
+      class QueueDetailTool < BaseTool
+        tool_name "pgbus_queue_detail"
+        title "Pgbus Queue Detail"
+        description <<~DESC
+          Detailed status for a single queue: depth, visible count, message
+          ages, lifetime total, whether the queue is paused, and per-table
+          health (live/dead tuples, bloat ratio, last vacuum age). Pass the
+          full physical queue name as shown by pgbus_queues (e.g.
+          "pgbus_default").
+        DESC
+
+        input_schema(
+          properties: {
+            name: {
+              type: "string",
+              description: "Full physical queue name, e.g. \"pgbus_default\"."
+            }
+          },
+          required: ["name"]
+        )
+
+        def self.call(name:, server_context: nil)
+          data_source = data_source_from(server_context)
+          detail = data_source.queue_detail(name)
+          return error_response("Queue not found: #{name}") unless detail
+
+          json_response(
+            detail.merge(
+              paused: data_source.queue_paused?(name),
+              health: data_source.queue_health_detail(name)
+            )
+          )
+        end
+      end
+    end
+  end
+end

data/lib/pgbus/mcp/tools/queues_tool.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Pgbus
+  module MCP
+    module Tools
+      # Lists every queue with depth, visible count, oldest-message age and
+      # throughput — the "are queues backed up?" diagnostic. Maps to
+      # DataSource#queues_with_metrics. No payloads are involved.
+      class QueuesTool < BaseTool
+        tool_name "pgbus_queues"
+        title "Pgbus Queues"
+        description <<~DESC
+          List all pgbus queues with their depth (queue_length), visible count
+          (messages whose visibility timeout has expired and are ready to be
+          claimed), oldest/newest message age in seconds, lifetime total, and
+          paused state. Use this to answer "are any queues backed up?".
+        DESC
+
+        input_schema(properties: {}, required: [])
+
+        def self.call(server_context: nil)
+          data_source = data_source_from(server_context)
+          json_response({ queues: data_source.queues_with_metrics })
+        end
+      end
+    end
+  end
+end

data/lib/pgbus/mcp/tools/recurring_tool.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Pgbus
+  module MCP
+    module Tools
+      # Recurring task schedule with last-run / next-run times. Maps to
+      # DataSource#recurring_tasks. The :arguments field is not part of the
+      # list payload, so no redaction is needed here.
+      class RecurringTool < BaseTool
+        tool_name "pgbus_recurring"
+        title "Pgbus Recurring Tasks"
+        description <<~DESC
+          List recurring (cron) tasks with their schedule, human-readable
+          schedule, queue, enabled state, last_run_at and next_run_at. Use this
+          to confirm scheduled work is firing on time.
+        DESC
+
+        input_schema(properties: {}, required: [])
+
+        def self.call(server_context: nil)
+          data_source = data_source_from(server_context)
+          json_response({ recurring_tasks: data_source.recurring_tasks })
+        end
+      end
+    end
+  end
+end

data/lib/pgbus/mcp/tools/stats_tool.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module Pgbus
+  module MCP
+    module Tools
+      # Recent job status counts + summary (success / failed / dead-lettered,
+      # durations). Maps to DataSource#job_status_counts + #job_stats_summary.
+      class StatsTool < BaseTool
+        tool_name "pgbus_stats"
+        title "Pgbus Stats"
+        description <<~DESC
+          Recent job statistics over the last N minutes (default 60, max 1440):
+          status counts (success/failed/dead_lettered) and a summary including
+          average and max duration. Use this to gauge error rates and latency.
+        DESC
+
+        MAX_MINUTES = 1440
+
+        input_schema(
+          properties: {
+            minutes: { type: "integer", description: "Look-back window in minutes (default 60, max 1440).", minimum: 1 }
+          },
+          required: []
+        )
+
+        def self.call(minutes: 60, server_context: nil)
+          data_source = data_source_from(server_context)
+          window = minutes.to_i.clamp(1, MAX_MINUTES)
+          json_response(
+            {
+              minutes: window,
+              status_counts: data_source.job_status_counts(minutes: window),
+              summary: data_source.job_stats_summary(minutes: window)
+            }
+          )
+        end
+      end
+    end
+  end
+end

data/lib/pgbus/mcp/tools/throughput_tool.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module Pgbus
+  module MCP
+    module Tools
+      # Recent job throughput as a time series. Maps to
+      # DataSource#job_throughput. Used to answer "is the system draining?".
+      class ThroughputTool < BaseTool
+        tool_name "pgbus_throughput"
+        title "Pgbus Throughput"
+        description <<~DESC
+          Recent job throughput as a time series of {time, count} buckets over
+          the last N minutes (default 60, max 1440). Use this to confirm the
+          system is draining — a flat-zero series alongside a backlog is the
+          wedge signature.
+        DESC
+
+        MAX_MINUTES = 1440
+
+        input_schema(
+          properties: {
+            minutes: { type: "integer", description: "Look-back window in minutes (default 60, max 1440).", minimum: 1 }
+          },
+          required: []
+        )
+
+        def self.call(minutes: 60, server_context: nil)
+          data_source = data_source_from(server_context)
+          window = minutes.to_i.clamp(1, MAX_MINUTES)
+          json_response({ minutes: window, throughput: data_source.job_throughput(minutes: window) })
+        end
+      end
+    end
+  end
+end

data/lib/pgbus/mcp.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+module Pgbus
+  # Optional read-only MCP (Model Context Protocol) diagnostic server.
+  #
+  # This namespace is deliberately kept out of pgbus's Zeitwerk loader because
+  # its tool classes subclass `MCP::Tool` from the optional `mcp` gem.
+  # Referencing that gem at autoload time would force every pgbus user to
+  # install it. Instead the subsystem is loaded explicitly via {load!}, which
+  # is called by the `pgbus mcp` CLI command (and by specs).
+  module MCP
+    # Files in dependency order: the gem first, then the base classes the
+    # tools depend on, then the tools, then the server/runner.
+    REQUIRES = %w[
+      pgbus/mcp/redactor
+      pgbus/mcp/base_tool
+      pgbus/mcp/health_analyzer
+      pgbus/mcp/tools/health_tool
+      pgbus/mcp/tools/queues_tool
+      pgbus/mcp/tools/queue_detail_tool
+      pgbus/mcp/tools/processes_tool
+      pgbus/mcp/tools/jobs_tool
+      pgbus/mcp/tools/job_detail_tool
+      pgbus/mcp/tools/dlq_tool
+      pgbus/mcp/tools/dlq_detail_tool
+      pgbus/mcp/tools/locks_tool
+      pgbus/mcp/tools/throughput_tool
+      pgbus/mcp/tools/stats_tool
+      pgbus/mcp/tools/recurring_tool
+      pgbus/mcp/server
+      pgbus/mcp/runner
+      pgbus/mcp/rack_app
+    ].freeze
+
+    module_function
+
+    # Require the `mcp` gem and the whole pgbus MCP subsystem. Idempotent.
+    # Raises Pgbus::Error with an actionable message if the gem is missing.
+    #
+    # Only the `require "mcp"` is wrapped — a LoadError from an internal
+    # `pgbus/mcp/...` path (typo, missing file) propagates verbatim so the
+    # real broken require shows up in the message instead of being masked
+    # as "add gem `mcp`".
+    def load!
+      begin
+        require "mcp"
+      rescue LoadError
+        raise Pgbus::Error,
+              "The pgbus MCP server requires the `mcp` gem. Add `gem \"mcp\"` to your Gemfile and run `bundle install`."
+      end
+
+      REQUIRES.each { |path| require path }
+    end
+  end
+end

data/lib/pgbus/process/heartbeat.rb

@@ -11,10 +11,11 @@ module Pgbus
 
       attr_reader :process_entry
 
-      def initialize(kind:, metadata: {}, on_beat: nil)
+      def initialize(kind:, metadata: {}, on_beat: nil, loop_tick_supplier: nil)
         @kind = kind
         @metadata = metadata
         @on_beat = on_beat
+        @loop_tick_supplier = loop_tick_supplier
         @timer = nil
       end
 
@@ -33,7 +34,12 @@ module Pgbus
         return unless @process_id
 
         @on_beat&.call
-        ProcessEntry.where(id: @process_id).update_all(last_heartbeat_at: Time.current)
+        updates = { last_heartbeat_at: Time.current }
+        if @loop_tick_supplier
+          tick = @loop_tick_supplier.call
+          updates[:metadata] = @metadata.merge("loop_tick_at" => tick&.to_f)
+        end
+        ProcessEntry.where(id: @process_id).update_all(updates)
       rescue StandardError => e
         Pgbus.logger.warn { "[Pgbus] Heartbeat failed: #{e.message}" }
       end

data/lib/pgbus/process/supervisor.rb

@@ -6,6 +6,7 @@ module Pgbus
       include SignalHandler
 
       FORK_WAIT = 1 # seconds between fork checks
+      WATCHDOG_INTERVAL = 10 # seconds between stall checks
 
       attr_reader :config
 
@@ -13,6 +14,7 @@ module Pgbus
         @config = config
         @forks = {}
         @shutting_down = false
+        @last_watchdog_at = monotonic_now
       end
 
       def run
@@ -240,6 +242,7 @@ module Pgbus
 
           process_signals
           reap_children
+          check_stalled_workers unless @shutting_down
           interruptible_sleep(FORK_WAIT)
         end
       end
@@ -280,6 +283,43 @@ module Pgbus
         end
       end
 
+      def check_stalled_workers
+        now = monotonic_now
+        return if (now - @last_watchdog_at) < WATCHDOG_INTERVAL
+
+        @last_watchdog_at = now
+        threshold = config.stall_threshold
+        return unless threshold&.positive?
+
+        worker_pids = @forks.select { |_, info| info[:type] == :worker }.keys
+        return if worker_pids.empty?
+
+        rows = ProcessEntry.where(kind: "worker", pid: worker_pids).to_a
+        rows.each do |entry|
+          meta = entry.metadata
+          next unless meta.is_a?(Hash)
+
+          loop_tick = meta["loop_tick_at"]
+          next unless loop_tick
+
+          age = Time.current.to_f - loop_tick.to_f
+          next unless age > threshold
+
+          pid = entry.pid
+          Pgbus.logger.error do
+            "[Pgbus] Supervisor watchdog: worker pid=#{pid} claim loop stalled " \
+              "(loop_tick_at age=#{age.round(1)}s > threshold=#{threshold}s), sending SIGKILL"
+          end
+          begin
+            ::Process.kill("KILL", pid)
+          rescue Errno::ESRCH
+            # already gone
+          end
+        end
+      rescue StandardError => e
+        Pgbus.logger.warn { "[Pgbus] Supervisor watchdog check failed: #{e.message}" }
+      end
+
       def signal_children(sig)
         @forks.each_key do |pid|
           ::Process.kill(sig, pid)
@@ -318,6 +358,10 @@ module Pgbus
         @heartbeat.start
       end
 
+      def monotonic_now
+        ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+      end
+
       def shutdown
         # Wait for all children with timeout
         deadline = Time.now + 30

data/lib/pgbus/process/worker.rb

@@ -37,6 +37,7 @@ module Pgbus
         @jobs_processed = Concurrent::AtomicFixnum.new(0)
         @jobs_failed = Concurrent::AtomicFixnum.new(0)
         @in_flight = Concurrent::AtomicFixnum.new(0)
+        @loop_tick_at = Concurrent::AtomicReference.new(nil)
         @rate_counter = RateCounter.new(:processed, :failed, :dequeued)
         @started_at = Time.current
         @started_at_monotonic = monotonic_now
@@ -82,6 +83,7 @@ module Pgbus
         end
 
         loop do
+          stamp_loop_tick
           process_signals
           check_recycle
           refresh_wildcard_queues
@@ -492,7 +494,8 @@ module Pgbus
             queues: queues, threads: threads, pid: ::Process.pid,
             execution_mode: @execution_mode, consumer_priority: @consumer_priority
           },
-          on_beat: -> { @rate_counter.snapshot! }
+          on_beat: -> { @rate_counter.snapshot! },
+          loop_tick_supplier: -> { @loop_tick_at.get }
         )
         @heartbeat.start
       end
@@ -512,6 +515,14 @@ module Pgbus
       def monotonic_now
         ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
       end
+
+      # Stamp the loop-progress beacon with a wall-clock timestamp.
+      # Wall clock is required because the supervisor watchdog reads
+      # this value from a different process (cross-fork) and the
+      # dashboard reads it from a different host.
+      def stamp_loop_tick
+        @loop_tick_at.set(Time.now.to_f)
+      end
     end
   end
 end

data/lib/pgbus/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Pgbus
-  VERSION = "0.9.4"
+  VERSION = "0.9.6"
 end

data/lib/pgbus/web/data_source.rb

@@ -41,7 +41,17 @@ module Pgbus
       # Queues — query via ActiveRecord for reliability in web processes
       # (avoids PGMQ client connection issues when the web server uses a
       # different connection lifecycle than the worker processes).
+      # Memoized for the lifetime of this data-source instance (one per web
+      # request — see ApplicationController#data_source). A single page can ask
+      # for queue metrics more than once (e.g. the DLQ page reads both the
+      # message rows and the total count); without the memo each call re-runs
+      # the meta query + batched metrics. Mutations redirect to a fresh request
+      # with a new instance, so a per-request memo never serves stale data.
       def queues_with_metrics
+        @queues_with_metrics ||= fetch_queues_with_metrics
+      end
+
+      def fetch_queues_with_metrics
         queue_names = connection.select_values("SELECT queue_name FROM pgmq.meta ORDER BY queue_name")
         return [] if queue_names.empty?
 
@@ -53,6 +63,7 @@ module Pgbus
         Pgbus.logger.error { "[Pgbus::Web] Error fetching queue metrics: #{e.class}: #{e.message}" }
         []
       end
+      private :fetch_queues_with_metrics
 
       # name is the full PGMQ queue name (e.g. "pgbus_default") as returned
       # by queues_with_metrics. No prefix is added.
@@ -276,6 +287,16 @@ module Pgbus
         []
       end
 
+      def dlq_total_count
+        dlq_suffix = Pgbus::DEAD_LETTER_SUFFIX
+        queues_with_metrics
+          .select { |q| q[:name].end_with?(dlq_suffix) }
+          .sum { |q| q[:queue_length] }
+      rescue StandardError => e
+        Pgbus.logger.debug { "[Pgbus::Web] Error fetching DLQ count: #{e.message}" }
+        0
+      end
+
       def dlq_message_detail(msg_id)
         dlq_suffix = Pgbus::DEAD_LETTER_SUFFIX
         queues = queues_with_metrics.select { |q| q[:name].end_with?(dlq_suffix) }
@@ -1195,19 +1216,36 @@ module Pgbus
         heartbeat = row["last_heartbeat_at"]
         heartbeat_time = heartbeat.is_a?(String) ? Time.parse(heartbeat) : heartbeat
         stale = heartbeat_time && (Time.now - heartbeat_time) > Process::Heartbeat::ALIVE_THRESHOLD
+        metadata = row["metadata"].is_a?(String) ? JSON.parse(row["metadata"]) : row["metadata"]
+        status = derive_process_status(stale, metadata, row["kind"])
 
         {
           id: row["id"].to_i,
           kind: row["kind"],
           hostname: row["hostname"],
           pid: row["pid"].to_i,
-          metadata: row["metadata"].is_a?(String) ? JSON.parse(row["metadata"]) : row["metadata"],
+          metadata: metadata,
           last_heartbeat_at: heartbeat_time,
-          healthy: !stale,
+          healthy: status == :healthy,
+          status: status,
           created_at: row["created_at"]
         }
       end
 
+      def derive_process_status(stale, metadata, kind)
+        return :stale if stale
+        return :healthy unless kind == "worker" && metadata.is_a?(Hash)
+
+        loop_tick = metadata["loop_tick_at"]
+        return :healthy unless loop_tick
+
+        threshold = Pgbus.configuration.stall_threshold
+        return :healthy unless threshold&.positive?
+
+        age = Time.now.to_f - loop_tick.to_f
+        age > threshold ? :stalled : :healthy
+      end
+
       def format_batch(record)
         total = record.total_jobs
         done = record.completed_jobs + record.discarded_jobs