pgbus 0.9.7 → 0.9.8

data/lib/pgbus/cli/dlq.rb

@@ -0,0 +1,164 @@
+# frozen_string_literal: true
+
+require "optparse"
+
+module Pgbus
+  module CLI
+    # Dead-letter queue management for headless deployments and incident
+    # runbooks. Every operation routes through +Web::DataSource+ so the CLI
+    # shares the dashboard's exact semantics — origin-queue re-enqueue,
+    # transactional retry, lock release on discard, sensitive-payload
+    # filtering — with zero raw SQL or direct PGMQ calls.
+    module DLQ
+      module_function
+
+      ROW_FORMAT = "%-10s %-32s %-28s %-8s %s"
+
+      def start(args, data_source: Pgbus::Web::DataSource.new)
+        subcommand = args.first
+        rest = args[1..] || []
+
+        case subcommand
+        when "list"       then list(rest, data_source)
+        when "show"       then show(rest, data_source)
+        when "retry"      then retry_one(rest, data_source)
+        when "retry-all"  then retry_all(data_source)
+        when "purge"      then purge(rest, data_source)
+        else
+          warn "Unknown dlq subcommand: #{subcommand.inspect}" if subcommand
+          print_help
+          exit 1
+        end
+      end
+
+      def list(args, data_source)
+        options = parse_list_options(args)
+        messages = data_source.dlq_messages(page: options[:page], per_page: options[:per_page])
+
+        if messages.empty?
+          puts "No dead-letter messages."
+          return
+        end
+
+        print_list_table(messages)
+        puts "-" * 96
+        puts "Total dead-letter messages: #{data_source.dlq_total_count} " \
+             "(page #{options[:page]}, #{options[:per_page]} per page)"
+      end
+
+      def show(args, data_source)
+        msg_id = require_msg_id(args)
+        detail = data_source.dlq_message_detail(msg_id)
+        abort_missing(msg_id) unless detail
+
+        puts "msg_id:      #{detail[:msg_id]}"
+        puts "dlq queue:   #{detail[:queue_name]}"
+        puts "origin:      #{origin_queue(detail[:queue_name])}"
+        puts "read_ct:     #{detail[:read_ct]}"
+        puts "enqueued_at: #{detail[:enqueued_at]}"
+        puts "payload:"
+        puts Pgbus::Web::PayloadFilter.filter_json(detail[:message])
+      end
+
+      def retry_one(args, data_source)
+        msg_id = require_msg_id(args)
+        detail = data_source.dlq_message_detail(msg_id)
+        abort_missing(msg_id) unless detail
+
+        if data_source.retry_dlq_message(detail[:queue_name], detail[:msg_id])
+          puts "Re-enqueued msg_id #{detail[:msg_id]} to #{origin_queue(detail[:queue_name])}."
+        else
+          warn "Failed to retry msg_id #{detail[:msg_id]}."
+          exit 1
+        end
+      end
+
+      def retry_all(data_source)
+        count = data_source.retry_all_dlq
+        puts "Re-enqueued #{count} dead-letter message#{"s" unless count == 1}."
+      end
+
+      def purge(args, data_source)
+        if args.include?("--all")
+          purge_all(args, data_source)
+        else
+          purge_one(args, data_source)
+        end
+      end
+
+      def purge_one(args, data_source)
+        msg_id = require_msg_id(args)
+        detail = data_source.dlq_message_detail(msg_id)
+        abort_missing(msg_id) unless detail
+
+        if data_source.discard_dlq_message(detail[:queue_name], detail[:msg_id])
+          puts "Purged msg_id #{detail[:msg_id]} from #{detail[:queue_name]}."
+        else
+          warn "Failed to purge msg_id #{detail[:msg_id]}."
+          exit 1
+        end
+      end
+
+      def purge_all(args, data_source)
+        unless args.include?("--yes")
+          warn "Refusing to purge all dead-letter messages without --yes."
+          exit 1
+        end
+
+        count = data_source.discard_all_dlq
+        puts "Purged #{count} dead-letter message#{"s" unless count == 1}."
+      end
+
+      def parse_list_options(args)
+        options = { page: 1, per_page: 25 }
+        OptionParser.new do |opts|
+          opts.banner = "Usage: pgbus dlq list [options]"
+          opts.on("--page N", Integer, "Page number (default: 1)") { |v| options[:page] = v }
+          opts.on("--per-page N", Integer, "Messages per page (default: 25)") { |v| options[:per_page] = v }
+        end.parse!(args.dup)
+        options
+      end
+
+      def print_list_table(messages)
+        puts format(ROW_FORMAT, "MSG_ID", "DLQ QUEUE", "ORIGIN QUEUE", "READ_CT", "ENQUEUED_AT")
+        puts "-" * 96
+        messages.each do |m|
+          puts format(ROW_FORMAT, m[:msg_id], m[:queue_name], origin_queue(m[:queue_name]),
+                      m[:read_ct], m[:enqueued_at])
+        end
+      end
+
+      def origin_queue(dlq_queue_name)
+        dlq_queue_name.delete_suffix(Pgbus::DEAD_LETTER_SUFFIX)
+      end
+
+      def require_msg_id(args)
+        msg_id = args.find { |a| a !~ /\A--/ }
+        unless msg_id
+          warn "A msg_id is required."
+          exit 1
+        end
+        msg_id
+      end
+
+      def abort_missing(msg_id)
+        warn "No dead-letter message found with msg_id #{msg_id}."
+        exit 1
+      end
+
+      def print_help
+        puts <<~HELP
+          Usage: pgbus dlq <subcommand> [options]
+
+          Subcommands:
+            list [--page N] [--per-page N]  List dead-letter messages
+            show MSG_ID                     Show one message (payload filtered)
+            retry MSG_ID                    Re-enqueue a message to its origin queue
+            retry-all                       Re-enqueue every dead-letter message
+            purge MSG_ID                    Discard a single message
+            purge --all --yes               Discard every dead-letter message
+        HELP
+      end
+    end
+  end
+end

data/lib/pgbus/cli.rb

@@ -16,8 +16,12 @@ module Pgbus
         show_status
       when "queues"
         list_queues
+      when "dlq"
+        DLQ.start(args[1..] || [])
       when "mcp"
         start_mcp_server
+      when "doctor"
+        run_doctor
       when "version"
         puts "pgbus #{Pgbus::VERSION}"
       when "help", "--help", "-h"
@@ -107,7 +111,7 @@ module Pgbus
     def apply_capsule_filter(name)
       capsule = Pgbus.configuration.capsule_named(name)
       unless capsule
-        available = (Pgbus.configuration.workers || []).filter_map { |c| c[:name] || c["name"] }
+        available = (Pgbus.configuration.workers || []).filter_map { |c| c[:name] }
         raise ArgumentError,
               "no capsule named #{name.inspect} (available: #{available.join(", ")})"
       end
@@ -141,6 +145,15 @@ module Pgbus
       Pgbus::MCP::Runner.run
     end
 
+    # Runs the deployment preflight diagnostics and prints the report. Exits 1
+    # on any failed check so it can gate a deploy or CI run; exits 0 when all
+    # checks pass (warnings do not fail). Doctor itself never raises.
+    def run_doctor
+      doctor = Pgbus::Doctor.new
+      puts doctor.report
+      exit 1 unless doctor.success?
+    end
+
     def list_queues
       Pgbus.client.list_queues
       metrics = Pgbus.client.metrics
@@ -163,7 +176,11 @@ module Pgbus
           start    Start the Pgbus supervisor (workers + dispatcher)
           status   Show running Pgbus processes
           queues   List queues with metrics
+          dlq      Inspect and drain dead-letter queues
+                   (list/show/retry/retry-all/purge)
           mcp      Start the read-only MCP diagnostic server over stdio
+          doctor   Run environment diagnostics (config, DB, PGMQ, queues,
+                   LISTEN/NOTIFY, process liveness); exits 1 on any failure
           version  Show version
           help     Show this help
 

data/lib/pgbus/client/connection_health.rb

@@ -0,0 +1,194 @@
+# frozen_string_literal: true
+
+module Pgbus
+  class Client
+    # In-memory, process-local circuit breaker for database-down conditions.
+    #
+    # Distinct from Pgbus::CircuitBreaker (which is per-queue and persists its
+    # pause state in the database — useless when the database itself is down).
+    # This latch is a single shared object owned by Pgbus::Client that trips on
+    # raw connection failures across *any* operation, so a fleet of workers
+    # stops hammering a dead database and flooding the error tracker for the
+    # whole outage (issue #197).
+    #
+    # State machine:
+    #   closed    -> open       after OPEN_THRESHOLD consecutive ConnectionErrors
+    #   open      -> half_open  once the backoff window elapses (admits ONE probe)
+    #   half_open -> closed     on a successful probe (resets backoff)
+    #   half_open -> open       on a failed probe (backoff doubles, capped)
+    #
+    # Thresholds are constants, not configuration, mirroring CircuitBreaker:
+    # the values rarely need tuning and exposing them never proved useful.
+    #
+    # Thread safety: a single Mutex serializes every state transition. The
+    # guarded operation itself runs *outside* the lock (only the gate decision
+    # and the outcome recording are inside), so a slow probe never blocks other
+    # worker threads from failing fast.
+    class ConnectionHealth
+      # Consecutive ConnectionErrors that trip the breaker from closed to open.
+      OPEN_THRESHOLD = 5
+
+      # Initial backoff (seconds) on the first trip. Doubles on each re-open.
+      BASE_BACKOFF = 1.0
+
+      # Cap on the exponential backoff (seconds). After ~6 re-opens the curve
+      # plateaus here so a perpetually-down database stops probing more than
+      # once a minute.
+      MAX_BACKOFF = 60.0
+
+      def initialize(clock: -> { ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) },
+                     on_open: nil, on_close: nil)
+        @clock = clock
+        @on_open = on_open
+        @on_close = on_close
+        @mutex = Mutex.new
+        @state = :closed
+        @failure_count = 0
+        @trip_count = 0
+        @open_until = nil
+      end
+
+      # Gate an operation through the breaker.
+      #
+      # - closed / half-open-probe-admitted: yields, records the outcome.
+      # - open (window not elapsed) or half-open (probe already in flight):
+      #   raises Pgbus::ConnectionCircuitOpenError without yielding — no pool
+      #   checkout, no ConnectionError, no ErrorReporter noise.
+      #
+      # The breaker never swallows an error — every exception propagates. A
+      # ConnectionError in the closed state counts toward the trip threshold; a
+      # failure of the single half-open probe (on ANY error) re-opens the
+      # breaker; any other closed-state error is not counted.
+      def run_guarded
+        admitted = admit_or_raise # :run (was closed) or :probe (was open)
+
+        begin
+          result = yield
+        rescue StandardError => e
+          record_failure(admitted, e)
+          raise
+        end
+
+        record_success(admitted)
+        result
+      end
+
+      def closed?
+        @mutex.synchronize { @state == :closed }
+      end
+
+      def open?
+        @mutex.synchronize { @state == :open }
+      end
+
+      private
+
+      # Decide whether this call may proceed. Runs entirely inside the mutex.
+      # Returns the admission kind so the outcome recorders can attribute the
+      # result to THIS call (the probe vs. an ordinary closed-state read) —
+      # critical because the block runs outside the lock, so between admit and
+      # record the state may have changed under another thread.
+      #
+      # When admitting a probe from the open state it flips @state to :half_open
+      # so a second racing thread sees :half_open and is rejected — that is the
+      # single-probe guarantee.
+      def admit_or_raise
+        @mutex.synchronize do
+          case @state
+          when :closed
+            :run
+          when :open
+            raise_open unless window_elapsed?
+
+            @state = :half_open # admit exactly this thread as the probe
+            :probe
+          when :half_open
+            raise_open # a probe is already in flight
+          end
+        end
+      end
+
+      def record_success(admitted)
+        callback = nil
+        @mutex.synchronize do
+          if admitted == :probe && @state == :half_open
+            # The single probe succeeded — the database is reachable again.
+            close!
+            callback = @on_close
+          elsif @state == :closed
+            # An ordinary read completed while closed: clear the failure streak.
+            # Do NOT force-close here — a straggler read admitted while closed
+            # can return AFTER other threads have tripped the breaker; closing
+            # would wipe a freshly-opened breaker (issue #197 straggler race).
+            @failure_count = 0
+          end
+        end
+        callback&.call
+      end
+
+      # Any exception ends the operation. A probe failure — whether a
+      # ConnectionError or something else (e.g. Pgbus::ReadTimeoutError from a
+      # hung socket, which the read paths raise from INSIDE the guard) — must
+      # re-open the breaker. If it didn't, a non-ConnectionError probe would
+      # leave the latch wedged in :half_open forever, admitting no further
+      # probes and never recovering (issue #197 wedge). Ordinary closed-state
+      # failures only count toward the threshold when they are ConnectionErrors.
+      def record_failure(admitted, error)
+        callback_backoff = nil
+        @mutex.synchronize do
+          if admitted == :probe && @state == :half_open
+            callback_backoff = reopen
+          elsif @state == :closed && connection_error?(error)
+            @failure_count += 1
+            callback_backoff = reopen if @failure_count >= OPEN_THRESHOLD
+          end
+          # Otherwise (a straggler failing after another thread already changed
+          # the state) leave the current window untouched.
+        end
+        @on_open&.call(callback_backoff) if callback_backoff
+      end
+
+      # Transition to closed and fully reset. Must be called with @mutex held.
+      def close!
+        @state = :closed
+        @failure_count = 0
+        @trip_count = 0
+        @open_until = nil
+      end
+
+      # Transition to open, arm the backoff window, and return the window size
+      # so the caller can fire the on_open callback outside the lock. Must be
+      # called with @mutex held.
+      def reopen
+        @trip_count += 1
+        backoff = calculate_backoff(@trip_count)
+        @state = :open
+        @failure_count = 0
+        @open_until = clock_now + backoff
+        backoff
+      end
+
+      def window_elapsed?
+        @open_until.nil? || clock_now >= @open_until
+      end
+
+      def calculate_backoff(trip_count)
+        backoff = BASE_BACKOFF * (2**(trip_count - 1))
+        [backoff, MAX_BACKOFF].min
+      end
+
+      def raise_open
+        raise Pgbus::ConnectionCircuitOpenError,
+              "database connection circuit is open (#{OPEN_THRESHOLD}+ consecutive connection failures)"
+      end
+
+      def connection_error?(error)
+        defined?(PGMQ::Errors::ConnectionError) && error.is_a?(PGMQ::Errors::ConnectionError)
+      end
+
+      def clock_now
+        @clock.call
+      end
+    end
+  end
+end