pgbus 0.17.0 → 0.17.2

data/lib/pgbus/process/supervisor.rb

@@ -171,7 +171,7 @@ module Pgbus
       private_constant :ROLE_FLAGS
 
       def log_boot_banner
-        Pgbus.logger.info { "[Pgbus] boot: pgbus #{Pgbus::VERSION} pid=#{::Process.pid}" }
+        Pgbus.logger.info { "[Pgbus] boot: pgbus #{Pgbus::VERSION} pid=#{::Process.pid} jit=#{RubyJit.label}" }
         Pgbus.logger.info do
           "[Pgbus] boot: connection=#{redacted_connection_target} pool=#{banner_field { config.resolved_pool_size }}"
         end
@@ -195,9 +195,11 @@ module Pgbus
           name = worker_config[:name] || worker_config["name"] || "anonymous"
           queues = worker_config[:queues] || worker_config["queues"] || [config.default_queue]
           threads = worker_config[:threads] || worker_config["threads"] || 5
+          processes = config.processes_for(worker_config)
           mode = banner_field { config.execution_mode_for(worker_config) }
           Pgbus.logger.info do
-            "[Pgbus] boot: capsule=#{name} queues=#{Array(queues).join(",")} threads=#{threads} mode=#{mode}"
+            "[Pgbus] boot: capsule=#{name} queues=#{Array(queues).join(",")} threads=#{threads} " \
+              "processes=#{processes} mode=#{mode}"
           end
         end
       end
@@ -284,12 +286,7 @@ module Pgbus
         # --dispatcher-only CLI flags). Each role is gated by
         # config.role_enabled?, which returns true unless +config.roles+ has
         # been narrowed.
-        if config.role_enabled?(:workers)
-          # slot is the child's position in the config array — it keys the
-          # crash-streak tracking so identically-configured siblings don't
-          # share (and reset) each other's restart backoff.
-          Array(config.workers).each_with_index { |worker_config, slot| fork_worker(worker_config, slot: slot) }
-        end
+        boot_workers if config.role_enabled?(:workers)
 
         fork_dispatcher if config.role_enabled?(:dispatcher)
         boot_scheduler if config.role_enabled?(:scheduler)
@@ -297,7 +294,22 @@ module Pgbus
         boot_outbox_poller if config.role_enabled?(:outbox)
       end
 
-      def fork_worker(worker_config, slot: nil)
+      # A capsule with processes: N forks N identical workers (issue #503).
+      # slot numbers every worker fork across all capsules — it keys the
+      # crash-streak tracking so identically-configured siblings (in one
+      # capsule or in several) don't share or reset each other's restart
+      # backoff. process_index is the fork's position within its capsule.
+      def boot_workers
+        slot = 0
+        Array(config.workers).each do |worker_config|
+          config.processes_for(worker_config).times do |process_index|
+            fork_worker(worker_config, slot: slot, process_index: process_index)
+            slot += 1
+          end
+        end
+      end
+
+      def fork_worker(worker_config, slot: nil, process_index: 0)
         note_intended_child
         queues = worker_config[:queues] || [config.default_queue]
         threads = worker_config[:threads] || 5
@@ -330,6 +342,9 @@ module Pgbus
             queues: queues, threads: threads, config: config,
             single_active_consumer: single_active, consumer_priority: priority,
             execution_mode: exec_mode, group_mode: grp_mode,
+            read_ahead: config.read_ahead_for(worker_config),
+            recycle_limits: worker_config.slice(*Configuration::RECYCLE_LIMIT_KEYS),
+            capsule: worker_config[:name], process: process_label(worker_config, process_index),
             liveness_pipe: liveness_writer, wake_pipe: wake_reader
           )
           worker.run
@@ -351,7 +366,7 @@ module Pgbus
         close_pipe(wake_reader)
         register_fork_with_hub(pid, wake_writer, queues)
         @forks[pid] = {
-          type: :worker, config: worker_config, slot: slot, spawned_at: monotonic_now,
+          type: :worker, config: worker_config, slot: slot, process_index: process_index, spawned_at: monotonic_now,
           liveness_reader: liveness_reader, last_pipe_tick_at: monotonic_now, pipe_seen: false,
           wake_writer: wake_writer
         }
@@ -364,6 +379,13 @@ module Pgbus
         ErrorReporter.report(e, { action: "fork_worker", queues: queues })
       end
 
+      # "2/4" for the second of a capsule's four processes; nil for a
+      # single-process capsule, where it would only add noise.
+      def process_label(worker_config, process_index)
+        total = config.processes_for(worker_config)
+        "#{process_index + 1}/#{total}" if total > 1
+      end
+
       # Hand the hub a worker fork's routing entry: explicit queues as
       # physical names, "*" as the unconditional wildcard flag (the hub wakes
       # wildcard forks for every channel, so the fork's own resolved set never
@@ -498,6 +520,7 @@ module Pgbus
           setup_child_process
           load_rails_app
           consumer = Consumer.new(topics: topics, threads: threads, config: config,
+                                  read_ahead: config.read_ahead_for(consumer_config),
                                   liveness_pipe: liveness_writer, wake_pipe: wake_reader)
           consumer.run
         end
@@ -680,7 +703,7 @@ module Pgbus
       def restart_child(info)
         case info[:type]
         when :worker
-          fork_worker(info[:config], slot: info[:slot])
+          fork_worker(info[:config], slot: info[:slot], process_index: info[:process_index] || 0)
         when :dispatcher
           fork_dispatcher
         when :scheduler
@@ -880,7 +903,7 @@ module Pgbus
       def start_heartbeat
         @heartbeat = Heartbeat.new(
           kind: "supervisor",
-          metadata: { pid: ::Process.pid, hostname: Socket.gethostname }
+          metadata: { pid: ::Process.pid, hostname: Socket.gethostname, started_at: Time.now.to_f }
         )
         @heartbeat.start
       end

data/lib/pgbus/process/worker.rb

@@ -7,7 +7,7 @@ module Pgbus
     class Worker
       include SignalHandler
 
-      attr_reader :queues, :threads, :config, :execution_mode,
+      attr_reader :queues, :threads, :config, :execution_mode, :read_ahead,
                   :rate_counter, :wake_signal, :restore_streak, :lifecycle
       # stat_buffer is writable so a test can swap in a buffer double after
       # construction to assert graceful_shutdown / check_recycle flush it. The
@@ -35,7 +35,8 @@ module Pgbus
                      rate_counter: nil, wake_signal: nil, stat_buffer: :default,
                      notify_listener: nil, notify_retry_at: 0.0,
                      notify_retry_backoff: NOTIFY_RETRY_BASE_SECONDS,
-                     started_at_monotonic: nil, wake_pipe: nil)
+                     started_at_monotonic: nil, wake_pipe: nil, read_ahead: nil,
+                     recycle_limits: {}, capsule: nil, process: nil)
         @queues = Array(queues)
         @initial_queues = @queues.dup.freeze
         @wildcard = @queues.include?("*")
@@ -65,7 +66,15 @@ module Pgbus
         @last_wildcard_resolve = nil
         @jobs_processed = Concurrent::AtomicFixnum.new(0)
         @jobs_failed = Concurrent::AtomicFixnum.new(0)
+        # in_flight counts every claimed message: buffered and running.
         @in_flight = Concurrent::AtomicFixnum.new(0)
+        @read_ahead = read_ahead || config.read_ahead
+        # Capsule-level recycle limits (issue #503); each unset key falls back
+        # to the global value at check time, so a config change still applies.
+        @recycle_limits = recycle_limits
+        @capsule = capsule
+        @process = process
+        @claim_buffer = ClaimBuffer.new(config: config)
         @loop_tick_at = Concurrent::AtomicReference.new(nil)
         @rate_counter = rate_counter || RateCounter.new(:processed, :failed, :dequeued)
         @started_at = Time.current
@@ -119,6 +128,7 @@ module Pgbus
           jobs_processed: @jobs_processed.value,
           jobs_failed: @jobs_failed.value,
           in_flight: @in_flight.value,
+          buffered: @claim_buffer.size,
           state: @lifecycle.state,
           execution_mode: @execution_mode,
           consumer_priority: @consumer_priority,
@@ -184,6 +194,9 @@ module Pgbus
           check_recycle
           refresh_wildcard_queues
           ensure_notify_listener
+          # Draining, paused or stopped: claims nobody will run here go back
+          # to the queue now rather than when their timeout runs out.
+          return_claims unless @lifecycle.can_process?
 
           break if @lifecycle.stopped?
           # quiesced? (all slots free), not idle? (any slot free) — exiting
@@ -240,32 +253,53 @@ module Pgbus
 
       private
 
+      # One loop step (issue #486): feed buffered claims to free slots, read
+      # the deficit (free slots + read_ahead - buffered, capped by
+      # prefetch_limit), feed again. With read_ahead 0 the buffer never
+      # outlives the step and qty is the free slot count, as before. Waits
+      # only when the step moved nothing, so a full pool with a topped-up
+      # buffer sleeps until a slot frees (the pool's on_state_change wakes it).
       def claim_and_execute
-        poll_interval = wake_timeout
-
-        idle = @pool.available_capacity
-        return @wake_signal.wait(timeout: poll_interval) if idle <= 0
-
-        if config.prefetch_limit
-          available = config.prefetch_limit - @in_flight.value
-          return @wake_signal.wait(timeout: poll_interval) if available <= 0
-
-          idle = [idle, available].min
+        drained = drain_claim_buffer
+        claimed = claim_deficit
+        drained += drain_claim_buffer
+        @wake_signal.wait(timeout: wake_timeout) if drained.zero? && claimed.zero?
+      end
+
+      # The hold is released before the executor's own heartbeat tracking
+      # registers the same key, so the message is never untracked in between.
+      def drain_claim_buffer
+        @claim_buffer.drain_into(@pool) do |claim|
+          @pool.post do
+            VisibilityHeartbeat.release(claim.hold)
+            process_message(claim.message, claim.queue_name, source_queue: claim.source_queue)
+          end
         end
+      end
 
-        tagged_messages = fetch_messages(idle)
+      def claim_deficit
+        prefetch_room = config.prefetch_limit && (config.prefetch_limit - @in_flight.value)
+        want = @claim_buffer.deficit(free_slots: @pool.available_capacity, read_ahead: @read_ahead,
+                                     prefetch_room: prefetch_room)
+        return 0 if want.zero?
 
-        if tagged_messages.empty?
-          @wake_signal.wait(timeout: poll_interval)
-          return
-        end
+        tagged_messages = fetch_messages(want)
+        return 0 if tagged_messages.empty?
 
         @rate_counter.increment(:dequeued, tagged_messages.size)
+        client = Pgbus.client
         tagged_messages.each do |queue_name, message, source_queue|
           detect_zombie(queue_name, message)
           @in_flight.increment
-          @pool.post { process_message(message, queue_name, source_queue: source_queue) }
+          @claim_buffer.push(queue_name, message, source_queue, client: client)
         end
+        tagged_messages.size
+      end
+
+      def return_claims
+        return if @claim_buffer.empty?
+
+        @in_flight.decrement(@claim_buffer.return_all!(client: Pgbus.client))
       end
 
       # Returns an array of [queue_name, message] pairs so we always know
@@ -616,26 +650,33 @@ module Pgbus
       end
 
       def exceeded_max_jobs?
-        return false unless config.max_jobs_per_worker && @jobs_processed.value >= config.max_jobs_per_worker
+        limit = recycle_limit(:max_jobs_per_worker)
+        return false unless limit && @jobs_processed.value >= limit
 
         Pgbus.logger.info { "[Pgbus] Worker recycling: max_jobs reached (#{@jobs_processed.value})" }
         true
       end
 
       def exceeded_max_memory?
-        return false unless config.max_memory_mb && current_memory_mb > config.max_memory_mb
+        limit = recycle_limit(:max_memory_mb)
+        return false unless limit && current_memory_mb > limit
 
-        Pgbus.logger.info { "[Pgbus] Worker recycling: memory limit (#{current_memory_mb}MB > #{config.max_memory_mb}MB)" }
+        Pgbus.logger.info { "[Pgbus] Worker recycling: memory limit (#{current_memory_mb}MB > #{limit}MB)" }
         true
       end
 
       def exceeded_max_lifetime?
-        return false unless config.max_worker_lifetime && (monotonic_now - @started_at_monotonic) > config.max_worker_lifetime
+        limit = recycle_limit(:max_worker_lifetime)
+        return false unless limit && (monotonic_now - @started_at_monotonic) > limit
 
         Pgbus.logger.info { "[Pgbus] Worker recycling: lifetime exceeded" }
         true
       end
 
+      def recycle_limit(key)
+        config.recycle_limit_for(@recycle_limits, key)
+      end
+
       # Instrumentation payload may report a value up to MEMORY_CHECK_TTL seconds old.
       def current_memory_mb
         MemoryUsage.current_mb
@@ -788,10 +829,13 @@ module Pgbus
       def start_heartbeat
         @heartbeat = Heartbeat.new(
           kind: "worker",
+          # capsule/process lead: the dashboard's metadata cell truncates.
           metadata: {
+            capsule: @capsule, process: @process,
             queues: queues, threads: threads, pid: ::Process.pid,
-            execution_mode: @execution_mode, consumer_priority: @consumer_priority
-          },
+            execution_mode: @execution_mode, consumer_priority: @consumer_priority,
+            **lifecycle_metadata
+          }.compact,
           on_beat: -> { on_heartbeat },
           loop_tick_supplier: -> { @loop_tick_at.get },
           metadata_supplier: -> { throughput_metadata }
@@ -831,10 +875,32 @@ module Pgbus
           "rates" => @rate_counter.rates.transform_keys(&:to_s),
           "jobs_processed" => @jobs_processed.value,
           "jobs_failed" => @jobs_failed.value,
-          "in_flight" => @in_flight.value
+          "in_flight" => @in_flight.value,
+          "buffered" => @claim_buffer.size,
+          **pool_metadata
         }
       end
 
+      # Facts the Processes page states (issue #493), fixed for the process's
+      # life: when it started and which supervisor forked it (ppid; the page
+      # only groups under it when that pid is a supervisor row on this host).
+      def lifecycle_metadata
+        { started_at: @started_at.to_f, supervisor_pid: ::Process.ppid }
+      end
+
+      # Read once per beat: one atomic (threads) or one mutex (async) for the
+      # pool, MemoryUsage's 5s cache for RSS (a ps subprocess on darwin), and
+      # the recycle limits resolved as the recycle check resolves them (a
+      # capsule's own when set), so the page follows a config change.
+      def pool_metadata
+        pool = @pool.metadata
+        {
+          "capacity" => pool[:capacity], "busy" => pool[:busy], "memory_mb" => current_memory_mb,
+          "max_jobs" => recycle_limit(:max_jobs_per_worker), "max_memory_mb" => recycle_limit(:max_memory_mb),
+          "max_lifetime" => recycle_limit(:max_worker_lifetime)
+        }.compact
+      end
+
       def shutdown
         Pgbus.logger.info { "[Pgbus] Worker draining thread pool..." }
         stop_wake_source

data/lib/pgbus/recurring/scheduler.rb

@@ -116,7 +116,8 @@ module Pgbus
       def start_heartbeat
         @heartbeat = Process::Heartbeat.new(
           kind: "scheduler",
-          metadata: { pid: ::Process.pid, tasks: schedule.tasks.size },
+          metadata: { pid: ::Process.pid, tasks: schedule.tasks.size, started_at: Time.now.to_f,
+                      supervisor_pid: ::Process.ppid },
           loop_tick_supplier: -> { @loop_tick_at.get }
         )
         @heartbeat.start

data/lib/pgbus/ruby_jit.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Pgbus
+  # Reports which Ruby JIT this process runs under (issue #484). Read-only:
+  # Rails owns turning YJIT on (config.yjit), pgbus only makes the state
+  # visible in the boot log and `pgbus doctor`.
+  module RubyJit
+    module_function
+
+    def label
+      return "yjit" if yjit_available? && RubyVM::YJIT.enabled?
+      return "zjit" if defined?(RubyVM::ZJIT) && RubyVM::ZJIT.enabled?
+
+      "none"
+    end
+
+    def yjit_available?
+      defined?(RubyVM::YJIT) ? true : false
+    end
+  end
+end

data/lib/pgbus/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Pgbus
-  VERSION = "0.17.0"
+  VERSION = "0.17.2"
 end

data/lib/pgbus/visibility_heartbeat.rb

@@ -30,8 +30,9 @@ module Pgbus
     #
     # The ninth member takes the struct out of the 80-byte slot it used to fit
     # (measured: 80 → 160). That is paid at most once per in-flight message, so
-    # the whole table is bounded by the execution pool's capacity — a handful of
-    # entries per process, not one per enqueued job.
+    # the whole table is bounded by the execution pool's capacity plus the
+    # read-ahead buffer (threads + read_ahead per process, issue #486) — a
+    # handful of entries per process, not one per enqueued job.
     Entry = Struct.new(:client, :queue_name, :prefixed, :msg_id, :job_class, :extended_at, :extensions,
                        :concurrency, :on_beat, keyword_init: true)
 
@@ -81,6 +82,25 @@ module Pgbus
         end
       end
 
+      # Keep a claimed-but-not-started message invisible until #release.
+      # Used by Process::ClaimBuffer for read-ahead (issue #486): a buffered
+      # message waits for a free slot and must not be redelivered meanwhile.
+      # Same key as #track, so the run's own tracking takes over after the
+      # hold is released. Returns the entry, or nil when the heartbeat is off.
+      def hold(client:, queue_name:, msg_id:, prefixed: true, job_class: nil, config: Pgbus.configuration)
+        return unless config.visibility_heartbeat
+
+        entry = Entry.new(client: client, queue_name: queue_name, prefixed: prefixed, msg_id: msg_id.to_i,
+                          job_class: job_class, extended_at: monotonic_now, extensions: 0)
+        register(entry, config)
+        entry
+      end
+
+      # Drop a #hold. A nil entry (heartbeat off) is a no-op.
+      def release(entry)
+        unregister(entry) if entry
+      end
+
       # Extend every tracked message whose last extension is older than the
       # heartbeat interval. Public so tests and callers without the thread
       # can drive it.
@@ -127,11 +147,22 @@ module Pgbus
         end
       end
 
+      # Only the entry that is registered under the key: a stale hold released
+      # after #track re-registered the same message must not drop the running
+      # job's entry (issue #486).
       def unregister(entry)
-        synchronize { entries.delete(key_for(entry)) }
+        synchronize do
+          key = key_for(entry)
+          entries.delete(key) if entries[key].equal?(entry)
+        end
       end
 
       def extend!(entry, now:, config:)
+        # tick! picked this entry under the mutex but runs here without it: if
+        # the entry was released meanwhile (a read-ahead claim handed back with
+        # vt: 0, a job that finished), extending it would undo that.
+        return unless registered?(entry)
+
         vt = config.visibility_timeout
         entry.client.set_visibility_timeout(entry.queue_name, entry.msg_id, vt: vt, prefixed: entry.prefixed)
         entry.extended_at = now
@@ -218,6 +249,10 @@ module Pgbus
         @entries ||= {}
       end
 
+      def registered?(entry)
+        synchronize { entries[key_for(entry)].equal?(entry) }
+      end
+
       def key_for(entry)
         [entry.queue_name, entry.msg_id]
       end

data/lib/pgbus/web/data_source/dashboard_signals.rb

@@ -0,0 +1,170 @@
+# frozen_string_literal: true
+
+module Pgbus
+  module Web
+    class DataSource
+      # The facts behind the home page's words (issue #492): what is retrying
+      # and when it runs next, what died last and why, how the processes and
+      # queues stand, the job rate, and the health verdict /readyz serves.
+      #
+      # summary_stats keeps its shape (the stats API, MetricsSerializer and the
+      # AppSignal probe read it); everything here is home-page only. Each signal
+      # degrades to its empty shape and logs, so one failing query never blanks
+      # the page.
+      module DashboardSignals
+        THROUGHPUT_WINDOW_MINUTES = 5
+
+        EMPTY_RETRYING = { count: 0, capped: false, next_attempt_at: nil, due_now_count: 0,
+                           newest_error_class: nil }.freeze
+
+        def dashboard_signals
+          {
+            retrying: retrying_signal, dlq: dlq_signal, processes: process_signal,
+            queues: queue_signal, throughput: throughput_signal, verdict: verdict_signal
+          }
+        end
+
+        # The Retrying tab of the Jobs page (handler queues left to the Events
+        # page), plus when the next backoff ends, how many are due now and the
+        # newest error among those rows. Each queue contributes its earliest
+        # COUNT_CAP + 1 rows by vt (the vt index), so the next attempt is exact
+        # and the due count is a lower bound past the cap, like the tab counts
+        # (capped: marks both).
+        def retrying_signal
+          counts = job_state_counts(exclude: handler_queue_physical_names)
+          return EMPTY_RETRYING if counts["retrying"].zero?
+
+          schedule = retry_schedule
+          {
+            count: counts["retrying"], capped: counts.capped?("retrying"),
+            next_attempt_at: JobState.time(schedule&.dig("next_at")), due_now_count: schedule&.dig("due").to_i,
+            newest_error_class: schedule&.dig("newest_error")
+          }
+        rescue StandardError => e
+          Pgbus.logger.error { "[Pgbus::Web] Error reading the retry signal: #{e.class}: #{e.message}" }
+          EMPTY_RETRYING
+        end
+
+        # Depth from the cached metrics; the newest message is read from each
+        # non-empty DLQ's tail, so the query stays one index-backed row per DLQ.
+        def dlq_signal
+          counts = dlq_counts_by_queue
+          depth = counts.values.sum
+          { depth: depth, newest: depth.zero? ? nil : newest_dead_letter(counts.select { |_, n| n.positive? }.keys) }
+        rescue StandardError => e
+          Pgbus.logger.error { "[Pgbus::Web] Error reading the dead-letter signal: #{e.class}: #{e.message}" }
+          { depth: depth.to_i, newest: nil }
+        end
+
+        def process_signal
+          rows = processes
+          by_status = rows.group_by { |p| p[:status].to_s }.transform_values(&:size)
+          by_kind = rows.group_by { |p| p[:kind].to_s }.transform_values(&:size)
+          { total: rows.size, healthy: by_status.fetch("healthy", 0), stalled: by_status.fetch("stalled", 0),
+            stale: by_status.fetch("stale", 0), workers: by_kind.fetch("worker", 0),
+            consumers: by_kind.fetch("consumer", 0), dispatchers: by_kind.fetch("dispatcher", 0) }
+        end
+
+        # undrained follows the health verdict: an active (unpaused) backlog on
+        # a job queue no capsule or handler drains. A wildcard drains everything.
+        # depth, claimable and waiting leave the DLQs out (their card counts
+        # them) and the paused queues (paused_depth), whose visible messages
+        # no worker claims.
+        def queue_signal
+          queues = queues_with_metrics
+          dlq, live = queues.partition { |q| q[:name].to_s.end_with?(Pgbus::DEAD_LETTER_SUFFIX) }
+          paused, active = live.partition { |q| q[:paused] }
+          depth = ->(list) { list.sum { |q| q[:queue_length].to_i } }
+          claimable = active.sum { |q| q[:queue_visible_length].to_i }
+          { total: queues.size, paused: paused.size, dlq: dlq.size, undrained: undrained_count(live),
+            depth: depth.call(live), claimable: claimable, waiting: [depth.call(active) - claimable, 0].max,
+            paused_depth: depth.call(paused) }
+        end
+
+        def throughput_signal
+          window = THROUGHPUT_WINDOW_MINUTES
+          return { available: false, jobs_per_minute: nil, window_minutes: window } unless JobStat.table_exists?
+
+          total = JobStat.summary(minutes: window)[:total].to_i
+          { available: true, jobs_per_minute: (total.to_f / window).round(1), window_minutes: window }
+        rescue StandardError => e
+          Pgbus.logger.error { "[Pgbus::Web] Error reading the throughput signal: #{e.class}: #{e.message}" }
+          { available: false, jobs_per_minute: nil, window_minutes: THROUGHPUT_WINDOW_MINUTES }
+        end
+
+        # The verdict /readyz and the MCP serve. The analyzer lives outside
+        # Zeitwerk (pgbus/mcp is ignored), as in HealthApp.
+        def verdict_signal
+          require "pgbus/mcp/health_analyzer" unless defined?(Pgbus::MCP::HealthAnalyzer)
+          Pgbus::MCP::HealthAnalyzer.new(self).verdict
+        rescue StandardError => e
+          Pgbus.logger.error { "[Pgbus::Web] Error computing the health verdict: #{e.class}: #{e.message}" }
+          nil
+        end
+
+        # { physical name => QueueSummary::Line } for the home queues table:
+        # the facts QueueSummary needs for every row, loaded once per render
+        # (pause states in one query, processes and stream names once).
+        def queue_status_lines
+          queues = queues_with_metrics
+          pauses = queue_pause_states(queues.map { |q| q[:name] })
+          healthy = processes.select { |p| p[:healthy] }
+          streams = stream_queue_names
+          max_retries = Pgbus.configuration.max_retries
+          queues.to_h do |q|
+            drainers = queue_drainers(q[:name], healthy: healthy, streams: streams)
+            lines = Web::QueueSummary.present(q, pauses.fetch(q[:name]), drainers, max_retries: max_retries)
+            [q[:name], Web::QueueSummary.headline(lines)]
+          end
+        rescue StandardError => e
+          Pgbus.logger.error { "[Pgbus::Web] Error reading the queue status lines: #{e.class}: #{e.message}" }
+          {}
+        end
+
+        private
+
+        def retry_schedule
+          scope = job_list_scope(nil, exclude: handler_queue_physical_names)
+          return unless scope && scope[:queues].any?
+
+          fragments = scope[:queues].map do |(qtable, logical)|
+            <<~SQL.strip
+              (SELECT m.vt, f.error_class, f.failed_at FROM pgmq.q_#{qtable} m #{failed_join(qtable, logical)}
+               WHERE #{JobList::STATE_PREFILTER["retrying"]} AND (#{JobList::STATE_CASE}) = 'retrying'
+               ORDER BY m.vt LIMIT #{JobList::COUNT_CAP + 1})
+            SQL
+          end
+          connection.select_one(<<~SQL, "Pgbus Retry Schedule")
+            SELECT MIN(vt) FILTER (WHERE vt > now()) AS next_at, COUNT(*) FILTER (WHERE vt <= now()) AS due,
+                   (array_agg(error_class ORDER BY failed_at DESC))[1] AS newest_error
+            FROM (#{fragments.join("\nUNION ALL\n")}) AS retrying
+          SQL
+        end
+
+        def newest_dead_letter(names)
+          fragments = names.map do |name|
+            "(SELECT msg_id, read_ct, enqueued_at, last_read_at, vt, message, headers, '#{sanitize_name(name)}' " \
+              "AS queue_name FROM pgmq.q_#{sanitize_name(name)} ORDER BY msg_id DESC LIMIT 1)"
+          end
+          row = connection.select_one(
+            "SELECT * FROM (#{fragments.join(" UNION ALL ")}) AS newest ORDER BY enqueued_at DESC LIMIT 1",
+            "Pgbus Newest Dead Letter"
+          )
+          row && format_message(row, row["queue_name"])
+        end
+
+        def undrained_count(live)
+          drained = drained_queue_names
+          return 0 if drained.nil?
+
+          backlog = live.reject { |q| q[:paused] || drained.include?(q[:name].to_s) }
+                        .select { |q| q[:queue_visible_length].to_i.positive? }
+          return 0 if backlog.empty?
+
+          streams = stream_queue_names
+          backlog.count { |q| !streams.include?(q[:name].to_s) }
+        end
+      end
+    end
+  end
+end