pgbus 0.17.1 → 0.17.3

data/lib/pgbus/web/data_source/dashboard_signals.rb

@@ -0,0 +1,170 @@
+# frozen_string_literal: true
+
+module Pgbus
+  module Web
+    class DataSource
+      # The facts behind the home page's words (issue #492): what is retrying
+      # and when it runs next, what died last and why, how the processes and
+      # queues stand, the job rate, and the health verdict /readyz serves.
+      #
+      # summary_stats keeps its shape (the stats API, MetricsSerializer and the
+      # AppSignal probe read it); everything here is home-page only. Each signal
+      # degrades to its empty shape and logs, so one failing query never blanks
+      # the page.
+      module DashboardSignals
+        THROUGHPUT_WINDOW_MINUTES = 5
+
+        EMPTY_RETRYING = { count: 0, capped: false, next_attempt_at: nil, due_now_count: 0,
+                           newest_error_class: nil }.freeze
+
+        def dashboard_signals
+          {
+            retrying: retrying_signal, dlq: dlq_signal, processes: process_signal,
+            queues: queue_signal, throughput: throughput_signal, verdict: verdict_signal
+          }
+        end
+
+        # The Retrying tab of the Jobs page (handler queues left to the Events
+        # page), plus when the next backoff ends, how many are due now and the
+        # newest error among those rows. Each queue contributes its earliest
+        # COUNT_CAP + 1 rows by vt (the vt index), so the next attempt is exact
+        # and the due count is a lower bound past the cap, like the tab counts
+        # (capped: marks both).
+        def retrying_signal
+          counts = job_state_counts(exclude: handler_queue_physical_names)
+          return EMPTY_RETRYING if counts["retrying"].zero?
+
+          schedule = retry_schedule
+          {
+            count: counts["retrying"], capped: counts.capped?("retrying"),
+            next_attempt_at: JobState.time(schedule&.dig("next_at")), due_now_count: schedule&.dig("due").to_i,
+            newest_error_class: schedule&.dig("newest_error")
+          }
+        rescue StandardError => e
+          Pgbus.logger.error { "[Pgbus::Web] Error reading the retry signal: #{e.class}: #{e.message}" }
+          EMPTY_RETRYING
+        end
+
+        # Depth from the cached metrics; the newest message is read from each
+        # non-empty DLQ's tail, so the query stays one index-backed row per DLQ.
+        def dlq_signal
+          counts = dlq_counts_by_queue
+          depth = counts.values.sum
+          { depth: depth, newest: depth.zero? ? nil : newest_dead_letter(counts.select { |_, n| n.positive? }.keys) }
+        rescue StandardError => e
+          Pgbus.logger.error { "[Pgbus::Web] Error reading the dead-letter signal: #{e.class}: #{e.message}" }
+          { depth: depth.to_i, newest: nil }
+        end
+
+        def process_signal
+          rows = processes
+          by_status = rows.group_by { |p| p[:status].to_s }.transform_values(&:size)
+          by_kind = rows.group_by { |p| p[:kind].to_s }.transform_values(&:size)
+          { total: rows.size, healthy: by_status.fetch("healthy", 0), stalled: by_status.fetch("stalled", 0),
+            stale: by_status.fetch("stale", 0), workers: by_kind.fetch("worker", 0),
+            consumers: by_kind.fetch("consumer", 0), dispatchers: by_kind.fetch("dispatcher", 0) }
+        end
+
+        # undrained follows the health verdict: an active (unpaused) backlog on
+        # a job queue no capsule or handler drains. A wildcard drains everything.
+        # depth, claimable and waiting leave the DLQs out (their card counts
+        # them) and the paused queues (paused_depth), whose visible messages
+        # no worker claims.
+        def queue_signal
+          queues = queues_with_metrics
+          dlq, live = queues.partition { |q| q[:name].to_s.end_with?(Pgbus::DEAD_LETTER_SUFFIX) }
+          paused, active = live.partition { |q| q[:paused] }
+          depth = ->(list) { list.sum { |q| q[:queue_length].to_i } }
+          claimable = active.sum { |q| q[:queue_visible_length].to_i }
+          { total: queues.size, paused: paused.size, dlq: dlq.size, undrained: undrained_count(live),
+            depth: depth.call(live), claimable: claimable, waiting: [depth.call(active) - claimable, 0].max,
+            paused_depth: depth.call(paused) }
+        end
+
+        def throughput_signal
+          window = THROUGHPUT_WINDOW_MINUTES
+          return { available: false, jobs_per_minute: nil, window_minutes: window } unless JobStat.table_exists?
+
+          total = JobStat.summary(minutes: window)[:total].to_i
+          { available: true, jobs_per_minute: (total.to_f / window).round(1), window_minutes: window }
+        rescue StandardError => e
+          Pgbus.logger.error { "[Pgbus::Web] Error reading the throughput signal: #{e.class}: #{e.message}" }
+          { available: false, jobs_per_minute: nil, window_minutes: THROUGHPUT_WINDOW_MINUTES }
+        end
+
+        # The verdict /readyz and the MCP serve. The analyzer lives outside
+        # Zeitwerk (pgbus/mcp is ignored), as in HealthApp.
+        def verdict_signal
+          require "pgbus/mcp/health_analyzer" unless defined?(Pgbus::MCP::HealthAnalyzer)
+          Pgbus::MCP::HealthAnalyzer.new(self).verdict
+        rescue StandardError => e
+          Pgbus.logger.error { "[Pgbus::Web] Error computing the health verdict: #{e.class}: #{e.message}" }
+          nil
+        end
+
+        # { physical name => QueueSummary::Line } for the home queues table:
+        # the facts QueueSummary needs for every row, loaded once per render
+        # (pause states in one query, processes and stream names once).
+        def queue_status_lines
+          queues = queues_with_metrics
+          pauses = queue_pause_states(queues.map { |q| q[:name] })
+          healthy = processes.select { |p| p[:healthy] }
+          streams = stream_queue_names
+          max_retries = Pgbus.configuration.max_retries
+          queues.to_h do |q|
+            drainers = queue_drainers(q[:name], healthy: healthy, streams: streams)
+            lines = Web::QueueSummary.present(q, pauses.fetch(q[:name]), drainers, max_retries: max_retries)
+            [q[:name], Web::QueueSummary.headline(lines)]
+          end
+        rescue StandardError => e
+          Pgbus.logger.error { "[Pgbus::Web] Error reading the queue status lines: #{e.class}: #{e.message}" }
+          {}
+        end
+
+        private
+
+        def retry_schedule
+          scope = job_list_scope(nil, exclude: handler_queue_physical_names)
+          return unless scope && scope[:queues].any?
+
+          fragments = scope[:queues].map do |(qtable, logical)|
+            <<~SQL.strip
+              (SELECT m.vt, f.error_class, f.failed_at FROM pgmq.q_#{qtable} m #{failed_join(qtable, logical)}
+               WHERE #{JobList::STATE_PREFILTER["retrying"]} AND (#{JobList::STATE_CASE}) = 'retrying'
+               ORDER BY m.vt LIMIT #{JobList::COUNT_CAP + 1})
+            SQL
+          end
+          connection.select_one(<<~SQL, "Pgbus Retry Schedule")
+            SELECT MIN(vt) FILTER (WHERE vt > now()) AS next_at, COUNT(*) FILTER (WHERE vt <= now()) AS due,
+                   (array_agg(error_class ORDER BY failed_at DESC))[1] AS newest_error
+            FROM (#{fragments.join("\nUNION ALL\n")}) AS retrying
+          SQL
+        end
+
+        def newest_dead_letter(names)
+          fragments = names.map do |name|
+            "(SELECT msg_id, read_ct, enqueued_at, last_read_at, vt, message, headers, '#{sanitize_name(name)}' " \
+              "AS queue_name FROM pgmq.q_#{sanitize_name(name)} ORDER BY msg_id DESC LIMIT 1)"
+          end
+          row = connection.select_one(
+            "SELECT * FROM (#{fragments.join(" UNION ALL ")}) AS newest ORDER BY enqueued_at DESC LIMIT 1",
+            "Pgbus Newest Dead Letter"
+          )
+          row && format_message(row, row["queue_name"])
+        end
+
+        def undrained_count(live)
+          drained = drained_queue_names
+          return 0 if drained.nil?
+
+          backlog = live.reject { |q| q[:paused] || drained.include?(q[:name].to_s) }
+                        .select { |q| q[:queue_visible_length].to_i.positive? }
+          return 0 if backlog.empty?
+
+          streams = stream_queue_names
+          backlog.count { |q| !streams.include?(q[:name].to_s) }
+        end
+      end
+    end
+  end
+end

data/lib/pgbus/web/data_source/dead_letter.rb

@@ -0,0 +1,142 @@
+# frozen_string_literal: true
+
+module Pgbus
+  module Web
+    class DataSource
+      # The dead-letter queue pages (issue #495): list, count, detail, and the
+      # retry/discard actions. Moved out of data_source.rb unchanged.
+      module DeadLetter
+        # The header path the error-class filter reads (DeadLetterHeader::KEY).
+        ERROR_CLASS_SQL = "headers #>> '{#{Pgbus::DeadLetterHeader::KEY},error_class}'".freeze
+
+        # DLQ queue names from queues_with_metrics are already fully qualified
+        # (e.g., "pgbus_default_dlq"), so we use them directly without re-prefixing.
+        # dlq: one DLQ by its full name (unknown names list nothing);
+        # error_class: only messages whose dead-letter block names that class.
+        def dlq_messages(page: 1, per_page: 25, dlq: nil, error_class: nil)
+          offset = (page - 1) * per_page
+          names = dlq_queue_names(dlq)
+          return [] if names.empty?
+          return paginated_queue_messages(names, per_page, offset) unless error_class
+
+          paginated_queue_messages(names, per_page, offset, where: "#{ERROR_CLASS_SQL} = $3", binds: [error_class])
+        rescue StandardError => e
+          Pgbus.logger.debug { "[Pgbus::Web] Error fetching DLQ messages: #{e.message}" }
+          []
+        end
+
+        def dlq_total_count(dlq: nil, error_class: nil)
+          names = dlq_queue_names(dlq)
+          return 0 if names.empty?
+          return dlq_counts_by_queue.values_at(*names).sum unless error_class
+
+          connection.select_value(
+            "SELECT COUNT(*) FROM (#{queue_message_fragments(names, "#{ERROR_CLASS_SQL} = $1")}) AS combined",
+            "Pgbus DLQ Count",
+            [error_class]
+          ).to_i
+        rescue StandardError => e
+          Pgbus.logger.debug { "[Pgbus::Web] Error fetching DLQ count: #{e.message}" }
+          0
+        end
+
+        # { "pgbus_default_dlq" => 12, ... } for the filter chips, from the
+        # cached metrics (no query of its own).
+        def dlq_counts_by_queue
+          queues_with_metrics
+            .select { |q| q[:name].end_with?(Pgbus::DEAD_LETTER_SUFFIX) }
+            .to_h { |q| [q[:name], q[:queue_length].to_i] }
+        rescue StandardError => e
+          Pgbus.logger.debug { "[Pgbus::Web] Error fetching DLQ counts: #{e.message}" }
+          {}
+        end
+
+        # msg_ids are unique per DLQ table only: queue_name: picks the table;
+        # without it the first DLQ holding the id wins.
+        def dlq_message_detail(msg_id, queue_name: nil)
+          dlq_queue_names(queue_name).each do |name|
+            row = connection.select_one(
+              "SELECT * FROM pgmq.q_#{sanitize_name(name)} WHERE msg_id = $1",
+              "Pgbus DLQ Detail",
+              [msg_id.to_i]
+            )
+            return format_message(row, name) if row
+          end
+          nil
+        rescue StandardError => e
+          Pgbus.logger.debug { "[Pgbus::Web] Error fetching DLQ message #{msg_id}: #{e.message}" }
+          nil
+        end
+
+        def retry_dlq_message(queue_name, msg_id)
+          # queue_name here is the full DLQ name (already prefixed)
+          dlq_suffix = Pgbus::DEAD_LETTER_SUFFIX
+          original_queue = queue_name.delete_suffix(dlq_suffix)
+
+          row = connection.select_one(
+            "SELECT * FROM pgmq.q_#{sanitize_name(queue_name)} WHERE msg_id = $1",
+            "Pgbus DLQ Read",
+            [msg_id.to_i]
+          )
+          return false unless row
+
+          @client.transaction do |txn|
+            # A live message must not claim to be dead: strip the block, count the trip.
+            txn.produce(original_queue, row["message"], headers: DeadLetterHeader.strip_for_retry(row["headers"]))
+            txn.delete(queue_name, msg_id.to_i)
+          end
+          true
+        rescue StandardError => e
+          Pgbus.logger.debug { "[Pgbus::Web] Error retrying DLQ message #{msg_id}: #{e.message}" }
+          false
+        end
+
+        def discard_dlq_message(queue_name, msg_id)
+          # queue_name here is the full DLQ name (already prefixed)
+          release_lock_for_message(queue_name, msg_id)
+          @client.delete_message(queue_name, msg_id.to_i, prefixed: false)
+          true
+        rescue StandardError => e
+          Pgbus.logger.debug { "[Pgbus::Web] Error discarding DLQ message #{msg_id}: #{e.message}" }
+          false
+        end
+
+        def retry_all_dlq
+          messages = dlq_messages(page: 1, per_page: 1000)
+          count = 0
+          messages.each do |m|
+            retry_dlq_message(m[:queue_name], m[:msg_id]) && count += 1
+          rescue StandardError => e
+            Pgbus.logger.debug { "[Pgbus::Web] Error retrying DLQ message #{m[:msg_id]}: #{e.message}" }
+            next
+          end
+          count
+        end
+
+        def discard_all_dlq
+          messages = dlq_messages(page: 1, per_page: 1000)
+          return 0 if messages.empty?
+
+          release_locks_for_messages(messages)
+
+          # Group by queue for batch delete — one call per DLQ instead of N calls
+          messages.group_by { |m| m[:queue_name] }.sum do |queue_name, msgs|
+            ids = msgs.map { |m| m[:msg_id].to_i }
+            @client.delete_batch(queue_name, ids, prefixed: false).size
+          rescue StandardError => e
+            Pgbus.logger.debug { "[Pgbus::Web] Error batch-discarding DLQ messages from #{queue_name}: #{e.message}" }
+            0
+          end
+        end
+
+        private
+
+        # Only names that are real DLQs ever reach sanitize_name and SQL.
+        def dlq_queue_names(dlq)
+          names = dlq_counts_by_queue.keys
+          dlq.nil? ? names : names & [dlq]
+        end
+      end
+    end
+  end
+end

data/lib/pgbus/web/data_source/event_list.rb

@@ -0,0 +1,161 @@
+# frozen_string_literal: true
+
+require "securerandom"
+
+module Pgbus
+  module Web
+    class DataSource
+      # The Events page (issue #494): the unified job list scoped to the
+      # EventBus handler queues, the words EventState needs about consumers,
+      # and Replay of a processed event from its archived message.
+      #
+      # Handler names and patterns come from the subscriber registry, in Ruby:
+      # rows match on the physical queue (queue rows) or the logical one
+      # (orphaned failed rows, which carry the name the consumer recorded).
+      module EventList
+        # Lower bound on archived_at below the oldest claim on a page: the
+        # claim always precedes the archive, the margin absorbs clock skew.
+        ARCHIVE_PROBE_MARGIN = 60
+
+        # An archived envelope's routing key, where Publisher and the outbox
+        # put it. Without one the consumer would archive a replay unrouted.
+        ROUTING_KEY_SQL = "COALESCE(a.message->'headers'->>'routing_key', a.message->>'routing_key')"
+
+        def event_rows(state: nil, queue_name: nil, page: 1, per_page: 25)
+          queues = handler_queue_physical_names
+          return [] if queues.empty?
+
+          rows = job_rows(state: state, queue_name: queue_name, queues: queues, page: page, per_page: per_page)
+          by_queue = subscribers_by_queue_name
+          rows.map do |row|
+            sub = by_queue[row[:queue_name]]
+            row.merge(handler_class: sub&.dig(:handler_class), pattern: sub&.dig(:pattern))
+          end
+        end
+
+        def event_state_counts(queue_name: nil)
+          job_state_counts(queue_name: queue_name, queues: handler_queue_physical_names)
+        end
+
+        def events_ahead(rows) = jobs_ahead(rows)
+
+        def event_list_context(now: Time.now)
+          EventState::Context.new(jobs: job_list_context(now: now), covered_queues: covered_handler_queues,
+                                  claim_window: EventBus::Handler.claim_ownership_window)
+        end
+
+        # { processed_event id => :replayable | :not_archived | :no_handler }
+        # for one page of processed events: one bounded archive probe per
+        # handler queue, never one per row.
+        def event_replay_states(events)
+          subs = replay_subscribers
+          states = events.to_h { |e| [e["id"], subs[e["handler_class"]] ? :not_archived : :no_handler] }
+
+          events.select { |e| subs[e["handler_class"]] }.group_by { |e| subs[e["handler_class"]] }.each do |sub, rows|
+            keys = archived_routing_keys(sub[:physical_queue_name], rows)
+            rows.each { |e| states[e["id"]] = :replayable if replayable_key?(sub, keys[e["event_id"]]) }
+          end
+          states
+        end
+
+        # Re-deliver a processed event's archived envelope to its handler's
+        # queue, under a new event_id (so the idempotency claim is new) that
+        # links back through replayed_from. Only that handler runs again: a
+        # topic publish would re-run every subscriber of the routing key.
+        def replay_event(event)
+          sub = replay_subscribers[event["handler_class"]]
+          return false unless sub
+
+          archived = archived_event(sub[:physical_queue_name], event)
+          return false unless archived
+
+          message = JSON.parse(archived["message"])
+          return false unless replayable_key?(sub, event_routing_key(message))
+
+          replay = message.merge("event_id" => SecureRandom.uuid, "replayed_from" => event["event_id"])
+          @client.transaction do |txn|
+            txn.produce(sub[:physical_queue_name], JSON.generate(replay), headers: archived["headers"])
+          end
+          true
+        rescue StandardError => e
+          Pgbus.logger.error { "[Pgbus::Web] Error replaying event #{event["event_id"]}: #{e.class}: #{e.message}" }
+          false
+        end
+
+        private
+
+        def subscribers_by_queue_name
+          registered_subscribers.each_with_object({}) do |s, out|
+            out[s[:physical_queue_name]] ||= s
+            out[s[:queue_name]] ||= s
+          end
+        end
+
+        # Physical handler queues some healthy consumer reads: its topics
+        # overlap the subscriber's pattern (Registry#queue_names_for_topics'
+        # rule). nil when a healthy consumer does not report its topics.
+        def covered_handler_queues
+          consumers = processes.select { |p| p[:kind].to_s == "consumer" && p[:healthy] }
+          topics = consumers.map { |p| p[:metadata].is_a?(Hash) ? p[:metadata]["topics"] : nil }
+          return nil if topics.any?(&:nil?)
+
+          registry = EventBus::Registry.instance
+          topics = topics.flatten
+          registered_subscribers.select { |s| topics.any? { |t| registry.pattern_overlaps?(t, s[:pattern]) } }
+                                .to_set { |s| s[:physical_queue_name] }
+        end
+
+        # { event_id => routing key } for the page's events still in the archive.
+        def archived_routing_keys(queue, events)
+          ids = events.map { |e| text_array_element(e["event_id"]) }
+          rows = connection.select_all(<<~SQL, "Pgbus Archived Events", ["{#{ids.join(",")}}", archive_floor(events)])
+            SELECT a.message->>'event_id' AS event_id, #{ROUTING_KEY_SQL} AS routing_key
+            FROM pgmq.a_#{sanitize_name(queue)} a
+            WHERE a.archived_at >= $2::timestamptz AND a.message->>'event_id' = ANY($1::text[])
+              AND #{ROUTING_KEY_SQL} IS NOT NULL
+          SQL
+          rows.to_a.to_h { |r| [r["event_id"], r["routing_key"]] }
+        rescue StandardError => e
+          Pgbus.logger.error { "[Pgbus::Web] Error reading the archive of #{queue}: #{e.class}: #{e.message}" }
+          {}
+        end
+
+        # The registration Replay uses per handler class: the first, so the
+        # availability check and the action never disagree.
+        def replay_subscribers
+          registered_subscribers.each_with_object({}) { |s, out| out[s[:handler_class]] ||= s }
+        end
+
+        # A replay the consumer would archive unrouted is no replay: the key
+        # must still match the subscriber's pattern.
+        def replayable_key?(sub, routing_key)
+          !routing_key.nil? && EventBus::Registry.instance.pattern_matches?(sub[:pattern], routing_key)
+        end
+
+        # event_id is any string an envelope carried (Edit & Retry accepts any
+        # payload), so each element of the bound text[] literal is quoted.
+        def text_array_element(value)
+          %("#{value.to_s.gsub(/["\\]/) { |c| "\\#{c}" }}")
+        end
+
+        def archived_event(queue, event)
+          connection.select_one(<<~SQL, "Pgbus Archived Event", [event["event_id"].to_s, archive_floor([event])])
+            SELECT a.message::text AS message, a.headers::text AS headers FROM pgmq.a_#{sanitize_name(queue)} a
+            WHERE a.message->>'event_id' = $1 AND a.archived_at >= $2::timestamptz
+            ORDER BY a.archived_at DESC LIMIT 1
+          SQL
+        end
+
+        def archive_floor(events)
+          oldest = events.filter_map { |e| JobState.time(e["processed_at"]) }.min || Time.at(0)
+          (oldest - ARCHIVE_PROBE_MARGIN).utc.iso8601(6)
+        end
+
+        def event_routing_key(message)
+          headers = message["headers"]
+          (headers["routing_key"] if headers.is_a?(Hash)).presence || message["routing_key"].presence
+        end
+      end
+    end
+  end
+end