pgbus 0.6.6 → 0.6.8

data/lib/pgbus/generators/migration_detector.rb

@@ -73,7 +73,8 @@ module Pgbus
         add_queue_states: "pgbus:add_queue_states",
         add_outbox: "pgbus:add_outbox",
         add_recurring: "pgbus:add_recurring",
-        add_failed_events_index: "pgbus:add_failed_events_index"
+        add_failed_events_index: "pgbus:add_failed_events_index",
+        tune_autovacuum: "pgbus:tune_autovacuum"
       }.freeze
 
       # Human-friendly description of each migration for the generator
@@ -88,7 +89,8 @@ module Pgbus
         add_queue_states: "queue states table (pause/resume)",
         add_outbox: "outbox entries table (transactional outbox)",
         add_recurring: "recurring tasks + executions tables",
-        add_failed_events_index: "unique index on pgbus_failed_events (queue_name, msg_id)"
+        add_failed_events_index: "unique index on pgbus_failed_events (queue_name, msg_id)",
+        tune_autovacuum: "autovacuum tuning for PGMQ queue and archive tables"
       }.freeze
 
       def initialize(connection)
@@ -110,7 +112,8 @@ module Pgbus
           *queue_states_migrations,
           *outbox_migrations,
           *recurring_migrations,
-          *failed_events_index_migrations
+          *failed_events_index_migrations,
+          *autovacuum_migrations
         ]
       end
 
@@ -193,6 +196,15 @@ module Pgbus
         [:add_failed_events_index]
       end
 
+      # Autovacuum tuning: check if any PGMQ queue table already has
+      # custom autovacuum settings applied. If not, queue the migration.
+      def autovacuum_migrations
+        return [] unless pgmq_schema_exists?
+        return [] if autovacuum_already_tuned?
+
+        [:tune_autovacuum]
+      end
+
       # --- schema probes -------------------------------------------------
 
       def table_exists?(name)
@@ -212,6 +224,29 @@ module Pgbus
       rescue StandardError
         false
       end
+
+      def pgmq_schema_exists?
+        result = connection.select_value("SELECT 1 FROM information_schema.schemata WHERE schema_name = 'pgmq'")
+        result.present?
+      rescue StandardError
+        false
+      end
+
+      def autovacuum_already_tuned?
+        queue_name = connection.select_value("SELECT queue_name FROM pgmq.meta ORDER BY queue_name LIMIT 1")
+        return true unless queue_name # no queues = nothing to tune, skip
+
+        result = connection.select_value(<<~SQL)
+          SELECT reloptions::text LIKE '%autovacuum_vacuum_scale_factor%'
+          FROM pg_class
+          WHERE relname = 'q_#{queue_name}'
+            AND relnamespace = (SELECT oid FROM pg_namespace WHERE nspname = 'pgmq')
+        SQL
+
+        [true, "t"].include?(result)
+      rescue StandardError
+        true # if we can't tell, assume already tuned (safe default)
+      end
     end
   end
 end

data/lib/pgbus/streams/turbo_stream_override.rb

@@ -17,7 +17,17 @@ module Pgbus
     # After this patch, `turbo_stream_from` renders a `<pgbus-stream-source>`
     # element instead, so both sides use PGMQ/SSE. When `streams_enabled`
     # is false, the original turbo-rails behavior is preserved via `super`.
+    #
+    # The `include Pgbus::StreamsHelper` is required because some callers
+    # invoke `turbo_stream_from` from a Rack middleware context (e.g.
+    # hotwire-livereload's Middleware uses `ActionController::Base.helpers`)
+    # where `Turbo::StreamsHelper` is available but `Pgbus::StreamsHelper`
+    # is not — the engine's `isolate_namespace` scopes helpers to its own
+    # views. Including it here ensures `pgbus_stream_from` is always
+    # reachable on the receiver.
     module TurboStreamOverride
+      include Pgbus::StreamsHelper
+
       def turbo_stream_from(*streamables, **attributes)
         if Pgbus.configuration.streams_enabled
           pgbus_stream_from(*streamables, **attributes)

data/lib/pgbus/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Pgbus
-  VERSION = "0.6.6"
+  VERSION = "0.6.8"
 end

data/lib/pgbus/web/data_source.rb

@@ -21,6 +21,8 @@ module Pgbus
 
         throughput = compute_throughput(queues)
 
+        health = queue_health_stats
+
         {
           total_queues: queues.size,
           total_depth: total_depth,
@@ -29,7 +31,10 @@ module Pgbus
           failed_count: failed_events_count,
           dlq_depth: dlq_depth,
           recurring_count: recurring_tasks_count,
-          throughput_rate: throughput
+          throughput_rate: throughput,
+          total_dead_tuples: health[:total_dead_tuples],
+          tables_needing_vacuum: health[:tables_needing_vacuum],
+          oldest_transaction_age_sec: health[:oldest_transaction_age_sec]
         }
       end
 
@@ -629,6 +634,50 @@ module Pgbus
         []
       end
 
+      # Queue health — vacuum stats, dead tuples, bloat, MVCC horizon.
+      # Returns aggregate health across all queue and archive tables, plus
+      # the oldest open transaction age (MVCC horizon pinning risk).
+      def queue_health_stats
+        tables = fetch_all_table_stats
+
+        total_dead = tables.sum { |t| t[:dead_tuples] }
+        total_live = tables.sum { |t| t[:live_tuples] }
+        worst_bloat = tables.map { |t| t[:bloat_ratio] }.max || 0.0
+        needs_vacuum = tables.count { |t| t[:bloat_ratio] > 0.1 }
+        oldest_vacuum = tables.filter_map { |t| t[:last_vacuum_ago_sec] }.max
+
+        {
+          total_dead_tuples: total_dead,
+          total_live_tuples: total_live,
+          worst_bloat_ratio: worst_bloat.round(4),
+          tables_needing_vacuum: needs_vacuum,
+          oldest_vacuum_ago_sec: oldest_vacuum,
+          oldest_transaction_age_sec: oldest_transaction_age,
+          tables: tables
+        }
+      rescue StandardError => e
+        Pgbus.logger.debug { "[Pgbus::Web] Error fetching queue health stats: #{e.class}: #{e.message}" }
+        {
+          total_dead_tuples: 0, total_live_tuples: 0, worst_bloat_ratio: 0.0,
+          tables_needing_vacuum: 0, oldest_vacuum_ago_sec: nil,
+          oldest_transaction_age_sec: nil, tables: []
+        }
+      end
+
+      # Per-queue health stats for the queue detail view.
+      def queue_health_detail(queue_name)
+        sanitized = sanitize_name(queue_name)
+        tables = [
+          fetch_table_stats("pgmq", "q_#{sanitized}", "queue"),
+          fetch_table_stats("pgmq", "a_#{sanitized}", "archive")
+        ].compact
+
+        { tables: tables, oldest_transaction_age_sec: oldest_transaction_age }
+      rescue StandardError => e
+        Pgbus.logger.debug { "[Pgbus::Web] Error fetching health detail for #{queue_name}: #{e.message}" }
+        { tables: [], oldest_transaction_age_sec: nil }
+      end
+
       # Stream stats — only populated when streams_stats_enabled is
       # true AND the migration has been run. Controllers should gate
       # rendering on `stream_stats_available?` to avoid showing empty
@@ -674,6 +723,89 @@ module Pgbus
         Pgbus::BusRecord.connection
       end
 
+      # Single query to fetch pg_stat_user_tables stats for all queue and
+      # archive tables. Avoids 2*N catalog queries on the dashboard.
+      def fetch_all_table_stats
+        rows = connection.select_all(<<~SQL, "Pgbus All Table Health")
+          WITH rels AS (
+            SELECT queue_name, 'q_' || queue_name AS relname, 'queue' AS kind FROM pgmq.meta
+            UNION ALL
+            SELECT queue_name, 'a_' || queue_name AS relname, 'archive' AS kind FROM pgmq.meta
+          )
+          SELECT
+            'pgmq.' || r.relname AS table_name,
+            r.kind,
+            s.n_live_tup,
+            s.n_dead_tup,
+            EXTRACT(epoch FROM (NOW() - COALESCE(s.last_vacuum, s.last_autovacuum)))::int AS last_vacuum_ago_sec,
+            s.last_vacuum,
+            s.last_autovacuum
+          FROM rels r
+          LEFT JOIN pg_stat_user_tables s
+            ON s.schemaname = 'pgmq' AND s.relname = r.relname
+          ORDER BY r.queue_name, r.kind
+        SQL
+
+        rows.to_a.filter_map { |row| build_table_health_row(row) }
+      end
+
+      # Fetch pg_stat_user_tables stats for a single table (used by queue_health_detail).
+      def fetch_table_stats(schema, table_name, kind)
+        row = connection.select_one(<<~SQL, "Pgbus Table Health", [schema, table_name])
+          SELECT
+            n_live_tup,
+            n_dead_tup,
+            EXTRACT(epoch FROM (NOW() - COALESCE(last_vacuum, last_autovacuum)))::int AS last_vacuum_ago_sec,
+            last_vacuum,
+            last_autovacuum
+          FROM pg_stat_user_tables
+          WHERE schemaname = $1 AND relname = $2
+        SQL
+
+        return nil unless row
+
+        build_table_health_row(row.merge("table_name" => "#{schema}.#{table_name}", "kind" => kind))
+      end
+
+      def build_table_health_row(row)
+        return nil unless row["n_live_tup"] || row["n_dead_tup"]
+
+        live = row["n_live_tup"].to_i
+        dead = row["n_dead_tup"].to_i
+        total = live + dead
+        bloat = total.positive? ? (dead.to_f / total) : 0.0
+
+        {
+          table: row["table_name"],
+          kind: row["kind"],
+          live_tuples: live,
+          dead_tuples: dead,
+          bloat_ratio: bloat.round(4),
+          last_vacuum_ago_sec: row["last_vacuum_ago_sec"]&.to_i,
+          last_vacuum: row["last_vacuum"],
+          last_autovacuum: row["last_autovacuum"]
+        }
+      end
+
+      # Age of the oldest open transaction in seconds — indicates MVCC
+      # horizon pinning risk. Returns nil if no active transactions.
+      def oldest_transaction_age
+        row = connection.select_one(<<~SQL, "Pgbus Oldest Transaction")
+          SELECT EXTRACT(epoch FROM (NOW() - xact_start))::int AS age_sec
+          FROM pg_stat_activity
+          WHERE state != 'idle'
+            AND xact_start IS NOT NULL
+            AND pid != pg_backend_pid()
+          ORDER BY xact_start ASC
+          LIMIT 1
+        SQL
+
+        row&.dig("age_sec")&.to_i
+      rescue StandardError => e
+        Pgbus.logger.debug { "[Pgbus::Web] Error fetching oldest transaction age: #{e.class}: #{e.message}" }
+        nil
+      end
+
       # name is the full PGMQ queue name (already prefixed)
       def query_queue_messages(name, limit, offset)
         query_queue_messages_raw(name, limit, offset).map { |m| m.merge(queue: name) }

data/lib/pgbus/web/metrics_serializer.rb

@@ -24,6 +24,7 @@ module Pgbus
         append_process_metrics(lines)
         append_summary_metrics(lines)
         append_stream_metrics(lines)
+        append_health_metrics(lines)
         "#{lines.join("\n")}\n"
       end
 
@@ -97,9 +98,41 @@ module Pgbus
       end
 
       def append_process_metrics(lines)
-        count = @data_source.processes.count
+        procs = @data_source.processes
         gauge(lines, "pgbus_active_processes", "Number of active pgbus worker processes") do
-          [[count]]
+          [[procs.count]]
+        end
+
+        workers = procs.select { |p| p[:kind] == "worker" && p[:metadata].is_a?(Hash) }
+        unless workers.empty?
+          gauge(lines, "pgbus_worker_pool_capacity", "Total thread/async pool capacity per worker") do
+            workers.filter_map do |w|
+              capacity = w[:metadata]["capacity"]
+              next unless capacity
+
+              [capacity, { pid: w[:pid], hostname: w[:hostname] }]
+            end
+          end
+
+          gauge(lines, "pgbus_worker_pool_busy", "Number of busy threads/slots per worker") do
+            workers.filter_map do |w|
+              busy = w[:metadata]["busy"]
+              next unless busy
+
+              [busy, { pid: w[:pid], hostname: w[:hostname] }]
+            end
+          end
+
+          gauge(lines, "pgbus_worker_pool_utilization", "Pool utilization ratio (busy / capacity)") do
+            workers.filter_map do |w|
+              capacity = w[:metadata]["capacity"].to_i
+              busy = w[:metadata]["busy"].to_i
+              next unless capacity.positive?
+
+              ratio = (busy.to_f / capacity).round(4)
+              [ratio, { pid: w[:pid], hostname: w[:hostname] }]
+            end
+          end
         end
       rescue StandardError => e
         Pgbus.logger.debug { "[Pgbus::Metrics] Error serializing process metrics: #{e.message}" }
@@ -141,6 +174,42 @@ module Pgbus
         Pgbus.logger.debug { "[Pgbus::Metrics] Error serializing stream metrics: #{e.message}" }
       end
 
+      def append_health_metrics(lines)
+        health = @data_source.queue_health_stats
+        return if health[:tables].empty? && health[:oldest_transaction_age_sec].nil?
+
+        tables = health[:tables]
+        unless tables.empty?
+          gauge(lines, "pgbus_table_dead_tuples", "Number of dead tuples in queue/archive table") do
+            tables.map { |t| [t[:dead_tuples], { table: t[:table], kind: t[:kind] }] }
+          end
+
+          gauge(lines, "pgbus_table_live_tuples", "Number of live tuples in queue/archive table") do
+            tables.map { |t| [t[:live_tuples], { table: t[:table], kind: t[:kind] }] }
+          end
+
+          gauge(lines, "pgbus_table_bloat_ratio", "Dead tuple ratio (dead / total) per table") do
+            tables.map { |t| [t[:bloat_ratio], { table: t[:table], kind: t[:kind] }] }
+          end
+
+          vacuum_tables = tables.select { |t| t[:last_vacuum_ago_sec] }
+          unless vacuum_tables.empty?
+            gauge(lines, "pgbus_table_last_vacuum_age_seconds", "Seconds since last vacuum") do
+              vacuum_tables.map { |t| [t[:last_vacuum_ago_sec], { table: t[:table], kind: t[:kind] }] }
+            end
+          end
+        end
+
+        if health[:oldest_transaction_age_sec]
+          gauge(lines, "pgbus_oldest_transaction_age_seconds",
+                "Age of the oldest open transaction (MVCC horizon pin risk)") do
+            [[health[:oldest_transaction_age_sec]]]
+          end
+        end
+      rescue StandardError => e
+        Pgbus.logger.debug { "[Pgbus::Metrics] Error serializing health metrics: #{e.message}" }
+      end
+
       # Emits a Prometheus gauge metric family. The block must return an array
       # of [value] or [value, { label: "val" }] pairs.
       def gauge(lines, name, help)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: pgbus
 version: !ruby/object:Gem::Version
-  version: 0.6.6
+  version: 0.6.8
 platform: ruby
 authors:
 - Mikael Henriksson
@@ -121,6 +121,7 @@ files:
 - LICENSE.txt
 - README.md
 - Rakefile
+- app/assets/javascripts/pgbus/stream_source_element.js
 - app/controllers/pgbus/api/insights_controller.rb
 - app/controllers/pgbus/api/metrics_controller.rb
 - app/controllers/pgbus/api/stats_controller.rb
@@ -145,7 +146,6 @@ files:
 - app/frontend/pgbus/vendor/turbo.js
 - app/helpers/pgbus/application_helper.rb
 - app/helpers/pgbus/streams_helper.rb
-- app/javascript/pgbus/stream_source_element.js
 - app/models/pgbus/application_record.rb
 - app/models/pgbus/batch_entry.rb
 - app/models/pgbus/blocked_execution.rb
@@ -162,6 +162,7 @@ files:
 - app/models/pgbus/uniqueness_key.rb
 - app/views/layouts/pgbus/application.html.erb
 - app/views/pgbus/dashboard/_processes_table.html.erb
+- app/views/pgbus/dashboard/_queue_health.html.erb
 - app/views/pgbus/dashboard/_queues_table.html.erb
 - app/views/pgbus/dashboard/_recent_failures.html.erb
 - app/views/pgbus/dashboard/_stats_cards.html.erb
@@ -230,12 +231,15 @@ files:
 - lib/generators/pgbus/templates/pgbus.yml.erb
 - lib/generators/pgbus/templates/pgbus_binstub.erb
 - lib/generators/pgbus/templates/recurring.yml.erb
+- lib/generators/pgbus/templates/tune_autovacuum.rb.erb
 - lib/generators/pgbus/templates/upgrade_pgmq.rb.erb
+- lib/generators/pgbus/tune_autovacuum_generator.rb
 - lib/generators/pgbus/update_generator.rb
 - lib/generators/pgbus/upgrade_pgmq_generator.rb
 - lib/pgbus.rb
 - lib/pgbus/active_job/adapter.rb
 - lib/pgbus/active_job/executor.rb
+- lib/pgbus/autovacuum_tuning.rb
 - lib/pgbus/batch.rb
 - lib/pgbus/bus_record.rb
 - lib/pgbus/circuit_breaker.rb