pgbus 0.6.0 → 0.6.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9428608c5f9419017f78a37cfe59c70ec03ff276654e516444ec25b8d9ee4fed
-  data.tar.gz: 8ad2651f7b6f25203442c819649c81d90c55c057a2d77219c10f6f74d2db18e9
+  metadata.gz: f9e7369bd90c71a1c25bc9fdafe8905d8cdca99e5ae1deffb9c7b304aecc881d
+  data.tar.gz: 44d3fb5bf94742541ba5f9474c06cc602b2a7490decb7daee846c4b8f5ce1bbe
 SHA512:
-  metadata.gz: d63781a6eac94d51d295ce57f47729f7e6d6fffcc7c9a1883db110dd5833f35bfdf14559e832431e0c1759dfb8fd060c6f09701da47f65355b966c35f5aaa2c0
-  data.tar.gz: 4acb342915df5bb5761ab534edcfbd051592735640d3fa9551b2c16b0e785ec8ffe82e45fec1e7dbc81c92d2e5408f23457c80c088dc7c2c49a1aa03584a80c8
+  metadata.gz: be5efa4a81ec3244475bdbbda0dd90aad02076f422ffc0f6def444e5a0553a875a414a3f98049dff7a3471700515168ac7cab1e4394d379d8a316cea4d1981e4
+  data.tar.gz: 515bf628c80f44252579432eb34d27c3f14fb830a7be0e479ca98b5d5651823c5dfb044276d44e9624e122accfab1faf93479827fcf92a5cacc75ef300a534f7

data/README.md

@@ -113,7 +113,12 @@ end
 
 The capsule string DSL is the shortest form for the common case. Use `c.capsule` when you need named capsules with advanced options like `single_active_consumer` or `consumer_priority`. See [Routing and ordering](#routing-and-ordering) for the full set.
 
-> **Migrating from `config/pgbus.yml`?** Run `rails generate pgbus:update` to convert your YAML config to a Ruby initializer using the modern DSL. The original YAML stays in place for review; delete it once the new initializer looks right.
+> **Upgrading from an older pgbus?** Run `rails generate pgbus:update`. It does two things in one pass:
+>
+> - Converts any legacy `config/pgbus.yml` to a Ruby initializer at `config/initializers/pgbus.rb` (skipped if the initializer already exists).
+> - Inspects your live database and adds any missing pgbus migrations to `db/migrate` (or `db/pgbus_migrate` if you use `connects_to`). The generator detects your separate-database config automatically from `Pgbus.configuration.connects_to` or by scanning the initializer / `config/application.rb`, so you don't have to re-specify `--database=pgbus` every time.
+>
+> Useful flags: `--dry-run` (print the plan without creating files), `--skip-config`, `--skip-migrations`, `--quiet`. Running it on a database with no pgbus tables at all will redirect you to `pgbus:install` instead of stacking individual add_* migrations.
 
 ### 2. Use as ActiveJob backend
 

data/app/models/pgbus/stream_stat.rb

@@ -14,9 +14,14 @@ module Pgbus
     EVENT_TYPES = %w[broadcast connect disconnect].freeze
 
     scope :since, ->(time) { where("created_at >= ?", time) }
-    scope :broadcasts, -> { where(event_type: "broadcast") }
-    scope :connects, -> { where(event_type: "connect") }
-    scope :disconnects, -> { where(event_type: "disconnect") }
+    # NOTE: scope names intentionally avoid `broadcasts` / `connects` /
+    # `disconnects`. `turbo-rails` auto-includes `Turbo::Broadcastable`
+    # into every AR model, which defines a class method `broadcasts`.
+    # AR's dangerous_class_method? guard then rejects a colliding
+    # scope name at load time, crashing eager_load. See issue #92.
+    scope :broadcast_events, -> { where(event_type: "broadcast") }
+    scope :connect_events, -> { where(event_type: "connect") }
+    scope :disconnect_events, -> { where(event_type: "disconnect") }
 
     # Records a stream event. Called from the Dispatcher when the
     # `streams_stats_enabled` flag is set. Errors are swallowed so a
@@ -79,7 +84,7 @@ module Pgbus
 
     # Top N streams by broadcast count in the window, with avg fanout.
     def self.top_streams(limit: 10, minutes: 60)
-      broadcasts
+      broadcast_events
         .since(minutes.minutes.ago)
         .group(:stream_name)
         .order(Arel.sql("COUNT(*) DESC"))
@@ -102,7 +107,7 @@ module Pgbus
 
     # Throughput: broadcast events per minute bucketed by minute.
     def self.throughput(minutes: 60)
-      broadcasts
+      broadcast_events
         .since(minutes.minutes.ago)
         .group("date_trunc('minute', created_at)")
         .order(Arel.sql("date_trunc('minute', created_at)"))

data/lib/generators/pgbus/add_job_stats_queue_index_generator.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+require "rails/generators"
+require "rails/generators/active_record"
+
+module Pgbus
+  module Generators
+    class AddJobStatsQueueIndexGenerator < Rails::Generators::Base
+      include ActiveRecord::Generators::Migration
+
+      source_root File.expand_path("templates", __dir__)
+
+      desc "Add composite index on pgbus_job_stats (queue_name, created_at) for Insights latency-by-queue aggregation"
+
+      class_option :database,
+                   type: :string,
+                   default: nil,
+                   desc: "Use a separate database for pgbus tables (e.g. --database=pgbus)"
+
+      def create_migration_file
+        if separate_database?
+          migration_template "add_job_stats_queue_index.rb.erb",
+                             "db/pgbus_migrate/add_pgbus_job_stats_queue_index.rb"
+        else
+          migration_template "add_job_stats_queue_index.rb.erb",
+                             "db/migrate/add_pgbus_job_stats_queue_index.rb"
+        end
+      end
+
+      def display_post_install
+        say ""
+        say "Pgbus job stats queue index added!", :green
+        say ""
+        say "Next steps:"
+        say "  1. Run: rails db:migrate#{":#{options[:database]}" if separate_database?}"
+        say "  2. The Insights 'latency by queue' aggregation will now use the index"
+        say "     instead of sequentially scanning pgbus_job_stats. Install this on"
+        say "     heavy-traffic deployments with a large job stats retention window."
+        say ""
+      end
+
+      private
+
+      def migration_version
+        "[#{ActiveRecord::Migration.current_version}]"
+      end
+
+      def separate_database?
+        options[:database].present?
+      end
+    end
+  end
+end

data/lib/generators/pgbus/templates/add_job_stats_latency.rb.erb

@@ -3,7 +3,10 @@ class AddPgbusJobStatsLatency < ActiveRecord::Migration<%= migration_version %>
     add_column :pgbus_job_stats, :enqueue_latency_ms, :bigint
     add_column :pgbus_job_stats, :retry_count, :integer, default: 0
 
+    # idempotent: the standalone add_job_stats_queue_index generator
+    # creates the same index. Either order is safe.
     add_index :pgbus_job_stats, [:queue_name, :created_at],
-      name: "idx_pgbus_job_stats_queue_time"
+      name: "idx_pgbus_job_stats_queue_time",
+      if_not_exists: true
   end
 end

data/lib/generators/pgbus/templates/add_job_stats_queue_index.rb.erb

@@ -0,0 +1,11 @@
+class AddPgbusJobStatsQueueIndex < ActiveRecord::Migration<%= migration_version %>
+  def change
+    # Supports InsightsController#latency_by_queue and any other
+    # per-queue aggregation over pgbus_job_stats. Without this, a
+    # 60-day window over 1M jobs/day seq-scans the whole table for
+    # every Insights page load with the latency tab open.
+    add_index :pgbus_job_stats, %i[queue_name created_at],
+              name: "idx_pgbus_job_stats_queue_time",
+              if_not_exists: true
+  end
+end

data/lib/generators/pgbus/update_generator.rb

@@ -2,58 +2,171 @@
 
 require "rails/generators"
 require "pgbus/generators/config_converter"
+require "pgbus/generators/migration_detector"
+require "pgbus/generators/database_target_detector"
 
 module Pgbus
   module Generators
-    # Converts an existing config/pgbus.yml to a Ruby initializer at
-    # config/initializers/pgbus.rb using the modern DSL.
+    # Upgrade command with two independent jobs:
     #
-    # The original YAML file is left in place — the user reviews the
-    # generated initializer and deletes the YAML when ready.
+    #   1. Config conversion: if config/pgbus.yml exists, convert it to
+    #      config/initializers/pgbus.rb using the modern Ruby DSL. Skip
+    #      silently if the initializer already exists or the YAML is
+    #      absent — safe to re-run.
+    #
+    #   2. Migration detection: inspect the live database and add any
+    #      missing pgbus migrations to db/migrate (or db/pgbus_migrate
+    #      if a separate database is configured). Invokes each matching
+    #      sub-generator in-process via Thor's invoke, so this mirrors
+    #      what the user would get running each generator by hand.
     #
     # Usage:
     #
     #   bin/rails generate pgbus:update
-    #   bin/rails generate pgbus:update --force      # overwrite existing initializer
-    #   bin/rails generate pgbus:update --source=path/to/pgbus.yml
+    #   bin/rails generate pgbus:update --dry-run
+    #   bin/rails generate pgbus:update --skip-config
+    #   bin/rails generate pgbus:update --skip-migrations
+    #   bin/rails generate pgbus:update --database=pgbus
+    #   bin/rails generate pgbus:update --quiet
     class UpdateGenerator < Rails::Generators::Base
-      desc "Convert config/pgbus.yml to config/initializers/pgbus.rb using the Ruby DSL"
+      desc "Upgrade pgbus: convert YAML config + add any missing migrations"
 
       class_option :source,
                    type: :string,
                    default: "config/pgbus.yml",
-                   desc: "Path to the existing YAML config (default: config/pgbus.yml)"
+                   desc: "Path to an existing YAML config to convert (default: config/pgbus.yml)"
 
       class_option :destination,
                    type: :string,
                    default: "config/initializers/pgbus.rb",
                    desc: "Path to the generated initializer (default: config/initializers/pgbus.rb)"
 
-      def convert
+      class_option :skip_config,
+                   type: :boolean,
+                   default: false,
+                   desc: "Skip the YAML → Ruby initializer conversion step"
+
+      class_option :skip_migrations,
+                   type: :boolean,
+                   default: false,
+                   desc: "Skip the migration detection step"
+
+      class_option :database,
+                   type: :string,
+                   default: nil,
+                   desc: "Use a separate database for pgbus tables (default: auto-detect " \
+                         "from Pgbus.configuration.connects_to or config/initializers/pgbus.rb)"
+
+      class_option :dry_run,
+                   type: :boolean,
+                   default: false,
+                   desc: "Print what would be done without creating any files"
+
+      class_option :quiet,
+                   type: :boolean,
+                   default: false,
+                   desc: "Suppress verbose per-step output"
+
+      def convert_yaml_if_present
+        return if options[:skip_config]
+
         source_path = File.expand_path(options[:source], destination_root)
         destination_path = File.expand_path(options[:destination], destination_root)
 
-        # Thor::Error is the idiomatic way to abort a Rails generator. Thor
-        # catches it, prints the message in red, and exits with status 1
-        # without a Ruby backtrace. exit 1 would skip the framework's
-        # cleanup hooks and is hard to test.
-        raise Thor::Error, "Source file not found: #{options[:source]}" unless File.exist?(source_path)
+        unless File.exist?(source_path)
+          log "YAML config not found at #{options[:source]}; skipping config conversion."
+          return
+        end
+
+        if File.exist?(destination_path)
+          log "Initializer already exists at #{options[:destination]}; skipping config conversion."
+          return
+        end
 
         ruby_source = load_and_convert(source_path)
-        create_file destination_path, ruby_source
+        if options[:dry_run]
+          log_change "[dry-run] would create #{options[:destination]}"
+        else
+          create_file destination_path, ruby_source
+        end
+      end
+
+      def detect_and_install_missing_migrations
+        return if options[:skip_migrations]
+
+        unless active_record_available?
+          log "ActiveRecord not loaded — skipping migration detection. Run this generator from a Rails app."
+          return
+        end
+
+        connection = resolve_connection
+        unless connection
+          log "No ActiveRecord connection available — skipping migration detection."
+          return
+        end
+
+        detector = MigrationDetector.new(connection)
+        missing = detector.missing_migrations
+
+        if missing.empty?
+          log "Database schema is up to date — no migrations needed."
+          return
+        end
+
+        if missing == [MigrationDetector::FRESH_INSTALL]
+          say ""
+          say "Database looks empty of pgbus tables — this is a fresh install.", :yellow
+          say "Run `rails generate pgbus:install` instead of `pgbus:update`.", :yellow
+          say ""
+          return
+        end
+
+        database_name = options[:database] || detected_database_name
+        log "Auto-detected separate database: #{database_name}" if options[:database].nil? && database_name
+
+        log "Found #{missing.size} missing migration(s):"
+        missing.each do |key|
+          description = MigrationDetector::DESCRIPTIONS[key] || key.to_s
+          log "  - #{key}: #{description}"
+        end
+
+        # Two loops on purpose: print the full plan first so operators
+        # see what's coming, then execute. Combining would interleave
+        # "  - add_presence: foo" with "Invoking pgbus:add_presence..."
+        # which hides the shape of the upgrade from the reader.
+        missing.each do |key| # rubocop:disable Style/CombinableLoops
+          generator = MigrationDetector::GENERATOR_MAP[key]
+          unless generator
+            say "  !  no generator mapped for #{key}, skipping", :red
+            next
+          end
+
+          if options[:dry_run]
+            log_change "[dry-run] would invoke #{generator}#{" --database=#{database_name}" if database_name}"
+            next
+          end
+
+          invoke_args = []
+          invoke_args << "--database=#{database_name}" if database_name
+          log "Invoking #{generator}#{" --database=#{database_name}" if database_name}..."
+          invoke generator, invoke_args
+        end
       end
 
       def display_post_install
+        return if options[:quiet]
+
         say ""
-        say "Pgbus initializer generated at #{options[:destination]}!", :green
-        say ""
-        say "Next steps:"
-        say "  1. Review the generated initializer for correctness"
-        say "  2. Boot your app and verify everything still works"
-        say "  3. Delete #{options[:source]} when satisfied (Pgbus will stop reading it)"
+        say "Pgbus update complete.", :green
         say ""
-        say "If you spot a setting that didn't translate cleanly, please open an issue:"
-        say "  https://github.com/mhenrixon/pgbus/issues", :cyan
+        if options[:dry_run]
+          say "Dry-run: no files were created.", :yellow
+        else
+          say "Next steps:"
+          say "  1. Review the generated migration files in db/migrate (or db/pgbus_migrate)"
+          say "  2. Run: rails db:migrate#{":#{effective_database_name}" if effective_database_name}"
+          say "  3. Restart pgbus: bin/pgbus start"
+        end
         say ""
       end
 
@@ -70,6 +183,46 @@ module Pgbus
       rescue ConfigConverter::Error, Psych::Exception, Errno::ENOENT, Errno::EACCES => e
         raise Thor::Error, "Failed to convert #{options[:source]}: #{e.message}"
       end
+
+      def active_record_available?
+        defined?(::ActiveRecord::Base) && ::ActiveRecord::Base.respond_to?(:connection)
+      end
+
+      # Resolve the AR connection to inspect. If pgbus is configured to
+      # use a separate database (via connects_to), use BusRecord's
+      # connection so the detector probes the right schema.
+      def resolve_connection
+        if defined?(Pgbus) && Pgbus.respond_to?(:configuration) && Pgbus.configuration.connects_to
+          Pgbus::BusRecord.connection
+        else
+          ::ActiveRecord::Base.connection
+        end
+      rescue StandardError => e
+        say "  !  could not resolve AR connection: #{e.class}: #{e.message}", :red
+        nil
+      end
+
+      def detected_database_name
+        @detected_database_name ||= DatabaseTargetDetector.new(
+          destination_root: destination_root
+        ).detect
+      end
+
+      def effective_database_name
+        options[:database] || detected_database_name
+      end
+
+      def log(message)
+        return if options[:quiet]
+
+        say message
+      end
+
+      def log_change(message)
+        return if options[:quiet]
+
+        say message, :yellow
+      end
     end
   end
 end

data/lib/pgbus/generators/config_converter.rb

@@ -151,15 +151,23 @@ module Pgbus
       end
 
       # Returns [constant_settings, varying_settings].
-      # constant_settings: { "key" => value }   (same value across all envs)
+      # constant_settings: { "key" => value }   (same value in EVERY env)
       # varying_settings:  { "key" => { env => value, ... } }
+      #
+      # A setting is only "constant" when it is present in every env
+      # and all envs agree on the value. If any env is missing the
+      # setting entirely (e.g. `polling_interval: 0.01` set only under
+      # `test:`), emitting it as an unconditional line would silently
+      # apply the value to envs that never asked for it — see #93.
       def partition_by_variance(all_settings)
         constant = {}
         varying = {}
         all_settings.each do |key, env_values|
           present_values = env_values.reject { |_, v| v == :__missing__ }
           unique_values = present_values.values.uniq
-          if unique_values.size <= 1
+          all_envs_present = present_values.size == env_values.size
+
+          if all_envs_present && unique_values.size <= 1
             constant[key] = unique_values.first
           else
             varying[key] = present_values
@@ -182,6 +190,18 @@ module Pgbus
 
       def render_varying_setting(key, env_values)
         envs = env_values.keys
+
+        # Single-env coverage: the setting exists in exactly one env.
+        # Emit an `if Rails.env.X?` modifier rather than a case block
+        # so other envs fall back to the gem default. This is the
+        # fix for #93 — without it, a `test:`-only `polling_interval`
+        # would leak into dev and prod as an unconditional assignment.
+        if envs.size == 1
+          env = envs.first
+          value = env_values[env]
+          return ["c.#{key} = #{render_value(key, value)} if Rails.env.#{env}?"]
+        end
+
         if envs.size == 2 && envs.include?("development")
           # Special case: "everything except dev" — common pattern
           non_dev_value = env_values.except("development").values.first

data/lib/pgbus/generators/database_target_detector.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+module Pgbus
+  module Generators
+    # Detects whether the host application has configured pgbus to use
+    # a separate database via connects_to, and returns the database name
+    # that --database flags should be set to for sub-generators.
+    #
+    # Detection sources, in priority order:
+    #
+    #   1. Pgbus.configuration.connects_to (runtime — authoritative if
+    #      the initializer has already booted)
+    #   2. config/initializers/pgbus.rb (text scan)
+    #   3. config/application.rb (text scan — fallback for apps that
+    #      call connects_to in application config instead of the pgbus
+    #      initializer)
+    #
+    # Returns a String (the database name, e.g. "pgbus") or nil if no
+    # separate database is configured.
+    class DatabaseTargetDetector
+      # Matches:
+      #   c.connects_to = { database: { writing: :pgbus } }
+      #   c.connects_to(database: { writing: :queue_db })
+      #   Pgbus.configuration.connects_to = { database: { writing: :pgbus } }
+      #
+      # Rejects:
+      #   c.connects_to = { role: :writing }        (different API shape)
+      #   connects_to :something                    (no database: key)
+      #
+      # The [^;\n]*? and [^}]*? laziness keeps the scan within a
+      # reasonable window so a stray "writing:" later in the file
+      # can't cross-contaminate.
+      CONNECTS_TO_PATTERN = /connects_to\b[^;\n]*?database\s*:\s*\{[^}]*?writing\s*:\s*:?(?<name>[a-zA-Z_][a-zA-Z0-9_]*)/m
+
+      def initialize(destination_root:)
+        @destination_root = destination_root
+      end
+
+      # Returns the database name string if a separate DB is configured,
+      # nil otherwise. Checks runtime config first, then falls back to
+      # static file scanning.
+      def detect
+        runtime_database_name || scan_initializer || scan_application_config
+      end
+
+      private
+
+      attr_reader :destination_root
+
+      # Runtime path: Pgbus.configuration.connects_to. Only works if the
+      # host app has loaded pgbus AND booted the initializer. The update
+      # generator runs after Rails app boot so this is usually available.
+      def runtime_database_name
+        return nil unless defined?(Pgbus) && Pgbus.respond_to?(:configuration)
+
+        extract_database_name(Pgbus.configuration.connects_to)
+      rescue StandardError
+        nil
+      end
+
+      def scan_initializer
+        scan_file(File.join(destination_root, "config", "initializers", "pgbus.rb"))
+      end
+
+      def scan_application_config
+        scan_file(File.join(destination_root, "config", "application.rb"))
+      end
+
+      def scan_file(path)
+        return nil unless File.exist?(path)
+
+        content = File.read(path)
+        match = content.match(CONNECTS_TO_PATTERN)
+        return nil unless match
+
+        match[:name]
+      rescue StandardError
+        nil
+      end
+
+      # Parses `{ database: { writing: :name } }` or the String variant
+      # into the database name. Returns nil for anything we don't
+      # recognize.
+      def extract_database_name(connects_to)
+        return nil unless connects_to.is_a?(Hash)
+
+        db = connects_to[:database] || connects_to["database"]
+        return nil unless db.is_a?(Hash)
+
+        (db[:writing] || db["writing"])&.to_s
+      end
+    end
+  end
+end

data/lib/pgbus/generators/migration_detector.rb

@@ -0,0 +1,217 @@
+# frozen_string_literal: true
+
+module Pgbus
+  module Generators
+    # Inspects a live ActiveRecord connection and determines which of
+    # pgbus's migration generators need to run to bring the schema up
+    # to date.
+    #
+    # Usage:
+    #
+    #   detector = Pgbus::Generators::MigrationDetector.new(connection)
+    #   detector.missing_migrations
+    #   # => [:add_uniqueness_keys, :add_job_stats_queue_index, ...]
+    #
+    # The returned symbols correspond to generator names that the
+    # pgbus:update generator invokes via Thor composition. Each symbol
+    # maps to exactly one generator (see GENERATOR_MAP).
+    #
+    # Detection rules:
+    #
+    # 1. Fresh install (no core tables) → returns [:fresh_install] as a
+    #    sentinel. The caller should tell the user to run pgbus:install
+    #    instead of stacking 8 migrations.
+    #
+    # 2. Core tables missing → queued unconditionally. These are features
+    #    pgbus assumes are present.
+    #
+    # 3. Opt-in feature tables missing → queued unconditionally. The user
+    #    asked for update-to-latest, so we add the migration files but
+    #    don't enable the feature in config. They'll opt in separately.
+    #
+    # 4. Columns missing on existing tables → queued. These are in-place
+    #    schema upgrades (e.g. add_job_stats_latency adds
+    #    enqueue_latency_ms + retry_count).
+    #
+    # 5. Indexes missing on existing tables → queued. Additive, safe.
+    #
+    # 6. Modern replacements for legacy tables (e.g. pgbus_job_locks →
+    #    pgbus_uniqueness_keys) → queue the migration path only if the
+    #    legacy table still exists. Otherwise queue the fresh install.
+    class MigrationDetector
+      # Sentinel returned when the database looks empty of pgbus tables.
+      # The caller (pgbus:update generator) should redirect the user to
+      # pgbus:install rather than trying to stack the full schema as
+      # individual add_* migrations.
+      FRESH_INSTALL = :fresh_install
+
+      # The set of tables that the base pgbus:install migration creates.
+      # If NONE of these exist, we treat the DB as a fresh install.
+      CORE_INSTALL_TABLES = %w[
+        pgbus_processed_events
+        pgbus_processes
+        pgbus_failed_events
+        pgbus_semaphores
+        pgbus_blocked_executions
+        pgbus_batches
+      ].freeze
+
+      # generator_key → Rails generator name. Passed to Thor's invoke.
+      #
+      # Note: uniqueness_keys uses the migrate_job_locks generator for
+      # both the fresh-install and upgrade-from-job_locks paths. The
+      # template is idempotent: `unless table_exists?(:pgbus_uniqueness_keys)`
+      # creates it, and `if table_exists?(:pgbus_job_locks)` drops the
+      # legacy table. One generator covers both cases.
+      GENERATOR_MAP = {
+        uniqueness_keys: "pgbus:migrate_job_locks",
+        add_job_stats: "pgbus:add_job_stats",
+        add_job_stats_latency: "pgbus:add_job_stats_latency",
+        add_job_stats_queue_index: "pgbus:add_job_stats_queue_index",
+        add_stream_stats: "pgbus:add_stream_stats",
+        add_presence: "pgbus:add_presence",
+        add_queue_states: "pgbus:add_queue_states",
+        add_outbox: "pgbus:add_outbox",
+        add_recurring: "pgbus:add_recurring",
+        add_failed_events_index: "pgbus:add_failed_events_index"
+      }.freeze
+
+      # Human-friendly description of each migration for the generator
+      # output. Keeps the update generator's run log readable.
+      DESCRIPTIONS = {
+        uniqueness_keys: "uniqueness keys table (job deduplication, also upgrades legacy job_locks if present)",
+        add_job_stats: "job stats table (Insights dashboard)",
+        add_job_stats_latency: "job stats latency columns (enqueue_latency_ms, retry_count)",
+        add_job_stats_queue_index: "job stats (queue_name, created_at) index",
+        add_stream_stats: "stream stats table (opt-in real-time Insights)",
+        add_presence: "presence members table (Turbo Streams presence)",
+        add_queue_states: "queue states table (pause/resume)",
+        add_outbox: "outbox entries table (transactional outbox)",
+        add_recurring: "recurring tasks + executions tables",
+        add_failed_events_index: "unique index on pgbus_failed_events (queue_name, msg_id)"
+      }.freeze
+
+      def initialize(connection)
+        @connection = connection
+      end
+
+      # Returns an Array of generator keys (symbols) in the order they
+      # should run. Dependencies are resolved implicitly via order: the
+      # base table creation for a feature always comes before the
+      # column/index add-ons.
+      def missing_migrations
+        return [FRESH_INSTALL] if fresh_install?
+
+        [
+          *uniqueness_key_migrations,
+          *job_stats_migrations,
+          *stream_stats_migrations,
+          *presence_migrations,
+          *queue_states_migrations,
+          *outbox_migrations,
+          *recurring_migrations,
+          *failed_events_index_migrations
+        ]
+      end
+
+      private
+
+      attr_reader :connection
+
+      def fresh_install?
+        CORE_INSTALL_TABLES.none? { |t| table_exists?(t) }
+      end
+
+      # Legacy pgbus_job_locks → modern pgbus_uniqueness_keys.
+      # The migrate_job_locks template is idempotent: it creates
+      # uniqueness_keys if missing and drops job_locks if present.
+      # One symbol covers both cases; see GENERATOR_MAP.
+      def uniqueness_key_migrations
+        return [] if table_exists?("pgbus_uniqueness_keys")
+
+        [:uniqueness_keys]
+      end
+
+      def job_stats_migrations
+        migrations = []
+
+        unless table_exists?("pgbus_job_stats")
+          migrations << :add_job_stats
+          # The latency columns and queue index are add-ons to job_stats.
+          # If we're creating the base table now, the add_job_stats
+          # template doesn't include them — add the upgrade migrations
+          # so a fresh install lands on the current schema.
+          migrations << :add_job_stats_latency
+          migrations << :add_job_stats_queue_index
+          return migrations
+        end
+
+        # Base table exists — check each add-on independently.
+        cols = column_names("pgbus_job_stats")
+        migrations << :add_job_stats_latency unless cols.include?("enqueue_latency_ms") && cols.include?("retry_count")
+
+        migrations << :add_job_stats_queue_index unless index_exists?("pgbus_job_stats", "idx_pgbus_job_stats_queue_time")
+
+        migrations
+      end
+
+      def stream_stats_migrations
+        table_exists?("pgbus_stream_stats") ? [] : [:add_stream_stats]
+      end
+
+      def presence_migrations
+        table_exists?("pgbus_presence_members") ? [] : [:add_presence]
+      end
+
+      def queue_states_migrations
+        table_exists?("pgbus_queue_states") ? [] : [:add_queue_states]
+      end
+
+      def outbox_migrations
+        table_exists?("pgbus_outbox_entries") ? [] : [:add_outbox]
+      end
+
+      def recurring_migrations
+        # pgbus_recurring_tasks + pgbus_recurring_executions are the two
+        # tables the recurring generator creates. If BOTH exist, nothing
+        # to do. If either is missing, we need the generator (which is
+        # idempotent via if_not_exists on both tables).
+        return [] if table_exists?("pgbus_recurring_tasks") && table_exists?("pgbus_recurring_executions")
+
+        [:add_recurring]
+      end
+
+      def failed_events_index_migrations
+        # pgbus_failed_events is created by pgbus:install, but the
+        # unique (queue_name, msg_id) index was added later via its own
+        # generator. If the table exists without the unique index,
+        # FailedEventRecorder's upsert silently swallows ON CONFLICT
+        # errors — so this is a real bug waiting to bite.
+        return [] unless table_exists?("pgbus_failed_events")
+        return [] if index_exists?("pgbus_failed_events", "idx_pgbus_failed_events_queue_msg")
+
+        [:add_failed_events_index]
+      end
+
+      # --- schema probes -------------------------------------------------
+
+      def table_exists?(name)
+        connection.table_exists?(name)
+      rescue StandardError
+        false
+      end
+
+      def column_names(table)
+        connection.columns(table).map(&:name)
+      rescue StandardError
+        []
+      end
+
+      def index_exists?(table, index_name)
+        connection.indexes(table).any? { |idx| idx.name == index_name }
+      rescue StandardError
+        false
+      end
+    end
+  end
+end

data/lib/pgbus/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Pgbus
-  VERSION = "0.6.0"
+  VERSION = "0.6.1"
 end

data/lib/pgbus/web/data_source.rb

@@ -38,7 +38,10 @@ module Pgbus
       # different connection lifecycle than the worker processes).
       def queues_with_metrics
         queue_names = connection.select_values("SELECT queue_name FROM pgmq.meta ORDER BY queue_name")
-        paused_queues = paused_queue_names
+        # paused_queue_names returns an Array; convert to Set so the
+        # per-queue membership check is O(1). With 100+ queues the
+        # Array#include? cost in the loop was O(n²) per dashboard load.
+        paused_queues = paused_queue_names.to_set
         queue_names.map { |name| queue_metrics_via_sql(name) }.compact.map do |q|
           q.merge(paused: paused_queues.include?(logical_queue_name(q[:name])))
         end
@@ -263,11 +266,7 @@ module Pgbus
         queues = queues_with_metrics.select { |q| q[:name].end_with?(dlq_suffix) }
         offset = (page - 1) * per_page
 
-        messages = queues.flat_map do |q|
-          query_queue_messages_raw(q[:name], per_page + offset, 0)
-        end
-
-        messages.sort_by { |m| -m[:msg_id].to_i }.slice(offset, per_page) || []
+        paginated_queue_messages(queues.map { |q| q[:name] }, per_page, offset)
       rescue StandardError => e
         Pgbus.logger.debug { "[Pgbus::Web] Error fetching DLQ messages: #{e.message}" }
         []
@@ -695,10 +694,41 @@ module Pgbus
       def all_queue_messages(limit, offset)
         dlq_suffix = Pgbus::DEAD_LETTER_SUFFIX
         queues = queues_with_metrics.reject { |q| q[:name].end_with?(dlq_suffix) }
-        messages = queues.flat_map do |q|
-          query_queue_messages_raw(q[:name], limit + offset, 0)
+        paginated_queue_messages(queues.map { |q| q[:name] }, limit, offset)
+      end
+
+      # Returns messages from multiple PGMQ queues in a single paginated
+      # query. Builds a UNION ALL across the target tables and pushes the
+      # ORDER BY msg_id DESC + LIMIT + OFFSET down to Postgres so we don't
+      # load (limit + offset) rows from every queue into Ruby just to slice
+      # out one page. Returns [] if queue_names is empty.
+      #
+      # Each UNION ALL fragment selects a literal queue name so the outer
+      # query can tag every row with its source queue — the pre-SQL
+      # implementation tagged it from the iteration variable. The queue
+      # name goes through sanitize_name (which calls QueueNameValidator)
+      # so it's safe to interpolate into both the schema-qualified table
+      # and the literal column. limit/offset are bound parameters.
+      def paginated_queue_messages(queue_names, limit, offset)
+        return [] if queue_names.empty?
+
+        sanitized = queue_names.map { |name| [name, sanitize_name(name)] }
+        fragments = sanitized.map do |(name, qtable)|
+          <<~SQL.strip
+            SELECT msg_id, read_ct, enqueued_at, last_read_at, vt, message, headers,
+                   '#{name}' AS queue_name
+            FROM pgmq.q_#{qtable}
+          SQL
         end
-        messages.sort_by { |m| -m[:msg_id].to_i }.slice(offset, limit) || []
+
+        sql = <<~SQL
+          SELECT * FROM (#{fragments.join("\nUNION ALL\n")}) AS combined
+          ORDER BY msg_id DESC
+          LIMIT $1 OFFSET $2
+        SQL
+
+        rows = connection.select_all(sql, "Pgbus Paginated Queue Messages", [limit, offset])
+        rows.to_a.map { |r| format_message(r, r["queue_name"]) }
       end
 
       def queue_metrics_via_sql(queue_name)
@@ -903,12 +933,31 @@ module Pgbus
       end
 
       # Archive every queue message referenced by a failed_event row.
+      # Groups by queue and uses archive_batch so an N-event discard is
+      # one SQL statement per queue instead of N per-row roundtrips.
+      # Falls back to per-row archive inside a rescue if a batch fails,
+      # so one bad queue can't block progress on the others.
       def archive_all_failed_messages
         rows = connection.select_all(
           "SELECT id, queue_name, msg_id FROM pgbus_failed_events WHERE msg_id IS NOT NULL",
           "Pgbus Collect Failed Messages"
         )
-        rows.to_a.each { |row| archive_failed_message(row) }
+
+        grouped = rows.to_a.group_by { |row| row["queue_name"] }
+        grouped.each do |queue_name, events|
+          msg_ids = events.filter_map { |row| row["msg_id"]&.to_i }
+          next if msg_ids.empty?
+
+          begin
+            @client.archive_batch(queue_name, msg_ids)
+          rescue StandardError => e
+            Pgbus.logger.debug do
+              "[Pgbus::Web] archive_batch failed for #{queue_name} (#{e.message}); " \
+                "falling back to per-row archive"
+            end
+            events.each { |row| archive_failed_message(row) }
+          end
+        end
       rescue StandardError => e
         Pgbus.logger.debug { "[Pgbus::Web] Error archiving failed messages: #{e.message}" }
       end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: pgbus
 version: !ruby/object:Gem::Version
-  version: 0.6.0
+  version: 0.6.1
 platform: ruby
 authors:
 - Mikael Henriksson
@@ -205,6 +205,7 @@ files:
 - lib/generators/pgbus/add_job_locks_generator.rb
 - lib/generators/pgbus/add_job_stats_generator.rb
 - lib/generators/pgbus/add_job_stats_latency_generator.rb
+- lib/generators/pgbus/add_job_stats_queue_index_generator.rb
 - lib/generators/pgbus/add_outbox_generator.rb
 - lib/generators/pgbus/add_presence_generator.rb
 - lib/generators/pgbus/add_queue_states_generator.rb
@@ -216,6 +217,7 @@ files:
 - lib/generators/pgbus/templates/add_job_locks.rb.erb
 - lib/generators/pgbus/templates/add_job_stats.rb.erb
 - lib/generators/pgbus/templates/add_job_stats_latency.rb.erb
+- lib/generators/pgbus/templates/add_job_stats_queue_index.rb.erb
 - lib/generators/pgbus/templates/add_outbox.rb.erb
 - lib/generators/pgbus/templates/add_presence.rb.erb
 - lib/generators/pgbus/templates/add_queue_states.rb.erb
@@ -255,6 +257,8 @@ files:
 - lib/pgbus/event_bus/subscriber.rb
 - lib/pgbus/failed_event_recorder.rb
 - lib/pgbus/generators/config_converter.rb
+- lib/pgbus/generators/database_target_detector.rb
+- lib/pgbus/generators/migration_detector.rb
 - lib/pgbus/instrumentation.rb
 - lib/pgbus/outbox.rb
 - lib/pgbus/outbox/poller.rb