pgbus 0.8.3 → 0.9.0

data/config/locales/nb.yml

@@ -1,6 +1,35 @@
 ---
 nb:
   pgbus:
+    batches:
+      index:
+        description: Jobbgrupper med fremdriftssporing og tilbakekall
+        empty: Ingen grupper funnet
+        headers:
+          batch_id: Gruppe-ID
+          created: Opprettet
+          description: Beskrivelse
+          jobs: Jobber
+          progress: Fremdrift
+          status: Status
+        title: Grupper
+      show:
+        back: Tilbake til grupper
+        batch_id: Gruppe-ID
+        completed: Fullført
+        created_at: Opprettet den
+        details: Detaljer
+        discarded: Forkastet
+        finished_at: Avsluttet den
+        not_found: Gruppe ikke funnet
+        on_discard: Ved forkastning
+        on_finish: Ved avslutning
+        on_success: Ved suksess
+        progress: Fremdrift
+        properties: Egenskaper
+        remaining: Gjenstående
+        title: Gruppe
+        total_jobs: Totalt antall jobber
     dashboard:
       processes_table:
         empty: Ingen prosesser kjører
@@ -189,6 +218,10 @@ nb:
         not_found: Hendelse ikke funnet
         title: Hendelse %{event_id}
     helpers:
+      batch_status:
+        finished: Fullført
+        pending: Avventer
+        processing: Behandler
       bulk_select_all: Velg alle
       bulk_select_row: Velg %{id}
       bulk_selected: valgt
@@ -346,6 +379,7 @@ nb:
     layout:
       brand: Pgbus
       nav:
+        batches: Grupper
         dashboard: Dashbord
         dlq: DLQ
         events: Hendelser

data/config/locales/nl.yml

@@ -1,6 +1,35 @@
 ---
 nl:
   pgbus:
+    batches:
+      index:
+        description: Taakgroepen met voortgangsregistratie en callbacks
+        empty: Geen groepen gevonden
+        headers:
+          batch_id: Groep-ID
+          created: Aangemaakt
+          description: Beschrijving
+          jobs: Taken
+          progress: Voortgang
+          status: Status
+        title: Groepen
+      show:
+        back: Terug naar groepen
+        batch_id: Groep-ID
+        completed: Voltooid
+        created_at: Aangemaakt op
+        details: Details
+        discarded: Verworpen
+        finished_at: Voltooid op
+        not_found: Groep niet gevonden
+        on_discard: Bij verwerping
+        on_finish: Bij voltooiing
+        on_success: Bij succes
+        progress: Voortgang
+        properties: Eigenschappen
+        remaining: Resterend
+        title: Groep
+        total_jobs: Totaal taken
     dashboard:
       processes_table:
         empty: Geen processen actief
@@ -189,6 +218,10 @@ nl:
         not_found: Gebeurtenis niet gevonden
         title: Gebeurtenis %{event_id}
     helpers:
+      batch_status:
+        finished: Voltooid
+        pending: In afwachting
+        processing: Bezig
       bulk_select_all: Alles selecteren
       bulk_select_row: Selecteer %{id}
       bulk_selected: geselecteerd
@@ -346,6 +379,7 @@ nl:
     layout:
       brand: Pgbus
       nav:
+        batches: Groepen
         dashboard: Dashboard
         dlq: DLQ
         events: Evenementen

data/config/locales/pt.yml

@@ -1,6 +1,35 @@
 ---
 pt:
   pgbus:
+    batches:
+      index:
+        description: Lotes de trabalhos com acompanhamento de progresso e callbacks
+        empty: Nenhum lote encontrado
+        headers:
+          batch_id: ID do lote
+          created: Criado
+          description: Descrição
+          jobs: Trabalhos
+          progress: Progresso
+          status: Status
+        title: Lotes
+      show:
+        back: Voltar para lotes
+        batch_id: ID do lote
+        completed: Concluídos
+        created_at: Criado em
+        details: Detalhes
+        discarded: Descartados
+        finished_at: Finalizado em
+        not_found: Lote não encontrado
+        on_discard: Ao descartar
+        on_finish: Ao finalizar
+        on_success: Ao ter sucesso
+        progress: Progresso
+        properties: Propriedades
+        remaining: Restantes
+        title: Lote
+        total_jobs: Total de trabalhos
     dashboard:
       processes_table:
         empty: Nenhum processo em execução
@@ -189,6 +218,10 @@ pt:
         not_found: Evento não encontrado
         title: Evento %{event_id}
     helpers:
+      batch_status:
+        finished: Finalizado
+        pending: Pendente
+        processing: Processando
       bulk_select_all: Selecionar todos
       bulk_select_row: Selecionar %{id}
       bulk_selected: selecionado
@@ -346,6 +379,7 @@ pt:
     layout:
       brand: Pgbus
       nav:
+        batches: Lotes
         dashboard: Painel
         dlq: DLQ
         events: Eventos

data/config/locales/sv.yml

@@ -1,6 +1,35 @@
 ---
 sv:
   pgbus:
+    batches:
+      index:
+        description: Jobbgrupper med framstegsspårning och callbacks
+        empty: Inga grupper hittades
+        headers:
+          batch_id: Grupp-ID
+          created: Skapad
+          description: Beskrivning
+          jobs: Jobb
+          progress: Framsteg
+          status: Status
+        title: Grupper
+      show:
+        back: Tillbaka till grupper
+        batch_id: Grupp-ID
+        completed: Slutförda
+        created_at: Skapad den
+        details: Detaljer
+        discarded: Kasserade
+        finished_at: Avslutad den
+        not_found: Grupp hittades inte
+        on_discard: Vid kassering
+        on_finish: Vid avslut
+        on_success: Vid framgång
+        progress: Framsteg
+        properties: Egenskaper
+        remaining: Återstående
+        title: Grupp
+        total_jobs: Totalt antal jobb
     dashboard:
       processes_table:
         empty: Inga processer körs
@@ -189,6 +218,10 @@ sv:
         not_found: Händelse hittades inte
         title: Händelse %{event_id}
     helpers:
+      batch_status:
+        finished: Slutförd
+        pending: Väntande
+        processing: Bearbetas
       bulk_select_all: Markera alla
       bulk_select_row: Markera %{id}
       bulk_selected: valda
@@ -346,6 +379,7 @@ sv:
     layout:
       brand: Pgbus
       nav:
+        batches: Grupper
         dashboard: Instrumentpanel
         dlq: DLQ
         events: Händelser

data/config/routes.rb

@@ -45,6 +45,7 @@ Pgbus::Engine.routes.draw do
     end
   end
 
+  resources :batches, only: %i[index show]
   resources :processes, only: [:index]
 
   resources :events, only: %i[index show] do

data/lib/pgbus/client.rb

@@ -351,6 +351,87 @@ module Pgbus
       total
     end
 
+    # --- Grouped reads (PGMQ v1.11.0+) ---
+
+    def read_grouped(queue_name, qty:, vt: nil)
+      full_name = config.queue_name(queue_name)
+      Instrumentation.instrument("pgbus.client.read_grouped", queue: full_name, qty: qty) do
+        with_stale_connection_retry do
+          synchronized { @pgmq.read_grouped(full_name, vt: vt || config.visibility_timeout, qty: qty) }
+        end
+      end
+    end
+
+    def read_grouped_rr(queue_name, qty:, vt: nil)
+      full_name = config.queue_name(queue_name)
+      Instrumentation.instrument("pgbus.client.read_grouped_rr", queue: full_name, qty: qty) do
+        with_stale_connection_retry do
+          synchronized { @pgmq.read_grouped_rr(full_name, vt: vt || config.visibility_timeout, qty: qty) }
+        end
+      end
+    end
+
+    def read_grouped_head(queue_name, qty:, vt: nil)
+      full_name = config.queue_name(queue_name)
+      with_stale_connection_retry do
+        synchronized { @pgmq.read_grouped_head(full_name, vt: vt || config.visibility_timeout, qty: qty) }
+      end
+    end
+
+    # --- FIFO index management (PGMQ v1.11.0+) ---
+
+    def create_fifo_index(queue_name)
+      full_name = config.queue_name(queue_name)
+      with_stale_connection_retry do
+        synchronized { @pgmq.create_fifo_index(full_name) }
+      end
+    end
+
+    def create_fifo_indexes_all
+      with_stale_connection_retry do
+        synchronized { @pgmq.create_fifo_indexes_all }
+      end
+    end
+
+    # --- LISTEN/NOTIFY management (PGMQ v1.11.0+) ---
+
+    def wait_for_notify(queue_name, timeout: nil, &block)
+      full_name = config.queue_name(queue_name)
+      with_stale_connection_retry do
+        synchronized { @pgmq.wait_for_notify(full_name, timeout: timeout, &block) }
+      end
+    end
+
+    def update_notify_insert(queue_name, throttle_interval_ms:)
+      full_name = config.queue_name(queue_name)
+      with_stale_connection_retry do
+        synchronized { @pgmq.update_notify_insert(full_name, throttle_interval_ms: throttle_interval_ms) }
+      end
+    end
+
+    def list_notify_insert_throttles
+      with_stale_connection_retry do
+        synchronized { @pgmq.list_notify_insert_throttles }
+      end
+    end
+
+    # --- Archive partitioning (requires pg_partman extension) ---
+
+    def convert_archive_partitioned(queue_name, partition_interval: "10000", retention_interval: "100000",
+                                    leading_partition: 10)
+      full_name = config.queue_name(queue_name)
+      with_stale_connection_retry do
+        synchronized do
+          @pgmq.convert_archive_partitioned(
+            full_name,
+            partition_interval: partition_interval,
+            retention_interval: retention_interval,
+            leading_partition: leading_partition
+          )
+        end
+      end
+    end
+
     # Topic routing
     def bind_topic(pattern, queue_name)
       full_name = config.queue_name(queue_name)
@@ -503,6 +584,7 @@ module Pgbus
           @pgmq.create(full_name)
           tune_autovacuum(full_name)
           enable_notify_if_needed(full_name, NOTIFY_THROTTLE_MS)
+          create_fifo_index_if_needed(full_name)
         end
         true
       end
@@ -515,6 +597,12 @@ module Pgbus
       @pgmq.enable_notify_insert(full_name, throttle_interval_ms: throttle_ms)
     end
 
+    def create_fifo_index_if_needed(full_name)
+      return unless config.group_mode
+
+      @pgmq.create_fifo_index(full_name)
+    end
+
     # Check whether the NOTIFY trigger already exists on this queue with the
     # expected throttle interval. When it does, we can skip the destructive
     # DROP TRIGGER + CREATE TRIGGER cycle that causes deadlocks when multiple
@@ -544,11 +632,21 @@ module Pgbus
       false
     end
 
+    # Apply PGMQ-tuned autovacuum + storage parameters to a queue's tables.
+    #
+    # Delegates to pgmq-ruby's tune_autovacuum (v0.7+), which sets the same
+    # queue/archive parameters pgbus used to apply by hand — vacuum scale
+    # factor 0.01/0.05, cost_delay 2/5, analyze scale factor 0.05, and
+    # fillfactor 70 on the queue table — plus a vacuum_threshold floor of 50.
+    # It quotes/lowercases the table name and runs both ALTER TABLEs in one
+    # pooled checkout. Tuning is best-effort: a failure here never blocks a
+    # queue from being usable, so we log and move on.
+    #
+    # Pgbus::AutovacuumTuning is still the source for the migration generators
+    # (sql_for_all_queues, sql_for_high_churn_tables) which tune pgbus-owned
+    # metadata tables the gem doesn't know about.
     def tune_autovacuum(queue_name)
-      with_raw_connection do |conn|
-        conn.exec(AutovacuumTuning.sql_for_queue(queue_name))
-        conn.exec(TableMaintenance.fillfactor_sql_for_queue(queue_name))
-      end
+      @pgmq.tune_autovacuum(queue_name)
     rescue StandardError => e
       Pgbus.logger.debug { "[Pgbus::Client] Autovacuum tuning failed for #{queue_name}: #{e.message}" }
     end

data/lib/pgbus/configuration.rb

@@ -44,6 +44,12 @@ module Pgbus
     # Priority queues
     attr_accessor :priority_levels, :default_priority
 
+    # Grouped reads (PGMQ v1.11.0+ FIFO grouping).
+    # nil = disabled (default read_batch behavior).
+    # :fifo = use read_grouped (drains oldest group first, throughput-optimized).
+    # :round_robin = use read_grouped_rr (fair round-robin across groups).
+    attr_reader :group_mode
+
     # Archive compaction. Only the user-facing retention window is configurable;
     # the loop interval and batch size are tuned via constants on
     # Pgbus::Process::Dispatcher.
@@ -106,7 +112,8 @@ module Pgbus
                   :streams_write_deadline_ms, :streams_falcon_streaming_body,
                   :streams_stats_enabled, :streams_test_mode,
                   :streams_orphan_sweep_interval, :streams_orphan_threshold,
-                  :streams_durable_patterns
+                  :streams_durable_patterns,
+                  :streams_host, :streams_port, :streams_database_url
     attr_reader :streams_default_broadcast_mode # rubocop:disable Style/AccessorGrouping
 
     # AppSignal integration (auto-loaded when ::Appsignal is defined and this is true).
@@ -145,6 +152,7 @@ module Pgbus
 
       @priority_levels = nil
       @default_priority = 1
+      @group_mode = nil
 
       @archive_retention = 7 * 24 * 3600 # 7 days
 
@@ -193,6 +201,22 @@ module Pgbus
       @streams_enabled = true
       @streams_path = nil
       @streams_queue_prefix = "pgbus_stream"
+      # Streamer-only connection overrides. The Streamer's Listener owns a
+      # dedicated long-lived `wait_for_notify` PG connection that can't go
+      # through a PgBouncer in transaction mode (LISTEN/NOTIFY don't survive
+      # transaction-pool COMMIT boundaries — see PlanetScale's docs). Setting
+      # any of these overrides only the Streamer's connection options; the
+      # worker, dispatcher, and client publish paths keep using the regular
+      # `database_url` / `connection_params` (typically pooled).
+      #
+      #   streams_host          — override host only
+      #   streams_port          — override port only (most common case:
+      #                           pooler is on 6432, direct is 5432)
+      #   streams_database_url  — full URL override; takes precedence over
+      #                           the host/port surgicals when set
+      @streams_host = nil
+      @streams_port = nil
+      @streams_database_url = nil
       @streams_signed_name_secret = nil
       @streams_default_retention = 5 * 60 # 5 minutes
       @streams_retention = {}
@@ -283,6 +307,24 @@ module Pgbus
       @streams_default_broadcast_mode = mode
     end
 
+    VALID_GROUP_MODES = [nil, :fifo, :round_robin].freeze
+
+    def group_mode=(mode)
+      coerced = case mode
+                when nil then nil
+                when Symbol then mode
+                when String then mode.to_sym
+                else
+                  raise ArgumentError,
+                        "Invalid group_mode type: #{mode.class}. Must be nil, String, or Symbol"
+                end
+      unless VALID_GROUP_MODES.include?(coerced)
+        raise ArgumentError, "Invalid group_mode: #{coerced.inspect}. Must be nil, :fifo, or :round_robin"
+      end
+
+      @group_mode = coerced
+    end
+
     VALID_PGMQ_SCHEMA_MODES = %i[auto extension embedded].freeze
 
     def pgmq_schema_mode=(mode)
@@ -608,6 +650,43 @@ module Pgbus
       end
     end
 
+    # Connection options the Streamer's dedicated LISTEN/NOTIFY PG connection
+    # should use. Defaults to `connection_options` (same as workers and the
+    # publish path). If any of `streams_database_url`, `streams_host`, or
+    # `streams_port` is set, the Streamer's connection is reconfigured —
+    # everything else keeps using the base options.
+    #
+    # The typical use is "workers go through PgBouncer, streamer goes direct":
+    #
+    #   c.connects_to = { database: { writing: :pgbus } }   # pooler
+    #   c.streams_port = 5432                               # direct
+    #
+    # Precedence: streams_database_url > streams_host/port override > base.
+    def streams_connection_options
+      return streams_database_url if streams_database_url
+
+      base = connection_options
+      return base unless streams_host || streams_port
+
+      case base
+      when Hash
+        result = base.dup
+        result[:host] = streams_host if streams_host
+        result[:port] = streams_port if streams_port
+        result
+      when String
+        # libpq's conninfo parser takes later key=value pairs as overrides
+        # for earlier ones, so we just append. Handles both URI form
+        # (postgres://...) and key=value form.
+        parts = [base]
+        parts << "host=#{streams_host}" if streams_host
+        parts << "port=#{streams_port}" if streams_port
+        parts.join(" ")
+      else
+        base
+      end
+    end
+
     private
 
     # Coerce a duration setting value to a positive Numeric.

data/lib/pgbus/process/supervisor.rb

@@ -69,6 +69,7 @@ module Pgbus
         single_active = worker_config[:single_active_consumer] || worker_config["single_active_consumer"] || false
         priority = worker_config[:consumer_priority] || worker_config["consumer_priority"] || 0
         exec_mode = config.execution_mode_for(worker_config)
+        grp_mode = worker_config[:group_mode] || worker_config["group_mode"] || config.group_mode
 
         pid = fork do
           restore_signals
@@ -78,7 +79,7 @@ module Pgbus
           worker = Worker.new(
             queues: queues, threads: threads, config: config,
             single_active_consumer: single_active, consumer_priority: priority,
-            execution_mode: exec_mode
+            execution_mode: exec_mode, group_mode: grp_mode
           )
           worker.run
         end

data/lib/pgbus/process/worker.rb

@@ -11,12 +11,24 @@ module Pgbus
 
       def initialize(queues:, threads: 5, config: Pgbus.configuration,
                      single_active_consumer: false, consumer_priority: 0,
-                     execution_mode: :threads)
+                     execution_mode: :threads, group_mode: nil)
         @queues = Array(queues)
         @wildcard = @queues.include?("*")
         @threads = threads
         @config = config
         @execution_mode = ExecutionPools.normalize_mode(execution_mode)
+        @group_mode = case group_mode
+                      when nil then nil
+                      when Symbol then group_mode
+                      when String then group_mode.to_sym
+                      else
+                        raise ArgumentError,
+                              "Invalid group_mode type: #{group_mode.class}. Must be nil, String, or Symbol"
+                      end
+        unless Pgbus::Configuration::VALID_GROUP_MODES.include?(@group_mode)
+          raise ArgumentError,
+                "Invalid group_mode: #{@group_mode.inspect}. Must be nil, :fifo, or :round_robin"
+        end
         @single_active_consumer = single_active_consumer
         @consumer_priority = consumer_priority
         @lifecycle = Lifecycle.new
@@ -141,6 +153,8 @@ module Pgbus
 
         if priority_enabled?
           fetch_prioritized(active_queues, qty)
+        elsif @group_mode
+          fetch_grouped(active_queues, qty)
         elsif active_queues.size == 1
           queue = active_queues.first
           messages = Pgbus.client.read_batch(queue, qty: qty) || []
@@ -210,6 +224,29 @@ module Pgbus
         end
       end
 
+      # Use grouped reads for fair or throughput-optimized multi-tenant processing.
+      # Each queue is read independently with the configured group strategy.
+      def fetch_grouped(active_queues, qty)
+        remaining = qty
+        results = []
+
+        active_queues.each do |queue|
+          break if remaining <= 0
+
+          messages = case @group_mode
+                     when :round_robin
+                       Pgbus.client.read_grouped_rr(queue, qty: remaining) || []
+                     else # :fifo
+                       Pgbus.client.read_grouped(queue, qty: remaining) || []
+                     end
+
+          messages.each { |m| results << [queue, m] }
+          remaining -= messages.size
+        end
+
+        results
+      end
+
       def priority_enabled?
         config.priority_levels && config.priority_levels > 1
       end

data/lib/pgbus/serializer.rb

@@ -58,7 +58,7 @@ module Pgbus
       raise ArgumentError, "Invalid GlobalID: #{gid_string.inspect}" unless gid
 
       allowed = Pgbus.configuration.allowed_global_id_models
-      if allowed&.empty?
+      if allowed && allowed.empty?
         raise ArgumentError,
               "GlobalID deserialization is disabled (allowed_global_id_models is empty). " \
               "Set to nil to allow all models, or add permitted classes."

data/lib/pgbus/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Pgbus
-  VERSION = "0.8.3"
+  VERSION = "0.9.0"
 end