pgbus 0.8.4 → 0.9.0

data/config/locales/nb.yml

@@ -1,6 +1,35 @@
 ---
 nb:
   pgbus:
+    batches:
+      index:
+        description: Jobbgrupper med fremdriftssporing og tilbakekall
+        empty: Ingen grupper funnet
+        headers:
+          batch_id: Gruppe-ID
+          created: Opprettet
+          description: Beskrivelse
+          jobs: Jobber
+          progress: Fremdrift
+          status: Status
+        title: Grupper
+      show:
+        back: Tilbake til grupper
+        batch_id: Gruppe-ID
+        completed: Fullført
+        created_at: Opprettet den
+        details: Detaljer
+        discarded: Forkastet
+        finished_at: Avsluttet den
+        not_found: Gruppe ikke funnet
+        on_discard: Ved forkastning
+        on_finish: Ved avslutning
+        on_success: Ved suksess
+        progress: Fremdrift
+        properties: Egenskaper
+        remaining: Gjenstående
+        title: Gruppe
+        total_jobs: Totalt antall jobber
     dashboard:
       processes_table:
         empty: Ingen prosesser kjører
@@ -189,6 +218,10 @@ nb:
         not_found: Hendelse ikke funnet
         title: Hendelse %{event_id}
     helpers:
+      batch_status:
+        finished: Fullført
+        pending: Avventer
+        processing: Behandler
       bulk_select_all: Velg alle
       bulk_select_row: Velg %{id}
       bulk_selected: valgt
@@ -346,6 +379,7 @@ nb:
     layout:
       brand: Pgbus
       nav:
+        batches: Grupper
         dashboard: Dashbord
         dlq: DLQ
         events: Hendelser

data/config/locales/nl.yml

@@ -1,6 +1,35 @@
 ---
 nl:
   pgbus:
+    batches:
+      index:
+        description: Taakgroepen met voortgangsregistratie en callbacks
+        empty: Geen groepen gevonden
+        headers:
+          batch_id: Groep-ID
+          created: Aangemaakt
+          description: Beschrijving
+          jobs: Taken
+          progress: Voortgang
+          status: Status
+        title: Groepen
+      show:
+        back: Terug naar groepen
+        batch_id: Groep-ID
+        completed: Voltooid
+        created_at: Aangemaakt op
+        details: Details
+        discarded: Verworpen
+        finished_at: Voltooid op
+        not_found: Groep niet gevonden
+        on_discard: Bij verwerping
+        on_finish: Bij voltooiing
+        on_success: Bij succes
+        progress: Voortgang
+        properties: Eigenschappen
+        remaining: Resterend
+        title: Groep
+        total_jobs: Totaal taken
     dashboard:
       processes_table:
         empty: Geen processen actief
@@ -189,6 +218,10 @@ nl:
         not_found: Gebeurtenis niet gevonden
         title: Gebeurtenis %{event_id}
     helpers:
+      batch_status:
+        finished: Voltooid
+        pending: In afwachting
+        processing: Bezig
       bulk_select_all: Alles selecteren
       bulk_select_row: Selecteer %{id}
       bulk_selected: geselecteerd
@@ -346,6 +379,7 @@ nl:
     layout:
       brand: Pgbus
       nav:
+        batches: Groepen
         dashboard: Dashboard
         dlq: DLQ
         events: Evenementen

data/config/locales/pt.yml

@@ -1,6 +1,35 @@
 ---
 pt:
   pgbus:
+    batches:
+      index:
+        description: Lotes de trabalhos com acompanhamento de progresso e callbacks
+        empty: Nenhum lote encontrado
+        headers:
+          batch_id: ID do lote
+          created: Criado
+          description: Descrição
+          jobs: Trabalhos
+          progress: Progresso
+          status: Status
+        title: Lotes
+      show:
+        back: Voltar para lotes
+        batch_id: ID do lote
+        completed: Concluídos
+        created_at: Criado em
+        details: Detalhes
+        discarded: Descartados
+        finished_at: Finalizado em
+        not_found: Lote não encontrado
+        on_discard: Ao descartar
+        on_finish: Ao finalizar
+        on_success: Ao ter sucesso
+        progress: Progresso
+        properties: Propriedades
+        remaining: Restantes
+        title: Lote
+        total_jobs: Total de trabalhos
     dashboard:
       processes_table:
         empty: Nenhum processo em execução
@@ -189,6 +218,10 @@ pt:
         not_found: Evento não encontrado
         title: Evento %{event_id}
     helpers:
+      batch_status:
+        finished: Finalizado
+        pending: Pendente
+        processing: Processando
       bulk_select_all: Selecionar todos
       bulk_select_row: Selecionar %{id}
       bulk_selected: selecionado
@@ -346,6 +379,7 @@ pt:
     layout:
       brand: Pgbus
       nav:
+        batches: Lotes
         dashboard: Painel
         dlq: DLQ
         events: Eventos

data/config/locales/sv.yml

@@ -1,6 +1,35 @@
 ---
 sv:
   pgbus:
+    batches:
+      index:
+        description: Jobbgrupper med framstegsspårning och callbacks
+        empty: Inga grupper hittades
+        headers:
+          batch_id: Grupp-ID
+          created: Skapad
+          description: Beskrivning
+          jobs: Jobb
+          progress: Framsteg
+          status: Status
+        title: Grupper
+      show:
+        back: Tillbaka till grupper
+        batch_id: Grupp-ID
+        completed: Slutförda
+        created_at: Skapad den
+        details: Detaljer
+        discarded: Kasserade
+        finished_at: Avslutad den
+        not_found: Grupp hittades inte
+        on_discard: Vid kassering
+        on_finish: Vid avslut
+        on_success: Vid framgång
+        progress: Framsteg
+        properties: Egenskaper
+        remaining: Återstående
+        title: Grupp
+        total_jobs: Totalt antal jobb
     dashboard:
       processes_table:
         empty: Inga processer körs
@@ -189,6 +218,10 @@ sv:
         not_found: Händelse hittades inte
         title: Händelse %{event_id}
     helpers:
+      batch_status:
+        finished: Slutförd
+        pending: Väntande
+        processing: Bearbetas
       bulk_select_all: Markera alla
       bulk_select_row: Markera %{id}
       bulk_selected: valda
@@ -346,6 +379,7 @@ sv:
     layout:
       brand: Pgbus
       nav:
+        batches: Grupper
         dashboard: Instrumentpanel
         dlq: DLQ
         events: Händelser

data/config/routes.rb

@@ -45,6 +45,7 @@ Pgbus::Engine.routes.draw do
     end
   end
 
+  resources :batches, only: %i[index show]
   resources :processes, only: [:index]
 
   resources :events, only: %i[index show] do

data/lib/pgbus/client.rb

@@ -351,6 +351,87 @@ module Pgbus
       total
     end
 
+    # --- Grouped reads (PGMQ v1.11.0+) ---
+
+    def read_grouped(queue_name, qty:, vt: nil)
+      full_name = config.queue_name(queue_name)
+      Instrumentation.instrument("pgbus.client.read_grouped", queue: full_name, qty: qty) do
+        with_stale_connection_retry do
+          synchronized { @pgmq.read_grouped(full_name, vt: vt || config.visibility_timeout, qty: qty) }
+        end
+      end
+    end
+
+    def read_grouped_rr(queue_name, qty:, vt: nil)
+      full_name = config.queue_name(queue_name)
+      Instrumentation.instrument("pgbus.client.read_grouped_rr", queue: full_name, qty: qty) do
+        with_stale_connection_retry do
+          synchronized { @pgmq.read_grouped_rr(full_name, vt: vt || config.visibility_timeout, qty: qty) }
+        end
+      end
+    end
+
+    def read_grouped_head(queue_name, qty:, vt: nil)
+      full_name = config.queue_name(queue_name)
+      with_stale_connection_retry do
+        synchronized { @pgmq.read_grouped_head(full_name, vt: vt || config.visibility_timeout, qty: qty) }
+      end
+    end
+
+    # --- FIFO index management (PGMQ v1.11.0+) ---
+
+    def create_fifo_index(queue_name)
+      full_name = config.queue_name(queue_name)
+      with_stale_connection_retry do
+        synchronized { @pgmq.create_fifo_index(full_name) }
+      end
+    end
+
+    def create_fifo_indexes_all
+      with_stale_connection_retry do
+        synchronized { @pgmq.create_fifo_indexes_all }
+      end
+    end
+
+    # --- LISTEN/NOTIFY management (PGMQ v1.11.0+) ---
+
+    def wait_for_notify(queue_name, timeout: nil, &block)
+      full_name = config.queue_name(queue_name)
+      with_stale_connection_retry do
+        synchronized { @pgmq.wait_for_notify(full_name, timeout: timeout, &block) }
+      end
+    end
+
+    def update_notify_insert(queue_name, throttle_interval_ms:)
+      full_name = config.queue_name(queue_name)
+      with_stale_connection_retry do
+        synchronized { @pgmq.update_notify_insert(full_name, throttle_interval_ms: throttle_interval_ms) }
+      end
+    end
+
+    def list_notify_insert_throttles
+      with_stale_connection_retry do
+        synchronized { @pgmq.list_notify_insert_throttles }
+      end
+    end
+
+    # --- Archive partitioning (requires pg_partman extension) ---
+
+    def convert_archive_partitioned(queue_name, partition_interval: "10000", retention_interval: "100000",
+                                    leading_partition: 10)
+      full_name = config.queue_name(queue_name)
+      with_stale_connection_retry do
+        synchronized do
+          @pgmq.convert_archive_partitioned(
+            full_name,
+            partition_interval: partition_interval,
+            retention_interval: retention_interval,
+            leading_partition: leading_partition
+          )
+        end
+      end
+    end
+
     # Topic routing
     def bind_topic(pattern, queue_name)
       full_name = config.queue_name(queue_name)
@@ -503,6 +584,7 @@ module Pgbus
           @pgmq.create(full_name)
           tune_autovacuum(full_name)
           enable_notify_if_needed(full_name, NOTIFY_THROTTLE_MS)
+          create_fifo_index_if_needed(full_name)
         end
         true
       end
@@ -515,6 +597,12 @@ module Pgbus
       @pgmq.enable_notify_insert(full_name, throttle_interval_ms: throttle_ms)
     end
 
+    def create_fifo_index_if_needed(full_name)
+      return unless config.group_mode
+
+      @pgmq.create_fifo_index(full_name)
+    end
+
     # Check whether the NOTIFY trigger already exists on this queue with the
     # expected throttle interval. When it does, we can skip the destructive
     # DROP TRIGGER + CREATE TRIGGER cycle that causes deadlocks when multiple
@@ -544,11 +632,21 @@ module Pgbus
       false
     end
 
+    # Apply PGMQ-tuned autovacuum + storage parameters to a queue's tables.
+    #
+    # Delegates to pgmq-ruby's tune_autovacuum (v0.7+), which sets the same
+    # queue/archive parameters pgbus used to apply by hand — vacuum scale
+    # factor 0.01/0.05, cost_delay 2/5, analyze scale factor 0.05, and
+    # fillfactor 70 on the queue table — plus a vacuum_threshold floor of 50.
+    # It quotes/lowercases the table name and runs both ALTER TABLEs in one
+    # pooled checkout. Tuning is best-effort: a failure here never blocks a
+    # queue from being usable, so we log and move on.
+    #
+    # Pgbus::AutovacuumTuning is still the source for the migration generators
+    # (sql_for_all_queues, sql_for_high_churn_tables) which tune pgbus-owned
+    # metadata tables the gem doesn't know about.
     def tune_autovacuum(queue_name)
-      with_raw_connection do |conn|
-        conn.exec(AutovacuumTuning.sql_for_queue(queue_name))
-        conn.exec(TableMaintenance.fillfactor_sql_for_queue(queue_name))
-      end
+      @pgmq.tune_autovacuum(queue_name)
     rescue StandardError => e
       Pgbus.logger.debug { "[Pgbus::Client] Autovacuum tuning failed for #{queue_name}: #{e.message}" }
     end

data/lib/pgbus/configuration.rb

@@ -44,6 +44,12 @@ module Pgbus
     # Priority queues
     attr_accessor :priority_levels, :default_priority
 
+    # Grouped reads (PGMQ v1.11.0+ FIFO grouping).
+    # nil = disabled (default read_batch behavior).
+    # :fifo = use read_grouped (drains oldest group first, throughput-optimized).
+    # :round_robin = use read_grouped_rr (fair round-robin across groups).
+    attr_reader :group_mode
+
     # Archive compaction. Only the user-facing retention window is configurable;
     # the loop interval and batch size are tuned via constants on
     # Pgbus::Process::Dispatcher.
@@ -146,6 +152,7 @@ module Pgbus
 
       @priority_levels = nil
       @default_priority = 1
+      @group_mode = nil
 
       @archive_retention = 7 * 24 * 3600 # 7 days
 
@@ -300,6 +307,24 @@ module Pgbus
       @streams_default_broadcast_mode = mode
     end
 
+    VALID_GROUP_MODES = [nil, :fifo, :round_robin].freeze
+
+    def group_mode=(mode)
+      coerced = case mode
+                when nil then nil
+                when Symbol then mode
+                when String then mode.to_sym
+                else
+                  raise ArgumentError,
+                        "Invalid group_mode type: #{mode.class}. Must be nil, String, or Symbol"
+                end
+      unless VALID_GROUP_MODES.include?(coerced)
+        raise ArgumentError, "Invalid group_mode: #{coerced.inspect}. Must be nil, :fifo, or :round_robin"
+      end
+
+      @group_mode = coerced
+    end
+
     VALID_PGMQ_SCHEMA_MODES = %i[auto extension embedded].freeze
 
     def pgmq_schema_mode=(mode)

data/lib/pgbus/process/supervisor.rb

@@ -69,6 +69,7 @@ module Pgbus
         single_active = worker_config[:single_active_consumer] || worker_config["single_active_consumer"] || false
         priority = worker_config[:consumer_priority] || worker_config["consumer_priority"] || 0
         exec_mode = config.execution_mode_for(worker_config)
+        grp_mode = worker_config[:group_mode] || worker_config["group_mode"] || config.group_mode
 
         pid = fork do
           restore_signals
@@ -78,7 +79,7 @@ module Pgbus
           worker = Worker.new(
             queues: queues, threads: threads, config: config,
             single_active_consumer: single_active, consumer_priority: priority,
-            execution_mode: exec_mode
+            execution_mode: exec_mode, group_mode: grp_mode
           )
           worker.run
         end

data/lib/pgbus/process/worker.rb

@@ -11,12 +11,24 @@ module Pgbus
 
       def initialize(queues:, threads: 5, config: Pgbus.configuration,
                      single_active_consumer: false, consumer_priority: 0,
-                     execution_mode: :threads)
+                     execution_mode: :threads, group_mode: nil)
         @queues = Array(queues)
         @wildcard = @queues.include?("*")
         @threads = threads
         @config = config
         @execution_mode = ExecutionPools.normalize_mode(execution_mode)
+        @group_mode = case group_mode
+                      when nil then nil
+                      when Symbol then group_mode
+                      when String then group_mode.to_sym
+                      else
+                        raise ArgumentError,
+                              "Invalid group_mode type: #{group_mode.class}. Must be nil, String, or Symbol"
+                      end
+        unless Pgbus::Configuration::VALID_GROUP_MODES.include?(@group_mode)
+          raise ArgumentError,
+                "Invalid group_mode: #{@group_mode.inspect}. Must be nil, :fifo, or :round_robin"
+        end
         @single_active_consumer = single_active_consumer
         @consumer_priority = consumer_priority
         @lifecycle = Lifecycle.new
@@ -141,6 +153,8 @@ module Pgbus
 
         if priority_enabled?
           fetch_prioritized(active_queues, qty)
+        elsif @group_mode
+          fetch_grouped(active_queues, qty)
         elsif active_queues.size == 1
           queue = active_queues.first
           messages = Pgbus.client.read_batch(queue, qty: qty) || []
@@ -210,6 +224,29 @@ module Pgbus
         end
       end
 
+      # Use grouped reads for fair or throughput-optimized multi-tenant processing.
+      # Each queue is read independently with the configured group strategy.
+      def fetch_grouped(active_queues, qty)
+        remaining = qty
+        results = []
+
+        active_queues.each do |queue|
+          break if remaining <= 0
+
+          messages = case @group_mode
+                     when :round_robin
+                       Pgbus.client.read_grouped_rr(queue, qty: remaining) || []
+                     else # :fifo
+                       Pgbus.client.read_grouped(queue, qty: remaining) || []
+                     end
+
+          messages.each { |m| results << [queue, m] }
+          remaining -= messages.size
+        end
+
+        results
+      end
+
       def priority_enabled?
         config.priority_levels && config.priority_levels > 1
       end

data/lib/pgbus/serializer.rb

@@ -58,7 +58,7 @@ module Pgbus
       raise ArgumentError, "Invalid GlobalID: #{gid_string.inspect}" unless gid
 
       allowed = Pgbus.configuration.allowed_global_id_models
-      if allowed&.empty?
+      if allowed && allowed.empty?
         raise ArgumentError,
               "GlobalID deserialization is disabled (allowed_global_id_models is empty). " \
               "Set to nil to allow all models, or add permitted classes."

data/lib/pgbus/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Pgbus
-  VERSION = "0.8.4"
+  VERSION = "0.9.0"
 end

data/lib/pgbus/web/data_source.rb

@@ -590,6 +590,43 @@ module Pgbus
         []
       end
 
+      # Batches
+      def batches(limit: 100)
+        BatchEntry.order(created_at: :desc).limit(limit).map { |r| format_batch(r) }
+      rescue StandardError => e
+        Pgbus.logger.debug { "[Pgbus::Web] Error fetching batches: #{e.message}" }
+        []
+      end
+
+      def batch_detail(batch_id)
+        record = BatchEntry.find_by(batch_id: batch_id)
+        return nil unless record
+
+        format_batch(record).merge(
+          properties: record.properties,
+          on_finish_class: record.on_finish_class,
+          on_success_class: record.on_success_class,
+          on_discard_class: record.on_discard_class
+        )
+      rescue StandardError => e
+        Pgbus.logger.debug { "[Pgbus::Web] Error fetching batch #{batch_id}: #{e.message}" }
+        nil
+      end
+
+      def batches_count
+        BatchEntry.count
+      rescue StandardError => e
+        Pgbus.logger.debug { "[Pgbus::Web] Error counting batches: #{e.message}" }
+        0
+      end
+
+      def active_batches_count
+        BatchEntry.where.not(status: "finished").count
+      rescue StandardError => e
+        Pgbus.logger.debug { "[Pgbus::Web] Error counting active batches: #{e.message}" }
+        0
+      end
+
       # Job stats
       def job_stats_summary(minutes: 60)
         JobStat.summary(minutes: minutes)
@@ -874,6 +911,33 @@ module Pgbus
         []
       end
 
+      # NOTIFY throttle status for all queues with notifications enabled.
+      # Returns an array of hashes: { queue_name:, throttle_interval_ms:, last_notified_at: }
+      def notify_throttles
+        @client.list_notify_insert_throttles.map do |throttle|
+          {
+            queue_name: throttle.queue_name,
+            throttle_interval_ms: throttle.throttle_interval_ms,
+            last_notified_at: throttle.last_notified_at
+          }
+        end
+      rescue StandardError => e
+        Pgbus.logger.debug { "[Pgbus::Web] Error fetching notify throttles: #{e.message}" }
+        []
+      end
+
+      # FIFO group head sampling for a specific queue.
+      # Returns the oldest visible message from each distinct group (up to qty).
+      # Useful for detecting head-of-line stalls in multi-tenant queues.
+      def queue_group_heads(queue_name, qty: 20)
+        logical = logical_queue_name(queue_name)
+        messages = @client.read_grouped_head(logical, qty: qty) || []
+        messages.map { |m| format_pgmq_message(m, queue_name) }
+      rescue StandardError => e
+        Pgbus.logger.debug { "[Pgbus::Web] Error fetching group heads for #{queue_name}: #{e.message}" }
+        []
+      end
+
       private
 
       def connection
@@ -1114,6 +1178,19 @@ module Pgbus
         }
       end
 
+      def format_pgmq_message(msg, queue_name)
+        {
+          msg_id: msg.msg_id.to_i,
+          read_ct: msg.read_ct.to_i,
+          enqueued_at: msg.enqueued_at,
+          last_read_at: msg.respond_to?(:last_read_at) ? msg.last_read_at : nil,
+          vt: msg.respond_to?(:vt) ? msg.vt : nil,
+          message: msg.message,
+          headers: msg.headers,
+          queue_name: queue_name
+        }
+      end
+
       def format_process(row)
         heartbeat = row["last_heartbeat_at"]
         heartbeat_time = heartbeat.is_a?(String) ? Time.parse(heartbeat) : heartbeat
@@ -1131,6 +1208,25 @@ module Pgbus
         }
       end
 
+      def format_batch(record)
+        total = record.total_jobs
+        done = record.completed_jobs + record.discarded_jobs
+        pct = total.positive? ? ((done * 100) / total) : 100
+
+        {
+          batch_id: record.batch_id,
+          description: record.description,
+          status: record.status,
+          total_jobs: total,
+          completed_jobs: record.completed_jobs,
+          discarded_jobs: record.discarded_jobs,
+          failed_jobs: record.failed_jobs,
+          progress_pct: pct,
+          created_at: record.created_at,
+          finished_at: record.finished_at
+        }
+      end
+
       def sanitize_name(name)
         QueueNameValidator.sanitize!(name)
       end

data/lib/tasks/pgbus_queues.rake

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+namespace :pgbus do
+  namespace :queues do
+    desc "Create FIFO indexes on all pgbus queues (required for grouped reads)"
+    task fifo_indexes: :environment do
+      puts "Creating FIFO indexes on all pgbus queues..."
+      Pgbus.client.create_fifo_indexes_all
+      puts "Done. FIFO indexes created on all queues."
+    end
+
+    desc "Create FIFO index on a specific queue (QUEUE=name)"
+    task fifo_index: :environment do
+      queue = ENV.fetch("QUEUE") do
+        abort "Usage: rake pgbus:queues:fifo_index QUEUE=<queue_name>"
+      end
+      puts "Creating FIFO index on queue '#{queue}'..."
+      Pgbus.client.create_fifo_index(queue)
+      puts "Done."
+    end
+  end
+
+  namespace :archives do
+    desc "Convert a queue's archive table to pg_partman-managed partitions (QUEUE=name)"
+    task partition: :environment do
+      queue = ENV.fetch("QUEUE") do
+        abort "Usage: rake pgbus:archives:partition QUEUE=<queue_name> " \
+              "[INTERVAL=10000] [RETENTION=100000] [LEADING_PARTITION=10]"
+      end
+      interval = ENV.fetch("INTERVAL", "10000")
+      retention = ENV.fetch("RETENTION", "100000")
+      leading_raw = ENV.fetch("LEADING_PARTITION", "10")
+      leading = begin
+        Integer(leading_raw, 10)
+      rescue ArgumentError, TypeError
+        abort "LEADING_PARTITION must be a positive integer, got #{leading_raw.inspect}"
+      end
+      abort "LEADING_PARTITION must be a positive integer" if leading <= 0
+
+      puts "Converting archive table for queue '#{queue}' to partitioned..."
+      puts "  Partition interval: #{interval}"
+      puts "  Retention interval: #{retention}"
+      puts "  Leading partitions: #{leading}"
+
+      Pgbus.client.convert_archive_partitioned(
+        queue,
+        partition_interval: interval,
+        retention_interval: retention,
+        leading_partition: leading
+      )
+      puts "Done. Archive table is now partitioned."
+    end
+  end
+end