pgbus 0.1.9 → 0.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b1e18f777c6b6f2dbc0f927994e19587a489f0551be9f5e228c4d47b3f346fa0
-  data.tar.gz: 433faca464a6f881a21cbcf950c3238cdd34e1e11fac0108a320dd98e23dfae1
+  metadata.gz: 35bd03aa1d30dde2aea609829f027d9a1355282fb049e22d4f3a20119039828a
+  data.tar.gz: 341150d6777d9747f19b8446692ffd262403eee0b2e687461176cce820af8177
 SHA512:
-  metadata.gz: 89418720dfb7433c1716cff8e238e74fee2be4c04a8023a4c2dc3b67fdf393b9ddb9f7c4df70b435a6cdcc9a3668c1c24e78bedce83c0c22d93d196ab1ee1574
-  data.tar.gz: b10d5f71b80c97e42573b2b034c0c1b4444342d6e729fef0fd0e8a97dba269eb35cdca8fc4bfcb4446dd17e630683859c1b5d0043467218baf15b72b05fb3e38
+  metadata.gz: dafeed2ede684fedb7a95b76ac3504c0987624ba49bdd5b3c9ea18c67a51f241ce626069b3d62f0dff617d9582da83490e8b527e600f396a334785d56010c405
+  data.tar.gz: 55a9207ec4e8bbe56ce2e8d7322f0bf242daf6b30f89ac87f4cd37dc4dad2565704503076e24b994a3c52aec31e822abfbdafe72ee11ef239227a703d823f875

data/lib/pgbus/active_job/executor.rb

@@ -156,7 +156,7 @@ module Pgbus
 
       def archive_from(queue_name, msg_id, source_queue: nil)
         if source_queue
-          client.archive_from_queue(source_queue, msg_id)
+          client.archive_message(source_queue, msg_id, prefixed: false)
         else
           client.archive_message(queue_name, msg_id)
         end

data/lib/pgbus/client.rb

@@ -19,27 +19,34 @@ module Pgbus
         require "pgmq"
       end
       @config = config
-      # Force pool_size=1. PG::Connection (libpq) is not thread-safe.
-      # When using the Rails lambda path (-> { AR::Base.connection.raw_connection }),
-      # the pool would return the same underlying PG::Connection that ActiveRecord
-      # also uses, causing concurrent access corruption (segfaults, result.ntuples
-      # NoMethodError). A single-connection pool combined with @pgmq_mutex ensures
-      # all PGMQ operations are serialized.
-      @pgmq = PGMQ::Client.new(
-        config.connection_options,
-        pool_size: 1,
-        pool_timeout: config.pool_timeout
-      )
-      @pgmq_mutex = Mutex.new
+      conn_opts = config.connection_options
+      @shared_connection = conn_opts.is_a?(Proc)
+
+      if @shared_connection
+        # When using the Rails lambda path (-> { AR::Base.connection.raw_connection }),
+        # the Proc returns the same underlying PG::Connection that ActiveRecord uses.
+        # PG::Connection (libpq) is not thread-safe — concurrent access causes
+        # segfaults and result corruption. Force pool_size=1 and serialize all
+        # operations through a mutex.
+        @pgmq = PGMQ::Client.new(conn_opts, pool_size: 1, pool_timeout: config.pool_timeout)
+        @pgmq_mutex = Mutex.new
+      else
+        # With a String URL or Hash params, pgmq-ruby creates its own dedicated
+        # PG::Connection per pool slot — no shared state with ActiveRecord.
+        # Use the configured pool_size and let pgmq-ruby's connection_pool handle
+        # concurrency internally (no mutex needed).
+        @pgmq = PGMQ::Client.new(conn_opts, pool_size: config.pool_size, pool_timeout: config.pool_timeout)
+        @pgmq_mutex = nil
+      end
+
       @queues_created = Concurrent::Map.new
       @queue_strategy = QueueFactory.for(config)
+      @schema_ensured = false
     end
 
     def ensure_queue(name)
+      ensure_pgmq_schema
       @queue_strategy.physical_queue_names(name).each { |pq| ensure_single_queue(pq) }
-    rescue PGMQ::Errors::ConnectionError => e
-      raise Pgbus::SchemaNotReady,
-            "PGMQ schema is not available (#{e.message}). Run `rails db:migrate` for the pgbus database."
     end
 
     def ensure_all_queues
@@ -70,7 +77,7 @@ module Pgbus
       full_name = config.queue_name(queue_name)
       ensure_queue(queue_name)
       serialized = payloads.map { |p| serialize(p) }
-      serialized_headers = headers&.map { |h| serialize(h) }
+      serialized_headers = headers&.map { |h| h.nil? ? nil : serialize(h) }
       Instrumentation.instrument("pgbus.client.send_batch", queue: full_name, size: payloads.size) do
         synchronized { @pgmq.produce_batch(full_name, serialized, headers: serialized_headers, delay: delay) }
       end
@@ -129,31 +136,47 @@ module Pgbus
       end
     end
 
-    def delete_message(queue_name, msg_id)
-      full_name = config.queue_name(queue_name)
-      synchronized { @pgmq.delete(full_name, msg_id) }
+    # Read from multiple queues in a single SQL query (UNION ALL).
+    # Each returned message includes a queue_name field identifying its source.
+    # queue_names should be logical names (prefix is added automatically).
+    def read_multi(queue_names, qty:, vt: nil)
+      full_names = queue_names.map { |q| config.queue_name(q) }
+      Instrumentation.instrument("pgbus.client.read_multi", queues: full_names, qty: qty) do
+        synchronized { @pgmq.read_multi(full_names, vt: vt || config.visibility_timeout, qty: qty) }
+      end
     end
 
-    def archive_message(queue_name, msg_id)
-      full_name = config.queue_name(queue_name)
-      synchronized { @pgmq.archive(full_name, msg_id) }
+    # Delete a message. Pass prefixed: false when queue_name is already
+    # the full PGMQ queue name (e.g. from priority sub-queues or dashboard).
+    def delete_message(queue_name, msg_id, prefixed: true)
+      name = prefixed ? config.queue_name(queue_name) : queue_name
+      synchronized { @pgmq.delete(name, msg_id) }
     end
 
-    def archive_from_queue(full_queue_name, msg_id)
-      synchronized { @pgmq.archive(full_queue_name, msg_id) }
+    # Archive a message. Pass prefixed: false when queue_name is already
+    # the full PGMQ queue name.
+    def archive_message(queue_name, msg_id, prefixed: true)
+      name = prefixed ? config.queue_name(queue_name) : queue_name
+      synchronized { @pgmq.archive(name, msg_id) }
     end
 
-    def extend_visibility(queue_name, msg_id, vt:)
-      full_name = config.queue_name(queue_name)
-      synchronized { @pgmq.set_vt(full_name, msg_id, vt: vt) }
+    # Batch archive — moves multiple messages to the archive table in one call.
+    def archive_batch(queue_name, msg_ids, prefixed: true)
+      name = prefixed ? config.queue_name(queue_name) : queue_name
+      synchronized { @pgmq.archive_batch(name, msg_ids) }
     end
 
-    def set_visibility_timeout(queue_name, msg_id, vt:)
-      synchronized { @pgmq.set_vt(queue_name, msg_id, vt: vt) }
+    # Batch delete — permanently removes multiple messages in one call.
+    def delete_batch(queue_name, msg_ids, prefixed: true)
+      name = prefixed ? config.queue_name(queue_name) : queue_name
+      synchronized { @pgmq.delete_batch(name, msg_ids) }
     end
 
-    def delete_from_queue(queue_name, msg_id)
-      synchronized { @pgmq.delete(queue_name, msg_id) }
+    # Set visibility timeout. Pass prefixed: false when queue_name is already
+    # the full PGMQ queue name.
+    def set_visibility_timeout(queue_name, msg_id, vt:, prefixed: true)
+      name = prefixed ? config.queue_name(queue_name) : queue_name
+      synchronized { @pgmq.set_vt(name, msg_id, vt: vt) }
     end
 
     def transaction(&block)
@@ -201,7 +224,9 @@ module Pgbus
 
       loop do
         deleted = synchronized do
-          @pgmq.pool.with { |conn| conn.exec_params(sql, [older_than, batch_size]).cmd_tuples }
+          with_raw_connection do |conn|
+            conn.exec_params(sql, [older_than, batch_size]).cmd_tuples
+          end
         end
         total += deleted
         break if deleted < batch_size
@@ -254,6 +279,66 @@ module Pgbus
       queues.to_a
     end
 
+    def ensure_pgmq_schema
+      return if @schema_ensured
+
+      synchronized do
+        return if @schema_ensured
+
+        with_raw_connection do |raw_conn|
+          exists = raw_conn.exec("SELECT 1 FROM pg_tables WHERE schemaname = 'pgmq' AND tablename = 'meta' LIMIT 1")
+          install_pgmq_schema(raw_conn) if exists.ntuples.zero?
+        end
+        @schema_ensured = true
+      end
+    rescue StandardError => e
+      raise Pgbus::SchemaNotReady,
+            "PGMQ schema installation failed (#{e.class}: #{e.message}). " \
+            "Ensure the pgbus database exists and migrations have been run."
+    end
+
+    def install_pgmq_schema(conn)
+      mode = config.pgmq_schema_mode
+
+      case mode
+      when :extension
+        Pgbus.logger.info { "[Pgbus] PGMQ schema not found — installing via extension" }
+        conn.exec("CREATE EXTENSION IF NOT EXISTS pgmq")
+      when :embedded
+        Pgbus.logger.info { "[Pgbus] PGMQ schema not found — installing embedded SQL" }
+        conn.exec(PgmqSchema.install_sql)
+      else # :auto
+        ext = conn.exec("SELECT 1 FROM pg_available_extensions WHERE name = 'pgmq' LIMIT 1")
+        if ext.ntuples.positive?
+          Pgbus.logger.info { "[Pgbus] PGMQ schema not found — installing via extension" }
+          conn.exec("CREATE EXTENSION IF NOT EXISTS pgmq")
+        else
+          Pgbus.logger.info { "[Pgbus] PGMQ schema not found — installing embedded SQL" }
+          conn.exec(PgmqSchema.install_sql)
+        end
+      end
+    end
+
+    def with_raw_connection
+      opts = config.connection_options
+      owned = false
+      conn = case opts
+             when Proc
+               opts.call
+             when String
+               owned = true
+               PG.connect(opts)
+             when Hash
+               owned = true
+               PG.connect(**opts)
+             else
+               raise ConfigurationError, "Cannot resolve raw PG connection from #{opts.class}"
+             end
+      yield conn
+    ensure
+      conn&.close if owned
+    end
+
     def ensure_single_queue(full_name)
       return if @queues_created[full_name]
 
@@ -266,11 +351,15 @@ module Pgbus
       end
     end
 
-    # Serialize all PGMQ operations through a single mutex.
-    # PG::Connection is not thread-safe — concurrent access from worker
-    # threads causes segfaults and result corruption.
+    # Serialize PGMQ operations through a mutex when sharing a connection
+    # with ActiveRecord (Proc path). When pgmq-ruby owns its own connections
+    # (String/Hash path), the internal connection_pool handles concurrency.
     def synchronized(&)
-      @pgmq_mutex.synchronize(&)
+      if @pgmq_mutex
+        @pgmq_mutex.synchronize(&)
+      else
+        yield
+      end
     end
 
     def serialize(data)

data/lib/pgbus/event_bus/publisher.rb

@@ -7,13 +7,7 @@ module Pgbus
 
       def publish(routing_key, payload, headers: nil, delay: 0)
         event_data = build_event_data(payload)
-
-        if Pgbus.client.pgmq.respond_to?(:produce_topic)
-          Pgbus.client.publish_to_topic(routing_key, event_data, headers: headers, delay: delay)
-        else
-          # Fallback: send directly to queues matching the routing key
-          Pgbus.client.send_message(routing_key, event_data, headers: headers, delay: delay)
-        end
+        Pgbus.client.publish_to_topic(routing_key, event_data, headers: headers, delay: delay)
       end
 
       def publish_later(routing_key, payload, delay:, headers: nil)

data/lib/pgbus/outbox/poller.rb

@@ -54,10 +54,7 @@ module Pgbus
                                  .to_a
             break if entries.empty?
 
-            entries.each do |entry|
-              succeeded += 1 if publish_entry(entry)
-            end
-
+            succeeded = publish_entries(entries)
             published += succeeded
             break if succeeded.zero? || entries.size < config.outbox_batch_size
           end
@@ -74,24 +71,65 @@ module Pgbus
 
       private
 
-      def publish_entry(entry)
-        if entry.routing_key.present?
-          Pgbus.client.publish_to_topic(
-            entry.routing_key,
-            entry.payload,
-            headers: entry.headers,
-            delay: entry.delay || 0
-          )
-        else
-          Pgbus.client.send_message(
-            entry.queue_name,
-            entry.payload,
-            headers: entry.headers,
-            delay: entry.delay || 0,
-            priority: entry.priority
-          )
+      def publish_entries(entries)
+        # Partition: topic-routed entries must be published individually
+        # (different routing keys), direct-queue entries can be batched.
+        topic_entries, queue_entries = entries.partition { |e| e.routing_key.present? }
+
+        succeeded = 0
+        topic_entries.each { |e| succeeded += 1 if publish_single(e) }
+        succeeded += publish_queue_batch(queue_entries)
+        succeeded
+      end
+
+      def publish_single(entry)
+        Pgbus.client.publish_to_topic(
+          entry.routing_key,
+          entry.payload,
+          headers: entry.headers,
+          delay: entry.delay || 0
+        )
+        entry.update!(published_at: Time.current)
+        true
+      rescue StandardError => e
+        Pgbus.logger.error { "[Pgbus] Failed to publish outbox entry #{entry.id}: #{e.message}" }
+        false
+      end
+
+      # Group direct-queue entries by (queue_name, priority, delay) and
+      # use send_batch for each group to reduce round-trips.
+      def publish_queue_batch(entries)
+        return 0 if entries.empty?
+
+        succeeded = 0
+        entries.group_by { |e| [e.queue_name, e.priority, e.delay || 0] }.each do |(queue, _priority, delay), group|
+          payloads = group.map(&:payload)
+          headers = group.map(&:headers)
+          headers = nil if headers.all?(&:blank?)
+
+          Pgbus.client.send_batch(queue, payloads, headers: headers, delay: delay)
+          now = Time.current
+          group.each { |e| e.update!(published_at: now) }
+          succeeded += group.size
+        rescue StandardError => e
+          Pgbus.logger.error { "[Pgbus] Failed to batch-publish #{group.size} outbox entries: #{e.message}" }
+          # Fall back to individual publishing for this group
+          group.each { |entry| succeeded += 1 if publish_single_queue(entry) }
         end
+        succeeded
+      end
 
+      # Fallback for individual publishing when a batch fails.
+      # Intentionally omits priority to match send_batch routing behavior
+      # (base queue only), ensuring consistent queue placement regardless
+      # of whether the batch or fallback path is used.
+      def publish_single_queue(entry)
+        Pgbus.client.send_message(
+          entry.queue_name,
+          entry.payload,
+          headers: entry.headers,
+          delay: entry.delay || 0
+        )
         entry.update!(published_at: Time.current)
         true
       rescue StandardError => e

data/lib/pgbus/process/consumer.rb

@@ -56,9 +56,12 @@ module Pgbus
         idle = @pool.max_length - @pool.queue_length
         return interruptible_sleep(config.polling_interval) if idle <= 0
 
-        tagged_messages = @queue_names.flat_map do |queue_name|
-          (Pgbus.client.read_batch(queue_name, qty: idle) || []).map { |m| [queue_name, m] }
-        end.first(idle)
+        tagged_messages = if @queue_names.size == 1
+                            queue = @queue_names.first
+                            (Pgbus.client.read_batch(queue, qty: idle) || []).map { |m| [queue, m] }
+                          else
+                            fetch_multi_consumer(idle)
+                          end
 
         if tagged_messages.empty?
           interruptible_sleep(config.polling_interval)
@@ -94,6 +97,16 @@ module Pgbus
         # the next read will route to DLQ above.
       end
 
+      def fetch_multi_consumer(qty)
+        messages = Pgbus.client.read_multi(@queue_names, qty: qty) || []
+        prefix = "#{config.queue_prefix}_"
+
+        messages.map do |m|
+          logical = m.queue_name&.delete_prefix(prefix) || @queue_names.first
+          [logical, m]
+        end
+      end
+
       def pattern_overlaps?(topic_filter, subscription_pattern)
         # Simple check: if either is a subset of the other
         topic_filter == subscription_pattern ||

data/lib/pgbus/process/worker.rb

@@ -107,7 +107,7 @@ module Pgbus
       end
 
       # Returns an array of [queue_name, message] pairs so we always know
-      # which queue each message came from (PGMQ messages don't carry this).
+      # which queue each message came from.
       def fetch_messages(qty)
         active_queues = queues.reject { |q| @circuit_breaker.paused?(q) }
         active_queues = active_queues.select { |q| @queue_lock.try_lock(q) } if @single_active_consumer
@@ -120,10 +120,7 @@ module Pgbus
           messages = Pgbus.client.read_batch(queue, qty: qty) || []
           messages.map { |m| [queue, m] }
         else
-          per_queue = [(qty / active_queues.size.to_f).ceil, 1].max
-          active_queues.flat_map do |q|
-            (Pgbus.client.read_batch(q, qty: per_queue) || []).map { |m| [q, m] }
-          end.first(qty)
+          fetch_multi(active_queues, qty)
         end
       rescue StandardError => e
         Pgbus.logger.error { "[Pgbus] Error fetching messages: #{e.message}" }
@@ -147,6 +144,19 @@ module Pgbus
         results
       end
 
+      # Use pgmq-ruby's read_multi to read from all queues in a single
+      # SQL query (UNION ALL). Each returned message carries a queue_name
+      # field so we can map it back to the logical queue.
+      def fetch_multi(active_queues, qty)
+        messages = Pgbus.client.read_multi(active_queues, qty: qty) || []
+        prefix = "#{config.queue_prefix}_"
+
+        messages.map do |m|
+          logical = m.queue_name&.delete_prefix(prefix) || active_queues.first
+          [logical, m]
+        end
+      end
+
       def priority_enabled?
         config.priority_levels && config.priority_levels > 1
       end

data/lib/pgbus/recurring/scheduler.rb

@@ -17,6 +17,7 @@ module Pgbus
       def run
         setup_signals
         start_heartbeat
+        sync_recurring_tasks
 
         Pgbus.logger.info do
           "[Pgbus] Scheduler started: #{schedule.tasks.size} recurring tasks, " \
@@ -84,6 +85,16 @@ module Pgbus
 
       private
 
+      def sync_recurring_tasks
+        tasks = config.recurring_tasks
+        return if tasks.nil?
+
+        RecurringTask.sync_from_config!(tasks)
+        Pgbus.logger.info { "[Pgbus] Synced #{tasks.size} recurring task(s) from configuration" }
+      rescue StandardError => e
+        Pgbus.logger.error { "[Pgbus] Failed to sync recurring tasks: #{e.class}: #{e.message}" }
+      end
+
       def start_heartbeat
         @heartbeat = Process::Heartbeat.new(
           kind: "scheduler",

data/lib/pgbus/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Pgbus
-  VERSION = "0.1.9"
+  VERSION = "0.2.1"
 end

data/lib/pgbus/web/data_source.rb

@@ -105,11 +105,11 @@ module Pgbus
       end
 
       def retry_job(queue_name, msg_id)
-        @client.set_visibility_timeout(queue_name, msg_id.to_i, vt: 0)
+        @client.set_visibility_timeout(queue_name, msg_id.to_i, vt: 0, prefixed: false)
       end
 
       def discard_job(queue_name, msg_id)
-        @client.archive_message(queue_name, msg_id.to_i)
+        @client.archive_message(queue_name, msg_id.to_i, prefixed: false)
       end
 
       # Failed events
@@ -269,7 +269,7 @@ module Pgbus
 
       def discard_dlq_message(queue_name, msg_id)
         # queue_name here is the full DLQ name (already prefixed)
-        @client.delete_from_queue(queue_name, msg_id.to_i)
+        @client.delete_message(queue_name, msg_id.to_i, prefixed: false)
         true
       rescue StandardError => e
         Pgbus.logger.debug { "[Pgbus::Web] Error discarding DLQ message #{msg_id}: #{e.message}" }
@@ -290,14 +290,16 @@ module Pgbus
 
       def discard_all_dlq
         messages = dlq_messages(page: 1, per_page: 1000)
-        count = 0
-        messages.each do |m|
-          discard_dlq_message(m[:queue_name], m[:msg_id]) && count += 1
+        return 0 if messages.empty?
+
+        # Group by queue for batch delete — one call per DLQ instead of N calls
+        messages.group_by { |m| m[:queue_name] }.sum do |queue_name, msgs|
+          ids = msgs.map { |m| m[:msg_id].to_i }
+          @client.delete_batch(queue_name, ids, prefixed: false).size
         rescue StandardError => e
-          Pgbus.logger.debug { "[Pgbus::Web] Error discarding DLQ message #{m[:msg_id]}: #{e.message}" }
-          next
+          Pgbus.logger.debug { "[Pgbus::Web] Error batch-discarding DLQ messages from #{queue_name}: #{e.message}" }
+          0
         end
-        count
       end
 
       # Processes

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: pgbus
 version: !ruby/object:Gem::Version
-  version: 0.1.9
+  version: 0.2.1
 platform: ruby
 authors:
 - Mikael Henriksson