pgbus 0.2.0 → 0.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e06c29b37491ad78b26d4799f11baea645e4380b2ebb6950bec74ad7be339f13
-  data.tar.gz: 712a67117a92021b45ab361476068c9885b03a8215ff02d8327e58f6b9a5abd2
+  metadata.gz: 35bd03aa1d30dde2aea609829f027d9a1355282fb049e22d4f3a20119039828a
+  data.tar.gz: 341150d6777d9747f19b8446692ffd262403eee0b2e687461176cce820af8177
 SHA512:
-  metadata.gz: f3e083cd40e0a1d02577c015283d83e3aeb8a1205bde8bb21e5e4c5e6df8b70f031f894fb1bfc93771caaeb4d26b5e1c7cca08cfa01dfad0d31a2b946db997f4
-  data.tar.gz: fe1d70f3a39bb849981a025b5d9fee6e2c447a0fc641ce771e606e42f2da8892a40e7cb68ecaa57aac47fe970c819cdd0c60f8c439d92fe1306662da309f1dfd
+  metadata.gz: dafeed2ede684fedb7a95b76ac3504c0987624ba49bdd5b3c9ea18c67a51f241ce626069b3d62f0dff617d9582da83490e8b527e600f396a334785d56010c405
+  data.tar.gz: 55a9207ec4e8bbe56ce2e8d7322f0bf242daf6b30f89ac87f4cd37dc4dad2565704503076e24b994a3c52aec31e822abfbdafe72ee11ef239227a703d823f875

data/lib/pgbus/active_job/executor.rb

@@ -156,7 +156,7 @@ module Pgbus
 
       def archive_from(queue_name, msg_id, source_queue: nil)
         if source_queue
-          client.archive_from_queue(source_queue, msg_id)
+          client.archive_message(source_queue, msg_id, prefixed: false)
         else
           client.archive_message(queue_name, msg_id)
         end

data/lib/pgbus/client.rb

@@ -19,18 +19,26 @@ module Pgbus
         require "pgmq"
       end
       @config = config
-      # Force pool_size=1. PG::Connection (libpq) is not thread-safe.
-      # When using the Rails lambda path (-> { AR::Base.connection.raw_connection }),
-      # the pool would return the same underlying PG::Connection that ActiveRecord
-      # also uses, causing concurrent access corruption (segfaults, result.ntuples
-      # NoMethodError). A single-connection pool combined with @pgmq_mutex ensures
-      # all PGMQ operations are serialized.
-      @pgmq = PGMQ::Client.new(
-        config.connection_options,
-        pool_size: 1,
-        pool_timeout: config.pool_timeout
-      )
-      @pgmq_mutex = Mutex.new
+      conn_opts = config.connection_options
+      @shared_connection = conn_opts.is_a?(Proc)
+
+      if @shared_connection
+        # When using the Rails lambda path (-> { AR::Base.connection.raw_connection }),
+        # the Proc returns the same underlying PG::Connection that ActiveRecord uses.
+        # PG::Connection (libpq) is not thread-safe — concurrent access causes
+        # segfaults and result corruption. Force pool_size=1 and serialize all
+        # operations through a mutex.
+        @pgmq = PGMQ::Client.new(conn_opts, pool_size: 1, pool_timeout: config.pool_timeout)
+        @pgmq_mutex = Mutex.new
+      else
+        # With a String URL or Hash params, pgmq-ruby creates its own dedicated
+        # PG::Connection per pool slot — no shared state with ActiveRecord.
+        # Use the configured pool_size and let pgmq-ruby's connection_pool handle
+        # concurrency internally (no mutex needed).
+        @pgmq = PGMQ::Client.new(conn_opts, pool_size: config.pool_size, pool_timeout: config.pool_timeout)
+        @pgmq_mutex = nil
+      end
+
       @queues_created = Concurrent::Map.new
       @queue_strategy = QueueFactory.for(config)
       @schema_ensured = false
@@ -69,7 +77,7 @@ module Pgbus
       full_name = config.queue_name(queue_name)
       ensure_queue(queue_name)
       serialized = payloads.map { |p| serialize(p) }
-      serialized_headers = headers&.map { |h| serialize(h) }
+      serialized_headers = headers&.map { |h| h.nil? ? nil : serialize(h) }
       Instrumentation.instrument("pgbus.client.send_batch", queue: full_name, size: payloads.size) do
         synchronized { @pgmq.produce_batch(full_name, serialized, headers: serialized_headers, delay: delay) }
       end
@@ -128,31 +136,47 @@ module Pgbus
       end
     end
 
-    def delete_message(queue_name, msg_id)
-      full_name = config.queue_name(queue_name)
-      synchronized { @pgmq.delete(full_name, msg_id) }
+    # Read from multiple queues in a single SQL query (UNION ALL).
+    # Each returned message includes a queue_name field identifying its source.
+    # queue_names should be logical names (prefix is added automatically).
+    def read_multi(queue_names, qty:, vt: nil)
+      full_names = queue_names.map { |q| config.queue_name(q) }
+      Instrumentation.instrument("pgbus.client.read_multi", queues: full_names, qty: qty) do
+        synchronized { @pgmq.read_multi(full_names, vt: vt || config.visibility_timeout, qty: qty) }
+      end
     end
 
-    def archive_message(queue_name, msg_id)
-      full_name = config.queue_name(queue_name)
-      synchronized { @pgmq.archive(full_name, msg_id) }
+    # Delete a message. Pass prefixed: false when queue_name is already
+    # the full PGMQ queue name (e.g. from priority sub-queues or dashboard).
+    def delete_message(queue_name, msg_id, prefixed: true)
+      name = prefixed ? config.queue_name(queue_name) : queue_name
+      synchronized { @pgmq.delete(name, msg_id) }
     end
 
-    def archive_from_queue(full_queue_name, msg_id)
-      synchronized { @pgmq.archive(full_queue_name, msg_id) }
+    # Archive a message. Pass prefixed: false when queue_name is already
+    # the full PGMQ queue name.
+    def archive_message(queue_name, msg_id, prefixed: true)
+      name = prefixed ? config.queue_name(queue_name) : queue_name
+      synchronized { @pgmq.archive(name, msg_id) }
     end
 
-    def extend_visibility(queue_name, msg_id, vt:)
-      full_name = config.queue_name(queue_name)
-      synchronized { @pgmq.set_vt(full_name, msg_id, vt: vt) }
+    # Batch archive — moves multiple messages to the archive table in one call.
+    def archive_batch(queue_name, msg_ids, prefixed: true)
+      name = prefixed ? config.queue_name(queue_name) : queue_name
+      synchronized { @pgmq.archive_batch(name, msg_ids) }
     end
 
-    def set_visibility_timeout(queue_name, msg_id, vt:)
-      synchronized { @pgmq.set_vt(queue_name, msg_id, vt: vt) }
+    # Batch delete — permanently removes multiple messages in one call.
+    def delete_batch(queue_name, msg_ids, prefixed: true)
+      name = prefixed ? config.queue_name(queue_name) : queue_name
+      synchronized { @pgmq.delete_batch(name, msg_ids) }
     end
 
-    def delete_from_queue(queue_name, msg_id)
-      synchronized { @pgmq.delete(queue_name, msg_id) }
+    # Set visibility timeout. Pass prefixed: false when queue_name is already
+    # the full PGMQ queue name.
+    def set_visibility_timeout(queue_name, msg_id, vt:, prefixed: true)
+      name = prefixed ? config.queue_name(queue_name) : queue_name
+      synchronized { @pgmq.set_vt(name, msg_id, vt: vt) }
     end
 
     def transaction(&block)
@@ -327,11 +351,15 @@ module Pgbus
       end
     end
 
-    # Serialize all PGMQ operations through a single mutex.
-    # PG::Connection is not thread-safe — concurrent access from worker
-    # threads causes segfaults and result corruption.
+    # Serialize PGMQ operations through a mutex when sharing a connection
+    # with ActiveRecord (Proc path). When pgmq-ruby owns its own connections
+    # (String/Hash path), the internal connection_pool handles concurrency.
     def synchronized(&)
-      @pgmq_mutex.synchronize(&)
+      if @pgmq_mutex
+        @pgmq_mutex.synchronize(&)
+      else
+        yield
+      end
     end
 
     def serialize(data)

data/lib/pgbus/event_bus/publisher.rb

@@ -7,13 +7,7 @@ module Pgbus
 
       def publish(routing_key, payload, headers: nil, delay: 0)
         event_data = build_event_data(payload)
-
-        if Pgbus.client.pgmq.respond_to?(:produce_topic)
-          Pgbus.client.publish_to_topic(routing_key, event_data, headers: headers, delay: delay)
-        else
-          # Fallback: send directly to queues matching the routing key
-          Pgbus.client.send_message(routing_key, event_data, headers: headers, delay: delay)
-        end
+        Pgbus.client.publish_to_topic(routing_key, event_data, headers: headers, delay: delay)
       end
 
       def publish_later(routing_key, payload, delay:, headers: nil)

data/lib/pgbus/outbox/poller.rb

@@ -54,10 +54,7 @@ module Pgbus
                                  .to_a
             break if entries.empty?
 
-            entries.each do |entry|
-              succeeded += 1 if publish_entry(entry)
-            end
-
+            succeeded = publish_entries(entries)
             published += succeeded
             break if succeeded.zero? || entries.size < config.outbox_batch_size
           end
@@ -74,24 +71,65 @@ module Pgbus
 
       private
 
-      def publish_entry(entry)
-        if entry.routing_key.present?
-          Pgbus.client.publish_to_topic(
-            entry.routing_key,
-            entry.payload,
-            headers: entry.headers,
-            delay: entry.delay || 0
-          )
-        else
-          Pgbus.client.send_message(
-            entry.queue_name,
-            entry.payload,
-            headers: entry.headers,
-            delay: entry.delay || 0,
-            priority: entry.priority
-          )
+      def publish_entries(entries)
+        # Partition: topic-routed entries must be published individually
+        # (different routing keys), direct-queue entries can be batched.
+        topic_entries, queue_entries = entries.partition { |e| e.routing_key.present? }
+
+        succeeded = 0
+        topic_entries.each { |e| succeeded += 1 if publish_single(e) }
+        succeeded += publish_queue_batch(queue_entries)
+        succeeded
+      end
+
+      def publish_single(entry)
+        Pgbus.client.publish_to_topic(
+          entry.routing_key,
+          entry.payload,
+          headers: entry.headers,
+          delay: entry.delay || 0
+        )
+        entry.update!(published_at: Time.current)
+        true
+      rescue StandardError => e
+        Pgbus.logger.error { "[Pgbus] Failed to publish outbox entry #{entry.id}: #{e.message}" }
+        false
+      end
+
+      # Group direct-queue entries by (queue_name, priority, delay) and
+      # use send_batch for each group to reduce round-trips.
+      def publish_queue_batch(entries)
+        return 0 if entries.empty?
+
+        succeeded = 0
+        entries.group_by { |e| [e.queue_name, e.priority, e.delay || 0] }.each do |(queue, _priority, delay), group|
+          payloads = group.map(&:payload)
+          headers = group.map(&:headers)
+          headers = nil if headers.all?(&:blank?)
+
+          Pgbus.client.send_batch(queue, payloads, headers: headers, delay: delay)
+          now = Time.current
+          group.each { |e| e.update!(published_at: now) }
+          succeeded += group.size
+        rescue StandardError => e
+          Pgbus.logger.error { "[Pgbus] Failed to batch-publish #{group.size} outbox entries: #{e.message}" }
+          # Fall back to individual publishing for this group
+          group.each { |entry| succeeded += 1 if publish_single_queue(entry) }
         end
+        succeeded
+      end
 
+      # Fallback for individual publishing when a batch fails.
+      # Intentionally omits priority to match send_batch routing behavior
+      # (base queue only), ensuring consistent queue placement regardless
+      # of whether the batch or fallback path is used.
+      def publish_single_queue(entry)
+        Pgbus.client.send_message(
+          entry.queue_name,
+          entry.payload,
+          headers: entry.headers,
+          delay: entry.delay || 0
+        )
         entry.update!(published_at: Time.current)
         true
       rescue StandardError => e

data/lib/pgbus/process/consumer.rb

@@ -56,9 +56,12 @@ module Pgbus
         idle = @pool.max_length - @pool.queue_length
         return interruptible_sleep(config.polling_interval) if idle <= 0
 
-        tagged_messages = @queue_names.flat_map do |queue_name|
-          (Pgbus.client.read_batch(queue_name, qty: idle) || []).map { |m| [queue_name, m] }
-        end.first(idle)
+        tagged_messages = if @queue_names.size == 1
+                            queue = @queue_names.first
+                            (Pgbus.client.read_batch(queue, qty: idle) || []).map { |m| [queue, m] }
+                          else
+                            fetch_multi_consumer(idle)
+                          end
 
         if tagged_messages.empty?
           interruptible_sleep(config.polling_interval)
@@ -94,6 +97,16 @@ module Pgbus
         # the next read will route to DLQ above.
       end
 
+      def fetch_multi_consumer(qty)
+        messages = Pgbus.client.read_multi(@queue_names, qty: qty) || []
+        prefix = "#{config.queue_prefix}_"
+
+        messages.map do |m|
+          logical = m.queue_name&.delete_prefix(prefix) || @queue_names.first
+          [logical, m]
+        end
+      end
+
       def pattern_overlaps?(topic_filter, subscription_pattern)
         # Simple check: if either is a subset of the other
         topic_filter == subscription_pattern ||

data/lib/pgbus/process/worker.rb

@@ -107,7 +107,7 @@ module Pgbus
       end
 
       # Returns an array of [queue_name, message] pairs so we always know
-      # which queue each message came from (PGMQ messages don't carry this).
+      # which queue each message came from.
       def fetch_messages(qty)
         active_queues = queues.reject { |q| @circuit_breaker.paused?(q) }
         active_queues = active_queues.select { |q| @queue_lock.try_lock(q) } if @single_active_consumer
@@ -120,10 +120,7 @@ module Pgbus
           messages = Pgbus.client.read_batch(queue, qty: qty) || []
           messages.map { |m| [queue, m] }
         else
-          per_queue = [(qty / active_queues.size.to_f).ceil, 1].max
-          active_queues.flat_map do |q|
-            (Pgbus.client.read_batch(q, qty: per_queue) || []).map { |m| [q, m] }
-          end.first(qty)
+          fetch_multi(active_queues, qty)
         end
       rescue StandardError => e
         Pgbus.logger.error { "[Pgbus] Error fetching messages: #{e.message}" }
@@ -147,6 +144,19 @@ module Pgbus
         results
       end
 
+      # Use pgmq-ruby's read_multi to read from all queues in a single
+      # SQL query (UNION ALL). Each returned message carries a queue_name
+      # field so we can map it back to the logical queue.
+      def fetch_multi(active_queues, qty)
+        messages = Pgbus.client.read_multi(active_queues, qty: qty) || []
+        prefix = "#{config.queue_prefix}_"
+
+        messages.map do |m|
+          logical = m.queue_name&.delete_prefix(prefix) || active_queues.first
+          [logical, m]
+        end
+      end
+
       def priority_enabled?
         config.priority_levels && config.priority_levels > 1
       end

data/lib/pgbus/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Pgbus
-  VERSION = "0.2.0"
+  VERSION = "0.2.1"
 end

data/lib/pgbus/web/data_source.rb

@@ -105,11 +105,11 @@ module Pgbus
       end
 
       def retry_job(queue_name, msg_id)
-        @client.set_visibility_timeout(queue_name, msg_id.to_i, vt: 0)
+        @client.set_visibility_timeout(queue_name, msg_id.to_i, vt: 0, prefixed: false)
       end
 
       def discard_job(queue_name, msg_id)
-        @client.archive_message(queue_name, msg_id.to_i)
+        @client.archive_message(queue_name, msg_id.to_i, prefixed: false)
       end
 
       # Failed events
@@ -269,7 +269,7 @@ module Pgbus
 
       def discard_dlq_message(queue_name, msg_id)
         # queue_name here is the full DLQ name (already prefixed)
-        @client.delete_from_queue(queue_name, msg_id.to_i)
+        @client.delete_message(queue_name, msg_id.to_i, prefixed: false)
         true
       rescue StandardError => e
         Pgbus.logger.debug { "[Pgbus::Web] Error discarding DLQ message #{msg_id}: #{e.message}" }
@@ -290,14 +290,16 @@ module Pgbus
 
       def discard_all_dlq
         messages = dlq_messages(page: 1, per_page: 1000)
-        count = 0
-        messages.each do |m|
-          discard_dlq_message(m[:queue_name], m[:msg_id]) && count += 1
+        return 0 if messages.empty?
+
+        # Group by queue for batch delete — one call per DLQ instead of N calls
+        messages.group_by { |m| m[:queue_name] }.sum do |queue_name, msgs|
+          ids = msgs.map { |m| m[:msg_id].to_i }
+          @client.delete_batch(queue_name, ids, prefixed: false).size
         rescue StandardError => e
-          Pgbus.logger.debug { "[Pgbus::Web] Error discarding DLQ message #{m[:msg_id]}: #{e.message}" }
-          next
+          Pgbus.logger.debug { "[Pgbus::Web] Error batch-discarding DLQ messages from #{queue_name}: #{e.message}" }
+          0
         end
-        count
       end
 
       # Processes

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: pgbus
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.2.1
 platform: ruby
 authors:
 - Mikael Henriksson