pgbus 0.1.3 → 0.1.5

data/lib/pgbus/client.rb

@@ -19,22 +19,26 @@ module Pgbus
         require "pgmq"
       end
       @config = config
+      # Force pool_size=1. PG::Connection (libpq) is not thread-safe.
+      # When using the Rails lambda path (-> { AR::Base.connection.raw_connection }),
+      # the pool would return the same underlying PG::Connection that ActiveRecord
+      # also uses, causing concurrent access corruption (segfaults, result.ntuples
+      # NoMethodError). A single-connection pool combined with @pgmq_mutex ensures
+      # all PGMQ operations are serialized.
       @pgmq = PGMQ::Client.new(
         config.connection_options,
-        pool_size: config.pool_size,
+        pool_size: 1,
         pool_timeout: config.pool_timeout
       )
+      @pgmq_mutex = Mutex.new
       @queues_created = Concurrent::Map.new
     end
 
     def ensure_queue(name)
-      full_name = config.queue_name(name)
-      return if @queues_created[full_name]
-
-      @queues_created.compute_if_absent(full_name) do
-        @pgmq.create(full_name)
-        @pgmq.enable_notify_insert(full_name, throttle_interval_ms: config.notify_throttle_ms) if config.listen_notify
-        true
+      if priority_enabled?
+        config.priority_queue_names(name).each { |pq| ensure_single_queue(pq) }
+      else
+        ensure_single_queue(config.queue_name(name))
       end
     rescue PGMQ::Errors::ConnectionError => e
       raise Pgbus::SchemaNotReady,
@@ -46,79 +50,107 @@ module Pgbus
       return if @queues_created[dlq_name]
 
       @queues_created.compute_if_absent(dlq_name) do
-        @pgmq.create(dlq_name)
+        synchronized { @pgmq.create(dlq_name) }
         true
       end
     end
 
-    def send_message(queue_name, payload, headers: nil, delay: 0)
-      full_name = config.queue_name(queue_name)
+    def send_message(queue_name, payload, headers: nil, delay: 0, priority: nil)
+      target = resolve_target_queue(queue_name, priority)
       ensure_queue(queue_name)
-      Instrumentation.instrument("pgbus.client.send_message", queue: full_name) do
-        @pgmq.produce(full_name, serialize(payload), headers: headers && serialize(headers), delay: delay)
+      Instrumentation.instrument("pgbus.client.send_message", queue: target) do
+        synchronized { @pgmq.produce(target, serialize(payload), headers: headers && serialize(headers), delay: delay) }
       end
     end
 
     def send_batch(queue_name, payloads, headers: nil, delay: 0)
       full_name = config.queue_name(queue_name)
       ensure_queue(queue_name)
+      serialized = payloads.map { |p| serialize(p) }
+      serialized_headers = headers&.map { |h| serialize(h) }
       Instrumentation.instrument("pgbus.client.send_batch", queue: full_name, size: payloads.size) do
-        serialized = payloads.map { |p| serialize(p) }
-        serialized_headers = headers&.map { |h| serialize(h) }
-        @pgmq.produce_batch(full_name, serialized, headers: serialized_headers, delay: delay)
+        synchronized { @pgmq.produce_batch(full_name, serialized, headers: serialized_headers, delay: delay) }
       end
     end
 
     def read_message(queue_name, vt: nil)
       full_name = config.queue_name(queue_name)
       Instrumentation.instrument("pgbus.client.read_message", queue: full_name) do
-        @pgmq.read(full_name, vt: vt || config.visibility_timeout)
+        synchronized { @pgmq.read(full_name, vt: vt || config.visibility_timeout) }
       end
     end
 
     def read_batch(queue_name, qty:, vt: nil)
       full_name = config.queue_name(queue_name)
       Instrumentation.instrument("pgbus.client.read_batch", queue: full_name, qty: qty) do
-        @pgmq.read_batch(full_name, vt: vt || config.visibility_timeout, qty: qty)
+        synchronized { @pgmq.read_batch(full_name, vt: vt || config.visibility_timeout, qty: qty) }
+      end
+    end
+
+    # Read from priority sub-queues, highest priority (p0) first.
+    # Returns [priority_queue_name, messages] pairs.
+    def read_batch_prioritized(queue_name, qty:, vt: nil)
+      return (read_batch(queue_name, qty: qty, vt: vt) || []).map { |m| [config.queue_name(queue_name), m] } unless priority_enabled?
+
+      remaining = qty
+      results = []
+
+      config.priority_queue_names(queue_name).each do |pq_name|
+        break if remaining <= 0
+
+        msgs = Instrumentation.instrument("pgbus.client.read_batch", queue: pq_name, qty: remaining) do
+          synchronized { @pgmq.read_batch(pq_name, vt: vt || config.visibility_timeout, qty: remaining) }
+        end || []
+
+        msgs.each { |m| results << [pq_name, m] }
+        remaining -= msgs.size
       end
+
+      results
     end
 
     def read_with_poll(queue_name, qty:, vt: nil, max_poll_seconds: 5, poll_interval_ms: 100)
       full_name = config.queue_name(queue_name)
-      @pgmq.read_with_poll(
-        full_name,
-        vt: vt || config.visibility_timeout,
-        qty: qty,
-        max_poll_seconds: max_poll_seconds,
-        poll_interval_ms: poll_interval_ms
-      )
+      synchronized do
+        @pgmq.read_with_poll(
+          full_name,
+          vt: vt || config.visibility_timeout,
+          qty: qty,
+          max_poll_seconds: max_poll_seconds,
+          poll_interval_ms: poll_interval_ms
+        )
+      end
     end
 
     def delete_message(queue_name, msg_id)
       full_name = config.queue_name(queue_name)
-      @pgmq.delete(full_name, msg_id)
+      synchronized { @pgmq.delete(full_name, msg_id) }
     end
 
     def archive_message(queue_name, msg_id)
       full_name = config.queue_name(queue_name)
-      @pgmq.archive(full_name, msg_id)
+      synchronized { @pgmq.archive(full_name, msg_id) }
+    end
+
+    def archive_from_queue(full_queue_name, msg_id)
+      synchronized { @pgmq.archive(full_queue_name, msg_id) }
     end
 
     def extend_visibility(queue_name, msg_id, vt:)
       full_name = config.queue_name(queue_name)
-      @pgmq.set_vt(full_name, msg_id, vt: vt)
+      synchronized { @pgmq.set_vt(full_name, msg_id, vt: vt) }
     end
 
     def set_visibility_timeout(queue_name, msg_id, vt:)
-      @pgmq.set_vt(queue_name, msg_id, vt: vt)
+      synchronized { @pgmq.set_vt(queue_name, msg_id, vt: vt) }
     end
 
     def delete_from_queue(queue_name, msg_id)
-      @pgmq.delete(queue_name, msg_id)
+      synchronized { @pgmq.delete(queue_name, msg_id) }
     end
 
-    def transaction(&)
-      @pgmq.transaction(&)
+    def transaction(&block)
+      synchronized { @pgmq.transaction(&block) }
     end
 
     def move_to_dead_letter(queue_name, message)
@@ -126,50 +158,109 @@ module Pgbus
       dlq_name = config.dead_letter_queue_name(queue_name)
       full_queue = config.queue_name(queue_name)
 
-      @pgmq.transaction do |txn|
-        txn.produce(dlq_name, message.message, headers: message.headers)
-        txn.delete(full_queue, message.msg_id.to_i)
+      synchronized do
+        @pgmq.transaction do |txn|
+          txn.produce(dlq_name, message.message, headers: message.headers)
+          txn.delete(full_queue, message.msg_id.to_i)
+        end
       end
     end
 
     def metrics(queue_name = nil)
-      if queue_name
-        @pgmq.metrics(config.queue_name(queue_name))
-      else
-        @pgmq.metrics_all
+      synchronized do
+        if queue_name
+          @pgmq.metrics(config.queue_name(queue_name))
+        else
+          @pgmq.metrics_all
+        end
       end
     end
 
     def list_queues
-      @pgmq.list_queues
+      synchronized { @pgmq.list_queues }
     end
 
     def purge_queue(queue_name)
-      @pgmq.purge_queue(config.queue_name(queue_name))
+      synchronized { @pgmq.purge_queue(config.queue_name(queue_name)) }
+    end
+
+    def purge_archive(queue_name, older_than:, batch_size: 1000)
+      full_name = config.queue_name(queue_name)
+      sanitized = full_name.gsub(/[^a-zA-Z0-9_]/, "")
+      total = 0
+
+      sql = "DELETE FROM pgmq.a_#{sanitized} " \
+            "WHERE ctid = ANY(ARRAY(SELECT ctid FROM pgmq.a_#{sanitized} WHERE enqueued_at < $1 LIMIT $2))"
+
+      loop do
+        deleted = synchronized do
+          @pgmq.pool.with { |conn| conn.exec_params(sql, [older_than, batch_size]).cmd_tuples }
+        end
+        total += deleted
+        break if deleted < batch_size
+      end
+
+      total
     end
 
     # Topic routing
     def bind_topic(pattern, queue_name)
       full_name = config.queue_name(queue_name)
       ensure_queue(queue_name)
-      @pgmq.bind_topic(pattern, full_name)
+      synchronized { @pgmq.bind_topic(pattern, full_name) }
     end
 
     def publish_to_topic(routing_key, payload, headers: nil, delay: 0)
-      @pgmq.produce_topic(
-        routing_key,
-        serialize(payload),
-        headers: headers && serialize(headers),
-        delay: delay
-      )
+      synchronized do
+        @pgmq.produce_topic(
+          routing_key,
+          serialize(payload),
+          headers: headers && serialize(headers),
+          delay: delay
+        )
+      end
     end
 
     def close
-      @pgmq.close
+      synchronized { @pgmq.close }
     end
 
     private
 
+    def ensure_single_queue(full_name)
+      return if @queues_created[full_name]
+
+      @queues_created.compute_if_absent(full_name) do
+        synchronized do
+          @pgmq.create(full_name)
+          @pgmq.enable_notify_insert(full_name, throttle_interval_ms: config.notify_throttle_ms) if config.listen_notify
+        end
+        true
+      end
+    end
+
+    def priority_enabled?
+      config.priority_levels && config.priority_levels > 1
+    end
+
+    def resolve_target_queue(queue_name, priority)
+      if priority_enabled? && priority
+        clamped = priority.clamp(0, config.priority_levels - 1)
+        config.priority_queue_name(queue_name, clamped)
+      elsif priority_enabled?
+        config.priority_queue_name(queue_name, config.default_priority.clamp(0, config.priority_levels - 1))
+      else
+        config.queue_name(queue_name)
+      end
+    end
+
+    # Serialize all PGMQ operations through a single mutex.
+    # PG::Connection is not thread-safe — concurrent access from worker
+    # threads causes segfaults and result corruption.
+    def synchronized(&)
+      @pgmq_mutex.synchronize(&)
+    end
+
     def serialize(data)
       case data
       when String

data/lib/pgbus/configuration.rb

@@ -11,7 +11,7 @@ module Pgbus
     attr_accessor :default_queue, :queue_prefix
 
     # Worker settings
-    attr_accessor :workers, :polling_interval, :visibility_timeout
+    attr_accessor :workers, :polling_interval, :visibility_timeout, :prefetch_limit
 
     # Worker recycling
     attr_accessor :max_jobs_per_worker, :max_memory_mb, :max_worker_lifetime
@@ -19,9 +19,22 @@ module Pgbus
     # Dispatcher settings
     attr_accessor :dispatch_interval
 
+    # Circuit breaker
+    attr_accessor :circuit_breaker_enabled, :circuit_breaker_threshold,
+                  :circuit_breaker_base_backoff, :circuit_breaker_max_backoff
+
     # Dead letter queue
     attr_accessor :max_retries, :dead_letter_queue_suffix
 
+    # Priority queues
+    attr_accessor :priority_levels, :default_priority
+
+    # Archive compaction
+    attr_accessor :archive_retention, :archive_compaction_interval, :archive_compaction_batch_size
+
+    # Transactional outbox
+    attr_accessor :outbox_enabled, :outbox_poll_interval, :outbox_batch_size, :outbox_retention
+
     # Event bus
     attr_accessor :idempotency_ttl
 
@@ -62,15 +75,34 @@ module Pgbus
       @polling_interval = 0.1
       @visibility_timeout = 30
 
+      @prefetch_limit = nil
+
       @max_jobs_per_worker = nil
       @max_memory_mb = nil
       @max_worker_lifetime = nil
 
       @dispatch_interval = 1.0
 
+      @circuit_breaker_enabled = true
+      @circuit_breaker_threshold = 5
+      @circuit_breaker_base_backoff = 30
+      @circuit_breaker_max_backoff = 600
+
       @max_retries = 5
       @dead_letter_queue_suffix = "_dlq"
 
+      @priority_levels = nil
+      @default_priority = 1
+
+      @archive_retention = 7 * 24 * 3600 # 7 days
+      @archive_compaction_interval = 3600
+      @archive_compaction_batch_size = 1000
+
+      @outbox_enabled = false
+      @outbox_poll_interval = 1.0
+      @outbox_batch_size = 100
+      @outbox_retention = 24 * 3600 # 1 day
+
       @idempotency_ttl = 7 * 24 * 3600 # 7 days
 
       @logger = defined?(Rails) ? Rails.logger : Logger.new($stdout)
@@ -105,6 +137,16 @@ module Pgbus
       "#{queue_name(name)}#{dead_letter_queue_suffix}"
     end
 
+    def priority_queue_name(name, priority)
+      "#{queue_name(name)}_p#{priority}"
+    end
+
+    def priority_queue_names(name)
+      return [queue_name(name)] unless priority_levels && priority_levels > 1
+
+      (0...priority_levels).map { |p| priority_queue_name(name, p) }
+    end
+
     VALID_PGMQ_SCHEMA_MODES = %i[auto extension embedded].freeze
 
     def pgmq_schema_mode=(mode)
@@ -128,6 +170,12 @@ module Pgbus
         raise ArgumentError, "worker threads must be > 0" unless threads.is_a?(Integer) && threads.positive?
       end
 
+      raise ArgumentError, "prefetch_limit must be > 0" if prefetch_limit && !(prefetch_limit.is_a?(Integer) && prefetch_limit.positive?)
+
+      if priority_levels && !(priority_levels.is_a?(Integer) && priority_levels >= 1 && priority_levels <= 10)
+        raise ArgumentError, "priority_levels must be an integer between 1 and 10"
+      end
+
       self
     end
 
@@ -137,7 +185,11 @@ module Pgbus
       elsif connection_params
         connection_params
       elsif defined?(ActiveRecord::Base)
-        -> { ActiveRecord::Base.connection.raw_connection }
+        if connects_to
+          -> { Pgbus::ApplicationRecord.connection.raw_connection }
+        else
+          -> { ActiveRecord::Base.connection.raw_connection }
+        end
       else
         raise ConfigurationError, "No database connection configured. " \
                                   "Set Pgbus.configuration.database_url, connection_params, or use with Rails."

data/lib/pgbus/dedup_cache.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+require "concurrent"
+
+module Pgbus
+  # Thread-safe in-memory dedup cache with TTL.
+  # Sits in front of the pgbus_processed_events table to avoid
+  # hitting the database for recently-seen (event_id, handler_class) pairs.
+  #
+  # Inspired by LavinMQ's built-in message deduplication with LRU + TTL.
+  class DedupCache
+    DEFAULT_MAX_SIZE = 10_000
+    DEFAULT_TTL = 300 # 5 minutes
+
+    def initialize(max_size: DEFAULT_MAX_SIZE, ttl: DEFAULT_TTL)
+      @max_size = max_size
+      @ttl = ttl
+      @cache = Concurrent::Map.new
+      @insertion_order = Concurrent::Array.new
+    end
+
+    # Returns true if this key was already seen (duplicate).
+    # Returns false if it's new (first time seen — caller should proceed).
+    def seen?(key)
+      entry = @cache[key]
+      return false unless entry
+
+      if expired?(entry)
+        evict(key)
+        return false
+      end
+
+      true
+    end
+
+    # Mark a key as seen. Call this after successfully claiming idempotency
+    # in the database so future lookups skip the DB.
+    def mark!(key)
+      already_present = @cache.key?(key)
+      evict_oldest if !already_present && @cache.size >= @max_size
+
+      @cache[key] = monotonic_now
+      @insertion_order << key unless already_present
+    end
+
+    def size
+      @cache.size
+    end
+
+    def clear!
+      @cache.clear
+      @insertion_order.clear
+    end
+
+    private
+
+    def expired?(timestamp)
+      (monotonic_now - timestamp) > @ttl
+    end
+
+    def evict(key)
+      @cache.delete(key)
+    end
+
+    def evict_oldest
+      while @cache.size >= @max_size && !@insertion_order.empty?
+        oldest_key = @insertion_order.shift
+        @cache.delete(oldest_key)
+      end
+    end
+
+    def monotonic_now
+      ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+    end
+  end
+end

data/lib/pgbus/engine.rb

@@ -6,6 +6,12 @@ module Pgbus
   class Engine < ::Rails::Engine
     isolate_namespace Pgbus
 
+    # When pgbus was loaded before Rails, a separate Zeitwerk loader manages
+    # app/models. Tear it down before Rails' autoloader claims the same paths.
+    initializer "pgbus.release_models_loader", before: :set_autoload_paths do
+      Pgbus.teardown_models_loader!
+    end
+
     initializer "pgbus.configure" do |app|
       config_path = app.root.join("config", "pgbus.yml")
       Pgbus::ConfigLoader.load(config_path) if config_path.exist?

data/lib/pgbus/event_bus/handler.rb

@@ -11,6 +11,10 @@ module Pgbus
         def idempotent?
           @idempotent == true
         end
+
+        def dedup_cache
+          @dedup_cache ||= DedupCache.new
+        end
       end
 
       def process(message)
@@ -50,13 +54,20 @@ module Pgbus
       # Atomically claim idempotency: INSERT ... ON CONFLICT DO NOTHING.
       # Returns true if this handler claimed the event (row was inserted),
       # false if another handler already processed it (conflict, no insert).
+      #
+      # Uses an in-memory dedup cache to skip the DB for recently-seen events.
       def claim_idempotency?(event_id)
+        cache_key = "#{event_id}:#{self.class.name}"
+        return false if self.class.dedup_cache.seen?(cache_key)
+
         result = ProcessedEvent.insert(
           { event_id: event_id, handler_class: self.class.name, processed_at: Time.now.utc },
           unique_by: %i[event_id handler_class]
         )
-        # insert returns an InsertAll::Result; inserted row count > 0 means we claimed it
-        result.rows.any?
+
+        claimed = result.rows.any?
+        self.class.dedup_cache.mark!(cache_key)
+        claimed
       end
     end
   end

data/lib/pgbus/outbox/poller.rb

@@ -0,0 +1,117 @@
+# frozen_string_literal: true
+
+module Pgbus
+  module Outbox
+    class Poller
+      include Process::SignalHandler
+
+      attr_reader :config
+
+      def initialize(config: Pgbus.configuration)
+        @config = config
+        @shutting_down = false
+      end
+
+      def run
+        setup_signals
+        start_heartbeat
+        Pgbus.logger.info { "[Pgbus] Outbox poller started: interval=#{config.outbox_poll_interval}s" }
+
+        loop do
+          break if @shutting_down
+
+          process_signals
+          break if @shutting_down
+
+          poll_and_publish
+          break if @shutting_down
+
+          interruptible_sleep(config.outbox_poll_interval)
+        end
+
+        shutdown
+      end
+
+      def graceful_shutdown
+        @shutting_down = true
+      end
+
+      def immediate_shutdown
+        @shutting_down = true
+      end
+
+      def poll_and_publish
+        published = 0
+
+        loop do
+          succeeded = 0
+
+          OutboxEntry.transaction do
+            entries = OutboxEntry.unpublished
+                                 .order(:id)
+                                 .limit(config.outbox_batch_size)
+                                 .lock("FOR UPDATE SKIP LOCKED")
+                                 .to_a
+            break if entries.empty?
+
+            entries.each do |entry|
+              succeeded += 1 if publish_entry(entry)
+            end
+
+            published += succeeded
+            break if succeeded.zero? || entries.size < config.outbox_batch_size
+          end
+
+          break if succeeded.zero?
+        end
+
+        Pgbus.logger.debug { "[Pgbus] Outbox published #{published} entries" } if published.positive?
+        published
+      rescue StandardError => e
+        Pgbus.logger.error { "[Pgbus] Outbox poll error: #{e.message}" }
+        0
+      end
+
+      private
+
+      def publish_entry(entry)
+        if entry.routing_key.present?
+          Pgbus.client.publish_to_topic(
+            entry.routing_key,
+            entry.payload,
+            headers: entry.headers,
+            delay: entry.delay || 0
+          )
+        else
+          Pgbus.client.send_message(
+            entry.queue_name,
+            entry.payload,
+            headers: entry.headers,
+            delay: entry.delay || 0,
+            priority: entry.priority
+          )
+        end
+
+        entry.update!(published_at: Time.current)
+        true
+      rescue StandardError => e
+        Pgbus.logger.error { "[Pgbus] Failed to publish outbox entry #{entry.id}: #{e.message}" }
+        false
+      end
+
+      def start_heartbeat
+        @heartbeat = Process::Heartbeat.new(
+          kind: "outbox_poller",
+          metadata: { pid: ::Process.pid }
+        )
+        @heartbeat.start
+      end
+
+      def shutdown
+        @heartbeat&.stop
+        restore_signals
+        Pgbus.logger.info { "[Pgbus] Outbox poller stopped" }
+      end
+    end
+  end
+end

data/lib/pgbus/outbox.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Pgbus
+  module Outbox
+    module_function
+
+    def publish(queue_name, payload, headers: nil, priority: nil, delay: 0)
+      OutboxEntry.create!(
+        queue_name: queue_name,
+        payload: payload,
+        headers: headers,
+        priority: priority || Pgbus.configuration.default_priority,
+        delay: delay
+      )
+    end
+
+    def publish_event(routing_key, payload, headers: nil)
+      event_data = EventBus::Publisher.build_event_data(payload)
+      OutboxEntry.create!(
+        routing_key: routing_key,
+        payload: event_data,
+        headers: headers
+      )
+    end
+
+    def flush!
+      Poller.new.poll_and_publish
+    end
+  end
+end