pgbus 0.1.4 → 0.1.6

data/lib/pgbus/circuit_breaker.rb

@@ -0,0 +1,112 @@
+# frozen_string_literal: true
+
+require "concurrent"
+
+module Pgbus
+  class CircuitBreaker
+    attr_reader :config
+
+    def initialize(config: Pgbus.configuration)
+      @config = config
+      @failure_counts = Concurrent::Map.new
+      @pause_cache = Concurrent::Map.new
+      @pause_cache_ttl = 30 # seconds
+    end
+
+    def record_success(queue_name)
+      @failure_counts.delete(queue_name)
+    end
+
+    def record_failure(queue_name)
+      return unless config.circuit_breaker_enabled
+
+      count = @failure_counts.compute(queue_name) { |val| (val || 0) + 1 }
+
+      return unless count >= config.circuit_breaker_threshold
+
+      trip!(queue_name, count)
+    end
+
+    def paused?(queue_name)
+      cached = @pause_cache[queue_name]
+      return cached[:paused] if cached && (Time.now - cached[:checked_at]) < @pause_cache_ttl
+
+      paused = check_paused(queue_name)
+      @pause_cache[queue_name] = { paused: paused, checked_at: Time.now }
+      paused
+    end
+
+    def pause!(queue_name, reason: nil)
+      QueueState.pause!(queue_name, reason: reason)
+      invalidate_cache(queue_name)
+    end
+
+    def resume!(queue_name)
+      QueueState.resume!(queue_name)
+      @failure_counts.delete(queue_name)
+      invalidate_cache(queue_name)
+    end
+
+    def invalidate_cache(queue_name = nil)
+      if queue_name
+        @pause_cache.delete(queue_name)
+      else
+        @pause_cache.clear
+      end
+    end
+
+    private
+
+    def trip!(queue_name, failure_count)
+      trip_count = current_trip_count(queue_name) + 1
+      backoff = calculate_backoff(trip_count)
+      resume_at = Time.current + backoff
+
+      Pgbus.logger.warn do
+        "[Pgbus] Circuit breaker tripped for #{queue_name}: #{failure_count} consecutive failures, " \
+          "backoff #{backoff}s (trip ##{trip_count})"
+      end
+
+      QueueState.find_or_initialize_by(queue_name: queue_name).update!(
+        paused: true,
+        paused_reason: "circuit_breaker: #{failure_count} consecutive failures",
+        paused_at: Time.current,
+        circuit_breaker_trip_count: trip_count,
+        circuit_breaker_resume_at: resume_at
+      )
+
+      @failure_counts.delete(queue_name)
+      invalidate_cache(queue_name)
+    rescue StandardError => e
+      Pgbus.logger.error { "[Pgbus] Circuit breaker trip failed for #{queue_name}: #{e.message}" }
+    end
+
+    def check_paused(queue_name)
+      state = QueueState.find_by(queue_name: queue_name)
+      return false unless state&.paused?
+
+      # Auto-resume if circuit breaker backoff has expired
+      if state.circuit_breaker_resume_at && Time.current >= state.circuit_breaker_resume_at
+        QueueState.resume!(queue_name)
+        Pgbus.logger.info { "[Pgbus] Circuit breaker auto-resumed #{queue_name}" }
+        return false
+      end
+
+      true
+    rescue StandardError => e
+      Pgbus.logger.warn { "[Pgbus] Circuit breaker pause check failed for #{queue_name}: #{e.message}" }
+      false
+    end
+
+    def current_trip_count(queue_name)
+      QueueState.find_by(queue_name: queue_name)&.circuit_breaker_trip_count || 0
+    rescue StandardError
+      0
+    end
+
+    def calculate_backoff(trip_count)
+      backoff = config.circuit_breaker_base_backoff * (2**(trip_count - 1))
+      [backoff, config.circuit_breaker_max_backoff].min
+    end
+  end
+end

data/lib/pgbus/client.rb

@@ -19,23 +19,24 @@ module Pgbus
         require "pgmq"
       end
       @config = config
+      # Force pool_size=1. PG::Connection (libpq) is not thread-safe.
+      # When using the Rails lambda path (-> { AR::Base.connection.raw_connection }),
+      # the pool would return the same underlying PG::Connection that ActiveRecord
+      # also uses, causing concurrent access corruption (segfaults, result.ntuples
+      # NoMethodError). A single-connection pool combined with @pgmq_mutex ensures
+      # all PGMQ operations are serialized.
       @pgmq = PGMQ::Client.new(
         config.connection_options,
-        pool_size: config.pool_size,
+        pool_size: 1,
         pool_timeout: config.pool_timeout
       )
+      @pgmq_mutex = Mutex.new
       @queues_created = Concurrent::Map.new
+      @queue_strategy = QueueFactory.for(config)
     end
 
     def ensure_queue(name)
-      full_name = config.queue_name(name)
-      return if @queues_created[full_name]
-
-      @queues_created.compute_if_absent(full_name) do
-        @pgmq.create(full_name)
-        @pgmq.enable_notify_insert(full_name, throttle_interval_ms: config.notify_throttle_ms) if config.listen_notify
-        true
-      end
+      @queue_strategy.physical_queue_names(name).each { |pq| ensure_single_queue(pq) }
     rescue PGMQ::Errors::ConnectionError => e
       raise Pgbus::SchemaNotReady,
             "PGMQ schema is not available (#{e.message}). Run `rails db:migrate` for the pgbus database."
@@ -46,79 +47,111 @@ module Pgbus
       return if @queues_created[dlq_name]
 
       @queues_created.compute_if_absent(dlq_name) do
-        @pgmq.create(dlq_name)
+        synchronized { @pgmq.create(dlq_name) }
         true
       end
     end
 
-    def send_message(queue_name, payload, headers: nil, delay: 0)
-      full_name = config.queue_name(queue_name)
+    def send_message(queue_name, payload, headers: nil, delay: 0, priority: nil)
+      target = @queue_strategy.target_queue(queue_name, priority)
       ensure_queue(queue_name)
-      Instrumentation.instrument("pgbus.client.send_message", queue: full_name) do
-        @pgmq.produce(full_name, serialize(payload), headers: headers && serialize(headers), delay: delay)
+      Instrumentation.instrument("pgbus.client.send_message", queue: target) do
+        synchronized { @pgmq.produce(target, serialize(payload), headers: headers && serialize(headers), delay: delay) }
       end
     end
 
     def send_batch(queue_name, payloads, headers: nil, delay: 0)
       full_name = config.queue_name(queue_name)
       ensure_queue(queue_name)
+      serialized = payloads.map { |p| serialize(p) }
+      serialized_headers = headers&.map { |h| serialize(h) }
       Instrumentation.instrument("pgbus.client.send_batch", queue: full_name, size: payloads.size) do
-        serialized = payloads.map { |p| serialize(p) }
-        serialized_headers = headers&.map { |h| serialize(h) }
-        @pgmq.produce_batch(full_name, serialized, headers: serialized_headers, delay: delay)
+        synchronized { @pgmq.produce_batch(full_name, serialized, headers: serialized_headers, delay: delay) }
       end
     end
 
     def read_message(queue_name, vt: nil)
       full_name = config.queue_name(queue_name)
       Instrumentation.instrument("pgbus.client.read_message", queue: full_name) do
-        @pgmq.read(full_name, vt: vt || config.visibility_timeout)
+        synchronized { @pgmq.read(full_name, vt: vt || config.visibility_timeout) }
       end
     end
 
     def read_batch(queue_name, qty:, vt: nil)
       full_name = config.queue_name(queue_name)
       Instrumentation.instrument("pgbus.client.read_batch", queue: full_name, qty: qty) do
-        @pgmq.read_batch(full_name, vt: vt || config.visibility_timeout, qty: qty)
+        synchronized { @pgmq.read_batch(full_name, vt: vt || config.visibility_timeout, qty: qty) }
       end
     end
 
+    # Read from priority sub-queues, highest priority (p0) first.
+    # Returns [priority_queue_name, messages] pairs.
+    def read_batch_prioritized(queue_name, qty:, vt: nil)
+      unless @queue_strategy.priority?
+        return (read_batch(queue_name, qty: qty, vt: vt) || []).map do |m|
+          [config.queue_name(queue_name), m]
+        end
+      end
+
+      remaining = qty
+      results = []
+
+      config.priority_queue_names(queue_name).each do |pq_name|
+        break if remaining <= 0
+
+        msgs = Instrumentation.instrument("pgbus.client.read_batch", queue: pq_name, qty: remaining) do
+          synchronized { @pgmq.read_batch(pq_name, vt: vt || config.visibility_timeout, qty: remaining) }
+        end || []
+
+        msgs.each { |m| results << [pq_name, m] }
+        remaining -= msgs.size
+      end
+
+      results
+    end
+
     def read_with_poll(queue_name, qty:, vt: nil, max_poll_seconds: 5, poll_interval_ms: 100)
       full_name = config.queue_name(queue_name)
-      @pgmq.read_with_poll(
-        full_name,
-        vt: vt || config.visibility_timeout,
-        qty: qty,
-        max_poll_seconds: max_poll_seconds,
-        poll_interval_ms: poll_interval_ms
-      )
+      synchronized do
+        @pgmq.read_with_poll(
+          full_name,
+          vt: vt || config.visibility_timeout,
+          qty: qty,
+          max_poll_seconds: max_poll_seconds,
+          poll_interval_ms: poll_interval_ms
+        )
+      end
     end
 
     def delete_message(queue_name, msg_id)
       full_name = config.queue_name(queue_name)
-      @pgmq.delete(full_name, msg_id)
+      synchronized { @pgmq.delete(full_name, msg_id) }
     end
 
     def archive_message(queue_name, msg_id)
       full_name = config.queue_name(queue_name)
-      @pgmq.archive(full_name, msg_id)
+      synchronized { @pgmq.archive(full_name, msg_id) }
+    end
+
+    def archive_from_queue(full_queue_name, msg_id)
+      synchronized { @pgmq.archive(full_queue_name, msg_id) }
     end
 
     def extend_visibility(queue_name, msg_id, vt:)
       full_name = config.queue_name(queue_name)
-      @pgmq.set_vt(full_name, msg_id, vt: vt)
+      synchronized { @pgmq.set_vt(full_name, msg_id, vt: vt) }
     end
 
     def set_visibility_timeout(queue_name, msg_id, vt:)
-      @pgmq.set_vt(queue_name, msg_id, vt: vt)
+      synchronized { @pgmq.set_vt(queue_name, msg_id, vt: vt) }
     end
 
     def delete_from_queue(queue_name, msg_id)
-      @pgmq.delete(queue_name, msg_id)
+      synchronized { @pgmq.delete(queue_name, msg_id) }
     end
 
-    def transaction(&)
-      @pgmq.transaction(&)
+    def transaction(&block)
+      synchronized { @pgmq.transaction(&block) }
     end
 
     def move_to_dead_letter(queue_name, message)
@@ -126,50 +159,94 @@ module Pgbus
       dlq_name = config.dead_letter_queue_name(queue_name)
       full_queue = config.queue_name(queue_name)
 
-      @pgmq.transaction do |txn|
-        txn.produce(dlq_name, message.message, headers: message.headers)
-        txn.delete(full_queue, message.msg_id.to_i)
+      synchronized do
+        @pgmq.transaction do |txn|
+          txn.produce(dlq_name, message.message, headers: message.headers)
+          txn.delete(full_queue, message.msg_id.to_i)
+        end
       end
     end
 
     def metrics(queue_name = nil)
-      if queue_name
-        @pgmq.metrics(config.queue_name(queue_name))
-      else
-        @pgmq.metrics_all
+      synchronized do
+        if queue_name
+          @pgmq.metrics(config.queue_name(queue_name))
+        else
+          @pgmq.metrics_all
+        end
       end
     end
 
     def list_queues
-      @pgmq.list_queues
+      synchronized { @pgmq.list_queues }
     end
 
     def purge_queue(queue_name)
-      @pgmq.purge_queue(config.queue_name(queue_name))
+      synchronized { @pgmq.purge_queue(config.queue_name(queue_name)) }
+    end
+
+    def purge_archive(queue_name, older_than:, batch_size: 1000)
+      full_name = config.queue_name(queue_name)
+      sanitized = full_name.gsub(/[^a-zA-Z0-9_]/, "")
+      total = 0
+
+      sql = "DELETE FROM pgmq.a_#{sanitized} " \
+            "WHERE ctid = ANY(ARRAY(SELECT ctid FROM pgmq.a_#{sanitized} WHERE enqueued_at < $1 LIMIT $2))"
+
+      loop do
+        deleted = synchronized do
+          @pgmq.pool.with { |conn| conn.exec_params(sql, [older_than, batch_size]).cmd_tuples }
+        end
+        total += deleted
+        break if deleted < batch_size
+      end
+
+      total
     end
 
     # Topic routing
     def bind_topic(pattern, queue_name)
       full_name = config.queue_name(queue_name)
       ensure_queue(queue_name)
-      @pgmq.bind_topic(pattern, full_name)
+      synchronized { @pgmq.bind_topic(pattern, full_name) }
     end
 
     def publish_to_topic(routing_key, payload, headers: nil, delay: 0)
-      @pgmq.produce_topic(
-        routing_key,
-        serialize(payload),
-        headers: headers && serialize(headers),
-        delay: delay
-      )
+      synchronized do
+        @pgmq.produce_topic(
+          routing_key,
+          serialize(payload),
+          headers: headers && serialize(headers),
+          delay: delay
+        )
+      end
     end
 
     def close
-      @pgmq.close
+      synchronized { @pgmq.close }
     end
 
     private
 
+    def ensure_single_queue(full_name)
+      return if @queues_created[full_name]
+
+      @queues_created.compute_if_absent(full_name) do
+        synchronized do
+          @pgmq.create(full_name)
+          @pgmq.enable_notify_insert(full_name, throttle_interval_ms: config.notify_throttle_ms) if config.listen_notify
+        end
+        true
+      end
+    end
+
+    # Serialize all PGMQ operations through a single mutex.
+    # PG::Connection is not thread-safe — concurrent access from worker
+    # threads causes segfaults and result corruption.
+    def synchronized(&)
+      @pgmq_mutex.synchronize(&)
+    end
+
     def serialize(data)
       case data
       when String

data/lib/pgbus/configuration.rb

@@ -11,7 +11,7 @@ module Pgbus
     attr_accessor :default_queue, :queue_prefix
 
     # Worker settings
-    attr_accessor :workers, :polling_interval, :visibility_timeout
+    attr_accessor :workers, :polling_interval, :visibility_timeout, :prefetch_limit
 
     # Worker recycling
     attr_accessor :max_jobs_per_worker, :max_memory_mb, :max_worker_lifetime
@@ -19,9 +19,22 @@ module Pgbus
     # Dispatcher settings
     attr_accessor :dispatch_interval
 
+    # Circuit breaker
+    attr_accessor :circuit_breaker_enabled, :circuit_breaker_threshold,
+                  :circuit_breaker_base_backoff, :circuit_breaker_max_backoff
+
     # Dead letter queue
     attr_accessor :max_retries, :dead_letter_queue_suffix
 
+    # Priority queues
+    attr_accessor :priority_levels, :default_priority
+
+    # Archive compaction
+    attr_accessor :archive_retention, :archive_compaction_interval, :archive_compaction_batch_size
+
+    # Transactional outbox
+    attr_accessor :outbox_enabled, :outbox_poll_interval, :outbox_batch_size, :outbox_retention
+
     # Event bus
     attr_accessor :idempotency_ttl
 
@@ -46,6 +59,9 @@ module Pgbus
     # Requires a matching entry in config/database.yml under the "pgbus" key.
     attr_accessor :connects_to
 
+    # Job stats
+    attr_accessor :stats_retention, :stats_enabled
+
     # Web dashboard
     attr_accessor :web_auth, :web_refresh_interval, :web_per_page, :web_live_updates, :web_data_source
 
@@ -62,15 +78,34 @@ module Pgbus
       @polling_interval = 0.1
       @visibility_timeout = 30
 
+      @prefetch_limit = nil
+
       @max_jobs_per_worker = nil
       @max_memory_mb = nil
       @max_worker_lifetime = nil
 
       @dispatch_interval = 1.0
 
+      @circuit_breaker_enabled = true
+      @circuit_breaker_threshold = 5
+      @circuit_breaker_base_backoff = 30
+      @circuit_breaker_max_backoff = 600
+
       @max_retries = 5
       @dead_letter_queue_suffix = "_dlq"
 
+      @priority_levels = nil
+      @default_priority = 1
+
+      @archive_retention = 7 * 24 * 3600 # 7 days
+      @archive_compaction_interval = 3600
+      @archive_compaction_batch_size = 1000
+
+      @outbox_enabled = false
+      @outbox_poll_interval = 1.0
+      @outbox_batch_size = 100
+      @outbox_retention = 24 * 3600 # 1 day
+
       @idempotency_ttl = 7 * 24 * 3600 # 7 days
 
       @logger = defined?(Rails) ? Rails.logger : Logger.new($stdout)
@@ -88,6 +123,9 @@ module Pgbus
       @skip_recurring = false
       @recurring_execution_retention = 7 * 24 * 3600 # 7 days
 
+      @stats_enabled = true
+      @stats_retention = 7 * 24 * 3600 # 7 days
+
       @connects_to = nil
 
       @web_auth = nil
@@ -105,6 +143,16 @@ module Pgbus
       "#{queue_name(name)}#{dead_letter_queue_suffix}"
     end
 
+    def priority_queue_name(name, priority)
+      "#{queue_name(name)}_p#{priority}"
+    end
+
+    def priority_queue_names(name)
+      return [queue_name(name)] unless priority_levels && priority_levels > 1
+
+      (0...priority_levels).map { |p| priority_queue_name(name, p) }
+    end
+
     VALID_PGMQ_SCHEMA_MODES = %i[auto extension embedded].freeze
 
     def pgmq_schema_mode=(mode)
@@ -128,6 +176,12 @@ module Pgbus
         raise ArgumentError, "worker threads must be > 0" unless threads.is_a?(Integer) && threads.positive?
       end
 
+      raise ArgumentError, "prefetch_limit must be > 0" if prefetch_limit && !(prefetch_limit.is_a?(Integer) && prefetch_limit.positive?)
+
+      if priority_levels && !(priority_levels.is_a?(Integer) && priority_levels >= 1 && priority_levels <= 10)
+        raise ArgumentError, "priority_levels must be an integer between 1 and 10"
+      end
+
       self
     end
 

data/lib/pgbus/dedup_cache.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+require "concurrent"
+
+module Pgbus
+  # Thread-safe in-memory dedup cache with TTL.
+  # Sits in front of the pgbus_processed_events table to avoid
+  # hitting the database for recently-seen (event_id, handler_class) pairs.
+  #
+  # Inspired by LavinMQ's built-in message deduplication with LRU + TTL.
+  class DedupCache
+    DEFAULT_MAX_SIZE = 10_000
+    DEFAULT_TTL = 300 # 5 minutes
+
+    def initialize(max_size: DEFAULT_MAX_SIZE, ttl: DEFAULT_TTL)
+      @max_size = max_size
+      @ttl = ttl
+      @cache = Concurrent::Map.new
+      @insertion_order = Concurrent::Array.new
+    end
+
+    # Returns true if this key was already seen (duplicate).
+    # Returns false if it's new (first time seen — caller should proceed).
+    def seen?(key)
+      entry = @cache[key]
+      return false unless entry
+
+      if expired?(entry)
+        evict(key)
+        return false
+      end
+
+      true
+    end
+
+    # Mark a key as seen. Call this after successfully claiming idempotency
+    # in the database so future lookups skip the DB.
+    def mark!(key)
+      already_present = @cache.key?(key)
+      evict_oldest if !already_present && @cache.size >= @max_size
+
+      @cache[key] = monotonic_now
+      @insertion_order << key unless already_present
+    end
+
+    def size
+      @cache.size
+    end
+
+    def clear!
+      @cache.clear
+      @insertion_order.clear
+    end
+
+    private
+
+    def expired?(timestamp)
+      (monotonic_now - timestamp) > @ttl
+    end
+
+    def evict(key)
+      @cache.delete(key)
+    end
+
+    def evict_oldest
+      while @cache.size >= @max_size && !@insertion_order.empty?
+        oldest_key = @insertion_order.shift
+        @cache.delete(oldest_key)
+      end
+    end
+
+    def monotonic_now
+      ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+    end
+  end
+end

data/lib/pgbus/engine.rb

@@ -34,6 +34,7 @@ module Pgbus
     initializer "pgbus.active_job" do
       ActiveSupport.on_load(:active_job) do
         include Pgbus::Concurrency
+        include Pgbus::Uniqueness
       end
     end
 

data/lib/pgbus/event_bus/handler.rb

@@ -11,6 +11,10 @@ module Pgbus
         def idempotent?
           @idempotent == true
         end
+
+        def dedup_cache
+          @dedup_cache ||= DedupCache.new
+        end
       end
 
       def process(message)
@@ -50,13 +54,20 @@ module Pgbus
       # Atomically claim idempotency: INSERT ... ON CONFLICT DO NOTHING.
       # Returns true if this handler claimed the event (row was inserted),
       # false if another handler already processed it (conflict, no insert).
+      #
+      # Uses an in-memory dedup cache to skip the DB for recently-seen events.
       def claim_idempotency?(event_id)
+        cache_key = "#{event_id}:#{self.class.name}"
+        return false if self.class.dedup_cache.seen?(cache_key)
+
         result = ProcessedEvent.insert(
           { event_id: event_id, handler_class: self.class.name, processed_at: Time.now.utc },
           unique_by: %i[event_id handler_class]
         )
-        # insert returns an InsertAll::Result; inserted row count > 0 means we claimed it
-        result.rows.any?
+
+        claimed = result.rows.any?
+        self.class.dedup_cache.mark!(cache_key)
+        claimed
       end
     end
   end