pgbus 0.1.3 → 0.1.5

data/lib/pgbus/process/dispatcher.rb

@@ -11,6 +11,8 @@ module Pgbus
       CONCURRENCY_INTERVAL = 300            # Run concurrency cleanup every 5 minutes
       BATCH_CLEANUP_INTERVAL = 3600         # Run batch cleanup every hour
       RECURRING_CLEANUP_INTERVAL = 3600     # Run recurring execution cleanup every hour
+      ARCHIVE_COMPACTION_INTERVAL = 3600    # Run archive compaction every hour
+      OUTBOX_CLEANUP_INTERVAL = 3600 # Run outbox cleanup every hour
 
       attr_reader :config
 
@@ -22,6 +24,8 @@ module Pgbus
         @last_concurrency_at = Time.now
         @last_batch_cleanup_at = Time.now
         @last_recurring_cleanup_at = Time.now
+        @last_archive_compaction_at = Time.now
+        @last_outbox_cleanup_at = Time.now
       end
 
       def run
@@ -64,6 +68,8 @@ module Pgbus
         run_if_due(now, :@last_concurrency_at, CONCURRENCY_INTERVAL) { cleanup_concurrency }
         run_if_due(now, :@last_batch_cleanup_at, BATCH_CLEANUP_INTERVAL) { cleanup_batches }
         run_if_due(now, :@last_recurring_cleanup_at, RECURRING_CLEANUP_INTERVAL) { cleanup_recurring_executions }
+        run_if_due(now, :@last_archive_compaction_at, archive_compaction_interval) { compact_archives }
+        run_if_due(now, :@last_outbox_cleanup_at, OUTBOX_CLEANUP_INTERVAL) { cleanup_outbox }
       end
 
       # Only update the timestamp when the block succeeds.
@@ -121,6 +127,46 @@ module Pgbus
         Pgbus.logger.warn { "[Pgbus] Batch cleanup failed: #{e.message}" }
       end
 
+      def cleanup_outbox
+        return unless config.outbox_enabled
+
+        retention = config.outbox_retention
+        return unless retention&.positive?
+
+        deleted = OutboxEntry.published_before(Time.now.utc - retention).delete_all
+        Pgbus.logger.debug { "[Pgbus] Cleaned up #{deleted} published outbox entries" } if deleted.positive?
+      rescue StandardError => e
+        Pgbus.logger.warn { "[Pgbus] Outbox cleanup failed: #{e.message}" }
+      end
+
+      def archive_compaction_interval
+        config.archive_compaction_interval || ARCHIVE_COMPACTION_INTERVAL
+      end
+
+      def compact_archives
+        retention = config.archive_retention
+        return unless retention&.positive?
+
+        cutoff = Time.now.utc - retention
+        batch_size = config.archive_compaction_batch_size || 1000
+        prefix = config.queue_prefix
+
+        conn = config.connects_to ? Pgbus::ApplicationRecord.connection : ActiveRecord::Base.connection
+        queue_names = conn.select_values("SELECT queue_name FROM pgmq.meta ORDER BY queue_name")
+
+        queue_names.each do |full_name|
+          next unless full_name.start_with?("#{prefix}_")
+
+          stripped = full_name.delete_prefix("#{prefix}_")
+          deleted = Pgbus.client.purge_archive(stripped, older_than: cutoff, batch_size: batch_size)
+          Pgbus.logger.debug { "[Pgbus] Compacted #{deleted} archive entries from #{full_name}" } if deleted.positive?
+        rescue StandardError => e
+          Pgbus.logger.warn { "[Pgbus] Archive compaction failed for #{full_name}: #{e.message}" }
+        end
+      rescue StandardError => e
+        Pgbus.logger.warn { "[Pgbus] Archive compaction failed: #{e.message}" }
+      end
+
       def cleanup_recurring_executions
         retention = config.recurring_execution_retention
         return unless retention&.positive?

data/lib/pgbus/process/heartbeat.rb

@@ -11,9 +11,10 @@ module Pgbus
 
       attr_reader :process_entry
 
-      def initialize(kind:, metadata: {})
+      def initialize(kind:, metadata: {}, on_beat: nil)
         @kind = kind
         @metadata = metadata
+        @on_beat = on_beat
         @timer = nil
       end
 
@@ -31,6 +32,7 @@ module Pgbus
       def beat
         return unless @process_id
 
+        @on_beat&.call
         ProcessEntry.where(id: @process_id).update_all(last_heartbeat_at: Time.current)
       rescue StandardError => e
         Pgbus.logger.warn { "[Pgbus] Heartbeat failed: #{e.message}" }

data/lib/pgbus/process/lifecycle.rb

@@ -0,0 +1,111 @@
+# frozen_string_literal: true
+
+require "concurrent"
+
+module Pgbus
+  module Process
+    # Thread-safe worker lifecycle state machine inspired by LavinMQ's QueueState.
+    #
+    # States:
+    #   :starting  → initial state, setting up
+    #   :running   → actively processing messages
+    #   :paused    → temporarily stopped (manual or circuit breaker)
+    #   :draining  → finishing in-flight work before stopping
+    #   :stopped   → terminal state
+    #
+    # Transitions:
+    #   starting → running
+    #   running  → paused | draining | stopped
+    #   paused   → running | draining | stopped
+    #   draining → stopped
+    class Lifecycle
+      STATES = %i[starting running paused draining stopped].freeze
+
+      TRANSITIONS = {
+        starting: %i[running stopped],
+        running: %i[paused draining stopped],
+        paused: %i[running draining stopped],
+        draining: %i[stopped],
+        stopped: []
+      }.freeze
+
+      attr_reader :state
+
+      def initialize
+        @state = :starting
+        @mutex = Mutex.new
+        @callbacks = Hash.new { |h, k| h[k] = [] }
+      end
+
+      def transition_to!(new_state)
+        @mutex.synchronize do
+          validate_transition!(new_state)
+          old_state = @state
+          @state = new_state
+          fire_callbacks(old_state, new_state)
+          new_state
+        end
+      end
+
+      def transition_to(new_state)
+        transition_to!(new_state)
+      rescue InvalidTransition
+        false
+      end
+
+      def on(event, &block)
+        @callbacks[event] << block
+      end
+
+      def starting?
+        @state == :starting
+      end
+
+      def running?
+        @state == :running
+      end
+
+      def paused?
+        @state == :paused
+      end
+
+      def draining?
+        @state == :draining
+      end
+
+      def stopped?
+        @state == :stopped
+      end
+
+      def active?
+        running? || paused?
+      end
+
+      def can_process?
+        running?
+      end
+
+      def terminal?
+        stopped?
+      end
+
+      private
+
+      def validate_transition!(new_state)
+        raise ArgumentError, "Unknown state: #{new_state}. Valid states: #{STATES.join(", ")}" unless STATES.include?(new_state)
+
+        return if TRANSITIONS[@state].include?(new_state)
+
+        raise InvalidTransition, "Cannot transition from #{@state} to #{new_state}. " \
+                                 "Valid transitions: #{TRANSITIONS[@state].join(", ")}"
+      end
+
+      def fire_callbacks(old_state, new_state)
+        @callbacks[:"#{old_state}_to_#{new_state}"].each(&:call)
+        @callbacks[:any].each { |cb| cb.call(old_state, new_state) }
+      end
+    end
+
+    class InvalidTransition < Pgbus::Error; end
+  end
+end

data/lib/pgbus/process/supervisor.rb

@@ -55,6 +55,9 @@ module Pgbus
 
         # Boot event consumers if configured
         boot_consumers
+
+        # Boot outbox poller if configured
+        boot_outbox_poller
       end
 
       def fork_worker(worker_config)
@@ -63,7 +66,7 @@ module Pgbus
 
         pid = fork do
           restore_signals
-          setup_child_signals
+          setup_child_process
           load_rails_app
           worker = Worker.new(queues: queues, threads: threads, config: config)
           worker.run
@@ -83,7 +86,7 @@ module Pgbus
       def fork_dispatcher
         pid = fork do
           restore_signals
-          setup_child_signals
+          setup_child_process
           load_rails_app
           dispatcher = Dispatcher.new(config: config)
           dispatcher.run
@@ -110,7 +113,7 @@ module Pgbus
       def fork_scheduler
         pid = fork do
           restore_signals
-          setup_child_signals
+          setup_child_process
           load_rails_app
           load_recurring_config
           scheduler = Recurring::Scheduler.new(config: config)
@@ -165,7 +168,7 @@ module Pgbus
 
         pid = fork do
           restore_signals
-          setup_child_signals
+          setup_child_process
           load_rails_app
           consumer = Consumer.new(topics: topics, threads: threads, config: config)
           consumer.run
@@ -182,6 +185,32 @@ module Pgbus
         Pgbus.logger.error { "[Pgbus] Fork failed for consumer: #{e.message}" }
       end
 
+      def boot_outbox_poller
+        return unless config.outbox_enabled
+
+        fork_outbox_poller
+      end
+
+      def fork_outbox_poller
+        pid = fork do
+          restore_signals
+          setup_child_process
+          load_rails_app
+          poller = Outbox::Poller.new(config: config)
+          poller.run
+        end
+
+        unless pid
+          Pgbus.logger.error { "[Pgbus] Failed to fork outbox poller" }
+          return
+        end
+
+        @forks[pid] = { type: :outbox_poller }
+        Pgbus.logger.info { "[Pgbus] Forked outbox poller pid=#{pid}" }
+      rescue Errno::EAGAIN, Errno::ENOMEM => e
+        Pgbus.logger.error { "[Pgbus] Fork failed for outbox poller: #{e.message}" }
+      end
+
       def monitor_loop
         loop do
           break if @shutting_down && @forks.empty?
@@ -223,6 +252,8 @@ module Pgbus
           fork_scheduler
         when :consumer
           fork_consumer(info[:config])
+        when :outbox_poller
+          fork_outbox_poller
         end
       end
 
@@ -234,7 +265,11 @@ module Pgbus
         end
       end
 
-      def setup_child_signals
+      def setup_child_process
+        # Reset the PGMQ client so this forked process gets a fresh
+        # PG::Connection instead of inheriting the parent's (which is
+        # in undefined state post-fork and not thread-safe to share).
+        Pgbus.reset_client!
         %w[INT TERM QUIT].each do |sig|
           trap(sig) { @shutting_down = true }
         end

data/lib/pgbus/process/worker.rb

@@ -13,31 +13,44 @@ module Pgbus
         @queues = Array(queues)
         @threads = threads
         @config = config
-        @shutting_down = false
+        @lifecycle = Lifecycle.new
         @jobs_processed = Concurrent::AtomicFixnum.new(0)
         @jobs_failed = Concurrent::AtomicFixnum.new(0)
+        @in_flight = Concurrent::AtomicFixnum.new(0)
+        @rate_counter = RateCounter.new(:processed, :failed, :dequeued)
         @started_at = Time.now
         @executor = Pgbus::ActiveJob::Executor.new
         @pool = Concurrent::FixedThreadPool.new(threads)
+        @circuit_breaker = Pgbus::CircuitBreaker.new(config: config)
       end
 
       def stats
-        { jobs_processed: @jobs_processed.value, jobs_failed: @jobs_failed.value, started_at: @started_at }
+        {
+          jobs_processed: @jobs_processed.value,
+          jobs_failed: @jobs_failed.value,
+          in_flight: @in_flight.value,
+          state: @lifecycle.state,
+          rates: @rate_counter.rates,
+          started_at: @started_at
+        }
       end
 
       def run
         setup_signals
         start_heartbeat
         resolve_wildcard_queues
+        @lifecycle.transition_to!(:running)
         Pgbus.logger.info { "[Pgbus] Worker started: queues=#{queues.join(",")} threads=#{threads} pid=#{::Process.pid}" }
 
         loop do
           process_signals
-          break if @shutting_down && @pool.queue_length.zero?
-          break if recycle_needed? && @pool.queue_length.zero?
+          check_recycle
 
-          claim_and_execute unless @shutting_down || recycle_needed?
-          interruptible_sleep(config.polling_interval) if (@shutting_down || recycle_needed?) && !@pool.queue_length.zero?
+          break if @lifecycle.stopped?
+          break if @lifecycle.draining? && @pool.queue_length.zero?
+
+          claim_and_execute if @lifecycle.can_process?
+          interruptible_sleep(config.polling_interval) if @lifecycle.draining? || @lifecycle.paused?
         end
 
         shutdown
@@ -45,12 +58,12 @@ module Pgbus
 
       def graceful_shutdown
         Pgbus.logger.info { "[Pgbus] Worker shutting down gracefully..." }
-        @shutting_down = true
+        @lifecycle.transition_to(:draining)
       end
 
       def immediate_shutdown
         Pgbus.logger.warn { "[Pgbus] Worker shutting down immediately!" }
-        @shutting_down = true
+        @lifecycle.transition_to!(:stopped)
         @pool.kill
       end
 
@@ -60,6 +73,13 @@ module Pgbus
         idle = @pool.max_length - @pool.queue_length
         return interruptible_sleep(config.polling_interval) if idle <= 0
 
+        if config.prefetch_limit
+          available = config.prefetch_limit - @in_flight.value
+          return interruptible_sleep(config.polling_interval) if available <= 0
+
+          idle = [idle, available].min
+        end
+
         tagged_messages = fetch_messages(idle)
 
         if tagged_messages.empty?
@@ -67,21 +87,28 @@ module Pgbus
           return
         end
 
-        tagged_messages.each do |queue_name, message|
-          @pool.post { process_message(message, queue_name) }
+        @rate_counter.increment(:dequeued, tagged_messages.size)
+        tagged_messages.each do |queue_name, message, source_queue|
+          @in_flight.increment
+          @pool.post { process_message(message, queue_name, source_queue: source_queue) }
         end
       end
 
       # Returns an array of [queue_name, message] pairs so we always know
       # which queue each message came from (PGMQ messages don't carry this).
       def fetch_messages(qty)
-        if queues.size == 1
-          queue = queues.first
+        active_queues = queues.reject { |q| @circuit_breaker.paused?(q) }
+        return [] if active_queues.empty?
+
+        if priority_enabled?
+          fetch_prioritized(active_queues, qty)
+        elsif active_queues.size == 1
+          queue = active_queues.first
           messages = Pgbus.client.read_batch(queue, qty: qty) || []
           messages.map { |m| [queue, m] }
         else
-          per_queue = [(qty / queues.size.to_f).ceil, 1].max
-          queues.flat_map do |q|
+          per_queue = [(qty / active_queues.size.to_f).ceil, 1].max
+          active_queues.flat_map do |q|
             (Pgbus.client.read_batch(q, qty: per_queue) || []).map { |m| [q, m] }
           end.first(qty)
         end
@@ -90,13 +117,45 @@ module Pgbus
         []
       end
 
-      def process_message(message, queue_name)
-        result = @executor.execute(message, queue_name)
+      def fetch_prioritized(active_queues, qty)
+        remaining = qty
+        results = []
+
+        active_queues.each do |q|
+          break if remaining <= 0
+
+          batch = Pgbus.client.read_batch_prioritized(q, qty: remaining)
+          batch.each do |physical_queue, message|
+            results << [q, message, physical_queue]
+          end
+          remaining -= batch.size
+        end
+
+        results
+      end
+
+      def priority_enabled?
+        config.priority_levels && config.priority_levels > 1
+      end
+
+      def process_message(message, queue_name, source_queue: nil)
+        result = @executor.execute(message, queue_name, source_queue: source_queue)
         @jobs_processed.increment
-        @jobs_failed.increment if result == :failed
+        @rate_counter.increment(:processed)
+        if result == :failed
+          @jobs_failed.increment
+          @rate_counter.increment(:failed)
+          @circuit_breaker.record_failure(queue_name)
+        else
+          @circuit_breaker.record_success(queue_name)
+        end
       rescue StandardError => e
         @jobs_failed.increment
+        @rate_counter.increment(:failed)
+        @circuit_breaker.record_failure(queue_name)
         Pgbus.logger.error { "[Pgbus] Unhandled error processing message: #{e.message}" }
+      ensure
+        @in_flight.decrement
       end
 
       # Resolve "*" to all non-DLQ queues from pgmq.meta, stripping the prefix.
@@ -107,7 +166,8 @@ module Pgbus
         dlq_suffix = config.dead_letter_queue_suffix
         prefix = "#{config.queue_prefix}_"
 
-        all_queues = ActiveRecord::Base.connection.select_values("SELECT queue_name FROM pgmq.meta ORDER BY queue_name")
+        conn = Pgbus.configuration.connects_to ? Pgbus::ApplicationRecord.connection : ActiveRecord::Base.connection
+        all_queues = conn.select_values("SELECT queue_name FROM pgmq.meta ORDER BY queue_name")
         resolved = all_queues
                    .reject { |q| q.end_with?(dlq_suffix) }
                    .map { |q| q.delete_prefix(prefix) }
@@ -124,6 +184,12 @@ module Pgbus
         @queues = [config.default_queue]
       end
 
+      def check_recycle
+        return unless @lifecycle.running? && recycle_needed?
+
+        @lifecycle.transition_to(:draining)
+      end
+
       def recycle_needed?
         exceeded_max_jobs? || exceeded_max_memory? || exceeded_max_lifetime?
       end
@@ -164,7 +230,8 @@ module Pgbus
       def start_heartbeat
         @heartbeat = Heartbeat.new(
           kind: "worker",
-          metadata: { queues: queues, threads: threads, pid: ::Process.pid }
+          metadata: { queues: queues, threads: threads, pid: ::Process.pid },
+          on_beat: -> { @rate_counter.snapshot! }
         )
         @heartbeat.start
       end

data/lib/pgbus/rate_counter.rb

@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+require "concurrent"
+
+module Pgbus
+  # Thread-safe rate counter inspired by LavinMQ's rate_stats macro.
+  # Tracks absolute counts via AtomicFixnum and computes rates as
+  # deltas between periodic snapshots.
+  #
+  # Usage:
+  #   counter = RateCounter.new(:enqueued, :dequeued, :failed)
+  #   counter.increment(:enqueued)
+  #   counter.snapshot!          # call periodically (e.g. every 5s)
+  #   counter.rate(:enqueued)    # => msgs/s since last snapshot
+  #   counter.rates              # => { enqueued: 12.4, dequeued: 10.1, failed: 0.2 }
+  class RateCounter
+    attr_reader :names
+
+    def initialize(*names)
+      @names = names.map(&:to_sym)
+      @counters = {}
+      @last_values = {}
+      @rates = {}
+      @last_snapshot_at = monotonic_now
+
+      @names.each do |name|
+        @counters[name] = Concurrent::AtomicFixnum.new(0)
+        @last_values[name] = 0
+        @rates[name] = 0.0
+      end
+    end
+
+    def increment(name, delta = 1)
+      @counters.fetch(name).increment(delta)
+    end
+
+    def count(name)
+      @counters.fetch(name).value
+    end
+
+    def rate(name)
+      @rates.fetch(name)
+    end
+
+    def rates
+      @names.to_h { |name| [name, @rates[name]] }
+    end
+
+    def counts
+      @names.to_h { |name| [name, @counters[name].value] }
+    end
+
+    def snapshot!
+      now = monotonic_now
+      elapsed = now - @last_snapshot_at
+      return if elapsed <= 0
+
+      @names.each do |name|
+        current = @counters[name].value
+        delta = current - @last_values[name]
+        @rates[name] = (delta / elapsed).round(1)
+        @last_values[name] = current
+      end
+
+      @last_snapshot_at = now
+    end
+
+    def to_h
+      {
+        counts: counts,
+        rates: rates
+      }
+    end
+
+    private
+
+    def monotonic_now
+      ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+    end
+  end
+end

data/lib/pgbus/recurring/schedule.rb

@@ -87,7 +87,7 @@ module Pgbus
       end
 
       def resolve_queue(task)
-        @config.queue_name(task.queue_name || @config.default_queue)
+        task.queue_name || @config.default_queue
       end
 
       def build_headers(task, run_at)

data/lib/pgbus/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Pgbus
-  VERSION = "0.1.3"
+  VERSION = "0.1.5"
 end