pgbus 0.0.1

data/lib/generators/pgbus/templates/pgbus.yml.erb

@@ -0,0 +1,74 @@
+# Pgbus configuration
+# https://github.com/mhenrixon/pgbus
+
+default: &default
+  # Queue prefix for all PGMQ queues
+  queue_prefix: pgbus
+
+  # Default queue name (without prefix)
+  default_queue: default
+
+  # Connection pool for PGMQ client
+  pool_size: 5
+  pool_timeout: 5
+
+  # Use PostgreSQL LISTEN/NOTIFY for instant job wake-up
+  listen_notify: true
+  notify_throttle_ms: 250
+
+  # Visibility timeout in seconds (how long a message is invisible after being read)
+  visibility_timeout: 30
+
+  # Dead letter queue
+  max_retries: 5
+
+  # Idempotency deduplication TTL (seconds)
+  idempotency_ttl: 604800  # 7 days
+
+  # Worker definitions
+  workers:
+    - queues:
+        - critical
+        - default
+      threads: 5
+    - queues:
+        - low
+      threads: 2
+
+  # Worker recycling (prevents memory bloat)
+  max_jobs_per_worker: 10000
+  max_memory_mb: 512
+  # max_worker_lifetime: 3600  # seconds
+
+  # Dispatcher settings (maintenance tasks)
+  dispatch_interval: 1.0
+
+  # Event consumers (uncomment to enable event bus)
+  # event_consumers:
+  #   - topics:
+  #       - "orders.#"
+  #     threads: 3
+  #   - topics:
+  #       - "notifications.#"
+  #     threads: 1
+
+development:
+  <<: *default
+  workers:
+    - queues:
+        - default
+      threads: 2
+  max_jobs_per_worker: ~
+  max_memory_mb: ~
+
+test:
+  <<: *default
+  workers:
+    - queues:
+        - default
+      threads: 1
+  polling_interval: 0.01
+  visibility_timeout: 5
+
+production:
+  <<: *default

data/lib/generators/pgbus/templates/pgbus_binstub.erb

@@ -0,0 +1,7 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require_relative "../config/environment"
+require "pgbus/cli"
+
+Pgbus::CLI.start(ARGV)

data/lib/pgbus/active_job/adapter.rb

@@ -0,0 +1,109 @@
+# frozen_string_literal: true
+
+require "active_job"
+
+module Pgbus
+  module ActiveJob
+    class Adapter
+      def enqueue(active_job)
+        queue = active_job.queue_name || Pgbus.configuration.default_queue
+        payload_hash = JSON.parse(Serializer.serialize_job(active_job))
+        payload_hash = Concurrency.inject_metadata(active_job, payload_hash)
+        payload_hash = inject_batch_metadata(payload_hash)
+
+        enqueue_with_concurrency(active_job, queue, payload_hash)
+      end
+
+      def enqueue_at(active_job, timestamp)
+        queue = active_job.queue_name || Pgbus.configuration.default_queue
+        payload_hash = JSON.parse(Serializer.serialize_job(active_job))
+        payload_hash = Concurrency.inject_metadata(active_job, payload_hash)
+        payload_hash = inject_batch_metadata(payload_hash)
+        delay = [(timestamp - Time.now.to_f).ceil, 0].max
+
+        enqueue_with_concurrency(active_job, queue, payload_hash, delay: delay)
+      end
+
+      def enqueue_all(active_jobs)
+        active_jobs.group_by { |j| j.queue_name || Pgbus.configuration.default_queue }.each do |queue, jobs|
+          enqueue_immediate(queue, jobs.reject { |j| j.scheduled_at && j.scheduled_at > Time.now })
+          jobs.select { |j| j.scheduled_at && j.scheduled_at > Time.now }.each { |j| enqueue_at(j, j.scheduled_at.to_f) }
+        end
+
+        active_jobs.count
+      end
+
+      private
+
+      def enqueue_with_concurrency(active_job, queue, payload_hash, delay: 0)
+        key = Concurrency.extract_key(payload_hash)
+        concurrency = concurrency_config(active_job)
+
+        if key && concurrency
+          result = Concurrency::Semaphore.acquire(key, concurrency[:limit], concurrency[:duration])
+
+          if result == :acquired
+            msg_id = Pgbus.client.send_message(queue, payload_hash, delay: delay)
+            active_job.provider_job_id = msg_id
+          else
+            handle_conflict(concurrency, active_job, key, queue, payload_hash)
+          end
+        else
+          msg_id = Pgbus.client.send_message(queue, payload_hash, delay: delay)
+          active_job.provider_job_id = msg_id
+        end
+
+        active_job
+      end
+
+      def concurrency_config(active_job)
+        active_job.class.respond_to?(:pgbus_concurrency) && active_job.class.pgbus_concurrency
+      end
+
+      def handle_conflict(concurrency, active_job, key, queue, payload_hash)
+        case concurrency[:on_conflict]
+        when :block
+          Concurrency::BlockedExecution.insert(
+            concurrency_key: key,
+            queue_name: queue,
+            payload: payload_hash,
+            priority: active_job.try(:priority) || 0,
+            duration: concurrency[:duration]
+          )
+        when :discard
+          Pgbus.logger.info { "[Pgbus] Discarding job #{active_job.class.name}: concurrency limit for #{key}" }
+        when :raise
+          raise ConcurrencyLimitExceeded, "Concurrency limit reached for key: #{key}"
+        end
+      end
+
+      def inject_batch_metadata(payload_hash)
+        batch_id = Thread.current[:pgbus_batch_id]
+        return payload_hash unless batch_id
+
+        Thread.current[:pgbus_batch_job_count] = (Thread.current[:pgbus_batch_job_count] || 0) + 1
+        payload_hash.merge(Batch::METADATA_KEY => batch_id)
+      end
+
+      def enqueue_immediate(queue, jobs)
+        return if jobs.empty?
+
+        payloads = jobs.map { |j| JSON.parse(Serializer.serialize_job(j)) }
+        msg_ids = Pgbus.client.send_batch(queue, payloads)
+
+        unless msg_ids.is_a?(Array) && msg_ids.size == jobs.size
+          raise "Pgbus batch enqueue failed: expected #{jobs.size} ids, got #{msg_ids&.size || 0}"
+        end
+
+        jobs.zip(msg_ids).each { |job, id| job.provider_job_id = id }
+      end
+
+      def enqueue_after_transaction_commit?
+        true
+      end
+    end
+  end
+end
+
+# Register the adapter with ActiveJob (register method added in Rails 7.2+)
+ActiveJob::QueueAdapters.register(:pgbus, Pgbus::ActiveJob::Adapter) if ActiveJob::QueueAdapters.respond_to?(:register)

data/lib/pgbus/active_job/executor.rb

@@ -0,0 +1,107 @@
+# frozen_string_literal: true
+
+module Pgbus
+  module ActiveJob
+    class Executor
+      attr_reader :client, :config
+
+      def initialize(client: Pgbus.client, config: Pgbus.configuration)
+        @client = client
+        @config = config
+      end
+
+      def execute(message, queue_name)
+        payload = JSON.parse(message.message)
+        read_count = message.read_ct.to_i
+
+        if read_count > config.max_retries
+          handle_dead_letter(message, queue_name, payload)
+          signal_concurrency(payload)
+          signal_batch_discarded(payload)
+          return :dead_lettered
+        end
+
+        job = ::ActiveJob::Base.deserialize(payload)
+        execute_job(job)
+        client.archive_message(queue_name, message.msg_id.to_i)
+        signal_concurrency(payload)
+        signal_batch_completed(payload)
+        instrument("pgbus.job_completed", queue: queue_name, job_class: payload["job_class"])
+        :success
+      rescue StandardError => e
+        handle_failure(message, queue_name, e)
+        instrument("pgbus.job_failed", queue: queue_name, job_class: payload&.dig("job_class"), error: e.class.name)
+        # Don't signal concurrency on transient failure — the job will be retried.
+        # Semaphore is released only on success or dead-lettering.
+        :failed
+      end
+
+      private
+
+      def execute_job(job)
+        if defined?(Rails) && Rails.application
+          Rails.application.executor.wrap { job.perform_now }
+        else
+          job.perform_now
+        end
+      end
+
+      def handle_failure(_message, _queue_name, error)
+        Pgbus.logger.error { "[Pgbus] Job failed: #{error.class}: #{error.message}" }
+        Pgbus.logger.debug { error.backtrace&.join("\n") }
+
+        # Message visibility timeout will expire and it becomes available again.
+        # read_ct tracks delivery attempts — when it exceeds max_retries,
+        # the next read will route to DLQ.
+      end
+
+      def instrument(event_name, payload = {})
+        return unless defined?(ActiveSupport::Notifications)
+
+        ActiveSupport::Notifications.instrument(event_name, payload)
+      rescue StandardError => e
+        Pgbus.logger.debug { "[Pgbus] Notification failure #{event_name}: #{e.class}: #{e.message}" }
+      end
+
+      def signal_concurrency(payload)
+        key = Concurrency.extract_key(payload)
+        return unless key
+
+        Concurrency::Semaphore.release(key)
+
+        released = Concurrency::BlockedExecution.release_next(key)
+        return unless released
+
+        client.send_message(released[:queue_name], released[:payload])
+      rescue StandardError => e
+        Pgbus.logger.warn { "[Pgbus] Concurrency signal failed: #{e.message}" }
+      end
+
+      def signal_batch_completed(payload)
+        batch_id = payload[Batch::METADATA_KEY]
+        return unless batch_id
+
+        Batch.job_completed(batch_id)
+      rescue StandardError => e
+        Pgbus.logger.warn { "[Pgbus] Batch completion signal failed: #{e.message}" }
+      end
+
+      def signal_batch_discarded(payload)
+        batch_id = payload[Batch::METADATA_KEY]
+        return unless batch_id
+
+        Batch.job_discarded(batch_id)
+      rescue StandardError => e
+        Pgbus.logger.warn { "[Pgbus] Batch discard signal failed: #{e.message}" }
+      end
+
+      def handle_dead_letter(message, queue_name, payload)
+        Pgbus.logger.warn do
+          job_class = payload["job_class"] || "unknown"
+          "[Pgbus] Moving job #{job_class} to dead letter queue after #{message.read_ct} attempts"
+        end
+        client.move_to_dead_letter(queue_name, message)
+      end
+    end
+  end
+end

data/lib/pgbus/batch.rb

@@ -0,0 +1,153 @@
+# frozen_string_literal: true
+
+require "securerandom"
+require "json"
+
+module Pgbus
+  class Batch
+    METADATA_KEY = "pgbus_batch_id"
+
+    attr_reader :batch_id, :properties, :description,
+                :on_finish, :on_success, :on_discard
+
+    def initialize(on_finish: nil, on_success: nil, on_discard: nil, description: nil, properties: {})
+      @batch_id = SecureRandom.uuid
+      @on_finish = on_finish
+      @on_success = on_success
+      @on_discard = on_discard
+      @description = description
+      @properties = properties
+      @job_count = 0
+    end
+
+    # Enqueue a group of jobs as a batch.
+    # Jobs enqueued inside the block are tracked as part of this batch.
+    def enqueue(&)
+      create_record
+      count_jobs(&)
+      update_total
+      self
+    end
+
+    # Record a completed job. Returns the batch status after update.
+    def self.job_completed(batch_id)
+      update_counter(batch_id, "completed_jobs")
+    end
+
+    # Record a discarded (dead-lettered) job. Returns the batch status after update.
+    def self.job_discarded(batch_id)
+      update_counter(batch_id, "discarded_jobs")
+    end
+
+    # Find a batch record by ID. Returns a hash or nil.
+    def self.find(batch_id)
+      return nil unless defined?(ActiveRecord::Base)
+
+      result = ActiveRecord::Base.connection.exec_query(
+        "SELECT * FROM pgbus_batches WHERE batch_id = $1",
+        "Pgbus Batch Find",
+        [batch_id]
+      )
+      result.first
+    end
+
+    # Delete finished batches older than the given threshold.
+    def self.cleanup(older_than:)
+      return 0 unless defined?(ActiveRecord::Base)
+
+      result = ActiveRecord::Base.connection.exec_query(
+        "DELETE FROM pgbus_batches WHERE status = 'finished' AND finished_at < $1 RETURNING id",
+        "Pgbus Batch Cleanup",
+        [older_than]
+      )
+      result.to_a.size
+    end
+
+    private
+
+    def create_record
+      return unless defined?(ActiveRecord::Base)
+
+      ActiveRecord::Base.connection.exec_query(
+        <<~SQL,
+          INSERT INTO pgbus_batches
+            (batch_id, description, on_finish_class, on_success_class, on_discard_class, properties, status)
+          VALUES ($1, $2, $3, $4, $5, $6, 'pending')
+        SQL
+        "Pgbus Batch Create",
+        [batch_id, description, on_finish&.name, on_success&.name, on_discard&.name, JSON.generate(properties)]
+      )
+    end
+
+    def count_jobs(&)
+      Thread.current[:pgbus_batch_id] = batch_id
+      @job_count = 0
+
+      yield
+
+      @job_count = Thread.current[:pgbus_batch_job_count] || 0
+    ensure
+      Thread.current[:pgbus_batch_id] = nil
+      Thread.current[:pgbus_batch_job_count] = nil
+    end
+
+    def update_total
+      return unless defined?(ActiveRecord::Base)
+
+      ActiveRecord::Base.connection.exec_query(
+        "UPDATE pgbus_batches SET total_jobs = $1, status = 'processing' WHERE batch_id = $2",
+        "Pgbus Batch Update Total",
+        [@job_count, batch_id]
+      )
+    end
+
+    class << self
+      private
+
+      def update_counter(batch_id, column)
+        return nil unless defined?(ActiveRecord::Base)
+
+        result = ActiveRecord::Base.connection.exec_query(
+          <<~SQL,
+            UPDATE pgbus_batches
+            SET #{column} = #{column} + 1,
+                status = CASE
+                  WHEN completed_jobs + discarded_jobs + 1 = total_jobs THEN 'finished'
+                  ELSE status
+                END,
+                finished_at = CASE
+                  WHEN completed_jobs + discarded_jobs + 1 = total_jobs THEN NOW()
+                  ELSE finished_at
+                END
+            WHERE batch_id = $1
+            RETURNING status, total_jobs, completed_jobs, discarded_jobs, on_finish_class, on_success_class, on_discard_class, properties
+          SQL
+          "Pgbus Batch Counter",
+          [batch_id]
+        )
+
+        row = result.first
+        return nil unless row
+
+        fire_callbacks(row) if row["status"] == "finished"
+        row
+      end
+
+      def fire_callbacks(row)
+        properties = JSON.parse(row["properties"] || "{}")
+        all_succeeded = row["discarded_jobs"].to_i.zero?
+
+        enqueue_callback(row["on_finish_class"], properties) if row["on_finish_class"]
+        enqueue_callback(row["on_success_class"], properties) if row["on_success_class"] && all_succeeded
+        enqueue_callback(row["on_discard_class"], properties) if row["on_discard_class"] && !all_succeeded
+      end
+
+      def enqueue_callback(class_name, properties)
+        job_class = class_name.constantize
+        job_class.perform_later(properties)
+      rescue NameError => e
+        Pgbus.logger.error { "[Pgbus] Batch callback class not found: #{class_name}: #{e.message}" }
+      end
+    end
+  end
+end

data/lib/pgbus/cli.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+module Pgbus
+  module CLI
+    module_function
+
+    def start(args)
+      command = args.first || "help"
+
+      case command
+      when "start"
+        start_supervisor
+      when "status"
+        show_status
+      when "queues"
+        list_queues
+      when "version"
+        puts "pgbus #{Pgbus::VERSION}"
+      when "help", "--help", "-h"
+        print_help
+      else
+        puts "Unknown command: #{command}"
+        print_help
+        exit 1
+      end
+    end
+
+    def start_supervisor
+      Pgbus.logger.info { "[Pgbus] Starting Pgbus #{Pgbus::VERSION}..." }
+
+      supervisor = Process::Supervisor.new
+      supervisor.run
+    end
+
+    def show_status
+      if defined?(ActiveRecord::Base)
+        processes = ActiveRecord::Base.connection.execute(
+          "SELECT kind, hostname, pid, metadata, last_heartbeat_at FROM pgbus_processes ORDER BY kind, created_at"
+        )
+
+        if processes.none?
+          puts "No Pgbus processes running."
+          return
+        end
+
+        puts "KIND         HOST                 PID      HEARTBEAT                      METADATA"
+        puts "-" * 100
+        processes.each do |p|
+          puts format("%-12s %-20s %-8s %-30s %s",
+                      p["kind"], p["hostname"], p["pid"], p["last_heartbeat_at"], p["metadata"])
+        end
+      else
+        puts "ActiveRecord not available. Run from a Rails context."
+      end
+    end
+
+    def list_queues
+      Pgbus.client.list_queues
+      metrics = Pgbus.client.metrics
+
+      puts "QUEUE                                    DEPTH      VISIBLE    OLDEST (s)      TOTAL          "
+      puts "-" * 95
+
+      Array(metrics).each do |m|
+        puts format("%-40s %-10s %-10s %-15s %-15s",
+                    m.queue_name, m.queue_length, m.queue_visible_length,
+                    m.oldest_msg_age_sec || "-", m.total_messages)
+      end
+    end
+
+    def print_help
+      puts <<~HELP
+        Usage: pgbus <command>
+
+        Commands:
+          start    Start the Pgbus supervisor (workers + dispatcher)
+          status   Show running Pgbus processes
+          queues   List queues with metrics
+          version  Show version
+          help     Show this help
+      HELP
+    end
+  end
+end

data/lib/pgbus/client.rb

@@ -0,0 +1,162 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Pgbus
+  class Client
+    attr_reader :pgmq, :config
+
+    def initialize(config = Pgbus.configuration)
+      require "pgmq-ruby"
+      @config = config
+      @pgmq = PGMQ::Client.new(
+        config.connection_options,
+        pool_size: config.pool_size,
+        pool_timeout: config.pool_timeout
+      )
+      @queues_created = {}
+      @mutex = Mutex.new
+    end
+
+    def ensure_queue(name)
+      full_name = config.queue_name(name)
+      @mutex.synchronize do
+        return if @queues_created[full_name]
+
+        @pgmq.create(full_name)
+        @pgmq.enable_notify_insert(full_name, throttle_interval_ms: config.notify_throttle_ms) if config.listen_notify
+        @queues_created[full_name] = true
+      end
+    end
+
+    def ensure_dead_letter_queue(name)
+      dlq_name = config.dead_letter_queue_name(name)
+      @mutex.synchronize do
+        return if @queues_created[dlq_name]
+
+        @pgmq.create(dlq_name)
+        @queues_created[dlq_name] = true
+      end
+    end
+
+    def send_message(queue_name, payload, headers: nil, delay: 0)
+      full_name = config.queue_name(queue_name)
+      ensure_queue(queue_name)
+      @pgmq.produce(full_name, serialize(payload), headers: headers && serialize(headers), delay: delay)
+    end
+
+    def send_batch(queue_name, payloads, headers: nil, delay: 0)
+      full_name = config.queue_name(queue_name)
+      ensure_queue(queue_name)
+      serialized = payloads.map { |p| serialize(p) }
+      serialized_headers = headers&.map { |h| serialize(h) }
+      @pgmq.produce_batch(full_name, serialized, headers: serialized_headers, delay: delay)
+    end
+
+    def read_message(queue_name, vt: nil)
+      full_name = config.queue_name(queue_name)
+      @pgmq.read(full_name, vt: vt || config.visibility_timeout)
+    end
+
+    def read_batch(queue_name, qty:, vt: nil)
+      full_name = config.queue_name(queue_name)
+      @pgmq.read_batch(full_name, vt: vt || config.visibility_timeout, qty: qty)
+    end
+
+    def read_with_poll(queue_name, qty:, vt: nil, max_poll_seconds: 5, poll_interval_ms: 100)
+      full_name = config.queue_name(queue_name)
+      @pgmq.read_with_poll(
+        full_name,
+        vt: vt || config.visibility_timeout,
+        qty: qty,
+        max_poll_seconds: max_poll_seconds,
+        poll_interval_ms: poll_interval_ms
+      )
+    end
+
+    def delete_message(queue_name, msg_id)
+      full_name = config.queue_name(queue_name)
+      @pgmq.delete(full_name, msg_id)
+    end
+
+    def archive_message(queue_name, msg_id)
+      full_name = config.queue_name(queue_name)
+      @pgmq.archive(full_name, msg_id)
+    end
+
+    def extend_visibility(queue_name, msg_id, vt:)
+      full_name = config.queue_name(queue_name)
+      @pgmq.set_vt(full_name, msg_id, vt: vt)
+    end
+
+    def set_visibility_timeout(queue_name, msg_id, vt:)
+      @pgmq.set_vt(queue_name, msg_id, vt: vt)
+    end
+
+    def delete_from_queue(queue_name, msg_id)
+      @pgmq.delete(queue_name, msg_id)
+    end
+
+    def transaction(&)
+      @pgmq.transaction(&)
+    end
+
+    def move_to_dead_letter(queue_name, message)
+      ensure_dead_letter_queue(queue_name)
+      dlq_name = config.dead_letter_queue_name(queue_name)
+      full_queue = config.queue_name(queue_name)
+
+      @pgmq.transaction do |txn|
+        txn.produce(dlq_name, message.message, headers: message.headers)
+        txn.delete(full_queue, message.msg_id.to_i)
+      end
+    end
+
+    def metrics(queue_name = nil)
+      if queue_name
+        @pgmq.metrics(config.queue_name(queue_name))
+      else
+        @pgmq.metrics_all
+      end
+    end
+
+    def list_queues
+      @pgmq.list_queues
+    end
+
+    def purge_queue(queue_name)
+      @pgmq.purge_queue(config.queue_name(queue_name))
+    end
+
+    # Topic routing
+    def bind_topic(pattern, queue_name)
+      full_name = config.queue_name(queue_name)
+      ensure_queue(queue_name)
+      @pgmq.bind_topic(pattern, full_name)
+    end
+
+    def publish_to_topic(routing_key, payload, headers: nil, delay: 0)
+      @pgmq.produce_topic(
+        routing_key,
+        serialize(payload),
+        headers: headers && serialize(headers),
+        delay: delay
+      )
+    end
+
+    def close
+      @pgmq.close
+    end
+
+    private
+
+    def serialize(data)
+      case data
+      when String
+        data
+      else
+        JSON.generate(data)
+      end
+    end
+  end
+end