pgbus 0.0.1

data/lib/pgbus/process/worker.rb

@@ -0,0 +1,153 @@
+# frozen_string_literal: true
+
+require "concurrent"
+
+module Pgbus
+  module Process
+    class Worker
+      include SignalHandler
+
+      attr_reader :queues, :threads, :config
+
+      def initialize(queues:, threads: 5, config: Pgbus.configuration)
+        @queues = Array(queues)
+        @threads = threads
+        @config = config
+        @shutting_down = false
+        @jobs_processed = Concurrent::AtomicFixnum.new(0)
+        @jobs_failed = Concurrent::AtomicFixnum.new(0)
+        @started_at = Time.now
+        @executor = Pgbus::ActiveJob::Executor.new
+        @pool = Concurrent::FixedThreadPool.new(threads)
+      end
+
+      def stats
+        { jobs_processed: @jobs_processed.value, jobs_failed: @jobs_failed.value, started_at: @started_at }
+      end
+
+      def run
+        setup_signals
+        start_heartbeat
+        Pgbus.logger.info { "[Pgbus] Worker started: queues=#{queues.join(",")} threads=#{threads} pid=#{::Process.pid}" }
+
+        loop do
+          break if @shutting_down
+          break if recycle_needed?
+
+          process_signals
+          claim_and_execute
+        end
+
+        shutdown
+      end
+
+      def graceful_shutdown
+        Pgbus.logger.info { "[Pgbus] Worker shutting down gracefully..." }
+        @shutting_down = true
+      end
+
+      def immediate_shutdown
+        Pgbus.logger.warn { "[Pgbus] Worker shutting down immediately!" }
+        @shutting_down = true
+        @pool.kill
+      end
+
+      private
+
+      def claim_and_execute
+        idle = @pool.max_length - @pool.queue_length
+        return sleep(config.polling_interval) if idle <= 0
+
+        messages = fetch_messages(idle)
+
+        if messages.empty?
+          sleep(config.polling_interval)
+          return
+        end
+
+        messages.each do |message|
+          queue_name = message.respond_to?(:queue_name) ? message.queue_name : queues.first
+          @pool.post { process_message(message, queue_name) }
+        end
+      end
+
+      def fetch_messages(qty)
+        if queues.size == 1
+          Pgbus.client.read_batch(queues.first, qty: qty) || []
+        else
+          # Multi-queue read: read from each queue proportionally
+          per_queue = [(qty / queues.size.to_f).ceil, 1].max
+          queues.flat_map do |q|
+            Pgbus.client.read_batch(q, qty: per_queue) || []
+          end.first(qty)
+        end
+      rescue StandardError => e
+        Pgbus.logger.error { "[Pgbus] Error fetching messages: #{e.message}" }
+        []
+      end
+
+      def process_message(message, queue_name)
+        result = @executor.execute(message, queue_name)
+        @jobs_processed.increment
+        @jobs_failed.increment if result == :failed
+      rescue StandardError => e
+        @jobs_failed.increment
+        Pgbus.logger.error { "[Pgbus] Unhandled error processing message: #{e.message}" }
+      end
+
+      def recycle_needed?
+        exceeded_max_jobs? || exceeded_max_memory? || exceeded_max_lifetime?
+      end
+
+      def exceeded_max_jobs?
+        return false unless config.max_jobs_per_worker && @jobs_processed.value >= config.max_jobs_per_worker
+
+        Pgbus.logger.info { "[Pgbus] Worker recycling: max_jobs reached (#{@jobs_processed.value})" }
+        true
+      end
+
+      def exceeded_max_memory?
+        return false unless config.max_memory_mb && current_memory_mb > config.max_memory_mb
+
+        Pgbus.logger.info { "[Pgbus] Worker recycling: memory limit (#{current_memory_mb}MB > #{config.max_memory_mb}MB)" }
+        true
+      end
+
+      def exceeded_max_lifetime?
+        return false unless config.max_worker_lifetime && (Time.now - @started_at) > config.max_worker_lifetime
+
+        Pgbus.logger.info { "[Pgbus] Worker recycling: lifetime exceeded" }
+        true
+      end
+
+      def current_memory_mb
+        if RUBY_PLATFORM.include?("darwin")
+          `ps -o rss= -p #{::Process.pid}`.to_i / 1024
+        else
+          begin
+            File.read("/proc/#{::Process.pid}/statm").split[1].to_i * 4096 / (1024 * 1024)
+          rescue Errno::ENOENT
+            0
+          end
+        end
+      end
+
+      def start_heartbeat
+        @heartbeat = Heartbeat.new(
+          kind: "worker",
+          metadata: { queues: queues, threads: threads, pid: ::Process.pid }
+        )
+        @heartbeat.start
+      end
+
+      def shutdown
+        Pgbus.logger.info { "[Pgbus] Worker draining thread pool..." }
+        @pool.shutdown
+        @pool.wait_for_termination(30)
+        @heartbeat&.stop
+        restore_signals
+        Pgbus.logger.info { "[Pgbus] Worker stopped. Processed: #{@jobs_processed.value}, Failed: #{@jobs_failed.value}" }
+      end
+    end
+  end
+end

data/lib/pgbus/serializer.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Pgbus
+  module Serializer
+    module_function
+
+    def serialize_job(active_job)
+      data = active_job.serialize
+      # GlobalID is handled by ActiveJob's serialize — it converts AR objects
+      # to GlobalID URIs automatically. We just JSON-encode the result.
+      JSON.generate(data)
+    end
+
+    def deserialize_job(json_string)
+      data = JSON.parse(json_string)
+      ActiveJob::Base.deserialize(data)
+    end
+
+    def serialize_event(event)
+      payload = event.respond_to?(:to_global_id) ? { "_global_id" => event.to_global_id.to_s } : event
+      JSON.generate({
+                      "event_id" => event.respond_to?(:event_id) ? event.event_id : SecureRandom.uuid,
+                      "payload" => payload,
+                      "published_at" => Time.now.utc.iso8601(6)
+                    })
+    end
+
+    def deserialize_event(json_string)
+      data = JSON.parse(json_string)
+      payload = data["payload"]
+
+      data["payload"] = GlobalID::Locator.locate(payload["_global_id"]) if payload.is_a?(Hash) && payload["_global_id"]
+
+      Event.new(
+        event_id: data["event_id"],
+        payload: data["payload"],
+        published_at: Time.parse(data["published_at"])
+      )
+    end
+  end
+end

data/lib/pgbus/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Pgbus
+  VERSION = "0.0.1"
+end

data/lib/pgbus/web/authentication.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Pgbus
+  module Web
+    module Authentication
+      extend ActiveSupport::Concern
+
+      included do
+        before_action :authenticate_pgbus!
+      end
+
+      private
+
+      def authenticate_pgbus!
+        auth_block = Pgbus.configuration.web_auth
+        return if auth_block.nil?
+
+        return if auth_block.respond_to?(:call) && auth_block.call(request)
+
+        head :unauthorized
+      end
+    end
+  end
+end

data/lib/pgbus/web/data_source.rb

@@ -0,0 +1,406 @@
+# frozen_string_literal: true
+
+require "time"
+
+module Pgbus
+  module Web
+    class DataSource
+      def initialize(client: Pgbus.client)
+        @client = client
+      end
+
+      # Dashboard summary
+      def summary_stats
+        queues = queues_with_metrics
+        total_depth = queues.sum { |q| q[:queue_length] }
+        total_visible = queues.sum { |q| q[:queue_visible_length] }
+        dlq_suffix = Pgbus.configuration.dead_letter_queue_suffix
+        dlq_depth = queues.select { |q| q[:name].end_with?(dlq_suffix) }.sum { |q| q[:queue_length] }
+
+        {
+          total_queues: queues.size,
+          total_depth: total_depth,
+          total_visible: total_visible,
+          active_processes: processes.count,
+          failed_count: failed_events_count,
+          dlq_depth: dlq_depth
+        }
+      end
+
+      # Queues
+      def queues_with_metrics
+        metrics = @client.metrics || []
+        Array(metrics).map { |m| format_metrics(m) }
+      rescue StandardError => e
+        Pgbus.logger.debug { "[Pgbus::Web] Error fetching queue metrics: #{e.message}" }
+        []
+      end
+
+      def queue_detail(name)
+        m = @client.metrics(name)
+        m ? format_metrics(m) : nil
+      rescue StandardError => e
+        Pgbus.logger.debug { "[Pgbus::Web] Error fetching queue detail for #{name}: #{e.message}" }
+        nil
+      end
+
+      def purge_queue(name)
+        @client.purge_queue(name)
+      end
+
+      # Jobs (messages in queue tables)
+      def jobs(queue_name: nil, page: 1, per_page: 25)
+        offset = (page - 1) * per_page
+
+        if queue_name
+          query_queue_messages(queue_name, per_page, offset)
+        else
+          all_queue_messages(per_page, offset)
+        end
+      rescue StandardError => e
+        Pgbus.logger.debug { "[Pgbus::Web] Error reading jobs: #{e.message}" }
+        []
+      end
+
+      def job_detail(queue_name, msg_id)
+        full_name = Pgbus.configuration.queue_name(queue_name)
+        row = connection.select_one(
+          "SELECT * FROM pgmq.q_#{sanitize_name(full_name)} WHERE msg_id = $1",
+          "Pgbus Job Detail",
+          [msg_id.to_i]
+        )
+        row ? format_message(row, queue_name) : nil
+      rescue StandardError => e
+        Pgbus.logger.debug { "[Pgbus::Web] Error fetching job detail: #{e.message}" }
+        nil
+      end
+
+      def retry_job(queue_name, msg_id)
+        full_name = Pgbus.configuration.queue_name(queue_name)
+        @client.set_visibility_timeout(full_name, msg_id.to_i, vt: 0)
+      end
+
+      def discard_job(queue_name, msg_id)
+        @client.archive_message(queue_name, msg_id.to_i)
+      end
+
+      # Failed events
+      def failed_events(page: 1, per_page: 25)
+        offset = (page - 1) * per_page
+        rows = connection.select_all(
+          "SELECT * FROM pgbus_failed_events ORDER BY failed_at DESC LIMIT $1 OFFSET $2",
+          "Pgbus Failed Events",
+          [per_page, offset]
+        )
+        rows.to_a
+      rescue StandardError => e
+        Pgbus.logger.debug { "[Pgbus::Web] Error fetching failed events: #{e.message}" }
+        []
+      end
+
+      def failed_events_count
+        result = connection.select_value("SELECT COUNT(*) FROM pgbus_failed_events")
+        result.to_i
+      rescue StandardError => e
+        Pgbus.logger.debug { "[Pgbus::Web] Error counting failed events: #{e.message}" }
+        0
+      end
+
+      def failed_event(id)
+        connection.select_one(
+          "SELECT * FROM pgbus_failed_events WHERE id = $1",
+          "Pgbus Failed Event",
+          [id.to_i]
+        )
+      rescue StandardError => e
+        Pgbus.logger.debug { "[Pgbus::Web] Error fetching failed event #{id}: #{e.message}" }
+        nil
+      end
+
+      def retry_failed_event(id)
+        event = failed_event(id)
+        return false unless event
+
+        payload = JSON.parse(event["payload"])
+        headers = event["headers"]
+        headers = JSON.parse(headers) if headers.is_a?(String)
+
+        connection.transaction do
+          @client.send_message(event["queue_name"], payload, headers: headers)
+          connection.execute("DELETE FROM pgbus_failed_events WHERE id = #{id.to_i}")
+        end
+        true
+      rescue StandardError => e
+        Pgbus.logger.debug { "[Pgbus::Web] Error retrying failed event #{id}: #{e.message}" }
+        false
+      end
+
+      def discard_failed_event(id)
+        connection.execute("DELETE FROM pgbus_failed_events WHERE id = #{id.to_i}")
+        true
+      rescue StandardError => e
+        Pgbus.logger.debug { "[Pgbus::Web] Error discarding failed event #{id}: #{e.message}" }
+        false
+      end
+
+      def retry_all_failed
+        count = 0
+        connection.select_all("SELECT * FROM pgbus_failed_events").each do |event|
+          payload = JSON.parse(event["payload"])
+          headers = event["headers"]
+          headers = JSON.parse(headers) if headers.is_a?(String)
+
+          connection.transaction do
+            @client.send_message(event["queue_name"], payload, headers: headers)
+            connection.execute("DELETE FROM pgbus_failed_events WHERE id = #{event["id"].to_i}")
+          end
+          count += 1
+        rescue StandardError => e
+          Pgbus.logger.error { "[Pgbus::Web] Failed to retry event #{event["id"]}: #{e.message}" }
+        end
+        count
+      end
+
+      def discard_all_failed
+        result = connection.execute("DELETE FROM pgbus_failed_events")
+        result.cmd_tuples
+      rescue StandardError => e
+        Pgbus.logger.debug { "[Pgbus::Web] Error discarding all failed events: #{e.message}" }
+        0
+      end
+
+      # Dead letter queue
+      # Note: DLQ queue names from queues_with_metrics are already fully qualified
+      # (e.g., "pgbus_default_dlq"), so we use them directly without re-prefixing.
+      def dlq_messages(page: 1, per_page: 25)
+        dlq_suffix = Pgbus.configuration.dead_letter_queue_suffix
+        queues = queues_with_metrics.select { |q| q[:name].end_with?(dlq_suffix) }
+        offset = (page - 1) * per_page
+
+        messages = queues.flat_map do |q|
+          query_queue_messages_raw(q[:name], per_page + offset, 0)
+        end
+
+        messages.sort_by { |m| -m[:msg_id].to_i }.slice(offset, per_page) || []
+      rescue StandardError => e
+        Pgbus.logger.debug { "[Pgbus::Web] Error fetching DLQ messages: #{e.message}" }
+        []
+      end
+
+      def dlq_message_detail(msg_id)
+        dlq_suffix = Pgbus.configuration.dead_letter_queue_suffix
+        queues = queues_with_metrics.select { |q| q[:name].end_with?(dlq_suffix) }
+        queues.each do |q|
+          row = connection.select_one(
+            "SELECT * FROM pgmq.q_#{sanitize_name(q[:name])} WHERE msg_id = $1",
+            "Pgbus DLQ Detail",
+            [msg_id.to_i]
+          )
+          return format_message(row, q[:name]) if row
+        end
+        nil
+      rescue StandardError => e
+        Pgbus.logger.debug { "[Pgbus::Web] Error fetching DLQ message #{msg_id}: #{e.message}" }
+        nil
+      end
+
+      def retry_dlq_message(queue_name, msg_id)
+        # queue_name here is the full DLQ name (already prefixed)
+        dlq_suffix = Pgbus.configuration.dead_letter_queue_suffix
+        original_queue = queue_name.delete_suffix(dlq_suffix)
+
+        row = connection.select_one(
+          "SELECT * FROM pgmq.q_#{sanitize_name(queue_name)} WHERE msg_id = $1",
+          "Pgbus DLQ Read",
+          [msg_id.to_i]
+        )
+        return false unless row
+
+        @client.transaction do |txn|
+          txn.produce(original_queue, row["message"], headers: row["headers"])
+          txn.delete(queue_name, msg_id.to_i)
+        end
+        true
+      rescue StandardError => e
+        Pgbus.logger.debug { "[Pgbus::Web] Error retrying DLQ message #{msg_id}: #{e.message}" }
+        false
+      end
+
+      def discard_dlq_message(queue_name, msg_id)
+        # queue_name here is the full DLQ name (already prefixed)
+        @client.delete_from_queue(queue_name, msg_id.to_i)
+        true
+      rescue StandardError => e
+        Pgbus.logger.debug { "[Pgbus::Web] Error discarding DLQ message #{msg_id}: #{e.message}" }
+        false
+      end
+
+      def retry_all_dlq
+        messages = dlq_messages(page: 1, per_page: 1000)
+        count = 0
+        messages.each do |m|
+          retry_dlq_message(m[:queue_name], m[:msg_id]) && count += 1
+        rescue StandardError => e
+          Pgbus.logger.debug { "[Pgbus::Web] Error retrying DLQ message #{m[:msg_id]}: #{e.message}" }
+          next
+        end
+        count
+      end
+
+      def discard_all_dlq
+        messages = dlq_messages(page: 1, per_page: 1000)
+        count = 0
+        messages.each do |m|
+          discard_dlq_message(m[:queue_name], m[:msg_id]) && count += 1
+        rescue StandardError => e
+          Pgbus.logger.debug { "[Pgbus::Web] Error discarding DLQ message #{m[:msg_id]}: #{e.message}" }
+          next
+        end
+        count
+      end
+
+      # Processes
+      def processes
+        rows = connection.select_all(
+          "SELECT * FROM pgbus_processes ORDER BY kind, created_at"
+        )
+        rows.to_a.map { |r| format_process(r) }
+      rescue StandardError => e
+        Pgbus.logger.debug { "[Pgbus::Web] Error fetching processes: #{e.message}" }
+        []
+      end
+
+      # Processed events (audit trail)
+      def processed_events(page: 1, per_page: 25)
+        offset = (page - 1) * per_page
+        rows = connection.select_all(
+          "SELECT * FROM pgbus_processed_events ORDER BY processed_at DESC LIMIT $1 OFFSET $2",
+          "Pgbus Processed Events",
+          [per_page, offset]
+        )
+        rows.to_a
+      rescue StandardError => e
+        Pgbus.logger.debug { "[Pgbus::Web] Error fetching processed events: #{e.message}" }
+        []
+      end
+
+      def processed_event(id)
+        connection.select_one(
+          "SELECT * FROM pgbus_processed_events WHERE id = $1",
+          "Pgbus Processed Event",
+          [id.to_i]
+        )
+      rescue StandardError => e
+        Pgbus.logger.debug { "[Pgbus::Web] Error fetching processed event #{id}: #{e.message}" }
+        nil
+      end
+
+      def processed_events_count
+        result = connection.select_value("SELECT COUNT(*) FROM pgbus_processed_events")
+        result.to_i
+      rescue StandardError => e
+        Pgbus.logger.debug { "[Pgbus::Web] Error counting processed events: #{e.message}" }
+        0
+      end
+
+      def replay_event(event)
+        # Re-publish the event payload to all matching subscribers
+        routing_key = event["routing_key"] || event["handler_class"]
+        return false unless routing_key
+
+        @client.publish_to_topic(routing_key, event["payload"] || "{}")
+        true
+      rescue StandardError => e
+        Pgbus.logger.debug { "[Pgbus::Web] Error replaying event: #{e.message}" }
+        false
+      end
+
+      # Subscriber registry
+      def registered_subscribers
+        EventBus::Registry.instance.subscribers.map do |s|
+          { pattern: s.pattern, handler_class: s.handler_class.name, queue_name: s.queue_name }
+        end
+      rescue StandardError => e
+        Pgbus.logger.debug { "[Pgbus::Web] Error fetching subscribers: #{e.message}" }
+        []
+      end
+
+      private
+
+      def connection
+        ActiveRecord::Base.connection
+      end
+
+      def query_queue_messages(name, limit, offset)
+        full_name = Pgbus.configuration.queue_name(name)
+        query_queue_messages_raw(full_name, limit, offset).map { |m| m.merge(queue: name) }
+      end
+
+      def query_queue_messages_raw(full_name, limit, offset)
+        rows = connection.select_all(
+          "SELECT * FROM pgmq.q_#{sanitize_name(full_name)} ORDER BY msg_id DESC LIMIT $1 OFFSET $2",
+          "Pgbus Queue Messages",
+          [limit, offset]
+        )
+        rows.to_a.map { |r| format_message(r, full_name) }
+      rescue StandardError => e
+        Pgbus.logger.debug { "[Pgbus::Web] Error querying messages from #{full_name}: #{e.message}" }
+        []
+      end
+
+      def all_queue_messages(limit, offset)
+        dlq_suffix = Pgbus.configuration.dead_letter_queue_suffix
+        queues = queues_with_metrics.reject { |q| q[:name].end_with?(dlq_suffix) }
+        messages = queues.flat_map do |q|
+          query_queue_messages_raw(q[:name], limit + offset, 0)
+        end
+        messages.sort_by { |m| -m[:msg_id].to_i }.slice(offset, limit) || []
+      end
+
+      def format_metrics(m)
+        {
+          name: m.queue_name.to_s,
+          queue_length: m.queue_length.to_i,
+          queue_visible_length: m.queue_visible_length.to_i,
+          oldest_msg_age_sec: m.oldest_msg_age_sec&.to_i,
+          newest_msg_age_sec: m.newest_msg_age_sec&.to_i,
+          total_messages: m.total_messages.to_i
+        }
+      end
+
+      def format_message(row, queue_name)
+        {
+          msg_id: row["msg_id"].to_i,
+          read_ct: row["read_ct"].to_i,
+          enqueued_at: row["enqueued_at"],
+          last_read_at: row["last_read_at"],
+          vt: row["vt"],
+          message: row["message"],
+          headers: row["headers"],
+          queue_name: queue_name
+        }
+      end
+
+      def format_process(row)
+        heartbeat = row["last_heartbeat_at"]
+        heartbeat_time = heartbeat.is_a?(String) ? Time.parse(heartbeat) : heartbeat
+        stale = heartbeat_time && (Time.now - heartbeat_time) > Process::Heartbeat::ALIVE_THRESHOLD
+
+        {
+          id: row["id"].to_i,
+          kind: row["kind"],
+          hostname: row["hostname"],
+          pid: row["pid"].to_i,
+          metadata: row["metadata"].is_a?(String) ? JSON.parse(row["metadata"]) : row["metadata"],
+          last_heartbeat_at: heartbeat_time,
+          healthy: !stale,
+          created_at: row["created_at"]
+        }
+      end
+
+      def sanitize_name(name)
+        name.gsub(/[^a-zA-Z0-9_]/, "")
+      end
+    end
+  end
+end

data/lib/pgbus.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+require "zeitwerk"
+
+module Pgbus
+  class Error < StandardError; end
+  class ConfigurationError < Error; end
+  class SerializationError < Error; end
+  class QueueNotFoundError < Error; end
+  class DeadLetterError < Error; end
+  class ConcurrencyLimitExceeded < Error; end
+
+  class << self
+    def loader
+      @loader ||= begin
+        loader = Zeitwerk::Loader.for_gem
+        loader.inflector.inflect("pgbus" => "Pgbus", "cli" => "CLI", "dsl" => "DSL")
+        loader.ignore("#{__dir__}/generators")
+        loader
+      end
+    end
+
+    def configuration
+      @configuration ||= Configuration.new
+    end
+
+    def configure
+      yield configuration
+    end
+
+    def client
+      @client ||= Client.new(configuration)
+    end
+
+    def reset!
+      @client&.close
+      @client = nil
+      @configuration = nil
+    end
+
+    def logger
+      configuration.logger
+    end
+  end
+
+  loader.setup
+end
+
+require "pgbus/engine" if defined?(Rails::Engine)

data/package.json

@@ -0,0 +1,9 @@
+{
+  "private": true,
+  "engines": {
+    "bun": ">=1.1.0"
+  },
+  "devDependencies": {
+    "playwright": "^1.50.0"
+  }
+}

data/sig/pgbus.rbs

@@ -0,0 +1,4 @@
+module Pgbus
+  VERSION: String
+  # See the writing guide of rbs: https://github.com/ruby/rbs#guides
+end