pgbus 0.0.1

data/lib/pgbus/process/consumer.rb

@@ -0,0 +1,113 @@
+# frozen_string_literal: true
+
+require "concurrent"
+
+module Pgbus
+  module Process
+    class Consumer
+      include SignalHandler
+
+      attr_reader :topics, :threads, :config
+
+      def initialize(topics:, threads: 3, config: Pgbus.configuration)
+        @topics = Array(topics)
+        @threads = threads
+        @config = config
+        @shutting_down = false
+        @pool = Concurrent::FixedThreadPool.new(threads)
+        @registry = EventBus::Registry.instance
+      end
+
+      def run
+        setup_signals
+        start_heartbeat
+        setup_subscriptions
+        Pgbus.logger.info { "[Pgbus] Consumer started: topics=#{topics.join(",")} threads=#{threads}" }
+
+        loop do
+          break if @shutting_down
+
+          process_signals
+          consume
+        end
+
+        shutdown
+      end
+
+      def graceful_shutdown
+        @shutting_down = true
+      end
+
+      def immediate_shutdown
+        @shutting_down = true
+        @pool.kill
+      end
+
+      private
+
+      def setup_subscriptions
+        matching = @registry.subscribers.select do |s|
+          topics.any? { |t| pattern_overlaps?(t, s.pattern) }
+        end
+        @queue_names = matching.map(&:queue_name).uniq
+      end
+
+      def consume
+        idle = @pool.max_length - @pool.queue_length
+        return sleep(config.polling_interval) if idle <= 0
+
+        messages = @queue_names.flat_map do |queue_name|
+          Pgbus.client.read_batch(queue_name, qty: idle) || []
+        end.first(idle)
+
+        if messages.empty?
+          sleep(config.polling_interval)
+          return
+        end
+
+        messages.each do |message|
+          @pool.post { handle_message(message) }
+        end
+      end
+
+      def handle_message(message)
+        raw = JSON.parse(message.message)
+        routing_key = raw.dig("headers", "routing_key") || raw["routing_key"]
+
+        handlers = @registry.handlers_for(routing_key || "")
+        handlers.each do |subscriber|
+          handler = subscriber.handler_class.new
+          handler.process(message)
+        end
+
+        queue_name = message.respond_to?(:queue_name) ? message.queue_name : @queue_names.first
+        Pgbus.client.archive_message(queue_name, message.msg_id.to_i)
+      rescue StandardError => e
+        Pgbus.logger.error { "[Pgbus] Consumer error: #{e.class}: #{e.message}" }
+      end
+
+      def pattern_overlaps?(topic_filter, subscription_pattern)
+        # Simple check: if either is a subset of the other
+        topic_filter == subscription_pattern ||
+          topic_filter.end_with?("#") ||
+          subscription_pattern.start_with?(topic_filter.delete_suffix(".#"))
+      end
+
+      def start_heartbeat
+        @heartbeat = Heartbeat.new(
+          kind: "consumer",
+          metadata: { topics: topics, threads: threads, pid: ::Process.pid }
+        )
+        @heartbeat.start
+      end
+
+      def shutdown
+        @pool.shutdown
+        @pool.wait_for_termination(30)
+        @heartbeat&.stop
+        restore_signals
+        Pgbus.logger.info { "[Pgbus] Consumer stopped" }
+      end
+    end
+  end
+end

data/lib/pgbus/process/dispatcher.rb

@@ -0,0 +1,154 @@
+# frozen_string_literal: true
+
+module Pgbus
+  module Process
+    class Dispatcher
+      include SignalHandler
+
+      # Maintenance runs on coarser intervals than the main loop
+      CLEANUP_INTERVAL = 3600       # Run idempotency cleanup every hour
+      REAP_INTERVAL = 300           # Run stale process reaping every 5 minutes
+      CONCURRENCY_INTERVAL = 300    # Run concurrency cleanup every 5 minutes
+      BATCH_CLEANUP_INTERVAL = 3600 # Run batch cleanup every hour
+
+      attr_reader :config
+
+      def initialize(config: Pgbus.configuration)
+        @config = config
+        @shutting_down = false
+        @last_cleanup_at = Time.now
+        @last_reap_at = Time.now
+        @last_concurrency_at = Time.now
+        @last_batch_cleanup_at = Time.now
+      end
+
+      def run
+        setup_signals
+        start_heartbeat
+        Pgbus.logger.info do
+          "[Pgbus] Dispatcher started: interval=#{config.dispatch_interval}s"
+        end
+
+        loop do
+          break if @shutting_down
+
+          process_signals
+          break if @shutting_down
+
+          run_maintenance
+          break if @shutting_down
+
+          sleep(config.dispatch_interval)
+        end
+
+        shutdown
+      end
+
+      def graceful_shutdown
+        @shutting_down = true
+      end
+
+      def immediate_shutdown
+        @shutting_down = true
+      end
+
+      private
+
+      def run_maintenance
+        now = Time.now
+
+        if now - @last_cleanup_at >= CLEANUP_INTERVAL
+          cleanup_processed_events
+          @last_cleanup_at = now
+        end
+
+        if now - @last_reap_at >= REAP_INTERVAL
+          reap_stale_processes
+          @last_reap_at = now
+        end
+
+        if now - @last_concurrency_at >= CONCURRENCY_INTERVAL
+          cleanup_concurrency
+          @last_concurrency_at = now
+        end
+
+        if now - @last_batch_cleanup_at >= BATCH_CLEANUP_INTERVAL
+          cleanup_batches
+          @last_batch_cleanup_at = now
+        end
+      rescue StandardError => e
+        Pgbus.logger.error { "[Pgbus] Dispatcher maintenance error: #{e.message}" }
+      end
+
+      def cleanup_processed_events
+        return unless defined?(ActiveRecord::Base)
+
+        ttl = config.idempotency_ttl
+        return unless ttl&.positive?
+
+        deleted = ActiveRecord::Base.connection.delete(
+          "DELETE FROM pgbus_processed_events WHERE processed_at < $1",
+          "Pgbus Idempotency Cleanup",
+          [Time.now.utc - ttl]
+        )
+        Pgbus.logger.debug { "[Pgbus] Cleaned up #{deleted} expired processed events" } if deleted.positive?
+      rescue StandardError => e
+        Pgbus.logger.warn { "[Pgbus] Idempotency cleanup failed: #{e.message}" }
+      end
+
+      def reap_stale_processes
+        return unless defined?(ActiveRecord::Base)
+
+        threshold = Heartbeat::ALIVE_THRESHOLD
+        deleted = ActiveRecord::Base.connection.delete(
+          "DELETE FROM pgbus_processes WHERE last_heartbeat_at < $1",
+          "Pgbus Stale Process Reap",
+          [Time.now.utc - threshold]
+        )
+        Pgbus.logger.info { "[Pgbus] Reaped #{deleted} stale processes" } if deleted.positive?
+      rescue StandardError => e
+        Pgbus.logger.warn { "[Pgbus] Stale process reaping failed: #{e.message}" }
+      end
+
+      def cleanup_concurrency
+        expired_keys = Concurrency::Semaphore.expire_stale
+        expired_keys.each do |row|
+          release_blocked_for_key(row["key"])
+        end
+
+        orphaned = Concurrency::BlockedExecution.expire_stale
+        Pgbus.logger.debug { "[Pgbus] Expired #{orphaned} orphaned blocked executions" } if orphaned.positive?
+      rescue StandardError => e
+        Pgbus.logger.warn { "[Pgbus] Concurrency cleanup failed: #{e.message}" }
+      end
+
+      def release_blocked_for_key(key)
+        released = Concurrency::BlockedExecution.release_next(key)
+        return unless released
+
+        Pgbus.client.send_message(released[:queue_name], released[:payload])
+        Pgbus.logger.debug { "[Pgbus] Released blocked execution for key: #{key}" }
+      rescue StandardError => e
+        Pgbus.logger.warn { "[Pgbus] Failed to release blocked execution for #{key}: #{e.message}" }
+      end
+
+      def cleanup_batches
+        deleted = Batch.cleanup(older_than: Time.now.utc - (7 * 24 * 3600)) # 7 days
+        Pgbus.logger.debug { "[Pgbus] Cleaned up #{deleted} finished batches" } if deleted.positive?
+      rescue StandardError => e
+        Pgbus.logger.warn { "[Pgbus] Batch cleanup failed: #{e.message}" }
+      end
+
+      def start_heartbeat
+        @heartbeat = Heartbeat.new(kind: "dispatcher", metadata: { pid: ::Process.pid })
+        @heartbeat.start
+      end
+
+      def shutdown
+        @heartbeat&.stop
+        restore_signals
+        Pgbus.logger.info { "[Pgbus] Dispatcher stopped" }
+      end
+    end
+  end
+end

data/lib/pgbus/process/heartbeat.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+require "concurrent"
+
+module Pgbus
+  module Process
+    class Heartbeat
+      INTERVAL = 60 # seconds
+      ALIVE_THRESHOLD = 300 # 5 minutes
+
+      attr_reader :process_record
+
+      def initialize(kind:, metadata: {})
+        @kind = kind
+        @metadata = metadata
+        @timer = nil
+      end
+
+      def start
+        register_process
+        @timer = Concurrent::TimerTask.new(execution_interval: INTERVAL) { beat }
+        @timer.execute
+      end
+
+      def stop
+        @timer&.shutdown
+        deregister_process
+      end
+
+      def beat
+        return unless @process_id && defined?(ActiveRecord::Base)
+
+        ActiveRecord::Base.connection.execute(
+          "UPDATE pgbus_processes SET last_heartbeat_at = NOW() WHERE id = $1",
+          "Pgbus Heartbeat",
+          [@process_id]
+        )
+      rescue StandardError => e
+        Pgbus.logger.warn { "[Pgbus] Heartbeat failed: #{e.message}" }
+      end
+
+      private
+
+      def register_process
+        return unless defined?(ActiveRecord::Base)
+
+        result = ActiveRecord::Base.connection.exec_insert(
+          "INSERT INTO pgbus_processes (kind, hostname, pid, metadata, last_heartbeat_at, created_at, updated_at) " \
+          "VALUES ($1, $2, $3, $4, NOW(), NOW(), NOW()) RETURNING id",
+          "Pgbus Register Process",
+          [@kind, Socket.gethostname, ::Process.pid, JSON.generate(@metadata)]
+        )
+        @process_id = result.first["id"]
+      rescue StandardError => e
+        Pgbus.logger.warn { "[Pgbus] Process registration failed: #{e.message}" }
+      end
+
+      def deregister_process
+        return unless @process_id && defined?(ActiveRecord::Base)
+
+        ActiveRecord::Base.connection.execute(
+          "DELETE FROM pgbus_processes WHERE id = $1",
+          "Pgbus Deregister Process",
+          [@process_id]
+        )
+      rescue StandardError
+        # Best effort — process is exiting anyway
+      end
+    end
+  end
+end

data/lib/pgbus/process/signal_handler.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+module Pgbus
+  module Process
+    module SignalHandler
+      def self.included(base)
+        base.attr_reader :signal_queue
+      end
+
+      def setup_signals
+        @signal_queue = Queue.new
+        @previous_handlers = {}
+
+        %w[INT TERM QUIT].each do |sig|
+          @previous_handlers[sig] = trap(sig) { @signal_queue << sig }
+        end
+      end
+
+      def restore_signals
+        @previous_handlers&.each do |sig, handler|
+          trap(sig, handler || "DEFAULT")
+        end
+      end
+
+      def process_signals
+        while (sig = begin
+          @signal_queue.pop(true)
+        rescue StandardError
+          nil
+        end)
+          case sig
+          when "INT", "TERM"
+            graceful_shutdown
+          when "QUIT"
+            immediate_shutdown
+          end
+        end
+      end
+
+      def graceful_shutdown
+        raise NotImplementedError
+      end
+
+      def immediate_shutdown
+        raise NotImplementedError
+      end
+    end
+  end
+end

data/lib/pgbus/process/supervisor.rb

@@ -0,0 +1,198 @@
+# frozen_string_literal: true
+
+module Pgbus
+  module Process
+    class Supervisor
+      include SignalHandler
+
+      FORK_WAIT = 1 # seconds between fork checks
+
+      attr_reader :config
+
+      def initialize(config: Pgbus.configuration)
+        @config = config
+        @forks = {}
+        @shutting_down = false
+      end
+
+      def run
+        setup_signals
+        start_heartbeat
+
+        Pgbus.logger.info { "[Pgbus] Supervisor starting pid=#{::Process.pid}" }
+
+        boot_processes
+        monitor_loop
+      ensure
+        shutdown
+      end
+
+      def graceful_shutdown
+        Pgbus.logger.info { "[Pgbus] Supervisor: graceful shutdown requested" }
+        @shutting_down = true
+        signal_children("TERM")
+      end
+
+      def immediate_shutdown
+        Pgbus.logger.warn { "[Pgbus] Supervisor: immediate shutdown requested" }
+        @shutting_down = true
+        signal_children("QUIT")
+      end
+
+      private
+
+      def boot_processes
+        # Boot workers
+        config.workers.each do |worker_config|
+          fork_worker(worker_config)
+        end
+
+        # Boot dispatcher
+        fork_dispatcher
+
+        # Boot event consumers if configured
+        boot_consumers
+      end
+
+      def fork_worker(worker_config)
+        queues = worker_config[:queues] || worker_config["queues"] || [config.default_queue]
+        threads = worker_config[:threads] || worker_config["threads"] || 5
+
+        pid = fork do
+          restore_signals
+          setup_child_signals
+          load_rails_app
+          worker = Worker.new(queues: queues, threads: threads, config: config)
+          worker.run
+        end
+
+        @forks[pid] = { type: :worker, config: worker_config }
+        Pgbus.logger.info { "[Pgbus] Forked worker pid=#{pid} queues=#{queues.join(",")}" }
+      end
+
+      def fork_dispatcher
+        pid = fork do
+          restore_signals
+          setup_child_signals
+          load_rails_app
+          dispatcher = Dispatcher.new(config: config)
+          dispatcher.run
+        end
+
+        @forks[pid] = { type: :dispatcher }
+        Pgbus.logger.info { "[Pgbus] Forked dispatcher pid=#{pid}" }
+      end
+
+      def boot_consumers
+        return unless config.event_consumers
+
+        config.event_consumers.each do |consumer_config|
+          fork_consumer(consumer_config)
+        end
+      end
+
+      def fork_consumer(consumer_config)
+        topics = consumer_config[:topics] || consumer_config["topics"]
+        threads = consumer_config[:threads] || consumer_config["threads"] || 3
+
+        pid = fork do
+          restore_signals
+          setup_child_signals
+          load_rails_app
+          consumer = Consumer.new(topics: topics, threads: threads, config: config)
+          consumer.run
+        end
+
+        @forks[pid] = { type: :consumer, config: consumer_config }
+        Pgbus.logger.info { "[Pgbus] Forked consumer pid=#{pid} topics=#{topics.join(",")}" }
+      end
+
+      def monitor_loop
+        loop do
+          break if @shutting_down && @forks.empty?
+
+          process_signals
+          reap_children
+          sleep(FORK_WAIT)
+        end
+      end
+
+      def reap_children
+        loop do
+          pid, status = ::Process.waitpid2(-1, ::Process::WNOHANG)
+          break unless pid
+
+          info = @forks.delete(pid)
+          next unless info
+
+          if @shutting_down
+            Pgbus.logger.info { "[Pgbus] Child #{info[:type]} pid=#{pid} exited (status=#{status.exitstatus})" }
+          else
+            Pgbus.logger.warn do
+              "[Pgbus] Child #{info[:type]} pid=#{pid} exited unexpectedly (status=#{status&.exitstatus}), restarting..."
+            end
+            restart_child(info)
+          end
+        rescue Errno::ECHILD
+          break
+        end
+      end
+
+      def restart_child(info)
+        case info[:type]
+        when :worker
+          fork_worker(info[:config])
+        when :dispatcher
+          fork_dispatcher
+        when :consumer
+          fork_consumer(info[:config])
+        end
+      end
+
+      def signal_children(sig)
+        @forks.each_key do |pid|
+          ::Process.kill(sig, pid)
+        rescue Errno::ESRCH
+          # Process already gone
+        end
+      end
+
+      def setup_child_signals
+        %w[INT TERM QUIT].each do |sig|
+          trap(sig) { @shutting_down = true }
+        end
+      end
+
+      def load_rails_app
+        return unless defined?(Rails)
+
+        Rails.application.eager_load! if Rails.application.respond_to?(:eager_load!)
+      end
+
+      def start_heartbeat
+        @heartbeat = Heartbeat.new(
+          kind: "supervisor",
+          metadata: { pid: ::Process.pid, hostname: Socket.gethostname }
+        )
+        @heartbeat.start
+      end
+
+      def shutdown
+        # Wait for all children with timeout
+        deadline = Time.now + 30
+
+        until @forks.empty? || Time.now > deadline
+          reap_children
+          sleep(0.5)
+        end
+
+        # Force kill any remaining
+        signal_children("KILL") unless @forks.empty?
+
+        @heartbeat&.stop
+        restore_signals
+        Pgbus.logger.info { "[Pgbus] Supervisor stopped" }
+      end
+    end
+  end
+end