pgbus 0.0.1 → 0.1.2

data/lib/pgbus/pgmq_schema.rb

@@ -0,0 +1,159 @@
+# frozen_string_literal: true
+
+module Pgbus
+  # Manages embedded PGMQ SQL schema for extension-free installations.
+  #
+  # Supports three modes:
+  #   :auto      - Try extension first, fall back to embedded SQL
+  #   :extension - Require the pgmq PostgreSQL extension
+  #   :embedded  - Use vendored SQL (no extension needed)
+  #
+  # Vendored SQL files live in lib/pgbus/pgmq_schema/pgmq_v{VERSION}.sql
+  # and are exact copies of the upstream pgmq-extension/sql/pgmq.sql at each release.
+  module PgmqSchema
+    class VersionNotFoundError < StandardError; end
+
+    SCHEMA_DIR = File.expand_path("pgmq_schema", __dir__).freeze
+
+    class << self
+      # Returns the latest vendored PGMQ version string.
+      def latest_version
+        available_versions.last
+      end
+
+      # Returns sorted list of all vendored PGMQ versions.
+      def available_versions
+        Dir.glob(File.join(SCHEMA_DIR, "pgmq_v*.sql"))
+           .map { |f| File.basename(f).match(/pgmq_v(.+)\.sql/)[1] }
+           .sort_by { |v| Gem::Version.new(v) }
+      end
+
+      # Returns the filesystem path to the vendored SQL file for a given version.
+      #
+      # @param version [String] e.g. "1.11.0"
+      # @return [String] absolute path
+      # @raise [VersionNotFoundError] if no SQL file exists for that version
+      def sql_path(version)
+        path = File.join(SCHEMA_DIR, "pgmq_v#{version}.sql")
+        raise VersionNotFoundError, "No vendored PGMQ SQL for version #{version}" unless File.exist?(path)
+
+        path
+      end
+
+      # Returns the raw SQL content for a given version.
+      def sql_for_version(version)
+        File.read(sql_path(version))
+      end
+
+      # Returns the SQL to install PGMQ schema without the extension.
+      # Strips the extension-only pg_dump config blocks since they're
+      # irrelevant when not installed as an extension.
+      #
+      # @param version [String] defaults to latest
+      # @return [String] SQL
+      def install_sql(version = latest_version)
+        sql = sql_for_version(version)
+        strip_extension_only_blocks(sql)
+      end
+
+      # Returns SQL to create the version tracking table and record an installation.
+      #
+      # @param version [String] defaults to latest
+      # @return [String] SQL
+      def version_tracking_sql(version = latest_version)
+        <<~SQL
+          CREATE TABLE IF NOT EXISTS pgbus_pgmq_schema_versions (
+            id SERIAL PRIMARY KEY,
+            version VARCHAR NOT NULL,
+            installed_at TIMESTAMP WITH TIME ZONE DEFAULT now() NOT NULL,
+            install_method VARCHAR NOT NULL DEFAULT 'embedded'
+          );
+
+          INSERT INTO pgbus_pgmq_schema_versions (version, install_method)
+          VALUES ('#{version}', 'embedded');
+        SQL
+      end
+
+      # Returns SQL to record an extension-based installation in the version tracking table.
+      #
+      # @param version [String]
+      # @return [String] SQL
+      def version_tracking_extension_sql(version = latest_version)
+        <<~SQL
+          CREATE TABLE IF NOT EXISTS pgbus_pgmq_schema_versions (
+            id SERIAL PRIMARY KEY,
+            version VARCHAR NOT NULL,
+            installed_at TIMESTAMP WITH TIME ZONE DEFAULT now() NOT NULL,
+            install_method VARCHAR NOT NULL DEFAULT 'embedded'
+          );
+
+          INSERT INTO pgbus_pgmq_schema_versions (version, install_method)
+          VALUES ('#{version}', 'extension');
+        SQL
+      end
+
+      # SQL to drop all pgmq functions/types (for clean upgrade).
+      # Uses CASCADE so dependent objects are also dropped.
+      def drop_pgmq_functions_sql
+        <<~SQL
+          DO $$
+          DECLARE
+            r RECORD;
+          BEGIN
+            -- Drop all functions in pgmq schema
+            FOR r IN
+              SELECT pg_catalog.pg_get_functiondef(p.oid) AS funcdef,
+                     n.nspname || '.' || p.proname || '(' ||
+                       pg_catalog.pg_get_function_identity_arguments(p.oid) || ')' AS func_sig
+              FROM pg_catalog.pg_proc p
+              JOIN pg_catalog.pg_namespace n ON n.oid = p.pronamespace
+              WHERE n.nspname = 'pgmq'
+            LOOP
+              EXECUTE 'DROP FUNCTION IF EXISTS ' || r.func_sig || ' CASCADE';
+            END LOOP;
+
+            -- Drop custom types in pgmq schema
+            FOR r IN
+              SELECT n.nspname || '.' || t.typname AS type_name
+              FROM pg_catalog.pg_type t
+              JOIN pg_catalog.pg_namespace n ON n.oid = t.typnamespace
+              WHERE n.nspname = 'pgmq'
+                AND t.typtype = 'c'
+                AND NOT EXISTS (
+                  SELECT 1 FROM pg_catalog.pg_class c
+                  WHERE c.reltype = t.oid
+                )
+            LOOP
+              EXECUTE 'DROP TYPE IF EXISTS ' || r.type_name || ' CASCADE';
+            END LOOP;
+          END $$;
+        SQL
+      end
+
+      private
+
+      # Strips extension-specific blocks (pg_extension_config_dump, pg_depend checks)
+      # that only work when pgmq is installed as an extension.
+      def strip_extension_only_blocks(sql)
+        # Remove the DO block that conditionally creates schema only when extension is missing.
+        # Replace with unconditional schema creation.
+        sql = sql.sub(
+          /DO\s*\$\$\s*BEGIN\s*IF\s*\(SELECT\s+NOT\s+EXISTS.*?END\s*\$\$;/m,
+          "CREATE SCHEMA IF NOT EXISTS pgmq;"
+        )
+
+        # Remove pg_extension_config_dump blocks
+        sql = sql.gsub(
+          /DO\s*\$\$\s*BEGIN\s*IF\s+EXISTS\(SELECT\s+1\s+FROM\s+pg_extension.*?END\s*\$\$;/m,
+          ""
+        )
+
+        # Remove _belongs_to_pgmq function (checks pg_depend on extension)
+        sql.gsub(
+          /CREATE FUNCTION pgmq\._belongs_to_pgmq.*?LANGUAGE plpgsql;/m,
+          ""
+        )
+      end
+    end
+  end
+end

data/lib/pgbus/process/consumer.rb

@@ -54,23 +54,29 @@ module Pgbus
 
       def consume
         idle = @pool.max_length - @pool.queue_length
-        return sleep(config.polling_interval) if idle <= 0
+        return interruptible_sleep(config.polling_interval) if idle <= 0
 
-        messages = @queue_names.flat_map do |queue_name|
-          Pgbus.client.read_batch(queue_name, qty: idle) || []
+        tagged_messages = @queue_names.flat_map do |queue_name|
+          (Pgbus.client.read_batch(queue_name, qty: idle) || []).map { |m| [queue_name, m] }
         end.first(idle)
 
-        if messages.empty?
-          sleep(config.polling_interval)
+        if tagged_messages.empty?
+          interruptible_sleep(config.polling_interval)
           return
         end
 
-        messages.each do |message|
-          @pool.post { handle_message(message) }
+        tagged_messages.each do |queue_name, message|
+          @pool.post { handle_message(message, queue_name) }
         end
       end
 
-      def handle_message(message)
+      def handle_message(message, queue_name)
+        if message.read_ct.to_i > config.max_retries
+          Pgbus.logger.warn { "[Pgbus] Consumer moving message #{message.msg_id} to DLQ after #{message.read_ct} reads" }
+          Pgbus.client.move_to_dead_letter(queue_name, message)
+          return
+        end
+
         raw = JSON.parse(message.message)
         routing_key = raw.dig("headers", "routing_key") || raw["routing_key"]
 
@@ -80,10 +86,12 @@ module Pgbus
           handler.process(message)
         end
 
-        queue_name = message.respond_to?(:queue_name) ? message.queue_name : @queue_names.first
         Pgbus.client.archive_message(queue_name, message.msg_id.to_i)
       rescue StandardError => e
         Pgbus.logger.error { "[Pgbus] Consumer error: #{e.class}: #{e.message}" }
+        # Message stays in queue; VT will expire and it becomes available again.
+        # read_ct tracks delivery attempts — when it exceeds max_retries,
+        # the next read will route to DLQ above.
       end
 
       def pattern_overlaps?(topic_filter, subscription_pattern)

data/lib/pgbus/process/dispatcher.rb

@@ -6,10 +6,11 @@ module Pgbus
       include SignalHandler
 
       # Maintenance runs on coarser intervals than the main loop
-      CLEANUP_INTERVAL = 3600       # Run idempotency cleanup every hour
-      REAP_INTERVAL = 300           # Run stale process reaping every 5 minutes
-      CONCURRENCY_INTERVAL = 300    # Run concurrency cleanup every 5 minutes
-      BATCH_CLEANUP_INTERVAL = 3600 # Run batch cleanup every hour
+      CLEANUP_INTERVAL = 3600               # Run idempotency cleanup every hour
+      REAP_INTERVAL = 300                   # Run stale process reaping every 5 minutes
+      CONCURRENCY_INTERVAL = 300            # Run concurrency cleanup every 5 minutes
+      BATCH_CLEANUP_INTERVAL = 3600         # Run batch cleanup every hour
+      RECURRING_CLEANUP_INTERVAL = 3600     # Run recurring execution cleanup every hour
 
       attr_reader :config
 
@@ -20,6 +21,7 @@ module Pgbus
         @last_reap_at = Time.now
         @last_concurrency_at = Time.now
         @last_batch_cleanup_at = Time.now
+        @last_recurring_cleanup_at = Time.now
       end
 
       def run
@@ -38,7 +40,7 @@ module Pgbus
           run_maintenance
           break if @shutting_down
 
-          sleep(config.dispatch_interval)
+          interruptible_sleep(config.dispatch_interval)
         end
 
         shutdown
@@ -57,54 +59,37 @@ module Pgbus
       def run_maintenance
         now = Time.now
 
-        if now - @last_cleanup_at >= CLEANUP_INTERVAL
-          cleanup_processed_events
-          @last_cleanup_at = now
-        end
+        run_if_due(now, :@last_cleanup_at, CLEANUP_INTERVAL) { cleanup_processed_events }
+        run_if_due(now, :@last_reap_at, REAP_INTERVAL) { reap_stale_processes }
+        run_if_due(now, :@last_concurrency_at, CONCURRENCY_INTERVAL) { cleanup_concurrency }
+        run_if_due(now, :@last_batch_cleanup_at, BATCH_CLEANUP_INTERVAL) { cleanup_batches }
+        run_if_due(now, :@last_recurring_cleanup_at, RECURRING_CLEANUP_INTERVAL) { cleanup_recurring_executions }
+      end
 
-        if now - @last_reap_at >= REAP_INTERVAL
-          reap_stale_processes
-          @last_reap_at = now
-        end
+      # Only update the timestamp when the block succeeds.
+      # On failure, the next tick retries instead of waiting the full interval.
+      def run_if_due(now, ivar, interval)
+        return unless now - instance_variable_get(ivar) >= interval
 
-        if now - @last_concurrency_at >= CONCURRENCY_INTERVAL
-          cleanup_concurrency
-          @last_concurrency_at = now
-        end
-
-        if now - @last_batch_cleanup_at >= BATCH_CLEANUP_INTERVAL
-          cleanup_batches
-          @last_batch_cleanup_at = now
-        end
+        yield
+        instance_variable_set(ivar, now)
       rescue StandardError => e
         Pgbus.logger.error { "[Pgbus] Dispatcher maintenance error: #{e.message}" }
       end
 
       def cleanup_processed_events
-        return unless defined?(ActiveRecord::Base)
-
         ttl = config.idempotency_ttl
         return unless ttl&.positive?
 
-        deleted = ActiveRecord::Base.connection.delete(
-          "DELETE FROM pgbus_processed_events WHERE processed_at < $1",
-          "Pgbus Idempotency Cleanup",
-          [Time.now.utc - ttl]
-        )
+        deleted = ProcessedEvent.expired(Time.now.utc - ttl).delete_all
         Pgbus.logger.debug { "[Pgbus] Cleaned up #{deleted} expired processed events" } if deleted.positive?
       rescue StandardError => e
         Pgbus.logger.warn { "[Pgbus] Idempotency cleanup failed: #{e.message}" }
       end
 
       def reap_stale_processes
-        return unless defined?(ActiveRecord::Base)
-
         threshold = Heartbeat::ALIVE_THRESHOLD
-        deleted = ActiveRecord::Base.connection.delete(
-          "DELETE FROM pgbus_processes WHERE last_heartbeat_at < $1",
-          "Pgbus Stale Process Reap",
-          [Time.now.utc - threshold]
-        )
+        deleted = ProcessEntry.stale(Time.now.utc - threshold).delete_all
         Pgbus.logger.info { "[Pgbus] Reaped #{deleted} stale processes" } if deleted.positive?
       rescue StandardError => e
         Pgbus.logger.warn { "[Pgbus] Stale process reaping failed: #{e.message}" }
@@ -123,11 +108,8 @@ module Pgbus
       end
 
       def release_blocked_for_key(key)
-        released = Concurrency::BlockedExecution.release_next(key)
-        return unless released
-
-        Pgbus.client.send_message(released[:queue_name], released[:payload])
-        Pgbus.logger.debug { "[Pgbus] Released blocked execution for key: #{key}" }
+        promoted = Concurrency::BlockedExecution.promote_next(key, client: Pgbus.client)
+        Pgbus.logger.debug { "[Pgbus] Released blocked execution for key: #{key}" } if promoted
       rescue StandardError => e
         Pgbus.logger.warn { "[Pgbus] Failed to release blocked execution for #{key}: #{e.message}" }
       end
@@ -139,6 +121,16 @@ module Pgbus
         Pgbus.logger.warn { "[Pgbus] Batch cleanup failed: #{e.message}" }
       end
 
+      def cleanup_recurring_executions
+        retention = config.recurring_execution_retention
+        return unless retention&.positive?
+
+        deleted = RecurringExecution.older_than(Time.now.utc - retention).delete_all
+        Pgbus.logger.debug { "[Pgbus] Cleaned up #{deleted} old recurring executions" } if deleted.positive?
+      rescue StandardError => e
+        Pgbus.logger.warn { "[Pgbus] Recurring execution cleanup failed: #{e.message}" }
+      end
+
       def start_heartbeat
         @heartbeat = Heartbeat.new(kind: "dispatcher", metadata: { pid: ::Process.pid })
         @heartbeat.start

data/lib/pgbus/process/heartbeat.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require "concurrent"
+require "socket"
 
 module Pgbus
   module Process
@@ -8,7 +9,7 @@ module Pgbus
       INTERVAL = 60 # seconds
       ALIVE_THRESHOLD = 300 # 5 minutes
 
-      attr_reader :process_record
+      attr_reader :process_entry
 
       def initialize(kind:, metadata: {})
         @kind = kind
@@ -28,13 +29,9 @@ module Pgbus
       end
 
       def beat
-        return unless @process_id && defined?(ActiveRecord::Base)
+        return unless @process_id
 
-        ActiveRecord::Base.connection.execute(
-          "UPDATE pgbus_processes SET last_heartbeat_at = NOW() WHERE id = $1",
-          "Pgbus Heartbeat",
-          [@process_id]
-        )
+        ProcessEntry.where(id: @process_id).update_all(last_heartbeat_at: Time.current)
       rescue StandardError => e
         Pgbus.logger.warn { "[Pgbus] Heartbeat failed: #{e.message}" }
       end
@@ -42,29 +39,24 @@ module Pgbus
       private
 
       def register_process
-        return unless defined?(ActiveRecord::Base)
-
-        result = ActiveRecord::Base.connection.exec_insert(
-          "INSERT INTO pgbus_processes (kind, hostname, pid, metadata, last_heartbeat_at, created_at, updated_at) " \
-          "VALUES ($1, $2, $3, $4, NOW(), NOW(), NOW()) RETURNING id",
-          "Pgbus Register Process",
-          [@kind, Socket.gethostname, ::Process.pid, JSON.generate(@metadata)]
+        record = ProcessEntry.create!(
+          kind: @kind,
+          hostname: Socket.gethostname,
+          pid: ::Process.pid,
+          metadata: @metadata,
+          last_heartbeat_at: Time.current
         )
-        @process_id = result.first["id"]
+        @process_id = record.id
       rescue StandardError => e
         Pgbus.logger.warn { "[Pgbus] Process registration failed: #{e.message}" }
       end
 
       def deregister_process
-        return unless @process_id && defined?(ActiveRecord::Base)
+        return unless @process_id
 
-        ActiveRecord::Base.connection.execute(
-          "DELETE FROM pgbus_processes WHERE id = $1",
-          "Pgbus Deregister Process",
-          [@process_id]
-        )
-      rescue StandardError
-        # Best effort — process is exiting anyway
+        ProcessEntry.where(id: @process_id).delete_all
+      rescue StandardError => e
+        Pgbus.logger.warn { "[Pgbus] Process deregistration failed: #{e.message}" }
       end
     end
   end

data/lib/pgbus/process/signal_handler.rb

@@ -10,9 +10,14 @@ module Pgbus
       def setup_signals
         @signal_queue = Queue.new
         @previous_handlers = {}
+        @self_pipe_r, @self_pipe_w = IO.pipe
 
         %w[INT TERM QUIT].each do |sig|
-          @previous_handlers[sig] = trap(sig) { @signal_queue << sig }
+          @previous_handlers[sig] = trap(sig) do
+            @signal_queue << sig
+            # Wake any IO.select / interruptible_sleep call
+            @self_pipe_w.write_nonblock(".", exception: false)
+          end
         end
       end
 
@@ -20,6 +25,8 @@ module Pgbus
         @previous_handlers&.each do |sig, handler|
           trap(sig, handler || "DEFAULT")
         end
+        @self_pipe_r&.close unless @self_pipe_r&.closed?
+        @self_pipe_w&.close unless @self_pipe_w&.closed?
       end
 
       def process_signals
@@ -37,6 +44,13 @@ module Pgbus
         end
       end
 
+      # Sleep that can be interrupted by signals. Use this instead of Kernel#sleep
+      # in any loop that needs to respond to INT/TERM/QUIT promptly.
+      def interruptible_sleep(seconds)
+        @self_pipe_r.wait_readable(seconds)
+        drain_self_pipe
+      end
+
       def graceful_shutdown
         raise NotImplementedError
       end
@@ -44,6 +58,14 @@ module Pgbus
       def immediate_shutdown
         raise NotImplementedError
       end
+
+      private
+
+      def drain_self_pipe
+        loop { @self_pipe_r.read_nonblock(256) }
+      rescue IO::WaitReadable, EOFError
+        # pipe drained
+      end
     end
   end
 end

data/lib/pgbus/process/supervisor.rb

@@ -50,6 +50,9 @@ module Pgbus
         # Boot dispatcher
         fork_dispatcher
 
+        # Boot recurring scheduler if configured
+        boot_scheduler
+
         # Boot event consumers if configured
         boot_consumers
       end
@@ -66,8 +69,15 @@ module Pgbus
           worker.run
         end
 
+        unless pid
+          Pgbus.logger.error { "[Pgbus] Failed to fork worker for queues=#{queues.join(",")}" }
+          return
+        end
+
         @forks[pid] = { type: :worker, config: worker_config }
         Pgbus.logger.info { "[Pgbus] Forked worker pid=#{pid} queues=#{queues.join(",")}" }
+      rescue Errno::EAGAIN, Errno::ENOMEM => e
+        Pgbus.logger.error { "[Pgbus] Fork failed for worker: #{e.message}" }
       end
 
       def fork_dispatcher
@@ -79,8 +89,66 @@ module Pgbus
           dispatcher.run
         end
 
+        unless pid
+          Pgbus.logger.error { "[Pgbus] Failed to fork dispatcher" }
+          return
+        end
+
         @forks[pid] = { type: :dispatcher }
         Pgbus.logger.info { "[Pgbus] Forked dispatcher pid=#{pid}" }
+      rescue Errno::EAGAIN, Errno::ENOMEM => e
+        Pgbus.logger.error { "[Pgbus] Fork failed for dispatcher: #{e.message}" }
+      end
+
+      def boot_scheduler
+        return if config.skip_recurring
+        return unless recurring_tasks_configured?
+
+        fork_scheduler
+      end
+
+      def fork_scheduler
+        pid = fork do
+          restore_signals
+          setup_child_signals
+          load_rails_app
+          load_recurring_config
+          scheduler = Recurring::Scheduler.new(config: config)
+          scheduler.run
+        end
+
+        unless pid
+          Pgbus.logger.error { "[Pgbus] Failed to fork scheduler" }
+          return
+        end
+
+        @forks[pid] = { type: :scheduler }
+        Pgbus.logger.info { "[Pgbus] Forked scheduler pid=#{pid}" }
+      rescue Errno::EAGAIN, Errno::ENOMEM => e
+        Pgbus.logger.error { "[Pgbus] Fork failed for scheduler: #{e.message}" }
+      end
+
+      def recurring_tasks_configured?
+        return true if config.recurring_tasks&.any?
+        return true if config.recurring_tasks_file && File.exist?(config.recurring_tasks_file.to_s)
+
+        # Check default location
+        if defined?(Rails)
+          default_path = Rails.root.join("config", "recurring.yml")
+          return File.exist?(default_path.to_s)
+        end
+
+        false
+      end
+
+      def load_recurring_config
+        return if config.recurring_tasks&.any?
+
+        path = config.recurring_tasks_file
+        path ||= defined?(Rails) ? Rails.root.join("config", "recurring.yml") : nil
+        return unless path && File.exist?(path.to_s)
+
+        config.recurring_tasks = Recurring::ConfigLoader.load(path)
       end
 
       def boot_consumers
@@ -103,8 +171,15 @@ module Pgbus
           consumer.run
         end
 
+        unless pid
+          Pgbus.logger.error { "[Pgbus] Failed to fork consumer for topics=#{topics.join(",")}" }
+          return
+        end
+
         @forks[pid] = { type: :consumer, config: consumer_config }
         Pgbus.logger.info { "[Pgbus] Forked consumer pid=#{pid} topics=#{topics.join(",")}" }
+      rescue Errno::EAGAIN, Errno::ENOMEM => e
+        Pgbus.logger.error { "[Pgbus] Fork failed for consumer: #{e.message}" }
       end
 
       def monitor_loop
@@ -113,7 +188,7 @@ module Pgbus
 
           process_signals
           reap_children
-          sleep(FORK_WAIT)
+          interruptible_sleep(FORK_WAIT)
         end
       end
 
@@ -144,6 +219,8 @@ module Pgbus
           fork_worker(info[:config])
         when :dispatcher
           fork_dispatcher
+        when :scheduler
+          fork_scheduler
         when :consumer
           fork_consumer(info[:config])
         end
@@ -183,7 +260,7 @@ module Pgbus
 
         until @forks.empty? || Time.now > deadline
           reap_children
-          sleep(0.5)
+          interruptible_sleep(0.5)
         end
 
         # Force kill any remaining