pgbus 0.0.1 → 0.1.1

data/lib/pgbus/pgmq_schema.rb

@@ -0,0 +1,159 @@
+# frozen_string_literal: true
+
+module Pgbus
+  # Manages embedded PGMQ SQL schema for extension-free installations.
+  #
+  # Supports three modes:
+  #   :auto      - Try extension first, fall back to embedded SQL
+  #   :extension - Require the pgmq PostgreSQL extension
+  #   :embedded  - Use vendored SQL (no extension needed)
+  #
+  # Vendored SQL files live in lib/pgbus/pgmq_schema/pgmq_v{VERSION}.sql
+  # and are exact copies of the upstream pgmq-extension/sql/pgmq.sql at each release.
+  module PgmqSchema
+    class VersionNotFoundError < StandardError; end
+
+    SCHEMA_DIR = File.expand_path("pgmq_schema", __dir__).freeze
+
+    class << self
+      # Returns the latest vendored PGMQ version string.
+      def latest_version
+        available_versions.last
+      end
+
+      # Returns sorted list of all vendored PGMQ versions.
+      def available_versions
+        Dir.glob(File.join(SCHEMA_DIR, "pgmq_v*.sql"))
+           .map { |f| File.basename(f).match(/pgmq_v(.+)\.sql/)[1] }
+           .sort_by { |v| Gem::Version.new(v) }
+      end
+
+      # Returns the filesystem path to the vendored SQL file for a given version.
+      #
+      # @param version [String] e.g. "1.11.0"
+      # @return [String] absolute path
+      # @raise [VersionNotFoundError] if no SQL file exists for that version
+      def sql_path(version)
+        path = File.join(SCHEMA_DIR, "pgmq_v#{version}.sql")
+        raise VersionNotFoundError, "No vendored PGMQ SQL for version #{version}" unless File.exist?(path)
+
+        path
+      end
+
+      # Returns the raw SQL content for a given version.
+      def sql_for_version(version)
+        File.read(sql_path(version))
+      end
+
+      # Returns the SQL to install PGMQ schema without the extension.
+      # Strips the extension-only pg_dump config blocks since they're
+      # irrelevant when not installed as an extension.
+      #
+      # @param version [String] defaults to latest
+      # @return [String] SQL
+      def install_sql(version = latest_version)
+        sql = sql_for_version(version)
+        strip_extension_only_blocks(sql)
+      end
+
+      # Returns SQL to create the version tracking table and record an installation.
+      #
+      # @param version [String] defaults to latest
+      # @return [String] SQL
+      def version_tracking_sql(version = latest_version)
+        <<~SQL
+          CREATE TABLE IF NOT EXISTS pgbus_pgmq_schema_versions (
+            id SERIAL PRIMARY KEY,
+            version VARCHAR NOT NULL,
+            installed_at TIMESTAMP WITH TIME ZONE DEFAULT now() NOT NULL,
+            install_method VARCHAR NOT NULL DEFAULT 'embedded'
+          );
+
+          INSERT INTO pgbus_pgmq_schema_versions (version, install_method)
+          VALUES ('#{version}', 'embedded');
+        SQL
+      end
+
+      # Returns SQL to record an extension-based installation in the version tracking table.
+      #
+      # @param version [String]
+      # @return [String] SQL
+      def version_tracking_extension_sql(version = latest_version)
+        <<~SQL
+          CREATE TABLE IF NOT EXISTS pgbus_pgmq_schema_versions (
+            id SERIAL PRIMARY KEY,
+            version VARCHAR NOT NULL,
+            installed_at TIMESTAMP WITH TIME ZONE DEFAULT now() NOT NULL,
+            install_method VARCHAR NOT NULL DEFAULT 'embedded'
+          );
+
+          INSERT INTO pgbus_pgmq_schema_versions (version, install_method)
+          VALUES ('#{version}', 'extension');
+        SQL
+      end
+
+      # SQL to drop all pgmq functions/types (for clean upgrade).
+      # Uses CASCADE so dependent objects are also dropped.
+      def drop_pgmq_functions_sql
+        <<~SQL
+          DO $$
+          DECLARE
+            r RECORD;
+          BEGIN
+            -- Drop all functions in pgmq schema
+            FOR r IN
+              SELECT pg_catalog.pg_get_functiondef(p.oid) AS funcdef,
+                     n.nspname || '.' || p.proname || '(' ||
+                       pg_catalog.pg_get_function_identity_arguments(p.oid) || ')' AS func_sig
+              FROM pg_catalog.pg_proc p
+              JOIN pg_catalog.pg_namespace n ON n.oid = p.pronamespace
+              WHERE n.nspname = 'pgmq'
+            LOOP
+              EXECUTE 'DROP FUNCTION IF EXISTS ' || r.func_sig || ' CASCADE';
+            END LOOP;
+
+            -- Drop custom types in pgmq schema
+            FOR r IN
+              SELECT n.nspname || '.' || t.typname AS type_name
+              FROM pg_catalog.pg_type t
+              JOIN pg_catalog.pg_namespace n ON n.oid = t.typnamespace
+              WHERE n.nspname = 'pgmq'
+                AND t.typtype = 'c'
+                AND NOT EXISTS (
+                  SELECT 1 FROM pg_catalog.pg_class c
+                  WHERE c.reltype = t.oid
+                )
+            LOOP
+              EXECUTE 'DROP TYPE IF EXISTS ' || r.type_name || ' CASCADE';
+            END LOOP;
+          END $$;
+        SQL
+      end
+
+      private
+
+      # Strips extension-specific blocks (pg_extension_config_dump, pg_depend checks)
+      # that only work when pgmq is installed as an extension.
+      def strip_extension_only_blocks(sql)
+        # Remove the DO block that conditionally creates schema only when extension is missing.
+        # Replace with unconditional schema creation.
+        sql = sql.sub(
+          /DO\s*\$\$\s*BEGIN\s*IF\s*\(SELECT\s+NOT\s+EXISTS.*?END\s*\$\$;/m,
+          "CREATE SCHEMA IF NOT EXISTS pgmq;"
+        )
+
+        # Remove pg_extension_config_dump blocks
+        sql = sql.gsub(
+          /DO\s*\$\$\s*BEGIN\s*IF\s+EXISTS\(SELECT\s+1\s+FROM\s+pg_extension.*?END\s*\$\$;/m,
+          ""
+        )
+
+        # Remove _belongs_to_pgmq function (checks pg_depend on extension)
+        sql.gsub(
+          /CREATE FUNCTION pgmq\._belongs_to_pgmq.*?LANGUAGE plpgsql;/m,
+          ""
+        )
+      end
+    end
+  end
+end

data/lib/pgbus/process/consumer.rb

@@ -54,23 +54,23 @@ module Pgbus
 
       def consume
         idle = @pool.max_length - @pool.queue_length
-        return sleep(config.polling_interval) if idle <= 0
+        return interruptible_sleep(config.polling_interval) if idle <= 0
 
-        messages = @queue_names.flat_map do |queue_name|
-          Pgbus.client.read_batch(queue_name, qty: idle) || []
+        tagged_messages = @queue_names.flat_map do |queue_name|
+          (Pgbus.client.read_batch(queue_name, qty: idle) || []).map { |m| [queue_name, m] }
         end.first(idle)
 
-        if messages.empty?
-          sleep(config.polling_interval)
+        if tagged_messages.empty?
+          interruptible_sleep(config.polling_interval)
           return
         end
 
-        messages.each do |message|
-          @pool.post { handle_message(message) }
+        tagged_messages.each do |queue_name, message|
+          @pool.post { handle_message(message, queue_name) }
         end
       end
 
-      def handle_message(message)
+      def handle_message(message, queue_name)
         raw = JSON.parse(message.message)
         routing_key = raw.dig("headers", "routing_key") || raw["routing_key"]
 
@@ -80,7 +80,6 @@ module Pgbus
           handler.process(message)
         end
 
-        queue_name = message.respond_to?(:queue_name) ? message.queue_name : @queue_names.first
         Pgbus.client.archive_message(queue_name, message.msg_id.to_i)
       rescue StandardError => e
         Pgbus.logger.error { "[Pgbus] Consumer error: #{e.class}: #{e.message}" }

data/lib/pgbus/process/dispatcher.rb

@@ -6,10 +6,11 @@ module Pgbus
       include SignalHandler
 
       # Maintenance runs on coarser intervals than the main loop
-      CLEANUP_INTERVAL = 3600       # Run idempotency cleanup every hour
-      REAP_INTERVAL = 300           # Run stale process reaping every 5 minutes
-      CONCURRENCY_INTERVAL = 300    # Run concurrency cleanup every 5 minutes
-      BATCH_CLEANUP_INTERVAL = 3600 # Run batch cleanup every hour
+      CLEANUP_INTERVAL = 3600               # Run idempotency cleanup every hour
+      REAP_INTERVAL = 300                   # Run stale process reaping every 5 minutes
+      CONCURRENCY_INTERVAL = 300            # Run concurrency cleanup every 5 minutes
+      BATCH_CLEANUP_INTERVAL = 3600         # Run batch cleanup every hour
+      RECURRING_CLEANUP_INTERVAL = 3600     # Run recurring execution cleanup every hour
 
       attr_reader :config
 
@@ -20,6 +21,7 @@ module Pgbus
         @last_reap_at = Time.now
         @last_concurrency_at = Time.now
         @last_batch_cleanup_at = Time.now
+        @last_recurring_cleanup_at = Time.now
       end
 
       def run
@@ -38,7 +40,7 @@ module Pgbus
           run_maintenance
           break if @shutting_down
 
-          sleep(config.dispatch_interval)
+          interruptible_sleep(config.dispatch_interval)
         end
 
         shutdown
@@ -76,35 +78,28 @@ module Pgbus
           cleanup_batches
           @last_batch_cleanup_at = now
         end
+
+        if now - @last_recurring_cleanup_at >= RECURRING_CLEANUP_INTERVAL
+          cleanup_recurring_executions
+          @last_recurring_cleanup_at = now
+        end
       rescue StandardError => e
         Pgbus.logger.error { "[Pgbus] Dispatcher maintenance error: #{e.message}" }
       end
 
       def cleanup_processed_events
-        return unless defined?(ActiveRecord::Base)
-
         ttl = config.idempotency_ttl
         return unless ttl&.positive?
 
-        deleted = ActiveRecord::Base.connection.delete(
-          "DELETE FROM pgbus_processed_events WHERE processed_at < $1",
-          "Pgbus Idempotency Cleanup",
-          [Time.now.utc - ttl]
-        )
+        deleted = ProcessedEvent.expired(Time.now.utc - ttl).delete_all
         Pgbus.logger.debug { "[Pgbus] Cleaned up #{deleted} expired processed events" } if deleted.positive?
       rescue StandardError => e
         Pgbus.logger.warn { "[Pgbus] Idempotency cleanup failed: #{e.message}" }
       end
 
       def reap_stale_processes
-        return unless defined?(ActiveRecord::Base)
-
         threshold = Heartbeat::ALIVE_THRESHOLD
-        deleted = ActiveRecord::Base.connection.delete(
-          "DELETE FROM pgbus_processes WHERE last_heartbeat_at < $1",
-          "Pgbus Stale Process Reap",
-          [Time.now.utc - threshold]
-        )
+        deleted = ProcessEntry.stale(Time.now.utc - threshold).delete_all
         Pgbus.logger.info { "[Pgbus] Reaped #{deleted} stale processes" } if deleted.positive?
       rescue StandardError => e
         Pgbus.logger.warn { "[Pgbus] Stale process reaping failed: #{e.message}" }
@@ -123,11 +118,8 @@ module Pgbus
       end
 
       def release_blocked_for_key(key)
-        released = Concurrency::BlockedExecution.release_next(key)
-        return unless released
-
-        Pgbus.client.send_message(released[:queue_name], released[:payload])
-        Pgbus.logger.debug { "[Pgbus] Released blocked execution for key: #{key}" }
+        promoted = Concurrency::BlockedExecution.promote_next(key, client: Pgbus.client)
+        Pgbus.logger.debug { "[Pgbus] Released blocked execution for key: #{key}" } if promoted
       rescue StandardError => e
         Pgbus.logger.warn { "[Pgbus] Failed to release blocked execution for #{key}: #{e.message}" }
       end
@@ -139,6 +131,16 @@ module Pgbus
         Pgbus.logger.warn { "[Pgbus] Batch cleanup failed: #{e.message}" }
       end
 
+      def cleanup_recurring_executions
+        retention = config.recurring_execution_retention
+        return unless retention&.positive?
+
+        deleted = RecurringExecution.older_than(Time.now.utc - retention).delete_all
+        Pgbus.logger.debug { "[Pgbus] Cleaned up #{deleted} old recurring executions" } if deleted.positive?
+      rescue StandardError => e
+        Pgbus.logger.warn { "[Pgbus] Recurring execution cleanup failed: #{e.message}" }
+      end
+
       def start_heartbeat
         @heartbeat = Heartbeat.new(kind: "dispatcher", metadata: { pid: ::Process.pid })
         @heartbeat.start

data/lib/pgbus/process/heartbeat.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require "concurrent"
+require "socket"
 
 module Pgbus
   module Process
@@ -8,7 +9,7 @@ module Pgbus
       INTERVAL = 60 # seconds
       ALIVE_THRESHOLD = 300 # 5 minutes
 
-      attr_reader :process_record
+      attr_reader :process_entry
 
       def initialize(kind:, metadata: {})
         @kind = kind
@@ -28,13 +29,9 @@ module Pgbus
       end
 
       def beat
-        return unless @process_id && defined?(ActiveRecord::Base)
+        return unless @process_id
 
-        ActiveRecord::Base.connection.execute(
-          "UPDATE pgbus_processes SET last_heartbeat_at = NOW() WHERE id = $1",
-          "Pgbus Heartbeat",
-          [@process_id]
-        )
+        ProcessEntry.where(id: @process_id).update_all(last_heartbeat_at: Time.current)
       rescue StandardError => e
         Pgbus.logger.warn { "[Pgbus] Heartbeat failed: #{e.message}" }
       end
@@ -42,29 +39,24 @@ module Pgbus
       private
 
       def register_process
-        return unless defined?(ActiveRecord::Base)
-
-        result = ActiveRecord::Base.connection.exec_insert(
-          "INSERT INTO pgbus_processes (kind, hostname, pid, metadata, last_heartbeat_at, created_at, updated_at) " \
-          "VALUES ($1, $2, $3, $4, NOW(), NOW(), NOW()) RETURNING id",
-          "Pgbus Register Process",
-          [@kind, Socket.gethostname, ::Process.pid, JSON.generate(@metadata)]
+        record = ProcessEntry.create!(
+          kind: @kind,
+          hostname: Socket.gethostname,
+          pid: ::Process.pid,
+          metadata: @metadata,
+          last_heartbeat_at: Time.current
         )
-        @process_id = result.first["id"]
+        @process_id = record.id
       rescue StandardError => e
         Pgbus.logger.warn { "[Pgbus] Process registration failed: #{e.message}" }
       end
 
       def deregister_process
-        return unless @process_id && defined?(ActiveRecord::Base)
+        return unless @process_id
 
-        ActiveRecord::Base.connection.execute(
-          "DELETE FROM pgbus_processes WHERE id = $1",
-          "Pgbus Deregister Process",
-          [@process_id]
-        )
-      rescue StandardError
-        # Best effort — process is exiting anyway
+        ProcessEntry.where(id: @process_id).delete_all
+      rescue StandardError => e
+        Pgbus.logger.warn { "[Pgbus] Process deregistration failed: #{e.message}" }
       end
     end
   end

data/lib/pgbus/process/signal_handler.rb

@@ -10,9 +10,14 @@ module Pgbus
       def setup_signals
         @signal_queue = Queue.new
         @previous_handlers = {}
+        @self_pipe_r, @self_pipe_w = IO.pipe
 
         %w[INT TERM QUIT].each do |sig|
-          @previous_handlers[sig] = trap(sig) { @signal_queue << sig }
+          @previous_handlers[sig] = trap(sig) do
+            @signal_queue << sig
+            # Wake any IO.select / interruptible_sleep call
+            @self_pipe_w.write_nonblock(".", exception: false)
+          end
         end
       end
 
@@ -20,6 +25,8 @@ module Pgbus
         @previous_handlers&.each do |sig, handler|
           trap(sig, handler || "DEFAULT")
         end
+        @self_pipe_r&.close unless @self_pipe_r&.closed?
+        @self_pipe_w&.close unless @self_pipe_w&.closed?
       end
 
       def process_signals
@@ -37,6 +44,13 @@ module Pgbus
         end
       end
 
+      # Sleep that can be interrupted by signals. Use this instead of Kernel#sleep
+      # in any loop that needs to respond to INT/TERM/QUIT promptly.
+      def interruptible_sleep(seconds)
+        @self_pipe_r.wait_readable(seconds)
+        drain_self_pipe
+      end
+
       def graceful_shutdown
         raise NotImplementedError
       end
@@ -44,6 +58,14 @@ module Pgbus
       def immediate_shutdown
         raise NotImplementedError
       end
+
+      private
+
+      def drain_self_pipe
+        loop { @self_pipe_r.read_nonblock(256) }
+      rescue IO::WaitReadable, EOFError
+        # pipe drained
+      end
     end
   end
 end

data/lib/pgbus/process/supervisor.rb

@@ -50,6 +50,9 @@ module Pgbus
         # Boot dispatcher
         fork_dispatcher
 
+        # Boot recurring scheduler if configured
+        boot_scheduler
+
         # Boot event consumers if configured
         boot_consumers
       end
@@ -83,6 +86,50 @@ module Pgbus
         Pgbus.logger.info { "[Pgbus] Forked dispatcher pid=#{pid}" }
       end
 
+      def boot_scheduler
+        return if config.skip_recurring
+        return unless recurring_tasks_configured?
+
+        fork_scheduler
+      end
+
+      def fork_scheduler
+        pid = fork do
+          restore_signals
+          setup_child_signals
+          load_rails_app
+          load_recurring_config
+          scheduler = Recurring::Scheduler.new(config: config)
+          scheduler.run
+        end
+
+        @forks[pid] = { type: :scheduler }
+        Pgbus.logger.info { "[Pgbus] Forked scheduler pid=#{pid}" }
+      end
+
+      def recurring_tasks_configured?
+        return true if config.recurring_tasks&.any?
+        return true if config.recurring_tasks_file && File.exist?(config.recurring_tasks_file.to_s)
+
+        # Check default location
+        if defined?(Rails)
+          default_path = Rails.root.join("config", "recurring.yml")
+          return File.exist?(default_path.to_s)
+        end
+
+        false
+      end
+
+      def load_recurring_config
+        return if config.recurring_tasks&.any?
+
+        path = config.recurring_tasks_file
+        path ||= defined?(Rails) ? Rails.root.join("config", "recurring.yml") : nil
+        return unless path && File.exist?(path.to_s)
+
+        config.recurring_tasks = Recurring::ConfigLoader.load(path)
+      end
+
       def boot_consumers
         return unless config.event_consumers
 
@@ -113,7 +160,7 @@ module Pgbus
 
           process_signals
           reap_children
-          sleep(FORK_WAIT)
+          interruptible_sleep(FORK_WAIT)
         end
       end
 
@@ -144,6 +191,8 @@ module Pgbus
           fork_worker(info[:config])
         when :dispatcher
           fork_dispatcher
+        when :scheduler
+          fork_scheduler
         when :consumer
           fork_consumer(info[:config])
         end
@@ -183,7 +232,7 @@ module Pgbus
 
         until @forks.empty? || Time.now > deadline
           reap_children
-          sleep(0.5)
+          interruptible_sleep(0.5)
         end
 
         # Force kill any remaining

data/lib/pgbus/process/worker.rb

@@ -28,6 +28,7 @@ module Pgbus
       def run
         setup_signals
         start_heartbeat
+        resolve_wildcard_queues
         Pgbus.logger.info { "[Pgbus] Worker started: queues=#{queues.join(",")} threads=#{threads} pid=#{::Process.pid}" }
 
         loop do
@@ -56,29 +57,31 @@ module Pgbus
 
       def claim_and_execute
         idle = @pool.max_length - @pool.queue_length
-        return sleep(config.polling_interval) if idle <= 0
+        return interruptible_sleep(config.polling_interval) if idle <= 0
 
-        messages = fetch_messages(idle)
+        tagged_messages = fetch_messages(idle)
 
-        if messages.empty?
-          sleep(config.polling_interval)
+        if tagged_messages.empty?
+          interruptible_sleep(config.polling_interval)
           return
         end
 
-        messages.each do |message|
-          queue_name = message.respond_to?(:queue_name) ? message.queue_name : queues.first
+        tagged_messages.each do |queue_name, message|
           @pool.post { process_message(message, queue_name) }
         end
       end
 
+      # Returns an array of [queue_name, message] pairs so we always know
+      # which queue each message came from (PGMQ messages don't carry this).
       def fetch_messages(qty)
         if queues.size == 1
-          Pgbus.client.read_batch(queues.first, qty: qty) || []
+          queue = queues.first
+          messages = Pgbus.client.read_batch(queue, qty: qty) || []
+          messages.map { |m| [queue, m] }
         else
-          # Multi-queue read: read from each queue proportionally
           per_queue = [(qty / queues.size.to_f).ceil, 1].max
           queues.flat_map do |q|
-            Pgbus.client.read_batch(q, qty: per_queue) || []
+            (Pgbus.client.read_batch(q, qty: per_queue) || []).map { |m| [q, m] }
           end.first(qty)
         end
       rescue StandardError => e
@@ -95,6 +98,31 @@ module Pgbus
         Pgbus.logger.error { "[Pgbus] Unhandled error processing message: #{e.message}" }
       end
 
+      # Resolve "*" to all non-DLQ queues from pgmq.meta, stripping the prefix.
+      # Called once at startup. If no wildcard, this is a no-op.
+      def resolve_wildcard_queues
+        return unless @queues.include?("*")
+
+        dlq_suffix = config.dead_letter_queue_suffix
+        prefix = "#{config.queue_prefix}_"
+
+        all_queues = ActiveRecord::Base.connection.select_values("SELECT queue_name FROM pgmq.meta ORDER BY queue_name")
+        resolved = all_queues
+                   .reject { |q| q.end_with?(dlq_suffix) }
+                   .map { |q| q.delete_prefix(prefix) }
+
+        if resolved.empty?
+          Pgbus.logger.warn { "[Pgbus] Wildcard queue '*' resolved to no queues — falling back to default" }
+          @queues = [config.default_queue]
+        else
+          @queues = resolved
+          Pgbus.logger.info { "[Pgbus] Wildcard queue '*' resolved to: #{@queues.join(", ")}" }
+        end
+      rescue StandardError => e
+        Pgbus.logger.error { "[Pgbus] Failed to resolve wildcard queues: #{e.message} — falling back to default" }
+        @queues = [config.default_queue]
+      end
+
       def recycle_needed?
         exceeded_max_jobs? || exceeded_max_memory? || exceeded_max_lifetime?
       end

data/lib/pgbus/recurring/already_recorded.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module Pgbus
+  module Recurring
+    class AlreadyRecorded < Pgbus::Error; end
+  end
+end

data/lib/pgbus/recurring/command_job.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Pgbus
+  module Recurring
+    # Job class for command-based recurring tasks.
+    # Executes a Ruby command string, similar to solid_queue's RecurringJob.
+    #
+    # NOTE: Only use this with trusted commands from config/recurring.yml.
+    # Never expose this to user input.
+    class CommandJob < ::ActiveJob::Base
+      def perform(command)
+        eval(command) # rubocop:disable Security/Eval
+      end
+    end
+  end
+end

data/lib/pgbus/recurring/config_loader.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+require "yaml"
+require "erb"
+
+module Pgbus
+  module Recurring
+    module ConfigLoader
+      module_function
+
+      def load(path, env: nil)
+        return {} unless path && File.exist?(path.to_s)
+
+        env ||= detect_env
+        raw = File.read(path)
+        parsed = YAML.safe_load(ERB.new(raw).result, permitted_classes: [Symbol], aliases: true)
+        return {} unless parsed.is_a?(Hash)
+
+        # If the parsed hash has an environment key, use that subtree
+        parsed.key?(env) ? parsed.fetch(env, {}) : parsed
+      rescue StandardError => e
+        Pgbus.logger.error { "[Pgbus] Failed to load recurring config from #{path}: #{e.message}" }
+        {}
+      end
+
+      def detect_env
+        if defined?(Rails)
+          Rails.env.to_s
+        else
+          ENV.fetch("PGBUS_ENV", "development")
+        end
+      end
+    end
+  end
+end