pgbus 0.19.0 → 0.19.1

data/lib/pgbus/deploy/task.rb

@@ -0,0 +1,105 @@
+# frozen_string_literal: true
+
+module Pgbus
+  module Deploy
+    # One declared deploy task (issue #536): an entry of
+    # config/deploy_tasks.yml. The sibling of Recurring::Task without a
+    # schedule: it runs once per (key, version) after a rollover, or after
+    # every rollover when every_deploy is set.
+    class Task
+      DEFAULT_VERSION = "1"
+
+      attr_reader :key, :class_name, :command, :version, :every_deploy, :queue_name,
+                  :arguments, :priority, :description
+
+      def self.from_configuration(key, **options)
+        options = options.transform_keys(&:to_sym)
+        new(
+          key: key.to_s,
+          class_name: options[:class],
+          command: options[:command],
+          version: options.fetch(:version, DEFAULT_VERSION),
+          explicit_version: options.key?(:version),
+          every_deploy: options.fetch(:every_deploy, false),
+          queue_name: options[:queue],
+          arguments: options[:args].is_a?(Hash) ? [options[:args]] : Array(options[:args]),
+          priority: options.fetch(:priority, 0).to_i,
+          description: options[:description]
+        )
+      end
+
+      def initialize(key:, class_name: nil, command: nil, version: DEFAULT_VERSION, explicit_version: false,
+                     every_deploy: false, queue_name: nil, arguments: [], priority: 0, description: nil)
+        @key = key
+        @class_name = class_name
+        @command = command
+        @version = version.to_s.strip
+        @explicit_version = explicit_version
+        @every_deploy = every_deploy
+        @queue_name = queue_name
+        @arguments = arguments || []
+        @priority = priority || 0
+        @description = description
+        @errors = []
+        @warnings = []
+      end
+
+      def valid?
+        @errors = []
+        @warnings = []
+        validate!
+        @errors.empty?
+      end
+
+      def errors
+        valid? unless @validated
+        @errors
+      end
+
+      def warnings
+        valid? unless @validated
+        @warnings
+      end
+
+      # What an arm for this task is stamped with: its version, or the
+      # revision it runs on for an every_deploy task.
+      def armed_for(revision)
+        every_deploy ? revision : version
+      end
+
+      def job_class
+        class_name&.safe_constantize
+      end
+
+      def to_h
+        {
+          key: key, class_name: class_name, command: command, version: version, every_deploy: every_deploy,
+          queue_name: queue_name, arguments: arguments, priority: priority, description: description
+        }
+      end
+
+      private
+
+      def validate!
+        @validated = true
+        @errors << "Key must not be blank" if key.to_s.strip.empty?
+        validate_target!
+        @errors << "Version must not be blank" if version.empty?
+        @errors << "every_deploy must be true or false" unless [true, false].include?(every_deploy)
+        return unless every_deploy == true && @explicit_version
+
+        @warnings << "every_deploy is set, so version is ignored (it runs after every rollover)"
+      end
+
+      def validate_target!
+        { class: class_name, command: command }.each do |name, value|
+          @errors << "#{name} must be a String" unless value.nil? || value.is_a?(String)
+        end
+        has_class = !class_name.to_s.strip.empty?
+        has_command = !command.to_s.strip.empty?
+        @errors << "Either class or command is required" unless has_class || has_command
+        @errors << "Declare either class or command, not both" if has_class && has_command
+      end
+    end
+  end
+end

data/lib/pgbus/engine.rb

@@ -58,6 +58,23 @@ module Pgbus
       end
     end
 
+    # config/deploy_tasks.yml (issue #536): same shape and loader as
+    # recurring.yml. Explicit deploy_tasks_files replace the default file, so
+    # this runs after the app's config/initializers set them.
+    initializer "pgbus.deploy_tasks", after: :load_config_initializers do |app|
+      config = Pgbus.configuration
+      next if config.deploy_tasks
+
+      files = config.deploy_tasks_files
+      default_path = app.root.join("config", "deploy_tasks.yml")
+      tasks = if files
+                Pgbus::Recurring::ConfigLoader.load_all(files)
+              elsif default_path.exist?
+                Pgbus::Recurring::ConfigLoader.load(default_path)
+              end
+      config.deploy_tasks = tasks if tasks.present?
+    end
+
     initializer "pgbus.db" do
       ActiveSupport.on_load(:active_record) do
         Pgbus::BusRecord.connects_to(**Pgbus.configuration.connects_to) if Pgbus.configuration.connects_to

data/lib/pgbus/generators/migration_detector.rb

@@ -77,6 +77,7 @@ module Pgbus
         add_queue_states: "pgbus:add_queue_states",
         add_outbox: "pgbus:add_outbox",
         add_recurring: "pgbus:add_recurring",
+        add_deploy_tasks: "pgbus:add_deploy_tasks",
         add_failed_events_index: "pgbus:add_failed_events_index",
         add_processed_event_completion: "pgbus:add_processed_event_completion",
         add_batch_executions: "pgbus:add_batch_executions",
@@ -99,6 +100,7 @@ module Pgbus
         add_queue_states: "queue states table (pause/resume)",
         add_outbox: "outbox entries table (transactional outbox)",
         add_recurring: "recurring tasks + executions tables",
+        add_deploy_tasks: "deploy tasks + runs tables (one-time jobs run after a rollover)",
         add_failed_events_index: "unique index on pgbus_failed_events (queue_name, msg_id)",
         add_processed_event_completion: "completed_at on pgbus_processed_events (two-phase idempotency claim)",
         add_batch_executions: "batch execution-row tracking (self-healing completion, on_failure rename)",
@@ -127,6 +129,7 @@ module Pgbus
           *queue_states_migrations,
           *outbox_migrations,
           *recurring_migrations,
+          *deploy_tasks_migrations,
           *failed_events_index_migrations,
           *processed_event_completion_migrations,
           *batch_executions_migrations,
@@ -215,6 +218,14 @@ module Pgbus
         [:add_recurring]
       end
 
+      # Same shape as recurring_migrations: plain create_table in the
+      # template, so the generator is queued only while a table is absent.
+      def deploy_tasks_migrations
+        return [] if table_exists?("pgbus_deploy_tasks") && table_exists?("pgbus_deploy_task_runs")
+
+        [:add_deploy_tasks]
+      end
+
       def failed_events_index_migrations
         # pgbus_failed_events is created by pgbus:install, but the
         # unique (queue_name, msg_id) index was added later via its own

data/lib/pgbus/instrumentation.rb

@@ -28,6 +28,9 @@ module Pgbus
   #   pgbus.stream.broadcast       — stream broadcast (sync or deferred)
   #   pgbus.outbox.publish         — outbox row created
   #   pgbus.recurring.enqueue      — scheduler enqueued a due recurring task
+  #   pgbus.deploy_task.enqueue    — a deploy task enqueue was attempted (after a rollover or by
+  #                                  Run now); payload: task, version, revision, triggered_by, plus
+  #                                  :exception/:exception_object when the enqueue raised
   #   pgbus.worker.recycle         — worker hit a recycle threshold (reason/jobs_processed/memory_mb/lifetime_seconds)
   #   pgbus.consumer.recycle       — consumer hit a recycle threshold (same payload shape as pgbus.worker.recycle)
   #   pgbus.serializer.serialize   — job/event serialization

data/lib/pgbus/mcp/server.rb

@@ -26,7 +26,8 @@ module Pgbus
         Tools::ConcurrencyTool,
         Tools::ThroughputTool,
         Tools::StatsTool,
-        Tools::RecurringTool
+        Tools::RecurringTool,
+        Tools::DeployTasksTool
       ].freeze
 
       INSTRUCTIONS = <<~TEXT

data/lib/pgbus/mcp/tools/deploy_tasks_tool.rb

@@ -0,0 +1,86 @@
+# frozen_string_literal: true
+
+module Pgbus
+  module MCP
+    module Tools
+      # Deploy tasks with their state and the rollover the scheduler waits on.
+      # Maps to DataSource#deploy_tasks and #deploy_rollover. Job arguments are
+      # not part of the list payload (and are dropped here if a data source
+      # ever adds them), so the tool never carries them.
+      class DeployTasksTool < BaseTool
+        tool_name "pgbus_deploy_tasks"
+        title "Pgbus Deploy Tasks"
+        description <<~DESC
+          Deploy tasks: one-time jobs the scheduler enqueues once every live pgbus process runs the new app revision. Lists each task's state (disabled, waiting, due, ran, never) and the rollover status the scheduler waits on.
+          Returns every task unless page or per_page is given; then it is
+          paginated (per_page defaults to 100, capped at 100). The response
+          carries page, per_page, total (capped at 10000 when paginated; the
+          exact row count when not) and has_more; per_page is null when
+          unpaginated.
+        DESC
+
+        MAX_PER_PAGE = 100
+        MAX_PAGE = 1_000
+
+        input_schema(
+          properties: {
+            page: { type: "integer", description: "1-based page number (default 1, max 1000).", minimum: 1 },
+            per_page: { type: "integer", description: "Rows per page (default 100 once paginating, max 100).", minimum: 1 }
+          },
+          required: []
+        )
+
+        def self.call(page: nil, per_page: nil, server_context: nil)
+          data_source = data_source_from(server_context)
+          return unpaged(data_source) if page.nil? && per_page.nil?
+
+          per_page = (per_page || MAX_PER_PAGE).to_i.clamp(1, MAX_PER_PAGE)
+          page = (page || 1).to_i.clamp(1, MAX_PAGE)
+          tasks = data_source.deploy_tasks(page: page, per_page: per_page)
+          count = data_source.list_count(:deploy_tasks)
+
+          json_response(
+            payload(data_source, tasks).merge(
+              page: page, per_page: per_page, total: count.total,
+              has_more: more_pages?(count, page: page, per_page: per_page, shown: tasks.size)
+            )
+          )
+        end
+
+        def self.unpaged(data_source)
+          tasks = data_source.deploy_tasks
+          json_response(
+            payload(data_source, tasks).merge(page: 1, per_page: nil, total: tasks.size, has_more: false)
+          )
+        end
+        private_class_method :unpaged
+
+        def self.payload(data_source, tasks)
+          rollover = data_source.deploy_rollover
+          { rollover: serialize(rollover), tasks: tasks.map { |row| task_hash(row, rollover) } }
+        end
+        private_class_method :payload
+
+        def self.task_hash(row, rollover)
+          state = Pgbus::Web::DeployTaskState.present(row, rollover)
+          serialize(row.except(:arguments)).merge(
+            "state" => state.state, "reason" => state.reason_key, "reason_args" => serialize(state.reason_args)
+          )
+        end
+        private_class_method :task_hash
+
+        # Times as ISO 8601 UTC, symbols as strings, string keys throughout.
+        def self.serialize(value)
+          case value
+          when Hash then value.to_h { |key, val| [key.to_s, serialize(val)] }
+          when Array then value.map { |val| serialize(val) }
+          when Symbol then value.to_s
+          when Time, DateTime, ActiveSupport::TimeWithZone then value.utc.iso8601
+          else value
+          end
+        end
+        private_class_method :serialize
+      end
+    end
+  end
+end

data/lib/pgbus/mcp.rb

@@ -28,6 +28,7 @@ module Pgbus
       pgbus/mcp/tools/throughput_tool
       pgbus/mcp/tools/stats_tool
       pgbus/mcp/tools/recurring_tool
+      pgbus/mcp/tools/deploy_tasks_tool
       pgbus/mcp/server
       pgbus/mcp/runner
       pgbus/mcp/rack_app

data/lib/pgbus/outbox/poller.rb

@@ -139,6 +139,7 @@ module Pgbus
       def start_heartbeat
         @heartbeat = Process::Heartbeat.new(
           kind: "outbox_poller",
+          config: @config,
           metadata: { pid: ::Process.pid, started_at: Time.now.to_f, supervisor_pid: ::Process.ppid }
         )
         @heartbeat.start

data/lib/pgbus/process/consumer.rb

@@ -615,6 +615,7 @@ module Pgbus
       def start_heartbeat
         @heartbeat = Heartbeat.new(
           kind: "consumer",
+          config: @config,
           metadata: {
             topics: topics, threads: threads, pid: ::Process.pid, execution_mode: @execution_mode,
             started_at: @started_at.to_f, supervisor_pid: ::Process.ppid

data/lib/pgbus/process/dispatcher.rb

@@ -13,6 +13,7 @@ module Pgbus
       CONCURRENCY_INTERVAL = 300            # Run concurrency cleanup every 5 minutes
       BATCH_CLEANUP_INTERVAL = 3600         # Run batch cleanup every hour
       RECURRING_CLEANUP_INTERVAL = 3600     # Run recurring execution cleanup every hour
+      DEPLOY_RUN_CLEANUP_INTERVAL = 3600    # Run deploy task run cleanup every hour
       ARCHIVE_COMPACTION_INTERVAL = 3600    # Run archive compaction every hour
       OUTBOX_CLEANUP_INTERVAL = 3600 # Run outbox cleanup every hour
       JOB_LOCK_CLEANUP_INTERVAL = 300 # Run job lock cleanup every 5 minutes
@@ -52,7 +53,8 @@ module Pgbus
       MAINTENANCE_TIMESTAMPS = %i[
         @last_cleanup_at @last_reap_at @last_concurrency_at @last_batch_cleanup_at
         @last_batch_sweep_at
-        @last_recurring_cleanup_at @last_archive_compaction_at @last_stream_archive_compaction_at
+        @last_recurring_cleanup_at @last_deploy_run_cleanup_at @last_archive_compaction_at
+        @last_stream_archive_compaction_at
         @last_outbox_cleanup_at @last_job_lock_cleanup_at @last_stats_cleanup_at
         @last_orphan_stream_sweep_at @last_table_maintenance_at
       ].freeze
@@ -236,6 +238,7 @@ module Pgbus
         results << run_if_due(now, :@last_batch_cleanup_at, BATCH_CLEANUP_INTERVAL) { cleanup_batches }
         results << run_if_due(now, :@last_batch_sweep_at, config.batch_sweep_interval) { sweep_stalled_batches }
         results << run_if_due(now, :@last_recurring_cleanup_at, RECURRING_CLEANUP_INTERVAL) { cleanup_recurring_executions }
+        results << run_if_due(now, :@last_deploy_run_cleanup_at, DEPLOY_RUN_CLEANUP_INTERVAL) { cleanup_deploy_task_runs }
         results << run_if_due(now, :@last_archive_compaction_at, ARCHIVE_COMPACTION_INTERVAL) { compact_archives }
         results << run_if_due(now, :@last_stream_archive_compaction_at, ARCHIVE_COMPACTION_INTERVAL) { prune_stream_archives }
         results << run_if_due(now, :@last_outbox_cleanup_at, OUTBOX_CLEANUP_INTERVAL) { cleanup_outbox }
@@ -681,6 +684,19 @@ module Pgbus
         e
       end
 
+      # Guarded on the table: deploy tasks default on, and an install that has
+      # not run pgbus:add_deploy_tasks must not warn every hour.
+      def cleanup_deploy_task_runs
+        retention = config.deploy_task_run_retention
+        return unless retention&.positive? && DeployTaskRun.table_exists?
+
+        deleted = DeployTaskRun.prunable_before(Time.current - retention).delete_all
+        Pgbus.logger.debug { "[Pgbus] Cleaned up #{deleted} old deploy task runs" } if deleted.positive?
+      rescue StandardError => e
+        log_maintenance_failure("Deploy task run cleanup", e)
+        e
+      end
+
       def monotonic_now
         ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
       end
@@ -697,6 +713,7 @@ module Pgbus
       def start_heartbeat
         @heartbeat = Heartbeat.new(
           kind: "dispatcher",
+          config: @config,
           metadata: { pid: ::Process.pid, started_at: Time.now.to_f, supervisor_pid: ::Process.ppid },
           loop_tick_supplier: -> { @loop_tick_at.get }
         )

data/lib/pgbus/process/heartbeat.rb

@@ -11,9 +11,14 @@ module Pgbus
 
       attr_reader :process_entry
 
-      def initialize(kind:, metadata: {}, on_beat: nil, loop_tick_supplier: nil, metadata_supplier: nil)
+      # config: the owning process's configuration, for the app revision.
+      def initialize(kind:, metadata: {}, on_beat: nil, loop_tick_supplier: nil, metadata_supplier: nil,
+                     config: Pgbus.configuration)
         @kind = kind
-        @metadata = metadata
+        # Every kind's row carries the app revision from its first
+        # registration on, nil included: the deploy-task rollover gate reads
+        # it (issue #536), and beat_metadata dups this hash so it survives.
+        @metadata = metadata.merge(revision: Deploy::Revision.current(config))
         @on_beat = on_beat
         @loop_tick_supplier = loop_tick_supplier
         @metadata_supplier = metadata_supplier

data/lib/pgbus/process/supervisor.rb

@@ -425,9 +425,12 @@ module Pgbus
         ErrorReporter.report(e, { action: "fork_dispatcher" })
       end
 
+      # The scheduler runs recurring tasks and deploy tasks (issue #536);
+      # either one being enabled and configured forks it.
       def boot_scheduler
-        return unless config.recurring_enabled
-        return unless recurring_tasks_configured?
+        recurring = config.recurring_enabled && recurring_tasks_configured?
+        deploy = config.deploy_tasks_enabled && deploy_tasks_configured?
+        return unless recurring || deploy
 
         fork_scheduler
       end
@@ -438,7 +441,8 @@ module Pgbus
           restore_signals
           setup_child_process
           load_rails_app
-          load_recurring_config
+          load_recurring_config if config.recurring_enabled
+          load_deploy_tasks_config if config.deploy_tasks_enabled
           bootstrap_queues!
           scheduler = Recurring::Scheduler.new(config: config)
           scheduler.run
@@ -488,6 +492,33 @@ module Pgbus
         config.recurring_tasks = Recurring::ConfigLoader.load(path)
       end
 
+      def deploy_tasks_configured?
+        return true if config.deploy_tasks&.any?
+        # Explicit files replace the default file, as in load_deploy_tasks_config.
+        return config.deploy_tasks_files.any? { |f| File.exist?(f.to_s) } if config.deploy_tasks_files
+
+        path = default_deploy_tasks_path
+        path ? File.exist?(path.to_s) : false
+      end
+
+      # The engine initializer loads config/deploy_tasks.yml in a Rails boot;
+      # this covers a scheduler fork whose config was not loaded that way.
+      def load_deploy_tasks_config
+        return if config.deploy_tasks&.any?
+
+        files = config.deploy_tasks_files
+        tasks = if files
+                  Recurring::ConfigLoader.load_all(files)
+                elsif (path = default_deploy_tasks_path) && File.exist?(path.to_s)
+                  Recurring::ConfigLoader.load(path)
+                end
+        config.deploy_tasks = tasks if tasks&.any?
+      end
+
+      def default_deploy_tasks_path
+        Rails.root.join("config", "deploy_tasks.yml") if defined?(Rails) && Rails.respond_to?(:root) && Rails.root
+      end
+
       def boot_consumers
         return unless config.event_consumers
 
@@ -903,6 +934,7 @@ module Pgbus
       def start_heartbeat
         @heartbeat = Heartbeat.new(
           kind: "supervisor",
+          config: @config,
           metadata: { pid: ::Process.pid, hostname: Socket.gethostname, started_at: Time.now.to_f }
         )
         @heartbeat.start

data/lib/pgbus/process/worker.rb

@@ -836,6 +836,7 @@ module Pgbus
       def start_heartbeat
         @heartbeat = Heartbeat.new(
           kind: "worker",
+          config: @config,
           # capsule/process lead: the dashboard's metadata cell truncates.
           metadata: {
             capsule: @capsule, process: @process,

data/lib/pgbus/recurring/scheduler.rb

@@ -7,24 +7,33 @@ module Pgbus
     class Scheduler
       include Process::SignalHandler
 
-      attr_reader :schedule, :config
+      attr_reader :schedule, :config, :deploy_runner
 
       def initialize(config: Pgbus.configuration)
         @config = config
         @schedule = Schedule.new(config: config)
+        # Deploy tasks (issue #536) ride this loop: the scheduler is already
+        # the one-per-deployment process, dedup'd across the overlap window.
+        @deploy_runner = Deploy::Runner.new(config: config) if config.deploy_tasks_enabled && config.deploy_tasks&.any?
         @shutting_down = false
         @last_runs = {}
         @loop_tick_at = Concurrent::AtomicReference.new(nil)
+        @deploy_error = nil
       end
 
+      # The order is load-bearing: the heartbeat registers this scheduler's row
+      # before it arms deploy tasks, so an old-revision scheduler in the
+      # overlap window sees it as lagging. A failed registration is logged and
+      # sync still runs; the new release's workers keep the old gate closed.
       def run
         setup_signals
         start_heartbeat
         sync_recurring_tasks
+        deploy_runner&.sync!
 
         Pgbus.logger.info do
           "[Pgbus] Scheduler started: #{schedule.tasks.size} recurring tasks, " \
-            "interval=#{config.recurring_schedule_interval}s"
+            "#{deploy_task_count} deploy tasks, interval=#{config.recurring_schedule_interval}s"
         end
 
         loop do
@@ -44,22 +53,12 @@ module Pgbus
       end
 
       def tick(now)
-        schedule.due_tasks(now).each do |task, run_at|
-          Pgbus::Instrumentation.instrument(
-            "pgbus.recurring.enqueue",
-            task: task.key,
-            class_name: task.class_name,
-            queue: task.queue_name,
-            run_at: run_at
-          ) do
-            schedule.enqueue_task(task, run_at: run_at)
-            @last_runs[task.key] = now
-          end
-        rescue StandardError => e
-          Pgbus.logger.error do
-            "[Pgbus] Error scheduling recurring task #{task.key}: #{e.class}: #{e.message}"
-          end
-        end
+        tick_recurring(now) if config.recurring_enabled
+        tick_deploy_tasks(now)
+      end
+
+      def deploy_task_count
+        deploy_runner&.task_count || 0
       end
 
       def last_run_at(key)
@@ -94,9 +93,43 @@ module Pgbus
 
       private
 
+      def tick_recurring(now)
+        schedule.due_tasks(now).each do |task, run_at|
+          Pgbus::Instrumentation.instrument(
+            "pgbus.recurring.enqueue",
+            task: task.key,
+            class_name: task.class_name,
+            queue: task.queue_name,
+            run_at: run_at
+          ) do
+            schedule.enqueue_task(task, run_at: run_at)
+            @last_runs[task.key] = now
+          end
+        rescue StandardError => e
+          Pgbus.logger.error do
+            "[Pgbus] Error scheduling recurring task #{task.key}: #{e.class}: #{e.message}"
+          end
+        end
+      end
+
+      # A persistent failure (the tables missing, the database down) is logged
+      # and reported once per distinct error, then at debug, not every tick.
+      def tick_deploy_tasks(now)
+        deploy_runner&.tick(now)
+        @deploy_error = nil
+      rescue StandardError => e
+        signature = "#{e.class}: #{e.message}"
+        if signature == @deploy_error
+          Pgbus.logger.debug { "[Pgbus] Error running deploy tasks (repeated): #{signature}" }
+        else
+          @deploy_error = signature
+          ErrorReporter.report(e, { action: "deploy_tasks_tick" }) # logs at error itself
+        end
+      end
+
       def sync_recurring_tasks
         tasks = config.recurring_tasks
-        return if tasks.nil?
+        return if tasks.nil? || !config.recurring_enabled
 
         RecurringTask.sync_from_config!(tasks)
         Pgbus.logger.info { "[Pgbus] Synced #{tasks.size} recurring task(s) from configuration" }
@@ -116,8 +149,9 @@ module Pgbus
       def start_heartbeat
         @heartbeat = Process::Heartbeat.new(
           kind: "scheduler",
+          config: @config,
           metadata: { pid: ::Process.pid, tasks: schedule.tasks.size, started_at: Time.now.to_f,
-                      supervisor_pid: ::Process.ppid },
+                      supervisor_pid: ::Process.ppid }.merge(deploy_runner ? { deploy_tasks: deploy_task_count } : {}),
           loop_tick_supplier: -> { @loop_tick_at.get }
         )
         @heartbeat.start

data/lib/pgbus/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Pgbus
-  VERSION = "0.19.0"
+  VERSION = "0.19.1"
 end