pgbus 0.18.2 → 0.19.1

data/lib/pgbus/deploy/runner.rb

@@ -0,0 +1,259 @@
+# frozen_string_literal: true
+
+module Pgbus
+  module Deploy
+    # Fires deploy tasks once the fleet has rolled over (issue #536). Ticked
+    # by the recurring scheduler's loop; one instance per scheduler process,
+    # so its throttle and log state are plain instance state on that loop.
+    #
+    # A tick: cheap pending check → the rollover gate (at most every
+    # ROLLOVER_CHECK_INTERVAL seconds) → for each declared, enabled, armed
+    # task in key order: claim the arm → perform_later → record the run. A
+    # failed enqueue re-arms the task, logs and reports. No surrounding
+    # transaction: the adapter defers an enqueue inside one to its commit.
+    class Runner
+      ROLLOVER_CHECK_INTERVAL = 5 # seconds
+      # A task whose enqueue failed is retried after 5 s, then 10, 20, … up to
+      # 5 minutes, so a deterministic failure (a before_enqueue that always
+      # aborts, a uniqueness reject while the duplicate is queued) does not
+      # retry and report on every tick.
+      RETRY_BACKOFF_BASE = 5 # seconds
+      RETRY_BACKOFF_MAX = 300 # seconds
+      ROLLOVER = "rollover"
+      MANUAL = "manual"
+
+      attr_reader :revision
+
+      def initialize(config: Pgbus.configuration, revision: Revision.current(config))
+        @config = config
+        @revision = revision
+        @tasks = nil
+        @status = nil
+        @checked_at = nil
+        @logged_wait = nil
+        @failures = {}
+        @synced = nil
+        @sync_failures = 0
+        @sync_retry_at = nil
+      end
+
+      # The valid declared tasks, in key order. A class: that does not resolve
+      # to an ActiveJob is skipped with a warning, like an invalid task.
+      def tasks
+        @tasks ||= load_tasks
+      end
+
+      def task_count
+        tasks.size
+      end
+
+      # true once synced. A failure (a database blip at boot) is retried from
+      # tick with the enqueue backoff, so the scheduler keeps no unarmed tasks.
+      def sync!(now: Time.current)
+        if revision.nil? && @synced.nil?
+          Pgbus.logger.warn do
+            "[Pgbus] No app revision configured (config.app_revision, PGBUS_REVISION, DASH_VERSION, KAMAL_VERSION " \
+              "or a REVISION file): deploy tasks wait until one is set"
+          end
+        end
+        DeployTask.sync_from_config!(tasks, revision: revision, now: now)
+        Pgbus.logger.info { "[Pgbus] Synced #{tasks.size} deploy task(s) for revision #{revision.inspect}" }
+        @synced = true
+      rescue StandardError => e
+        @sync_failures += 1
+        delay = retry_delay(@sync_failures)
+        @sync_retry_at = monotonic_now + delay
+        # Reported (and logged at error by the reporter) once; repeats of a
+        # persistent failure drop to debug instead of reporting every retry.
+        if @sync_failures == 1
+          ErrorReporter.report(e, { action: "deploy_tasks_sync", retry_in: delay })
+        else
+          Pgbus.logger.debug { "[Pgbus] Deploy task sync still failing (retrying in #{delay}s): #{e.class}: #{e.message}" }
+        end
+        @synced = false
+      end
+
+      def tick(now = Time.current)
+        return if tasks.empty?
+        return if @synced == false && !retry_sync(now)
+        return unless DeployTask.pending?
+
+        status = gate_status
+        return log_waiting(status) unless status.rolled_over?
+
+        @logged_wait = nil
+        rows = DeployTask.where(key: tasks.map(&:key)).index_by(&:key)
+        tasks.each do |task|
+          row = rows[task.key]
+          fire(task, row, now) if row&.enabled && row.armed? && !backing_off?(task.key)
+        end
+      end
+
+      # Enqueue +task+ through ActiveJob and record the run. :enqueued, or
+      # :failed (logged and reported) when perform_later raised or did not
+      # enqueue the job. Run now calls this directly: no gate, no claim.
+      def enqueue(task, triggered_by:, now: Time.current)
+        payload = { task: task.key, version: task.version, revision: revision, triggered_by: triggered_by }
+        job = Instrumentation.instrument("pgbus.deploy_task.enqueue", payload) { submit(task) }
+        record_run(task, job, triggered_by, now)
+        :enqueued
+      rescue StandardError => e
+        Pgbus.logger.error do
+          "[Pgbus] Deploy task #{task.key} (version #{task.version}, #{triggered_by}) was not enqueued: " \
+            "#{e.class}: #{e.message}"
+        end
+        ErrorReporter.report(e, { action: "deploy_task_enqueue", task: task.key, version: task.version,
+                                  revision: revision, triggered_by: triggered_by })
+        :failed
+      end
+
+      private
+
+      def load_tasks
+        (@config.deploy_tasks || {}).filter_map { |key, options| build_task(key, options) }.sort_by(&:key)
+      end
+
+      def build_task(key, options)
+        options = options.is_a?(Hash) ? options.transform_keys(&:to_sym) : {}
+        task = Task.from_configuration(key, **options)
+        return skip_task(key, task.errors.join(", ")) unless task.valid?
+        return skip_task(key, "#{task.class_name} is not an ActiveJob") if task.class_name && !active_job?(task.job_class)
+
+        task.warnings.each { |warning| Pgbus.logger.warn { "[Pgbus] Deploy task '#{key}': #{warning}" } }
+        task
+      rescue StandardError => e
+        # A malformed entry (a Hash where a scalar belongs) skips that task
+        # instead of crashing the scheduler at boot.
+        skip_task(key, "#{e.class}: #{e.message}")
+      end
+
+      def active_job?(klass)
+        klass.is_a?(Class) && defined?(::ActiveJob::Base) && klass <= ::ActiveJob::Base
+      end
+
+      def skip_task(key, reason)
+        Pgbus.logger.warn { "[Pgbus] Skipping invalid deploy task '#{key}': #{reason}" }
+        nil
+      end
+
+      def gate_status
+        at = monotonic_now
+        return @status if @checked_at && at - @checked_at < ROLLOVER_CHECK_INTERVAL
+
+        # Stamped only after a successful check: a raising check must not
+        # leave a nil status cached for the next ticks.
+        @status = Rollover.check(revision: revision)
+        @checked_at = at
+        @status
+      end
+
+      def retry_sync(now)
+        return false if monotonic_now < @sync_retry_at
+
+        sync!(now: now)
+      end
+
+      def log_waiting(status)
+        summary = waiting_summary(status)
+        return if summary == @logged_wait
+
+        @logged_wait = summary
+        Pgbus.logger.info { "[Pgbus] Deploy tasks waiting for rollover: #{summary}" }
+      end
+
+      def waiting_summary(status)
+        case status.reason
+        when :no_revision then "app revision not configured"
+        when :no_workers then "no worker on #{revision} yet"
+        else
+          processes = status.lagging.map { |p| "#{p[:kind]}@#{p[:hostname]} pid #{p[:pid]} (#{p[:revision] || "no revision"})" }
+          "#{status.lagging.size} process(es) not on #{revision}: #{processes.join(", ")}"
+        end
+      end
+
+      # An arm for another version or revision belongs to code no live
+      # process runs any more (the gate is open on this revision), so when
+      # this runner's own version is owed it takes the arm over; otherwise
+      # the claim below fails and nothing fires.
+      def fire(task, row, now)
+        armed_for = task.armed_for(revision)
+        row.take_over!(armed_for: armed_for, now: now) if row.armed_for != armed_for && DeployTask.owed?(task, revision)
+        return unless row.claim!(armed_for: armed_for, now: now)
+        return @failures.delete(task.key) if enqueue(task, triggered_by: ROLLOVER, now: now) == :enqueued
+
+        rearm(task, row, armed_for)
+      end
+
+      def rearm(task, row, armed_for)
+        delay = note_failure(task.key)
+        row.rearm!(armed_for: armed_for, now: Time.current)
+        Pgbus.logger.warn { "[Pgbus] Deploy task #{task.key} re-armed; retrying in #{delay}s" }
+      rescue StandardError => e
+        Pgbus.logger.error { "[Pgbus] Could not re-arm deploy task #{task.key}: #{e.class}: #{e.message}" }
+        ErrorReporter.report(e, { action: "deploy_task_rearm", task: task.key, armed_for: armed_for })
+      end
+
+      def backing_off?(key)
+        failure = @failures[key]
+        !failure.nil? && monotonic_now < failure[:retry_at]
+      end
+
+      # Records one more failed enqueue for +key+ and returns the seconds until its retry.
+      def note_failure(key)
+        count = @failures.fetch(key, { count: 0 })[:count] + 1
+        delay = retry_delay(count)
+        @failures[key] = { count: count, retry_at: monotonic_now + delay }
+        delay
+      end
+
+      def retry_delay(count)
+        [RETRY_BACKOFF_BASE * (2**(count - 1)), RETRY_BACKOFF_MAX].min
+      end
+
+      def monotonic_now
+        ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+      end
+
+      # Builds the job and enqueues it directly (what perform_later does), so
+      # the job, and its enqueue_error, is in hand on every Rails version:
+      # perform_later returns false on failure, and on Rails 7.1 a .set(...)
+      # job never yields it to perform_later's block.
+      def submit(task)
+        job = job_class_for(task).new(*job_arguments(task))
+        return job if job.enqueue(job_options(task))
+
+        error = job.enqueue_error
+        raise EnqueueError, error ? "#{error.class}: #{error.message}" : "the job was not enqueued (a before_enqueue callback aborted)"
+      end
+
+      def job_class_for(task)
+        return Recurring::CommandJob if task.command
+
+        klass = task.job_class
+        raise EnqueueError, "#{task.class_name} does not resolve to a job class" unless active_job?(klass)
+
+        klass
+      end
+
+      def job_options(task)
+        priority = task.command || task.priority.zero? ? nil : task.priority
+        { queue: task.queue_name, priority: priority }.compact
+      end
+
+      def job_arguments(task)
+        task.command ? [task.command] : task.arguments
+      end
+
+      def record_run(task, job, triggered_by, now)
+        DeployTaskRun.create!(task_key: task.key, version: task.version, revision: revision,
+                              triggered_by: triggered_by, queue_name: job.queue_name,
+                              msg_id: job.provider_job_id, run_at: now)
+      rescue StandardError => e
+        Pgbus.logger.error do
+          "[Pgbus] Deploy task #{task.key} was enqueued but its run was not recorded: #{e.class}: #{e.message}"
+        end
+        ErrorReporter.report(e, { action: "deploy_task_record_run", task: task.key, triggered_by: triggered_by })
+      end
+    end
+  end
+end

data/lib/pgbus/deploy/task.rb

@@ -0,0 +1,105 @@
+# frozen_string_literal: true
+
+module Pgbus
+  module Deploy
+    # One declared deploy task (issue #536): an entry of
+    # config/deploy_tasks.yml. The sibling of Recurring::Task without a
+    # schedule: it runs once per (key, version) after a rollover, or after
+    # every rollover when every_deploy is set.
+    class Task
+      DEFAULT_VERSION = "1"
+
+      attr_reader :key, :class_name, :command, :version, :every_deploy, :queue_name,
+                  :arguments, :priority, :description
+
+      def self.from_configuration(key, **options)
+        options = options.transform_keys(&:to_sym)
+        new(
+          key: key.to_s,
+          class_name: options[:class],
+          command: options[:command],
+          version: options.fetch(:version, DEFAULT_VERSION),
+          explicit_version: options.key?(:version),
+          every_deploy: options.fetch(:every_deploy, false),
+          queue_name: options[:queue],
+          arguments: options[:args].is_a?(Hash) ? [options[:args]] : Array(options[:args]),
+          priority: options.fetch(:priority, 0).to_i,
+          description: options[:description]
+        )
+      end
+
+      def initialize(key:, class_name: nil, command: nil, version: DEFAULT_VERSION, explicit_version: false,
+                     every_deploy: false, queue_name: nil, arguments: [], priority: 0, description: nil)
+        @key = key
+        @class_name = class_name
+        @command = command
+        @version = version.to_s.strip
+        @explicit_version = explicit_version
+        @every_deploy = every_deploy
+        @queue_name = queue_name
+        @arguments = arguments || []
+        @priority = priority || 0
+        @description = description
+        @errors = []
+        @warnings = []
+      end
+
+      def valid?
+        @errors = []
+        @warnings = []
+        validate!
+        @errors.empty?
+      end
+
+      def errors
+        valid? unless @validated
+        @errors
+      end
+
+      def warnings
+        valid? unless @validated
+        @warnings
+      end
+
+      # What an arm for this task is stamped with: its version, or the
+      # revision it runs on for an every_deploy task.
+      def armed_for(revision)
+        every_deploy ? revision : version
+      end
+
+      def job_class
+        class_name&.safe_constantize
+      end
+
+      def to_h
+        {
+          key: key, class_name: class_name, command: command, version: version, every_deploy: every_deploy,
+          queue_name: queue_name, arguments: arguments, priority: priority, description: description
+        }
+      end
+
+      private
+
+      def validate!
+        @validated = true
+        @errors << "Key must not be blank" if key.to_s.strip.empty?
+        validate_target!
+        @errors << "Version must not be blank" if version.empty?
+        @errors << "every_deploy must be true or false" unless [true, false].include?(every_deploy)
+        return unless every_deploy == true && @explicit_version
+
+        @warnings << "every_deploy is set, so version is ignored (it runs after every rollover)"
+      end
+
+      def validate_target!
+        { class: class_name, command: command }.each do |name, value|
+          @errors << "#{name} must be a String" unless value.nil? || value.is_a?(String)
+        end
+        has_class = !class_name.to_s.strip.empty?
+        has_command = !command.to_s.strip.empty?
+        @errors << "Either class or command is required" unless has_class || has_command
+        @errors << "Declare either class or command, not both" if has_class && has_command
+      end
+    end
+  end
+end

data/lib/pgbus/engine.rb

@@ -58,6 +58,23 @@ module Pgbus
       end
     end
 
+    # config/deploy_tasks.yml (issue #536): same shape and loader as
+    # recurring.yml. Explicit deploy_tasks_files replace the default file, so
+    # this runs after the app's config/initializers set them.
+    initializer "pgbus.deploy_tasks", after: :load_config_initializers do |app|
+      config = Pgbus.configuration
+      next if config.deploy_tasks
+
+      files = config.deploy_tasks_files
+      default_path = app.root.join("config", "deploy_tasks.yml")
+      tasks = if files
+                Pgbus::Recurring::ConfigLoader.load_all(files)
+              elsif default_path.exist?
+                Pgbus::Recurring::ConfigLoader.load(default_path)
+              end
+      config.deploy_tasks = tasks if tasks.present?
+    end
+
     initializer "pgbus.db" do
       ActiveSupport.on_load(:active_record) do
         Pgbus::BusRecord.connects_to(**Pgbus.configuration.connects_to) if Pgbus.configuration.connects_to
@@ -81,6 +98,9 @@ module Pgbus
     end
 
     initializer "pgbus.logger" do
+      # Rails.logger exists now: a log_format set before it did went to the
+      # $stdout fallback only.
+      Pgbus.configuration.apply_log_format!
       ActiveSupport.on_load(:after_initialize) do
         Pgbus.configuration.logger ||= Rails.logger
       end
@@ -209,5 +229,13 @@ module Pgbus
         end
       end
     end
+
+    # Last: once the app's initializers have configured pgbus, freeze the
+    # configuration's plain-data settings into Ractor-shareable values
+    # (Pgbus.finalize!, issue #527). Registered after the app's initializer
+    # files, so it runs after the after_initialize blocks they add.
+    initializer "pgbus.finalize", after: :load_config_initializers do
+      ActiveSupport.on_load(:after_initialize) { Pgbus.finalize! }
+    end
   end
 end

data/lib/pgbus/event_bus/handler.rb

@@ -4,6 +4,19 @@ module Pgbus
   module EventBus
     class Handler
       class << self
+        # One DedupCache per handler class, built when the class is defined
+        # (this class body, and #inherited for every subclass), never by a
+        # lazy `||=` that two first deliveries could each win (issue #527).
+        attr_reader :dedup_cache
+
+        def inherited(subclass)
+          super
+          subclass.instance_variable_set(:@dedup_cache, DedupCache.new)
+        end
+
+        # Class-body macro: `idempotent!` runs while the handler class loads,
+        # and #idempotent? only reads the flag afterwards.
+        # rubocop:disable ThreadSafety/ClassInstanceVariable
         def idempotent!
           @idempotent = true
         end
@@ -11,10 +24,7 @@ module Pgbus
         def idempotent?
           @idempotent == true
         end
-
-        def dedup_cache
-          @dedup_cache ||= DedupCache.new
-        end
+        # rubocop:enable ThreadSafety/ClassInstanceVariable
 
         # How long a pending claim may stay silent before its holder counts as
         # dead. ClaimBeat refreshes a live claim from the visibility heartbeat,
@@ -34,6 +44,8 @@ module Pgbus
         end
       end
 
+      @dedup_cache = DedupCache.new
+
       # Outcome of the two-phase claim. `age` is how long the losing delivery
       # found the existing claim to have been silent, in seconds (nil unless
       # the claim was pending).

data/lib/pgbus/execution_pools/async_pool.rb

@@ -3,6 +3,10 @@
 module Pgbus
   module ExecutionPools
     class AsyncPool
+      # How long #kill waits for the killed reactor thread (and the job
+      # fibers it cancels) to unwind before giving up on it.
+      KILL_JOIN_TIMEOUT = 1
+
       attr_reader :capacity
 
       def initialize(capacity:, on_state_change: nil)
@@ -70,9 +74,15 @@ module Pgbus
         @reactor_thread&.join(timeout)
       end
 
+      # Killing the reactor thread cancels its job fibers; wait for them to
+      # unwind so the caller can reclaim what they held (ExecutionPools.terminate).
       def kill
         shutdown
-        @reactor_thread&.kill
+        thread = @reactor_thread
+        return unless thread
+
+        thread.kill
+        join_killed(thread)
       end
 
       def metadata
@@ -86,6 +96,14 @@ module Pgbus
 
       private
 
+      # The reactor's own failure is already recorded (register_fatal_error);
+      # Thread#join would raise it again here.
+      def join_killed(thread)
+        thread.join(KILL_JOIN_TIMEOUT)
+      rescue StandardError => e
+        Pgbus.logger.debug { "[Pgbus] Async pool reactor ended with #{e.class}: #{e.message}" }
+      end
+
       def validate_dependencies!
         require "async"
         require "async/semaphore"
@@ -95,9 +113,12 @@ module Pgbus
               "Add `gem \"async\"` to your Gemfile. Original error: #{e.message}"
       end
 
+      # The pool's one reactor thread, owned by this pool: started once by
+      # #initialize, it exits when #shutdown lets the reactor drain, and
+      # #wait_for_termination / #kill join it.
       # rubocop:disable-next Lint/RescueException
       def start_reactor
-        Thread.new do
+        Thread.new do # rubocop:disable ThreadSafety/NewThread
           Thread.current.name = "pgbus-async-reactor-#{object_id}"
           Async do |task|
             semaphore = Async::Semaphore.new(@capacity, parent: task)

data/lib/pgbus/execution_pools/thread_pool.rb

@@ -5,6 +5,10 @@ require "concurrent"
 module Pgbus
   module ExecutionPools
     class ThreadPool
+      # How long #kill waits, in all, for the killed jobs to unwind (run their
+      # ensure blocks) before giving up on those still running.
+      KILL_JOIN_TIMEOUT = 1
+
       attr_reader :capacity
 
       def initialize(capacity:, on_state_change: nil)
@@ -12,14 +16,17 @@ module Pgbus
         @on_state_change = on_state_change
         @available_capacity = Concurrent::AtomicFixnum.new(capacity)
         @pool = Concurrent::FixedThreadPool.new(capacity)
+        @running = Concurrent::Map.new
       end
 
       def post(&block)
         @available_capacity.decrement
         begin
           @pool.post do
+            @running[Thread.current] = true
             block.call
           ensure
+            @running.delete(Thread.current)
             @available_capacity.increment
             @on_state_change&.call
           end
@@ -52,8 +59,17 @@ module Pgbus
         @pool.wait_for_termination(timeout)
       end
 
+      # Thread#kill only asks a thread to stop: wait for the killed jobs to
+      # unwind, so the caller can reclaim what they held (ExecutionPools.terminate).
+      # The running jobs are listed after the kill: a job that started in
+      # between is killed too, and a job leaves the list only once it has
+      # unwound, so every one still unwinding is joined. One budget for all
+      # of them: stuck jobs must not hold a shutdown for capacity x the timeout.
       def kill
         @pool.kill
+        unwinding = @running.keys
+        deadline = monotonic_now + KILL_JOIN_TIMEOUT
+        unwinding.each { |thread| thread.join([deadline - monotonic_now, 0].max) }
       end
 
       def metadata
@@ -63,6 +79,12 @@ module Pgbus
           busy: @capacity - available_capacity
         }
       end
+
+      private
+
+      def monotonic_now
+        ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+      end
     end
   end
 end

data/lib/pgbus/execution_pools.rb

@@ -22,6 +22,32 @@ module Pgbus
           raise ArgumentError, "Unknown execution_mode: #{mode.inspect}. Expected one of: :threads, :async, :fiber"
         end
       end
+
+      # Stop +pool+ taking work, give the jobs already running +grace+
+      # seconds to finish, and kill whatever is still running after that.
+      # Either way, reclaim the ActiveRecord connections the jobs left checked
+      # out: a killed thread or fiber never checks its own back in, and the
+      # pool would stay that many connections short until its reaper ran.
+      # Also when the wait raises (an async pool re-raises a fatal reactor
+      # error). Returns true when every job finished within the grace (issue #527).
+      def terminate(pool, grace:)
+        pool.shutdown
+        finished = pool.wait_for_termination(grace) ? true : false
+        pool.kill unless finished
+        finished
+      ensure
+        reclaim_connections
+      end
+
+      # The caller's own connection, then every connection whose owning
+      # thread or fiber is dead. A no-op without ActiveRecord.
+      def reclaim_connections
+        return unless defined?(::ActiveRecord::Base)
+
+        handler = ::ActiveRecord::Base.connection_handler
+        handler.clear_active_connections!(:all)
+        handler.connection_pool_list(:all).each(&:reap)
+      end
     end
   end
 end

data/lib/pgbus/generators/migration_detector.rb

@@ -77,6 +77,7 @@ module Pgbus
         add_queue_states: "pgbus:add_queue_states",
         add_outbox: "pgbus:add_outbox",
         add_recurring: "pgbus:add_recurring",
+        add_deploy_tasks: "pgbus:add_deploy_tasks",
         add_failed_events_index: "pgbus:add_failed_events_index",
         add_processed_event_completion: "pgbus:add_processed_event_completion",
         add_batch_executions: "pgbus:add_batch_executions",
@@ -99,6 +100,7 @@ module Pgbus
         add_queue_states: "queue states table (pause/resume)",
         add_outbox: "outbox entries table (transactional outbox)",
         add_recurring: "recurring tasks + executions tables",
+        add_deploy_tasks: "deploy tasks + runs tables (one-time jobs run after a rollover)",
         add_failed_events_index: "unique index on pgbus_failed_events (queue_name, msg_id)",
         add_processed_event_completion: "completed_at on pgbus_processed_events (two-phase idempotency claim)",
         add_batch_executions: "batch execution-row tracking (self-healing completion, on_failure rename)",
@@ -127,6 +129,7 @@ module Pgbus
           *queue_states_migrations,
           *outbox_migrations,
           *recurring_migrations,
+          *deploy_tasks_migrations,
           *failed_events_index_migrations,
           *processed_event_completion_migrations,
           *batch_executions_migrations,
@@ -215,6 +218,14 @@ module Pgbus
         [:add_recurring]
       end
 
+      # Same shape as recurring_migrations: plain create_table in the
+      # template, so the generator is queued only while a table is absent.
+      def deploy_tasks_migrations
+        return [] if table_exists?("pgbus_deploy_tasks") && table_exists?("pgbus_deploy_task_runs")
+
+        [:add_deploy_tasks]
+      end
+
       def failed_events_index_migrations
         # pgbus_failed_events is created by pgbus:install, but the
         # unique (queue_name, msg_id) index was added later via its own

data/lib/pgbus/instrumentation.rb

@@ -28,6 +28,9 @@ module Pgbus
   #   pgbus.stream.broadcast       — stream broadcast (sync or deferred)
   #   pgbus.outbox.publish         — outbox row created
   #   pgbus.recurring.enqueue      — scheduler enqueued a due recurring task
+  #   pgbus.deploy_task.enqueue    — a deploy task enqueue was attempted (after a rollover or by
+  #                                  Run now); payload: task, version, revision, triggered_by, plus
+  #                                  :exception/:exception_object when the enqueue raised
   #   pgbus.worker.recycle         — worker hit a recycle threshold (reason/jobs_processed/memory_mb/lifetime_seconds)
   #   pgbus.consumer.recycle       — consumer hit a recycle threshold (same payload shape as pgbus.worker.recycle)
   #   pgbus.serializer.serialize   — job/event serialization