pgbus 0.19.0 → 0.19.2

data/lib/pgbus/batch.rb

@@ -21,21 +21,57 @@ module Pgbus
     LOCK_QUEUE_PREFIX = "batch:"
     VALID_CONFLICTS = %i[reject discard log].freeze
 
+    # The callback slots, each mapped to the prefix of its pgbus_batches
+    # columns. on_complete keeps the on_finish_* columns until the 1.0 schema
+    # cleanup, so rows written before the rename still fire.
+    CALLBACK_COLUMNS = { on_complete: "on_finish", on_success: "on_success", on_failure: "on_failure" }.freeze
+
     attr_reader :batch_id, :properties, :description,
-                :on_finish, :on_success, :on_failure,
+                :on_complete, :on_success, :on_failure,
                 :uniqueness_key, :on_conflict
 
+    # Deprecated alias of #on_complete until 1.0.
+    def on_finish = on_complete
+
     def on_discard
       on_failure
     end
 
+    # Open a batch, yield a Pgbus::Batch::Builder, and start the batch when
+    # the block returns. Every job the block enqueues (through +b.add+ or a
+    # plain +perform_later+) joins the batch, which stays pending, and so
+    # cannot finish, until the block is done. Keywords go to Batch.new.
+    # Membership is the block's own thread or fiber (the tag lives in
+    # Thread.current): b.add raises anywhere else, and a plain perform_later
+    # from a thread or fiber the block spawns does not join the batch.
+    #
+    #   Pgbus::Batch.start(description: "Import") do |b|
+    #     b.on_complete ImportDoneJob
+    #     b.add(*rows.map { |row| ImportRowJob.new(row.id) })
+    #   end
+    def self.start(**, &block)
+      raise ArgumentError, "Pgbus.batch needs a block: Pgbus.batch { |b| b.add(...) }" unless block
+
+      batch = new(**)
+      builder = Builder.new(batch)
+      begin
+        batch.enqueue { yield builder }
+      ensure
+        builder.close
+      end
+      batch
+    end
+
+    # @param on_complete [Class, ActiveJob::Base, nil] fired when every job
+    #   reached a terminal state (+on_finish:+ is a deprecated alias)
     # @param uniqueness_key [String, nil] at most one unfinished batch with
     #   this key may exist; see #enqueue for what happens to the next one
     # @param on_conflict [Symbol] :reject (raise AlreadyRunning), :discard
     #   (skip the block silently) or :log (skip the block, warn)
-    def initialize(on_finish: nil, on_success: nil, on_discard: nil, on_failure: nil, description: nil, properties: {},
-                   uniqueness_key: nil, on_conflict: :reject)
+    def initialize(on_complete: nil, on_finish: nil, on_success: nil, on_discard: nil, on_failure: nil,
+                   description: nil, properties: {}, uniqueness_key: nil, on_conflict: :reject)
       raise ArgumentError, "pass on_failure: only — on_discard: is a deprecated alias" if on_discard && on_failure
+      raise ArgumentError, "pass on_complete: only — on_finish: is a deprecated alias" if on_finish && on_complete
       unless VALID_CONFLICTS.include?(on_conflict)
         raise ArgumentError, "on_conflict must be one of #{VALID_CONFLICTS.join(", ")}, got #{on_conflict.inspect}"
       end
@@ -45,9 +81,10 @@ module Pgbus
           "[Pgbus] Batch on_discard: is deprecated and will be removed in 1.0 — use on_failure: instead"
         end
       end
+      self.class.warn_on_finish_deprecated if on_finish
 
       @batch_id = SecureRandom.uuid
-      @on_finish = on_finish
+      @on_complete = on_complete || on_finish
       @on_success = on_success
       @on_failure = on_failure || on_discard
       @description = description
@@ -91,6 +128,23 @@ module Pgbus
       self
     end
 
+    # Enqueue ActiveJob instances (and anything the block enqueues) into this
+    # batch: +enqueue { ActiveJob.perform_all_later(jobs) }+. Instances keep
+    # their own +.set(queue:, wait:, priority:)+ options. On a started batch
+    # this is an open-batch add, so a job running inside the batch can call
+    # +batch.add(NextJob.new(id))+. Returns the jobs; on a unique batch that
+    # another run discarded (#discarded?) nothing was sent.
+    def add(*jobs, &block)
+      jobs = Builder.job_instances(jobs)
+      return jobs if jobs.empty? && block.nil?
+
+      enqueue do
+        ::ActiveJob.perform_all_later(jobs) if jobs.any?
+        yield if block
+      end
+      jobs
+    end
+
     # --- readers on a live batch ---------------------------------------
 
     def status = record&.status
@@ -210,11 +264,22 @@ module Pgbus
     end
 
     # Schema-probe memos (a true answer is kept for the life of the process)
-    # and the warn-once flag, read and written under @schema_lock (issue
+    # and the warn-once flags, read and written under @schema_lock (issue
     # #527). The probes run outside it: a duplicate probe on a race is
     # harmless, a lock held across a query is not.
     @schema_lock = Mutex.new
 
+    # on_finish: is in most batch calls written before the rename (issue
+    # #538), so it warns once per process rather than on every batch.
+    def self.warn_on_finish_deprecated
+      first = @schema_lock.synchronize { @warned_on_finish ? false : (@warned_on_finish = true) }
+      return unless first
+
+      Pgbus.logger.warn do
+        "[Pgbus] Batch on_finish is deprecated and will be removed in 1.0 — use on_complete instead"
+      end
+    end
+
     def self.executions_migrated?
       return true if @schema_lock.synchronize { @executions_migrated }
 
@@ -243,6 +308,7 @@ module Pgbus
         @executions_migrated = nil
         @callback_jobs_migrated = nil
         @warned_callback_jobs_unmigrated = nil
+        @warned_on_finish = nil
       end
     end
 
@@ -521,7 +587,7 @@ module Pgbus
       @batch_id = record.batch_id
       @description = record.description
       @properties = parse_properties(record.properties)
-      @on_finish = nil
+      @on_complete = nil
       @on_success = nil
       @on_failure = nil
       @uniqueness_key = nil
@@ -572,20 +638,37 @@ module Pgbus
       attrs = {
         batch_id: batch_id,
         description: description,
-        on_finish_class: callback_class_name(on_finish),
-        on_success_class: callback_class_name(on_success),
         properties: JSON.generate(properties),
         status: "pending"
       }
-      if self.class.executions_migrated?
-        attrs[:on_failure_class] = callback_class_name(on_failure)
-      else
-        attrs[:on_discard_class] = callback_class_name(on_failure)
-      end
-      attrs.merge!(callback_job_attributes) if self.class.callback_jobs_migrated?
+      CALLBACK_COLUMNS.each_key { |slot| attrs.merge!(callback_attributes(slot, public_send(slot))) }
       @record = BatchEntry.create!(attrs)
     end
 
+    # Replace one callback on a batch that has not finished yet (the
+    # builder's setters, inside the Pgbus.batch block, while the row is
+    # still pending and so cannot fire its callbacks).
+    def write_callback(slot, callback)
+      updated = BatchEntry.where(batch_id: batch_id, status: %w[pending processing])
+                          .update_all(callback_attributes(slot, callback))
+      raise AlreadyFinished, "Can't change the callbacks of an already finished batch" if updated.zero?
+
+      instance_variable_set(:"@#{slot}", callback)
+      reload
+    end
+
+    # The pgbus_batches columns for one callback slot. Before the
+    # add_batch_executions migration the failure class lives in
+    # on_discard_class; before add_batch_callback_jobs there is no *_job
+    # column, and a configured instance degrades to its class.
+    def callback_attributes(slot, callback)
+      prefix = CALLBACK_COLUMNS.fetch(slot)
+      class_column = slot == :on_failure && !self.class.executions_migrated? ? "on_discard" : prefix
+      attrs = { "#{class_column}_class": callback_class_name(callback) }
+      attrs[:"#{prefix}_job"] = serialize_callback(callback) if self.class.callback_jobs_migrated?
+      attrs
+    end
+
     # A callback given as a bare class keeps the legacy *_class column; a
     # configured ActiveJob instance is serialized now (so .set options resolve
     # at creation, matching solid_queue) into the *_job jsonb column. Before
@@ -600,14 +683,6 @@ module Pgbus
       callback.class.name
     end
 
-    def callback_job_attributes
-      {
-        on_finish_job: serialize_callback(on_finish),
-        on_success_job: serialize_callback(on_success),
-        on_failure_job: serialize_callback(on_failure)
-      }
-    end
-
     def serialize_callback(callback)
       return nil if callback.nil? || callback.is_a?(Class)
 

data/lib/pgbus/configuration/deploy_settings.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module Pgbus
+  class Configuration
+    # Deploy tasks (issue #536): one-time jobs the scheduler enqueues once
+    # every live pgbus process runs the new app revision.
+    #
+    #   config.app_revision = -> { File.read("REVISION").strip } # or a String
+    #   config.deploy_tasks_files = ["config/deploy_tasks.yml"]
+    #   config.deploy_task_run_retention = 30.days
+    module DeploySettings
+      DEFAULT_DEPLOY_TASK_RUN_RETENTION = 90 * 24 * 3600
+
+      attr_accessor :app_revision, :deploy_tasks, :deploy_tasks_files, :deploy_tasks_enabled
+      attr_reader :deploy_task_run_retention
+
+      def deploy_task_run_retention=(value)
+        @deploy_task_run_retention = coerce_duration!(value, :deploy_task_run_retention)
+      end
+
+      private
+
+      def initialize_deploy_settings
+        @app_revision = nil
+        @deploy_tasks = nil
+        @deploy_tasks_files = nil
+        @deploy_tasks_enabled = true
+        @deploy_task_run_retention = DEFAULT_DEPLOY_TASK_RUN_RETENTION
+      end
+
+      def validate_deploy_settings!
+        unless app_revision.nil? || app_revision.is_a?(String) || app_revision.respond_to?(:call)
+          raise Pgbus::ConfigurationError, "app_revision must be a String, respond to #call, or be nil"
+        end
+
+        raise Pgbus::ConfigurationError, "deploy_tasks must be a Hash or nil" unless deploy_tasks.nil? || deploy_tasks.is_a?(Hash)
+        unless deploy_tasks_files.nil? || deploy_tasks_files.is_a?(Array)
+          raise Pgbus::ConfigurationError, "deploy_tasks_files must be an Array of paths or nil"
+        end
+        return if [true, false].include?(deploy_tasks_enabled)
+
+        raise Pgbus::ConfigurationError, "deploy_tasks_enabled must be true or false"
+      end
+    end
+  end
+end

data/lib/pgbus/configuration.rb

@@ -6,6 +6,7 @@ module Pgbus
   class Configuration
     include PresenceSettings
     include StreamAuthorization
+    include DeploySettings
 
     # Connection settings
     attr_accessor :database_url, :connection_params, :pool_size, :pool_timeout
@@ -363,6 +364,7 @@ module Pgbus
       @recurring_tasks_files = nil
       @recurring_enabled = true
       @recurring_execution_retention = 7 * 24 * 3600 # 7 days
+      initialize_deploy_settings
 
       @zombie_detection = true
 
@@ -944,6 +946,7 @@ module Pgbus
       validate_streams!
       validate_metrics_backend!
       validate_fair_share!
+      validate_deploy_settings!
 
       self
     end

data/lib/pgbus/deploy/revision.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+module Pgbus
+  module Deploy
+    # The app revision this process runs (issue #536). Every heartbeat row
+    # carries it in metadata["revision"]; the deploy-task rollover gate
+    # compares those against the scheduler's own.
+    #
+    # First match wins, blank counts as absent:
+    #   config.app_revision (String or callable) → PGBUS_REVISION → DASH_VERSION
+    #   → KAMAL_VERSION → the REVISION file at Rails.root (Capistrano) → nil
+    #
+    # Computed per call, never memoized: it is read once per process
+    # registration, and a module-level memo would be unguarded shared state.
+    module Revision
+      ENV_KEYS = %w[PGBUS_REVISION DASH_VERSION KAMAL_VERSION].freeze
+
+      module_function
+
+      def current(config = Pgbus.configuration)
+        from_config(config) || from_env || from_file
+      end
+
+      def from_config(config)
+        value = config.app_revision
+        value = value.call if value.respond_to?(:call)
+        presence(value)
+      rescue StandardError => e
+        Pgbus.logger.warn { "[Pgbus] app_revision raised #{e.class}: #{e.message}; trying the environment" }
+        nil
+      end
+
+      def from_env
+        ENV_KEYS.each do |key|
+          value = presence(ENV.fetch(key, nil))
+          return value if value
+        end
+        nil
+      end
+
+      def from_file
+        return nil unless defined?(Rails) && Rails.respond_to?(:root) && Rails.root
+
+        path = Rails.root.join("REVISION")
+        File.exist?(path) ? presence(File.read(path)) : nil
+      rescue SystemCallError, IOError => e
+        # Read while every heartbeat initializes: an unreadable optional file
+        # must not stop a process from starting.
+        Pgbus.logger.warn { "[Pgbus] Could not read #{path}: #{e.class}: #{e.message}" }
+        nil
+      end
+
+      def presence(value)
+        string = value.to_s.strip
+        string.empty? ? nil : string
+      end
+    end
+  end
+end

data/lib/pgbus/deploy/rollover.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Pgbus
+  module Deploy
+    # Has the fleet rolled over to +revision+? (issue #536). A pure read of
+    # pgbus_processes: live rows (heartbeat within Heartbeat::ALIVE_THRESHOLD)
+    # whose metadata["revision"] differs from +revision+ are lagging (a missing
+    # revision is a pre-upgrade process, so it lags too), and at least one live
+    # worker on +revision+ must exist. The scheduler and the dashboard both
+    # consume the Status, so the page prints the reason the scheduler acts on.
+    module Rollover
+      # reason – :ready, :no_revision, :lagging or :no_workers.
+      # lagging – [{ kind:, hostname:, pid:, revision: }].
+      # live_by_revision – { revision => live process count }, nil for rows without one.
+      Status = Data.define(:revision, :lagging, :workers_live, :reason, :live_by_revision) do
+        def rolled_over?
+          reason == :ready
+        end
+      end
+
+      module_function
+
+      def check(revision:, now: Time.current)
+        cutoff = now - Process::Heartbeat::ALIVE_THRESHOLD
+        rows = ProcessEntry.where(last_heartbeat_at: cutoff..).pluck(:kind, :hostname, :pid, :metadata)
+        evaluate(rows.map { |kind, hostname, pid, metadata| { kind:, hostname:, pid:, metadata: } }, revision: revision)
+      end
+
+      # [live, revision] of the newest live scheduler row: the revision the
+      # dashboard judges the rollover against (the one the scheduler acts on).
+      def scheduler_revision(now: Time.current)
+        row = ProcessEntry.where(kind: "scheduler", last_heartbeat_at: (now - Process::Heartbeat::ALIVE_THRESHOLD)..)
+                          .order(created_at: :desc, id: :desc).first
+        row ? [true, revision_of(row.metadata)] : [false, nil]
+      end
+
+      # rows – live process rows as { kind:, hostname:, pid:, metadata: }.
+      def evaluate(rows, revision:)
+        live = rows.map { |r| { kind: r[:kind], hostname: r[:hostname], pid: r[:pid], revision: revision_of(r[:metadata]) } }
+        lagging = live.reject { |r| revision && r[:revision] == revision }
+        workers_live = live.count { |r| r[:kind] == "worker" && revision && r[:revision] == revision }
+
+        Status.new(revision: revision, lagging: revision ? lagging : [], workers_live: workers_live,
+                   reason: reason_for(revision, lagging, workers_live), live_by_revision: live.map { |r| r[:revision] }.tally)
+      end
+
+      def reason_for(revision, lagging, workers_live)
+        return :no_revision if revision.nil?
+        return :lagging if lagging.any?
+        return :no_workers if workers_live.zero?
+
+        :ready
+      end
+
+      def revision_of(metadata)
+        metadata = JSON.parse(metadata) if metadata.is_a?(String)
+        metadata.is_a?(Hash) ? metadata["revision"] : nil
+      rescue JSON::ParserError
+        nil
+      end
+    end
+  end
+end

data/lib/pgbus/deploy/runner.rb

@@ -0,0 +1,259 @@
+# frozen_string_literal: true
+
+module Pgbus
+  module Deploy
+    # Fires deploy tasks once the fleet has rolled over (issue #536). Ticked
+    # by the recurring scheduler's loop; one instance per scheduler process,
+    # so its throttle and log state are plain instance state on that loop.
+    #
+    # A tick: cheap pending check → the rollover gate (at most every
+    # ROLLOVER_CHECK_INTERVAL seconds) → for each declared, enabled, armed
+    # task in key order: claim the arm → perform_later → record the run. A
+    # failed enqueue re-arms the task, logs and reports. No surrounding
+    # transaction: the adapter defers an enqueue inside one to its commit.
+    class Runner
+      ROLLOVER_CHECK_INTERVAL = 5 # seconds
+      # A task whose enqueue failed is retried after 5 s, then 10, 20, … up to
+      # 5 minutes, so a deterministic failure (a before_enqueue that always
+      # aborts, a uniqueness reject while the duplicate is queued) does not
+      # retry and report on every tick.
+      RETRY_BACKOFF_BASE = 5 # seconds
+      RETRY_BACKOFF_MAX = 300 # seconds
+      ROLLOVER = "rollover"
+      MANUAL = "manual"
+
+      attr_reader :revision
+
+      def initialize(config: Pgbus.configuration, revision: Revision.current(config))
+        @config = config
+        @revision = revision
+        @tasks = nil
+        @status = nil
+        @checked_at = nil
+        @logged_wait = nil
+        @failures = {}
+        @synced = nil
+        @sync_failures = 0
+        @sync_retry_at = nil
+      end
+
+      # The valid declared tasks, in key order. A class: that does not resolve
+      # to an ActiveJob is skipped with a warning, like an invalid task.
+      def tasks
+        @tasks ||= load_tasks
+      end
+
+      def task_count
+        tasks.size
+      end
+
+      # true once synced. A failure (a database blip at boot) is retried from
+      # tick with the enqueue backoff, so the scheduler keeps no unarmed tasks.
+      def sync!(now: Time.current)
+        if revision.nil? && @synced.nil?
+          Pgbus.logger.warn do
+            "[Pgbus] No app revision configured (config.app_revision, PGBUS_REVISION, DASH_VERSION, KAMAL_VERSION " \
+              "or a REVISION file): deploy tasks wait until one is set"
+          end
+        end
+        DeployTask.sync_from_config!(tasks, revision: revision, now: now)
+        Pgbus.logger.info { "[Pgbus] Synced #{tasks.size} deploy task(s) for revision #{revision.inspect}" }
+        @synced = true
+      rescue StandardError => e
+        @sync_failures += 1
+        delay = retry_delay(@sync_failures)
+        @sync_retry_at = monotonic_now + delay
+        # Reported (and logged at error by the reporter) once; repeats of a
+        # persistent failure drop to debug instead of reporting every retry.
+        if @sync_failures == 1
+          ErrorReporter.report(e, { action: "deploy_tasks_sync", retry_in: delay })
+        else
+          Pgbus.logger.debug { "[Pgbus] Deploy task sync still failing (retrying in #{delay}s): #{e.class}: #{e.message}" }
+        end
+        @synced = false
+      end
+
+      def tick(now = Time.current)
+        return if tasks.empty?
+        return if @synced == false && !retry_sync(now)
+        return unless DeployTask.pending?
+
+        status = gate_status
+        return log_waiting(status) unless status.rolled_over?
+
+        @logged_wait = nil
+        rows = DeployTask.where(key: tasks.map(&:key)).index_by(&:key)
+        tasks.each do |task|
+          row = rows[task.key]
+          fire(task, row, now) if row&.enabled && row.armed? && !backing_off?(task.key)
+        end
+      end
+
+      # Enqueue +task+ through ActiveJob and record the run. :enqueued, or
+      # :failed (logged and reported) when perform_later raised or did not
+      # enqueue the job. Run now calls this directly: no gate, no claim.
+      def enqueue(task, triggered_by:, now: Time.current)
+        payload = { task: task.key, version: task.version, revision: revision, triggered_by: triggered_by }
+        job = Instrumentation.instrument("pgbus.deploy_task.enqueue", payload) { submit(task) }
+        record_run(task, job, triggered_by, now)
+        :enqueued
+      rescue StandardError => e
+        Pgbus.logger.error do
+          "[Pgbus] Deploy task #{task.key} (version #{task.version}, #{triggered_by}) was not enqueued: " \
+            "#{e.class}: #{e.message}"
+        end
+        ErrorReporter.report(e, { action: "deploy_task_enqueue", task: task.key, version: task.version,
+                                  revision: revision, triggered_by: triggered_by })
+        :failed
+      end
+
+      private
+
+      def load_tasks
+        (@config.deploy_tasks || {}).filter_map { |key, options| build_task(key, options) }.sort_by(&:key)
+      end
+
+      def build_task(key, options)
+        options = options.is_a?(Hash) ? options.transform_keys(&:to_sym) : {}
+        task = Task.from_configuration(key, **options)
+        return skip_task(key, task.errors.join(", ")) unless task.valid?
+        return skip_task(key, "#{task.class_name} is not an ActiveJob") if task.class_name && !active_job?(task.job_class)
+
+        task.warnings.each { |warning| Pgbus.logger.warn { "[Pgbus] Deploy task '#{key}': #{warning}" } }
+        task
+      rescue StandardError => e
+        # A malformed entry (a Hash where a scalar belongs) skips that task
+        # instead of crashing the scheduler at boot.
+        skip_task(key, "#{e.class}: #{e.message}")
+      end
+
+      def active_job?(klass)
+        klass.is_a?(Class) && defined?(::ActiveJob::Base) && klass <= ::ActiveJob::Base
+      end
+
+      def skip_task(key, reason)
+        Pgbus.logger.warn { "[Pgbus] Skipping invalid deploy task '#{key}': #{reason}" }
+        nil
+      end
+
+      def gate_status
+        at = monotonic_now
+        return @status if @checked_at && at - @checked_at < ROLLOVER_CHECK_INTERVAL
+
+        # Stamped only after a successful check: a raising check must not
+        # leave a nil status cached for the next ticks.
+        @status = Rollover.check(revision: revision)
+        @checked_at = at
+        @status
+      end
+
+      def retry_sync(now)
+        return false if monotonic_now < @sync_retry_at
+
+        sync!(now: now)
+      end
+
+      def log_waiting(status)
+        summary = waiting_summary(status)
+        return if summary == @logged_wait
+
+        @logged_wait = summary
+        Pgbus.logger.info { "[Pgbus] Deploy tasks waiting for rollover: #{summary}" }
+      end
+
+      def waiting_summary(status)
+        case status.reason
+        when :no_revision then "app revision not configured"
+        when :no_workers then "no worker on #{revision} yet"
+        else
+          processes = status.lagging.map { |p| "#{p[:kind]}@#{p[:hostname]} pid #{p[:pid]} (#{p[:revision] || "no revision"})" }
+          "#{status.lagging.size} process(es) not on #{revision}: #{processes.join(", ")}"
+        end
+      end
+
+      # An arm for another version or revision belongs to code no live
+      # process runs any more (the gate is open on this revision), so when
+      # this runner's own version is owed it takes the arm over; otherwise
+      # the claim below fails and nothing fires.
+      def fire(task, row, now)
+        armed_for = task.armed_for(revision)
+        row.take_over!(armed_for: armed_for, now: now) if row.armed_for != armed_for && DeployTask.owed?(task, revision)
+        return unless row.claim!(armed_for: armed_for, now: now)
+        return @failures.delete(task.key) if enqueue(task, triggered_by: ROLLOVER, now: now) == :enqueued
+
+        rearm(task, row, armed_for)
+      end
+
+      def rearm(task, row, armed_for)
+        delay = note_failure(task.key)
+        row.rearm!(armed_for: armed_for, now: Time.current)
+        Pgbus.logger.warn { "[Pgbus] Deploy task #{task.key} re-armed; retrying in #{delay}s" }
+      rescue StandardError => e
+        Pgbus.logger.error { "[Pgbus] Could not re-arm deploy task #{task.key}: #{e.class}: #{e.message}" }
+        ErrorReporter.report(e, { action: "deploy_task_rearm", task: task.key, armed_for: armed_for })
+      end
+
+      def backing_off?(key)
+        failure = @failures[key]
+        !failure.nil? && monotonic_now < failure[:retry_at]
+      end
+
+      # Records one more failed enqueue for +key+ and returns the seconds until its retry.
+      def note_failure(key)
+        count = @failures.fetch(key, { count: 0 })[:count] + 1
+        delay = retry_delay(count)
+        @failures[key] = { count: count, retry_at: monotonic_now + delay }
+        delay
+      end
+
+      def retry_delay(count)
+        [RETRY_BACKOFF_BASE * (2**(count - 1)), RETRY_BACKOFF_MAX].min
+      end
+
+      def monotonic_now
+        ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+      end
+
+      # Builds the job and enqueues it directly (what perform_later does), so
+      # the job, and its enqueue_error, is in hand on every Rails version:
+      # perform_later returns false on failure, and on Rails 7.1 a .set(...)
+      # job never yields it to perform_later's block.
+      def submit(task)
+        job = job_class_for(task).new(*job_arguments(task))
+        return job if job.enqueue(job_options(task))
+
+        error = job.enqueue_error
+        raise EnqueueError, error ? "#{error.class}: #{error.message}" : "the job was not enqueued (a before_enqueue callback aborted)"
+      end
+
+      def job_class_for(task)
+        return Recurring::CommandJob if task.command
+
+        klass = task.job_class
+        raise EnqueueError, "#{task.class_name} does not resolve to a job class" unless active_job?(klass)
+
+        klass
+      end
+
+      def job_options(task)
+        priority = task.command || task.priority.zero? ? nil : task.priority
+        { queue: task.queue_name, priority: priority }.compact
+      end
+
+      def job_arguments(task)
+        task.command ? [task.command] : task.arguments
+      end
+
+      def record_run(task, job, triggered_by, now)
+        DeployTaskRun.create!(task_key: task.key, version: task.version, revision: revision,
+                              triggered_by: triggered_by, queue_name: job.queue_name,
+                              msg_id: job.provider_job_id, run_at: now)
+      rescue StandardError => e
+        Pgbus.logger.error do
+          "[Pgbus] Deploy task #{task.key} was enqueued but its run was not recorded: #{e.class}: #{e.message}"
+        end
+        ErrorReporter.report(e, { action: "deploy_task_record_run", task: task.key, triggered_by: triggered_by })
+      end
+    end
+  end
+end