pgbus 0.19.0 → 0.19.1

data/lib/generators/pgbus/add_deploy_tasks_generator.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+require "rails/generators"
+require "rails/generators/active_record"
+require_relative "migration_path"
+
+module Pgbus
+  module Generators
+    class AddDeployTasksGenerator < Rails::Generators::Base
+      include ActiveRecord::Generators::Migration
+      include MigrationPath
+
+      source_root File.expand_path("templates", __dir__)
+
+      desc "Add deploy tasks tables (one-time jobs run after a rollover) to an existing Pgbus installation"
+
+      class_option :database,
+                   type: :string,
+                   default: nil,
+                   desc: "Use a separate database for pgbus tables (e.g. --database=pgbus)"
+
+      def create_migration_file
+        migration_template "add_deploy_tasks_tables.rb.erb",
+                           File.join(pgbus_migrate_path, "add_pgbus_deploy_tasks_tables.rb")
+      end
+
+      def create_deploy_tasks_config
+        template "deploy_tasks.yml.erb", "config/deploy_tasks.yml"
+      end
+
+      def display_post_install
+        say ""
+        say "Pgbus deploy tasks installed!", :green
+        say ""
+        say "Next steps:"
+        say "  1. Run: rails db:migrate#{migrate_command_suffix}"
+        say "  2. Edit config/deploy_tasks.yml to declare your one-time tasks"
+        say "  3. Make sure every pgbus process knows the app revision (config.app_revision,"
+        say "     PGBUS_REVISION, DASH_VERSION, KAMAL_VERSION or a REVISION file)"
+        say "  4. Restart pgbus: bin/pgbus start"
+        say ""
+      end
+
+      private
+
+      def migration_version
+        "[#{ActiveRecord::Migration.current_version}]"
+      end
+    end
+  end
+end

data/lib/generators/pgbus/templates/add_deploy_tasks_tables.rb.erb

@@ -0,0 +1,43 @@
+class AddPgbusDeployTasksTables < ActiveRecord::Migration<%= migration_version %>
+  # if_not_exists throughout: pgbus:update queues this migration when either
+  # table is missing, so a half-present schema must not fail it.
+  def change
+    create_table :pgbus_deploy_tasks, if_not_exists: true do |t|
+      t.string :key, null: false
+      t.string :class_name
+      t.string :command, limit: 2048
+      t.text :arguments
+      t.string :queue_name
+      t.integer :priority, default: 0
+      t.string :version, null: false, default: "1"
+      t.boolean :every_deploy, null: false, default: false
+      t.boolean :static, null: false, default: true
+      t.boolean :enabled, null: false, default: true
+      t.text :description
+      t.datetime :armed_at
+      t.string :armed_for
+      t.timestamps
+    end
+
+    add_index :pgbus_deploy_tasks, :key,
+      unique: true, name: "idx_pgbus_deploy_tasks_key", if_not_exists: true
+    add_index :pgbus_deploy_tasks, :armed_at,
+      where: "armed_at IS NOT NULL", name: "idx_pgbus_deploy_tasks_armed", if_not_exists: true
+
+    create_table :pgbus_deploy_task_runs, if_not_exists: true do |t|
+      t.string :task_key, null: false
+      t.string :version, null: false
+      t.string :revision
+      t.string :triggered_by, null: false
+      t.string :queue_name
+      t.bigint :msg_id
+      t.datetime :run_at, null: false
+      t.datetime :created_at, null: false, default: -> { "CURRENT_TIMESTAMP" }
+    end
+
+    add_index :pgbus_deploy_task_runs, [:task_key, :run_at],
+      name: "idx_pgbus_deploy_task_runs_task", if_not_exists: true
+    add_index :pgbus_deploy_task_runs, :run_at,
+      name: "idx_pgbus_deploy_task_runs_cleanup", if_not_exists: true
+  end
+end

data/lib/generators/pgbus/templates/deploy_tasks.yml.erb

@@ -0,0 +1,42 @@
+# Pgbus Deploy Tasks Configuration
+#
+# One-time jobs that run once the whole fleet has rolled over to a new app
+# revision: a backfill, a data fix, a cache warm. Same shape as
+# config/recurring.yml, without schedule:. Supports environment-scoped or
+# flat configuration.
+#
+# The scheduler enqueues a task only when every live pgbus process reports
+# the scheduler's own app revision and at least one worker on it is alive,
+# so a task never runs on old code during the deploy overlap window. The
+# revision comes from config.app_revision, then PGBUS_REVISION, DASH_VERSION,
+# KAMAL_VERSION, then the REVISION file at Rails.root. Without one, deploy
+# tasks wait (the dashboard says so) and only "Run now" enqueues them.
+#
+# Format:
+#   task_key:
+#     class: JobClassName      # ActiveJob class to run (or use command:)
+#     version: 1               # Runs once per version; bump it to run again after the next deploy
+#     every_deploy: false      # true: run after every rollover instead (version is ignored)
+#     queue: default           # Optional: queue name (default: pgbus default queue)
+#     args: [arg1, arg2]       # Optional: job arguments
+#     priority: 0              # Optional: numeric priority
+#     description: "text"      # Optional: human-readable description
+#
+# Command-based tasks (no job class needed):
+#   warm_fragment_caches:
+#     command: "FragmentCacheWarmer.run"
+#     every_deploy: true
+#
+# Examples:
+#
+# production:
+#   backfill_order_totals:
+#     class: BackfillOrderTotalsJob
+#     version: 2
+#     queue: maintenance
+#     args: [{ batch_size: 500 }]
+#     description: Recompute totals after the currency migration
+#
+#   warm_fragment_caches:
+#     command: "FragmentCacheWarmer.run"
+#     every_deploy: true

data/lib/generators/pgbus/templates/migration.rb.erb

@@ -170,6 +170,47 @@ class CreatePgbusTables < ActiveRecord::Migration<%= migration_version %>
     add_index :pgbus_recurring_executions, :run_at,
       name: "idx_pgbus_recurring_executions_cleanup"
 
+    # Deploy tasks (synced from config/deploy_tasks.yml): one-time jobs the
+    # scheduler enqueues once the fleet has rolled over to a new revision.
+    # armed_at/armed_for is the compare-and-set claim; runs are the history.
+    create_table :pgbus_deploy_tasks do |t|
+      t.string :key, null: false
+      t.string :class_name
+      t.string :command, limit: 2048
+      t.text :arguments
+      t.string :queue_name
+      t.integer :priority, default: 0
+      t.string :version, null: false, default: "1"
+      t.boolean :every_deploy, null: false, default: false
+      t.boolean :static, null: false, default: true
+      t.boolean :enabled, null: false, default: true
+      t.text :description
+      t.datetime :armed_at
+      t.string :armed_for
+      t.timestamps
+    end
+
+    add_index :pgbus_deploy_tasks, :key,
+      unique: true, name: "idx_pgbus_deploy_tasks_key"
+    add_index :pgbus_deploy_tasks, :armed_at,
+      where: "armed_at IS NOT NULL", name: "idx_pgbus_deploy_tasks_armed"
+
+    create_table :pgbus_deploy_task_runs do |t|
+      t.string :task_key, null: false
+      t.string :version, null: false
+      t.string :revision
+      t.string :triggered_by, null: false
+      t.string :queue_name
+      t.bigint :msg_id
+      t.datetime :run_at, null: false
+      t.datetime :created_at, null: false, default: -> { "CURRENT_TIMESTAMP" }
+    end
+
+    add_index :pgbus_deploy_task_runs, [:task_key, :run_at],
+      name: "idx_pgbus_deploy_task_runs_task"
+    add_index :pgbus_deploy_task_runs, :run_at,
+      name: "idx_pgbus_deploy_task_runs_cleanup"
+
     # Stream queue registry — records which PGMQ queues back streams (vs jobs).
     # Stream and job queues share the pgbus_ namespace, so maintenance
     # (per-stream retention, orphan sweep) and wildcard workers rely on this
@@ -204,6 +245,8 @@ class CreatePgbusTables < ActiveRecord::Migration<%= migration_version %>
     execute "SELECT pgmq.drop_queue('pgbus_default')"
 
     drop_table :pgbus_stream_queues
+    drop_table :pgbus_deploy_task_runs
+    drop_table :pgbus_deploy_tasks
     drop_table :pgbus_recurring_executions
     drop_table :pgbus_recurring_tasks
     drop_table :pgbus_batch_executions

data/lib/pgbus/configuration/deploy_settings.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module Pgbus
+  class Configuration
+    # Deploy tasks (issue #536): one-time jobs the scheduler enqueues once
+    # every live pgbus process runs the new app revision.
+    #
+    #   config.app_revision = -> { File.read("REVISION").strip } # or a String
+    #   config.deploy_tasks_files = ["config/deploy_tasks.yml"]
+    #   config.deploy_task_run_retention = 30.days
+    module DeploySettings
+      DEFAULT_DEPLOY_TASK_RUN_RETENTION = 90 * 24 * 3600
+
+      attr_accessor :app_revision, :deploy_tasks, :deploy_tasks_files, :deploy_tasks_enabled
+      attr_reader :deploy_task_run_retention
+
+      def deploy_task_run_retention=(value)
+        @deploy_task_run_retention = coerce_duration!(value, :deploy_task_run_retention)
+      end
+
+      private
+
+      def initialize_deploy_settings
+        @app_revision = nil
+        @deploy_tasks = nil
+        @deploy_tasks_files = nil
+        @deploy_tasks_enabled = true
+        @deploy_task_run_retention = DEFAULT_DEPLOY_TASK_RUN_RETENTION
+      end
+
+      def validate_deploy_settings!
+        unless app_revision.nil? || app_revision.is_a?(String) || app_revision.respond_to?(:call)
+          raise Pgbus::ConfigurationError, "app_revision must be a String, respond to #call, or be nil"
+        end
+
+        raise Pgbus::ConfigurationError, "deploy_tasks must be a Hash or nil" unless deploy_tasks.nil? || deploy_tasks.is_a?(Hash)
+        unless deploy_tasks_files.nil? || deploy_tasks_files.is_a?(Array)
+          raise Pgbus::ConfigurationError, "deploy_tasks_files must be an Array of paths or nil"
+        end
+        return if [true, false].include?(deploy_tasks_enabled)
+
+        raise Pgbus::ConfigurationError, "deploy_tasks_enabled must be true or false"
+      end
+    end
+  end
+end

data/lib/pgbus/configuration.rb

@@ -6,6 +6,7 @@ module Pgbus
   class Configuration
     include PresenceSettings
     include StreamAuthorization
+    include DeploySettings
 
     # Connection settings
     attr_accessor :database_url, :connection_params, :pool_size, :pool_timeout
@@ -363,6 +364,7 @@ module Pgbus
       @recurring_tasks_files = nil
       @recurring_enabled = true
       @recurring_execution_retention = 7 * 24 * 3600 # 7 days
+      initialize_deploy_settings
 
       @zombie_detection = true
 
@@ -944,6 +946,7 @@ module Pgbus
       validate_streams!
       validate_metrics_backend!
       validate_fair_share!
+      validate_deploy_settings!
 
       self
     end

data/lib/pgbus/deploy/revision.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+module Pgbus
+  module Deploy
+    # The app revision this process runs (issue #536). Every heartbeat row
+    # carries it in metadata["revision"]; the deploy-task rollover gate
+    # compares those against the scheduler's own.
+    #
+    # First match wins, blank counts as absent:
+    #   config.app_revision (String or callable) → PGBUS_REVISION → DASH_VERSION
+    #   → KAMAL_VERSION → the REVISION file at Rails.root (Capistrano) → nil
+    #
+    # Computed per call, never memoized: it is read once per process
+    # registration, and a module-level memo would be unguarded shared state.
+    module Revision
+      ENV_KEYS = %w[PGBUS_REVISION DASH_VERSION KAMAL_VERSION].freeze
+
+      module_function
+
+      def current(config = Pgbus.configuration)
+        from_config(config) || from_env || from_file
+      end
+
+      def from_config(config)
+        value = config.app_revision
+        value = value.call if value.respond_to?(:call)
+        presence(value)
+      rescue StandardError => e
+        Pgbus.logger.warn { "[Pgbus] app_revision raised #{e.class}: #{e.message}; trying the environment" }
+        nil
+      end
+
+      def from_env
+        ENV_KEYS.each do |key|
+          value = presence(ENV.fetch(key, nil))
+          return value if value
+        end
+        nil
+      end
+
+      def from_file
+        return nil unless defined?(Rails) && Rails.respond_to?(:root) && Rails.root
+
+        path = Rails.root.join("REVISION")
+        File.exist?(path) ? presence(File.read(path)) : nil
+      rescue SystemCallError, IOError => e
+        # Read while every heartbeat initializes: an unreadable optional file
+        # must not stop a process from starting.
+        Pgbus.logger.warn { "[Pgbus] Could not read #{path}: #{e.class}: #{e.message}" }
+        nil
+      end
+
+      def presence(value)
+        string = value.to_s.strip
+        string.empty? ? nil : string
+      end
+    end
+  end
+end

data/lib/pgbus/deploy/rollover.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Pgbus
+  module Deploy
+    # Has the fleet rolled over to +revision+? (issue #536). A pure read of
+    # pgbus_processes: live rows (heartbeat within Heartbeat::ALIVE_THRESHOLD)
+    # whose metadata["revision"] differs from +revision+ are lagging (a missing
+    # revision is a pre-upgrade process, so it lags too), and at least one live
+    # worker on +revision+ must exist. The scheduler and the dashboard both
+    # consume the Status, so the page prints the reason the scheduler acts on.
+    module Rollover
+      # reason – :ready, :no_revision, :lagging or :no_workers.
+      # lagging – [{ kind:, hostname:, pid:, revision: }].
+      # live_by_revision – { revision => live process count }, nil for rows without one.
+      Status = Data.define(:revision, :lagging, :workers_live, :reason, :live_by_revision) do
+        def rolled_over?
+          reason == :ready
+        end
+      end
+
+      module_function
+
+      def check(revision:, now: Time.current)
+        cutoff = now - Process::Heartbeat::ALIVE_THRESHOLD
+        rows = ProcessEntry.where(last_heartbeat_at: cutoff..).pluck(:kind, :hostname, :pid, :metadata)
+        evaluate(rows.map { |kind, hostname, pid, metadata| { kind:, hostname:, pid:, metadata: } }, revision: revision)
+      end
+
+      # [live, revision] of the newest live scheduler row: the revision the
+      # dashboard judges the rollover against (the one the scheduler acts on).
+      def scheduler_revision(now: Time.current)
+        row = ProcessEntry.where(kind: "scheduler", last_heartbeat_at: (now - Process::Heartbeat::ALIVE_THRESHOLD)..)
+                          .order(created_at: :desc, id: :desc).first
+        row ? [true, revision_of(row.metadata)] : [false, nil]
+      end
+
+      # rows – live process rows as { kind:, hostname:, pid:, metadata: }.
+      def evaluate(rows, revision:)
+        live = rows.map { |r| { kind: r[:kind], hostname: r[:hostname], pid: r[:pid], revision: revision_of(r[:metadata]) } }
+        lagging = live.reject { |r| revision && r[:revision] == revision }
+        workers_live = live.count { |r| r[:kind] == "worker" && revision && r[:revision] == revision }
+
+        Status.new(revision: revision, lagging: revision ? lagging : [], workers_live: workers_live,
+                   reason: reason_for(revision, lagging, workers_live), live_by_revision: live.map { |r| r[:revision] }.tally)
+      end
+
+      def reason_for(revision, lagging, workers_live)
+        return :no_revision if revision.nil?
+        return :lagging if lagging.any?
+        return :no_workers if workers_live.zero?
+
+        :ready
+      end
+
+      def revision_of(metadata)
+        metadata = JSON.parse(metadata) if metadata.is_a?(String)
+        metadata.is_a?(Hash) ? metadata["revision"] : nil
+      rescue JSON::ParserError
+        nil
+      end
+    end
+  end
+end

data/lib/pgbus/deploy/runner.rb

@@ -0,0 +1,259 @@
+# frozen_string_literal: true
+
+module Pgbus
+  module Deploy
+    # Fires deploy tasks once the fleet has rolled over (issue #536). Ticked
+    # by the recurring scheduler's loop; one instance per scheduler process,
+    # so its throttle and log state are plain instance state on that loop.
+    #
+    # A tick: cheap pending check → the rollover gate (at most every
+    # ROLLOVER_CHECK_INTERVAL seconds) → for each declared, enabled, armed
+    # task in key order: claim the arm → perform_later → record the run. A
+    # failed enqueue re-arms the task, logs and reports. No surrounding
+    # transaction: the adapter defers an enqueue inside one to its commit.
+    class Runner
+      ROLLOVER_CHECK_INTERVAL = 5 # seconds
+      # A task whose enqueue failed is retried after 5 s, then 10, 20, … up to
+      # 5 minutes, so a deterministic failure (a before_enqueue that always
+      # aborts, a uniqueness reject while the duplicate is queued) does not
+      # retry and report on every tick.
+      RETRY_BACKOFF_BASE = 5 # seconds
+      RETRY_BACKOFF_MAX = 300 # seconds
+      ROLLOVER = "rollover"
+      MANUAL = "manual"
+
+      attr_reader :revision
+
+      def initialize(config: Pgbus.configuration, revision: Revision.current(config))
+        @config = config
+        @revision = revision
+        @tasks = nil
+        @status = nil
+        @checked_at = nil
+        @logged_wait = nil
+        @failures = {}
+        @synced = nil
+        @sync_failures = 0
+        @sync_retry_at = nil
+      end
+
+      # The valid declared tasks, in key order. A class: that does not resolve
+      # to an ActiveJob is skipped with a warning, like an invalid task.
+      def tasks
+        @tasks ||= load_tasks
+      end
+
+      def task_count
+        tasks.size
+      end
+
+      # true once synced. A failure (a database blip at boot) is retried from
+      # tick with the enqueue backoff, so the scheduler keeps no unarmed tasks.
+      def sync!(now: Time.current)
+        if revision.nil? && @synced.nil?
+          Pgbus.logger.warn do
+            "[Pgbus] No app revision configured (config.app_revision, PGBUS_REVISION, DASH_VERSION, KAMAL_VERSION " \
+              "or a REVISION file): deploy tasks wait until one is set"
+          end
+        end
+        DeployTask.sync_from_config!(tasks, revision: revision, now: now)
+        Pgbus.logger.info { "[Pgbus] Synced #{tasks.size} deploy task(s) for revision #{revision.inspect}" }
+        @synced = true
+      rescue StandardError => e
+        @sync_failures += 1
+        delay = retry_delay(@sync_failures)
+        @sync_retry_at = monotonic_now + delay
+        # Reported (and logged at error by the reporter) once; repeats of a
+        # persistent failure drop to debug instead of reporting every retry.
+        if @sync_failures == 1
+          ErrorReporter.report(e, { action: "deploy_tasks_sync", retry_in: delay })
+        else
+          Pgbus.logger.debug { "[Pgbus] Deploy task sync still failing (retrying in #{delay}s): #{e.class}: #{e.message}" }
+        end
+        @synced = false
+      end
+
+      def tick(now = Time.current)
+        return if tasks.empty?
+        return if @synced == false && !retry_sync(now)
+        return unless DeployTask.pending?
+
+        status = gate_status
+        return log_waiting(status) unless status.rolled_over?
+
+        @logged_wait = nil
+        rows = DeployTask.where(key: tasks.map(&:key)).index_by(&:key)
+        tasks.each do |task|
+          row = rows[task.key]
+          fire(task, row, now) if row&.enabled && row.armed? && !backing_off?(task.key)
+        end
+      end
+
+      # Enqueue +task+ through ActiveJob and record the run. :enqueued, or
+      # :failed (logged and reported) when perform_later raised or did not
+      # enqueue the job. Run now calls this directly: no gate, no claim.
+      def enqueue(task, triggered_by:, now: Time.current)
+        payload = { task: task.key, version: task.version, revision: revision, triggered_by: triggered_by }
+        job = Instrumentation.instrument("pgbus.deploy_task.enqueue", payload) { submit(task) }
+        record_run(task, job, triggered_by, now)
+        :enqueued
+      rescue StandardError => e
+        Pgbus.logger.error do
+          "[Pgbus] Deploy task #{task.key} (version #{task.version}, #{triggered_by}) was not enqueued: " \
+            "#{e.class}: #{e.message}"
+        end
+        ErrorReporter.report(e, { action: "deploy_task_enqueue", task: task.key, version: task.version,
+                                  revision: revision, triggered_by: triggered_by })
+        :failed
+      end
+
+      private
+
+      def load_tasks
+        (@config.deploy_tasks || {}).filter_map { |key, options| build_task(key, options) }.sort_by(&:key)
+      end
+
+      def build_task(key, options)
+        options = options.is_a?(Hash) ? options.transform_keys(&:to_sym) : {}
+        task = Task.from_configuration(key, **options)
+        return skip_task(key, task.errors.join(", ")) unless task.valid?
+        return skip_task(key, "#{task.class_name} is not an ActiveJob") if task.class_name && !active_job?(task.job_class)
+
+        task.warnings.each { |warning| Pgbus.logger.warn { "[Pgbus] Deploy task '#{key}': #{warning}" } }
+        task
+      rescue StandardError => e
+        # A malformed entry (a Hash where a scalar belongs) skips that task
+        # instead of crashing the scheduler at boot.
+        skip_task(key, "#{e.class}: #{e.message}")
+      end
+
+      def active_job?(klass)
+        klass.is_a?(Class) && defined?(::ActiveJob::Base) && klass <= ::ActiveJob::Base
+      end
+
+      def skip_task(key, reason)
+        Pgbus.logger.warn { "[Pgbus] Skipping invalid deploy task '#{key}': #{reason}" }
+        nil
+      end
+
+      def gate_status
+        at = monotonic_now
+        return @status if @checked_at && at - @checked_at < ROLLOVER_CHECK_INTERVAL
+
+        # Stamped only after a successful check: a raising check must not
+        # leave a nil status cached for the next ticks.
+        @status = Rollover.check(revision: revision)
+        @checked_at = at
+        @status
+      end
+
+      def retry_sync(now)
+        return false if monotonic_now < @sync_retry_at
+
+        sync!(now: now)
+      end
+
+      def log_waiting(status)
+        summary = waiting_summary(status)
+        return if summary == @logged_wait
+
+        @logged_wait = summary
+        Pgbus.logger.info { "[Pgbus] Deploy tasks waiting for rollover: #{summary}" }
+      end
+
+      def waiting_summary(status)
+        case status.reason
+        when :no_revision then "app revision not configured"
+        when :no_workers then "no worker on #{revision} yet"
+        else
+          processes = status.lagging.map { |p| "#{p[:kind]}@#{p[:hostname]} pid #{p[:pid]} (#{p[:revision] || "no revision"})" }
+          "#{status.lagging.size} process(es) not on #{revision}: #{processes.join(", ")}"
+        end
+      end
+
+      # An arm for another version or revision belongs to code no live
+      # process runs any more (the gate is open on this revision), so when
+      # this runner's own version is owed it takes the arm over; otherwise
+      # the claim below fails and nothing fires.
+      def fire(task, row, now)
+        armed_for = task.armed_for(revision)
+        row.take_over!(armed_for: armed_for, now: now) if row.armed_for != armed_for && DeployTask.owed?(task, revision)
+        return unless row.claim!(armed_for: armed_for, now: now)
+        return @failures.delete(task.key) if enqueue(task, triggered_by: ROLLOVER, now: now) == :enqueued
+
+        rearm(task, row, armed_for)
+      end
+
+      def rearm(task, row, armed_for)
+        delay = note_failure(task.key)
+        row.rearm!(armed_for: armed_for, now: Time.current)
+        Pgbus.logger.warn { "[Pgbus] Deploy task #{task.key} re-armed; retrying in #{delay}s" }
+      rescue StandardError => e
+        Pgbus.logger.error { "[Pgbus] Could not re-arm deploy task #{task.key}: #{e.class}: #{e.message}" }
+        ErrorReporter.report(e, { action: "deploy_task_rearm", task: task.key, armed_for: armed_for })
+      end
+
+      def backing_off?(key)
+        failure = @failures[key]
+        !failure.nil? && monotonic_now < failure[:retry_at]
+      end
+
+      # Records one more failed enqueue for +key+ and returns the seconds until its retry.
+      def note_failure(key)
+        count = @failures.fetch(key, { count: 0 })[:count] + 1
+        delay = retry_delay(count)
+        @failures[key] = { count: count, retry_at: monotonic_now + delay }
+        delay
+      end
+
+      def retry_delay(count)
+        [RETRY_BACKOFF_BASE * (2**(count - 1)), RETRY_BACKOFF_MAX].min
+      end
+
+      def monotonic_now
+        ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+      end
+
+      # Builds the job and enqueues it directly (what perform_later does), so
+      # the job, and its enqueue_error, is in hand on every Rails version:
+      # perform_later returns false on failure, and on Rails 7.1 a .set(...)
+      # job never yields it to perform_later's block.
+      def submit(task)
+        job = job_class_for(task).new(*job_arguments(task))
+        return job if job.enqueue(job_options(task))
+
+        error = job.enqueue_error
+        raise EnqueueError, error ? "#{error.class}: #{error.message}" : "the job was not enqueued (a before_enqueue callback aborted)"
+      end
+
+      def job_class_for(task)
+        return Recurring::CommandJob if task.command
+
+        klass = task.job_class
+        raise EnqueueError, "#{task.class_name} does not resolve to a job class" unless active_job?(klass)
+
+        klass
+      end
+
+      def job_options(task)
+        priority = task.command || task.priority.zero? ? nil : task.priority
+        { queue: task.queue_name, priority: priority }.compact
+      end
+
+      def job_arguments(task)
+        task.command ? [task.command] : task.arguments
+      end
+
+      def record_run(task, job, triggered_by, now)
+        DeployTaskRun.create!(task_key: task.key, version: task.version, revision: revision,
+                              triggered_by: triggered_by, queue_name: job.queue_name,
+                              msg_id: job.provider_job_id, run_at: now)
+      rescue StandardError => e
+        Pgbus.logger.error do
+          "[Pgbus] Deploy task #{task.key} was enqueued but its run was not recorded: #{e.class}: #{e.message}"
+        end
+        ErrorReporter.report(e, { action: "deploy_task_record_run", task: task.key, triggered_by: triggered_by })
+      end
+    end
+  end
+end