pgbus 0.19.3 → 0.20.0

data/lib/generators/pgbus/templates/add_workflows.rb.erb

@@ -0,0 +1,150 @@
+class AddPgbusWorkflows < ActiveRecord::Migration<%= migration_version %>
+  # if_not_exists throughout: pgbus:update queues this migration when any of
+  # the three core tables is missing, so a half-present schema must not fail
+  # it. The waiting-step columns and pgbus_workflow_signals (issue #541) are
+  # here too, so a fresh install needs no pgbus:add_workflow_signals.
+  # up/down rather than change: Rails 7.1 cannot reverse an add_foreign_key
+  # that names its primary_key, and dropping the tables drops the keys.
+  def up
+    # One row per workflow run (Pgbus::WorkflowEntry).
+    create_table :pgbus_workflows, if_not_exists: true do |t|
+      t.string :workflow_id, null: false
+      t.string :workflow_class, null: false
+      t.string :subject_type
+      t.string :subject_id
+      t.string :subject_gid
+      t.jsonb :arguments, null: false, default: {}
+      t.string :status, null: false, default: "ready"
+      t.string :current_step
+      t.jsonb :step_job_options, null: false, default: {}
+      t.string :error_class
+      t.text :error_message
+      t.text :error_backtrace
+      t.datetime :paused_at
+      t.datetime :finished_at
+      t.datetime :cancelled_at
+      t.string :idempotency_key
+      t.string :definition_digest, null: false
+      t.string :rerun_of_workflow_id
+      t.datetime :created_at, null: false
+      t.datetime :updated_at, null: false
+    end
+
+    add_index :pgbus_workflows, :workflow_id,
+      unique: true, name: "idx_pgbus_workflows_workflow_id", if_not_exists: true
+    add_index :pgbus_workflows, [:workflow_class, :subject_type, :subject_id],
+      unique: true, where: "status IN ('ready','performing','waiting','paused')",
+      name: "idx_pgbus_workflows_one_ongoing", if_not_exists: true
+    add_index :pgbus_workflows, [:workflow_class, :idempotency_key],
+      unique: true, where: "idempotency_key IS NOT NULL",
+      name: "idx_pgbus_workflows_idempotency", if_not_exists: true
+    add_index :pgbus_workflows, [:status, :updated_at],
+      name: "idx_pgbus_workflows_status", if_not_exists: true
+    add_index :pgbus_workflows, [:workflow_class, :status, :created_at],
+      name: "idx_pgbus_workflows_class_status", if_not_exists: true
+    add_index :pgbus_workflows, [:subject_type, :subject_id],
+      name: "idx_pgbus_workflows_subject", if_not_exists: true
+
+    # One row per step attempt (Pgbus::WorkflowStep).
+    create_table :pgbus_workflow_steps, if_not_exists: true do |t|
+      t.string :workflow_id, null: false
+      t.string :workflow_class, null: false
+      t.string :step_name, null: false
+      t.integer :attempt, null: false, default: 1
+      t.string :status, null: false, default: "scheduled"
+      t.string :job_id
+      t.bigint :msg_id
+      t.datetime :scheduled_at, null: false
+      t.datetime :enqueued_at
+      t.datetime :started_at
+      t.datetime :finished_at
+      t.datetime :stall_at
+      t.string :outcome
+      t.jsonb :output
+      t.string :error_class
+      t.text :error_message
+      t.text :error_backtrace
+      t.jsonb :error_causes
+      t.string :worker
+      t.string :app_revision
+      # Waiting steps (fan_out, wait_for:): also added by pgbus:add_workflow_signals.
+      t.string :batch_id
+      t.string :wait_for
+      t.bigint :signal_id
+      t.datetime :waiting_since
+      t.datetime :timeout_at
+      t.datetime :created_at, null: false
+    end
+
+    add_index :pgbus_workflow_steps, [:workflow_id, :step_name, :attempt],
+      unique: true, name: "idx_pgbus_workflow_steps_attempt", if_not_exists: true
+    add_index :pgbus_workflow_steps, [:status, :scheduled_at],
+      name: "idx_pgbus_workflow_steps_scheduled", if_not_exists: true
+    add_index :pgbus_workflow_steps, [:status, :stall_at],
+      name: "idx_pgbus_workflow_steps_stall", if_not_exists: true
+    add_index :pgbus_workflow_steps, [:workflow_class, :step_name, :finished_at],
+      name: "idx_pgbus_workflow_steps_insights", if_not_exists: true
+    # A steps table that predates waiting steps lacks these columns (the
+    # queued pgbus:add_workflow_signals adds them and their indexes).
+    if column_exists?(:pgbus_workflow_steps, :batch_id)
+      add_index :pgbus_workflow_steps, :batch_id,
+        name: "idx_pgbus_workflow_steps_batch", if_not_exists: true
+    end
+    if column_exists?(:pgbus_workflow_steps, :timeout_at)
+      add_index :pgbus_workflow_steps, [:status, :timeout_at],
+        name: "idx_pgbus_workflow_steps_timeout", if_not_exists: true
+    end
+    add_foreign_key :pgbus_workflow_steps, :pgbus_workflows,
+      column: :workflow_id, primary_key: :workflow_id, on_delete: :cascade, if_not_exists: true
+
+    # Append-only audit trail: one row per transition or operator action
+    # (Pgbus::WorkflowEvent).
+    create_table :pgbus_workflow_events, if_not_exists: true do |t|
+      t.string :workflow_id, null: false
+      t.bigint :step_id
+      t.string :step_name
+      t.integer :attempt
+      t.string :event, null: false
+      t.string :actor, null: false
+      t.text :reason
+      t.jsonb :details, null: false, default: {}
+      t.datetime :created_at, null: false
+    end
+
+    add_index :pgbus_workflow_events, [:workflow_id, :id],
+      name: "idx_pgbus_workflow_events_workflow", if_not_exists: true
+    add_foreign_key :pgbus_workflow_events, :pgbus_workflows,
+      column: :workflow_id, primary_key: :workflow_id, on_delete: :cascade, if_not_exists: true
+
+    # One row per signal delivered to a run (Pgbus::WorkflowSignal):
+    # pending → claimed → consumed.
+    create_table :pgbus_workflow_signals, if_not_exists: true do |t|
+      t.string :workflow_id, null: false
+      t.string :name, null: false
+      t.jsonb :payload, null: false, default: {}
+      t.string :idempotency_key
+      t.string :status, null: false, default: "pending"
+      t.bigint :claimed_by_step_id
+      t.datetime :created_at, null: false
+      t.datetime :claimed_at
+      t.datetime :consumed_at
+    end
+
+    add_index :pgbus_workflow_signals, [:workflow_id, :idempotency_key],
+      unique: true, where: "idempotency_key IS NOT NULL",
+      name: "idx_pgbus_workflow_signals_idempotency", if_not_exists: true
+    add_index :pgbus_workflow_signals, [:workflow_id, :name, :status],
+      name: "idx_pgbus_workflow_signals_pending", if_not_exists: true
+    add_index :pgbus_workflow_signals, [:workflow_id, :status, :claimed_by_step_id],
+      name: "idx_pgbus_workflow_signals_claimed", if_not_exists: true
+    add_foreign_key :pgbus_workflow_signals, :pgbus_workflows,
+      column: :workflow_id, primary_key: :workflow_id, on_delete: :cascade, if_not_exists: true
+  end
+
+  def down
+    drop_table :pgbus_workflow_signals, if_exists: true
+    drop_table :pgbus_workflow_events, if_exists: true
+    drop_table :pgbus_workflow_steps, if_exists: true
+    drop_table :pgbus_workflows, if_exists: true
+  end
+end

data/lib/generators/pgbus/templates/migration.rb.erb

@@ -211,6 +211,129 @@ class CreatePgbusTables < ActiveRecord::Migration<%= migration_version %>
     add_index :pgbus_deploy_task_runs, :run_at,
       name: "idx_pgbus_deploy_task_runs_cleanup"
 
+    # Workflows (Pgbus::Workflow): one row per run, one per step attempt, and
+    # the append-only audit trail. At most one ongoing run per (class, subject).
+    create_table :pgbus_workflows do |t|
+      t.string :workflow_id, null: false
+      t.string :workflow_class, null: false
+      t.string :subject_type
+      t.string :subject_id
+      t.string :subject_gid
+      t.jsonb :arguments, null: false, default: {}
+      t.string :status, null: false, default: "ready"
+      t.string :current_step
+      t.jsonb :step_job_options, null: false, default: {}
+      t.string :error_class
+      t.text :error_message
+      t.text :error_backtrace
+      t.datetime :paused_at
+      t.datetime :finished_at
+      t.datetime :cancelled_at
+      t.string :idempotency_key
+      t.string :definition_digest, null: false
+      t.string :rerun_of_workflow_id
+      t.datetime :created_at, null: false
+      t.datetime :updated_at, null: false
+    end
+
+    add_index :pgbus_workflows, :workflow_id,
+      unique: true, name: "idx_pgbus_workflows_workflow_id"
+    add_index :pgbus_workflows, [:workflow_class, :subject_type, :subject_id],
+      unique: true, where: "status IN ('ready','performing','waiting','paused')",
+      name: "idx_pgbus_workflows_one_ongoing"
+    add_index :pgbus_workflows, [:workflow_class, :idempotency_key],
+      unique: true, where: "idempotency_key IS NOT NULL",
+      name: "idx_pgbus_workflows_idempotency"
+    add_index :pgbus_workflows, [:status, :updated_at],
+      name: "idx_pgbus_workflows_status"
+    add_index :pgbus_workflows, [:workflow_class, :status, :created_at],
+      name: "idx_pgbus_workflows_class_status"
+    add_index :pgbus_workflows, [:subject_type, :subject_id],
+      name: "idx_pgbus_workflows_subject"
+
+    create_table :pgbus_workflow_steps do |t|
+      t.string :workflow_id, null: false
+      t.string :workflow_class, null: false
+      t.string :step_name, null: false
+      t.integer :attempt, null: false, default: 1
+      t.string :status, null: false, default: "scheduled"
+      t.string :job_id
+      t.bigint :msg_id
+      t.datetime :scheduled_at, null: false
+      t.datetime :enqueued_at
+      t.datetime :started_at
+      t.datetime :finished_at
+      t.datetime :stall_at
+      t.string :outcome
+      t.jsonb :output
+      t.string :error_class
+      t.text :error_message
+      t.text :error_backtrace
+      t.jsonb :error_causes
+      t.string :worker
+      t.string :app_revision
+      t.string :batch_id
+      t.string :wait_for
+      t.bigint :signal_id
+      t.datetime :waiting_since
+      t.datetime :timeout_at
+      t.datetime :created_at, null: false
+    end
+
+    add_index :pgbus_workflow_steps, [:workflow_id, :step_name, :attempt],
+      unique: true, name: "idx_pgbus_workflow_steps_attempt"
+    add_index :pgbus_workflow_steps, [:status, :scheduled_at],
+      name: "idx_pgbus_workflow_steps_scheduled"
+    add_index :pgbus_workflow_steps, [:status, :stall_at],
+      name: "idx_pgbus_workflow_steps_stall"
+    add_index :pgbus_workflow_steps, [:workflow_class, :step_name, :finished_at],
+      name: "idx_pgbus_workflow_steps_insights"
+    add_index :pgbus_workflow_steps, :batch_id,
+      name: "idx_pgbus_workflow_steps_batch"
+    add_index :pgbus_workflow_steps, [:status, :timeout_at],
+      name: "idx_pgbus_workflow_steps_timeout"
+    add_foreign_key :pgbus_workflow_steps, :pgbus_workflows,
+      column: :workflow_id, primary_key: :workflow_id, on_delete: :cascade
+
+    create_table :pgbus_workflow_events do |t|
+      t.string :workflow_id, null: false
+      t.bigint :step_id
+      t.string :step_name
+      t.integer :attempt
+      t.string :event, null: false
+      t.string :actor, null: false
+      t.text :reason
+      t.jsonb :details, null: false, default: {}
+      t.datetime :created_at, null: false
+    end
+
+    add_index :pgbus_workflow_events, [:workflow_id, :id],
+      name: "idx_pgbus_workflow_events_workflow"
+    add_foreign_key :pgbus_workflow_events, :pgbus_workflows,
+      column: :workflow_id, primary_key: :workflow_id, on_delete: :cascade
+
+    create_table :pgbus_workflow_signals do |t|
+      t.string :workflow_id, null: false
+      t.string :name, null: false
+      t.jsonb :payload, null: false, default: {}
+      t.string :idempotency_key
+      t.string :status, null: false, default: "pending"
+      t.bigint :claimed_by_step_id
+      t.datetime :created_at, null: false
+      t.datetime :claimed_at
+      t.datetime :consumed_at
+    end
+
+    add_index :pgbus_workflow_signals, [:workflow_id, :idempotency_key],
+      unique: true, where: "idempotency_key IS NOT NULL",
+      name: "idx_pgbus_workflow_signals_idempotency"
+    add_index :pgbus_workflow_signals, [:workflow_id, :name, :status],
+      name: "idx_pgbus_workflow_signals_pending"
+    add_index :pgbus_workflow_signals, [:workflow_id, :status, :claimed_by_step_id],
+      name: "idx_pgbus_workflow_signals_claimed"
+    add_foreign_key :pgbus_workflow_signals, :pgbus_workflows,
+      column: :workflow_id, primary_key: :workflow_id, on_delete: :cascade
+
     # Stream queue registry — records which PGMQ queues back streams (vs jobs).
     # Stream and job queues share the pgbus_ namespace, so maintenance
     # (per-stream retention, orphan sweep) and wildcard workers rely on this
@@ -245,6 +368,10 @@ class CreatePgbusTables < ActiveRecord::Migration<%= migration_version %>
     execute "SELECT pgmq.drop_queue('pgbus_default')"
 
     drop_table :pgbus_stream_queues
+    drop_table :pgbus_workflow_signals
+    drop_table :pgbus_workflow_events
+    drop_table :pgbus_workflow_steps
+    drop_table :pgbus_workflows
     drop_table :pgbus_deploy_task_runs
     drop_table :pgbus_deploy_tasks
     drop_table :pgbus_recurring_executions

data/lib/pgbus/active_job/adapter.rb

@@ -11,14 +11,16 @@ module Pgbus
         payload_hash = Concurrency.inject_metadata(active_job, payload_hash)
         payload_hash = Uniqueness.inject_metadata(active_job, payload_hash)
         payload_hash = FairShare.inject_metadata(active_job, payload_hash)
-        payload_hash = inject_batch_metadata(payload_hash, active_job: active_job)
+        payload_hash = inject_batch_metadata(payload_hash, active_job)
 
         if uniqueness_rejected?(active_job, payload_hash, queue: queue)
-          uncount_batch_job(payload_hash)
+          uncount_batch_job(payload_hash, active_job)
           return active_job
         end
 
         enqueue_with_concurrency(active_job, queue, payload_hash)
+      ensure
+        forget_enqueue_batch_id(active_job)
       end
 
       def enqueue_at(active_job, timestamp)
@@ -27,15 +29,17 @@ module Pgbus
         payload_hash = Concurrency.inject_metadata(active_job, payload_hash)
         payload_hash = Uniqueness.inject_metadata(active_job, payload_hash)
         payload_hash = FairShare.inject_metadata(active_job, payload_hash)
-        payload_hash = inject_batch_metadata(payload_hash, active_job: active_job)
+        payload_hash = inject_batch_metadata(payload_hash, active_job)
         delay = [(timestamp - Time.current.to_f).ceil, 0].max
 
         if uniqueness_rejected?(active_job, payload_hash, queue: queue)
-          uncount_batch_job(payload_hash)
+          uncount_batch_job(payload_hash, active_job)
           return active_job
         end
 
         enqueue_with_concurrency(active_job, queue, payload_hash, delay: delay)
+      ensure
+        forget_enqueue_batch_id(active_job)
       end
 
       def enqueue_all(active_jobs)
@@ -114,7 +118,7 @@ module Pgbus
         Batch.backfill_execution(payload_hash, msg_id, physical_queue(queue, priority)) if msg_id
         # A retry re-enqueue that is now live (sent, or parked as a blocked
         # execution) must stop the original attempt from signalling completion.
-        Batch.note_retry_reenqueued(payload_hash["job_id"]) if (msg_id || blocked) && retry_retagged?(payload_hash)
+        Batch.note_retry_reenqueued(payload_hash["job_id"]) if (msg_id || blocked) && retry_retagged?(payload_hash, active_job)
         uniqueness_key = Thread.current[:pgbus_acquired_uniqueness_key]
         UniquenessKey.clear_bind_stamp!(uniqueness_key) if uniqueness_key
         Thread.current[:pgbus_acquired_uniqueness_key] = nil
@@ -131,7 +135,7 @@ module Pgbus
         # database error from the slot upsert or the park is not ambiguous.
         if msg_id.nil? && !(sending && ambiguous_delivery?(e))
           rollback_acquired_uniqueness_lock
-          uncount_batch_job(payload_hash)
+          uncount_batch_job(payload_hash, active_job)
         else
           # Drop the thread-local so a later discard on this thread cannot
           # release that job's uniqueness lock, but do not DELETE the
@@ -210,7 +214,7 @@ module Pgbus
           # acquired earlier in this enqueue (no executor will release it), and
           # uncount it from its batch so completion is not waited on forever.
           rollback_acquired_uniqueness_lock
-          uncount_batch_job(payload_hash)
+          uncount_batch_job(payload_hash, active_job)
         when :raise
           raise ConcurrencyLimitExceeded, "Concurrency limit reached for key: #{key}"
         end
@@ -283,25 +287,39 @@ module Pgbus
 
       # Reverses inject_batch_metadata for a job discarded at enqueue time
       # (uniqueness duplicate or concurrency :discard conflict): the message is
-      # never sent, so it can never signal completion. Only applies while the
-      # tagging batch's block is still active on this thread.
-      def uncount_batch_job(payload_hash)
+      # never sent, so it can never signal completion. Only applies to a job
+      # counted in as a member (see #member_batch_id).
+      def uncount_batch_job(payload_hash, active_job)
         batch_id = payload_hash[Batch::METADATA_KEY]
         return unless batch_id
 
-        if batch_id == Thread.current[:pgbus_batch_id]
+        if batch_id == member_batch_id(active_job)
           Batch.untrack_enqueue(payload_hash)
-        elsif retry_retagged?(payload_hash)
+        elsif retry_retagged?(payload_hash, active_job)
           # The retry never became live; the original attempt's row and its
           # normal completion signal stand.
           Batch.forget_retry_reenqueued(payload_hash["job_id"])
         end
       end
 
-      # Tagged for a batch while no Batch#enqueue block is active on this
-      # thread — only a retry_on re-enqueue gets there (issue #424).
-      def retry_retagged?(payload_hash)
-        payload_hash[Batch::METADATA_KEY] && Thread.current[:pgbus_batch_id].nil?
+      # Tagged for a batch the job was neither added to nor enqueued inside
+      # the block of — only a retry_on re-enqueue gets there (issue #424).
+      def retry_retagged?(payload_hash, active_job)
+        payload_hash[Batch::METADATA_KEY] && member_batch_id(active_job).nil?
+      end
+
+      # The batch a job joins as a new member: the one it was added to, kept
+      # on the instance so it survives an enqueue deferred to after commit
+      # (issue #553), else the Batch#enqueue block active on this thread. The
+      # instance wins: a deferred enqueue can run while another batch's
+      # block is open on the committing thread.
+      def member_batch_id(active_job)
+        (active_job.enqueue_batch_id if active_job.respond_to?(:enqueue_batch_id)) ||
+          Thread.current[:pgbus_batch_id]
+      end
+
+      def forget_enqueue_batch_id(active_job)
+        active_job.enqueue_batch_id = nil if active_job.respond_to?(:enqueue_batch_id=)
       end
 
       # A retry_on re-enqueue of a job that is already a batch member: same
@@ -316,18 +334,17 @@ module Pgbus
         active_job.batch_id
       end
 
-      # Tag the payload with the active batch and count it in (guarded
-      # increment + execution row, before the send — Batch.track_enqueue). Pass
-      # track: false to only tag, when the caller counts a bulk once.
-      def inject_batch_metadata(payload_hash, active_job: nil, track: true)
-        batch_id = Thread.current[:pgbus_batch_id]
+      # Tag the payload with the job's batch and count it in (guarded
+      # increment + execution row, before the send — Batch.track_enqueue).
+      def inject_batch_metadata(payload_hash, active_job)
+        batch_id = member_batch_id(active_job)
         if batch_id
           tagged = payload_hash.merge(Batch::METADATA_KEY => batch_id)
-          Batch.track_enqueue(tagged) if track
+          Batch.track_enqueue(tagged)
           return tagged
         end
 
-        retry_batch_id = active_job && retry_batch_id_for(active_job)
+        retry_batch_id = retry_batch_id_for(active_job)
         return payload_hash unless retry_batch_id
 
         tagged = payload_hash.merge(Batch::METADATA_KEY => retry_batch_id)
@@ -335,14 +352,20 @@ module Pgbus
         tagged
       end
 
+      # Only tags: the bulk is counted once per batch below. A retry re-enqueue
+      # never takes the bulk path (retry_on enqueues one job at a time).
+      def tag_bulk_member(payload_hash, active_job)
+        batch_id = member_batch_id(active_job)
+        batch_id ? payload_hash.merge(Batch::METADATA_KEY => batch_id) : payload_hash
+      end
+
       def enqueue_immediate(queue, jobs, priority: nil)
         return if jobs.empty?
 
         payloads = jobs.map do |j|
-          inject_batch_metadata(FairShare.inject_metadata(j, Serializer.serialize_job_hash(j)), track: false)
+          tag_bulk_member(FairShare.inject_metadata(j, Serializer.serialize_job_hash(j)), j)
         end
-        # One guarded increment for the whole bulk, not one per job.
-        Batch.track_enqueue(payloads) if Thread.current[:pgbus_batch_id]
+        track_bulk(payloads)
         physical = physical_queue(queue, priority)
         msg_ids = nil
         msg_ids = Pgbus.client.send_batch(queue, payloads, priority: priority)
@@ -361,6 +384,24 @@ module Pgbus
       rescue Pgbus::SchemaNotReady => e
         Pgbus.logger.error { "[Pgbus] #{e.message}" }
         raise
+      ensure
+        jobs.each { |j| forget_enqueue_batch_id(j) }
+      end
+
+      # One guarded increment per batch for the whole bulk, not one per job.
+      # When a later batch refuses (finished), the batches already counted are
+      # uncounted before the error goes up: nothing has been sent yet.
+      def track_bulk(payloads)
+        tracked = []
+        payloads.select { |p| p[Batch::METADATA_KEY] }
+                .group_by { |p| p[Batch::METADATA_KEY] }
+                .each_value do |members|
+                  Batch.track_enqueue(members)
+                  tracked.concat(members)
+                end
+      rescue StandardError
+        tracked.each { |payload| Batch.untrack_enqueue(payload) }
+        raise
       end
 
       def scheduled_in_future?(job)

data/lib/pgbus/active_job/batch_id.rb

@@ -16,11 +16,33 @@ module Pgbus
     # Both round-trip through +serialize+/+deserialize+, and both are omitted
     # from the serialized hash when unset — an unbatched job's payload is
     # byte-for-byte what it was before this mixin existed.
+    #
+    # A third, transient id, +enqueue_batch_id+, is the batch this instance
+    # was *added* to. +b.add+ / +batch.add+ set it, and +enqueue+ copies the
+    # block's tag onto it, before Active Job decides whether to defer the
+    # enqueue to after commit (+enqueue_after_transaction_commit+). The
+    # deferred enqueue keeps the instance, so the adapter still finds the
+    # batch once the block's thread tag is gone (issue #553). It is never
+    # serialized and the adapter clears it once the enqueue is done.
     module BatchId
       extend ActiveSupport::Concern
 
       included do
-        attr_accessor :batch_id, :callback_batch_id
+        attr_accessor :batch_id, :callback_batch_id, :enqueue_batch_id
+      end
+
+      # Mark +jobs+ as added to +batch_id+ (instances without this mixin are
+      # left alone; they join through the block's thread tag only).
+      def self.capture(jobs, batch_id)
+        jobs.each { |job| job.enqueue_batch_id = batch_id if job.respond_to?(:enqueue_batch_id=) }
+      end
+
+      # Plain assignment, not ||=: an id left by an add whose enqueue never
+      # ran (rolled back, or aborted by a before_enqueue) must not follow the
+      # instance into this one. b.add / batch.add never come through here.
+      def enqueue(...)
+        self.enqueue_batch_id = Thread.current[:pgbus_batch_id]
+        super
       end
 
       def serialize

data/lib/pgbus/active_job/delivery.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+module Pgbus
+  module ActiveJob
+    # The PGMQ delivery of the job running in this fiber:
+    #
+    #   Pgbus::ActiveJob::Delivery.current # => { msg_id: 42, read_ct: 2, queue_name: "default" }
+    #
+    # Set by Executor#execute around perform_now and cleared after, also when
+    # the job raises; nil outside the executor (perform_now in a test, a
+    # top-level inline-adapter job). A job run inline from inside an executing
+    # job sees the enclosing job's delivery; StepJob only acts on it when the
+    # step row's msg_id matches, so that is harmless. Fiber-local (Thread.current[]), so concurrent jobs under
+    # execution_mode: :async each see their own. Pgbus::Workflow::StepJob reads
+    # read_ct to tell a redelivery after a worker died from a first delivery.
+    #
+    # The three values sit in three fiber-locals and the Hash is built only
+    # when read: setting them allocates nothing, which keeps the executor's
+    # per-job allocation budget (spec/pgbus/active_job/executor_allocation_budget_spec.rb).
+    module Delivery
+      MSG_ID_KEY = :pgbus_delivery_msg_id
+      READ_CT_KEY = :pgbus_delivery_read_ct
+      QUEUE_NAME_KEY = :pgbus_delivery_queue_name
+
+      module_function
+
+      def current
+        msg_id = Thread.current[MSG_ID_KEY]
+        return nil if msg_id.nil?
+
+        { msg_id: msg_id, read_ct: Thread.current[READ_CT_KEY], queue_name: Thread.current[QUEUE_NAME_KEY] }
+      end
+
+      def with(msg_id:, read_ct:, queue_name:)
+        previous_msg_id = Thread.current[MSG_ID_KEY]
+        previous_read_ct = Thread.current[READ_CT_KEY]
+        previous_queue_name = Thread.current[QUEUE_NAME_KEY]
+        Thread.current[MSG_ID_KEY] = msg_id
+        Thread.current[READ_CT_KEY] = read_ct
+        Thread.current[QUEUE_NAME_KEY] = queue_name
+        yield
+      ensure
+        Thread.current[MSG_ID_KEY] = previous_msg_id
+        Thread.current[READ_CT_KEY] = previous_read_ct
+        Thread.current[QUEUE_NAME_KEY] = previous_queue_name
+      end
+    end
+  end
+end

data/lib/pgbus/active_job/executor.rb

@@ -99,7 +99,9 @@ module Pgbus
           # `batch` (and `batch.enqueue` for open batches) work inside a job.
           assign_batch_id(job, payload)
           Pgbus.logger.debug { "[Pgbus::Executor] running #{log_tag(message, queue_name)} job_class=#{job_class}" }
-          with_visibility_heartbeat(job, queue_name, msg_id, source_queue, payload) { execute_job(job) }
+          with_visibility_heartbeat(job, queue_name, msg_id, source_queue, payload) do
+            Delivery.with(msg_id: msg_id, read_ct: read_count, queue_name: queue_name) { execute_job(job) }
+          end
           # retry_on re-enqueues from inside perform_now and returns normally:
           # this attempt is done (archive it) but the job is not — the retry
           # message carries the batch tag and signals on its own outcome.

data/lib/pgbus/batch/builder.rb

@@ -59,6 +59,7 @@ module Pgbus
         raise ArgumentError, UNTAGGED_MESSAGE unless Thread.current[:pgbus_batch_id] == @batch.batch_id
 
         jobs = self.class.job_instances(jobs)
+        Pgbus::ActiveJob::BatchId.capture(jobs, @batch.batch_id)
         ::ActiveJob.perform_all_later(jobs) if jobs.any?
         yield if block_given?
         jobs

data/lib/pgbus/batch.rb

@@ -45,6 +45,19 @@ module Pgbus
     # Thread.current): b.add raises anywhere else, and a plain perform_later
     # from a thread or fiber the block spawns does not join the batch.
     #
+    # Inside a database transaction the batch stays pending until the
+    # transaction commits: a job with enqueue_after_transaction_commit keeps
+    # the batch on its instance (b.add, batch.add and perform_later put it
+    # there) and counts itself in at commit, before the batch starts. On a
+    # rollback no job is sent and, when pgbus shares the application's
+    # database, the batch row is gone with the rest (a separate pgbus
+    # database commits the row on its own connection). A bare
+    # ActiveJob.perform_all_later in the block does not keep it: use b.add.
+    # Add jobs that defer: one sent at once, on a connection other than the
+    # caller's, can run before the batch commits and is then accounted for
+    # only by the stalled-execution sweep. A job instance is one member (one
+    # job_id): enqueue it once.
+    #
     #   Pgbus::Batch.start(description: "Import") do |b|
     #     b.on_complete ImportDoneJob
     #     b.add(*rows.map { |row| ImportRowJob.new(row.id) })
@@ -124,7 +137,7 @@ module Pgbus
       end
       @started = true
       count_jobs(&)
-      start_processing
+      self.class.send(:after_transactions_commit) { start_processing }
       self
     end
 
@@ -139,6 +152,7 @@ module Pgbus
       return jobs if jobs.empty? && block.nil?
 
       enqueue do
+        Pgbus::ActiveJob::BatchId.capture(jobs, batch_id)
         ::ActiveJob.perform_all_later(jobs) if jobs.any?
         yield if block
       end
@@ -473,6 +487,22 @@ module Pgbus
         finish_if_needed(try_finish!(batch_id))
       end
 
+      # Run the end-of-block start (or an open batch's re-check) once every
+      # open transaction has committed, and at once when none is open
+      # (issue #553). Jobs with enqueue_after_transaction_commit register
+      # their enqueue as an after_commit callback during the block; Active
+      # Record runs a transaction's callbacks in registration order and
+      # appends a savepoint's to its parent's on release, so every one of
+      # them has counted itself into the still-pending row before this runs.
+      # On a rollback none of them runs, and the row (created in the same
+      # transaction) is gone. Active Record 7.1 has no
+      # after_all_transactions_commit, and Active Job 7.1 never defers.
+      def after_transactions_commit(&)
+        return yield unless ::ActiveRecord.respond_to?(:after_all_transactions_commit)
+
+        ::ActiveRecord.after_all_transactions_commit(&)
+      end
+
       def finish_if_needed(result)
         return result unless result&.fetch(:just_finished, false) && result[:record]
 
@@ -629,8 +659,10 @@ module Pgbus
       raise AlreadyFinished, "Can't add jobs into an already finished batch" if finished?
 
       count_jobs(&)
-      reload
-      self.class.send(:finish_if_needed, BatchEntry.check_finished!(batch_id))
+      self.class.send(:after_transactions_commit) do
+        reload
+        self.class.send(:finish_if_needed, BatchEntry.check_finished!(batch_id))
+      end
       self
     end