@adonis-agora/durable 0.56.4 → 0.56.6

package/dist/src/engine.js

@@ -7,6 +7,7 @@ import { instantCheckpoint, lostStepEvent, stepCheckpoint } from './checkpoints.
 import { CHILD_DELIVERED_TAG, markChildDelivered } from './child-delivery.js';
 import { completionError, failedCompletion, isFailedCompletion, } from './completion.js';
 import { assertConcurrency, concurrencyTag, countRuns, resolveConcurrency, } from './concurrency.js';
+import { DISPATCH_KEY, dispatchKey } from './dispatch-key.js';
 import { controlPlaneDescriptor, planDispatch, WorkflowBlocked, } from './dispatch-routing.js';
 import { parseDuration } from './duration.js';
 import { Entities } from './entities.js';
@@ -140,6 +141,19 @@ const BUFFERED_SWEEP_BUDGET_MS = 250;
 const BUFFERED_SWEEP_REST_MS = 10 * 60_000;
 /** The most results of one run settled by one batched write (see `completeRemoteResult`). */
 const RESULT_BATCH_MAX = 256;
+/** The most runs one owed-resume write marks (see `WorkflowEngine.markResumeOwed`). */
+const OWED_RESUME_BATCH_MAX = 500;
+/**
+ * How long a settled step result's acknowledgement waits for the run's resume before it marks the
+ * run as owed one instead (see `WorkflowEngine.resumeOwed`). A turn that ends within it needs no mark.
+ */
+const OWED_RESUME_GRACE_MS = 3_000;
+/**
+ * The results `execute` returns when another instance holds the run's lease — no turn ran here, so the
+ * wake that asked for it is not served yet — each with the durable request that wake made instead
+ * (see `WorkflowEngine.resumeOwed`). By identity, so the result keeps the shape every caller sees.
+ */
+const leaseRefused = new WeakMap();
 /**
  * How many runs this instance executes in-process at once when it runs them itself (the default
  * dispatcher without a run queue, deferred child starts): past it, run ids wait in memory in arrival
@@ -320,6 +334,10 @@ export class WorkflowEngine {
     pending = new Map();
     /** Each run's step results waiting to be settled on this instance — see {@link completeRemoteResult}. */
     resultQueues = new Map();
+    /** Runs waiting to be marked as owed a resume by the next batched write — see {@link markResumeOwed}. */
+    owedResumes = [];
+    /** Whether an owed-resume write is in flight (the runs marked meanwhile go out in the next). */
+    markingOwedResumes = false;
     /** Per-step "reset the liveness timer" callbacks, called with the beat's worker timestamp when a
      *  heartbeat arrives. */
     heartbeatResets = new Map();
@@ -2196,8 +2214,13 @@ export class WorkflowEngine {
         }
         // A resume issued from INSIDE this run's own execution (its ambient ctx is this run) can only be
         // satisfied once that execution ends — awaiting it there would deadlock. Queue it, answer now.
-        if (workflowAls.getStore()?.runId === runId)
-            return this.echoCurrentStatus(runId);
+        if (workflowAls.getStore()?.runId === runId) {
+            // Queued, not run: the wake is not served yet (see `resumeOwed`).
+            return this.echoCurrentStatus(runId).then((echoed) => {
+                leaseRefused.set(echoed, Promise.resolve('failed'));
+                return echoed;
+            });
+        }
         return queued.result;
     }
     /** Drop `gate` once `tail` settles, unless another execution queued behind it meanwhile. */
@@ -2597,8 +2620,22 @@ export class WorkflowEngine {
             // the contract instead: park it `suspended` on the step's deadline or the reconcile timer. The
             // worker's result resumes it; `resumeDueTimers` is the safety net. A LOST dispatch stays the job
             // of `remoteRedispatchMs` / `redispatchPending`.
-            const dispatched = await this.inFlightRemoteStep(run.id);
+            // The durable marker of work on the transport: a `pending` remote checkpoint is written before
+            // the dispatch and settled by the result.
+            const checkpoints = await this.store.listCheckpoints(run.id);
+            const dispatched = checkpoints.find((cp) => cp.kind === 'remote' && cp.status === 'pending');
             if (dispatched) {
+                // `running` with a lapsed lease: the turn that wrote these checkpoints died before settling
+                // the run, possibly before its dispatch reached the broker. Send what it may have lost.
+                try {
+                    await this.redispatchCrashedTurnSteps(run, checkpoints, true);
+                }
+                catch (error) {
+                    // Not sent: leave the run `running` for the next pass rather than park it on a lost step.
+                    await this.store.releaseRunLock(run.id, this.instanceId).catch(() => undefined);
+                    console.error(`[adonis-durable] could not re-dispatch the steps of crashed run ${run.id}; retrying on a later tick:`, error);
+                    continue;
+                }
                 // A REMOTE workflow reads `running` only while a turn is being computed or applied, so a free
                 // lease here means that turn was lost — the call in flight is not what the run waits on next.
                 // Park it on a near wake so the timer pass re-drives the turn promptly (never later than the
@@ -2693,13 +2730,87 @@ export class WorkflowEngine {
         return released;
     }
     /**
-     * The remote step this run dispatched and still awaits a result for, if any — the durable marker
-     * that tells {@link recoverIncomplete} "this run's work is on the transport, not in a crashed
-     * process". A `pending` remote checkpoint is written before the dispatch and settled by the result.
+     * {@link redispatchCrashedTurnSteps} for a `suspended` run a timer pass just leased, when the run as
+     * listed shows the previous lease holder died holding it: a lease is released (both columns
+     * cleared) by every path that ends normally, so one still set but lapsed is a holder that never
+     * reached its release — a turn SIGKILLed mid-way, most likely. A resumed turn keeps the run
+     * `suspended` while it executes, so this is the only trace such a crash leaves. Called with the
+     * lease held; a failure is reported and does not stop the resume.
      */
-    async inFlightRemoteStep(runId) {
-        const checkpoints = await this.store.listCheckpoints(runId);
-        return checkpoints.find((cp) => cp.kind === 'remote' && cp.status === 'pending');
+    async redispatchIfHolderCrashed(listed, nowMs) {
+        if (listed.status !== 'suspended')
+            return;
+        if (listed.lockedBy === undefined || listed.lockedUntil === undefined)
+            return;
+        if (listed.lockedUntil > nowMs)
+            return;
+        try {
+            await this.redispatchCrashedTurnSteps(listed, await this.store.listCheckpoints(listed.id));
+        }
+        catch (error) {
+            console.error(`[adonis-durable] could not re-dispatch the steps of run ${listed.id} after its lease holder died; its turn proceeds:`, error);
+        }
+    }
+    /**
+     * Re-dispatch the remote steps a CRASHED turn of `run` wrote `pending` and may never have sent.
+     *
+     * A turn persists a step's `pending` checkpoint BEFORE handing the job to the transport (so a fast
+     * result always finds it), then settles the run. A process that dies between the two leaves a
+     * `pending` step no worker ever received — and every later turn replays straight back to "pending,
+     * keep waiting" (re-dispatching on replay is the opt-in `remoteRedispatchMs`, because a step may
+     * legitimately run longer than any window). Observed under a SIGKILLed pod: runs stuck forever.
+     *
+     * The caller holds the run's lease and has evidence the turn died; this narrows it to the steps
+     * THAT turn wrote: the ones enqueued at or after the run's last settle (`updatedAt` — a turn that
+     * ends normally settles after its dispatches, so a step it sent is always older than its settle).
+     * `sinceInclusive` is for a run still `running`: its `updatedAt` is that same turn's start.
+     *
+     * Each step goes out again under its own `stepId` and attempt, keyed like its original dispatch
+     * (see `dispatch-key.ts`): a broker that deduplicates by key (BullMQ) adds nothing while the original
+     * job still waits or runs, or has just finished. Elsewhere a step whose original dispatch did land
+     * runs twice (at-least-once, as documented), and the second result finds the checkpoint settled and
+     * is dropped. Only on a crash — a healthy run never gets here. Remote (polyglot) workflows are left to
+     * their own turn re-drive. Returns how many steps were sent.
+     */
+    async redispatchCrashedTurnSteps(run, checkpoints, sinceInclusive = false) {
+        const registered = this.workflows.get(versionKey(run.workflow, run.workflowVersion)) ??
+            this.latest.get(run.workflow);
+        // No transport: nothing could have been dispatched, nothing can be sent again.
+        if (registered?.remote || this.pool.size === 0)
+            return 0;
+        const since = run.updatedAt.getTime();
+        const lost = checkpoints.filter((cp) => {
+            if (cp.kind !== 'remote' || cp.status !== 'pending' || !cp.stepId)
+                return false;
+            const at = cp.enqueuedAt.getTime();
+            return sinceInclusive ? at >= since : at > since;
+        });
+        for (const cp of lost) {
+            await this.dispatchRemoteTask({
+                runId: run.id,
+                seq: cp.seq,
+                name: cp.name,
+                stepId: cp.stepId,
+                group: cp.workerGroup ?? sanitizeQueueToken(cp.name),
+                input: cp.input,
+                attempt: cp.attempts,
+                priority: run.priority,
+                // The original dispatch's key: a broker that still holds that job adds nothing.
+                enqueuedAt: cp.enqueuedAt,
+            });
+            this.emit({
+                type: 'step.started',
+                runId: run.id,
+                seq: cp.seq,
+                name: cp.name,
+                kind: 'remote',
+                redispatched: true,
+            });
+        }
+        if (lost.length > 0) {
+            console.warn(`[adonis-durable] re-dispatched ${lost.length} step(s) of run ${run.id} written by a turn that died before settling (a step that did reach a worker may run twice)`);
+        }
+        return lost.length;
     }
     /**
      * Per-recovery bookkeeping (called once the lease is held): count the attempt, or — past
@@ -2789,6 +2900,7 @@ export class WorkflowEngine {
                         });
                         if (!claimed)
                             continue;
+                        await this.redispatchIfHolderCrashed(run, nowMs);
                     }
                     finally {
                         await this.store.releaseRunLock(run.id, this.instanceId);
@@ -3162,6 +3274,7 @@ export class WorkflowEngine {
                     return;
                 }
                 try {
+                    await this.redispatchIfHolderCrashed(run, nowMs);
                     // A per-run hook (recovery counting / dead-lettering) may settle the run terminally instead.
                     const settled = onLocked ? await onLocked(run) : undefined;
                     results.push(settled ?? (await this.resumeImplicit(run.id)));
@@ -3896,21 +4009,12 @@ export class WorkflowEngine {
         });
     }
     /**
-     * A wake that found the run leased by another instance (its turn is executing) asks for another
-     * turn, durably: it stamps {@link WorkflowRun.wakeRequestedAt} on the run. The holder does not let
-     * go of the lease while the stamp is there ({@link releaseSuspendedLease}) and runs another turn,
-     * which sees what the wake wrote; and a `suspended` run carrying it is due for the timer poll, so
-     * the request survives both this instance and the holder going down.
-     *
-     * The holder may have let go between the refused lease and the stamp: nobody but the timer poll
-     * would see it then, so the lease is tried once more here. The in-memory re-check
-     * ({@link recheckAfterLease}) runs too, as the fast path when the holder goes down mid-turn; it
-     * stops as soon as a turn served the request.
+     * The durable half of {@link requestWake}: keep the wake's reason, then stamp
+     * {@link WorkflowRun.wakeRequestedAt} on the run. `stamped`; `gone` when the run is no longer live
+     * (nothing to wake); `failed` when the stamp could not be kept (a schema without the column, a store
+     * error). Never rejects.
      */
-    async requestWake(runId, lockedUntil, reason = { generic: true, fans: new Map() }) {
-        if (this.draining)
-            return;
-        let marked;
+    async stampWakeRequest(runId, reason) {
         try {
             // The reason first, the stamp second: a holder that finds the stamp finds the reason too.
             await this.store
@@ -3921,16 +4025,37 @@ export class WorkflowEngine {
                 publishedAt: this.clock(),
             })
                 .catch(() => undefined);
-            marked = await this.store.updateRunIf(runId, ['running', 'suspended', 'blocked'], {
+            const marked = await this.store.updateRunIf(runId, ['running', 'suspended', 'blocked'], {
                 wakeRequestedAt: this.clock(),
             });
+            return marked ? 'stamped' : 'gone';
         }
         catch {
+            return 'failed';
+        }
+    }
+    /**
+     * A wake that found the run leased by another instance (its turn is executing) asks for another
+     * turn, durably: it stamps {@link WorkflowRun.wakeRequestedAt} on the run. The holder does not let
+     * go of the lease while the stamp is there ({@link releaseSuspendedLease}) and runs another turn,
+     * which sees what the wake wrote; and a `suspended` run carrying it is due for the timer poll, so
+     * the request survives both this instance and the holder going down.
+     *
+     * The holder may have let go between the refused lease and the stamp: nobody but the timer poll
+     * would see it then, so the lease is tried once more here. The in-memory re-check
+     * ({@link recheckAfterLease}) runs too, as the fast path when the holder goes down mid-turn; it
+     * stops as soon as a turn served the request.
+     */
+    async requestWake(runId, lockedUntil, reason = { generic: true, fans: new Map() }, stamping) {
+        if (!stamping && this.draining)
+            return;
+        const stamp = await (stamping ?? this.stampWakeRequest(runId, reason));
+        if (stamp === 'failed') {
             // Not kept (a schema without the column, a store error): the in-memory re-check is all there is.
             this.recheckAfterLease(runId, lockedUntil);
             return;
         }
-        if (!marked)
+        if (stamp === 'gone')
             return;
         // The fast path for a holder that went down mid-turn: look again in memory too (it stops once a
         // turn served the request).
@@ -4074,8 +4199,11 @@ export class WorkflowEngine {
         this.wakeReasons.delete(run.id);
         if (run.lockedBy !== this.instanceId || (run.lockedUntil ?? 0) <= lockNow) {
             if (!(await this.store.tryLockRun(run.id, this.instanceId, lockNow + this.leaseMs, lockNow))) {
-                this.trackEffect(this.requestWake(run.id, run.lockedUntil, reason));
-                return { runId: run.id, status: run.status };
+                const stamping = this.draining ? undefined : this.stampWakeRequest(run.id, reason);
+                this.trackEffect(this.requestWake(run.id, run.lockedUntil, reason, stamping));
+                const refused = { runId: run.id, status: run.status };
+                leaseRefused.set(refused, stamping ?? Promise.resolve('failed'));
+                return refused;
             }
         }
         // This turn reads the run's checkpoints after this point, so it sees whatever the wakes that
@@ -6404,6 +6532,7 @@ export class WorkflowEngine {
             priority: admission?.priority,
             attempt,
             transport,
+            enqueuedAt,
         });
         this.emit({ type: 'step.started', runId, seq, name: step.name, kind: 'remote' });
         throw new WorkflowSuspended(undefined, [seq]);
@@ -6607,7 +6736,9 @@ export class WorkflowEngine {
                     traceparent,
                     context,
                     priority: opts.admission?.priority,
+                    lazyPickupBeat: true,
                     attempt: 1,
+                    [DISPATCH_KEY]: dispatchKey(cp.stepId, 1, enqueuedAt),
                 })), opts.transport);
                 // Awaited below; a rejection is not left unhandled while the next round is written.
                 dispatch.catch(() => { });
@@ -6632,7 +6763,7 @@ export class WorkflowEngine {
      * {@link redispatchPending}. Carries the current trace/context so a re-dispatch is traceable.
      */
     dispatchRemoteTask(task) {
-        return this.pool.dispatch({
+        const remote = {
             runId: task.runId,
             seq: task.seq,
             name: task.name,
@@ -6643,7 +6774,10 @@ export class WorkflowEngine {
             context: this.context?.(),
             priority: task.priority,
             attempt: task.attempt,
-        }, task.transport);
+        };
+        if (task.enqueuedAt)
+            remote[DISPATCH_KEY] = dispatchKey(task.stepId, task.attempt, task.enqueuedAt);
+        return this.pool.dispatch(remote, task.transport);
     }
     /** A turn's read of its run's checkpoints: {@link StateStore.listCheckpointsForReplay}, or every
      *  column through {@link StateStore.listCheckpoints} on a store without it. */
@@ -6729,14 +6863,15 @@ export class WorkflowEngine {
     }
     /** The single-result path: settle `result`, then resume its run unless its fan is still running. */
     async completeOneRemoteResult(result, queuedIn) {
-        const cp = await this.settleRemoteCheckpoint(result);
-        if (cp === undefined)
+        const settled = await this.settleRemoteCheckpoint(result);
+        if (settled === undefined)
             return;
+        const { cp, run } = settled;
         if (await this.stepFanStillRunning(cp, result, queuedIn))
             return;
         if (!parseStepFanGroup(cp.parallelGroup))
             this.noteSettledHere(result.runId, [cp.seq]);
-        this.resumeInBackground(result.runId, fanWakeOf(cp, result));
+        await this.resumeOwed(run, result.runId, fanWakeOf(cp, result));
     }
     /**
      * Record `seqs` of `runId` in {@link settledHere}, dropping the entries older than a lease. A key is
@@ -6756,10 +6891,156 @@ export class WorkflowEngine {
             this.settledHere.delete(key);
         }
     }
-    resumeInBackground(runId, fan) {
-        this.trackEffect(this.resumeImplicit(runId, fan).catch((err) => {
+    /**
+     * Resume the run a settled step result owes a turn, and make that turn DURABLE before the result
+     * is acknowledged (the caller's return is the acknowledgement).
+     *
+     * Settling a result does two things — settle the checkpoint, then resume the run — and only the
+     * first is durable on its own: the resume runs in the background, so the transport is never held
+     * by a turn that awaits another result (the self-deadlock the background resume exists for). A pod
+     * SIGKILLed after the acknowledgement and before (or mid-way through) the resumed turn, which keeps
+     * the run `suspended` and so is invisible to the orphan sweep, left the run parked on whatever
+     * `wakeAt` its last turn wrote — for a step that is the reconcile fallback, minutes out — with
+     * nothing else to wake it: the result was already acknowledged, so no redelivery re-drives it.
+     *
+     * So the acknowledgement waits for one of two things, at most {@link OWED_RESUME_GRACE_MS}:
+     *  - the resumed turn ending here: it settled or re-parked the run itself, so nothing is owed any
+     *    more — the common case, and it costs no write;
+     *  - else ({@link OWED_RESUME_GRACE_MS} passed, or the resume failed) a durable mark: for a
+     *    `suspended` run {@link markResumeOwed} pulls its timer in to one lease from now, so every
+     *    instance's due-timer poll resumes it once the lease a dying turn may hold has lapsed; for a run
+     *    whose turn is still in flight, a {@link requestWake} stamp (see {@link owedDurably}). A resume
+     *    refused by another instance's lease already made that stamp, which is then enough.
+     * A pod dying while the acknowledgement waits never acknowledged: the transport redelivers the
+     * result, whose settled checkpoint re-drives the resume on a live instance.
+     *
+     * The wait is bounded, so a turn that awaits another result in memory holds this one for at most
+     * the grace before it is marked and acknowledged — never a deadlock.
+     */
+    async resumeOwed(run, runId, fan) {
+        let served = false;
+        let refusal;
+        const resumed = this.resumeImplicit(runId, fan).then((result) => {
+            refusal = leaseRefused.get(result);
+            served = refusal === undefined;
+        }, (err) => {
             console.warn(`[adonis-durable] background resume of run ${runId} failed (the run recovers on its next reconcile wake): ${err instanceof Error ? err.message : String(err)}`);
-        }));
+        });
+        this.trackEffect(resumed);
+        const owed = this.owedDurably(run);
+        if (owed === undefined)
+            return;
+        let timer;
+        await Promise.race([
+            resumed,
+            new Promise((resolve) => {
+                timer = setTimeout(resolve, OWED_RESUME_GRACE_MS);
+            }),
+        ]);
+        clearTimeout(timer);
+        if (served)
+            return;
+        // Refused: the wake already asked the lease holder for a turn, durably, unless that failed.
+        if (refusal !== undefined && (await refusal) !== 'failed')
+            return;
+        if (owed === 'mark') {
+            await this.markResumeOwed(run, () => served);
+            return;
+        }
+        if ((await this.stampWakeRequest(runId, { generic: true, fans: new Map() })) === 'failed') {
+            console.warn(`[adonis-durable] could not mark run ${runId} as owed a turn (it recovers on its next reconcile wake)`);
+        }
+    }
+    /**
+     * What makes the turn a result settled for `run` (as read when it settled) owes durable, if the
+     * resume does not end in time: `mark` ({@link markResumeOwed}) for a `suspended` run whose timer is
+     * not already due within a lease; `stamp` (a {@link requestWake} stamp) for a run whose turn is in
+     * flight (`running`, or `blocked`) — its turn may park it after this, overwriting any timer, and a
+     * holder that dies after parking leaves a `suspended` run the orphan sweep never sees, so only the
+     * stamp (kept across the park, and due for the timer poll at once) survives both. Else nothing.
+     */
+    owedDurably(run) {
+        if (run?.status === 'suspended')
+            return this.owesMark(run) ? 'mark' : undefined;
+        if (run?.status === 'running' || run?.status === 'blocked')
+            return 'stamp';
+        return undefined;
+    }
+    /**
+     * Whether a result settled for `run` (as read when it settled) would need {@link markResumeOwed}: only
+     * a `suspended` run whose timer is not already due within a lease (a run whose turn is in flight
+     * gets a wake request instead, see {@link owedDurably}).
+     */
+    owesMark(run) {
+        if (run?.status !== 'suspended')
+            return false;
+        return run.wakeAt === undefined || run.wakeAt > this.clock() + this.leaseMs;
+    }
+    /**
+     * Mark `run` as owed a resume: pull its timer in to one lease from now (only IN — a timer already
+     * due sooner is kept, and only while the run is still `suspended`). Leased like any due run, so a
+     * live turn is never doubled — at worst a resume that lands after the mark replays and re-parks.
+     * `served` reports a resume that ended meanwhile, whose mark is then dropped if not yet written.
+     *
+     * Batched across runs when the store has {@link StateStore.pullRunWakesIn}. Best-effort: a store that
+     * refuses the write leaves the run exactly where it was before.
+     */
+    async markResumeOwed(run, served = () => false) {
+        if (!this.owesMark(run))
+            return;
+        if (!this.store.pullRunWakesIn) {
+            await this.markRunResumeOwed(run.id, this.clock() + this.leaseMs);
+            return;
+        }
+        // Batched across runs: the results of a busy instance arrive many runs at a time (a results batch
+        // settles each of its runs concurrently), and each would otherwise cost its own UPDATE. The first
+        // mark goes out at once; the ones arriving while it is in flight go out together in the next.
+        await new Promise((done) => {
+            this.owedResumes.push({ runId: run.id, served, done });
+            if (!this.markingOwedResumes)
+                void this.markOwedResumes();
+        });
+    }
+    /** Write the queued owed-resume marks, a batch at a time, until none is left (see {@link markResumeOwed}). */
+    async markOwedResumes() {
+        this.markingOwedResumes = true;
+        try {
+            while (this.owedResumes.length > 0) {
+                const batch = this.owedResumes.splice(0, OWED_RESUME_BATCH_MAX);
+                // One lease from the write, not from each mark's arrival: never later than a single write.
+                const owedBy = this.clock() + this.leaseMs;
+                // A resume that ended while its mark queued owes nothing any more.
+                const runIds = batch.filter((entry) => !entry.served()).map((entry) => entry.runId);
+                if (runIds.length === 0) {
+                    for (const entry of batch)
+                        entry.done();
+                    continue;
+                }
+                try {
+                    await this.store.pullRunWakesIn?.(runIds, owedBy);
+                }
+                catch {
+                    // A batch the store refused (a deadlock victim, a dropped connection) is retried one run at
+                    // a time, so one bad row costs no other run its mark.
+                    for (const runId of new Set(runIds))
+                        await this.markRunResumeOwed(runId, owedBy);
+                }
+                for (const entry of batch)
+                    entry.done();
+            }
+        }
+        finally {
+            this.markingOwedResumes = false;
+        }
+    }
+    /** {@link markResumeOwed} for one run, through {@link StateStore.updateRunIf}. Best-effort. */
+    async markRunResumeOwed(runId, owedBy) {
+        try {
+            await this.store.updateRunIf(runId, ['suspended'], { wakeAt: owedBy });
+        }
+        catch (err) {
+            console.warn(`[adonis-durable] could not mark run ${runId} as owed a resume (it recovers on its next reconcile wake): ${err instanceof Error ? err.message : String(err)}`);
+        }
     }
     /**
      * Settle a batch of one run's step results; per result, `true` (settled or dropped) or the error
@@ -6877,7 +7158,7 @@ export class WorkflowEngine {
             const fan = parseStepFanGroup(cp.parallelGroup);
             if (!fan || result.status === 'failed') {
                 this.noteSettledHere(runId, singles);
-                this.resumeInBackground(runId);
+                await this.resumeOwed(run, runId);
                 return outcomes;
             }
             let closed = barriers.get(fan.firstSeq);
@@ -6890,7 +7171,7 @@ export class WorkflowEngine {
             }
             if (!(await closed)) {
                 this.noteSettledHere(runId, singles);
-                this.resumeInBackground(runId, {
+                await this.resumeOwed(run, runId, {
                     first: fan.firstSeq,
                     seq: highest.get(fan.firstSeq) ?? cp.seq,
                 });
@@ -7000,7 +7281,7 @@ export class WorkflowEngine {
     }
     /**
      * The durable half of {@link completeRemoteResult}: settle `result` onto its checkpoint (or re-drive
-     * the resume when a redelivered result finds it already settled). Returns the step's checkpoint as
+     * the resume when a redelivered result finds it already settled). Returns the step's checkpoint (and its run) as
      * read when its run should be resumed, or `undefined` when no resume is wanted (no checkpoint, or a
      * late result for a terminal run — terminal is terminal).
      */
@@ -7033,7 +7314,7 @@ export class WorkflowEngine {
         // explicit `requeue`/`durable:retry`, never to a stray result.
         if (cp.status !== 'pending') {
             if (!terminal)
-                return cp;
+                return { cp, run };
             return undefined;
         }
         // A result settling this step frees its flow-control slot (no-op if it wasn't queued). Released
@@ -7070,7 +7351,7 @@ export class WorkflowEngine {
         if (typeof claim === 'object')
             throw claim.error;
         if (!claim)
-            return terminal ? undefined : cp;
+            return terminal ? undefined : { cp, run };
         await this.releaseQueueSlot(cp);
         this.emit({
             type: result.status === 'completed' ? 'step.completed' : 'step.failed',
@@ -7083,7 +7364,7 @@ export class WorkflowEngine {
             queueMs: startedAt.getTime() - cp.enqueuedAt.getTime(),
             durationMs: finishedAt.getTime() - startedAt.getTime(),
         });
-        return cp;
+        return { cp, run };
     }
     /**
      * A slot freed on `queue` (a fleet-wide signal): resume this instance's runs blocked on it so they