@adonis-agora/durable 0.51.1 → 0.52.1

package/dist/src/engine.js

@@ -1,7 +1,9 @@
 import { AsyncLocalStorage } from 'node:async_hooks';
 import { InMemoryAdmissionBackend } from './admission.js';
 import { backoffDelay, MAX_BACKOFF_MS } from './backoff.js';
+import { BoundedRunner } from './bounded-runner.js';
 import { instantCheckpoint, lostStepEvent, stepCheckpoint } from './checkpoints.js';
+import { CHILD_DELIVERED_TAG, markChildDelivered } from './child-delivery.js';
 import { completionError, failedCompletion, isFailedCompletion, } from './completion.js';
 import { assertConcurrency, concurrencyTag, countRuns, resolveConcurrency, } from './concurrency.js';
 import { controlPlaneDescriptor, planDispatch, WorkflowBlocked, } from './dispatch-routing.js';
@@ -71,6 +73,13 @@ const SETTLE_ELIGIBLE_STATUSES = [
 /** How long a read of the onError routes a fleet announces is reused. */
 const ANNOUNCED_ROUTES_TTL_MS = 5_000;
 const versionKey = (name, version) => `${name}@${version}`;
+/**
+ * The {@link RunDispatch.key} of a run put back to `pending` (a retry, a recovery): a new pending
+ * state, so it must never match the key of the run's earlier start, even within the same millisecond.
+ */
+const pendingKey = (at) => `${at.getTime()}-${globalThis.crypto.randomUUID().slice(0, 8)}`;
+/** Whether `status` is one a run never leaves on its own. */
+const isTerminalRun = (status) => TERMINAL_RUN_STATUSES.includes(status);
 /**
  * First checkpoint seq reserved for saga compensations: compensation `idx` (registration order)
  * persists at `COMPENSATION_SEQ_BASE - idx` (-2, -3, …). Negative so it can never collide with the
@@ -84,6 +93,24 @@ const STEP_FAN_CHUNK = 200;
 const DEFAULT_TIMER_BATCH = 500;
 /** The most results of one run settled by one batched write (see `completeRemoteResult`). */
 const RESULT_BATCH_MAX = 256;
+/**
+ * How many runs this instance executes in-process at once when it runs them itself (the default
+ * dispatcher without a run queue, deferred child starts): past it, run ids wait in memory in arrival
+ * order. The runs are already persisted, so a dropped id is recovered by the pending-run poll or the
+ * parent's next turn.
+ */
+const LOCAL_RUN_CONCURRENCY = 256;
+/** How many dispatched runs this instance takes from the transport's run queue at once. */
+const RUN_CONSUMER_CONCURRENCY = 64;
+/** The most runs handed to {@link Transport.dispatchRuns} in one call. */
+const RUN_DISPATCH_BATCH = 500;
+/** Tries, and the first backoff, for telling a parent that its child finished (see `deliverToParent`). */
+const PARENT_NOTIFY_ATTEMPTS = 4;
+const PARENT_NOTIFY_BACKOFF_MS = 250;
+/** How often, at most, and how many times a wake dropped on a held lease looks again (see `recheckAfterLease`). */
+const LEASE_RECHECK_MAX_MS = 2_000;
+const LEASE_RECHECK_SLACK_MS = 100;
+const LEASE_RECHECK_ATTEMPTS = 30;
 /**
  * How soon (ms) orphan recovery re-drives a REMOTE workflow run found `running` with a call in flight:
  * its turn was lost (an inline advance timed out, or the replica applying a decision died), so it is
@@ -314,6 +341,26 @@ export class WorkflowEngine {
      * escapes it.
      */
     postSettle = new Set();
+    /** Runs this instance dispatched to itself, executed at most {@link LOCAL_RUN_CONCURRENCY} at once. */
+    localRuns = new BoundedRunner(LOCAL_RUN_CONCURRENCY);
+    /** Deferred child starts (see {@link startChildDeferred}), at most {@link LOCAL_RUN_CONCURRENCY} at once. */
+    childStarts = new BoundedRunner(LOCAL_RUN_CONCURRENCY);
+    /**
+     * The transport whose run queue this engine dispatches runs through and consumes (see
+     * {@link Transport.dispatchRuns}), when the primary transport has one and no custom
+     * `runDispatcher` was supplied. Unset: runs execute on the instance that dispatches them.
+     */
+    runQueue;
+    /** Stops this instance's run-queue consumer (see {@link Transport.onRun}); unset once stopped. */
+    stopRunConsumer;
+    /** Runs waiting to go out in the next {@link Transport.dispatchRuns} batch, by run id. */
+    runBuffer = new Map();
+    runFlushScheduled = false;
+    /** When a failed {@link Transport.dispatchRuns} was last reported, and how many failed since. */
+    runEnqueueWarnedAt = Number.NEGATIVE_INFINITY;
+    runEnqueueFailures = 0;
+    /** Re-checks scheduled for runs whose wake found another instance's lease (see {@link recheckAfterLease}). */
+    leaseRetries = new Map();
     draining = false;
     /** Shutdown wait already spent before/alongside {@link drain} (see {@link beginDrain}). */
     drainSpentMs = 0;
@@ -419,18 +466,29 @@ export class WorkflowEngine {
         // run before its execution enters `inflight`, so a bare `queueMicrotask` let `drain()` find both
         // registries empty in that gap and return while the pickup was still about to write — onto a
         // closed connection, or a dropped schema. The one-microtask defer is kept.
+        //
+        // Bounded: a wide fan's children wait here as run ids instead of all executing at once.
         this.runDispatcher = deps.runDispatcher ?? {
-            dispatch: (runId) => this.trackEffect(Promise.resolve()
-                .then(() => this.runOne(runId))
-                .catch((err) => {
+            dispatch: (runId) => this.trackEffect(this.localRuns.submit(runId, () => this.runOne(runId).catch((err) => {
                 console.warn(`[adonis-durable] in-process pickup of run ${runId} failed (a pending-run poll or recovery retries it): ${err instanceof Error ? err.message : String(err)}`);
-            })),
+            }))),
         };
+        // A primary transport with a run queue shared by every instance distributes the runs instead:
+        // every run this engine would execute on itself goes through the queue, and every instance
+        // consuming it (each driving instance) takes its share. A custom dispatcher owns execution
+        // itself, so it never uses the queue.
+        const runQueue = this.primaryTransport;
+        this.runQueue =
+            this.usesInProcessDispatch &&
+                typeof runQueue?.dispatchRuns === 'function' &&
+                typeof runQueue.onDispatchedRun === 'function'
+                ? runQueue
+                : undefined;
         this.singletons = new SingletonGate({
             store: this.store,
             clock: this.clock,
             dispatch: (runId) => {
-                this.runDispatcher.dispatch(runId);
+                void this.dispatchRun(runId, 'wake', { lookup: true });
             },
             configFor: (run) => (this.workflows.get(versionKey(run.workflow, run.workflowVersion)) ??
                 this.latest.get(run.workflow))?.singleton,
@@ -525,6 +583,184 @@ export class WorkflowEngine {
             wakeWaiter: (token, payload) => this.wakeSignalWaiter(token, payload),
             ensureStarted: (workflow, input, runId) => this.start(workflow, input, runId),
         });
+        // Consume the run queue last, once the transport has this engine's namespace. A process whose
+        // transport defers its consumers (an ace command) still dispatches through the queue and leaves
+        // the runs to the instances that consume it.
+        this.consumeRuns();
+    }
+    /**
+     * Consume this engine's namespace's run queue (see {@link Transport.onDispatchedRun}), and return
+     * the function that stops it; `undefined` when the engine dispatches runs without one (no run queue
+     * on the transport, or a custom `runDispatcher`). The engine calls it itself when it is built, so a
+     * driving instance consumes with no setup; a second call returns the same stop. {@link drain} stops it.
+     */
+    consumeRuns() {
+        if (!this.runQueue?.onDispatchedRun || this.draining)
+            return this.stopRunConsumer;
+        this.stopRunConsumer ??= this.runQueue.onDispatchedRun((run) => this.runEnqueued(run), {
+            concurrency: RUN_CONSUMER_CONCURRENCY,
+            namespace: this.namespace,
+        });
+        return this.stopRunConsumer;
+    }
+    /**
+     * Execute `runId` wherever this engine executes runs: through the transport's run queue when it has
+     * one (see {@link runQueue}), else through the {@link RunDispatcher}. The queue is this engine's
+     * namespace's, the one its instances consume (an operator, with no namespace, drives every pool's
+     * runs from the transport's own queue). The caller passes the run when it has it at hand, for its
+     * `priority`, or `lookup` to read it. A custom dispatcher receives the run id alone.
+     */
+    dispatchRun(runId, reason, opts = {}) {
+        if (!this.runQueue)
+            return this.runDispatcher.dispatch(runId);
+        const enqueue = (run) => this.enqueueRun({
+            runId,
+            reason,
+            ...(opts.key !== undefined ? { key: opts.key } : {}),
+            ...(run?.priority !== undefined ? { priority: run.priority } : {}),
+        }, this.namespace);
+        if (opts.run || !opts.lookup) {
+            enqueue(opts.run);
+            return;
+        }
+        return this.store.getRun(runId).then((run) => enqueue(run ?? undefined), () => enqueue(undefined));
+    }
+    /**
+     * Add `run` to the next {@link Transport.dispatchRuns} batch of `namespace` (this engine's). A batch goes out when
+     * the buffer reaches {@link RUN_DISPATCH_BATCH} runs or on the next macrotask, so the starts of a
+     * fan's children, which land over many store round trips, are sent together. A wake for a run
+     * already in the batch replaces its start: the consumer acts on the run's state either way.
+     */
+    enqueueRun(run, namespace) {
+        const prior = this.runBuffer.get(run.runId);
+        if (!prior || run.reason === 'wake')
+            this.runBuffer.set(run.runId, { run, namespace });
+        if (this.runBuffer.size >= RUN_DISPATCH_BATCH) {
+            this.flushRuns();
+            return;
+        }
+        if (this.runFlushScheduled)
+            return;
+        this.runFlushScheduled = true;
+        this.trackEffect(new Promise((resolve) => {
+            setImmediate(() => {
+                this.runFlushScheduled = false;
+                this.flushRuns();
+                resolve();
+            });
+        }));
+    }
+    /**
+     * Send every buffered run (see {@link enqueueRun}) now: one {@link Transport.dispatchRuns} per
+     * namespace and {@link RUN_DISPATCH_BATCH} runs.
+     */
+    flushRuns() {
+        const queue = this.runQueue;
+        if (!queue?.dispatchRuns || this.runBuffer.size === 0)
+            return;
+        const byNamespace = new Map();
+        for (const { run, namespace } of this.runBuffer.values()) {
+            const list = byNamespace.get(namespace) ?? [];
+            list.push(run);
+            byNamespace.set(namespace, list);
+        }
+        this.runBuffer.clear();
+        this.trackEffect((async () => {
+            for (const [namespace, runs] of byNamespace) {
+                for (let at = 0; at < runs.length; at += RUN_DISPATCH_BATCH) {
+                    const batch = runs.slice(at, at + RUN_DISPATCH_BATCH);
+                    try {
+                        await queue.dispatchRuns?.(batch, namespace !== undefined ? { namespace } : undefined);
+                    }
+                    catch (err) {
+                        // A start stays `pending` for the pending-run poll. A wake is not left to the run's
+                        // timer (minutes away): it resumes the run here, as it would without a run queue.
+                        for (const run of batch) {
+                            if (run.reason === 'wake' && !this.draining) {
+                                this.trackEffect(this.resumeImplicit(run.runId).catch(() => undefined));
+                            }
+                        }
+                        this.reportEnqueueFailure(err, "a pending-run poll or the runs' timers pick them up");
+                    }
+                }
+            }
+        })());
+    }
+    /** Report a failed {@link Transport.dispatchRuns}, at most once a minute: a queue that is down fails every batch. */
+    reportEnqueueFailure(err, fallback) {
+        this.runEnqueueFailures += 1;
+        const now = Date.now();
+        if (now - this.runEnqueueWarnedAt < 60_000)
+            return;
+        this.runEnqueueWarnedAt = now;
+        const failures = this.runEnqueueFailures;
+        this.runEnqueueFailures = 0;
+        console.warn(`[adonis-durable] could not enqueue runs on the run queue (${failures} failure(s) since the last report; ${fallback}): ${err instanceof Error ? err.message : String(err)}`);
+    }
+    /**
+     * Take a run dispatched through the transport's run queue (see {@link Transport.onDispatchedRun})
+     * and act on its state: a `pending` run is run, a suspended (or blocked) one is leased and resumed,
+     * a finished one is skipped. A run another instance already took (through its own consumer or its
+     * pending-run poll) is no longer `pending`, and its lease admits one executor either way; a resume
+     * that finds the lease held looks again once it is free. Resolves once the turn settled here.
+     */
+    async runEnqueued(dispatch) {
+        const runId = typeof dispatch === 'string' ? dispatch : dispatch.runId;
+        // Taken from the queue while this instance shuts down: hand it back for another instance.
+        const handBack = async (run) => {
+            const queue = this.runQueue;
+            if (!queue?.dispatchRuns)
+                return;
+            const namespace = this.namespace;
+            try {
+                await queue.dispatchRuns([
+                    {
+                        runId,
+                        reason: run?.status === 'pending' ? 'start' : 'wake',
+                        ...(run?.priority !== undefined ? { priority: run.priority } : {}),
+                    },
+                ], namespace !== undefined ? { namespace } : undefined);
+            }
+            catch (err) {
+                this.reportEnqueueFailure(err, `run ${runId} handed back while draining waits for a pending-run poll or its timer`);
+            }
+        };
+        if (this.draining) {
+            await handBack(await this.store.getRun(runId).catch(() => undefined));
+            return;
+        }
+        await this.localRuns.submit(runId, async () => {
+            try {
+                const current = await this.store.getRun(runId);
+                if (!current || isTerminalRun(current.status))
+                    return;
+                if (this.draining) {
+                    await handBack(current);
+                    return;
+                }
+                await this.leaseAndResume(runId, current.status !== 'pending');
+            }
+            catch (error) {
+                console.error(`[adonis-durable] could not run dispatched run ${runId}:`, error);
+            }
+        });
+    }
+    /**
+     * Wake a suspended run that has something new to see: resume it here, or, with a run queue, on
+     * whichever instance of this engine's namespace takes it from the queue (the result then only says
+     * it was handed over).
+     */
+    async wake(runId) {
+        if (!this.runQueue)
+            return this.resumeImplicit(runId);
+        const run = await this.store.getRun(runId);
+        if (!run)
+            return { runId, status: 'cancelled' };
+        if (isTerminalRun(run.status)) {
+            return { runId, status: run.status, output: run.output, error: run.error };
+        }
+        this.enqueueRun({ runId, reason: 'wake', ...(run.priority !== undefined ? { priority: run.priority } : {}) }, this.namespace);
+        return { runId, status: run.status };
     }
     /**
      * Register a **durable entity** (a virtual object): a keyed actor whose `handlers` run **serialized
@@ -1079,7 +1315,10 @@ export class WorkflowEngine {
             // dispatcher — doesn't sit `pending` until the next poll. Idempotent: execution is leased, so
             // a duplicate dispatch is a cheap no-op at the worker.
             if (dispatch && prior.status === 'pending') {
-                await this.runDispatcher.dispatch(runId);
+                await this.dispatchRun(runId, 'start', {
+                    key: String(prior.updatedAt.getTime()),
+                    run: prior,
+                });
             }
             return { runId, status: prior.status, output: prior.output, error: prior.error };
         }
@@ -1098,7 +1337,7 @@ export class WorkflowEngine {
         ];
         // The admission marker is owned by the store's atomic claim: a fresh run can neither supply it nor
         // inherit it (`retryWithInput` copies the original run's tags).
-        const tags = mergeTags(registered.tags, minted.length ? [...(opts?.tags ?? []), ...minted] : opts?.tags)?.filter((t) => !SINGLETON_RESERVED_TAGS.includes(t));
+        const tags = mergeTags(registered.tags, minted.length ? [...(opts?.tags ?? []), ...minted] : opts?.tags)?.filter((t) => !SINGLETON_RESERVED_TAGS.includes(t) && t !== CHILD_DELIVERED_TAG);
         // Singleton back-pressure: reject a start that would grow the same-key backlog past
         // `limit + maxQueueDepth` (no-op when no maxQueueDepth is configured).
         if (registered.singleton) {
@@ -1146,10 +1385,11 @@ export class WorkflowEngine {
         // NOT run the body inline. Await the terminal/suspended state with `waitForRun(runId)` if needed.
         // An internal in-process handoff persists with `dispatch: false` and drives the pickup itself.
         if (dispatch && !delayed) {
-            await this.runDispatcher.dispatch(runId);
+            await this.dispatchRun(runId, 'start', { key: String(run.updatedAt.getTime()), run });
             // Nudge worker instances to pick it up now instead of on their next poll (no-op without a control
-            // plane; self-receipt is filtered, so it only helps OTHER pods — e.g. an API pod's enqueue).
-            if (this.controlPlane) {
+            // plane; self-receipt is filtered, so it only helps OTHER pods — e.g. an API pod's enqueue). A run
+            // queue already delivers it to a consuming instance.
+            if (this.controlPlane && !this.runQueue) {
                 void this.controlPlane
                     .publishControl({ kind: 'enqueued', runId, from: this.instanceId })
                     .catch(() => undefined);
@@ -1175,14 +1415,18 @@ export class WorkflowEngine {
      * which is harmless because {@link drain}'s loop re-snapshots both each iteration. With a custom
      * dispatcher the run executes off-instance, so only the durable persist is guaranteed. A start that
      * throws is handed to `onStartError` (a child delivers it to its waiting parent) instead of vanishing.
+     * With a run queue ({@link runQueue}) the run is persisted and enqueued like any start, so it
+     * executes on whichever instance takes it.
+     *
+     * `limiter` bounds how many of these run at once (a fan's children wait in it as ids).
      */
-    handoffRun(workflow, input, runId, opts, onStartError, onAlreadySettled) {
-        this.trackEffect((async () => {
+    handoffRun(workflow, input, runId, opts, onStartError, onAlreadySettled, limiter) {
+        const handoff = async () => {
             // Defer one microtask: never reentrantly resume the parent that is still suspending/settling.
             await Promise.resolve();
             let result;
             try {
-                if (this.usesInProcessDispatch) {
+                if (this.usesInProcessDispatch && !this.runQueue) {
                     // Persist WITHOUT the fire-and-forget dispatch, then drive the pickup ourselves so this
                     // promise is the SOLE executor of the run (no race with a detached dispatcher `runOne`) and
                     // stays tracked until the run has entered `inflight` and settled. `runOne` no-ops (returns
@@ -1212,7 +1456,8 @@ export class WorkflowEngine {
                 TERMINAL_RUN_STATUSES.includes(result.status)) {
                 await onAlreadySettled(result);
             }
-        })());
+        };
+        this.trackEffect(limiter ? limiter.submit(runId, handoff) : handoff());
     }
     /** Read a run's current persisted state (or null if unknown). A thin pass-through to the store. */
     getRun(runId) {
@@ -1838,6 +2083,11 @@ export class WorkflowEngine {
      */
     beginDrain(spentMs = 0) {
         this.draining = true;
+        // Take no more runs from the run queue; the ones in flight here finish (and are waited for).
+        const stopRunConsumer = this.stopRunConsumer;
+        this.stopRunConsumer = undefined;
+        if (stopRunConsumer)
+            this.trackEffect(stopRunConsumer());
         if (Number.isFinite(spentMs) && spentMs > 0) {
             this.drainSpentMs += spentMs;
             for (const rearm of this.drainRearms)
@@ -1873,6 +2123,15 @@ export class WorkflowEngine {
      */
     async drain(timeoutMs = 10_000) {
         this.beginDrain();
+        // Runs waiting for an in-process slot are dropped: they are persisted, so the pending-run poll
+        // (a start) or the parent's next turn (a child start) takes them up on another instance.
+        this.localRuns.clear();
+        this.childStarts.clear();
+        for (const timer of this.leaseRetries.values())
+            clearTimeout(timer);
+        this.leaseRetries.clear();
+        // Runs buffered for the run queue go out now rather than on the next macrotask.
+        this.flushRuns();
         if (this.inflight.size === 0 && this.postSettle.size === 0)
             return;
         // The deadline is `start + timeoutMs - spent`, re-read whenever `beginDrain(spentMs)` reports more
@@ -2111,14 +2370,15 @@ export class WorkflowEngine {
             // the whole run (double `run.completed`, double parent notify). Same reasoning as every settle
             // path's SETTLE_ELIGIBLE_STATUSES guard. The lock is released only AFTER the flip (owner-scoped,
             // so a takeover can't be wiped), closing the release→flip gap the old order left open.
+            const repended = new Date();
             const applied = await this.store.updateRunIf(run.id, ['running'], {
                 status: 'pending',
-                updatedAt: new Date(),
+                updatedAt: repended,
             });
             await this.store.releaseRunLock(run.id, this.instanceId);
             if (!applied)
                 continue;
-            await this.runDispatcher.dispatch(run.id);
+            await this.dispatchRun(run.id, 'start', { key: pendingKey(repended), run });
             results.push({ runId: run.id, status: 'pending' });
         }
         return results;
@@ -2226,8 +2486,56 @@ export class WorkflowEngine {
         // them all contend for the same leases one row at a time.
         const due = shuffle(await asHeartbeat(() => this.store.listDueTimers(nowMs, this.namespace, this.timerBatchSize)));
         const blocked = await this.dueBlockedRuns(nowMs);
+        if (this.runQueue)
+            return this.handOffDue([...due, ...blocked], nowMs);
         return this.resumeLeased([...due, ...blocked], nowMs);
     }
+    /**
+     * {@link resumeDueTimers} with a run queue: each due run is claimed here, then woken through the
+     * queue, so the instance whose poll found it does not execute every due run itself. The claim (the
+     * lease, held while the run's `wakeAt` moves {@link leaseMs} ahead) keeps every other instance's
+     * poll from handing it off again meanwhile; a wake lost on the way comes due again then.
+     */
+    async handOffDue(runs, nowMs) {
+        if (this.draining)
+            return [];
+        const results = [];
+        let next = 0;
+        const worker = async () => {
+            for (;;) {
+                const run = runs[next];
+                next += 1;
+                if (!run || this.draining)
+                    return;
+                try {
+                    if (!(await this.store.tryLockRun(run.id, this.instanceId, nowMs + this.leaseMs, nowMs))) {
+                        continue;
+                    }
+                    try {
+                        const claimed = await this.store.updateRunIf(run.id, [run.status], {
+                            wakeAt: nowMs + this.leaseMs,
+                        });
+                        if (!claimed)
+                            continue;
+                    }
+                    finally {
+                        await this.store.releaseRunLock(run.id, this.instanceId);
+                    }
+                    this.enqueueRun({
+                        runId: run.id,
+                        reason: 'wake',
+                        ...(run.priority !== undefined ? { priority: run.priority } : {}),
+                    }, this.namespace);
+                    results.push({ runId: run.id, status: run.status });
+                }
+                catch (error) {
+                    console.error(`[adonis-durable] could not hand off due run ${run.id}; leaving it for another instance or a later tick:`, error);
+                }
+            }
+        };
+        await Promise.all(Array.from({ length: Math.min(8, runs.length) }, () => worker()));
+        return results;
+    }
     /**
      * Lease and execute one run by id — the worker side of dispatch. Acquires the recovery lease (so
      * exactly one instance runs it), then resumes the body. Returns the result, or null if another
@@ -2246,11 +2554,15 @@ export class WorkflowEngine {
      * work {@link drain} is waiting on — so it must run even while draining, exactly as a signal-driven
      * resume ({@link deliverSignal}) already does. Returns null if another instance holds the lease.
      */
-    async leaseAndResume(runId) {
+    async leaseAndResume(runId, recheck = false) {
         const nowMs = this.clock();
         const acquired = await this.store.tryLockRun(runId, this.instanceId, nowMs + this.leaseMs, nowMs);
-        if (!acquired)
+        if (!acquired) {
+            // A wake (not a start) that lost the lease race may postdate the holder's snapshot.
+            if (recheck)
+                this.recheckAfterLease(runId, undefined);
             return null;
+        }
         // resume() checks the namespace and throws NamespaceMismatch when it doesn't match (releasing the
         // lease) — a foreign run leased here is simply skipped rather than run.
         return this.resumeImplicit(runId).catch((err) => {
@@ -2384,16 +2696,21 @@ export class WorkflowEngine {
         //
         // And it clears a remote workflow turn's decision marker (`awaitingDecisionTaskId`): the requeued
         // run dispatches a FRESH turn, so a late decision of the old one must find no marker to match.
-        const staleAdmission = resurrecting ? withoutAdmission(run.tags) : undefined;
+        // A run executed again owes its parent a new completion: drop the stamp that says it was delivered.
+        const requeuedAt = new Date();
+        const kept = resurrecting ? (withoutAdmission(run.tags) ?? run.tags) : run.tags;
+        const staleAdmission = kept !== run.tags || kept?.includes(CHILD_DELIVERED_TAG)
+            ? (kept ?? []).filter((t) => t !== CHILD_DELIVERED_TAG)
+            : undefined;
         await this.store.updateRun(runId, {
             status: 'pending',
             error: undefined,
             awaitingDecisionTaskId: undefined,
             ...(resurrecting ? { recoveryAttempts: 0 } : {}),
             ...(staleAdmission ? { tags: staleAdmission } : {}),
-            updatedAt: new Date(),
+            updatedAt: requeuedAt,
         });
-        await this.runDispatcher.dispatch(runId);
+        await this.dispatchRun(runId, 'start', { key: pendingKey(requeuedAt), run });
         return { runId, status: 'pending' };
     }
     /**
@@ -2695,7 +3012,7 @@ export class WorkflowEngine {
         if (await this.fanStillWaiting(waiter, payload)) {
             return { runId: waiter.runId, status: 'suspended' };
         }
-        return this.resumeImplicit(waiter.runId);
+        return this.wake(waiter.runId);
     }
     /**
      * The `ctx.all` join barrier: true when `waiter` was one child of a fan (or of one wave of it)
@@ -2826,16 +3143,11 @@ export class WorkflowEngine {
      * knowing about the child feature.
      */
     notifyParent(runId, completion) {
-        // Tracked, not bare fire-and-forget: `signal` can wake a suspended parent and resume it, issuing
-        // store writes that must be awaited by `drain()` even though the settling child returned already.
-        // One retry + a warning before giving up: a transiently-failed delivery here used to vanish into
-        // trackEffect's swallow, stranding the parent until its reconcile wake. The parent's `ctx.child`
-        // re-registration now also RE-DERIVES a terminal child's completion from the run row itself, so
-        // a lost notify self-heals — but that costs a reconcile interval; don't lose it silently.
-        const deliver = (token) => this.signal(token, completion).catch(() => this.signal(token, completion).catch((err) => {
-            console.warn(`[adonis-durable] failed to deliver child completion on ${token} (the parent recovers on its next reconcile wake): ${err instanceof Error ? err.message : String(err)}`);
-        }));
-        this.trackEffect(deliver(`child:${runId}`));
+        // Tracked, not bare fire-and-forget: `signal` can wake a suspended parent, issuing store writes
+        // `drain()` must wait for even though the settling child returned already. The child is already
+        // persisted terminal, so a lost delivery leaves its parent waiting until the parent's next turn
+        // re-derives the completion from the child's run row: hence the retries.
+        this.trackEffect(this.deliverToParent(`child:${runId}`, completion, runId));
         // A fix-and-replay run (`<origin>~retry~<hash>`) is standalone, but its SUCCESS is the
         // origin's outcome for all practical purposes: deliver it on the ORIGIN's token too, so a
         // parent that failed on that child and is retried later consumes this success (buffered or
@@ -2843,7 +3155,51 @@ export class WorkflowEngine {
         // fix attempt must not poison the origin's token.
         const at = runId.lastIndexOf('~retry~');
         if (at !== -1 && completion.ok) {
-            this.trackEffect(deliver(`child:${runId.slice(0, at)}`));
+            this.trackEffect(this.deliverToParent(`child:${runId.slice(0, at)}`, completion));
+        }
+    }
+    /**
+     * {@link signal} a child's completion to its parent, retrying a store error with backoff
+     * ({@link PARENT_NOTIFY_ATTEMPTS} tries). A waiter already taken is delivered to directly on a
+     * retry, so the completion is not buffered for a waiter that no longer exists.
+     */
+    async deliverToParent(token, completion, childId) {
+        let waiter;
+        for (let attempt = 1;; attempt += 1) {
+            try {
+                if (waiter === undefined)
+                    waiter = await this.store.takeSignalWaiter(token);
+                if (waiter) {
+                    await this.deliverSignal(waiter, completion);
+                    return;
+                }
+                // No waiter: the parent may already have settled this child from its run row (see
+                // `markChildDelivered`); a buffered copy would then never be consumed.
+                const delivered = async () => childId !== undefined &&
+                    ((await this.store.getRun(childId))?.tags?.includes(CHILD_DELIVERED_TAG) ?? false);
+                if (await delivered())
+                    return;
+                const resumed = await this.signal(token, completion);
+                // The parent may settle the child from its run row between the look above and the buffer
+                // write (a join that registers its waiter only now): it then stamped the child first, and the
+                // copy just buffered would never be read.
+                if (resumed === null && (await delivered())) {
+                    await this.store.takeBufferedSignal(token);
+                }
+                return;
+            }
+            catch (err) {
+                if (err instanceof NamespaceMismatch)
+                    return;
+                if (attempt >= PARENT_NOTIFY_ATTEMPTS) {
+                    console.warn(`[adonis-durable] failed to deliver child completion on ${token} (the parent recovers on its next turn): ${err instanceof Error ? err.message : String(err)}`);
+                    return;
+                }
+                await new Promise((resolve) => {
+                    const timer = setTimeout(resolve, PARENT_NOTIFY_BACKOFF_MS * 4 ** (attempt - 1));
+                    timer.unref?.();
+                });
+            }
         }
     }
     /**
@@ -2868,7 +3224,11 @@ export class WorkflowEngine {
                 ok: false,
                 error: `child workflow "${workflow}" failed to start: ${message}`,
             });
-        }, (settled) => this.redeliverSettledChild(childId, settled));
+        }, (settled) => this.redeliverSettledChild(childId, settled), 
+        // Bounded: a fan's starts wait here instead of all hitting the store at once. A start still
+        // waiting at shutdown is dropped, not failed: the parent's placeholder remains, and its next
+        // turn starts every child that has no run yet.
+        this.childStarts);
     }
     /**
      * A child start that found the child already terminal: its completion may never reach the waiter
@@ -2952,6 +3312,8 @@ export class WorkflowEngine {
             error,
         });
         await this.cancelChildren(runId, opts);
+        // A parent awaiting this run as a child learns now, as a failed child, not on its next turn.
+        this.notifyParent(runId, failedCompletion(error));
         // Notify local cancel listeners now (a worker on this pod), and broadcast so the instance/worker
         // actually running this run learns of it and can abort cooperatively (the store already records
         // `cancelled`, but a busy worker won't re-read it).
@@ -3252,6 +3614,62 @@ export class WorkflowEngine {
             durationMs: Date.now() - step.startedAt.getTime(),
         });
     }
+    /**
+     * A wake that found the run leased by another instance is dropped, and normally that is fine: the
+     * holder's turn either sees what the wake wrote, or re-checks once it releases the lease
+     * ({@link recheckWaitSeqs}, {@link redriveSettledChildren}). Not if the holder goes down first (a
+     * scale-down mid-turn, an OOM): its lease lapses with the run `suspended` on something that has
+     * finished, and nothing wakes it before its timer.
+     *
+     * So the dropped wake is kept here: once the lease is free (looked at every
+     * {@link LEASE_RECHECK_MAX_MS} at most, {@link LEASE_RECHECK_ATTEMPTS} times at most), a run still
+     * `suspended` is woken. A holder that already moved the run on makes this one spare replay that
+     * re-parks. Best effort; the run's timer remains the fallback if this instance goes down too.
+     */
+    recheckAfterLease(runId, lockedUntil, attempt = 1) {
+        if (this.draining || this.leaseRetries.has(runId))
+            return;
+        const remaining = Math.max(0, (lockedUntil ?? this.clock()) - this.clock());
+        const timer = setTimeout(() => {
+            this.leaseRetries.delete(runId);
+            this.trackEffect(this.retryDroppedWake(runId, attempt));
+        }, Math.min(remaining, LEASE_RECHECK_MAX_MS) + LEASE_RECHECK_SLACK_MS);
+        timer.unref?.();
+        this.leaseRetries.set(runId, timer);
+    }
+    async retryDroppedWake(runId, attempt) {
+        if (this.draining)
+            return;
+        const run = await this.store.getRun(runId);
+        if (run?.status !== 'suspended')
+            return;
+        const nowMs = this.clock();
+        if (run.lockedUntil !== undefined && run.lockedUntil > nowMs) {
+            if (attempt < LEASE_RECHECK_ATTEMPTS) {
+                this.recheckAfterLease(runId, run.lockedUntil, attempt + 1);
+            }
+            return;
+        }
+        if (this.runQueue) {
+            this.enqueueRun({
+                runId,
+                reason: 'wake',
+                ...(run.priority !== undefined ? { priority: run.priority } : {}),
+            }, this.namespace);
+            return;
+        }
+        if (!(await this.store.tryLockRun(runId, this.instanceId, nowMs + this.leaseMs, nowMs)))
+            return;
+        const fresh = await this.store.getRun(runId);
+        if (fresh?.status !== 'suspended') {
+            await this.store.releaseRunLock(runId, this.instanceId);
+            return;
+        }
+        await this.resumeImplicit(runId).catch((err) => {
+            if (!(err instanceof NamespaceMismatch))
+                throw err;
+        });
+    }
     async execute(run, fn) {
         const registered = this.workflows.get(versionKey(run.workflow, run.workflowVersion));
         // Hold the lease for the WHOLE execution — whatever path got us here (leased sweep, a signal, a
@@ -3262,6 +3680,7 @@ export class WorkflowEngine {
         const lockNow = this.clock();
         if (run.lockedBy !== this.instanceId || (run.lockedUntil ?? 0) <= lockNow) {
             if (!(await this.store.tryLockRun(run.id, this.instanceId, lockNow + this.leaseMs, lockNow))) {
+                this.recheckAfterLease(run.id, run.lockedUntil);
                 return { runId: run.id, status: run.status };
             }
         }
@@ -4594,6 +5013,7 @@ export class WorkflowEngine {
                         child.status === 'failed' ||
                         child.status === 'cancelled' ||
                         child.status === 'dead') {
+                        await markChildDelivered(this.store, child);
                         await deliverChild(child.status === 'completed'
                             ? { ok: true, value: child.output }
                             : failedCompletion(child.error ?? { message: child.status }));
@@ -4661,6 +5081,7 @@ export class WorkflowEngine {
             namespace: run.namespace,
             error,
         });
+        this.notifyParent(run.id, failedCompletion(error));
         return { runId: run.id, status: 'cancelled', error };
     }
     /** The run body, lease held + renewed by {@link execute}. */
@@ -5353,10 +5774,14 @@ export class WorkflowEngine {
                 if (!(await this.savePendingCheckpoint({ ...existing, wakeAt }))) {
                     return this.callRemote(runId, seq, step, input, queue, transport, undefined, admission, runNamespace, parallelGroup);
                 }
-                throw new WorkflowSuspended(wakeAt, [seq]);
+                // Parked on the deadline, but no later than `reconcileMs`: the result's wake is the run's only
+                // other way forward, and a wake lost to a crash or a held lease would otherwise leave a
+                // finished step unnoticed until the deadline (an hour, say). An early wake re-parks here.
+                throw new WorkflowSuspended(this.stepLeaseWakeAt(wakeAt), [seq]);
+            }
+            if (this.clock() < existing.wakeAt) {
+                throw new WorkflowSuspended(this.stepLeaseWakeAt(existing.wakeAt), [seq]);
             }
-            if (this.clock() < existing.wakeAt)
-                throw new WorkflowSuspended(existing.wakeAt, [seq]);
             // Past the deadline with no result — the dispatch is presumed lost. Re-dispatch (bounded by
             // `remoteRedispatchMax` so a step that never settles fails the run instead of looping forever).
             if (existing.attempts >= this.remoteRedispatchMax) {
@@ -5404,7 +5829,7 @@ export class WorkflowEngine {
                 kind: 'remote',
                 redispatched: true,
             });
-            throw new WorkflowSuspended(nextDeadline, [seq]);
+            throw new WorkflowSuspended(this.stepLeaseWakeAt(nextDeadline), [seq]);
         }
         // Durable retry: a failed attempt re-dispatches up to `retries`, spacing attempts by `backoff` —
         // unless the worker marked the error non-retryable (a deterministic verdict like a declined card).
@@ -6016,7 +6441,7 @@ export class WorkflowEngine {
         const runIds = [...waiters];
         waiters.clear();
         for (const runId of runIds)
-            void this.resumeImplicit(runId).catch(() => undefined);
+            void this.wake(runId).catch(() => undefined);
     }
     /**
      * Release the flow-control slot a dispatched step held (if any). The queue comes from the