@adonis-agora/durable 0.16.0 → 0.17.0

package/dist/src/engine.js

@@ -79,6 +79,14 @@ export class WorkflowEngine {
     trackStepStart;
     /** Where a freshly-started run executes — in-process by default (see {@link RunDispatcher}). */
     runDispatcher;
+    /**
+     * True when NO custom `runDispatcher` was supplied — i.e. runs execute in-process via the default
+     * fire-and-forget dispatcher. Only then does an internal handoff (continue-as-new, a deferred child)
+     * own the run's pickup itself (a tracked {@link runOne}) so {@link drain} spans it end to end; with a
+     * custom (no-op / broker) dispatcher the run executes off this instance, so the handoff guarantees
+     * only the durable persist (the in-process rolled-back-transaction hazard doesn't apply there).
+     */
+    usesInProcessDispatch;
     /** The control-plane's OWN handshake descriptor — negotiated against each worker's descriptor to
      *  detect protocol incompatibility before dispatch (design §7.3/§7.4). Built once at construction. */
     cpDescriptor;
@@ -123,6 +131,17 @@ export class WorkflowEngine {
     stepQueue = new Map();
     /** Executions currently in flight, so a graceful shutdown can wait for them to settle. */
     inflight = new Set();
+    /**
+     * Post-settle side effects (parent notify, singleton wake) fired fire-and-forget AFTER a run's
+     * status was persisted. They run OFF the `execute()` path — the caller of `execute()` must never
+     * block on them — but they still issue store writes (waking a suspended parent, dispatching a
+     * gated singleton run), so `drain()` has to wait for them too. Left untracked, those writes can
+     * land after a test's global Lucid transaction rolled back ("Transaction query already complete"
+     * as an unhandled rejection) or after a graceful shutdown thought it was done. See {@link track}
+     * vs {@link trackEffect}: `inflight` is the execute() path, `postSettle` is everything that
+     * escapes it.
+     */
+    postSettle = new Set();
     draining = false;
     constructor(deps) {
         this.store = deps.store;
@@ -169,6 +188,7 @@ export class WorkflowEngine {
         // Default: execute the run on this instance, asynchronously, so `start` never blocks on the body.
         // A failed pickup is swallowed here (the run stays `pending` for a `runPending` poll to retry);
         // run failures themselves are handled inside `execute` and surfaced as the run's `failed` state.
+        this.usesInProcessDispatch = deps.runDispatcher === undefined;
         this.runDispatcher = deps.runDispatcher ?? {
             dispatch: (runId) => queueMicrotask(() => void this.runOne(runId).catch(() => { })),
         };
@@ -383,6 +403,16 @@ export class WorkflowEngine {
     registerQueue(config) {
         this.admission.register(config);
     }
+    /**
+     * Declare this process a worker: flush consumer subscriptions parked by a transport whose
+     * consumption was deferred (see {@link Transport.deferConsumers}). The `durable:work` loop calls
+     * this before its first tick; until then a deferred-transport process (an ace command, a REPL)
+     * dispatches and reads but never claims broker jobs. Idempotent; a no-op on transports that
+     * never defer (in-process transports, or a broker transport left eager).
+     */
+    startConsumers() {
+        this.pool.startConsumers();
+    }
     /** Subscribe to lifecycle events. Returns an unsubscribe function. */
     subscribe(listener) {
         this.listeners.add(listener);
@@ -474,7 +504,20 @@ export class WorkflowEngine {
             }
         }
     }
-    async start(workflow, input, runId, opts) {
+    start(workflow, input, runId, opts) {
+        // Return startCore's promise directly (no async wrapper): the extra microtask an `async`
+        // delegator inserts would shift `start`'s resolution one tick later, which is observable in the
+        // dispatch timing (a freshly-dispatched run reaching `running` before `start` resolves).
+        return this.startCore(workflow, input, runId, opts, true);
+    }
+    /**
+     * The shared body of {@link start}. `dispatch` gates the fire-and-forget hand-off to the run
+     * dispatcher (+ the control-plane `enqueued` nudge): the public `start` always dispatches, while an
+     * internal in-process handoff ({@link handoffRun}) passes `false` to persist the run and then OWN
+     * its pickup with a tracked {@link runOne} — so no detached dispatcher run races it and {@link drain}
+     * spans the handoff from the parent's settle until the new run has entered `inflight` and settled.
+     */
+    async startCore(workflow, input, runId, opts, dispatch) {
         const name = workflowName(workflow);
         let registered = this.latest.get(name);
         // Convention dispatch: an unregistered workflow whose name matches a LIVE worker group is routed
@@ -520,16 +563,68 @@ export class WorkflowEngine {
         await this.store.createRun(run);
         // The run is durably enqueued; a dispatcher (in-process by default) executes it — `start` does
         // NOT run the body inline. Await the terminal/suspended state with `waitForRun(runId)` if needed.
-        await this.runDispatcher.dispatch(runId);
-        // Nudge worker instances to pick it up now instead of on their next poll (no-op without a control
-        // plane; self-receipt is filtered, so it only helps OTHER pods — e.g. an API pod's enqueue).
-        if (this.controlPlane) {
-            void this.controlPlane
-                .publishControl({ kind: 'enqueued', runId, from: this.instanceId })
-                .catch(() => undefined);
+        // An internal in-process handoff persists with `dispatch: false` and drives the pickup itself.
+        if (dispatch) {
+            await this.runDispatcher.dispatch(runId);
+            // Nudge worker instances to pick it up now instead of on their next poll (no-op without a control
+            // plane; self-receipt is filtered, so it only helps OTHER pods — e.g. an API pod's enqueue).
+            if (this.controlPlane) {
+                void this.controlPlane
+                    .publishControl({ kind: 'enqueued', runId, from: this.instanceId })
+                    .catch(() => undefined);
+            }
         }
         return { runId, status: 'pending' };
     }
+    /**
+     * Bridge an INTERNAL run handoff — the next execution of a `continue-as-new`, or a deferred child
+     * start — into the {@link postSettle} registry so {@link drain} cannot slip through the microtask +
+     * store-I/O gap between the parent settling and the new run entering `inflight`. The bare
+     * `queueMicrotask(() => void this.start(...))` these replaced left exactly that window open: the
+     * parent's `execute()` promise had already left `inflight` and its post-settle effects could have
+     * drained before the deferred `start` even ran — so `drain()` could observe both sets empty and
+     * return early, letting the continuation's/child's store writes escape onto a torn-down connection
+     * (the rolled-back Lucid test transaction → "Transaction query already complete").
+     *
+     * The tracked promise is registered SYNCHRONOUSLY (at the parent's settle), then defers ONE microtask
+     * — preserving the original reentrancy guard so a fast child can't resume a still-suspending parent —
+     * before starting. With the default in-process dispatcher it then OWNS the pickup via {@link runOne}
+     * (persisting with `dispatch: false` so no detached dispatcher run races it), so the promise stays in
+     * `postSettle` until the run has entered `inflight` and settled here; the run is briefly in BOTH sets,
+     * which is harmless because {@link drain}'s loop re-snapshots both each iteration. With a custom
+     * dispatcher the run executes off-instance, so only the durable persist is guaranteed. A start that
+     * throws is handed to `onStartError` (a child delivers it to its waiting parent) instead of vanishing.
+     */
+    handoffRun(workflow, input, runId, opts, onStartError) {
+        this.trackEffect((async () => {
+            // Defer one microtask: never reentrantly resume the parent that is still suspending/settling.
+            await Promise.resolve();
+            try {
+                if (this.usesInProcessDispatch) {
+                    // Persist WITHOUT the fire-and-forget dispatch, then drive the pickup ourselves so this
+                    // promise is the SOLE executor of the run (no race with a detached dispatcher `runOne`) and
+                    // stays tracked until the run has entered `inflight` and settled. `runOne` no-ops (returns
+                    // null) on replay (the run already exists) or while draining (the run is left `pending` for
+                    // recovery) — the durable persist above is what matters on those paths.
+                    const result = await this.startCore(workflow, input, runId, opts, false);
+                    // Guard-free pickup: the continuation/child is in-flight work `drain()` is waiting on, so it
+                    // runs even while draining (a `runOne` here would no-op on `this.draining`, stranding it
+                    // `pending` after its persist — persisted-but-not-processed). Mirrors a signal-driven resume.
+                    if (result.status === 'pending')
+                        await this.leaseAndResume(runId);
+                }
+                else {
+                    // Custom dispatcher owns execution (possibly on another pod); guarantee the persist + its
+                    // normal dispatch/broadcast. The in-process rolled-back-transaction hazard doesn't apply.
+                    await this.startCore(workflow, input, runId, opts, true);
+                }
+            }
+            catch (err) {
+                if (onStartError)
+                    onStartError(err);
+            }
+        })());
+    }
     /** Read a run's current persisted state (or null if unknown). A thin pass-through to the store. */
     getRun(runId) {
         return this.store.getRun(runId);
@@ -603,20 +698,50 @@ export class WorkflowEngine {
         void p.finally(() => this.inflight.delete(p));
         return p;
     }
+    /**
+     * Track a post-settle side effect (parent notify, singleton wake) so {@link drain} waits for it,
+     * WITHOUT the `execute()`/`settleRun` caller ever blocking on it — the run's result is returned
+     * to its caller the moment its status is persisted; this only holds the effect for `drain()`.
+     * Errors are swallowed (these are best-effort wakeups; the durable timer is the real fallback),
+     * exactly as the original `void ....catch(() => undefined)` fire-and-forget did.
+     */
+    trackEffect(p) {
+        const tracked = p.catch(() => undefined);
+        this.postSettle.add(tracked);
+        void tracked.finally(() => this.postSettle.delete(tracked));
+    }
     /**
      * Graceful shutdown: stop picking up new runs (recovery/timer become no-ops) and wait for
      * in-flight executions to settle, up to `timeoutMs`. Call from your app's shutdown hook so a
      * deploy hands off cleanly instead of leaving runs to the lease timeout.
+     *
+     * Because an internal handoff ({@link handoffRun}) is finishing work already in flight, `drain()`
+     * follows a `continue-as-new` chain across its links (and a deferred child through its parent's
+     * resume). A HOT continue-as-new loop — one that hands off to a fresh link faster than it settles —
+     * therefore keeps `postSettle`/`inflight` non-empty and consumes the ENTIRE `timeoutMs` before this
+     * returns. That is by design (tearing down mid-continuation would orphan the continuation's writes);
+     * on timeout the frontier link is left persisted and leased, so the next boot's recovery re-drives it
+     * (no loss) — the cost is spending the full timeout on shutdown. Size `timeoutMs` accordingly.
      */
     async drain(timeoutMs = 10_000) {
         this.draining = true;
-        if (this.inflight.size === 0)
+        if (this.inflight.size === 0 && this.postSettle.size === 0)
             return;
         const timer = new Promise((resolve) => {
-            const t = setTimeout(resolve, timeoutMs);
+            const t = setTimeout(() => resolve('timeout'), timeoutMs);
             t.unref?.();
         });
-        await Promise.race([Promise.allSettled([...this.inflight]), timer]);
+        // Loop, don't snapshot-once: settling an in-flight run fires post-settle effects (parent notify,
+        // singleton wake) that can resume — and thus re-inflight — another run, which settles and fires
+        // more effects. Wait until BOTH sets reach steady-state empty, or the shared timer wins. The
+        // `Promise<'timeout'>` sentinel is distinguishable from `allSettled`'s array so the race is
+        // unambiguous even when everything resolves in the same tick.
+        while (this.inflight.size > 0 || this.postSettle.size > 0) {
+            const pending = Promise.allSettled([...this.inflight, ...this.postSettle]);
+            const outcome = await Promise.race([pending, timer]);
+            if (outcome === 'timeout')
+                return;
+        }
     }
     /**
      * Cancel in-flight runs that have outlived their workflow's `executionTimeout`. Call it from the
@@ -733,6 +858,16 @@ export class WorkflowEngine {
     async runOne(runId) {
         if (this.draining)
             return null;
+        return this.leaseAndResume(runId);
+    }
+    /**
+     * Lease and resume a run WITHOUT the {@link runOne} draining guard. The guard's job is to stop
+     * picking up NEW work once shutdown began; but an internal handoff ({@link handoffRun}) completing a
+     * continue-as-new continuation or a deferred child is finishing work already in flight — the very
+     * work {@link drain} is waiting on — so it must run even while draining, exactly as a signal-driven
+     * resume ({@link deliverSignal}) already does. Returns null if another instance holds the lease.
+     */
+    async leaseAndResume(runId) {
         const nowMs = this.clock();
         const acquired = await this.store.tryLockRun(runId, this.instanceId, nowMs + this.leaseMs, nowMs);
         if (!acquired)
@@ -1147,7 +1282,9 @@ export class WorkflowEngine {
      * knowing about the child feature.
      */
     notifyParent(runId, completion) {
-        void this.signal(`child:${runId}`, completion).catch(() => undefined);
+        // Tracked, not bare fire-and-forget: `signal` can wake a suspended parent and resume it, issuing
+        // store writes that must be awaited by `drain()` even though the settling child returned already.
+        this.trackEffect(this.signal(`child:${runId}`, completion));
         // A fix-and-replay run (`<origin>~retry~<hash>`) is standalone, but its SUCCESS is the
         // origin's outcome for all practical purposes: deliver it on the ORIGIN's token too, so a
         // parent that failed on that child and is retried later consumes this success (buffered or
@@ -1155,7 +1292,7 @@ export class WorkflowEngine {
         // fix attempt must not poison the origin's token.
         const at = runId.lastIndexOf('~retry~');
         if (at !== -1 && completion.ok) {
-            void this.signal(`child:${runId.slice(0, at)}`, completion).catch(() => undefined);
+            this.trackEffect(this.signal(`child:${runId.slice(0, at)}`, completion));
         }
     }
     /**
@@ -1171,13 +1308,16 @@ export class WorkflowEngine {
      * correctly observes the failed start.
      */
     startChildDeferred(workflow, input, childId, opts) {
-        queueMicrotask(() => void this.start(workflow, input, childId, opts).catch((err) => {
+        // Tracked handoff (see {@link handoffRun}): the deferral + reentrancy guard are preserved, but the
+        // start is now held in `postSettle` so `drain()` waits for the child to enter `inflight` and settle
+        // rather than slipping through the gap before the deferred start ran.
+        this.handoffRun(workflow, input, childId, opts, (err) => {
             const message = err instanceof Error ? err.message : String(err);
             this.notifyParent(childId, {
                 ok: false,
                 error: `child workflow "${workflow}" failed to start: ${message}`,
             });
-        }));
+        });
     }
     /**
      * Cancel a run (e.g. from the dashboard). Returns null if the run does not exist. Pass
@@ -1206,11 +1346,17 @@ export class WorkflowEngine {
                     .publishControl({ kind: 'cancel', runId, from: this.instanceId })
                     .catch(() => undefined);
             }
-            queueMicrotask(() => {
-                void this.resume(runId)
-                    .then(() => this.notifyCancelled(runId))
-                    .catch(() => undefined);
-            });
+            // Tracked (see {@link trackEffect}): the background resume replays the run and drives its saga
+            // compensations to the terminal `cancelled` write. As a bare `queueMicrotask` that whole sequence
+            // ran untracked — and because `resume()` only enters `inflight` once it is actually called (a
+            // microtask later), `drain()` could observe both registries empty in the gap and return BEFORE the
+            // compensation writes, letting them land on a torn-down connection ("Transaction query already
+            // complete"). Holding the deferred resume in `postSettle` closes that gap; `resume()` then also
+            // tracks its own execution in `inflight`, so the run is briefly in both sets (harmless — the drain
+            // loop re-snapshots each iteration). NOT a `handoffRun`: that STARTS a new run via `startCore`;
+            // here an EXISTING run is resumed, whose execution already self-tracks — only the pre-call gap
+            // needed covering. The one-microtask defer is preserved; the promise is registered synchronously.
+            this.trackEffect(Promise.resolve().then(() => this.resume(runId).then(() => this.notifyCancelled(runId))));
             await this.cancelChildren(runId, opts);
             return { runId, status: run.status };
         }
@@ -1234,7 +1380,7 @@ export class WorkflowEngine {
                 .catch(() => undefined);
         }
         // A cancelled singleton run frees its slot — wake the next gated waiter now (notify-on-release).
-        void this.singletons.wakeNext(run).catch(() => undefined);
+        this.trackEffect(this.singletons.wakeNext(run));
         return { runId, status: 'cancelled', error };
     }
     /**
@@ -1506,7 +1652,7 @@ export class WorkflowEngine {
                     result.status === 'failed' ||
                     result.status === 'cancelled' ||
                     result.status === 'dead')) {
-                void this.singletons.wakeNext(run).catch(() => undefined);
+                this.trackEffect(this.singletons.wakeNext(run));
             }
             return result;
         }
@@ -1544,7 +1690,7 @@ export class WorkflowEngine {
                 namespace: run.namespace,
                 output: outcome.output,
             });
-            void this.notifyParent(run.id, { ok: true, value: outcome.output });
+            this.notifyParent(run.id, { ok: true, value: outcome.output });
             return { runId: run.id, status: 'completed', output: outcome.output };
         }
         if (outcome.kind === 'failed') {
@@ -1556,7 +1702,7 @@ export class WorkflowEngine {
                 namespace: run.namespace,
                 error: outcome.error,
             });
-            void this.notifyParent(run.id, { ok: false, error: outcome.error.message });
+            this.notifyParent(run.id, { ok: false, error: outcome.error.message });
             return { runId: run.id, status: 'failed', error: outcome.error };
         }
         // This outcome was computed by a turn that started from a possibly-stale run snapshot. If the run
@@ -1948,9 +2094,12 @@ export class WorkflowEngine {
                     workflow: run.workflow,
                     namespace: run.namespace,
                 });
-                void this.notifyParent(run.id, { ok: true, value: undefined });
+                this.notifyParent(run.id, { ok: true, value: undefined });
                 const nextId = nextContinuationId(run.id);
-                queueMicrotask(() => void this.start(run.workflow, err.input, nextId).catch(() => undefined));
+                // Tracked handoff (see {@link handoffRun}): held in `postSettle` from this settle until the
+                // continuation has entered `inflight` and settled, so `drain()` can't return before the next
+                // run's store writes land. Still deferred + idempotent by the continuation id.
+                this.handoffRun(run.workflow, err.input, nextId);
                 return { runId: run.id, status: 'completed' };
             }
             if (err instanceof WorkflowBlocked) {
@@ -2104,8 +2253,13 @@ export class WorkflowEngine {
             // Deferred for the same reentrancy reason as `startChild` above. `cancel()` is already
             // idempotent on a terminal/cancelled run (returns its existing status without side effects), so
             // no extra guard is needed here for the failFast replay case (re-issuing the same cancel calls).
+            // Tracked (see {@link trackEffect}): `cancel()` writes the child — and its whole cancel cascade —
+            // to the store, so this fire-and-forget is held in `postSettle`. Fired from a settling parent
+            // (e.g. failFast cancelling surviving siblings), an untracked cancel's writes could otherwise
+            // escape `drain()` onto a torn-down connection ("Transaction query already complete"). The
+            // one-microtask defer is preserved; the promise is registered synchronously.
             cancelChild: (childId) => {
-                queueMicrotask(() => void this.cancel(childId).catch(() => undefined));
+                this.trackEffect(Promise.resolve().then(() => this.cancel(childId)));
             },
             // Shallow-merge into the run's searchAttributes (the ctx primitive makes this exactly-once).
             upsertSearchAttributes: async (runId, attrs) => {
@@ -2115,8 +2269,13 @@ export class WorkflowEngine {
                     updatedAt: new Date(),
                 });
             },
+            // Tracked (see {@link trackEffect}): `entities.dispatch` persists via `signalWithStart`
+            // (createRun/signal on the entity's run), so this fire-and-forget is held in `postSettle` — else
+            // an entity op signalled from a settling workflow could write after `drain()` returned (torn-down
+            // connection → "Transaction query already complete"). The one-microtask defer is preserved; the
+            // promise is registered synchronously.
             signalEntity: (name, key, op, arg, reply) => {
-                queueMicrotask(() => void this.entities.dispatch(name, key, op, arg, reply).catch(() => undefined));
+                this.trackEffect(Promise.resolve().then(() => this.entities.dispatch(name, key, op, arg, reply)));
             },
             interceptStep: (invocation, body) => this.interceptStep(invocation, body),
         };