@adonis-agora/durable 0.7.0 → 0.8.0

package/dist/src/engine.js

@@ -11,6 +11,7 @@ import { RemoteWorkflowExecutor } from './remote-workflow-executor.js';
 import { SingletonGate } from './singleton-gate.js';
 import { sanitizeQueueToken, tenantGroup } from './tenant-group.js';
 import { TransportPool } from './transport-pool.js';
+import { workflowAls } from './workflow-als.js';
 import { createWorkflowCtx, } from './workflow-ctx.js';
 import { workflowName, } from './workflow-ref.js';
 /** Union of a workflow's static tags and a run's start-time tags, de-duplicated, or undefined if none. */
@@ -592,7 +593,13 @@ export class WorkflowEngine {
                     continue;
                 const error = { message: 'execution timeout', code: 'execution_timeout' };
                 await this.store.updateRun(run.id, { status: 'cancelled', error, updatedAt: new Date() });
-                this.emit({ type: 'run.failed', runId: run.id, workflow: run.workflow, namespace: run.namespace, error });
+                this.emit({
+                    type: 'run.failed',
+                    runId: run.id,
+                    workflow: run.workflow,
+                    namespace: run.namespace,
+                    error,
+                });
             }
         }
     }
@@ -639,7 +646,13 @@ export class WorkflowEngine {
             };
             await this.store.updateRun(run.id, { status: 'dead', error, updatedAt: new Date() });
             await this.store.releaseRunLock(run.id);
-            this.emit({ type: 'run.failed', runId: run.id, workflow: run.workflow, namespace: run.namespace, error });
+            this.emit({
+                type: 'run.failed',
+                runId: run.id,
+                workflow: run.workflow,
+                namespace: run.namespace,
+                error,
+            });
             this.notifyDead({ ...run, status: 'dead', error, recoveryAttempts: attempts });
             return { runId: run.id, status: 'dead', error };
         }
@@ -721,9 +734,17 @@ export class WorkflowEngine {
      * Resolve once `runId` reaches a settled state — terminal (completed/failed/cancelled/dead) or
      * suspended (handed off to a timer/signal/event). The async counterpart to dispatch: pair it with
      * `start` when a call site needs the outcome — `await start(...); const r = await waitForRun(id)`.
+     *
+     * Pass `{ terminal: true }` to wait for a strictly TERMINAL state only (completed/failed/cancelled/
+     * dead) — a `suspended` run (e.g. one parked on `ctx.sleep`/`ctx.waitForSignal`) does NOT resolve;
+     * the wait continues until the run reaches an end state. This is what `BaseWorkflow.start` needs so
+     * "I want the result" blocks through a suspension instead of returning early with no output. The
+     * default (`terminal` unset/false) resolves on any settled state, suspended included.
      */
     waitForRun(runId, opts) {
+        const isTerminal = (s) => s === 'completed' || s === 'failed' || s === 'cancelled' || s === 'dead';
         const isSettled = (s) => s !== 'pending' && s !== 'running';
+        const shouldResolve = opts?.terminal ? isTerminal : isSettled;
         const toResult = (run) => ({
             runId,
             status: run.status,
@@ -753,7 +774,7 @@ export class WorkflowEngine {
             };
             const check = () => {
                 void this.store.getRun(runId).then((run) => {
-                    if (run && isSettled(run.status))
+                    if (run && shouldResolve(run.status))
                         finish(run);
                 });
             };
@@ -797,6 +818,17 @@ export class WorkflowEngine {
      *  2. **Starts** a fresh run of every workflow registered with `onEvent: [name]`, passing the
      *     payload as input. Idempotent by `evt:<id>:<workflow>` — pass `opts.id` to dedupe redeliveries
      *     of the same logical event (default: a fresh uuid, so each publish triggers once).
+     *
+     * Reliable (buffered) delivery — mirrors `signalWithStart`'s reliability contract, so document it
+     * exactly: a publish that resumes ≥1 live waiter, OR routes into an `eventBatch` accumulator /
+     * starts ≥1 subscriber, is NOT buffered (fan-out stays live-only). A publish that touches NEITHER
+     * buffers ONE copy (via {@link StateStore.bufferEvent}) unless `opts.buffer === false`, so it isn't
+     * silently dropped just because nobody was listening yet. That buffered copy is consumed by the
+     * FIRST future `ctx.waitForEvent(name, { match })` whose match accepts its payload — point-to-point
+     * on redelivery, never by a later-registered onEvent subscriber. Right after buffering, this
+     * re-checks `listSignalWaiters` ONCE (sandwich parity with `signal`'s take → buffer → re-check) so a
+     * waiter that registers in the sliver between the initial miss and the buffer write is still paired
+     * instead of leaving both rows stranded (`waitForEvent`'s post-registration scan is the mirror half).
      */
     async publishEvent(name, payload, opts) {
         let touched = 0;
@@ -830,20 +862,45 @@ export class WorkflowEngine {
                 }
             }
         }
+        // Nobody received it live (no waiter matched, no subscriber exists/started) — buffer ONE copy
+        // unless the caller opted out. `touched > 0` covers BOTH the waiter fan-out and the subscriber loop.
+        if (touched > 0 || opts?.buffer === false)
+            return touched;
+        const bufferedId = globalThis.crypto.randomUUID();
+        await this.store.bufferEvent({ name, payload, id: bufferedId, publishedAt: this.clock() });
+        // Re-check: a waiter may have registered in the window between the miss above and this buffer
+        // write (the events-side mirror of engine.signal's interleaving proof). Only the FIRST
+        // late-registered matching waiter matters — only one buffered copy exists.
+        const lateWaiters = await this.store.listSignalWaiters(eventPrefix(name));
+        const lateWaiter = lateWaiters.find((w) => eventMatches(payload, eventMatchOf(w.token)));
+        if (!lateWaiter)
+            return touched;
+        if (!(await this.store.removeBufferedEvent(bufferedId)))
+            return touched; // claimed elsewhere already
+        // `takeSignalWaiter` is safe here even though it deletes ANY row for the token: an event token
+        // embeds this ONE `waitForEvent` call's own `runId#seq` (see eventToken in events.ts), so no other
+        // registration could ever share it.
+        const waiter = await this.store.takeSignalWaiter(lateWaiter.token);
+        if (waiter) {
+            const settled = await this.deliverSignal(waiter, payload);
+            if (settled)
+                touched += 1;
+        }
+        // If `waiter` is null, that exact registration resolved itself some other way in the interim (most
+        // likely its own timeout deadline) — the buffered copy is already spent, so drop it rather than
+        // resuming a run a second time or re-buffering for a THIRD, unrelated waiter to pick up later.
         return touched;
     }
-    async signal(token, payload) {
-        const waiter = await this.store.takeSignalWaiter(token);
-        if (!waiter) {
-            // No one is waiting yet — buffer it so the next `waitForSignal(token)` consumes it instead of
-            // dropping it (reliable signals; the basis of `signalWithStart`).
-            await this.store.bufferSignal(token, payload);
-            return null;
-        }
+    /**
+     * Deliver a signal to an already-known waiter: write its resolving `signal:<token>` checkpoint and
+     * resume the run. Shared by {@link signal}'s direct hit and its post-buffer re-check reclaim — both
+     * resolve a waiter the exact same way.
+     */
+    async deliverSignal(waiter, payload) {
         await this.store.saveCheckpoint(instantCheckpoint({
             runId: waiter.runId,
             seq: waiter.seq,
-            name: `signal:${token}`,
+            name: `signal:${waiter.token}`,
             kind: 'signal',
             output: payload,
             // Carry the awaiting command's fan group (set when a `ctx.gather_children`/`ctx.all` fan-out
@@ -854,6 +911,40 @@ export class WorkflowEngine {
         }));
         return this.resume(waiter.runId);
     }
+    /**
+     * Deliver an external signal to the run waiting on `token`, resume it with `payload`, and return
+     * the run result — or null if no run is (or ends up) waiting for it.
+     *
+     * Race with the waiter side (`waitForSignal`'s check → `putSignalWaiter` → re-check): a waiter can
+     * register in the sliver between this method's initial `takeSignalWaiter` miss and its
+     * `bufferSignal` write, and the naive old flow (buffer and return) would then leave BOTH a buffered
+     * payload and a registered waiter sitting in the store with nothing left to pair them — a lost wake,
+     * forever suspended. The interleaving proof: the waiter side does check → put → re-check, this side
+     * does take → buffer → re-check; whichever side's SECOND look runs last always observes the other
+     * side's write, and the destructive `take*` ops arbitrate so a signal is never delivered twice.
+     */
+    async signal(token, payload) {
+        const waiter = await this.store.takeSignalWaiter(token);
+        if (waiter)
+            return this.deliverSignal(waiter, payload);
+        // No one was waiting yet — buffer it so the next `waitForSignal(token)` consumes it instead of
+        // dropping it (reliable signals; the basis of `signalWithStart`).
+        await this.store.bufferSignal(token, payload);
+        // Re-check: a waiter may have registered in the window between the miss above and this buffer
+        // write (see the interleaving proof above).
+        const lateWaiter = await this.store.takeSignalWaiter(token);
+        if (!lateWaiter)
+            return null;
+        const reclaimed = await this.store.takeBufferedSignal(token);
+        if (!reclaimed) {
+            // The waiter side won the race to consume the buffer itself (its own checkpoint+resume already
+            // delivered the payload) in the sliver between our takeSignalWaiter and takeBufferedSignal just
+            // above. We've already (destructively) taken `lateWaiter` off the store, but there is nothing
+            // left to deliver — drop it deliberately rather than resuming the run a second time.
+            return null;
+        }
+        return this.deliverSignal(lateWaiter, reclaimed.payload);
+    }
     /**
      * Ensure a run exists for `runId`, then deliver a signal to it — atomically race-free thanks to
      * signal buffering: if the run is new (or busy / not yet waiting), the signal is buffered and
@@ -892,6 +983,27 @@ export class WorkflowEngine {
     notifyParent(runId, completion) {
         void this.signal(`child:${runId}`, completion).catch(() => undefined);
     }
+    /**
+     * Deferred child start shared by the in-process ctx host and the remote `startChild` command.
+     * Deferred (microtask) so a fast child can't reentrantly resume a still-suspending parent. A start
+     * that THROWS (unregistered/unroutable workflow, input validation, singleton back-pressure, store
+     * failure) must NOT be swallowed: the parent is already suspended on `child:<childId>` (the waiter
+     * is put BEFORE start on every path), so a silent drop parks it in suspended-forever, invisibly
+     * re-attempting on every recovery wake — a misconfigured remote child looks exactly like a healthy
+     * long wait. Instead the failure is delivered to that waiter like a failed child (notifyParent):
+     * the parent resumes and fails loudly with the cause. For a fire-and-forget `ctx.startChild`
+     * (spawn — no waiter) the completion is buffered; a later join by the same id consumes it and
+     * correctly observes the failed start.
+     */
+    startChildDeferred(workflow, input, childId, opts) {
+        queueMicrotask(() => void this.start(workflow, input, childId, opts).catch((err) => {
+            const message = err instanceof Error ? err.message : String(err);
+            this.notifyParent(childId, {
+                ok: false,
+                error: `child workflow "${workflow}" failed to start: ${message}`,
+            });
+        }));
+    }
     /**
      * Cancel a run (e.g. from the dashboard). Returns null if the run does not exist. Pass
      * `{ compensate: true }` to undo the saga first: the suspended run is resumed so its completed
@@ -929,7 +1041,13 @@ export class WorkflowEngine {
         }
         const error = { message: 'cancelled' };
         await this.store.updateRun(runId, { status: 'cancelled', error, updatedAt: new Date() });
-        this.emit({ type: 'run.failed', runId, workflow: run.workflow, namespace: run.namespace, error });
+        this.emit({
+            type: 'run.failed',
+            runId,
+            workflow: run.workflow,
+            namespace: run.namespace,
+            error,
+        });
         await this.cancelChildren(runId, opts);
         // Notify local cancel listeners now (a worker on this pod), and broadcast so the instance/worker
         // actually running this run learns of it and can abort cooperatively (the store already records
@@ -1266,8 +1384,22 @@ export class WorkflowEngine {
             void this.notifyParent(run.id, { ok: false, error: outcome.error.message });
             return { runId: run.id, status: 'failed', error: outcome.error };
         }
+        // This outcome was computed by a turn that started from a possibly-stale run snapshot. If the run
+        // was cancelled WHILE that turn was still executing (e.g. `ctx.all`'s failFast cancelling a
+        // sibling mid-turn — plain `cancel()` writes `cancelled` directly, without waiting for the target's
+        // in-flight turn to notice), this now-stale "suspended" outcome must not resurrect it: re-check the
+        // CURRENT persisted status right before writing and echo it instead of clobbering a real cancel.
+        const latest = await this.store.getRun(run.id);
+        if (latest?.status === 'cancelled') {
+            return { runId: run.id, status: 'cancelled', error: latest.error };
+        }
         await this.store.updateRun(run.id, { status: 'suspended', wakeAt: outcome.wakeAt, updatedAt });
-        this.emit({ type: 'run.suspended', runId: run.id, workflow: run.workflow, namespace: run.namespace });
+        this.emit({
+            type: 'run.suspended',
+            runId: run.id,
+            workflow: run.workflow,
+            namespace: run.namespace,
+        });
         return { runId: run.id, status: 'suspended' };
     }
     async runRemoteExecution(run, registered) {
@@ -1275,12 +1407,22 @@ export class WorkflowEngine {
         if (run.status === 'pending') {
             await this.store.updateRun(run.id, { status: 'running', updatedAt: new Date() });
             run.status = 'running';
-            this.emit({ type: 'run.started', runId: run.id, workflow: run.workflow, namespace: run.namespace });
+            this.emit({
+                type: 'run.started',
+                runId: run.id,
+                workflow: run.workflow,
+                namespace: run.namespace,
+            });
         }
         if (registered.singleton && !(await this.singletons.admit(run, registered.singleton))) {
             const wakeAt = this.singletons.retryWakeAt();
             await this.store.updateRun(run.id, { status: 'suspended', wakeAt, updatedAt: new Date() });
-            this.emit({ type: 'run.suspended', runId: run.id, workflow: run.workflow, namespace: run.namespace });
+            this.emit({
+                type: 'run.suspended',
+                runId: run.id,
+                workflow: run.workflow,
+                namespace: run.namespace,
+            });
             await this.store.releaseRunLock(run.id);
             return { runId: run.id, status: 'suspended' };
         }
@@ -1295,7 +1437,13 @@ export class WorkflowEngine {
                 stack: err instanceof Error ? err.stack : undefined,
             };
             await this.store.updateRun(run.id, { status: 'failed', error, updatedAt: new Date() });
-            this.emit({ type: 'run.failed', runId: run.id, workflow: run.workflow, namespace: run.namespace, error });
+            this.emit({
+                type: 'run.failed',
+                runId: run.id,
+                workflow: run.workflow,
+                namespace: run.namespace,
+                error,
+            });
             return { runId: run.id, status: 'failed', error };
         }
         if (decision.status === 'completed') {
@@ -1483,19 +1631,33 @@ export class WorkflowEngine {
                 // engine.signal(name, payload) delivers it. If the signal was already delivered (buffered
                 // before the workflow reached this point — e.g. signalWithStart), resolve it now and re-drive
                 // on a macrotask, AFTER this turn suspends and frees the run lock (a re-entrant resume bails).
-                const buffered = await this.store.takeBufferedSignal(cmd.signal);
-                if (buffered) {
+                const deliverBuffered = async (payload) => {
                     await this.store.saveCheckpoint(instantCheckpoint({
                         runId: run.id,
                         seq: cmd.seq,
                         name: `signal:${cmd.signal}`,
                         kind: 'signal',
-                        output: buffered.payload,
+                        output: payload,
                     }));
                     setTimeout(() => void this.resume(run.id).catch(() => undefined), 0);
+                };
+                const buffered = await this.store.takeBufferedSignal(cmd.signal);
+                if (buffered) {
+                    await deliverBuffered(buffered.payload);
                 }
                 else {
+                    // Same reorder as the in-process ctx.waitForSignal: register on a miss, then re-check once
+                    // more before suspending — closing the lost-wake window where a signal races in between the
+                    // miss above and the registration below (see the interleaving proof at engine.signal).
                     await this.store.putSignalWaiter({ token: cmd.signal, runId: run.id, seq: cmd.seq });
+                    const lateBuffered = await this.store.takeBufferedSignal(cmd.signal);
+                    if (lateBuffered) {
+                        // Resolved it ourselves — remove OUR OWN row via the exact match (not
+                        // takeSignalWaiter(cmd.signal), which deletes ANY row for this token and could steal a
+                        // different run's waiter that has since claimed the same token).
+                        await this.store.removeSignalWaiter({ token: cmd.signal, runId: run.id, seq: cmd.seq });
+                        await deliverBuffered(lateBuffered.payload);
+                    }
                 }
             }
             else if (cmd.kind === 'startChild') {
@@ -1516,7 +1678,7 @@ export class WorkflowEngine {
                     parallelGroup: cmd.parallelGroup,
                 });
                 if (!(await this.store.getRun(childId))) {
-                    queueMicrotask(() => void this.start(cmd.workflow, cmd.input, childId).catch(() => undefined));
+                    this.startChildDeferred(cmd.workflow, cmd.input, childId);
                 }
             }
             else {
@@ -1533,14 +1695,24 @@ export class WorkflowEngine {
         if (run.status === 'pending') {
             await this.store.updateRun(run.id, { status: 'running', updatedAt: new Date() });
             run.status = 'running';
-            this.emit({ type: 'run.started', runId: run.id, workflow: run.workflow, namespace: run.namespace });
+            this.emit({
+                type: 'run.started',
+                runId: run.id,
+                workflow: run.workflow,
+                namespace: run.namespace,
+            });
         }
         // Singleton admission gate: if this run shares its key with `limit` older in-flight runs, wait
         // (suspend on a short timer) until a slot frees instead of running now. Re-checked on each resume.
         if (registered?.singleton && !(await this.singletons.admit(run, registered.singleton))) {
             const wakeAt = this.singletons.retryWakeAt();
             await this.store.updateRun(run.id, { status: 'suspended', wakeAt, updatedAt: new Date() });
-            this.emit({ type: 'run.suspended', runId: run.id, workflow: run.workflow, namespace: run.namespace });
+            this.emit({
+                type: 'run.suspended',
+                runId: run.id,
+                workflow: run.workflow,
+                namespace: run.namespace,
+            });
             await this.store.releaseRunLock(run.id);
             return { runId: run.id, status: 'suspended' };
         }
@@ -1557,7 +1729,10 @@ export class WorkflowEngine {
             replay.set(cp.seq, cp);
         const ctx = createWorkflowCtx(this.ctxHostFor(replay), run.id, compensations, run.workflow);
         try {
-            const output = await fn(ctx, run.input);
+            // Establish the ambient ctx for the duration of this body turn so context-aware statics
+            // (`BaseWorkflow.start`/`dispatch`) reachable from `fn` route through this run's `ctx.child`/
+            // `ctx.startChild`. Re-set on every replay turn (each is its own async scope) — correct.
+            const output = await workflowAls.run(ctx, () => fn(ctx, run.input));
             return this.settleRun(run, { kind: 'completed', output });
         }
         catch (err) {
@@ -1571,7 +1746,12 @@ export class WorkflowEngine {
                     error: undefined,
                     updatedAt: new Date(),
                 });
-                this.emit({ type: 'run.completed', runId: run.id, workflow: run.workflow, namespace: run.namespace });
+                this.emit({
+                    type: 'run.completed',
+                    runId: run.id,
+                    workflow: run.workflow,
+                    namespace: run.namespace,
+                });
                 void this.notifyParent(run.id, { ok: true, value: undefined });
                 const nextId = nextContinuationId(run.id);
                 queueMicrotask(() => void this.start(run.workflow, err.input, nextId).catch(() => undefined));
@@ -1589,7 +1769,13 @@ export class WorkflowEngine {
                     }
                     const error = { message: 'cancelled' };
                     await this.store.updateRun(run.id, { status: 'cancelled', error, updatedAt: new Date() });
-                    this.emit({ type: 'run.failed', runId: run.id, workflow: run.workflow, namespace: run.namespace, error });
+                    this.emit({
+                        type: 'run.failed',
+                        runId: run.id,
+                        workflow: run.workflow,
+                        namespace: run.namespace,
+                        error,
+                    });
                     return { runId: run.id, status: 'cancelled', error };
                 }
                 return this.settleRun(run, { kind: 'suspended', wakeAt: err.wakeAt });
@@ -1711,7 +1897,7 @@ export class WorkflowEngine {
             callRemote: (runId, seq, step, input, queue, transport, admission) => this.callRemote(runId, seq, step, input, queue, transport, replay, admission),
             // Defer so a fast child can't reentrantly resume a still-running parent.
             startChild: (workflow, input, id, priority) => {
-                queueMicrotask(() => void this.start(workflow, input, id, { priority }).catch(() => undefined));
+                this.startChildDeferred(workflow, input, id, { priority });
             },
             // Shallow-merge into the run's searchAttributes (the ctx primitive makes this exactly-once).
             upsertSearchAttributes: async (runId, attrs) => {