@adonis-agora/durable 0.6.1 → 0.8.0

package/dist/src/engine.js

@@ -7,8 +7,11 @@ import { ContinueAsNew, NonDeterminismError, RemoteStepTimeout, WorkflowSuspende
 import { EventAccumulators } from './event-accumulators.js';
 import { eventMatchOf, eventMatches, eventPrefix } from './events.js';
 import { breakpointToken, stepId } from './protocol.js';
+import { RemoteWorkflowExecutor } from './remote-workflow-executor.js';
 import { SingletonGate } from './singleton-gate.js';
+import { sanitizeQueueToken, tenantGroup } from './tenant-group.js';
 import { TransportPool } from './transport-pool.js';
+import { workflowAls } from './workflow-als.js';
 import { createWorkflowCtx, } from './workflow-ctx.js';
 import { workflowName, } from './workflow-ref.js';
 /** Union of a workflow's static tags and a run's start-time tags, de-duplicated, or undefined if none. */
@@ -52,6 +55,11 @@ export class WorkflowEngine {
     store;
     /** Ordered transport pool (dispatch + failover). Empty = no remote steps. */
     pool;
+    /** The primary task transport (first of the pool), used to build an on-the-fly remote-workflow
+     *  executor for {@link WorkflowEngineDeps.remoteByConvention}. Undefined when no transport is wired. */
+    primaryTransport;
+    /** Convention-dispatch opt-in — see {@link WorkflowEngineDeps.remoteByConvention}. */
+    remoteByConvention;
     controlPlane;
     clock;
     instanceId;
@@ -110,6 +118,8 @@ export class WorkflowEngine {
     constructor(deps) {
         this.store = deps.store;
         this.pool = new TransportPool(deps.transports ?? (deps.transport ? [{ id: 'default', transport: deps.transport }] : []));
+        this.primaryTransport = deps.transports?.[0]?.transport ?? deps.transport;
+        this.remoteByConvention = deps.remoteByConvention ?? false;
         this.controlPlane = deps.controlPlane;
         this.clock = deps.clock ?? Date.now;
         this.admission = deps.admission ?? new InMemoryAdmissionBackend(this.clock);
@@ -294,7 +304,28 @@ export class WorkflowEngine {
             this.latest.set(name, registered);
     }
     /**
-     * Register a flow-control queue referenced by `ctx.call(step, input, { queue })`. Caps concurrent
+     * Convention-dispatch resolution (opt-in via {@link WorkflowEngineDeps.remoteByConvention}): when
+     * `name` isn't locally registered but a LIVE worker group of the same name exists, auto-register a
+     * remote workflow routed to that group (bare group, no partition) so its runs advance over the
+     * broker like any `registerRemote`'d one. Returns whether `name` is registered afterwards. A no-op
+     * (returns `false`) when the opt-in is off, no transport is wired, or no matching group is live.
+     */
+    async ensureConventionWorkflow(name) {
+        if (!this.remoteByConvention || !this.primaryTransport)
+            return false;
+        if (this.latest.get(name))
+            return true;
+        const groups = await this.pool.listWorkerGroups();
+        if (!groups.includes(name))
+            return false;
+        this.registerRemote(name, '1', {
+            group: name,
+            executor: new RemoteWorkflowExecutor(this.primaryTransport, name),
+        });
+        return true;
+    }
+    /**
+     * Register a flow-control queue referenced by `ctx.step(name, input, { queue })`. Caps concurrent
      * in-flight steps and/or the admission rate; blocked calls re-suspend and retry, so the limit is
      * durable. Per engine instance (see {@link QueueConfig}). Registering the same name replaces it.
      */
@@ -394,7 +425,12 @@ export class WorkflowEngine {
     }
     async start(workflow, input, runId, opts) {
         const name = workflowName(workflow);
-        const registered = this.latest.get(name);
+        let registered = this.latest.get(name);
+        // remoteByConvention: an unregistered workflow whose name matches a LIVE worker group is routed to
+        // it as a remote workflow (bare group). Off by default → the original "not registered" throw.
+        if (!registered && (await this.ensureConventionWorkflow(name))) {
+            registered = this.latest.get(name);
+        }
         if (!registered)
             throw new Error(`workflow ${name} is not registered`);
         // Validate the input up front — a bad payload is rejected before any run is created.
@@ -421,7 +457,7 @@ export class WorkflowEngine {
             workflow: name,
             workflowVersion: registered.version,
             status: 'pending',
-            namespace: this.namespace,
+            namespace: opts?.namespace ?? this.namespace,
             input,
             tags,
             searchAttributes: opts?.searchAttributes,
@@ -496,7 +532,14 @@ export class WorkflowEngine {
         }
         // Pin to the version the run STARTED on — replay is positional, so running a changed
         // workflow body against old checkpoints would corrupt the run.
-        const registered = this.workflows.get(versionKey(run.workflow, run.workflowVersion));
+        let registered = this.workflows.get(versionKey(run.workflow, run.workflowVersion));
+        // remoteByConvention: a run recovered on an instance that never registered this workflow (e.g. a
+        // crash-recovery pickup) is re-routed to a live worker group of the same name, if one exists.
+        if (!registered && (await this.ensureConventionWorkflow(run.workflow))) {
+            registered =
+                this.workflows.get(versionKey(run.workflow, run.workflowVersion)) ??
+                    this.latest.get(run.workflow);
+        }
         if (!registered) {
             throw new Error(`workflow ${run.workflow}@${run.workflowVersion} is not registered — keep the prior version deployed so in-flight runs can drain (skew protection)`);
         }
@@ -550,7 +593,13 @@ export class WorkflowEngine {
                     continue;
                 const error = { message: 'execution timeout', code: 'execution_timeout' };
                 await this.store.updateRun(run.id, { status: 'cancelled', error, updatedAt: new Date() });
-                this.emit({ type: 'run.failed', runId: run.id, workflow: run.workflow, error });
+                this.emit({
+                    type: 'run.failed',
+                    runId: run.id,
+                    workflow: run.workflow,
+                    namespace: run.namespace,
+                    error,
+                });
             }
         }
     }
@@ -597,7 +646,13 @@ export class WorkflowEngine {
             };
             await this.store.updateRun(run.id, { status: 'dead', error, updatedAt: new Date() });
             await this.store.releaseRunLock(run.id);
-            this.emit({ type: 'run.failed', runId: run.id, workflow: run.workflow, error });
+            this.emit({
+                type: 'run.failed',
+                runId: run.id,
+                workflow: run.workflow,
+                namespace: run.namespace,
+                error,
+            });
             this.notifyDead({ ...run, status: 'dead', error, recoveryAttempts: attempts });
             return { runId: run.id, status: 'dead', error };
         }
@@ -659,7 +714,9 @@ export class WorkflowEngine {
         if (!run)
             return null;
         const id = newRunId ?? `${runId}~retry~${globalThis.crypto.randomUUID().slice(0, 8)}`;
-        await this.start(run.workflow, input, id, { tags: run.tags });
+        // Inherit the original run's namespace so a retry lands in the SAME worker-pool partition (an
+        // operator-routed run keeps its tenant on retry, rather than falling back to the engine's own).
+        await this.start(run.workflow, input, id, { tags: run.tags, namespace: run.namespace });
         return { runId: id };
     }
     /**
@@ -677,9 +734,17 @@ export class WorkflowEngine {
      * Resolve once `runId` reaches a settled state — terminal (completed/failed/cancelled/dead) or
      * suspended (handed off to a timer/signal/event). The async counterpart to dispatch: pair it with
      * `start` when a call site needs the outcome — `await start(...); const r = await waitForRun(id)`.
+     *
+     * Pass `{ terminal: true }` to wait for a strictly TERMINAL state only (completed/failed/cancelled/
+     * dead) — a `suspended` run (e.g. one parked on `ctx.sleep`/`ctx.waitForSignal`) does NOT resolve;
+     * the wait continues until the run reaches an end state. This is what `BaseWorkflow.start` needs so
+     * "I want the result" blocks through a suspension instead of returning early with no output. The
+     * default (`terminal` unset/false) resolves on any settled state, suspended included.
      */
     waitForRun(runId, opts) {
+        const isTerminal = (s) => s === 'completed' || s === 'failed' || s === 'cancelled' || s === 'dead';
         const isSettled = (s) => s !== 'pending' && s !== 'running';
+        const shouldResolve = opts?.terminal ? isTerminal : isSettled;
         const toResult = (run) => ({
             runId,
             status: run.status,
@@ -709,7 +774,7 @@ export class WorkflowEngine {
             };
             const check = () => {
                 void this.store.getRun(runId).then((run) => {
-                    if (run && isSettled(run.status))
+                    if (run && shouldResolve(run.status))
                         finish(run);
                 });
             };
@@ -753,6 +818,17 @@ export class WorkflowEngine {
      *  2. **Starts** a fresh run of every workflow registered with `onEvent: [name]`, passing the
      *     payload as input. Idempotent by `evt:<id>:<workflow>` — pass `opts.id` to dedupe redeliveries
      *     of the same logical event (default: a fresh uuid, so each publish triggers once).
+     *
+     * Reliable (buffered) delivery — mirrors `signalWithStart`'s reliability contract, so document it
+     * exactly: a publish that resumes ≥1 live waiter, OR routes into an `eventBatch` accumulator /
+     * starts ≥1 subscriber, is NOT buffered (fan-out stays live-only). A publish that touches NEITHER
+     * buffers ONE copy (via {@link StateStore.bufferEvent}) unless `opts.buffer === false`, so it isn't
+     * silently dropped just because nobody was listening yet. That buffered copy is consumed by the
+     * FIRST future `ctx.waitForEvent(name, { match })` whose match accepts its payload — point-to-point
+     * on redelivery, never by a later-registered onEvent subscriber. Right after buffering, this
+     * re-checks `listSignalWaiters` ONCE (sandwich parity with `signal`'s take → buffer → re-check) so a
+     * waiter that registers in the sliver between the initial miss and the buffer write is still paired
+     * instead of leaving both rows stranded (`waitForEvent`'s post-registration scan is the mirror half).
      */
     async publishEvent(name, payload, opts) {
         let touched = 0;
@@ -786,20 +862,45 @@ export class WorkflowEngine {
                 }
             }
         }
+        // Nobody received it live (no waiter matched, no subscriber exists/started) — buffer ONE copy
+        // unless the caller opted out. `touched > 0` covers BOTH the waiter fan-out and the subscriber loop.
+        if (touched > 0 || opts?.buffer === false)
+            return touched;
+        const bufferedId = globalThis.crypto.randomUUID();
+        await this.store.bufferEvent({ name, payload, id: bufferedId, publishedAt: this.clock() });
+        // Re-check: a waiter may have registered in the window between the miss above and this buffer
+        // write (the events-side mirror of engine.signal's interleaving proof). Only the FIRST
+        // late-registered matching waiter matters — only one buffered copy exists.
+        const lateWaiters = await this.store.listSignalWaiters(eventPrefix(name));
+        const lateWaiter = lateWaiters.find((w) => eventMatches(payload, eventMatchOf(w.token)));
+        if (!lateWaiter)
+            return touched;
+        if (!(await this.store.removeBufferedEvent(bufferedId)))
+            return touched; // claimed elsewhere already
+        // `takeSignalWaiter` is safe here even though it deletes ANY row for the token: an event token
+        // embeds this ONE `waitForEvent` call's own `runId#seq` (see eventToken in events.ts), so no other
+        // registration could ever share it.
+        const waiter = await this.store.takeSignalWaiter(lateWaiter.token);
+        if (waiter) {
+            const settled = await this.deliverSignal(waiter, payload);
+            if (settled)
+                touched += 1;
+        }
+        // If `waiter` is null, that exact registration resolved itself some other way in the interim (most
+        // likely its own timeout deadline) — the buffered copy is already spent, so drop it rather than
+        // resuming a run a second time or re-buffering for a THIRD, unrelated waiter to pick up later.
         return touched;
     }
-    async signal(token, payload) {
-        const waiter = await this.store.takeSignalWaiter(token);
-        if (!waiter) {
-            // No one is waiting yet — buffer it so the next `waitForSignal(token)` consumes it instead of
-            // dropping it (reliable signals; the basis of `signalWithStart`).
-            await this.store.bufferSignal(token, payload);
-            return null;
-        }
+    /**
+     * Deliver a signal to an already-known waiter: write its resolving `signal:<token>` checkpoint and
+     * resume the run. Shared by {@link signal}'s direct hit and its post-buffer re-check reclaim — both
+     * resolve a waiter the exact same way.
+     */
+    async deliverSignal(waiter, payload) {
         await this.store.saveCheckpoint(instantCheckpoint({
             runId: waiter.runId,
             seq: waiter.seq,
-            name: `signal:${token}`,
+            name: `signal:${waiter.token}`,
             kind: 'signal',
             output: payload,
             // Carry the awaiting command's fan group (set when a `ctx.gather_children`/`ctx.all` fan-out
@@ -810,6 +911,40 @@ export class WorkflowEngine {
         }));
         return this.resume(waiter.runId);
     }
+    /**
+     * Deliver an external signal to the run waiting on `token`, resume it with `payload`, and return
+     * the run result — or null if no run is (or ends up) waiting for it.
+     *
+     * Race with the waiter side (`waitForSignal`'s check → `putSignalWaiter` → re-check): a waiter can
+     * register in the sliver between this method's initial `takeSignalWaiter` miss and its
+     * `bufferSignal` write, and the naive old flow (buffer and return) would then leave BOTH a buffered
+     * payload and a registered waiter sitting in the store with nothing left to pair them — a lost wake,
+     * forever suspended. The interleaving proof: the waiter side does check → put → re-check, this side
+     * does take → buffer → re-check; whichever side's SECOND look runs last always observes the other
+     * side's write, and the destructive `take*` ops arbitrate so a signal is never delivered twice.
+     */
+    async signal(token, payload) {
+        const waiter = await this.store.takeSignalWaiter(token);
+        if (waiter)
+            return this.deliverSignal(waiter, payload);
+        // No one was waiting yet — buffer it so the next `waitForSignal(token)` consumes it instead of
+        // dropping it (reliable signals; the basis of `signalWithStart`).
+        await this.store.bufferSignal(token, payload);
+        // Re-check: a waiter may have registered in the window between the miss above and this buffer
+        // write (see the interleaving proof above).
+        const lateWaiter = await this.store.takeSignalWaiter(token);
+        if (!lateWaiter)
+            return null;
+        const reclaimed = await this.store.takeBufferedSignal(token);
+        if (!reclaimed) {
+            // The waiter side won the race to consume the buffer itself (its own checkpoint+resume already
+            // delivered the payload) in the sliver between our takeSignalWaiter and takeBufferedSignal just
+            // above. We've already (destructively) taken `lateWaiter` off the store, but there is nothing
+            // left to deliver — drop it deliberately rather than resuming the run a second time.
+            return null;
+        }
+        return this.deliverSignal(lateWaiter, reclaimed.payload);
+    }
     /**
      * Ensure a run exists for `runId`, then deliver a signal to it — atomically race-free thanks to
      * signal buffering: if the run is new (or busy / not yet waiting), the signal is buffered and
@@ -848,6 +983,27 @@ export class WorkflowEngine {
     notifyParent(runId, completion) {
         void this.signal(`child:${runId}`, completion).catch(() => undefined);
     }
+    /**
+     * Deferred child start shared by the in-process ctx host and the remote `startChild` command.
+     * Deferred (microtask) so a fast child can't reentrantly resume a still-suspending parent. A start
+     * that THROWS (unregistered/unroutable workflow, input validation, singleton back-pressure, store
+     * failure) must NOT be swallowed: the parent is already suspended on `child:<childId>` (the waiter
+     * is put BEFORE start on every path), so a silent drop parks it in suspended-forever, invisibly
+     * re-attempting on every recovery wake — a misconfigured remote child looks exactly like a healthy
+     * long wait. Instead the failure is delivered to that waiter like a failed child (notifyParent):
+     * the parent resumes and fails loudly with the cause. For a fire-and-forget `ctx.startChild`
+     * (spawn — no waiter) the completion is buffered; a later join by the same id consumes it and
+     * correctly observes the failed start.
+     */
+    startChildDeferred(workflow, input, childId, opts) {
+        queueMicrotask(() => void this.start(workflow, input, childId, opts).catch((err) => {
+            const message = err instanceof Error ? err.message : String(err);
+            this.notifyParent(childId, {
+                ok: false,
+                error: `child workflow "${workflow}" failed to start: ${message}`,
+            });
+        }));
+    }
     /**
      * Cancel a run (e.g. from the dashboard). Returns null if the run does not exist. Pass
      * `{ compensate: true }` to undo the saga first: the suspended run is resumed so its completed
@@ -885,7 +1041,13 @@ export class WorkflowEngine {
         }
         const error = { message: 'cancelled' };
         await this.store.updateRun(runId, { status: 'cancelled', error, updatedAt: new Date() });
-        this.emit({ type: 'run.failed', runId, workflow: run.workflow, error });
+        this.emit({
+            type: 'run.failed',
+            runId,
+            workflow: run.workflow,
+            namespace: run.namespace,
+            error,
+        });
         await this.cancelChildren(runId, opts);
         // Notify local cancel listeners now (a worker on this pod), and broadcast so the instance/worker
         // actually running this run learns of it and can abort cooperatively (the store already records
@@ -1204,6 +1366,7 @@ export class WorkflowEngine {
                 type: 'run.completed',
                 runId: run.id,
                 workflow: run.workflow,
+                namespace: run.namespace,
                 output: outcome.output,
             });
             void this.notifyParent(run.id, { ok: true, value: outcome.output });
@@ -1215,13 +1378,28 @@ export class WorkflowEngine {
                 type: 'run.failed',
                 runId: run.id,
                 workflow: run.workflow,
+                namespace: run.namespace,
                 error: outcome.error,
             });
             void this.notifyParent(run.id, { ok: false, error: outcome.error.message });
             return { runId: run.id, status: 'failed', error: outcome.error };
         }
+        // This outcome was computed by a turn that started from a possibly-stale run snapshot. If the run
+        // was cancelled WHILE that turn was still executing (e.g. `ctx.all`'s failFast cancelling a
+        // sibling mid-turn — plain `cancel()` writes `cancelled` directly, without waiting for the target's
+        // in-flight turn to notice), this now-stale "suspended" outcome must not resurrect it: re-check the
+        // CURRENT persisted status right before writing and echo it instead of clobbering a real cancel.
+        const latest = await this.store.getRun(run.id);
+        if (latest?.status === 'cancelled') {
+            return { runId: run.id, status: 'cancelled', error: latest.error };
+        }
         await this.store.updateRun(run.id, { status: 'suspended', wakeAt: outcome.wakeAt, updatedAt });
-        this.emit({ type: 'run.suspended', runId: run.id, workflow: run.workflow });
+        this.emit({
+            type: 'run.suspended',
+            runId: run.id,
+            workflow: run.workflow,
+            namespace: run.namespace,
+        });
         return { runId: run.id, status: 'suspended' };
     }
     async runRemoteExecution(run, registered) {
@@ -1229,12 +1407,22 @@ export class WorkflowEngine {
         if (run.status === 'pending') {
             await this.store.updateRun(run.id, { status: 'running', updatedAt: new Date() });
             run.status = 'running';
-            this.emit({ type: 'run.started', runId: run.id, workflow: run.workflow });
+            this.emit({
+                type: 'run.started',
+                runId: run.id,
+                workflow: run.workflow,
+                namespace: run.namespace,
+            });
         }
         if (registered.singleton && !(await this.singletons.admit(run, registered.singleton))) {
             const wakeAt = this.singletons.retryWakeAt();
             await this.store.updateRun(run.id, { status: 'suspended', wakeAt, updatedAt: new Date() });
-            this.emit({ type: 'run.suspended', runId: run.id, workflow: run.workflow });
+            this.emit({
+                type: 'run.suspended',
+                runId: run.id,
+                workflow: run.workflow,
+                namespace: run.namespace,
+            });
             await this.store.releaseRunLock(run.id);
             return { runId: run.id, status: 'suspended' };
         }
@@ -1249,7 +1437,13 @@ export class WorkflowEngine {
                 stack: err instanceof Error ? err.stack : undefined,
             };
             await this.store.updateRun(run.id, { status: 'failed', error, updatedAt: new Date() });
-            this.emit({ type: 'run.failed', runId: run.id, workflow: run.workflow, error });
+            this.emit({
+                type: 'run.failed',
+                runId: run.id,
+                workflow: run.workflow,
+                namespace: run.namespace,
+                error,
+            });
             return { runId: run.id, status: 'failed', error };
         }
         if (decision.status === 'completed') {
@@ -1375,6 +1569,10 @@ export class WorkflowEngine {
                 // Mirrors the `startChild` `getRun(childId)` guard below.
                 if (await this.store.getCheckpoint(run.id, cmd.seq))
                     continue;
+                // Route this cross-SDK-worker `call` by the SAME name-based token a native `ctx.step`
+                // dispatches with (and a worker subscribes to per handler name), so a decision-driven remote
+                // step and an in-process one land on the identical queue. The command carries no partition.
+                const callToken = tenantGroup(sanitizeQueueToken(cmd.name), undefined);
                 await this.store.saveCheckpoint(stepCheckpoint({
                     runId: run.id,
                     seq: cmd.seq,
@@ -1383,13 +1581,13 @@ export class WorkflowEngine {
                     status: 'pending',
                     input: cmd.input,
                     attempts: 1,
-                    workerGroup: cmd.group,
+                    workerGroup: callToken,
                     enqueuedAt: at,
                     startedAt: at,
                     finishedAt: at,
                     // A `ctx.gather_calls([...])` fan-out stamps every dispatched `call` in the fan with the
                     // same group, so the dashboard renders the remote steps as one parallel fan (parity with
-                    // the gathered `recordStep`/`startChild` tags). Undefined for a lone sequential `ctx.call`.
+                    // the gathered `recordStep`/`startChild` tags). Undefined for a lone sequential `ctx.step`.
                     parallelGroup: cmd.parallelGroup,
                 }));
                 await this.pool.dispatch({
@@ -1397,7 +1595,7 @@ export class WorkflowEngine {
                     seq: cmd.seq,
                     name: cmd.name,
                     stepId: id,
-                    group: cmd.group,
+                    group: callToken,
                     input: cmd.input,
                     traceparent: this.traceparent?.(),
                     context: this.context?.(),
@@ -1433,19 +1631,33 @@ export class WorkflowEngine {
                 // engine.signal(name, payload) delivers it. If the signal was already delivered (buffered
                 // before the workflow reached this point — e.g. signalWithStart), resolve it now and re-drive
                 // on a macrotask, AFTER this turn suspends and frees the run lock (a re-entrant resume bails).
-                const buffered = await this.store.takeBufferedSignal(cmd.signal);
-                if (buffered) {
+                const deliverBuffered = async (payload) => {
                     await this.store.saveCheckpoint(instantCheckpoint({
                         runId: run.id,
                         seq: cmd.seq,
                         name: `signal:${cmd.signal}`,
                         kind: 'signal',
-                        output: buffered.payload,
+                        output: payload,
                     }));
                     setTimeout(() => void this.resume(run.id).catch(() => undefined), 0);
+                };
+                const buffered = await this.store.takeBufferedSignal(cmd.signal);
+                if (buffered) {
+                    await deliverBuffered(buffered.payload);
                 }
                 else {
+                    // Same reorder as the in-process ctx.waitForSignal: register on a miss, then re-check once
+                    // more before suspending — closing the lost-wake window where a signal races in between the
+                    // miss above and the registration below (see the interleaving proof at engine.signal).
                     await this.store.putSignalWaiter({ token: cmd.signal, runId: run.id, seq: cmd.seq });
+                    const lateBuffered = await this.store.takeBufferedSignal(cmd.signal);
+                    if (lateBuffered) {
+                        // Resolved it ourselves — remove OUR OWN row via the exact match (not
+                        // takeSignalWaiter(cmd.signal), which deletes ANY row for this token and could steal a
+                        // different run's waiter that has since claimed the same token).
+                        await this.store.removeSignalWaiter({ token: cmd.signal, runId: run.id, seq: cmd.seq });
+                        await deliverBuffered(lateBuffered.payload);
+                    }
                 }
             }
             else if (cmd.kind === 'startChild') {
@@ -1466,7 +1678,7 @@ export class WorkflowEngine {
                     parallelGroup: cmd.parallelGroup,
                 });
                 if (!(await this.store.getRun(childId))) {
-                    queueMicrotask(() => void this.start(cmd.workflow, cmd.input, childId).catch(() => undefined));
+                    this.startChildDeferred(cmd.workflow, cmd.input, childId);
                 }
             }
             else {
@@ -1483,14 +1695,24 @@ export class WorkflowEngine {
         if (run.status === 'pending') {
             await this.store.updateRun(run.id, { status: 'running', updatedAt: new Date() });
             run.status = 'running';
-            this.emit({ type: 'run.started', runId: run.id, workflow: run.workflow });
+            this.emit({
+                type: 'run.started',
+                runId: run.id,
+                workflow: run.workflow,
+                namespace: run.namespace,
+            });
         }
         // Singleton admission gate: if this run shares its key with `limit` older in-flight runs, wait
         // (suspend on a short timer) until a slot frees instead of running now. Re-checked on each resume.
         if (registered?.singleton && !(await this.singletons.admit(run, registered.singleton))) {
             const wakeAt = this.singletons.retryWakeAt();
             await this.store.updateRun(run.id, { status: 'suspended', wakeAt, updatedAt: new Date() });
-            this.emit({ type: 'run.suspended', runId: run.id, workflow: run.workflow });
+            this.emit({
+                type: 'run.suspended',
+                runId: run.id,
+                workflow: run.workflow,
+                namespace: run.namespace,
+            });
             await this.store.releaseRunLock(run.id);
             return { runId: run.id, status: 'suspended' };
         }
@@ -1507,7 +1729,10 @@ export class WorkflowEngine {
             replay.set(cp.seq, cp);
         const ctx = createWorkflowCtx(this.ctxHostFor(replay), run.id, compensations, run.workflow);
         try {
-            const output = await fn(ctx, run.input);
+            // Establish the ambient ctx for the duration of this body turn so context-aware statics
+            // (`BaseWorkflow.start`/`dispatch`) reachable from `fn` route through this run's `ctx.child`/
+            // `ctx.startChild`. Re-set on every replay turn (each is its own async scope) — correct.
+            const output = await workflowAls.run(ctx, () => fn(ctx, run.input));
             return this.settleRun(run, { kind: 'completed', output });
         }
         catch (err) {
@@ -1521,7 +1746,12 @@ export class WorkflowEngine {
                     error: undefined,
                     updatedAt: new Date(),
                 });
-                this.emit({ type: 'run.completed', runId: run.id, workflow: run.workflow });
+                this.emit({
+                    type: 'run.completed',
+                    runId: run.id,
+                    workflow: run.workflow,
+                    namespace: run.namespace,
+                });
                 void this.notifyParent(run.id, { ok: true, value: undefined });
                 const nextId = nextContinuationId(run.id);
                 queueMicrotask(() => void this.start(run.workflow, err.input, nextId).catch(() => undefined));
@@ -1539,7 +1769,13 @@ export class WorkflowEngine {
                     }
                     const error = { message: 'cancelled' };
                     await this.store.updateRun(run.id, { status: 'cancelled', error, updatedAt: new Date() });
-                    this.emit({ type: 'run.failed', runId: run.id, workflow: run.workflow, error });
+                    this.emit({
+                        type: 'run.failed',
+                        runId: run.id,
+                        workflow: run.workflow,
+                        namespace: run.namespace,
+                        error,
+                    });
                     return { runId: run.id, status: 'cancelled', error };
                 }
                 return this.settleRun(run, { kind: 'suspended', wakeAt: err.wakeAt });
@@ -1569,14 +1805,21 @@ export class WorkflowEngine {
     /**
      * Run one saga compensation, retried up to `compensationRetries`, emitting a `compensate:<step>`
      * step event for its outcome so a stranded undo is visible. Never throws — a permanently-failing
-     * compensation is skipped so it can't mask the original failure.
+     * compensation is skipped so it can't mask the original failure. Handles both compensation shapes:
+     * a LOCAL `fn` (run in-process) and a DISPATCHED undo (an ordinary step def sent to a worker and
+     * awaited inline — the worker serving its name runs it with the {@link StepUndo} envelope).
      */
     async runCompensation(run, comp) {
         const name = `compensate:${comp.name}`;
         for (let attempt = 1; attempt <= this.compensationRetries; attempt += 1) {
             const startedAt = Date.now();
             try {
-                await comp.fn();
+                if ('fn' in comp) {
+                    await comp.fn();
+                }
+                else {
+                    await this.dispatchCompensation(run, comp.dispatch.def, comp.dispatch.args, attempt);
+                }
                 this.emit({
                     type: 'step.completed',
                     runId: run.id,
@@ -1602,6 +1845,43 @@ export class WorkflowEngine {
             }
         }
     }
+    /**
+     * Dispatch a saga undo step to a worker and await its result INLINE (the saga unwind already runs
+     * outside the positional replay, so this is not checkpointed — it's a one-shot dispatch + await).
+     * Routes by the undo def's own name/partition token, applies its own optional input schema, and
+     * honours its own liveness `timeoutMs` via the heartbeat window. Throws on a worker-reported failure
+     * or timeout so `runCompensation`'s retry/skip logic applies uniformly to both compensation shapes.
+     */
+    async dispatchCompensation(run, def, args, attempt) {
+        if (this.pool.size === 0)
+            throw new Error('dispatched compensation requires a Transport');
+        const token = tenantGroup(sanitizeQueueToken(def.name), def.partition);
+        const validInput = def.input ? def.input.parse(args) : args;
+        const id = `${run.id}:compensate:${def.name}:${attempt}`;
+        const resultPromise = new Promise((resolve, reject) => {
+            this.pending.set(id, { resolve, reject });
+        });
+        try {
+            await this.pool.dispatch({
+                runId: run.id,
+                seq: -1,
+                name: def.name,
+                stepId: id,
+                group: token,
+                input: validInput,
+                traceparent: this.traceparent?.(),
+                context: this.context?.(),
+                attempt: 1,
+            });
+            await (def.timeoutMs
+                ? this.awaitWithHeartbeat(id, resultPromise, def.timeoutMs)
+                : resultPromise);
+        }
+        catch (err) {
+            this.pending.delete(id);
+            throw err;
+        }
+    }
     /** The seam handed to {@link createWorkflowCtx}: the authoring API reaches durability + lifecycle
      *  (checkpointing, dispatch, child start) through this, so the ctx primitives live in their own
      *  module and the engine stays the orchestrator. */
@@ -1617,7 +1897,7 @@ export class WorkflowEngine {
             callRemote: (runId, seq, step, input, queue, transport, admission) => this.callRemote(runId, seq, step, input, queue, transport, replay, admission),
             // Defer so a fast child can't reentrantly resume a still-running parent.
             startChild: (workflow, input, id, priority) => {
-                queueMicrotask(() => void this.start(workflow, input, id, { priority }).catch(() => undefined));
+                this.startChildDeferred(workflow, input, id, { priority });
             },
             // Shallow-merge into the run's searchAttributes (the ctx primitive makes this exactly-once).
             upsertSearchAttributes: async (runId, attrs) => {
@@ -1760,7 +2040,13 @@ export class WorkflowEngine {
             this.queueWaiters.get(queue)?.delete(runId);
             this.stepQueue.set(id, queue);
         }
-        const validInput = step.input.parse(input);
+        // Optional runtime schema: validate only when the def carries one (a bare `@Step()` / string-name
+        // call carries none — the engine passes the input through untouched then).
+        const validInput = step.input ? step.input.parse(input) : input;
+        // Routing token: BY NAME (sanitized for brokers), optionally partition-suffixed. Computed once and
+        // used at BOTH the checkpoint's workerGroup and the dispatched task's routing `group`, so the same
+        // token the worker subscribes to per handler name serves this step.
+        const token = tenantGroup(sanitizeQueueToken(step.name), step.partition);
         const enqueuedAt = new Date();
         // Persist the pending checkpoint BEFORE dispatching, so a fast result always finds it to complete.
         await this.store.saveCheckpoint({
@@ -1772,7 +2058,7 @@ export class WorkflowEngine {
             status: 'pending',
             input: validInput,
             attempts: attempt,
-            workerGroup: step.group,
+            workerGroup: token,
             enqueuedAt,
             startedAt: enqueuedAt, // placeholders until the worker result lands
             finishedAt: enqueuedAt,
@@ -1782,7 +2068,7 @@ export class WorkflowEngine {
             seq,
             name: step.name,
             stepId: id,
-            group: step.group,
+            group: token,
             input: validInput,
             traceparent: this.traceparent?.(),
             context: this.context?.(),
@@ -1859,7 +2145,8 @@ export class WorkflowEngine {
     async callRemoteInMemory(runId, seq, step, input, transport) {
         if (this.pool.size === 0)
             throw new Error('remote steps require a Transport');
-        const validInput = step.input.parse(input);
+        const validInput = step.input ? step.input.parse(input) : input;
+        const token = tenantGroup(sanitizeQueueToken(step.name), step.partition);
         const id = stepId(runId, seq);
         const enqueuedAt = new Date();
         this.emit({ type: 'step.started', runId, seq, name: step.name, kind: 'remote' });
@@ -1878,7 +2165,7 @@ export class WorkflowEngine {
                 seq,
                 name: step.name,
                 stepId: id,
-                group: step.group,
+                group: token,
                 input: validInput,
                 traceparent: this.traceparent?.(),
                 context: this.context?.(),
@@ -1888,7 +2175,7 @@ export class WorkflowEngine {
                 const resolution = step.timeoutMs
                     ? await this.awaitWithHeartbeat(id, resultPromise, step.timeoutMs)
                     : await resultPromise;
-                const output = step.output.parse(resolution.output);
+                const output = (step.output ? step.output.parse(resolution.output) : resolution.output);
                 // The worker reports when it actually picked the task up; fall back to dispatch time if a
                 // transport doesn't carry it (queue-wait then reads as zero rather than going negative).
                 const startedAt = resolution.startedAt ? new Date(resolution.startedAt) : enqueuedAt;
@@ -1901,7 +2188,7 @@ export class WorkflowEngine {
                     output,
                     events: resolution.events,
                     attempts: attempt,
-                    workerGroup: step.group,
+                    workerGroup: token,
                     enqueuedAt,
                     startedAt,
                 });