@dudousxd/nestjs-durable-core 0.5.0 → 0.6.0

package/dist/engine.js

@@ -1,21 +1,13 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.RemoteStepError = exports.WorkflowEngine = void 0;
-const duration_1 = require("./duration");
+const backoff_1 = require("./backoff");
+const checkpoints_1 = require("./checkpoints");
 const errors_1 = require("./errors");
 const protocol_1 = require("./protocol");
-const step_logger_1 = require("./step-logger");
-/** Deterministic signal token a breakpoint suspends on — derived from its logical position. */
-const breakpointToken = (runId, seq) => `bp:${runId}:${seq}`;
-/** Delay in ms before the next retry of a local step, per its `StepOptions` backoff config. */
-function backoffDelay(attempt, options) {
-    const base = options?.backoffMs ?? 0;
-    if (base <= 0)
-        return 0;
-    const raw = options?.backoff === 'exp' ? base * 2 ** (attempt - 1) : base;
-    const capped = options?.backoffMaxMs ? Math.min(raw, options.backoffMaxMs) : raw;
-    return options?.jitter ? Math.round(capped * (0.5 + Math.random() * 0.5)) : capped;
-}
+const queue_1 = require("./queue");
+const transport_pool_1 = require("./transport-pool");
+const workflow_ctx_1 = require("./workflow-ctx");
 /** A breakpoint checkpoint's `name` is `breakpoint` (or `breakpoint:<label>`). This name — not the
  *  reused `signal` kind — is the explicit marker the dashboard and `continue()` detect it by. */
 const BREAKPOINT = 'breakpoint';
@@ -29,14 +21,6 @@ function isNewerVersion(a, b) {
         return na > nb;
     return a.localeCompare(b, undefined, { numeric: true }) > 0;
 }
-/** Read a `Completion` from a signal payload: return the value, or throw a FatalError if it failed. */
-function unwrapCompletion(payload, label) {
-    const c = payload;
-    if (c && typeof c === 'object' && 'ok' in c && c.ok === false) {
-        throw new errors_1.FatalError(`${label} failed: ${c.error}`);
-    }
-    return c?.value;
-}
 /**
  * The orchestrator. Owns workflow state and replays runs deterministically: each step's
  * result is checkpointed, so on resume a completed step returns its saved output instead of
@@ -45,10 +29,16 @@ function unwrapCompletion(payload, label) {
  */
 class WorkflowEngine {
     store;
-    transport;
+    /** Ordered transport pool (dispatch + failover). Empty = no remote steps. */
+    pool;
+    controlPlane;
     clock;
     instanceId;
     leaseMs;
+    maxRecoveryAttempts;
+    webhookUrl;
+    traceparent;
+    compensationRetries;
     /** Every registered workflow, keyed by `name@version` — so old versions stay runnable. */
     workflows = new Map();
     /** The newest registered version per workflow name — used to `start` new runs. */
@@ -60,16 +50,29 @@ class WorkflowEngine {
     listeners = new Set();
     /** Callbacks notified (on any instance) when a run is cancelled — for cooperative cancellation. */
     cancelListeners = new Set();
+    /** Validators gating `engine.update`, keyed by `<workflow>:<updateName>`. */
+    updateValidators = new Map();
+    /** Runs being cancelled WITH saga compensation — see `cancel({ compensate: true })`. */
+    cancelRequested = new Set();
+    /** Flow-control queues for remote steps, keyed by name (see {@link registerQueue}). */
+    queues = new Map();
+    /** Which queue a dispatched step took a slot from, by stepId — so the result can release it. */
+    stepQueue = new Map();
     /** Executions currently in flight, so a graceful shutdown can wait for them to settle. */
     inflight = new Set();
     draining = false;
     constructor(deps) {
         this.store = deps.store;
-        this.transport = deps.transport;
+        this.pool = new transport_pool_1.TransportPool(deps.transports ?? (deps.transport ? [{ id: 'default', transport: deps.transport }] : []));
+        this.controlPlane = deps.controlPlane;
         this.clock = deps.clock ?? Date.now;
         this.instanceId = deps.instanceId ?? globalThis.crypto.randomUUID();
         this.leaseMs = deps.leaseMs ?? 30_000;
-        this.transport?.onResult(async (result) => {
+        this.maxRecoveryAttempts = deps.maxRecoveryAttempts;
+        this.webhookUrl = deps.webhookUrl;
+        this.traceparent = deps.traceparent;
+        this.compensationRetries = Math.max(1, deps.compensationRetries ?? 1);
+        this.pool.bind(async (result) => {
             // In-memory path (a `timeoutMs` step awaiting on THIS instance): resolve its pending promise.
             const waiter = this.pending.get(result.stepId);
             if (waiter) {
@@ -89,15 +92,15 @@ class WorkflowEngine {
             // Durable path: no in-memory waiter (the step suspended the run, possibly on another
             // instance) → complete the checkpoint and resume the run here.
             await this.completeRemoteResult(result);
-        });
+        }, 
         // A heartbeat for an in-flight long step resets its liveness window (see callRemote).
-        this.transport?.onHeartbeat(async (beat) => {
+        async (beat) => {
             this.heartbeatResets.get(beat.stepId)?.();
         });
         // Control plane: re-broadcast lifecycle events from OTHER instances to this instance's
         // subscribers (cross-pod live-tail), and act on cancellations issued elsewhere. A broker may
         // echo our own publish back — ignore those (we already handled them locally) to avoid duplicates.
-        this.transport?.onControl?.((msg) => {
+        this.controlPlane?.onControl((msg) => {
             if (msg.from === this.instanceId)
                 return;
             if (msg.kind === 'event') {
@@ -132,6 +135,14 @@ class WorkflowEngine {
         if (!current || isNewerVersion(version, current.version))
             this.latest.set(name, registered);
     }
+    /**
+     * Register a flow-control queue referenced by `ctx.call(step, input, { queue })`. Caps concurrent
+     * in-flight steps and/or the admission rate; blocked calls re-suspend and retry, so the limit is
+     * durable. Per engine instance (see {@link QueueConfig}). Registering the same name replaces it.
+     */
+    registerQueue(config) {
+        this.queues.set(config.name, new queue_1.QueueController(config, this.clock));
+    }
     /** Subscribe to lifecycle events. Returns an unsubscribe function. */
     subscribe(listener) {
         this.listeners.add(listener);
@@ -151,8 +162,8 @@ class WorkflowEngine {
     emit(event) {
         const full = { ...event, at: new Date() };
         this.deliver(full);
-        if (this.transport?.publishControl) {
-            void this.transport
+        if (this.controlPlane) {
+            void this.controlPlane
                 .publishControl({ kind: 'event', event: full, from: this.instanceId })
                 .catch(() => {
                 // control-plane delivery is best-effort observability; never break execution
@@ -202,7 +213,7 @@ class WorkflowEngine {
         // A definitively-finished run must not be re-executed (e.g. a worker result landing after the
         // run was cancelled, or a duplicate resume) — that would replay the body and clobber the
         // terminal state. `failed` is intentionally NOT terminal here: retry resumes a failed run.
-        if (run.status === 'cancelled' || run.status === 'completed') {
+        if (run.status === 'cancelled' || run.status === 'completed' || run.status === 'dead') {
             return { runId, status: run.status, output: run.output, error: run.error };
         }
         // Pin to the version the run STARTED on — replay is positional, so running a changed
@@ -238,8 +249,29 @@ class WorkflowEngine {
      * Resume every run left incomplete by a crash or deploy. Called on boot. Completed steps
      * replay from their checkpoints, so only the work that had not finished runs again.
      */
-    async recoverIncomplete() {
-        return this.resumeLeased(await this.store.listIncompleteRuns());
+    async recoverIncomplete(nowMs = this.clock()) {
+        return this.resumeLeased(await this.store.listIncompleteRuns(), nowMs, (run) => this.countRecovery(run));
+    }
+    /**
+     * Per-recovery bookkeeping (called once the lease is held): count the attempt, or — past
+     * `maxRecoveryAttempts` — move a poison pill to the `dead` dead-letter state. Returns a terminal
+     * result to skip the resume, or `undefined` to proceed.
+     */
+    async countRecovery(run) {
+        const attempts = (run.recoveryAttempts ?? 0) + 1;
+        if (this.maxRecoveryAttempts != null && attempts > this.maxRecoveryAttempts) {
+            const error = {
+                message: `run exceeded maxRecoveryAttempts (${this.maxRecoveryAttempts}) — moved to dead-letter`,
+                code: 'max_recovery_attempts',
+            };
+            await this.store.updateRun(run.id, { status: 'dead', error, updatedAt: new Date() });
+            await this.store.releaseRunLock(run.id);
+            this.emit({ type: 'run.failed', runId: run.id, workflow: run.workflow, error });
+            return { runId: run.id, status: 'dead', error };
+        }
+        // Count BEFORE resuming, so a crash mid-resume still advances the counter.
+        await this.store.updateRun(run.id, { recoveryAttempts: attempts, updatedAt: new Date() });
+        return undefined;
     }
     /**
      * Resume every suspended run whose durable timer is due. Call periodically (a poller) and on
@@ -252,14 +284,17 @@ class WorkflowEngine {
      * Resume each run only if this instance can acquire its recovery lease — so when several
      * replicas recover or poll at once, each run is picked up by exactly one of them.
      */
-    async resumeLeased(runs, nowMs = this.clock()) {
+    async resumeLeased(runs, nowMs = this.clock(), onLocked) {
         if (this.draining)
             return []; // shutting down — don't pick up new runs
         const results = [];
         for (const run of runs) {
             const acquired = await this.store.tryLockRun(run.id, this.instanceId, nowMs + this.leaseMs, nowMs);
-            if (acquired)
-                results.push(await this.resume(run.id));
+            if (!acquired)
+                continue;
+            // A per-run hook (recovery counting / dead-lettering) may settle the run terminally instead.
+            const settled = onLocked ? await onLocked(run) : undefined;
+            results.push(settled ?? (await this.resume(run.id)));
         }
         return results;
     }
@@ -271,20 +306,13 @@ class WorkflowEngine {
         const waiter = await this.store.takeSignalWaiter(token);
         if (!waiter)
             return null;
-        const at = new Date();
-        await this.store.saveCheckpoint({
+        await this.store.saveCheckpoint((0, checkpoints_1.instantCheckpoint)({
             runId: waiter.runId,
             seq: waiter.seq,
             name: `signal:${token}`,
             kind: 'signal',
-            stepId: (0, protocol_1.stepId)(waiter.runId, waiter.seq),
-            status: 'completed',
             output: payload,
-            attempts: 1,
-            enqueuedAt: at,
-            startedAt: at,
-            finishedAt: at,
-        });
+        }));
         return this.resume(waiter.runId);
     }
     /**
@@ -310,11 +338,29 @@ class WorkflowEngine {
     notifyParent(runId, completion) {
         void this.signal(`child:${runId}`, completion).catch(() => undefined);
     }
-    /** Cancel a run (e.g. from the dashboard). Returns null if the run does not exist. */
-    async cancel(runId) {
+    /**
+     * Cancel a run (e.g. from the dashboard). Returns null if the run does not exist. Pass
+     * `{ compensate: true }` to undo the saga first: the suspended run is resumed so its completed
+     * steps' compensations run in reverse (visible as `compensate:<step>` events), then it's marked
+     * cancelled. Without it, cancellation is immediate (no undo).
+     */
+    async cancel(runId, opts) {
         const run = await this.store.getRun(runId);
         if (!run)
             return null;
+        // Compensating cancel: resume the run with a cancellation pending. Replay re-registers the
+        // saga, and at the run's suspension point execute() runs the undo and marks it cancelled.
+        if (opts?.compensate && (run.status === 'suspended' || run.status === 'running')) {
+            this.cancelRequested.add(runId);
+            const result = await this.resume(runId);
+            this.notifyCancelled(runId);
+            if (this.controlPlane) {
+                void this.controlPlane
+                    .publishControl({ kind: 'cancel', runId, from: this.instanceId })
+                    .catch(() => undefined);
+            }
+            return result;
+        }
         const error = { message: 'cancelled' };
         await this.store.updateRun(runId, { status: 'cancelled', error, updatedAt: new Date() });
         this.emit({ type: 'run.failed', runId, workflow: run.workflow, error });
@@ -322,8 +368,8 @@ class WorkflowEngine {
         // actually running this run learns of it and can abort cooperatively (the store already records
         // `cancelled`, but a busy worker won't re-read it).
         this.notifyCancelled(runId);
-        if (this.transport?.publishControl) {
-            void this.transport
+        if (this.controlPlane) {
+            void this.controlPlane
                 .publishControl({ kind: 'cancel', runId, from: this.instanceId })
                 .catch(() => undefined);
         }
@@ -390,7 +436,7 @@ class WorkflowEngine {
     async execute(run, fn) {
         // Saga compensations registered by completed steps; run in reverse if the run later fails.
         const compensations = [];
-        const ctx = this.makeCtx(run.id, compensations);
+        const ctx = (0, workflow_ctx_1.createWorkflowCtx)(this.ctxHost, run.id, compensations);
         try {
             const output = await fn(ctx, run.input);
             // Clear any error from an earlier failed-then-retried attempt: a completed run is a success
@@ -408,6 +454,20 @@ class WorkflowEngine {
         }
         catch (err) {
             if (err instanceof errors_1.WorkflowSuspended) {
+                // A compensating cancel resumed this run to reach here: the replay re-registered the saga,
+                // so undo the completed steps in reverse and mark it cancelled instead of re-suspending.
+                if (this.cancelRequested.has(run.id)) {
+                    this.cancelRequested.delete(run.id);
+                    for (let i = compensations.length - 1; i >= 0; i -= 1) {
+                        const comp = compensations[i];
+                        if (comp)
+                            await this.runCompensation(run, comp);
+                    }
+                    const error = { message: 'cancelled' };
+                    await this.store.updateRun(run.id, { status: 'cancelled', error, updatedAt: new Date() });
+                    this.emit({ type: 'run.failed', runId: run.id, workflow: run.workflow, error });
+                    return { runId: run.id, status: 'cancelled', error };
+                }
                 await this.store.updateRun(run.id, {
                     status: 'suspended',
                     wakeAt: err.wakeAt,
@@ -420,15 +480,15 @@ class WorkflowEngine {
                 message: err instanceof Error ? err.message : String(err),
                 stack: err instanceof Error ? err.stack : undefined,
             };
-            // Saga: undo completed steps in reverse. A compensation that throws is logged-and-skipped so
-            // one bad undo can't strand the rest. (Compensations are best-effort and should be idempotent.)
+            // Saga: undo completed steps in reverse, each retried up to `compensationRetries`. Outcomes
+            // are emitted as `compensate:<step>` step events so a stranded undo is VISIBLE (not silently
+            // swallowed) in the dashboard/telescope; a failing one is still skipped so it can't mask the
+            // original failure or strand the rest. (Compensations should be idempotent.)
             for (let i = compensations.length - 1; i >= 0; i -= 1) {
-                try {
-                    await compensations[i]?.();
-                }
-                catch {
-                    // a failing compensation must not mask the original failure
-                }
+                const comp = compensations[i];
+                if (!comp)
+                    continue;
+                await this.runCompensation(run, comp);
             }
             await this.store.updateRun(run.id, { status: 'failed', error, updatedAt: new Date() });
             this.emit({ type: 'run.failed', runId: run.id, workflow: run.workflow, error });
@@ -441,219 +501,59 @@ class WorkflowEngine {
             await this.store.releaseRunLock(run.id);
         }
     }
-    makeCtx(runId, compensations) {
-        let seq = -1;
-        const nextSeq = () => {
-            seq += 1;
-            return seq;
-        };
-        const store = this.store;
-        // Each ctx primitive is a named closure over `nextSeq` (the per-run logical position counter)
-        // and `compensations` (the saga undo stack). They're defined up front so `task`/`child` can
-        // compose `step`/`waitForSignal` directly — no post-construction mutation, no cast.
-        const step = async (name, fn, options) => {
-            const current = nextSeq();
-            const existing = await store.getCheckpoint(runId, current);
-            if (existing && existing.name !== name) {
-                throw new errors_1.NonDeterminismError(runId, current, name, existing.name);
-            }
-            if (existing && existing.status === 'completed') {
-                // Register the compensation on replay too, so a saga undoes ALL completed steps — even
-                // those done in an earlier (since-suspended) pass — not just the ones run this pass.
-                if (options?.compensate)
-                    compensations.push(options.compensate);
-                return existing.output;
+    /**
+     * Run one saga compensation, retried up to `compensationRetries`, emitting a `compensate:<step>`
+     * step event for its outcome so a stranded undo is visible. Never throws — a permanently-failing
+     * compensation is skipped so it can't mask the original failure.
+     */
+    async runCompensation(run, comp) {
+        const name = `compensate:${comp.name}`;
+        for (let attempt = 1; attempt <= this.compensationRetries; attempt += 1) {
+            const startedAt = Date.now();
+            try {
+                await comp.fn();
+                this.emit({
+                    type: 'step.completed',
+                    runId: run.id,
+                    workflow: run.workflow,
+                    name,
+                    kind: 'local',
+                    durationMs: Date.now() - startedAt,
+                });
+                return;
             }
-            const maxAttempts = Math.max(1, options?.retries ?? 1);
-            const startedAt = new Date();
-            for (let attempt = 1;; attempt += 1) {
-                // Events are scoped per attempt — a retry starts a clean log, so the checkpoint reflects
-                // only the attempt that actually completed (or the final failing one).
-                const events = [];
-                try {
-                    const output = await fn((0, step_logger_1.createStepLogger)(events, this.clock));
-                    await this.completeStep({
-                        runId,
-                        seq: current,
+            catch (err) {
+                if (attempt >= this.compensationRetries) {
+                    this.emit({
+                        type: 'step.failed',
+                        runId: run.id,
+                        workflow: run.workflow,
                         name,
                         kind: 'local',
-                        output,
-                        events,
-                        attempts: attempt,
-                        enqueuedAt: startedAt,
-                        startedAt,
+                        error: { message: err instanceof Error ? err.message : String(err) },
+                        durationMs: Date.now() - startedAt,
                     });
-                    if (options?.compensate)
-                        compensations.push(options.compensate);
-                    return output;
-                }
-                catch (err) {
-                    if (err instanceof errors_1.FatalError || attempt >= maxAttempts) {
-                        await this.failStep({
-                            runId,
-                            seq: current,
-                            name,
-                            kind: 'local',
-                            error: {
-                                message: err instanceof Error ? err.message : String(err),
-                                stack: err instanceof Error ? err.stack : undefined,
-                            },
-                            events,
-                            attempts: attempt,
-                            enqueuedAt: startedAt,
-                            startedAt,
-                        });
-                        throw err;
-                    }
-                    // Wait out the backoff before the next attempt (no-op when backoffMs is unset).
-                    const wait = backoffDelay(attempt, options);
-                    if (wait > 0)
-                        await new Promise((resolve) => setTimeout(resolve, wait));
                 }
             }
-        };
-        const sleep = async (duration) => {
-            const current = nextSeq();
-            const now = this.clock();
-            const existing = await store.getCheckpoint(runId, current);
-            if (existing) {
-                // Timer already recorded: resume if due, otherwise re-suspend cheaply.
-                if (now >= (existing.wakeAt ?? 0))
-                    return;
-                throw new errors_1.WorkflowSuspended(existing.wakeAt ?? now);
-            }
-            const wakeAt = now + (0, duration_1.parseDuration)(duration);
-            await this.recordTimer(runId, current, 'sleep', wakeAt);
-            throw new errors_1.WorkflowSuspended(wakeAt);
-        };
-        // NOTE (determinism): a bounded wait consumes TWO logical positions (deadline + wait), an
-        // unbounded one consumes ONE. So adding or removing `{ timeoutMs }` on an existing
-        // `waitForSignal` shifts the seq of every later step — treat it as a workflow-version change
-        // (register a new @Workflow version) for runs already in flight.
-        const waitForSignal = async (token, opts) => {
-            if (opts?.timeoutMs == null) {
-                const current = nextSeq();
-                const existing = await store.getCheckpoint(runId, current);
-                if (existing && existing.status === 'completed')
-                    return existing.output;
-                await store.putSignalWaiter({ token, runId, seq: current });
-                throw new errors_1.WorkflowSuspended();
-            }
-            const timeoutMs = opts.timeoutMs;
-            const deadlineSeq = nextSeq();
-            const waitSeq = nextSeq();
-            // The deadline is recorded durably as a timer checkpoint so replay knows it; the run also gets
-            // a run-level wakeAt (via WorkflowSuspended) so the timer poller resumes it at the deadline.
-            const recorded = await store.getCheckpoint(runId, deadlineSeq);
-            const deadline = recorded?.wakeAt ?? this.clock() + timeoutMs;
-            if (!recorded)
-                await this.recordTimer(runId, deadlineSeq, `timeout:${token}`, deadline);
-            const waited = await store.getCheckpoint(runId, waitSeq);
-            if (waited && waited.status === 'completed')
-                return waited.output;
-            if (this.clock() >= deadline) {
-                await store.takeSignalWaiter(token).catch(() => undefined);
-                throw new errors_1.SignalTimeoutError(token, timeoutMs);
-            }
-            await store.putSignalWaiter({ token, runId, seq: waitSeq });
-            throw new errors_1.WorkflowSuspended(deadline);
-        };
-        // An external task = a checkpointed dispatch + a wait for its async-completion `Completion`
-        // (delivered by engine.completeTask/failTask). The whole "fire at a foreign system, suspend,
-        // resume when it reports back" pattern as one call.
-        const task = async (name, dispatch, options) => {
-            await step(`task:dispatch:${name}`, dispatch, options);
-            return unwrapCompletion(await waitForSignal(`task:${runId}:${name}`), `task "${name}"`);
-        };
-        // Child workflow: start it once, then suspend on a `child:<id>` waiter the child signals on its
-        // terminal state (see notifyParent). The start is deferred to a microtask so it runs AFTER this
-        // run suspends — a fast child can't reentrantly resume a parent that's still running.
-        // A breakpoint = a visible `pending` checkpoint + a signal waiter the dashboard resumes via
-        // `engine.continue`. Reuses the signal machinery (kind 'signal'), so resume overwrites the
-        // pending checkpoint with a completed one and the run replays past it.
-        const breakpoint = async (label) => {
-            const current = nextSeq();
-            const existing = await store.getCheckpoint(runId, current);
-            if (existing && existing.status === 'completed')
-                return;
-            if (!existing) {
-                await this.recordBreakpoint(runId, current, label);
-                await store.putSignalWaiter({
-                    token: breakpointToken(runId, current),
-                    runId,
-                    seq: current,
-                });
-            }
-            throw new errors_1.WorkflowSuspended();
-        };
-        const child = async (workflow, input, childId) => {
-            const current = nextSeq();
-            const id = childId ?? `${runId}.child.${current}`;
-            const existing = await store.getCheckpoint(runId, current);
-            if (existing && existing.status === 'completed') {
-                return unwrapCompletion(existing.output, `child "${id}"`);
-            }
-            await store.putSignalWaiter({ token: `child:${id}`, runId, seq: current });
-            if (!(await store.getRun(id))) {
-                queueMicrotask(() => {
-                    void this.start(workflow, input, id).catch(() => undefined);
-                });
-            }
-            throw new errors_1.WorkflowSuspended();
-        };
-        // Deterministic non-deterministic sources: each is a checkpointed local step, so the value is
-        // captured on the first run and replayed verbatim afterwards (a raw Date.now()/Math.random()
-        // inside a workflow would differ across replays and corrupt the run).
-        const now = () => step('now', async () => this.clock());
-        const random = () => step('random', async () => Math.random());
-        const uuid = () => step('uuid', async () => globalThis.crypto.randomUUID());
+        }
+    }
+    /** The seam handed to {@link createWorkflowCtx}: the authoring API reaches durability + lifecycle
+     *  (checkpointing, dispatch, child start) through this, so the ctx primitives live in their own
+     *  module and the engine stays the orchestrator. */
+    get ctxHost() {
         return {
-            runId,
-            step,
-            sleep,
-            waitForSignal,
-            task,
-            child,
-            breakpoint,
-            now,
-            random,
-            uuid,
-            call: (remote, input) => this.callRemote(runId, nextSeq(), remote, input),
+            store: this.store,
+            clock: this.clock,
+            webhookUrl: this.webhookUrl,
+            completeStep: (s) => this.completeStep(s),
+            failStep: (s) => this.failStep(s),
+            callRemote: (runId, seq, step, input, queue, transport) => this.callRemote(runId, seq, step, input, queue, transport),
+            // Defer so a fast child can't reentrantly resume a still-running parent.
+            startChild: (workflow, input, id) => {
+                queueMicrotask(() => void this.start(workflow, input, id).catch(() => undefined));
+            },
         };
     }
-    /** Persist a completed timer checkpoint (a durable sleep / signal deadline) at a logical position. */
-    async recordTimer(runId, seq, name, wakeAt) {
-        const at = new Date();
-        await this.store.saveCheckpoint({
-            runId,
-            seq,
-            name,
-            kind: 'sleep',
-            stepId: (0, protocol_1.stepId)(runId, seq),
-            status: 'completed',
-            wakeAt,
-            attempts: 1,
-            enqueuedAt: at,
-            startedAt: at,
-            finishedAt: at,
-        });
-    }
-    /** Persist a `pending` checkpoint marking a breakpoint, so it's visible (and resumable) in the UI. */
-    async recordBreakpoint(runId, seq, label) {
-        const at = new Date();
-        await this.store.saveCheckpoint({
-            runId,
-            seq,
-            name: label ? `${BREAKPOINT}:${label}` : BREAKPOINT,
-            kind: 'signal',
-            stepId: (0, protocol_1.stepId)(runId, seq),
-            status: 'pending',
-            attempts: 1,
-            enqueuedAt: at,
-            startedAt: at,
-            finishedAt: at,
-        });
-    }
     /**
      * Resume a run paused at a {@link WorkflowCtx.breakpoint} (e.g. the dashboard "continue" button).
      * Finds the run's pending breakpoint checkpoint and signals it. Returns null if the run isn't
@@ -664,29 +564,103 @@ class WorkflowEngine {
         const bp = checkpoints.find(isBreakpoint);
         if (!bp)
             return null;
-        return this.signal(breakpointToken(runId, bp.seq), undefined);
+        return this.signal((0, protocol_1.breakpointToken)(runId, bp.seq), undefined);
     }
-    async callRemote(runId, seq, step, input) {
+    /**
+     * Read the latest value a run published for `key` via {@link WorkflowCtx.setEvent} — a
+     * side-effect-free query of a live (or finished) run's state. Returns `undefined` if the run
+     * never published that key. The suspend-model counterpart of a Temporal query.
+     */
+    async getEvent(runId, key) {
+        const name = `event:${key}`;
+        const checkpoints = await this.store.listCheckpoints(runId);
+        // listCheckpoints is ordered by seq ascending, so the last match is the most recent value.
+        let latest;
+        for (const cp of checkpoints)
+            if (cp.name === name)
+                latest = cp.output;
+        return latest;
+    }
+    /**
+     * Register a validator gating `engine.update(runId, name, …)` for runs of `workflow`. The
+     * validator runs BEFORE the update is delivered, so a rejection leaves the run untouched. One
+     * validator per (workflow, update name); registering again replaces it.
+     */
+    registerUpdateValidator(workflow, name, validate) {
+        this.updateValidators.set(`${workflow}:${name}`, validate);
+    }
+    /**
+     * Deliver a validated update to the run waiting at `ctx.onUpdate(name)`. Runs the registered
+     * validator (if any) first: on rejection returns `{ accepted: false, reason }` without disturbing
+     * the run; otherwise delivers `arg` and resumes, returning `{ accepted: true, run }` (`run` is null
+     * if nothing was waiting — a too-early or duplicate update).
+     */
+    async update(runId, name, arg) {
+        const run = await this.store.getRun(runId);
+        if (!run)
+            return { accepted: false, reason: `run ${runId} not found` };
+        const validate = this.updateValidators.get(`${run.workflow}:${name}`);
+        if (validate) {
+            try {
+                const reason = await validate(arg);
+                if (typeof reason === 'string' && reason.length > 0)
+                    return { accepted: false, reason };
+            }
+            catch (err) {
+                return { accepted: false, reason: err instanceof Error ? err.message : String(err) };
+            }
+        }
+        const result = await this.signal(`update:${runId}:${name}`, arg);
+        return { accepted: true, run: result };
+    }
+    async callRemote(runId, seq, step, input, queue, transport) {
         const existing = await this.store.getCheckpoint(runId, seq);
         if (existing && existing.name !== step.name) {
             throw new errors_1.NonDeterminismError(runId, seq, step.name, existing.name);
         }
         if (existing?.status === 'completed')
             return existing.output;
-        if (existing?.status === 'failed')
-            throw new RemoteStepError(existing.error);
-        if (!this.transport)
+        if (this.pool.size === 0)
             throw new Error('remote steps require a Transport');
         // A step with a liveness `timeoutMs` keeps the in-memory await + heartbeat path (re-dispatch a
         // presumed-dead worker). Without one, the call SUSPENDS DURABLY: dispatch, persist a `pending`
         // checkpoint, and let the result resume the run on whichever instance receives it — so a worker
         // pod can scale down or crash mid-step without losing the run or re-running completed work.
         if (step.timeoutMs)
-            return this.callRemoteInMemory(runId, seq, step, input);
+            return this.callRemoteInMemory(runId, seq, step, input, transport);
         if (existing?.status === 'pending')
             throw new errors_1.WorkflowSuspended(); // dispatched; keep waiting
-        const validInput = step.input.parse(input);
+        // Durable retry: a failed attempt re-dispatches up to `retries`, spacing attempts by `backoff` —
+        // unless the worker marked the error non-retryable (a deterministic verdict like a declined card).
+        // The retry deadline is stamped on the failed checkpoint as `wakeAt` (clock-space, persisted) the
+        // first time we see it, so it's stable across replays and survives a crash.
+        let attempt = 1;
+        if (existing?.status === 'failed') {
+            const maxAttempts = Math.max(1, step.retries ?? 1);
+            const retryable = existing.error?.retryable !== false;
+            if (!retryable || existing.attempts >= maxAttempts)
+                throw new RemoteStepError(existing.error);
+            if (existing.wakeAt == null) {
+                const wakeAt = this.clock() + (0, backoff_1.backoffDelay)(existing.attempts, step);
+                await this.store.saveCheckpoint({ ...existing, wakeAt });
+                throw new errors_1.WorkflowSuspended(wakeAt);
+            }
+            if (this.clock() < existing.wakeAt)
+                throw new errors_1.WorkflowSuspended(existing.wakeAt);
+            attempt = existing.attempts + 1;
+        }
         const id = (0, protocol_1.stepId)(runId, seq);
+        // Flow control: a queued call that can't be admitted (concurrency/rate) does NOT dispatch — the
+        // run re-suspends with the queue's retry time and the timer poller re-tries admission later, so
+        // the limit is durable. The admitted slot is released when the result lands (completeRemoteResult).
+        const controller = queue ? this.queues.get(queue) : undefined;
+        if (controller) {
+            const admission = controller.tryAdmit();
+            if (!admission.ok)
+                throw new errors_1.WorkflowSuspended(admission.retryAt);
+            this.stepQueue.set(id, queue);
+        }
+        const validInput = step.input.parse(input);
         const enqueuedAt = new Date();
         // Persist the pending checkpoint BEFORE dispatching, so a fast result always finds it to complete.
         await this.store.saveCheckpoint({
@@ -697,21 +671,22 @@ class WorkflowEngine {
             stepId: id,
             status: 'pending',
             input: validInput,
-            attempts: 1,
+            attempts: attempt,
             workerGroup: step.group,
             enqueuedAt,
             startedAt: enqueuedAt, // placeholders until the worker result lands
             finishedAt: enqueuedAt,
         });
-        await this.transport.dispatch({
+        await this.pool.dispatch({
             runId,
             seq,
             name: step.name,
             stepId: id,
             group: step.group,
             input: validInput,
-            attempt: 1,
-        });
+            traceparent: this.traceparent?.(),
+            attempt,
+        }, transport);
         this.emit({ type: 'step.started', runId, seq, name: step.name, kind: 'remote' });
         throw new errors_1.WorkflowSuspended();
     }
@@ -724,6 +699,9 @@ class WorkflowEngine {
         const cp = await this.store.getCheckpoint(result.runId, result.seq);
         if (!cp || cp.status !== 'pending')
             return;
+        // A result settling this step frees its flow-control slot (no-op if it wasn't queued). Done
+        // before the cancelled-run early-return below, so a cancellation can't leak the slot.
+        this.releaseQueueSlot(cp.stepId);
         // Drop a late result for a run that was cancelled/finished meanwhile — don't complete the step
         // or resume (the run is already terminal). This is the engine side of cooperative cancellation.
         const run = await this.store.getRun(result.runId);
@@ -753,9 +731,17 @@ class WorkflowEngine {
         });
         await this.resume(result.runId);
     }
+    /** Release the flow-control slot a dispatched step held (if any), by its stepId. */
+    releaseQueueSlot(id) {
+        const queue = this.stepQueue.get(id);
+        if (queue === undefined)
+            return;
+        this.stepQueue.delete(id);
+        this.queues.get(queue)?.release();
+    }
     /** In-memory await path for a remote step with a liveness `timeoutMs` (re-dispatch on timeout). */
-    async callRemoteInMemory(runId, seq, step, input) {
-        if (!this.transport)
+    async callRemoteInMemory(runId, seq, step, input, transport) {
+        if (this.pool.size === 0)
             throw new Error('remote steps require a Transport');
         const validInput = step.input.parse(input);
         const id = (0, protocol_1.stepId)(runId, seq);
@@ -771,15 +757,16 @@ class WorkflowEngine {
             const resultPromise = new Promise((resolve, reject) => {
                 this.pending.set(id, { resolve, reject });
             });
-            await this.transport.dispatch({
+            await this.pool.dispatch({
                 runId,
                 seq,
                 name: step.name,
                 stepId: id,
                 group: step.group,
                 input: validInput,
+                traceparent: this.traceparent?.(),
                 attempt,
-            });
+            }, transport);
             try {
                 const resolution = step.timeoutMs
                     ? await this.awaitWithHeartbeat(id, resultPromise, step.timeoutMs)