@dudousxd/nestjs-durable-core 0.78.0 → 0.80.0

package/dist/index.js

@@ -265,10 +265,16 @@ var FatalError = class extends Error {
     __name(this, "FatalError");
   }
   code;
-  constructor(message, code) {
+  /** Structured, JSON-serializable detail (e.g. validation issues). It travels with the failure to
+  *  the run's error, a parent's `allSettled` item and an onError handler's input. */
+  details;
+  /** Read by the step result builders, so a dispatched step's retry policy honours it too. */
+  retryable = false;
+  constructor(message, code, details) {
     super(message);
     this.name = "FatalError";
     this.code = code;
+    this.details = details;
   }
 };
 var ConcurrencyLimitError = class extends Error {
@@ -346,6 +352,8 @@ var NonDeterminismError = class extends Error {
   static {
     __name(this, "NonDeterminismError");
   }
+  /** Marks the failure as a broken deploy, which an `onError` route never takes. */
+  code = "non_determinism";
   runId;
   seq;
   constructor(runId, seq, expected, recorded) {
@@ -757,6 +765,9 @@ async function runStepHandler(task, handler) {
         ...typeof e?.code === "string" ? {
           code: e.code
         } : {},
+        ...e?.details !== void 0 ? {
+          details: e.details
+        } : {},
         ...typeof e?.retryable === "boolean" ? {
           retryable: e.retryable
         } : {}
@@ -833,11 +844,36 @@ __name(isFailedCompletion, "isFailedCompletion");
 function unwrapCompletion(payload, label) {
   const c = payload;
   if (c && typeof c === "object" && "ok" in c && c.ok === false) {
-    throw new FatalError(`${label} failed: ${c.error}`);
+    throw new FatalError(`${label} failed: ${c.error}`, c.code, c.details);
   }
   return c?.value;
 }
 __name(unwrapCompletion, "unwrapCompletion");
+function failedCompletion(error2) {
+  return {
+    ok: false,
+    error: error2.message,
+    ...error2.code !== void 0 ? {
+      code: error2.code
+    } : {},
+    ...error2.details !== void 0 ? {
+      details: error2.details
+    } : {}
+  };
+}
+__name(failedCompletion, "failedCompletion");
+function completionError(c) {
+  return {
+    message: c.error,
+    ...c.code !== void 0 ? {
+      code: c.code
+    } : {},
+    ...c.details !== void 0 ? {
+      details: c.details
+    } : {}
+  };
+}
+__name(completionError, "completionError");
 
 // src/concurrency.ts
 var DEFAULT_CONCURRENCY_STATUSES = [
@@ -1158,7 +1194,7 @@ function allWaves(size, concurrency) {
   };
 }
 __name(allWaves, "allWaves");
-var ENGINE_CHILD_ID = /^(.+)\.(?:all\.\d+\.(?:w\d+\.)?\d+|child\.\d+)$/;
+var ENGINE_CHILD_ID = /^(.+)\.(?:all\.\d+\.(?:w\d+\.)?\d+|child\.\d+|onError\.\d+)$/;
 function parentRunIdOf(runId) {
   return ENGINE_CHILD_ID.exec(runId)?.[1];
 }
@@ -1243,7 +1279,11 @@ function canonicalizeRegistrations(regs) {
     requires: r.requires ? [
       ...new Set(r.requires)
     ].sort() : null,
-    origin: r.origin ?? null
+    origin: r.origin ?? null,
+    // Conditional, so a registration that states no routes keeps the hash it had before routes existed.
+    ...r.onError !== void 0 ? {
+      onError: canonicalRoutes(r.onError)
+    } : {}
   }));
   const byJson = /* @__PURE__ */ new Map();
   for (const entry of canonical) byJson.set(JSON.stringify(entry), entry);
@@ -1252,6 +1292,20 @@ function canonicalizeRegistrations(regs) {
   ].sort().map((k) => byJson.get(k));
 }
 __name(canonicalizeRegistrations, "canonicalizeRegistrations");
+function canonicalRoutes(routes) {
+  const byJson = /* @__PURE__ */ new Map();
+  for (const r of routes) {
+    const entry = {
+      code: r.code ?? null,
+      workflow: r.workflow
+    };
+    byJson.set(JSON.stringify(entry), entry);
+  }
+  return [
+    ...byJson.keys()
+  ].sort().map((k) => byJson.get(k));
+}
+__name(canonicalRoutes, "canonicalRoutes");
 function canonicalizeForHash(d) {
   const set = /* @__PURE__ */ __name((xs) => [
     ...new Set(xs)
@@ -1286,13 +1340,13 @@ function fnv1a64Hex(input) {
   const OFFSET = 0xcbf29ce484222325n;
   const PRIME = 0x100000001b3n;
   const MASK = 0xffffffffffffffffn;
-  let hash = OFFSET;
+  let hash2 = OFFSET;
   for (let i = 0; i < input.length; i += 1) {
-    hash ^= BigInt(input.charCodeAt(i) & 255);
-    hash ^= BigInt(input.charCodeAt(i) >> 8 & 255);
-    hash = hash * PRIME & MASK;
+    hash2 ^= BigInt(input.charCodeAt(i) & 255);
+    hash2 ^= BigInt(input.charCodeAt(i) >> 8 & 255);
+    hash2 = hash2 * PRIME & MASK;
   }
-  return hash.toString(16).padStart(16, "0");
+  return hash2.toString(16).padStart(16, "0");
 }
 __name(fnv1a64Hex, "fnv1a64Hex");
 function descriptorHash(descriptor) {
@@ -1408,6 +1462,20 @@ function aggregateAnnouncements(descriptors) {
     const requires = [
       ...requiresByJson.values()
     ].sort((x, y) => x.join(",").localeCompare(y.join(",")));
+    const routesByJson = /* @__PURE__ */ new Map();
+    for (const a of announcements) {
+      if (a.onError === void 0) continue;
+      const canonical = canonicalRoutes(a.onError).map((r) => r.code === null ? {
+        workflow: r.workflow
+      } : {
+        code: r.code,
+        workflow: r.workflow
+      });
+      routesByJson.set(JSON.stringify(canonical), canonical);
+    }
+    const onError = [
+      ...routesByJson.keys()
+    ].sort().map((k) => routesByJson.get(k) ?? []);
     const disagreements = [];
     if (groups.length > 1) disagreements.push({
       axis: "group",
@@ -1423,6 +1491,14 @@ function aggregateAnnouncements(descriptors) {
         values: requires.map((r) => r.join(","))
       });
     }
+    if (onError.length > 1) {
+      disagreements.push({
+        axis: "onError",
+        values: [
+          ...routesByJson.keys()
+        ].sort()
+      });
+    }
     const first = announcements[0];
     if (!first) continue;
     out.push({
@@ -1435,6 +1511,7 @@ function aggregateAnnouncements(descriptors) {
       groups,
       origins,
       requires,
+      onError,
       runtimes: [
         ...new Set(announcements.map((a) => a.runtime))
       ].sort(),
@@ -1468,6 +1545,7 @@ function observedAnnouncements(heartbeats, covered) {
     ],
     origins: [],
     requires: [],
+    onError: [],
     runtimes: [],
     instances: [
       ...instances
@@ -1826,6 +1904,13 @@ function planDispatch(requires, liveDescriptors, cp, token) {
 __name(planDispatch, "planDispatch");
 
 // src/interfaces.ts
+function checkpointNameKind(name) {
+  if (name.startsWith("signal:child:")) return "child";
+  if (name.startsWith("fan:")) return "folded";
+  if (name.startsWith("spawn:")) return "spawn";
+  return "step";
+}
+__name(checkpointNameKind, "checkpointNameKind");
 var TERMINAL_RUN_STATUSES = [
   "completed",
   "failed",
@@ -1838,6 +1923,71 @@ function isNonEmptyRunScope(scope) {
 }
 __name(isNonEmptyRunScope, "isNonEmptyRunScope");
 
+// src/on-error.ts
+var UNROUTABLE_CODES = /* @__PURE__ */ new Set([
+  "non_determinism",
+  "nested_workflow_call"
+]);
+var HANDOFF_SEQ_BASE = 1e9;
+var onErrorRunId = /* @__PURE__ */ __name((runId, attempt) => `${runId}.onError.${attempt}`, "onErrorRunId");
+var handoffName = /* @__PURE__ */ __name((handlerId) => `signal:child:${handlerId}`, "handoffName");
+var retiredHandoffName = /* @__PURE__ */ __name((handlerId) => `onError:retried:${handlerId}`, "retiredHandoffName");
+function handoffHandlerId(runId, cp) {
+  const prefix = `signal:child:${runId}.onError.`;
+  return cp.seq >= HANDOFF_SEQ_BASE && cp.name.startsWith(prefix) ? cp.name.slice("signal:child:".length) : void 0;
+}
+__name(handoffHandlerId, "handoffHandlerId");
+function liveHandoff(runId, checkpoints) {
+  let live;
+  for (const cp of checkpoints) {
+    if (handoffHandlerId(runId, cp) !== void 0 && (!live || cp.seq > live.seq)) live = cp;
+  }
+  return live;
+}
+__name(liveHandoff, "liveHandoff");
+function routeFor(routes, error2) {
+  if (!routes?.length || error2.code && UNROUTABLE_CODES.has(error2.code)) return void 0;
+  return (error2.code ? routes.find((r) => r.code === error2.code) : void 0) ?? routes.find((r) => r.code === void 0);
+}
+__name(routeFor, "routeFor");
+function assertRoutes(workflow, routes) {
+  const seen = /* @__PURE__ */ new Set();
+  for (const r of routes) {
+    if (seen.has(r.code)) {
+      throw new Error(r.code === void 0 ? `workflow ${workflow}: more than one onError route without a code` : `workflow ${workflow}: more than one onError route for code "${r.code}"`);
+    }
+    seen.add(r.code);
+  }
+}
+__name(assertRoutes, "assertRoutes");
+function codeOf(err) {
+  const e = err;
+  return [
+    e?.code,
+    e?.stepError?.code,
+    e?.error?.code
+  ].find((c) => typeof c === "string");
+}
+__name(codeOf, "codeOf");
+function detailsOf(err) {
+  const e = err;
+  return e?.details ?? e?.stepError?.details ?? e?.error?.details;
+}
+__name(detailsOf, "detailsOf");
+function codeAndDetails(err) {
+  const code = codeOf(err);
+  const details = detailsOf(err);
+  return {
+    ...code !== void 0 ? {
+      code
+    } : {},
+    ...details !== void 0 ? {
+      details
+    } : {}
+  };
+}
+__name(codeAndDetails, "codeAndDetails");
+
 // src/tenant-group.ts
 function tenantGroup(baseGroup, tenant) {
   return tenant !== void 0 && tenant !== "" && tenant !== "default" ? `${baseGroup}@${tenant}` : baseGroup;
@@ -2897,7 +3047,8 @@ function createWorkflowCtx(host, runId, compensations, workflow = "", searchAttr
             kind: "local",
             error: {
               message: err instanceof Error ? err.message : String(err),
-              stack: err instanceof Error ? err.stack : void 0
+              stack: err instanceof Error ? err.stack : void 0,
+              ...codeAndDetails(err)
             },
             events,
             attempts: attempt,
@@ -3153,12 +3304,12 @@ function createWorkflowCtx(host, runId, compensations, workflow = "", searchAttr
       }
     ]));
   }, "child");
-  const all = /* @__PURE__ */ __name(async (workflow2, inputs, opts) => {
+  const fan = /* @__PURE__ */ __name(async (workflow2, inputs, opts, settle) => {
     if (inputs.length === 0) return [];
     if (stepNameOf(workflow2) !== void 0) {
-      return allSteps(workflow2, inputs, opts);
+      return allSteps(workflow2, inputs, opts, settle);
     }
-    const mode = opts?.mode ?? "waitAll";
+    const mode = settle ? "waitAll" : opts?.mode ?? "waitAll";
     const name = workflowName(workflow2);
     const waves = allWaves(inputs.length, opts?.concurrency);
     const positions = inputs.map(() => pos.next());
@@ -3280,24 +3431,45 @@ function createWorkflowCtx(host, runId, compensations, workflow = "", searchAttr
       throw new WorkflowSuspended(void 0, fanJoin(active.map((k) => allWait(runId, firstSeq, waves ? k : void 0))));
     }
     const outputs = [];
+    const results = [];
     const failures = [];
     for (let i = 0; i < inputs.length; i += 1) {
       const c = existing[i]?.output;
       if (c && typeof c === "object" && "ok" in c && c.ok === false) {
+        const { message, ...rest } = completionError({
+          ...c,
+          error: c.error ?? "unknown"
+        });
+        const failure = {
+          id: id(i),
+          error: message,
+          ...rest
+        };
         failures.push({
           index: i,
-          id: id(i),
-          error: c.error ?? "unknown"
+          ...failure
+        });
+        results.push({
+          status: "rejected",
+          ...failure
         });
         outputs.push(void 0);
       } else {
-        outputs.push(c?.value);
+        const value = c?.value;
+        results.push({
+          status: "fulfilled",
+          value
+        });
+        outputs.push(value);
       }
     }
+    if (settle) return results;
     if (failures.length > 0) throw new GatherError(failures, inputs.length);
     return outputs;
-  }, "all");
-  const allSteps = /* @__PURE__ */ __name(async (ref, inputs, opts) => {
+  }, "fan");
+  const all = /* @__PURE__ */ __name((workflow2, inputs, opts) => fan(workflow2, inputs, opts, false), "all");
+  const allSettled = /* @__PURE__ */ __name((workflow2, inputs, opts) => fan(workflow2, inputs, opts, true), "allSettled");
+  const allSteps = /* @__PURE__ */ __name(async (ref, inputs, opts, settle) => {
     if (opts?.queue !== void 0) {
       throw new Error("ctx.all: a flow-control queue is not supported on a step fan");
     }
@@ -3308,7 +3480,8 @@ function createWorkflowCtx(host, runId, compensations, workflow = "", searchAttr
     };
     const positions = inputs.map(() => pos.next());
     return host.callRemoteAll(runId, positions, def, inputs, {
-      mode: opts?.mode ?? "waitAll",
+      mode: settle ? "waitAll" : opts?.mode ?? "waitAll",
+      settle,
       concurrency: opts?.concurrency,
       transport: opts?.transport,
       admission: {
@@ -3626,6 +3799,7 @@ function createWorkflowCtx(host, runId, compensations, workflow = "", searchAttr
     task: guard("task", task),
     child: guard("child", child),
     all: guard("all", all),
+    allSettled: guard("allSettled", allSettled),
     startChild: guard("startChild", startChild),
     breakpoint: guard("breakpoint", breakpoint),
     webhook: guardedWebhook,
@@ -3651,9 +3825,11 @@ function mergeTags(staticTags, runTags) {
 __name(mergeTags, "mergeTags");
 var BREAKPOINT = "breakpoint";
 var isBreakpoint = /* @__PURE__ */ __name((cp) => cp.status === "pending" && cp.name.startsWith(BREAKPOINT), "isBreakpoint");
+var ANNOUNCED_ROUTES_TTL_MS = 5e3;
 var VERSION_UNDECLARED_TAG = "version:undeclared";
 var versionKey = /* @__PURE__ */ __name((name, version) => `${name}@${version}`, "versionKey");
 var STEP_FAN_CHUNK = 200;
+var RESULT_BATCH_MAX = 256;
 function shuffle(items) {
   for (let i = items.length - 1; i > 0; i -= 1) {
     const j = Math.floor(Math.random() * (i + 1));
@@ -3683,6 +3859,16 @@ function isNewerVersion(a, b) {
   }) > 0;
 }
 __name(isNewerVersion, "isNewerVersion");
+var SETTLE_ELIGIBLE_STATUSES = [
+  "pending",
+  "running",
+  "suspended",
+  "blocked",
+  "failed",
+  "cancelling"
+];
+var SUSPEND_ELIGIBLE_STATUSES = SETTLE_ELIGIBLE_STATUSES.filter((status) => status !== "cancelling");
+var RECOVERY_BATCH = 500;
 var NamespaceMismatch = class NamespaceMismatch2 extends Error {
   static {
     __name(this, "NamespaceMismatch");
@@ -3761,6 +3947,8 @@ var WorkflowEngine = class {
   updateValidators = /* @__PURE__ */ new Map();
   /** Runs being cancelled WITH saga compensation — see `cancel({ compensate: true })`. */
   cancelRequested = /* @__PURE__ */ new Set();
+  /** {@link announcedRoutes} reads, by group + workflow + version. */
+  announcedRoutesCache = /* @__PURE__ */ new Map();
   /**
   * Seqs of the dispatched remote steps a turn suspended on, by run, while that turn executes on
   * THIS instance — so the turn can re-check them once its lease is released (see
@@ -3780,8 +3968,22 @@ var WorkflowEngine = class {
   awaitedStepFans = /* @__PURE__ */ new Map();
   /** Step fans whose join barrier this instance opened, by `runId:firstSeq` — see {@link settledHere}. */
   stepFansFiredHere = /* @__PURE__ */ new Map();
+  /**
+  * The in-process execution gate, by run id: the execution of the run currently in flight in THIS
+  * process (`tail`), and at most one resume queued behind it (`queued`). See {@link resumeRun}.
+  *
+  * The run lease is keyed by engine instance, so it fences OTHER processes only: to the store, a
+  * second execution in this same process holds the very same lease (and {@link execute} skips
+  * `tryLockRun` when its snapshot says this instance holds it). Without this gate, a resume landing
+  * while this process was still executing the run (a signal, a late step result, a child wake, an
+  * explicit `resume()`) went straight past the lease and ran the body concurrently: a
+  * not-yet-checkpointed step executed twice.
+  */
+  runGates = /* @__PURE__ */ new Map();
   /** The tail of each run's queue of results being handled on this instance — see {@link serializedPerRun}. */
   resultChains = /* @__PURE__ */ new Map();
+  /** Each run's results waiting to be settled on this instance — see {@link completeRemoteResult}. */
+  resultQueues = /* @__PURE__ */ new Map();
   /** Flow-control admission backend for remote steps (see {@link registerQueue}). */
   admission;
   /** Runs on THIS instance blocked on admission, by queue — woken early on a freed-slot signal. */
@@ -3988,6 +4190,7 @@ var WorkflowEngine = class {
       name,
       version,
       fn,
+      onError: this.checkedRoutes(name, opts?.onError),
       tags: opts?.tags,
       singleton: opts?.singleton,
       concurrency: opts?.concurrency,
@@ -4057,6 +4260,7 @@ var WorkflowEngine = class {
       executionTimeoutMs: opts.executionTimeout != null ? parseDuration(opts.executionTimeout) : void 0,
       validateInput: opts.validateInput,
       requires: opts.requires,
+      onError: this.checkedRoutes(name, opts.onError),
       remote: {
         group: opts.group,
         executor: opts.executor
@@ -4244,11 +4448,16 @@ var WorkflowEngine = class {
     if (registered.singleton) {
       await this.singletons.assertCapacity(name, registered.singleton, input);
     }
+    const startAtMs = opts?.startAt instanceof Date ? opts.startAt.getTime() : opts?.startAt;
+    const delayed = startAtMs !== void 0 && Number.isFinite(startAtMs) && startAtMs > this.clock();
     const run = {
       id: runId,
       workflow: name,
       workflowVersion: registered.version,
-      status: "pending",
+      status: delayed ? "suspended" : "pending",
+      ...delayed ? {
+        wakeAt: startAtMs
+      } : {},
       namespace: opts?.namespace ?? this.namespace,
       // DERIVED from the registration, exactly like `namespace` is derived from this engine — there is
       // deliberately no `opts.origin`, so a caller cannot label its run as another package's work.
@@ -4274,6 +4483,18 @@ var WorkflowEngine = class {
         error: winner.error
       };
     }
+    if (delayed) {
+      this.emit({
+        type: "run.suspended",
+        runId,
+        workflow: name,
+        namespace: run.namespace
+      });
+      return {
+        runId,
+        status: "suspended"
+      };
+    }
     await this.runDispatcher.dispatch(runId);
     if (this.controlPlane) {
       void this.controlPlane.publishControl({
@@ -4311,7 +4532,104 @@ var WorkflowEngine = class {
   resumeImplicit(runId) {
     return this.resumeRun(runId, false);
   }
-  async resumeRun(runId, allowFailed) {
+  /**
+  * One execution of a run at a time in this process. A resume that lands while the run executes
+  * here neither joins that execution (the double-run) nor no-ops against its held lease (a lost
+  * wake: its checkpoint may postdate the running turn's snapshot): it is QUEUED, and re-drives the
+  * run once the running execution has settled and released its lease, re-reading the run then
+  * (terminal: echoed, never re-executed). Resumes landing while one is already queued coalesce
+  * into it: it has not started yet, so it will observe every write they made. Their callers get
+  * the queued execution's result, i.e. the run's state after their own wake was applied.
+  *
+  * Across processes, exclusion stays the lease's job: {@link execute} still `tryLockRun`s when it
+  * does not hold the lease.
+  */
+  resumeRun(runId, allowFailed) {
+    const gate = this.runGates.get(runId);
+    if (!gate) {
+      const tail = this.resumeRunNow(runId, allowFailed);
+      const fresh = {
+        tail
+      };
+      this.runGates.set(runId, fresh);
+      this.closeGateAfter(runId, fresh, tail);
+      return tail;
+    }
+    let queued = gate.queued;
+    if (queued) {
+      queued.allowFailed ||= allowFailed;
+    } else {
+      const entry = {
+        allowFailed
+      };
+      const result = gate.tail.catch(() => void 0).then(() => {
+        if (gate.queued === entry) gate.queued = void 0;
+        return this.resumeRunNow(runId, entry.allowFailed);
+      });
+      entry.result = result;
+      queued = entry;
+      gate.queued = queued;
+      gate.tail = result;
+      this.closeGateAfter(runId, gate, result);
+      void this.track(result).catch(() => void 0);
+    }
+    if (currentWorkflowCtx()?.runId === runId) return this.echoCurrentStatus(runId);
+    return queued.result;
+  }
+  /** Drop `gate` once `tail` settles, unless another execution queued behind it meanwhile. */
+  closeGateAfter(runId, gate, tail) {
+    const close = /* @__PURE__ */ __name(() => {
+      if (this.runGates.get(runId) === gate && gate.tail === tail) this.runGates.delete(runId);
+    }, "close");
+    void tail.then(close, close);
+  }
+  /**
+  * Release a run's lease AFTER its outcome is decided (persisted, or about to be returned). Best
+  * effort, and that matters most on the failure path: the thing that failed the run is frequently
+  * the very store this release talks to (a step that violated a DB constraint leaves Postgres
+  * refusing every further statement on that connection), so the release throws too. A throwing
+  * release in a `finally` REPLACES the run's real outcome with a lease write nobody asked about,
+  * skips the post-turn re-drives after it, and, for a turn driven as a background resume, surfaces
+  * as an unhandled rejection instead of a failed run.
+  *
+  * Letting it go is safe: the lease expires on its own and `recoverIncomplete` re-drives whatever
+  * is still held. A lost release costs one lease window of latency; a masked error costs the
+  * diagnosis.
+  */
+  async releaseLeaseBestEffort(runId) {
+    try {
+      await this.store.releaseRunLock(runId);
+    } catch (err) {
+      console.warn(`[nestjs-durable] releasing the lease of run ${runId} failed (it expires on its own and recovery re-drives): ${err instanceof Error ? err.message : String(err)}`);
+    }
+  }
+  /**
+  * The run's persisted state as a {@link RunResult}, without executing anything: the answer to a
+  * resume issued from inside the run's own execution, and to a settle whose compare-and-set lost
+  * (the path that won already emitted and notified). A run that is gone reads as `cancelled`, the
+  * safest terminal reading, never a fabricated success.
+  */
+  async echoCurrentStatus(runId) {
+    const run = await this.store.getRun(runId);
+    return {
+      runId,
+      status: run?.status ?? "cancelled",
+      output: run?.output,
+      error: run?.error
+    };
+  }
+  /**
+  * The engine's stand-in for {@link StateStore.updateRunIf} on a store without it, called with the
+  * store as `this`: read-then-write, which narrows the race to the gap between the read and the
+  * write (the pre-CAS behavior, now applied to every settle) but cannot close it.
+  */
+  async updateRunIfFallback(runId, expected, patch) {
+    const current = await this.getRun(runId);
+    if (!current || !expected.includes(current.status)) return false;
+    await this.updateRun(runId, patch);
+    return true;
+  }
+  async resumeRunNow(runId, allowFailed) {
     const run = await this.store.getRun(runId);
     if (!run) throw new Error(`run ${runId} not found`);
     if (run.status === "cancelled" || run.status === "completed" || run.status === "dead" || run.status === "failed" && !allowFailed) {
@@ -4617,7 +4935,7 @@ var WorkflowEngine = class {
   async recoverIncomplete(nowMs = this.clock()) {
     if (this.draining) return [];
     const results = [];
-    for (const listed of await this.store.listIncompleteRuns(this.namespace)) {
+    for (const listed of await this.orphanedRuns(nowMs)) {
       if (this.draining) break;
       const acquired = await this.store.tryLockRun(listed.id, this.instanceId, nowMs + this.leaseMs, nowMs);
       if (!acquired) continue;
@@ -4663,6 +4981,21 @@ var WorkflowEngine = class {
     return results;
   }
   /**
+  * The runs {@link recoverIncomplete} should try to reclaim: incomplete runs whose lease is free or
+  * expired, never the healthy ones a live turn is renewing. Every worker runs recovery on every
+  * timer tick, so listing every running run and lock-probing each (one doomed UPDATE per healthy
+  * run per worker per tick) was pure database load. The store answers it directly when it can
+  * ({@link StateStore.listOrphanedRuns}, bounded by {@link RECOVERY_BATCH}); else the full list is
+  * filtered here, which still spares the probes. Shuffled, like the due-timer pass, so instances
+  * recovering at once spread over the batch instead of contending for the same leases.
+  */
+  async orphanedRuns(nowMs) {
+    if (this.store.listOrphanedRuns) {
+      return shuffle(await this.store.listOrphanedRuns(nowMs, RECOVERY_BATCH, this.namespace));
+    }
+    return shuffle((await this.store.listIncompleteRuns(this.namespace)).filter((run) => run.lockedUntil == null || run.lockedUntil <= nowMs));
+  }
+  /**
   * The remote step this run dispatched and is still awaiting a result for, if any — the signature
   * that tells {@link recoverIncomplete} "this run's work is ON THE TRANSPORT, not in a crashed
   * process". A `pending` remote checkpoint is written BEFORE the dispatch and settled by the
@@ -4834,6 +5167,24 @@ var WorkflowEngine = class {
     if (resurrecting) {
       const isFailureCompletion = /* @__PURE__ */ __name((v) => typeof v === "object" && v !== null && v.ok === false, "isFailureCompletion");
       for (const cp of await this.store.listCheckpoints(runId)) {
+        const handlerId = handoffHandlerId(runId, cp);
+        if (handlerId !== void 0) {
+          await this.store.takeSignalWaiter(`child:${handlerId}`);
+          await this.store.saveCheckpoint({
+            ...cp,
+            name: retiredHandoffName(handlerId),
+            status: "completed"
+          });
+          const tags = run.tags?.filter((t) => !t.startsWith("onError:"));
+          if (tags && tags.length !== run.tags?.length) {
+            await this.store.updateRun(runId, {
+              tags,
+              updatedAt: /* @__PURE__ */ new Date()
+            });
+            run.tags = tags;
+          }
+          continue;
+        }
         if (cp.status === "failed") {
           await this.store.saveCheckpoint({
             ...cp,
@@ -4978,13 +5329,13 @@ var WorkflowEngine = class {
   * `start` when a call site needs the outcome — `await start(...); const r = await waitForRun(id)`.
   */
   waitForRun(runId, opts) {
-    const TERMINAL = /* @__PURE__ */ new Set([
+    const TERMINAL2 = /* @__PURE__ */ new Set([
       "completed",
       "failed",
       "cancelled",
       "dead"
     ]);
-    const isSettled = /* @__PURE__ */ __name((run) => opts?.until === "terminal" ? TERMINAL.has(run.status) : run.status !== "pending" && run.status !== "running" && run.status !== "cancelling" && !(run.status === "suspended" && run.awaitingDecisionTaskId !== void 0), "isSettled");
+    const isSettled = /* @__PURE__ */ __name((run) => opts?.until === "terminal" ? TERMINAL2.has(run.status) : run.status !== "pending" && run.status !== "running" && run.status !== "cancelling" && !(run.status === "suspended" && run.awaitingDecisionTaskId !== void 0), "isSettled");
     const toResult = /* @__PURE__ */ __name((run) => ({
       runId,
       status: run.status,
@@ -5239,10 +5590,7 @@ var WorkflowEngine = class {
   async redriveSettledStepFans(runId, fans) {
     for (const fan of fans) {
       if (this.stepFansFiredHere.has(`${runId}:${fan.from}`)) continue;
-      const pending = this.store.countCheckpoints ? await this.store.countCheckpoints(runId, fan.from, fan.to, [
-        "pending"
-      ]) : (await this.store.listCheckpoints(runId)).filter((c) => c.seq >= fan.from && c.seq <= fan.to && c.status === "pending").length;
-      if (pending > 0) continue;
+      if (await this.stepFanPending(runId, fan.from, fan.to) > 0) continue;
       const nowMs = this.clock();
       if (!await this.store.tryLockRun(runId, this.instanceId, nowMs + this.leaseMs, nowMs)) {
         return;
@@ -5952,6 +6300,209 @@ var WorkflowEngine = class {
   * us back here (a remote step finished, a timer fired) goes through `resume` like any TS workflow.
   */
   /**
+  * The registered onError routes whose handler workflow nothing can run: not registered on this
+  * engine, and no live worker group serves it. Meant for a boot-time warning: a typo in a handler's
+  * name would otherwise surface only when the first run fails.
+  */
+  async unservedOnErrorRoutes() {
+    const unserved = [];
+    let groups;
+    for (const registered of this.workflows.values()) {
+      for (const route of registered.onError ?? []) {
+        if (this.latest.has(route.workflow)) continue;
+        groups ??= await this.pool.listWorkerGroups().catch(() => []);
+        const token = sanitizeQueueToken(route.workflow);
+        if (groups.some((g) => g === token || g.startsWith(`${token}@`))) continue;
+        unserved.push({
+          workflow: registered.name,
+          route
+        });
+      }
+    }
+    return unserved;
+  }
+  /** Validate a workflow's {@link OnErrorRoute}s at registration. */
+  checkedRoutes(name, routes) {
+    if (!routes?.length) return void 0;
+    assertRoutes(name, routes);
+    return routes;
+  }
+  /**
+  * Route a failed run to its {@link OnErrorRoute} handler, read from the workflow's local registration
+  * or, for one served by convention, from what its live workers announce. Starts the handler run and
+  * waits for it the way a parent waits for a `ctx.child`: a `child:<handlerId>` waiter plus a
+  * `running` placeholder checkpoint carrying the handler's name and input, so the handler's terminal
+  * state wakes the run (buffered and reconciled like any child). From then on every execution of the
+  * run only follows the handler ({@link followHandoff}); the body never runs again. Undefined when no
+  * route matches: the run fails as usual.
+  *
+  * The placeholder sits past every seq the body used and is left out of {@link remoteHistory}.
+  */
+  async routeFailure(run, error2) {
+    const registered = this.workflows.get(versionKey(run.workflow, run.workflowVersion));
+    const routes = registered ? registered.onError : await this.announcedRoutes(run, error2);
+    const route = routeFor(routes, error2);
+    if (!route) return void 0;
+    const checkpoints = await this.handoffCheckpoints(run.id);
+    const live = liveHandoff(run.id, checkpoints);
+    if (live) return this.followHandoff(run, live);
+    const attempt = checkpoints.length;
+    const seq = HANDOFF_SEQ_BASE + attempt;
+    const handlerId = onErrorRunId(run.id, attempt);
+    const handler = {
+      workflow: route.workflow,
+      input: {
+        runId: run.id,
+        workflow: run.workflow,
+        input: run.input,
+        error: error2
+      }
+    };
+    await this.store.putSignalWaiter({
+      token: `child:${handlerId}`,
+      runId: run.id,
+      seq
+    });
+    await this.store.saveCheckpoint({
+      ...instantCheckpoint({
+        runId: run.id,
+        seq,
+        name: handoffName(handlerId),
+        kind: "signal",
+        status: "running"
+      }),
+      input: handler
+    });
+    const tag = `onError:${route.code ?? "*"}`;
+    if (!run.tags?.includes(tag)) {
+      const tags = [
+        ...run.tags ?? [],
+        tag
+      ];
+      await this.store.updateRun(run.id, {
+        tags,
+        updatedAt: /* @__PURE__ */ new Date()
+      });
+      run.tags = tags;
+    }
+    this.emit({
+      type: "run.routed",
+      runId: run.id,
+      workflow: run.workflow,
+      namespace: run.namespace,
+      name: route.workflow,
+      error: error2
+    });
+    this.startChildDeferred(handler.workflow, handler.input, handlerId, {
+      namespace: run.namespace
+    });
+    return {
+      kind: "suspended"
+    };
+  }
+  /** A run's handoff checkpoints, live and retired: the only ones at {@link HANDOFF_SEQ_BASE} and up. */
+  async handoffCheckpoints(runId) {
+    const handoffs = await this.store.listCheckpointsByNamePrefix?.(runId, [
+      `signal:child:${runId}.onError.`,
+      `onError:retried:${runId}.onError.`
+    ]);
+    return (handoffs ?? await this.store.listCheckpoints(runId)).filter((cp) => cp.seq >= HANDOFF_SEQ_BASE);
+  }
+  /**
+  * Where a run that handed its failure off stands: the handler's outcome once it settled (from the
+  * wake's checkpoint, or off the handler run when the wake never arrived), else still waiting. Starts
+  * the handler if the turn that handed off crashed before its deferred start ran.
+  */
+  async followHandoff(run, handoff) {
+    if (handoff.status === "completed") {
+      const outcome = handoff.output;
+      return outcome.ok ? {
+        kind: "completed",
+        output: outcome.value
+      } : {
+        kind: "failed",
+        error: completionError(outcome)
+      };
+    }
+    const handlerId = handoffHandlerId(run.id, handoff);
+    const handlerRun = await this.store.getRun(handlerId);
+    if (handlerRun?.status === "completed") return {
+      kind: "completed",
+      output: handlerRun.output
+    };
+    if (handlerRun && TERMINAL_RUN_STATUSES.includes(handlerRun.status)) {
+      return {
+        kind: "failed",
+        error: handlerRun.error ?? {
+          message: `onError handler ${handlerId} ${handlerRun.status}`
+        }
+      };
+    }
+    if (!handlerRun) {
+      const handler = handoff.input;
+      this.startChildDeferred(handler.workflow, handler.input, handlerId, {
+        namespace: run.namespace
+      });
+    }
+    return {
+      kind: "suspended"
+    };
+  }
+  /**
+  * The onError routes the live workers announce for a workflow the engine serves by convention: the
+  * one set they agree on, or none when they state none or cannot be read. Workers that announce
+  * different sets (a rollout in progress) route nothing, and the run's failure is reported as a
+  * `run.unrouted` event. Announcers of another version of the workflow are ignored. The read is
+  * cached for {@link ANNOUNCED_ROUTES_TTL_MS}, so a burst of failures costs one descriptor read.
+  */
+  async announcedRoutes(run, error2) {
+    const group = tenantGroup(sanitizeQueueToken(run.workflow), run.namespace);
+    const key = `${group}
+${run.workflow}
+${run.workflowVersion}`;
+    const now = this.clock();
+    let cached = this.announcedRoutesCache.get(key);
+    if (!cached || now - cached.at > ANNOUNCED_ROUTES_TTL_MS) {
+      cached = {
+        at: now,
+        sets: await this.readAnnouncedRoutes(run, group)
+      };
+      this.announcedRoutesCache.set(key, cached);
+    }
+    if (cached.sets.length > 1) {
+      this.emit({
+        type: "run.unrouted",
+        runId: run.id,
+        workflow: run.workflow,
+        namespace: run.namespace,
+        error: error2,
+        reason: `live workers announce ${cached.sets.length} different onError route sets`
+      });
+      return void 0;
+    }
+    return cached.sets[0];
+  }
+  /** Every distinct onError route set the live workers of `group` announce for the run's workflow. */
+  async readAnnouncedRoutes(run, group) {
+    const transport = await this.pool.transportWithLiveGroup(group);
+    if (!transport?.readWorkerDescriptors) return [];
+    let descriptors;
+    try {
+      descriptors = await transport.readWorkerDescriptors(group);
+    } catch {
+      return [];
+    }
+    const sets = /* @__PURE__ */ new Map();
+    for (const announced of aggregateAnnouncements(descriptors)) {
+      if (announced.name !== run.workflow) continue;
+      if (announced.version !== void 0 && announced.version !== run.workflowVersion) continue;
+      for (const routes of announced.onError) sets.set(JSON.stringify(routes), routes);
+    }
+    return [
+      ...sets.values()
+    ];
+  }
+  /**
   * Apply a terminal/suspended transition once: persist the new run status, emit the matching
   * lifecycle event, and (on completion/failure) wake a waiting parent. Both the TS and remote
   * executors funnel their completed/failed/suspended outcomes through here so a status update can
@@ -5961,12 +6512,13 @@ var WorkflowEngine = class {
   async settleRun(run, outcome) {
     const updatedAt = /* @__PURE__ */ new Date();
     if (outcome.kind === "completed") {
-      await this.store.updateRun(run.id, {
+      const applied2 = await (this.store.updateRunIf ?? this.updateRunIfFallback).call(this.store, run.id, SETTLE_ELIGIBLE_STATUSES, {
         status: "completed",
         output: outcome.output,
         error: void 0,
         updatedAt
       });
+      if (!applied2) return this.echoCurrentStatus(run.id);
       this.emit({
         type: "run.completed",
         runId: run.id,
@@ -5985,42 +6537,36 @@ var WorkflowEngine = class {
       };
     }
     if (outcome.kind === "failed") {
-      await this.store.updateRun(run.id, {
+      const handed = await this.routeFailure(run, outcome.error);
+      if (handed && handed.kind !== "failed") return this.settleRun(run, handed);
+      const error2 = handed?.error ?? outcome.error;
+      const applied2 = await (this.store.updateRunIf ?? this.updateRunIfFallback).call(this.store, run.id, SETTLE_ELIGIBLE_STATUSES, {
         status: "failed",
-        error: outcome.error,
+        error: error2,
         updatedAt
       });
+      if (!applied2) return this.echoCurrentStatus(run.id);
       this.emit({
         type: "run.failed",
         runId: run.id,
         workflow: run.workflow,
         namespace: run.namespace,
-        error: outcome.error
-      });
-      void this.notifyParent(run.id, {
-        ok: false,
-        error: outcome.error.message
+        error: error2
       });
+      void this.notifyParent(run.id, failedCompletion(error2));
       return {
         runId: run.id,
         status: "failed",
-        error: outcome.error
+        error: error2
       };
     }
     if (outcome.kind === "blocked") return this.parkBlocked(run, outcome.blocked);
-    const latest = await this.store.getRun(run.id);
-    if (latest?.status === "cancelled") {
-      return {
-        runId: run.id,
-        status: "cancelled",
-        error: latest.error
-      };
-    }
-    await this.store.updateRun(run.id, {
+    const applied = await (this.store.updateRunIf ?? this.updateRunIfFallback).call(this.store, run.id, SUSPEND_ELIGIBLE_STATUSES, {
       status: "suspended",
       wakeAt: this.reconcileWakeAt(outcome.wakeAt),
       updatedAt
     });
+    if (!applied) return this.echoCurrentStatus(run.id);
     this.emit({
       type: "run.suspended",
       runId: run.id,
@@ -6067,7 +6613,7 @@ var WorkflowEngine = class {
           from: this.instanceId
         }).catch(() => void 0);
       }
-      await this.store.releaseRunLock(run.id);
+      await this.releaseLeaseBestEffort(run.id);
       return {
         runId: run.id,
         status: "cancelled",
@@ -6093,7 +6639,14 @@ var WorkflowEngine = class {
     }
     const wfPlan = await this.planRoute(tenantGroup(sanitizeQueueToken(remote.group), void 0), registered.requires);
     if (wfPlan?.status === "blocked") return this.parkBlocked(run, wfPlan);
-    const history = await this.remoteHistory(run.id);
+    const checkpoints = await this.store.listCheckpoints(run.id);
+    const handoff = liveHandoff(run.id, checkpoints);
+    if (handoff) {
+      const result = await this.settleRun(run, await this.followHandoff(run, handoff));
+      await this.releaseLeaseBestEffort(run.id);
+      return result;
+    }
+    const history = await this.remoteHistory(run.id, checkpoints);
     if (remote.executor.dispatch) {
       const taskId = `${run.id}:wf:${globalThis.crypto.randomUUID().slice(0, 8)}`;
       const silenceMs = this.remoteAdvanceSilenceMs;
@@ -6128,7 +6681,7 @@ var WorkflowEngine = class {
       decision = silenceMs != null ? await this.awaitWithLivenessDeadline(run.id, advance, silenceMs, () => new RemoteWorkflowTimeout(run.id, silenceMs)) : await advance;
     } catch (err) {
       if (err instanceof RemoteWorkflowTimeout) {
-        await this.store.releaseRunLock(run.id);
+        await this.releaseLeaseBestEffort(run.id);
         return {
           runId: run.id,
           status: "running"
@@ -6417,8 +6970,8 @@ var WorkflowEngine = class {
     }
   }
   /** The run's resolved durable ops as replay inputs: completed/failed steps + elapsed timers. */
-  async remoteHistory(runId) {
-    const checkpoints = await this.store.listCheckpoints(runId);
+  async remoteHistory(runId, listed) {
+    const checkpoints = listed ?? await this.store.listCheckpoints(runId);
     const kindOf = {
       remote: "call",
       local: "step",
@@ -6428,15 +6981,14 @@ var WorkflowEngine = class {
     const events = [];
     for (const cp of checkpoints) {
       if (cp.status === "completed" || cp.status === "failed") {
+        if (cp.seq >= HANDOFF_SEQ_BASE) continue;
         if (cp.kind === "signal" && cp.name.startsWith("signal:child:")) {
           const completion = cp.output;
           events.push({
             seq: cp.seq,
             kind: "child",
             output: completion?.ok ? completion.value : void 0,
-            error: completion && completion.ok === false ? {
-              message: completion.error
-            } : void 0
+            error: completion && completion.ok === false ? completionError(completion) : void 0
           });
           continue;
         }
@@ -6717,6 +7269,10 @@ var WorkflowEngine = class {
     this.awaitedStepFans.set(run.id, awaitedFans);
     const ctx = createWorkflowCtx(this.ctxHostFor(replay, run.namespace), run.id, compensations, run.workflow, registered?.searchAttributesSchema);
     try {
+      const handoff = liveHandoff(run.id, snapshot);
+      if (handoff && !this.cancelRequested.has(run.id)) {
+        return await this.settleRun(run, await this.followHandoff(run, handoff));
+      }
       const output = await runInWorkflowCtx(ctx, () => fn(ctx, run.input));
       if (this.cancelRequested.has(run.id)) {
         return await this.compensateAndCancel(run, compensations, replay);
@@ -6779,7 +7335,8 @@ var WorkflowEngine = class {
       }
       const error2 = {
         message: err instanceof Error ? err.message : String(err),
-        stack: err instanceof Error ? err.stack : void 0
+        stack: err instanceof Error ? err.stack : void 0,
+        ...codeAndDetails(err)
       };
       const pending = await this.unwindCompensations(run, compensations, replay);
       if (pending) return await this.settleRun(run, {
@@ -6795,7 +7352,7 @@ var WorkflowEngine = class {
         this.awaitedRemoteSteps.delete(run.id);
       }
       if (this.awaitedStepFans.get(run.id) === awaitedFans) this.awaitedStepFans.delete(run.id);
-      await this.store.releaseRunLock(run.id);
+      await this.releaseLeaseBestEffort(run.id);
       if (suspendedOnFan) {
         void this.redriveSettledChildren(run.id, suspendedOnFan).catch(() => void 0);
       }
@@ -7166,27 +7723,21 @@ var WorkflowEngine = class {
   * own `finally` also releases (the run-execution path) is safe.
   */
   async parkBlocked(run, blocked) {
-    const current = await this.store.getRun(run.id);
-    if (current && (current.status === "cancelled" || current.status === "completed")) {
-      await this.store.releaseRunLock(run.id);
-      return {
-        runId: run.id,
-        status: current.status,
-        output: current.output,
-        error: current.error
-      };
-    }
     const error2 = {
       message: blocked.reason,
       code: blocked.code,
       retryable: true
     };
-    await this.store.updateRun(run.id, {
+    const applied = await (this.store.updateRunIf ?? this.updateRunIfFallback).call(this.store, run.id, SUSPEND_ELIGIBLE_STATUSES, {
       status: "blocked",
       error: error2,
       wakeAt: this.clock() + this.blockedPollMs,
       updatedAt: /* @__PURE__ */ new Date()
     });
+    if (!applied) {
+      await this.store.releaseRunLock(run.id);
+      return this.echoCurrentStatus(run.id);
+    }
     this.emit({
       type: "run.blocked",
       runId: run.id,
@@ -7415,7 +7966,8 @@ var WorkflowEngine = class {
           const failure = {
             index: i,
             id: stepId(runId, seq),
-            error: err.message
+            error: err.message,
+            ...codeAndDetails(err)
           };
           if (opts.mode === "failFast") throw new GatherError([
             failure
@@ -7437,6 +7989,24 @@ var WorkflowEngine = class {
       });
       throw new WorkflowSuspended(wakeAt);
     }
+    if (opts.settle) {
+      const failed = new Map(failures.map((f) => [
+        f.index,
+        f
+      ]));
+      return Array.from(outputs, (value, i) => {
+        const f = failed.get(i);
+        if (!f) return {
+          status: "fulfilled",
+          value
+        };
+        const { index: _index, ...rejected } = f;
+        return {
+          status: "rejected",
+          ...rejected
+        };
+      });
+    }
     if (failures.length > 0) throw new GatherError(failures, inputs.length);
     return outputs;
   }
@@ -7508,6 +8078,25 @@ var WorkflowEngine = class {
     await this.store.saveCheckpoint(checkpoint);
     return true;
   }
+  /**
+  * {@link savePendingCheckpoint} for checkpoints at distinct seqs of one run: one
+  * {@link StateStore.saveCheckpointsIfPending} call when the store has it, else one write each.
+  * Never rejects as a whole: a write that failed is reported as that entry's error, and the batch
+  * call rejecting means it wrote nothing, so the entries are then written one by one.
+  */
+  async savePendingCheckpoints(checkpoints) {
+    if (checkpoints.length > 1 && this.store.saveCheckpointsIfPending) {
+      try {
+        const wrote = await this.store.saveCheckpointsIfPending(checkpoints);
+        return checkpoints.map((_, i) => wrote[i] === true);
+      } catch {
+      }
+    }
+    const settled = await Promise.allSettled(checkpoints.map((cp) => this.savePendingCheckpoint(cp)));
+    return settled.map((s) => s.status === "fulfilled" ? s.value : {
+      error: s.reason
+    });
+  }
   /** Enqueue a remote step task to its worker group via the transport pool. Shared by the initial
   *  dispatch, the opt-in self-heal re-dispatch (see `remoteRedispatchMs`), and the explicit
   *  {@link redispatchPending}. Carries the current trace/context so a re-dispatch is traceable. */
@@ -7546,57 +8135,161 @@ var WorkflowEngine = class {
   * result (the dispatching one may be gone), so the run is crash/scale-safe. Resolves whether the
   * run should now be resumed: not for a duplicate or late delivery (the checkpoint isn't `pending`),
   * a run already finished, or a step fan whose barrier is still closed.
+  *
+  * Results of one run are settled in batches: the first starts a batch, and the ones that arrive
+  * while it is being written form the next. Under load a wide fan's results are written by one
+  * {@link StateStore.saveCheckpointsIfPending} call per batch, and the fan's barrier is asked once
+  * per batch, not once per result. A result resolves only once its own write has committed, so the
+  * transport acknowledges nothing that is not durable yet.
+  */
+  completeRemoteResult(result) {
+    return new Promise((resolve, reject) => {
+      const entry = {
+        result,
+        resolve,
+        reject
+      };
+      const queued = this.resultQueues.get(result.runId);
+      if (queued) {
+        queued.push(entry);
+        return;
+      }
+      const queue = [
+        entry
+      ];
+      this.resultQueues.set(result.runId, queue);
+      void this.drainResultQueue(result.runId, queue);
+    });
+  }
+  /** Settle `queue` (a run's queued results) batch by batch until it is empty, then retire it. */
+  async drainResultQueue(runId, queue) {
+    await Promise.resolve();
+    while (queue.length > 0) {
+      const batch = queue.splice(0, RESULT_BATCH_MAX);
+      let outcomes;
+      try {
+        outcomes = await this.completeRemoteResults(runId, batch.map((q) => q.result));
+      } catch (err) {
+        outcomes = batch.map(() => ({
+          error: err
+        }));
+      }
+      batch.forEach((q, i) => {
+        const outcome = outcomes[i] ?? false;
+        if (typeof outcome === "boolean") q.resolve(outcome);
+        else q.reject(outcome.error);
+      });
+    }
+    this.resultQueues.delete(runId);
+  }
+  /**
+  * Settle a batch of one run's step results. At most one of them resolves `true` (resume the run):
+  * that resume replays every write of the batch, so a second would only replay it again. A result
+  * whose read or write failed resolves its error, and the transport redelivers just that one.
   */
-  async completeRemoteResult(result) {
-    const cp = await this.store.getCheckpoint(result.runId, result.seq);
-    if (!cp || cp.status !== "pending") return false;
-    void this.releaseQueueSlot(cp.stepId).catch(() => void 0);
-    const run = await this.store.getRun(result.runId);
-    if (run && (run.status === "cancelled" || run.status === "completed")) return false;
+  async completeRemoteResults(runId, results) {
+    const outcomes = results.map(() => false);
+    const seen = /* @__PURE__ */ new Set();
+    const live = results.flatMap((r, i) => {
+      if (seen.has(r.seq)) return [];
+      seen.add(r.seq);
+      return [
+        i
+      ];
+    });
+    const reads = await Promise.allSettled(live.map((i) => this.store.getCheckpoint(runId, results[i].seq)));
+    const pending = [];
+    reads.forEach((read, k) => {
+      const index = live[k];
+      if (read.status === "rejected") outcomes[index] = {
+        error: read.reason
+      };
+      else if (read.value?.status === "pending") pending.push({
+        index,
+        cp: read.value
+      });
+    });
+    if (pending.length === 0) return outcomes;
+    for (const { cp } of pending) void this.releaseQueueSlot(cp.stepId).catch(() => void 0);
+    const run = await this.store.getRun(runId);
+    if (run && (run.status === "cancelled" || run.status === "completed")) return outcomes;
     const finishedAt = /* @__PURE__ */ new Date();
-    const startedAt = result.startedAt ? new Date(result.startedAt) : cp.startedAt;
-    const claimed = await this.savePendingCheckpoint({
-      ...cp,
-      status: result.status,
-      output: result.status === "completed" ? result.output : cp.output,
-      error: result.error,
-      events: result.events ?? cp.events,
-      startedAt,
-      finishedAt
+    const writes = pending.map(({ index, cp }) => {
+      const result = results[index];
+      return {
+        ...cp,
+        status: result.status,
+        output: result.status === "completed" ? result.output : cp.output,
+        error: result.error,
+        events: result.events ?? cp.events,
+        startedAt: result.startedAt ? new Date(result.startedAt) : cp.startedAt,
+        finishedAt
+      };
     });
-    if (!claimed) return false;
-    this.noteSettledHere(`${result.runId}:${result.seq}`);
-    this.emit({
-      type: result.status === "completed" ? "step.completed" : "step.failed",
-      runId: result.runId,
-      seq: result.seq,
-      name: cp.name,
-      kind: cp.kind,
-      output: result.output,
-      error: result.error,
-      queueMs: startedAt.getTime() - cp.enqueuedAt.getTime(),
-      durationMs: finishedAt.getTime() - startedAt.getTime()
+    const claims = await this.savePendingCheckpoints(writes);
+    const settled = [];
+    claims.forEach((claim, k) => {
+      const { index, cp } = pending[k];
+      if (claim === true) settled.push({
+        index,
+        cp,
+        result: results[index]
+      });
+      else if (claim !== false) outcomes[index] = claim;
     });
-    return !await this.stepFanStillRunning(cp, result);
+    for (const { cp, result } of settled) {
+      this.noteSettledHere(`${runId}:${result.seq}`);
+      const startedAt = result.startedAt ? new Date(result.startedAt) : cp.startedAt;
+      this.emit({
+        type: result.status === "completed" ? "step.completed" : "step.failed",
+        runId,
+        seq: result.seq,
+        name: cp.name,
+        kind: cp.kind,
+        output: result.output,
+        error: result.error,
+        queueMs: startedAt.getTime() - cp.enqueuedAt.getTime(),
+        durationMs: finishedAt.getTime() - startedAt.getTime()
+      });
+    }
+    const barriers = /* @__PURE__ */ new Map();
+    let carrier;
+    for (const { index, cp, result } of settled) {
+      if (!await this.stepFanStillRunning(cp, result, barriers)) carrier = index;
+    }
+    if (carrier !== void 0) outcomes[carrier] = true;
+    return outcomes;
   }
   /**
   * The join barrier of a step fan: a result for one of its steps resumes the parent only once no
   * step of the fan is still `pending` (a settled wave), or on a failure (failFast surfaces it, and a
   * retry needs the parent to schedule it). Without it every result would replay the whole parent,
-  * which makes a fan of N steps cost O(N²) checkpoint reads.
+  * which makes a fan of N steps cost O(N²) checkpoint reads. `barriers` holds the answer per fan
+  * for the rest of a batch. A barrier that cannot be read counts as open: a spare resume replays and
+  * re-suspends, a lost one strands the run.
   */
-  async stepFanStillRunning(cp, result) {
+  async stepFanStillRunning(cp, result, barriers) {
     const fan = parseStepFanGroup(cp.parallelGroup);
     if (!fan) return false;
     if (result.status === "failed") return false;
-    const to = fan.firstSeq + fan.size - 1;
-    const pending = this.store.countCheckpoints ? await this.store.countCheckpoints(result.runId, fan.firstSeq, to, [
-      "pending"
-    ]) : (await this.store.listCheckpoints(result.runId)).filter((c) => c.seq >= fan.firstSeq && c.seq <= to && c.status === "pending").length;
-    if (pending > 0) return true;
+    let closed = barriers.get(fan.firstSeq);
+    if (!closed) {
+      closed = this.stepFanPending(result.runId, fan.firstSeq, fan.firstSeq + fan.size - 1).then((pending) => pending > 0, () => false);
+      barriers.set(fan.firstSeq, closed);
+    }
+    if (await closed) return true;
     this.noteRecent(this.stepFansFiredHere, `${result.runId}:${fan.firstSeq}`);
     return false;
   }
+  /** How many steps of a step fan's seq range are still `pending`. */
+  async stepFanPending(runId, from, to) {
+    if (this.store.countCheckpoints) {
+      return this.store.countCheckpoints(runId, from, to, [
+        "pending"
+      ]);
+    }
+    return (await this.store.listCheckpoints(runId)).filter((c) => c.seq >= from && c.seq <= to && c.status === "pending").length;
+  }
   /**
   * Apply a remote workflow-turn {@link WorkflowDecision} durably — runs on WHATEVER instance receives
   * it over the broker (the dispatching one may be gone, or a point-to-point broker handed it to a
@@ -8043,14 +8736,339 @@ function summarizeTimeline(timeline, over) {
 __name(summarizeTimeline, "summarizeTimeline");
 function pageFan(timeline, group, query) {
   const matching = timeline.filter((cp) => cp.parallelGroup === group && (!query.failedOnly || isFailedMember(cp)));
-  const offset = query.offset !== void 0 && Number.isFinite(query.offset) ? Math.max(0, Math.floor(query.offset)) : 0;
-  const limit = query.limit !== void 0 && Number.isFinite(query.limit) ? Math.min(FAN_PAGE_MAX, Math.max(1, Math.floor(query.limit))) : 100;
+  const { offset, limit } = fanPageBounds(query);
   return {
     items: matching.slice(offset, offset + limit),
     total: matching.length
   };
 }
 __name(pageFan, "pageFan");
+function fanPageBounds(query) {
+  const offset = query.offset !== void 0 && Number.isFinite(query.offset) ? Math.max(0, Math.floor(query.offset)) : 0;
+  const limit = query.limit !== void 0 && Number.isFinite(query.limit) ? Math.min(FAN_PAGE_MAX, Math.max(1, Math.floor(query.limit))) : 100;
+  return {
+    offset,
+    limit
+  };
+}
+__name(fanPageBounds, "fanPageBounds");
+
+// src/run-detail-store.ts
+function aggregatesCheckpoints(store) {
+  return store.checkpointStats !== void 0 && store.queryCheckpoints !== void 0;
+}
+__name(aggregatesCheckpoints, "aggregatesCheckpoints");
+function checkpointStatsOf(checkpoints, opts) {
+  const buckets = /* @__PURE__ */ new Map();
+  for (const cp of checkpoints) {
+    const group = cp.parallelGroup ?? null;
+    if (opts?.group !== void 0 && group !== opts.group) continue;
+    const nameKind = checkpointNameKind(cp.name);
+    const key = JSON.stringify([
+      group,
+      cp.status,
+      nameKind
+    ]);
+    const enqueuedAt = cp.enqueuedAt ?? cp.startedAt;
+    const bucket = buckets.get(key);
+    if (!bucket) {
+      buckets.set(key, {
+        group,
+        status: cp.status,
+        nameKind,
+        count: 1,
+        minSeq: cp.seq,
+        maxSeq: cp.seq,
+        minName: cp.name,
+        maxName: cp.name,
+        minEnqueuedAt: enqueuedAt,
+        minStartedAt: cp.startedAt,
+        maxStartedAt: cp.startedAt,
+        maxFinishedAt: cp.finishedAt,
+        sumAttempts: cp.attempts
+      });
+      continue;
+    }
+    bucket.count += 1;
+    bucket.minSeq = Math.min(bucket.minSeq, cp.seq);
+    bucket.maxSeq = Math.max(bucket.maxSeq, cp.seq);
+    if (cp.name < bucket.minName) bucket.minName = cp.name;
+    if (cp.name > bucket.maxName) bucket.maxName = cp.name;
+    if (enqueuedAt < bucket.minEnqueuedAt) bucket.minEnqueuedAt = enqueuedAt;
+    if (cp.startedAt < bucket.minStartedAt) bucket.minStartedAt = cp.startedAt;
+    if (cp.startedAt > bucket.maxStartedAt) bucket.maxStartedAt = cp.startedAt;
+    if (cp.finishedAt > bucket.maxFinishedAt) bucket.maxFinishedAt = cp.finishedAt;
+    bucket.sumAttempts += cp.attempts;
+  }
+  return [
+    ...buckets.values()
+  ];
+}
+__name(checkpointStatsOf, "checkpointStatsOf");
+function queryCheckpointsOf(checkpoints, query) {
+  const excluded = new Set(query.excludeGroups ?? []);
+  const matching = checkpoints.filter((cp) => (query.group === void 0 || cp.parallelGroup === query.group) && (cp.parallelGroup === void 0 || !excluded.has(cp.parallelGroup)) && (query.statuses === void 0 || query.statuses.includes(cp.status)) && (query.namePrefixes === void 0 || query.namePrefixes.some((prefix) => cp.name.startsWith(prefix)))).sort((a, b) => a.seq - b.seq);
+  const offset = query.offset ?? 0;
+  return matching.slice(offset, query.limit === void 0 ? void 0 : offset + query.limit);
+}
+__name(queryCheckpointsOf, "queryCheckpointsOf");
+var TERMINAL = /* @__PURE__ */ __name((status) => status !== "pending" && status !== "running", "TERMINAL");
+function isWide(group, rows, over) {
+  return rows > over || (parseFanGroup(group)?.size ?? 0) > over;
+}
+__name(isWide, "isWide");
+function mergeBySeq(a, b) {
+  return [
+    ...a,
+    ...b
+  ].sort((x, y) => x.seq - y.seq);
+}
+__name(mergeBySeq, "mergeBySeq");
+async function failedChildren(store, runId, group, buckets) {
+  if (!buckets.some((b) => b.nameKind === "child" && b.status === "completed")) {
+    return {
+      completed: [],
+      failed: []
+    };
+  }
+  const completed = await store.queryCheckpoints(runId, {
+    group,
+    statuses: [
+      "completed"
+    ],
+    namePrefixes: [
+      "signal:child:"
+    ]
+  });
+  return {
+    completed,
+    failed: completed.filter(isFailedMember)
+  };
+}
+__name(failedChildren, "failedChildren");
+async function summarizeFanFromStats(store, runId, group, buckets, folded) {
+  const counts = {};
+  const add = /* @__PURE__ */ __name((status, n) => {
+    counts[status] = (counts[status] ?? 0) + n;
+  }, "add");
+  let size = 0;
+  let startedAt;
+  let finishedAt;
+  let firstSeq = Number.POSITIVE_INFINITY;
+  let lastSeq = Number.NEGATIVE_INFINITY;
+  let minName;
+  let maxName;
+  let childRuns = false;
+  for (const b of buckets) {
+    firstSeq = Math.min(firstSeq, b.minSeq);
+    lastSeq = Math.max(lastSeq, b.maxSeq);
+    if (minName === void 0 || b.minName < minName) minName = b.minName;
+    if (maxName === void 0 || b.maxName > maxName) maxName = b.maxName;
+    if (b.nameKind === "child" || b.nameKind === "folded") childRuns = true;
+    if (TERMINAL(b.status)) {
+      if (!startedAt || b.minStartedAt < startedAt) startedAt = b.minStartedAt;
+      if (!finishedAt || b.maxFinishedAt > finishedAt) finishedAt = b.maxFinishedAt;
+    } else if (!startedAt || b.minEnqueuedAt < startedAt) {
+      startedAt = b.minEnqueuedAt;
+    }
+    if (b.nameKind === "folded") continue;
+    if (b.nameKind === "child" && b.status === "completed") continue;
+    add(b.status, b.count);
+    size += b.count;
+  }
+  for (const cp of folded) {
+    const n = foldedCount(cp) ?? 1;
+    add(isFailedMember(cp) ? "failed" : cp.status, n);
+    size += n;
+  }
+  const children = await failedChildren(store, runId, group, buckets);
+  const failedChildSet = new Set(children.failed);
+  for (const cp of children.completed) add(failedChildSet.has(cp) ? "failed" : "completed", 1);
+  size += children.completed.length;
+  const failedStatusRows = buckets.some((b) => b.status === "failed") ? await store.queryCheckpoints(runId, {
+    group,
+    statuses: [
+      "failed"
+    ],
+    limit: FAN_SUMMARY_FAILURES
+  }) : [];
+  const failures = mergeBySeq(failedStatusRows, children.failed).slice(0, FAN_SUMMARY_FAILURES);
+  const parsed = parseFanGroup(group);
+  const total = parsed?.size;
+  return {
+    group,
+    label: childRuns ? "child runs" : minName === maxName ? minName ?? "" : "steps",
+    firstSeq,
+    lastSeq,
+    size,
+    total,
+    notStarted: total === void 0 ? void 0 : Math.max(0, total - size),
+    failFast: parsed?.failFast ?? false,
+    counts,
+    startedAt,
+    finishedAt,
+    failures
+  };
+}
+__name(summarizeFanFromStats, "summarizeFanFromStats");
+async function firstChildIds(store, runId) {
+  const ids = [];
+  for (let offset = 0; ids.length < SUMMARY_CHILD_LIMIT; offset += SUMMARY_CHILD_LIMIT) {
+    const page = await store.queryCheckpoints(runId, {
+      namePrefixes: CHILD_EDGE_PREFIXES,
+      offset,
+      limit: SUMMARY_CHILD_LIMIT
+    });
+    ids.push(...childIdsOf(page));
+    if (page.length < SUMMARY_CHILD_LIMIT) break;
+  }
+  return ids.slice(0, SUMMARY_CHILD_LIMIT);
+}
+__name(firstChildIds, "firstChildIds");
+async function summarizeRunFromStore(store, runId, over, stats) {
+  const all = stats ?? await store.checkpointStats(runId);
+  const byGroup = /* @__PURE__ */ new Map();
+  let edgeRows = 0;
+  let hasFolded = false;
+  for (const b of all) {
+    if (b.nameKind === "child" || b.nameKind === "spawn") edgeRows += b.count;
+    if (b.nameKind === "folded") hasFolded = true;
+    if (b.group === null) continue;
+    const buckets = byGroup.get(b.group);
+    if (buckets) buckets.push(b);
+    else byGroup.set(b.group, [
+      b
+    ]);
+  }
+  const wide = [
+    ...byGroup
+  ].filter(([group, buckets]) => isWide(group, buckets.reduce((n, b) => n + b.count, 0), over));
+  const [timeline, folded, children] = await Promise.all([
+    store.queryCheckpoints(runId, {
+      excludeGroups: wide.map(([group]) => group)
+    }),
+    hasFolded ? store.queryCheckpoints(runId, {
+      namePrefixes: [
+        FOLDED_PREFIX
+      ]
+    }) : [],
+    edgeRows > 0 || hasFolded ? firstChildIds(store, runId) : []
+  ]);
+  const fans = await Promise.all(wide.map(([group, buckets]) => summarizeFanFromStats(store, runId, group, buckets, folded.filter((cp) => cp.parallelGroup === group))));
+  const childCount = folded.reduce((n, cp) => n + childIdsOf([
+    cp
+  ]).length, edgeRows);
+  return {
+    timeline,
+    fans: fans.sort((a, b) => a.firstSeq - b.firstSeq),
+    children,
+    childCount
+  };
+}
+__name(summarizeRunFromStore, "summarizeRunFromStore");
+async function pageFanFromStore(store, runId, group, query, stats) {
+  const buckets = stats ?? await store.checkpointStats(runId, {
+    group
+  });
+  const { offset, limit } = fanPageBounds(query);
+  if (!query.failedOnly) {
+    const total2 = buckets.reduce((n, b) => n + b.count, 0);
+    const items = total2 > offset ? await store.queryCheckpoints(runId, {
+      group,
+      offset,
+      limit
+    }) : [];
+    return {
+      items,
+      total: total2
+    };
+  }
+  const failedStatus = buckets.filter((b) => b.status === "failed").reduce((n, b) => n + b.count, 0);
+  const children = await failedChildren(store, runId, group, buckets);
+  const total = failedStatus + children.failed.length;
+  if (total <= offset) return {
+    items: [],
+    total
+  };
+  if (children.failed.length === 0) {
+    return {
+      items: await store.queryCheckpoints(runId, {
+        group,
+        statuses: [
+          "failed"
+        ],
+        offset,
+        limit
+      }),
+      total
+    };
+  }
+  const failedRows = await store.queryCheckpoints(runId, {
+    group,
+    statuses: [
+      "failed"
+    ],
+    limit: offset + limit
+  });
+  return {
+    items: mergeBySeq(failedRows, children.failed).slice(offset, offset + limit),
+    total
+  };
+}
+__name(pageFanFromStore, "pageFanFromStore");
+function hash(text) {
+  let h1 = 3735928559;
+  let h2 = 1103547991;
+  for (let i = 0; i < text.length; i++) {
+    const ch = text.charCodeAt(i);
+    h1 = Math.imul(h1 ^ ch, 2654435761);
+    h2 = Math.imul(h2 ^ ch, 1597334677);
+  }
+  h1 = Math.imul(h1 ^ h1 >>> 16, 2246822507) ^ Math.imul(h2 ^ h2 >>> 13, 3266489909);
+  h2 = Math.imul(h2 ^ h2 >>> 16, 2246822507) ^ Math.imul(h1 ^ h1 >>> 13, 3266489909);
+  return (4294967296 * (2097151 & h2) + (h1 >>> 0)).toString(16);
+}
+__name(hash, "hash");
+function statsFingerprint(stats) {
+  return stats.map((b) => [
+    b.group ?? "",
+    b.status,
+    b.nameKind,
+    b.count,
+    b.minSeq,
+    b.maxSeq,
+    b.minName,
+    b.maxName,
+    b.minEnqueuedAt.getTime(),
+    b.minStartedAt.getTime(),
+    b.maxStartedAt.getTime(),
+    b.maxFinishedAt.getTime(),
+    b.sumAttempts
+  ].join("
")).sort().join("");
+}
+__name(statsFingerprint, "statsFingerprint");
+function runDetailEtag(run, stats, over) {
+  return `"rd-${hash(`${over}${JSON.stringify(run)}${statsFingerprint(stats)}`)}"`;
+}
+__name(runDetailEtag, "runDetailEtag");
+function fanPageEtag(group, stats, query) {
+  const { offset, limit } = fanPageBounds(query);
+  const key = [
+    group,
+    offset,
+    limit,
+    query.failedOnly === true
+  ].join("
");
+  return `"fp-${hash(`${key}${statsFingerprint(stats)}`)}"`;
+}
+__name(fanPageEtag, "fanPageEtag");
+function etagMatches(ifNoneMatch, etag) {
+  if (!ifNoneMatch) return false;
+  const bare = /* @__PURE__ */ __name((tag) => tag.trim().replace(/^W\//, ""), "bare");
+  const target = bare(etag);
+  return ifNoneMatch.split(",").some((tag) => tag.trim() === "*" || bare(tag) === target);
+}
+__name(etagMatches, "etagMatches");
 
 // src/codec-state-store.ts
 var CodecStateStore = class {
@@ -8077,14 +9095,28 @@ var CodecStateStore = class {
   /** Forwarded verbatim (deletes touch no payload). Bound only when the inner store has it, so the
   *  engine's per-run fallback still kicks in through the wrapper. */
   deleteRuns;
+  /** Decodes the runs like {@link listIncompleteRuns}. Bound only when the inner store has it, so
+  *  the engine's list-and-filter fallback still kicks in through the wrapper. */
+  listOrphanedRuns;
+  /** Encodes the patch like {@link updateRun}. Bound only when the inner store has it, so the
+  *  engine's read-then-write fallback still kicks in through the wrapper. */
+  updateRunIf;
   /** Encodes like {@link saveCheckpoint}. Bound only when the inner store has it, so the engine's
   *  plain-save fallback still kicks in through the wrapper. */
   saveCheckpointIfPending;
+  /** Encodes each checkpoint like {@link saveCheckpoint}. Bound only when the inner store has it. */
+  saveCheckpointsIfPending;
   /** Forwarded verbatim — a count reads no payload. Bound only when the inner store has it. */
   countCheckpoints;
   /** Forwarded verbatim — retention selects by status/updatedAt/scope, never by payload. Bound only
   *  when the inner store prunes, so the retention poller still sees (and warns about) its absence. */
   pruneTerminalRuns;
+  /** Forwarded verbatim — the buckets aggregate names, statuses and timestamps, never a payload.
+  *  Bound only when the inner store aggregates, so the run detail's fallback still applies. */
+  checkpointStats;
+  /** Filters on unencoded columns, then decodes the rows like {@link listCheckpoints}. Bound only
+  *  when the inner store has it. */
+  queryCheckpoints;
   /** Forwarded verbatim — a count reads no payload. Bound only when the inner store counts, so the
   *  engine's facet/listing fallback still applies through the wrapper. */
   countRuns;
@@ -8109,10 +9141,20 @@ var CodecStateStore = class {
     if (inner.tryAdmitSingleton) this.tryAdmitSingleton = inner.tryAdmitSingleton.bind(inner);
     const deleteRuns = inner.deleteRuns;
     if (deleteRuns) this.deleteRuns = (ids) => deleteRuns.call(inner, ids);
+    const orphans = inner.listOrphanedRuns;
+    if (orphans) this.listOrphanedRuns = async (nowMs, limit, namespace) => (await orphans.call(inner, nowMs, limit, namespace)).map((r) => this.decRun(r));
+    const updateIf = inner.updateRunIf;
+    if (updateIf) this.updateRunIf = (runId, expected, patch) => updateIf.call(inner, runId, expected, this.encRunPatch(patch));
     const saveIfPending = inner.saveCheckpointIfPending;
     if (saveIfPending) this.saveCheckpointIfPending = (cp) => saveIfPending.call(inner, this.encCp(cp));
+    const saveManyIfPending = inner.saveCheckpointsIfPending;
+    if (saveManyIfPending) this.saveCheckpointsIfPending = (cps) => saveManyIfPending.call(inner, cps.map((cp) => this.encCp(cp)));
     const countCps = inner.countCheckpoints;
     if (countCps) this.countCheckpoints = (runId, from, to, statuses) => countCps.call(inner, runId, from, to, statuses);
+    const stats = inner.checkpointStats;
+    if (stats) this.checkpointStats = (runId, opts) => stats.call(inner, runId, opts);
+    const queryCps = inner.queryCheckpoints;
+    if (queryCps) this.queryCheckpoints = async (runId, query) => (await queryCps.call(inner, runId, query)).map((c) => this.decCp(c));
     const prune = inner.pruneTerminalRuns;
     if (prune) this.pruneTerminalRuns = (policy, now, limit) => prune.call(inner, policy, now, limit);
     const count = inner.countRuns;
@@ -8209,12 +9251,15 @@ var CodecStateStore = class {
     return this.inner.createRun(this.encRun(run));
   }
   updateRun(runId, patch) {
+    return this.inner.updateRun(runId, this.encRunPatch(patch));
+  }
+  encRunPatch(patch) {
     const next = {
       ...patch
     };
     if ("input" in patch) next.input = this.enc(patch.input);
     if ("output" in patch) next.output = this.enc(patch.output);
-    return this.inner.updateRun(runId, next);
+    return next;
   }
   async getRun(runId) {
     return this.decRun(await this.inner.getRun(runId));
@@ -8320,8 +9365,11 @@ function collectMetrics(engine) {
       started: 0,
       completed: 0,
       failed: 0,
-      suspended: 0
+      suspended: 0,
+      routed: 0,
+      unrouted: 0
     },
+    routedByCode: {},
     steps: {
       started: 0,
       completed: 0,
@@ -8363,6 +9411,15 @@ function collectMetrics(engine) {
       case "run.suspended":
         m.runs.suspended += 1;
         break;
+      case "run.routed": {
+        m.runs.routed += 1;
+        const code = e.error?.code ?? "*";
+        m.routedByCode[code] = (m.routedByCode[code] ?? 0) + 1;
+        break;
+      }
+      case "run.unrouted":
+        m.runs.unrouted += 1;
+        break;
       case "step.started":
         m.steps.started += 1;
         break;
@@ -8389,6 +9446,8 @@ function collectMetrics(engine) {
       for (const [k, v] of Object.entries(m.steps)) lines.push(`durable_steps_total{event="${k}"} ${v}`);
       lines.push("# TYPE durable_runs_by_workflow_total counter");
       for (const [name, v] of Object.entries(m.byWorkflow)) for (const [event, n] of Object.entries(v)) lines.push(`durable_runs_by_workflow_total{workflow="${name}",event="${event}"} ${n}`);
+      lines.push("# TYPE durable_runs_routed_total counter");
+      for (const [code, n] of Object.entries(m.routedByCode)) lines.push(`durable_runs_routed_total{code="${code}"} ${n}`);
       lines.push("# TYPE durable_step_duration_ms_sum counter");
       lines.push(`durable_step_duration_ms_sum ${m.stepDuration.sumMs}`);
       lines.push("# TYPE durable_step_duration_count counter");
@@ -8559,6 +9618,11 @@ var InMemoryStateStore = class {
   * so this lets the benchmark contrast "fetch N, scan in JS" against "fetch the k that match".
   */
   lastCheckpointRowsRead = 0;
+  /**
+  * Test/observability hook: checkpoint rows materialized since construction, across every
+  * checkpoint read that returns rows — what a run-detail benchmark compares between paths.
+  */
+  checkpointRowsRead = 0;
   key(runId, seq) {
     return `${runId}:${seq}`;
   }
@@ -8604,6 +9668,14 @@ var InMemoryStateStore = class {
     this.reindexAttributes(run.id, run.searchAttributes);
   }
   async updateRun(runId, inputPatch) {
+    this.applyRunPatch(runId, inputPatch);
+  }
+  updateRunIf(runId, expectedStatuses, patch) {
+    const existing = this.runs.get(runId);
+    if (!existing || !expectedStatuses.includes(existing.status)) return Promise.resolve(false);
+    return this.updateRun(runId, patch).then(() => true);
+  }
+  applyRunPatch(runId, inputPatch) {
     const patch = inputPatch.status && TERMINAL_RUN_STATUSES.includes(inputPatch.status) ? {
       ...inputPatch,
       tags: ("tags" in inputPatch ? inputPatch.tags : this.runs.get(runId)?.tags)?.filter((tag) => tag !== SINGLETON_ADMITTED_TAG)
@@ -8684,6 +9756,23 @@ var InMemoryStateStore = class {
     });
     return true;
   }
+  async saveCheckpointsIfPending(checkpoints) {
+    return checkpoints.map((checkpoint) => {
+      const key = this.key(checkpoint.runId, checkpoint.seq);
+      if (this.checkpoints.get(key)?.status !== "pending") return false;
+      this.checkpoints.set(key, {
+        ...checkpoint
+      });
+      return true;
+    });
+  }
+  async listOrphanedRuns(nowMs, limit, namespace) {
+    return [
+      ...this.runs.values()
+    ].filter((r) => (r.status === "running" || r.status === "cancelling") && (r.lockedUntil == null || r.lockedUntil <= nowMs) && (namespace === void 0 || r.namespace === namespace)).sort((a, b) => a.createdAt.getTime() - b.createdAt.getTime()).slice(0, Math.max(0, limit)).map((r) => ({
+      ...r
+    }));
+  }
   async listIncompleteRuns(namespace) {
     return [
       ...this.runs.values()
@@ -8963,6 +10052,23 @@ var InMemoryStateStore = class {
       ...cp
     }));
     this.lastCheckpointRowsRead = rows.length;
+    this.checkpointRowsRead += rows.length;
+    return rows;
+  }
+  runCheckpoints(runId) {
+    return [
+      ...this.checkpoints.values()
+    ].filter((cp) => cp.runId === runId);
+  }
+  async checkpointStats(runId, opts) {
+    return checkpointStatsOf(this.runCheckpoints(runId), opts);
+  }
+  async queryCheckpoints(runId, query) {
+    const rows = queryCheckpointsOf(this.runCheckpoints(runId), query).map((cp) => ({
+      ...cp
+    }));
+    this.lastCheckpointRowsRead = rows.length;
+    this.checkpointRowsRead += rows.length;
     return rows;
   }
   async getLatestCheckpointByName(runId, name) {
@@ -8990,6 +10096,7 @@ var InMemoryStateStore = class {
       ...cp
     }));
     this.lastCheckpointRowsRead = rows.length;
+    this.checkpointRowsRead += rows.length;
     return rows;
   }
 };
@@ -9040,6 +10147,7 @@ var InMemoryTransport = class {
   }
 };
 export {
+  CHILD_EDGE_PREFIXES,
   CONTROL_FLOW_SIGNAL,
   CURRENT_PROTOCOL_VERSION,
   CodecStateStore,
@@ -9060,6 +10168,7 @@ export {
   FAN_SUMMARY_FAILURES,
   FatalError,
   GatherError,
+  HANDOFF_SEQ_BASE,
   InMemoryAdmissionBackend,
   InMemoryStateStore,
   InMemoryTransport,
@@ -9094,9 +10203,11 @@ export {
   WorkflowEngine,
   WorkflowSuspended,
   aggregateAnnouncements,
+  aggregatesCheckpoints,
   announcementsOf,
   applyAttributeQuery,
   assertConcurrency,
+  assertRoutes,
   attributeColumnFor,
   attributeOperand,
   attributePredicateOperands,
@@ -9106,10 +10217,14 @@ export {
   buildWorkflowDirectory,
   canRoute,
   capableWorkers,
+  checkpointNameKind,
+  checkpointStatsOf,
   childIdsOf,
   classifyWaiterToken,
   clearDurableExecutionWrappers,
   clearTerminalSingletonAdmission,
+  codeAndDetails,
+  codeOf,
   collectMetrics,
   concurrencyTag,
   controlPlaneDescriptor,
@@ -9122,13 +10237,20 @@ export {
   debug,
   describeWorker,
   descriptorHash,
+  detailsOf,
   error,
+  etagMatches,
   eventMatchOf,
   eventMatches,
   eventNameOf,
   eventPrefix,
   eventToken,
   facetOrigin,
+  fanPageBounds,
+  fanPageEtag,
+  foldedCount,
+  handoffHandlerId,
+  handoffName,
   hasDurableExecutionWrappers,
   heartbeatStatus,
   indexWaitersByRun,
@@ -9140,6 +10262,7 @@ export {
   isNonEmptyRunScope,
   isWorkflowControlFlowSignal,
   listSignalWaitersOfRuns,
+  liveHandoff,
   log,
   matchesAttributes,
   mergeRunFacetRows,
@@ -9150,17 +10273,23 @@ export {
   normalizeAttributeRows,
   normalizeDescriptor,
   observedAnnouncements,
+  onErrorRunId,
   pageFan,
+  pageFanFromStore,
   parentRunIdOf,
   parseDuration,
   partitionOfToken,
   planDispatch,
   prevCronFireMs,
+  queryCheckpointsOf,
   readSearchAttributes,
   requiredCapabilities,
   resolveConcurrency,
   resolveRouting,
   resolveRunWaiting,
+  retiredHandoffName,
+  routeFor,
+  runDetailEtag,
   runDurableExecution,
   runInStepLogger,
   runInStepScope,
@@ -9181,6 +10310,7 @@ export {
   subEvent,
   subProcess,
   summarizeFan,
+  summarizeRunFromStore,
   summarizeTimeline,
   tenantGroup,
   useDurableExecution,