@bastani/atomic 0.9.11-alpha.5 → 0.9.11-alpha.7

package/dist/builtin/workflows/src/durable/ui-primitive.ts

@@ -14,10 +14,45 @@ import type { DurableUiCheckpoint, UiPromptKind } from "./types.js";
 import { recordCheckpointDurably } from "./tool-primitive.js";
 import { claimDurablePromptToken, durablePromptScope, releaseDurablePrompt, reserveDurablePrompt, type PromptReservationToken } from "./prompt-reservations.js";
 
+export type DurableUiReplayRequest =
+  | {
+      readonly kind: "input";
+      readonly promptText: string;
+      readonly response: string;
+      readonly callerStack: string | undefined;
+    }
+  | {
+      readonly kind: "confirm";
+      readonly message: string;
+      readonly response: boolean;
+      readonly callerStack: string | undefined;
+    }
+  | {
+      readonly kind: "select";
+      readonly message: string;
+      readonly options: readonly string[];
+      readonly response: string;
+      readonly callerStack: string | undefined;
+    }
+  | {
+      readonly kind: "editor";
+      readonly initial: string | undefined;
+      readonly response: string;
+      readonly callerStack: string | undefined;
+    }
+  | {
+      readonly kind: "custom";
+      readonly factory: WorkflowCustomUiFactory<WorkflowSerializableValue>;
+      readonly options: WorkflowCustomUiOptions | undefined;
+      readonly response: WorkflowSerializableValue | undefined;
+      readonly callerStack: string | undefined;
+    };
+
 export interface DurableUiDeps {
   readonly workflowId: string;
   readonly backend: DurableWorkflowBackend;
   readonly nextCheckpointId: () => string;
+  readonly onReplay?: (request: DurableUiReplayRequest) => Promise<void>;
 }
 
 export function wrapUiWithDurable(base: WorkflowUIContext, deps: DurableUiDeps): WorkflowUIContext {
@@ -104,6 +139,7 @@ export function wrapUiWithDurable(base: WorkflowUIContext, deps: DurableUiDeps):
       const identity = nextIdentity("input", promptText, callerStack);
       const hit = cached(identity);
       if (typeof hit === "string") {
+        await deps.onReplay?.({ kind: "input", promptText, response: hit, callerStack });
         await releaseCached(identity);
         return hit;
       }
@@ -120,6 +156,7 @@ export function wrapUiWithDurable(base: WorkflowUIContext, deps: DurableUiDeps):
       const identity = nextIdentity("confirm", message, callerStack);
       const hit = cached(identity);
       if (typeof hit === "boolean") {
+        await deps.onReplay?.({ kind: "confirm", message, response: hit, callerStack });
         await releaseCached(identity);
         return hit;
       }
@@ -136,6 +173,7 @@ export function wrapUiWithDurable(base: WorkflowUIContext, deps: DurableUiDeps):
       const identity = nextIdentity("select", message, callerStack, [...options]);
       const hit = cached(identity);
       if (typeof hit === "string") {
+        await deps.onReplay?.({ kind: "select", message, options, response: hit, callerStack });
         await releaseCached(identity);
         return hit as T;
       }
@@ -152,6 +190,7 @@ export function wrapUiWithDurable(base: WorkflowUIContext, deps: DurableUiDeps):
       const identity = nextIdentity("editor", initial ?? "", callerStack, initial ?? null);
       const hit = cached(identity);
       if (typeof hit === "string") {
+        await deps.onReplay?.({ kind: "editor", initial, response: hit, callerStack });
         await releaseCached(identity);
         return hit;
       }
@@ -169,6 +208,13 @@ export function wrapUiWithDurable(base: WorkflowUIContext, deps: DurableUiDeps):
       const identity = nextIdentity("custom", replayIdentity, callerStack, { replayIdentity });
       const hit = cachedCustom(identity);
       if (hit.found) {
+        await deps.onReplay?.({
+          kind: "custom",
+          factory: factory as WorkflowCustomUiFactory<WorkflowSerializableValue>,
+          options,
+          response: hit.response,
+          callerStack,
+        });
         await releaseCached(identity);
         return hit.response as T;
       }

package/dist/builtin/workflows/src/durable/workflow-child-result.ts

@@ -0,0 +1,58 @@
+import type {
+  WorkflowChildResult,
+  WorkflowExitStatus,
+  WorkflowOutputValues,
+  WorkflowSerializableValue,
+} from "../shared/types.js";
+
+/** Parse the exact durable child-result union without trusting persisted casts. */
+export function parseWorkflowChildResult(
+  value: WorkflowSerializableValue | undefined,
+): WorkflowChildResult<WorkflowOutputValues> | undefined {
+  if (!isOrdinaryObject(value)) return undefined;
+  const workflow = value["workflow"];
+  const runId = value["runId"];
+  const status = value["status"];
+  const exited = value["exited"];
+  const outputs = value["outputs"];
+  const exitReason = value["exitReason"];
+  if (typeof workflow !== "string" || typeof runId !== "string" || !isOrdinaryObject(outputs)) return undefined;
+  if (exited === false) {
+    if (status !== "completed" || exitReason !== undefined) return undefined;
+    return value as WorkflowChildResult<WorkflowOutputValues>;
+  }
+  if (exited !== true || !isWorkflowExitStatus(status)
+    || (exitReason !== undefined && typeof exitReason !== "string")) return undefined;
+  return value as WorkflowChildResult<WorkflowOutputValues>;
+}
+
+export function parseLegacyWorkflowChildResult(
+  value: WorkflowSerializableValue | undefined,
+): WorkflowChildResult<WorkflowOutputValues> | undefined {
+  if (!isOrdinaryObject(value)) return undefined;
+  if (typeof value["workflow"] !== "string" || typeof value["runId"] !== "string"
+    || value["status"] !== "completed" || value["exited"] !== undefined
+    || value["exitReason"] !== undefined || !isOrdinaryObject(value["outputs"])) return undefined;
+  return value as WorkflowChildResult<WorkflowOutputValues>;
+}
+
+export function isWorkflowChildResult(
+  value: WorkflowSerializableValue | undefined,
+): value is WorkflowChildResult<WorkflowOutputValues> {
+  return parseWorkflowChildResult(value) !== undefined;
+}
+
+
+function isWorkflowExitStatus(
+  value: WorkflowSerializableValue | undefined,
+): value is WorkflowExitStatus {
+  return value === "completed" || value === "skipped" || value === "cancelled" || value === "blocked";
+}
+
+function isOrdinaryObject(
+  value: WorkflowSerializableValue | undefined,
+): value is Readonly<Record<string, WorkflowSerializableValue | undefined>> {
+  if (typeof value !== "object" || value === null || Array.isArray(value)) return false;
+  const prototype = Object.getPrototypeOf(value);
+  return prototype === Object.prototype || prototype === null;
+}

package/dist/builtin/workflows/src/engine/options.ts

@@ -40,4 +40,5 @@ export type EngineChildRunOptions = Pick<
   | "onStageEnd"
   | "onStageSession"
   | "durableBackend"
+  | "durableRootBackend"
 >;

package/dist/builtin/workflows/src/engine/primitives/workflow.ts

@@ -17,18 +17,14 @@ import {
   workflowChildReplaySnapshot,
   workflowDefinitionRequirementMessage,
 } from "../../runs/foreground/executor-child-helpers.js";
-import type { DurableScope } from "../../durable/scoped-backend.js";
+import type { DurableChildInvocation } from "../../durable/boundary-topology.js";
 
 export function createChildWorkflowRunner(input: {
   readonly runtime: EngineRuntime;
   readonly resolveWorkflowCwd: () => string;
   readonly nextWorkflowBoundaryReplayKey: (name: string) => string;
-  /**
-   * Consume the durable scope published by the durable child primitive for the
-   * current invocation, if any. Routes child internal side-effect checkpoints
-   * under the root workflow.
-   */
-  readonly consumeDurableScope?: () => DurableScope | undefined;
+  /** Consume the identity prepared by the durable outer primitive. */
+  readonly consumeDurableInvocation?: () => DurableChildInvocation | undefined;
   readonly runWorkflow: <
     TInputs extends WorkflowInputValues,
     TRunInputs extends WorkflowInputValues = TInputs,
@@ -60,7 +56,16 @@ export function createChildWorkflowRunner(input: {
     const childName = child.normalizedName;
     const boundaryName = options.stageName ?? `workflow:${childName}`;
     const boundaryReplayKey = input.nextWorkflowBoundaryReplayKey(boundaryName);
-    const boundary = runtime.spawnStage(boundaryName, { kind: "workflow-boundary", replayKey: boundaryReplayKey }).boundary;
+    const durableInvocation = input.consumeDurableInvocation?.();
+    const boundary = runtime.spawnStage(boundaryName, {
+      kind: "workflow-boundary",
+      replayKey: boundaryReplayKey,
+      ...(durableInvocation !== undefined ? { identity: durableInvocation.identity } : {}),
+    }).boundary;
+    if (boundary.replayedChild !== undefined) {
+      durableInvocation?.adoptReplayedChildRunId(boundary.replayedChild.runId);
+    }
+    await durableInvocation?.recordStart(boundary.parentIds, boundary.sourceOrder);
     let childRunId: string | undefined;
     let detachParentAbort: (() => void) | undefined;
     try {
@@ -68,13 +73,14 @@ export function createChildWorkflowRunner(input: {
         await Promise.resolve();
         runtime.exit.throwIfWorkflowExitSelected();
         boundary.finalizeReplay();
+        await durableInvocation?.recordTerminal("completed", boundary.replayedChild);
         return boundary.replayedChild as WorkflowChildResult<TChildOutputs>;
       }
 
       const childInputs = resolveAndValidateInputs(child.inputs, options.inputs ?? {}, `child workflow "${childName}" (${child.name})`);
       runtime.exit.throwIfWorkflowExitSelected();
 
-      childRunId = crypto.randomUUID();
+      childRunId = durableInvocation?.identity.childRunId ?? crypto.randomUUID();
       const childController = new AbortController();
       const childRef: WorkflowChildRunRef = { alias: childName, workflow: child.normalizedName, runId: childRunId };
       boundary.linkChildRun(childRef, childController);
@@ -107,7 +113,7 @@ export function createChildWorkflowRunner(input: {
         },
         signal: childController.signal,
         deferWorkflowStart: false,
-        ...(input.consumeDurableScope !== undefined ? { durableScope: input.consumeDurableScope() } : {}),
+        ...(durableInvocation !== undefined ? { durableScope: durableInvocation.scope } : {}),
       });
       boundary.observeChildRun(childRunPromise);
       const childRun = await childRunPromise;
@@ -150,14 +156,17 @@ export function createChildWorkflowRunner(input: {
         `Workflow "${child.name}" ${childRun.status} (runId: ${childRun.runId}; outputs: ${outputKeys.length > 0 ? outputKeys.join(", ") : "(none)"})`,
         workflowChild,
       );
+      await durableInvocation?.recordTerminal("completed", childResult);
       return childResult;
     } catch (err) {
       const exit = findWorkflowExitSignal(err, runtime.exit.exitScope) ?? findWorkflowExitSignal(runtime.signal.reason, runtime.exit.exitScope);
       if (exit !== undefined) {
         await boundary.skipForWorkflowExit(exit.reason);
+        await durableInvocation?.recordTerminal("skipped");
         throw exit;
       }
       boundary.fail(err);
+      await durableInvocation?.recordTerminal("failed");
       throw err;
     } finally {
       detachParentAbort?.();

package/dist/builtin/workflows/src/engine/run-durable-finalize.ts

@@ -39,7 +39,17 @@ export async function finalizeDurableTerminalStatus(input: DurableTerminalFinali
     if (durableStatus === "failed" || durableStatus === "blocked") {
       recordRunTimingCheckpoint(input.durableBackend, input.runSnapshot);
     }
-    input.durableBackend.setWorkflowStatus(input.runId, durableStatus, undefined, input.runSnapshot.resumable);
+    const failure = durableStatus === "failed" && input.runSnapshot.error !== undefined
+      ? {
+          error: input.runSnapshot.error,
+          ...(input.runSnapshot.failureKind !== undefined ? { failureKind: input.runSnapshot.failureKind } : {}),
+          ...(input.runSnapshot.failureCode !== undefined ? { failureCode: input.runSnapshot.failureCode } : {}),
+          ...(input.runSnapshot.failureRecoverability !== undefined ? { failureRecoverability: input.runSnapshot.failureRecoverability } : {}),
+          ...(input.runSnapshot.failureDisposition !== undefined ? { failureDisposition: input.runSnapshot.failureDisposition } : {}),
+          ...(input.runSnapshot.failedToolNodeId !== undefined ? { failedToolNodeId: input.runSnapshot.failedToolNodeId } : {}),
+        }
+      : undefined;
+    input.durableBackend.setWorkflowStatus(input.runId, durableStatus, undefined, input.runSnapshot.resumable, failure);
   }
   await input.durableBackend.flush();
 }

package/dist/builtin/workflows/src/engine/run-durable-topology.ts

@@ -1,16 +1,24 @@
 import { durableHash, type DurableWorkflowBackend } from "../durable/backend.js";
-import { durableNestedRunSnapshots } from "../durable/completed-catalog.js";
+import { durableCompletedNestedRunSubtree } from "../durable/completed-subtree.js";
+import { DurableNestedTopologyError } from "../durable/boundary-topology.js";
 import {
-  durableStageCheckpointMetadata,
   recordCachedStageWithTracker,
+  recordStageCheckpoint,
   type DurableCompletedStageCheckpoint,
   type DurableStageDeps,
 } from "../durable/stage-primitive.js";
-import type { DurableStageRunTopology } from "../durable/types.js";
+import { durableStageCheckpointMetadata } from "../durable/stage-topology.js";
+import { promptOccurrenceIdentityError } from "../durable/stage-topology-validation.js";
+import type { DurableStageCheckpoint, DurableStageRunTopology, DurableStageTopology } from "../durable/types.js";
+import { parseLegacyWorkflowChildResult, parseWorkflowChildResult } from "../durable/workflow-child-result.js";
 import type { GraphFrontierTracker } from "./graph-inference.js";
-import type { RunSnapshot } from "../shared/store-types.js";
+import type { RunSnapshot, StageSnapshot } from "../shared/store-types.js";
 import type { Store } from "../shared/store.js";
-import type { ParallelFailFastScope } from "../runs/foreground/executor-types.js";
+import type { ParallelFailFastScope, RunOpts } from "../runs/foreground/executor-types.js";
+import {
+  authoritativeWorkflowChildRunId,
+  reciprocalWorkflowRootRunId,
+} from "../shared/workflow-run-ownership.js";
 
 export function durableRunTopology(run: RunSnapshot): DurableStageRunTopology {
   return {
@@ -36,9 +44,82 @@ export function createDurableStageDeps(input: {
     nextReplayKey: input.nextReplayKey,
     replayKeyForCompletedStage: (stage) => input.completedReplayKeys.get(stage.id),
     runTopology: durableRunTopology(input.run),
+    sourceOrderForStage: (stage) => {
+      const order = input.run.stages.findIndex((candidate) => candidate.id === stage.id);
+      return order >= 0 ? order : undefined;
+    },
+  };
+}
+
+export function createDurableStageEndRecorder(input: {
+  readonly rootRunId: string;
+  readonly deps: DurableStageDeps;
+  readonly user?: RunOpts["onStageEnd"];
+  readonly metadataOnly?: boolean;
+}): (runId: string, snapshot: StageSnapshot) => Promise<void> {
+  return async (runId, snapshot): Promise<void> => {
+    if (runId === input.rootRunId) {
+      await recordStageCheckpoint(input.deps, snapshot, { metadataOnly: input.metadataOnly });
+    }
+    await input.user?.(runId, snapshot);
   };
 }
 
+/** Snapshot prior-process stage identities so live writes cannot be mistaken for resume topology. */
+export function createDurableStageTopologyResolver(
+  backend: DurableWorkflowBackend,
+  workflowId: string,
+): (replayKey: string) => DurableStageTopology | undefined {
+  const byReplayKey = new Map<string, Map<string, DurableStageTopology>>();
+  const ownedStages: DurableStageCheckpoint[] = [];
+  for (const checkpoint of backend.listCheckpoints(workflowId)) {
+    if (checkpoint.kind !== "stage" || checkpoint.topology === undefined) continue;
+    const topology = checkpoint.topology;
+    if (topology.run !== undefined && topology.run.runId !== workflowId) continue;
+    ownedStages.push(checkpoint);
+    const byStage = byReplayKey.get(checkpoint.replayKey) ?? new Map<string, DurableStageTopology>();
+    byStage.set(topology.stageId, topology);
+    byReplayKey.set(checkpoint.replayKey, byStage);
+  }
+  const occurrenceError = promptOccurrenceIdentityError(ownedStages);
+  const queues = new Map<string, readonly DurableStageTopology[]>();
+  for (const [replayKey, byStage] of byReplayKey) {
+    const stages = [...byStage.values()];
+    const sourceOrders = stages.map((stage) => stage.sourceOrder);
+    const safe = stages.length <= 1 || (sourceOrders.every((order) => order !== undefined)
+      && new Set(sourceOrders).size === sourceOrders.length);
+    queues.set(replayKey, safe
+      ? stages.sort((left, right) => (left.sourceOrder ?? 0) - (right.sourceOrder ?? 0))
+      : []);
+  }
+  const indexes = new Map<string, number>();
+  return (replayKey): DurableStageTopology | undefined => {
+    if (occurrenceError !== undefined) throw new DurableNestedTopologyError(occurrenceError);
+    const index = indexes.get(replayKey) ?? 0;
+    indexes.set(replayKey, index + 1);
+    return queues.get(replayKey)?.[index];
+  };
+}
+
+/** Persist an in-progress stage identity before an external wait can outlive this process. */
+export async function recordDurableActiveStage(
+  deps: DurableStageDeps,
+  stage: StageSnapshot,
+): Promise<boolean> {
+  if (stage.replayKey === undefined
+    || stage.status === "completed" || stage.status === "failed" || stage.status === "skipped") return false;
+  await deps.backend.recordCheckpointAsync({
+    kind: "stage",
+    workflowId: deps.workflowId,
+    checkpointId: `stage-active-meta:${durableHash({ replayKey: stage.replayKey, stageId: stage.id })}`,
+    name: stage.name,
+    replayKey: stage.replayKey,
+    completedAt: Date.now(),
+    ...durableStageCheckpointMetadata(stage, deps.runTopology, deps.sourceOrderForStage?.(stage)),
+  });
+  return true;
+}
+
 export function createDurableCachedStageRecorder(input: {
   readonly store: Store;
   readonly tracker: GraphFrontierTracker;
@@ -46,16 +127,39 @@ export function createDurableCachedStageRecorder(input: {
   readonly backend: DurableWorkflowBackend;
   readonly rootBackend: DurableWorkflowBackend;
   readonly completedStageReplayKeys: Map<string, string>;
+  readonly sourceToReplayedNodeIds: Map<string, string>;
 }): {
   readonly record: (name: string, replayKey: string, checkpoint: DurableCompletedStageCheckpoint, scope?: ParallelFailFastScope) => void;
   readonly metadata: (replayKey: string) => Partial<DurableCompletedStageCheckpoint>;
 } {
+  const replaySourceOrders = new Map<string, number>();
   return {
     record(name, replayKey, checkpoint, scope): void {
+      const cachedChildRunId = workflowChildRunId(checkpoint);
+      const runById = new Map(input.store.runs().map((run) => [run.id, run]));
+      if (!runById.has(input.run.id)) runById.set(input.run.id, input.run);
+      const durableRootId = reciprocalWorkflowRootRunId(runById, input.run.id);
+      const cachedDescendants = cachedChildRunId === undefined || durableRootId === undefined
+        ? undefined
+        : durableCompletedNestedRunSubtree(input.rootBackend, durableRootId, cachedChildRunId);
+      if (cachedChildRunId !== undefined && durableRootId !== undefined && cachedDescendants === undefined) {
+        throw new DurableNestedTopologyError(
+          `cached completed child subtree is incomplete or inconsistent at ${replayKey}`,
+        );
+      }
       recordCachedStageWithTracker(
         input.store, input.tracker, input.run.id, name, replayKey, checkpoint,
-        input.completedStageReplayKeys, scope,
+        input.completedStageReplayKeys, scope, input.sourceToReplayedNodeIds,
       );
+      const sourceOrder = checkpoint.topology?.sourceOrder;
+      if (sourceOrder !== undefined) {
+        const replayedStage = input.run.stages.find((candidate) => candidate.replayKey === replayKey);
+        if (replayedStage !== undefined) replaySourceOrders.set(replayedStage.id, sourceOrder);
+        input.run.stages.sort((left, right) =>
+          (replaySourceOrders.get(left.id) ?? Number.MAX_SAFE_INTEGER)
+          - (replaySourceOrders.get(right.id) ?? Number.MAX_SAFE_INTEGER)
+        );
+      }
       const stage = input.store.runs().find((run) => run.id === input.run.id)?.stages
         .find((candidate) => candidate.replayKey === replayKey);
       if (stage !== undefined) {
@@ -63,27 +167,95 @@ export function createDurableCachedStageRecorder(input: {
           kind: "stage", workflowId: input.run.id,
           checkpointId: `stage-replay-meta:${durableHash({ replayKey, stageId: stage.id, parentIds: stage.parentIds })}`,
           name, replayKey, completedAt: Date.now(),
-          ...durableStageCheckpointMetadata(stage, durableRunTopology(input.run)),
+          ...durableStageCheckpointMetadata(
+            stage,
+            durableRunTopology(input.run),
+            sourceOrder ?? input.run.stages.findIndex((candidate) => candidate.id === stage.id),
+          ),
         });
       }
-      if (workflowChildRunId(checkpoint) === undefined) return;
-      const durableRootId = input.run.rootRunId ?? input.run.id;
-      for (const childRun of durableNestedRunSnapshots(input.rootBackend, durableRootId)) {
-        if (!input.store.runs().some((candidate) => candidate.id === childRun.id)) input.store.recordRunStart(childRun);
+      for (const childRun of cachedDescendants ?? []) {
+        const existingRun = input.store.runs().find((candidate) => candidate.id === childRun.id);
+        const isDirectChild = childRun.id === cachedChildRunId
+          && childRun.parentRunId === input.run.id
+          && stage?.workflowChild?.runId === cachedChildRunId;
+        if (existingRun !== undefined) {
+          if (stage !== undefined && isDirectChild) {
+            reconcileCachedDirectChildParentStage({
+              store: input.store,
+              parentRun: input.run,
+              catalogRun: childRun,
+              checkpointChildRunId: cachedChildRunId!,
+              boundary: stage,
+            });
+          }
+          continue;
+        }
+        const hydratedRun = isDirectChild && stage !== undefined
+          ? { ...childRun, parentStageId: stage.id }
+          : childRun;
+        input.store.recordRunStart(hydratedRun);
+
       }
     },
     metadata(replayKey) {
       const stage = input.run.stages.find((candidate) => candidate.replayKey === replayKey);
-      return stage === undefined ? {} : durableStageCheckpointMetadata(stage, durableRunTopology(input.run));
+      return stage === undefined ? {} : durableStageCheckpointMetadata(
+        stage,
+        durableRunTopology(input.run),
+        input.run.stages.findIndex((candidate) => candidate.id === stage.id),
+      );
     },
   };
 }
 
+export function reconcileCachedDirectChildParentStage(input: {
+  readonly store: Store;
+  readonly parentRun: RunSnapshot;
+  readonly catalogRun: RunSnapshot;
+  readonly checkpointChildRunId: string;
+  readonly boundary: StageSnapshot;
+}): boolean {
+  const existingRun = input.store.runs().find((run) => run.id === input.checkpointChildRunId);
+  const runById = new Map(input.store.runs().map((run) => [run.id, run]));
+  const storedParentRun = runById.get(input.parentRun.id);
+  if (
+    storedParentRun !== undefined &&
+    (storedParentRun.parentRunId !== input.parentRun.parentRunId ||
+      storedParentRun.parentStageId !== input.parentRun.parentStageId)
+  ) return false;
+  if (storedParentRun === undefined) runById.set(input.parentRun.id, input.parentRun);
+  const expectedRootRunId = reciprocalWorkflowRootRunId(runById, input.parentRun.id);
+  if (expectedRootRunId === undefined) return false;
+  const storedBoundary = storedParentRun?.stages.find((stage) => stage.id === input.boundary.id);
+  if (
+    !rootMatches(input.parentRun.rootRunId, expectedRootRunId) ||
+    !rootMatches(storedParentRun?.rootRunId, expectedRootRunId) ||
+    authoritativeWorkflowChildRunId(input.boundary) !== input.checkpointChildRunId ||
+    (storedParentRun !== undefined &&
+      authoritativeWorkflowChildRunId(storedBoundary) !== input.checkpointChildRunId) ||
+    input.catalogRun.id !== input.checkpointChildRunId ||
+    existingRun?.status !== "completed" || input.catalogRun.status !== "completed" ||
+    existingRun.parentRunId !== input.parentRun.id || input.catalogRun.parentRunId !== input.parentRun.id ||
+    input.catalogRun.parentStageId === undefined || existingRun.parentStageId !== input.catalogRun.parentStageId ||
+    !rootMatches(existingRun.rootRunId, expectedRootRunId) ||
+    !rootMatches(input.catalogRun.rootRunId, expectedRootRunId)
+  ) return false;
+  return input.store.reconcileRunParentStage(
+    existingRun.id,
+    input.catalogRun.parentStageId,
+    input.boundary.id,
+  );
+}
+
+function rootMatches(rootRunId: string | undefined, expectedRootRunId: string): boolean {
+  return rootRunId === undefined || rootRunId === expectedRootRunId;
+}
+
 function workflowChildRunId(checkpoint: DurableCompletedStageCheckpoint): string | undefined {
-  const output = checkpoint.output;
-  if (typeof output !== "object" || output === null || Array.isArray(output)) return undefined;
-  const record = output as Record<string, import("../shared/types.js").WorkflowSerializableValue>;
-  const runId = record["runId"];
-  const workflow = record["workflow"];
-  return typeof runId === "string" && typeof workflow === "string" ? runId : undefined;
+  const topology = checkpoint.topology;
+  const hasCurrentIdentity = topology?.sourceOrder !== undefined || topology?.status !== undefined
+    || topology?.occurrenceKey !== undefined || topology?.boundary !== undefined;
+  return (parseWorkflowChildResult(checkpoint.output)
+    ?? (hasCurrentIdentity ? undefined : parseLegacyWorkflowChildResult(checkpoint.output)))?.runId;
 }

package/dist/builtin/workflows/src/engine/run-terminal-event.ts

@@ -0,0 +1,37 @@
+export type RunTerminalEvent =
+  | { readonly kind: "failure"; readonly error: unknown; readonly nodeId?: string }
+  | { readonly kind: "cancellation"; readonly reason: unknown };
+
+export interface RunTerminalEventArbiter {
+  selectFailure(error: unknown, nodeId?: string): RunTerminalEvent;
+  selectCancellation(reason: unknown): RunTerminalEvent;
+  winner(): RunTerminalEvent | undefined;
+  register(): void;
+  dispose(): void;
+}
+
+const activeArbiters = new Map<string, RunTerminalEventArbiter>();
+
+/** Latches the first executor failure or real run cancellation for one live run. */
+export function createRunTerminalEventArbiter(runId: string): RunTerminalEventArbiter {
+  let selected: RunTerminalEvent | undefined;
+  const select = (event: RunTerminalEvent): RunTerminalEvent => {
+    selected ??= event;
+    return selected;
+  };
+  const arbiter: RunTerminalEventArbiter = {
+    selectFailure: (error, nodeId) => select({ kind: "failure", error, ...(nodeId === undefined ? {} : { nodeId }) }),
+    selectCancellation: (reason) => select({ kind: "cancellation", reason }),
+    winner: () => selected,
+    register() { activeArbiters.set(runId, arbiter); },
+    dispose() {
+      if (activeArbiters.get(runId) === arbiter) activeArbiters.delete(runId);
+    },
+  };
+  return arbiter;
+}
+
+/** Read the selected event so an external kill cannot overwrite an earlier failure. */
+export function selectedRunTerminalEvent(runId: string): RunTerminalEvent | undefined {
+  return activeArbiters.get(runId)?.winner();
+}

package/dist/builtin/workflows/src/engine/run-terminal-failure.ts

@@ -0,0 +1,50 @@
+import type { RunSnapshot } from "../shared/store-types.js";
+import type { Store } from "../shared/store.js";
+import type { WorkflowPersistencePort } from "../shared/types.js";
+import {
+  appendRunEndWhenRecorded,
+  reconcileTerminalRunResult,
+  type SelectedRunFailureMetadata,
+} from "../runs/foreground/executor-lifecycle.js";
+import type { RunOpts, RunResult } from "../runs/foreground/executor-types.js";
+
+/** Record, persist, and return one ordinary terminal workflow failure. */
+export function finalizeTerminalFailure(input: {
+  readonly runId: string;
+  readonly runSnapshot: RunSnapshot;
+  readonly store: Store;
+  readonly persistence?: WorkflowPersistencePort;
+  readonly metadata: SelectedRunFailureMetadata & { readonly failedToolNodeId?: string };
+  readonly onRunEnd: RunOpts["onRunEnd"];
+}): RunResult {
+  const { runId, runSnapshot, store, persistence, metadata, onRunEnd } = input;
+  const failedToolNode = metadata.failedToolNodeId === undefined
+    ? undefined
+    : runSnapshot.toolNodes?.find((node) => node.id === metadata.failedToolNodeId);
+  const recorded = store.recordRunEnd(runId, "failed", undefined, metadata.errorMessage, metadata);
+  appendRunEndWhenRecorded(persistence, recorded, {
+    runId,
+    status: "failed",
+    error: metadata.errorMessage,
+    failureKind: metadata.failureKind,
+    ...(metadata.failureCode !== undefined ? { failureCode: metadata.failureCode } : {}),
+    ...(metadata.failureRecoverability !== undefined ? { failureRecoverability: metadata.failureRecoverability } : {}),
+    ...(metadata.failureDisposition !== undefined ? { failureDisposition: metadata.failureDisposition } : {}),
+    failureMessage: metadata.failureMessage,
+    ...(metadata.failedStageId !== undefined ? { failedStageId: metadata.failedStageId } : {}),
+    ...(metadata.failedToolNodeId !== undefined ? { failedToolNodeId: metadata.failedToolNodeId } : {}),
+    ...(failedToolNode !== undefined ? { failedToolNode } : {}),
+    resumable: metadata.resumable,
+    ...(metadata.retryAfterMs !== undefined ? { retryAfterMs: metadata.retryAfterMs } : {}),
+    ...(runSnapshot.endedAt !== undefined ? { endedAt: runSnapshot.endedAt } : {}),
+    ...(runSnapshot.durationMs !== undefined ? { durationMs: runSnapshot.durationMs } : {}),
+    ts: Date.now(),
+  });
+  return reconcileTerminalRunResult(
+    runId,
+    runSnapshot,
+    store,
+    { status: "failed", error: metadata.errorMessage },
+    onRunEnd,
+  );
+}