@bastani/atomic 0.9.11-alpha.5 → 0.9.11-alpha.7

package/dist/builtin/workflows/src/engine/run.ts

@@ -1,5 +1,5 @@
 import { nextEventLoopTurn, runWorkflowDefinitionCallback } from "./workflow-activity.js";
-import type { RunSnapshot, StageSnapshot } from "../shared/store-types.js";
+import type { RunSnapshot } from "../shared/store-types.js";
 import type { WorkflowDefinition, WorkflowInputValues, WorkflowOutputValues, WorkflowRunContext } from "../shared/types.js";
 import type { WorkflowFailure } from "../shared/workflow-failures.js";
 import { classifyWorkflowFailure } from "../shared/workflow-failures.js";
@@ -26,11 +26,12 @@ import { createRunFinalizers } from "../runs/foreground/executor-run-finalizers.
 import { buildPromptNodeUiAdapter } from "../runs/foreground/executor-prompt-nodes.js";
 import {
   appendRunEndWhenRecorded,
-  assertWorkflowCreatedStage,
+  assertWorkflowCreatedExecution,
   finalizeKilled,
   finalizeKilledByFailure,
   recordActiveBlockedFailure,
   normalizeStageLessFailureMetadata,
+  normalizeFailureWinnerMetadata,
   reconcileTerminalRunResult,
   selectRunFailureDisposition,
 } from "../runs/foreground/executor-lifecycle.js";
@@ -38,19 +39,21 @@ import { assertWorkflowRunOutputs, normalizeWorkflowRunOutput } from "../runs/fo
 import { isWorkflowDefinition, workflowDefinitionRequirementMessage } from "../runs/foreground/executor-child-helpers.js";
 import { getDurableBackend } from "../durable/factory.js";
 import { classifyReturnedRunStatus } from "./run-returned-status.js";
-import { createToolPrimitive, createCheckpointIdGenerator } from "../durable/tool-primitive.js";
-import { createDurableStagePrimitive, createDurableTaskPrimitive, recordStageCheckpoint, createStageReplayKeyGenerator } from "../durable/stage-primitive.js";
+import { createCheckpointIdGenerator } from "../durable/tool-primitive.js";
+import { createDurableStagePrimitive, createDurableTaskPrimitive, createStageReplayKeyGenerator } from "../durable/stage-primitive.js";
 import { inheritedRunElapsedMs, recordRunTimingCheckpoint } from "../durable/run-timing.js";
 import { createDurableChildWorkflowPrimitive } from "../durable/child-primitive.js";
-import { ScopedDurableBackend, type DurableScope } from "../durable/scoped-backend.js";
+import { ScopedDurableBackend } from "../durable/scoped-backend.js";
+import type { DurableChildInvocation } from "../durable/boundary-topology.js";
 import { finalizeDurableTerminalStatus } from "./run-durable-finalize.js";
 import { createDurableStageSessionRecorder } from "./run-durable-stage-session.js";
 import type { DurableWorkflowBackend } from "../durable/backend.js";
-import { createDurableCachedStageRecorder, createDurableStageDeps } from "./run-durable-topology.js";
+import { createDurableCachedStageRecorder, createDurableStageDeps, createDurableStageEndRecorder, createDurableStageTopologyResolver, durableRunTopology, recordDurableActiveStage } from "./run-durable-topology.js";
 import { admitDurableRootRun, durableRootRegistrationForRun } from "./run-durable-admission.js";
-
+import { createTrackedToolPrimitive } from "./run-tool-node-lifecycle.js";
+import { createRunTerminalEventArbiter } from "./run-terminal-event.js";
+import { finalizeTerminalFailure } from "./run-terminal-failure.js";
 type WorkflowRunInputArgument = Parameters<typeof resolveAndValidateInputs>[1];
-
 export function run<
   TInputs extends WorkflowInputValues,
   TOutputs extends WorkflowOutputValues,
@@ -69,7 +72,6 @@ export async function run<
   opts: RunOpts = {},
 ): Promise<RunResult> {
   if (!isWorkflowDefinition(def)) throw new Error(workflowDefinitionRequirementMessage("run(definition, inputs)", def));
-
   const activeStore = opts.store ?? defaultStore;
   const adapters = opts.adapters ?? {};
   // Prompt-node UI is the graph-mode transport and intentionally takes precedence
@@ -77,7 +79,6 @@ export async function run<
   if (opts.usePromptNodesForUi === true && opts.ui !== undefined) {
     console.warn("atomic-workflows: usePromptNodesForUi ignores the provided RunOpts.ui adapter");
   }
-
   const depth = opts.depth ?? 0;
   const maxDepth = opts.config?.maxDepth ?? 4;
   if (depth >= maxDepth) {
@@ -86,6 +87,7 @@ export async function run<
       status: "failed",
       error: `atomic-workflows: maxDepth exceeded (max ${maxDepth})`,
       stages: [],
+      toolNodes: [],
     };
   }
 
@@ -93,32 +95,32 @@ export async function run<
   const runId = opts.runId ?? crypto.randomUUID();
   const exitScope = Symbol(`workflow-exit:${runId}`);
   const ownController = new AbortController();
+  const terminalEvents = createRunTerminalEventArbiter(runId);
+  ownController.signal.addEventListener("abort", () => { terminalEvents.selectCancellation(ownController.signal.reason); }, { once: true });
   const callerSignal = opts.signal;
   if (callerSignal) {
     if (callerSignal.aborted) ownController.abort(callerSignal.reason);
     else callerSignal.addEventListener("abort", () => { ownController.abort(callerSignal.reason); }, { once: true });
   }
   const exit = createWorkflowExitManager({ runId, exitScope, controller: ownController });
-  // Durable workflow backend — registers this run and wires ctx.tool/ui/stage.
-  // Declared early so the stage-end recorder can attach to stageOptions and so
-  // resume can seed inherited elapsed time. Child runs with a durable scope
-  // route their internal side-effect checkpoints under the root workflow so
-  // interrupted children do not re-execute completed side effects on parent
-  // resume. cross-ref: issue #1498 — DBOS-backed cross-session resumability.
-  const rootBackend: DurableWorkflowBackend = opts.durableBackend ?? getDurableBackend();
+  // Durable child operations stay on stacked scoped views, while cached graph
+  // reconstruction keeps the physical root backend through arbitrary depth.
+  // cross-ref: issue #1498 — DBOS-backed cross-session resumability.
+  const backendView: DurableWorkflowBackend = opts.durableBackend ?? getDurableBackend();
+  const rootBackend: DurableWorkflowBackend = opts.durableRootBackend ?? backendView;
   const durableBackend: DurableWorkflowBackend = opts.durableScope !== undefined
-    ? new ScopedDurableBackend(rootBackend, opts.durableScope)
-    : rootBackend;
+    ? new ScopedDurableBackend(backendView, opts.durableScope)
+    : backendView;
   const inheritedElapsedMs = opts.parentRun === undefined
     ? inheritedRunElapsedMs({ backend: durableBackend, runId, continuationSource: opts.continuation?.source })
     : undefined;
-
   const runSnapshot: RunSnapshot = {
     id: runId,
     name: def.name,
     inputs: Object.freeze(resolvedInputs),
     status: "running" as const,
     stages: [],
+    toolNodes: [],
     startedAt: Date.now(),
     ...(opts.parentRun !== undefined ? {
       parentRunId: opts.parentRun.runId,
@@ -131,7 +133,6 @@ export async function run<
     } : {}),
     ...(inheritedElapsedMs !== undefined ? { accumulatedDurationMs: inheritedElapsedMs } : {}),
   };
-
   const classifiedFailures = new Map<unknown, WorkflowFailure>();
   const classifyExecutorFailure = (error: unknown): WorkflowFailure => {
     const cached = classifiedFailures.get(error);
@@ -145,7 +146,6 @@ export async function run<
     classifiedFailures.set(error, classified);
     return classified;
   };
-
   activeStore.recordRunStart(runSnapshot);
   if (!opts.signal) opts.cancellation?.register(runId, ownController);
   opts.onRunStart?.(runSnapshot);
@@ -163,7 +163,6 @@ export async function run<
       ts: runSnapshot.startedAt,
     });
   }
-
   const tracker = new GraphFrontierTracker();
   const inputConcurrency = resolveInputConcurrency(def.inputs, resolvedInputs);
   const inputRuntimeDefaults = resolveInputRuntimeDefaults(def, resolvedInputs), gitWorktreeSetupCacheOwner = createGitWorktreeSetupCacheOwner(opts.gitWorktreeSetupCache);
@@ -189,7 +188,6 @@ export async function run<
     () => scheduler.rejectReleaseBarriers(ownController.signal.reason ?? new Error("atomic-workflows: run aborted")),
     { once: true },
   );
-
   const finalizers = createRunFinalizers({
     def,
     runId,
@@ -202,18 +200,16 @@ export async function run<
   const checkpointIdGenerator = createCheckpointIdGenerator();
   const stageReplayKeyGenerator = createStageReplayKeyGenerator(runId);
   const completedStageReplayKeys = new Map<string, string>();
+  const sourceToReplayedNodeIds = new Map<string, string>();
   const durableStageDeps = createDurableStageDeps({
     backend: durableBackend, run: runSnapshot,
     nextCheckpointId: checkpointIdGenerator, nextReplayKey: stageReplayKeyGenerator,
     completedReplayKeys: completedStageReplayKeys,
   });
-  const userOnStageEnd = opts.onStageEnd;
-  const durableOnStageEnd = async (stageRunId: string, snapshot: StageSnapshot): Promise<void> => {
-    if (stageRunId === runId && snapshot.status === "completed") {
-      await recordStageCheckpoint(durableStageDeps, snapshot);
-    }
-    await userOnStageEnd?.(stageRunId, snapshot);
-  };
+  const durableOnStageEnd = createDurableStageEndRecorder({ rootRunId: runId, deps: durableStageDeps, user: opts.onStageEnd });
+  const durableOnPromptNodeEnd = createDurableStageEndRecorder({
+    rootRunId: runId, deps: durableStageDeps, user: opts.onStageEnd, metadataOnly: true,
+  });
   const durableOnStageSession = createDurableStageSessionRecorder({
     runId, deps: durableStageDeps, onStageSession: opts.onStageSession,
     ...(opts.parentRun === undefined ? { runSnapshot } : {}),
@@ -255,6 +251,7 @@ export async function run<
     onStageEnd: opts.onStageEnd,
     onStageSession: opts.onStageSession,
     durableBackend,
+    durableRootBackend: rootBackend,
   };
   const runtime = new EngineRuntime({
     runId,
@@ -280,36 +277,34 @@ export async function run<
   });
   const workflowBoundaryReplayCounts = new Map<string, number>();
   const nextWorkflowBoundaryReplayKey = (name: string): string => {
-    const durableScopePrefix = pendingChildDurableScope?.scopePrefix;
+    const durableScopePrefix = pendingChildDurableInvocation?.scope.scopePrefix;
     if (durableScopePrefix !== undefined && durableScopePrefix.startsWith(`workflow:${name}:`)) return durableScopePrefix;
     const next = (workflowBoundaryReplayCounts.get(name) ?? 0) + 1;
     workflowBoundaryReplayCounts.set(name, next);
     return `workflow:${name}:${next}`;
   };
-  // Durable child workflow replay keys use a SEPARATE counter so that cache
-  // hits (which do not invoke the inner workflow runner) do not desync the
-  // ordinal sequence. Without this, repeated ctx.workflow(child) calls would
-  // shift replay keys on resume and re-execute completed children.
-  // cross-ref: issue #1498.
+  // Durable scopes are keyed by definition+validated-input fingerprint. Only
+  // truly identical invocations share an ordinal sequence, so reversed
+  // parallel dispatch cannot exchange distinct cached child results.
   const durableChildReplayCounts = new Map<string, number>();
-  const nextDurableChildReplayKey = (name: string): string => {
-    const next = (durableChildReplayCounts.get(name) ?? 0) + 1;
-    durableChildReplayCounts.set(name, next);
-    return `workflow:${name}:${next}`;
+  const nextDurableChildReplayKey = (name: string, invocationFingerprint: string): string => {
+    const identity = `${name}:${invocationFingerprint}`;
+    const next = (durableChildReplayCounts.get(identity) ?? 0) + 1;
+    durableChildReplayCounts.set(identity, next);
+    return `workflow:${name}:${invocationFingerprint}:${next}`;
   };
   const taskRunners = createWorkflowTaskRunners({ runtime });
-  // Durable scope holder: the durable child primitive publishes the scope for
-  // the next child invocation; the child runner consumes it when launching the
-  // run. This routes child internal side-effect checkpoints under the root.
-  let pendingChildDurableScope: DurableScope | undefined;
+  // The outer primitive publishes one validated boundary/child identity; the
+  // inner runner consumes it before dispatching child workflow code.
+  let pendingChildDurableInvocation: DurableChildInvocation | undefined;
   const workflow = createChildWorkflowRunner({
     runtime,
     resolveWorkflowCwd,
     nextWorkflowBoundaryReplayKey,
-    consumeDurableScope: () => {
-      const scope = pendingChildDurableScope;
-      pendingChildDurableScope = undefined;
-      return scope;
+    consumeDurableInvocation: () => {
+      const invocation = pendingChildDurableInvocation;
+      pendingChildDurableInvocation = undefined;
+      return invocation;
     },
     runWorkflow: run,
   });
@@ -318,23 +313,45 @@ export async function run<
     hasPersistence: opts.persistence !== undefined, isChildRun: opts.parentRun !== undefined,
     continuationSourceId: opts.continuation?.source.id,
   });
-  const tool = createToolPrimitive({
+  const { tool, admittedTools } = createTrackedToolPrimitive({
     workflowId: runId,
     backend: durableBackend,
     nextCheckpointId: checkpointIdGenerator,
-    throwIfCancelled: () => {
-      if (ownController.signal.aborted) {
-        throw new Error("atomic-workflows: workflow cancelled");
-      }
-    },
-    signal: ownController.signal,
+    controller: ownController,
+    terminalEvents,
+    store: activeStore,
+    tracker,
+    run: runSnapshot,
+    sourceToReplayedNodeIds,
   });
-
-  // Durable ctx.ui wrapper — caches completed user responses so a resumed workflow does not re-ask answered prompts.
-  const durableUiDeps = { workflowId: runId, backend: durableBackend, nextCheckpointId: checkpointIdGenerator };
+  let selectedAdmittedToolFailure: ReturnType<typeof admittedTools.firstFailure> = undefined;
+  // Prompt-node mode re-materializes metadata before returning a durable ctx.ui cache hit.
+  const resolvePromptNodeTopology = createDurableStageTopologyResolver(durableBackend, runId);
+  let promptNodeUi: ReturnType<typeof buildPromptNodeUiAdapter> | undefined;
+  const getPromptNodeUi = (): ReturnType<typeof buildPromptNodeUiAdapter> => {
+    promptNodeUi ??= buildPromptNodeUiAdapter({
+      runId, activeStore, tracker, replayIndex, classifyExecutorFailure,
+      opts: { ...opts, onStageEnd: durableOnPromptNodeEnd }, stageControlRegistry: stageRegistry,
+      signal: ownController.signal,
+      throwIfWorkflowExitSelected: exit.throwIfWorkflowExitSelected,
+      registerWorkflowExitCleanup: exit.registerWorkflowExitCleanup,
+      workflowExitSkippedReason: exit.workflowExitSkippedReason,
+      preserveWorkflowExitSkippedReason: exit.preserveWorkflowExitSkippedReason,
+      durableTopologyForReplayKey: resolvePromptNodeTopology,
+      onPendingStage: async (pendingRunId, snapshot) => pendingRunId === runId
+        ? void await recordDurableActiveStage(durableStageDeps, snapshot) : undefined,
+    });
+    return promptNodeUi;
+  };
+  const durableUiDeps = {
+    workflowId: runId, backend: durableBackend, nextCheckpointId: checkpointIdGenerator,
+    ...(opts.usePromptNodesForUi === true ? { onReplay: async (
+      request: Parameters<ReturnType<typeof buildPromptNodeUiAdapter>["replayDurable"]>[0],
+    ) => getPromptNodeUi().replayDurable(request) } : {}),
+  };
   const cachedStage = createDurableCachedStageRecorder({
     store: activeStore, tracker, run: runSnapshot, backend: durableBackend,
-    rootBackend, completedStageReplayKeys,
+    rootBackend, completedStageReplayKeys, sourceToReplayedNodeIds,
   });
   const durableTask = createDurableTaskPrimitive({
     workflowId: runId, backend: durableBackend,
@@ -343,9 +360,10 @@ export async function run<
   });
   const durableWorkflow = createDurableChildWorkflowPrimitive({
     workflowId: runId, rootWorkflowId: opts.parentRun?.rootRunId ?? runId, backend: durableBackend,
-    nextReplayKey: nextDurableChildReplayKey, setChildDurableScope: (scope) => { pendingChildDurableScope = scope; },
+    nextReplayKey: nextDurableChildReplayKey,
+    setChildDurableInvocation: (invocation) => { pendingChildDurableInvocation = invocation; },
     recordCachedStage: cachedStage.record,
-    checkpointMetadata: cachedStage.metadata,
+    runTopology: durableRunTopology(runSnapshot),
     workflow,
   });
   const ctx: WorkflowRunContext<TInputs> = {
@@ -356,20 +374,7 @@ export async function run<
       opts,
       throwIfWorkflowExitSelected: exit.throwIfWorkflowExitSelected,
       durableUi: durableUiDeps,
-      baseFromPromptNodes: () => buildPromptNodeUiAdapter({
-        runId,
-        activeStore,
-        opts,
-        stageControlRegistry: stageRegistry,
-        tracker,
-        replayIndex,
-        signal: ownController.signal,
-        throwIfWorkflowExitSelected: exit.throwIfWorkflowExitSelected,
-        registerWorkflowExitCleanup: exit.registerWorkflowExitCleanup,
-        workflowExitSkippedReason: exit.workflowExitSkippedReason,
-        preserveWorkflowExitSkippedReason: exit.preserveWorkflowExitSkippedReason,
-        classifyExecutorFailure,
-      }),
+      baseFromPromptNodes: getPromptNodeUi,
     }),
     stage: createDurableStagePrimitive({
       workflowId: runId,
@@ -390,11 +395,12 @@ export async function run<
     tool,
     ...(opts.models !== undefined ? { models: opts.models } : {}),
   };
-
+  terminalEvents.register();
   try {
     if (opts.deferWorkflowStart === true) {
       await nextEventLoopTurn();
       if (ownController.signal.aborted) {
+        await admittedTools.closeAndDrain();
         const selectedExit = findWorkflowExitSignal(ownController.signal.reason, exitScope);
         if (selectedExit !== undefined) return await finalizers.finalizeWorkflowExit(selectedExit);
         const parentExit = parentWorkflowExitAbortReason(ownController.signal.reason);
@@ -402,7 +408,6 @@ export async function run<
         return finalizeKilled(runId, runSnapshot, activeStore, opts.persistence, opts.onRunEnd);
       }
     }
-
     await admitDurableRootRun({
       backend: durableBackend, runId, isChildRun: opts.parentRun !== undefined,
       registration: durableRootRegistration === undefined ? undefined
@@ -410,17 +415,21 @@ export async function run<
     });
     if (opts.deferWorkflowStart === true) opts.onWorkflowStartReady?.();
     const rawResult = await runWorkflowDefinitionCallback(def.name, runId, () => def.run(ctx));
-    if (ownController.signal.aborted) {
-      const selectedExit = findWorkflowExitSignal(ownController.signal.reason, exitScope);
+    await admittedTools.closeAndDrain();
+    const normalTerminalEvent = terminalEvents.winner();
+    if (normalTerminalEvent?.kind === "cancellation") {
+      const selectedExit = findWorkflowExitSignal(normalTerminalEvent.reason, exitScope);
       if (selectedExit !== undefined) return await finalizers.finalizeWorkflowExit(selectedExit);
-      const parentExit = parentWorkflowExitAbortReason(ownController.signal.reason);
+      const parentExit = parentWorkflowExitAbortReason(normalTerminalEvent.reason);
       if (parentExit !== undefined) return await finalizers.finalizeParentWorkflowExitCancellation(parentExit);
       return finalizeKilled(runId, runSnapshot, activeStore, opts.persistence, opts.onRunEnd);
     }
+    selectedAdmittedToolFailure = admittedTools.firstFailure();
+    if (normalTerminalEvent?.kind === "failure") throw normalTerminalEvent.error;
 
     const result = normalizeWorkflowRunOutput(def.name, rawResult);
     assertWorkflowRunOutputs(def.name, result, def.outputs);
-    assertWorkflowCreatedStage(runSnapshot);
+    assertWorkflowCreatedExecution(runSnapshot);
     await durableBackend.flush();
     const returned = classifyReturnedRunStatus(result, runSnapshot);
     const recorded = activeStore.recordRunEnd(runId, returned.status, result, returned.error, returned.metadata);
@@ -430,22 +439,25 @@ export async function run<
     await durableBackend.flush();
     return reconcileTerminalRunResult(runId, runSnapshot, activeStore, { status: returned.status, result, error: returned.error }, opts.onRunEnd);
   } catch (err) {
+    terminalEvents.selectFailure(err);
+    await admittedTools.closeAndDrain();
+    const observedAdmittedToolFailure = selectedAdmittedToolFailure ?? admittedTools.uniqueFailureFor(err);
     const selectedExit = findWorkflowExitSignal(err, exitScope) ?? findWorkflowExitSignal(ownController.signal.reason, exitScope);
     if (selectedExit !== undefined) return await finalizers.finalizeWorkflowExit(selectedExit);
-
-    if (ownController.signal.aborted) {
-      const parentExit = parentWorkflowExitAbortReason(ownController.signal.reason);
+    const catchTerminalEvent = terminalEvents.winner();
+    if (catchTerminalEvent?.kind === "cancellation") {
+      const parentExit = parentWorkflowExitAbortReason(catchTerminalEvent.reason);
       if (parentExit !== undefined) return await finalizers.finalizeParentWorkflowExitCancellation(parentExit);
       return finalizeKilled(runId, runSnapshot, activeStore, opts.persistence, opts.onRunEnd);
     }
-
     const failure = classifyExecutorFailure(err);
-    const metadata = normalizeStageLessFailureMetadata(selectRunFailureDisposition({
-      outerFailure: failure,
-      thrownError: err,
-      stages: runSnapshot.stages,
-      classifyFailure: classifyExecutorFailure,
-    }), runSnapshot.stages.length);
+    const selectedMetadata = normalizeFailureWinnerMetadata(normalizeStageLessFailureMetadata(selectRunFailureDisposition({ outerFailure: failure, thrownError: err, stages: runSnapshot.stages, classifyFailure: classifyExecutorFailure }), runSnapshot.stages.length));
+    const failedToolNodeId = selectedMetadata.failedStageId === undefined && selectedMetadata.failureKind !== "cancelled" && selectedMetadata.failureDisposition !== "terminal_killed"
+      ? catchTerminalEvent?.kind === "failure" && Object.is(catchTerminalEvent.error, err)
+        ? catchTerminalEvent.nodeId ?? observedAdmittedToolFailure?.nodeId
+        : observedAdmittedToolFailure?.nodeId
+      : undefined;
+    const metadata = failedToolNodeId === undefined ? selectedMetadata : { ...selectedMetadata, failedToolNodeId };
 
     if (metadata.failureDisposition === "terminal_killed") {
       for (const failedStageId of metadata.failedStageIds) scheduler.blockKnownNonTerminalDescendants(failedStageId);
@@ -462,24 +474,9 @@ export async function run<
       });
     }
 
-    const recorded = activeStore.recordRunEnd(runId, "failed", undefined, metadata.errorMessage, metadata);
-    appendRunEndWhenRecorded(opts.persistence, recorded, {
-      runId,
-      status: "failed",
-      error: metadata.errorMessage,
-      failureKind: metadata.failureKind,
-      ...(metadata.failureCode !== undefined ? { failureCode: metadata.failureCode } : {}),
-      ...(metadata.failureRecoverability !== undefined ? { failureRecoverability: metadata.failureRecoverability } : {}),
-      ...(metadata.failureDisposition !== undefined ? { failureDisposition: metadata.failureDisposition } : {}),
-      failureMessage: metadata.failureMessage,
-      ...(metadata.failedStageId !== undefined ? { failedStageId: metadata.failedStageId } : {}),
-      resumable: metadata.resumable,
-      ...(metadata.retryAfterMs !== undefined ? { retryAfterMs: metadata.retryAfterMs } : {}),
-      ...(runSnapshot.endedAt !== undefined ? { endedAt: runSnapshot.endedAt } : {}),
-      ...(runSnapshot.durationMs !== undefined ? { durationMs: runSnapshot.durationMs } : {}),
-      ts: Date.now(),
+    return finalizeTerminalFailure({
+      runId, runSnapshot, store: activeStore, persistence: opts.persistence, metadata, onRunEnd: opts.onRunEnd,
     });
-    return reconcileTerminalRunResult(runId, runSnapshot, activeStore, { status: "failed", error: metadata.errorMessage }, opts.onRunEnd);
   } finally {
     try {
       await finalizeDurableTerminalStatus({
@@ -489,7 +486,11 @@ export async function run<
         durableBackend,
       });
     } finally {
-      gitWorktreeSetupCacheOwner.release(() => opts.cancellation?.unregister(runId));
+      try {
+        gitWorktreeSetupCacheOwner.release(() => opts.cancellation?.unregister(runId));
+      } finally {
+        terminalEvents.dispose();
+      }
     }
   }
 }

package/dist/builtin/workflows/src/engine/runtime.ts

@@ -1,6 +1,7 @@
 import type { RunSnapshot } from "../shared/store-types.js";
 import type { Store } from "../shared/store.js";
 import type { StageOptions } from "../shared/types.js";
+import { workflowInvocationIntercomGroup } from "../shared/intercom-group.js";
 import type { ConcurrencyLimiter } from "../runs/shared/concurrency.js";
 import type { ParallelFailFastScope } from "../runs/foreground/executor-types.js";
 import type { WorkflowExitManager } from "../runs/foreground/executor-exit-manager.js";
@@ -14,6 +15,7 @@ import { createWorkflowBoundaryFactory, type WorkflowBoundaryStage } from "../ru
 import type { GraphFrontierTracker } from "./graph-inference.js";
 import type { EngineChildRunOptions, EngineStageRuntimeOptions, EngineWorkflowBoundaryOptions } from "./options.js";
 import type { GitWorktreeSetupCache } from "../runs/foreground/executor-direct-helpers.js";
+import type { DurableBoundaryRuntimeIdentity } from "../durable/boundary-topology.js";
 
 export interface EngineRuntimeInput {
   readonly runId: string;
@@ -48,6 +50,7 @@ export interface EngineSpawnAgentStageOptions {
 export interface EngineSpawnWorkflowBoundaryOptions {
   readonly kind: "workflow-boundary";
   readonly replayKey: string;
+  readonly identity?: DurableBoundaryRuntimeIdentity;
 }
 
 export type EngineSpawnStageOptions = EngineSpawnAgentStageOptions | EngineSpawnWorkflowBoundaryOptions;
@@ -66,6 +69,7 @@ export type StageHandle = EngineAgentStageHandle | EngineWorkflowBoundaryHandle;
 
 export class EngineRuntime {
   readonly runId: string;
+  readonly workflowIntercomGroup: string;
   readonly depth: number;
   readonly activeStore: Store;
   readonly childRunOptions: EngineChildRunOptions;
@@ -84,10 +88,15 @@ export class EngineRuntime {
     options?: StageOptions,
     stageFailFastScope?: ParallelFailFastScope,
   ) => StageContextWithMeta;
-  private readonly spawnWorkflowBoundary: (name: string, replayKey: string) => WorkflowBoundaryStage;
+  private readonly spawnWorkflowBoundary: (
+    name: string,
+    replayKey: string,
+    identity?: DurableBoundaryRuntimeIdentity,
+  ) => WorkflowBoundaryStage;
 
   constructor(input: EngineRuntimeInput) {
     this.runId = input.runId;
+    this.workflowIntercomGroup = workflowInvocationIntercomGroup(input.parentRootRunId ?? input.runId);
     this.depth = input.depth;
     this.activeStore = input.activeStore;
     this.childRunOptions = input.childRunOptions;
@@ -105,6 +114,7 @@ export class EngineRuntime {
     // created lazily by the stage runner through input.adapters.agentSession.
     this.spawnAgentStage = createWorkflowStageFactory({
       runId: input.runId,
+      workflowIntercomGroup: this.workflowIntercomGroup,
       activeStore: input.activeStore,
       opts: input.stageOptions,
       adapters: input.adapters,
@@ -138,7 +148,10 @@ export class EngineRuntime {
   spawnStage(name: string, opts?: EngineSpawnAgentStageOptions): EngineAgentStageHandle;
   spawnStage(name: string, opts: EngineSpawnStageOptions = {}): StageHandle {
     if (opts.kind === "workflow-boundary") {
-      return { kind: "workflow-boundary", boundary: this.spawnWorkflowBoundary(name, opts.replayKey) };
+      return {
+        kind: "workflow-boundary",
+        boundary: this.spawnWorkflowBoundary(name, opts.replayKey, opts.identity),
+      };
     }
     return { kind: "agent", context: this.spawnAgentStage(name, opts.options, opts.failFastScope) };
   }

package/dist/builtin/workflows/src/extension/extension-factory.ts

@@ -120,6 +120,7 @@ function factory(pi: ExtensionAPI): void {
       overlay,
       runtimeForContext: runtimeState.runtimeForContext,
       ensureWorkflowResourcesLoaded: runtimeState.ensureWorkflowResourcesLoaded,
+      beforeRestoreCompleted: runtimeState.beforeRestoreCompleted,
     },
   });
   registerWorkflowMessageRenderers(pi);

package/dist/builtin/workflows/src/extension/extension-runtime-state.ts

@@ -1,6 +1,7 @@
 import type { CreateAgentSessionOptions } from "@bastani/atomic";
 import type { StageAdapters } from "../runs/foreground/stage-runner.js";
 import type { SessionManager } from "../shared/persistence-restore.js";
+import type { RunSnapshot } from "../shared/store-types.js";
 import type {
   WorkflowExecutionPolicy,
   WorkflowMcpPort,
@@ -25,6 +26,7 @@ import { createStatusWriter, type StatusWriter } from "./status-writer.js";
 import {
   createWorkflowLifecycleNotificationState,
   installWorkflowLifecycleNotifications,
+  seedWorkflowLifecycleNotificationState,
   registerLifecycleNoticeRenderer,
   withWorkflowLifecycleNotificationsSuppressedAsync,
   type WorkflowLifecycleNotificationConfig,
@@ -50,6 +52,8 @@ export interface WorkflowExtensionRuntimeState {
   discoveryRef: { current: DiscoveryResult | null };
   lifecycleNotificationState: ReturnType<typeof createWorkflowLifecycleNotificationState>;
   hilAnswerNotificationState: ReturnType<typeof createWorkflowHilAnswerNotificationState>;
+  /** Seed lifecycle notification state before completed historical snapshots are inserted. */
+  beforeRestoreCompleted(snapshots: readonly RunSnapshot[]): void;
   runtimeForContext(ctx?: PiModelContext): ExtensionRuntime;
   resetWorkflowDiscoveryForSession(): void;
   ensureWorkflowConfigLoaded(): Promise<void>;
@@ -87,6 +91,9 @@ export function createWorkflowExtensionRuntimeState(
   let notificationGeneration = 0;
   const lifecycleNotificationState = createWorkflowLifecycleNotificationState();
   const hilAnswerNotificationState = createWorkflowHilAnswerNotificationState();
+  const beforeRestoreCompleted = (snapshots: readonly RunSnapshot[]): void => {
+    seedWorkflowLifecycleNotificationState(lifecycleNotificationState, { ...store.snapshot(), runs: snapshots });
+  };
   const lifecycleNotificationConfigRef: { current: WorkflowLifecycleNotificationConfig } = {
     current: WORKFLOW_CONFIG_DEFAULTS.workflowNotifications,
   };
@@ -135,6 +142,7 @@ export function createWorkflowExtensionRuntimeState(
       mcp: mcpPort,
       config: runtimeConfigRef.current,
       resolveDefaultStageSessionDir,
+      beforeRestoreCompleted,
     }),
   };
   const discoveryRef: { current: DiscoveryResult | null } = { current: null };
@@ -197,6 +205,7 @@ export function createWorkflowExtensionRuntimeState(
       config: runtimeConfigRef.current,
       models,
       resolveDefaultStageSessionDir,
+      beforeRestoreCompleted,
     });
   }
 
@@ -235,6 +244,7 @@ export function createWorkflowExtensionRuntimeState(
       mcp: mcpPort,
       config: runtimeConfigRef.current,
       resolveDefaultStageSessionDir,
+      beforeRestoreCompleted,
     });
   }
 
@@ -413,6 +423,7 @@ export function createWorkflowExtensionRuntimeState(
     discoveryRef,
     lifecycleNotificationState,
     hilAnswerNotificationState,
+    beforeRestoreCompleted,
     runtimeForContext,
     resetWorkflowDiscoveryForSession,
     ensureWorkflowConfigLoaded,