@tea-agent/loop-agent 0.44.0-next.10 → 0.44.0-next.12

package/dist/workflows/dag/checkpoint.js

@@ -0,0 +1,686 @@
+/** DAG runner checkpoint 生命周期：rank 调度、中断归属、恢复 lineage 与 state 写入的显式生命周期。 */
+import path from "node:path";
+import { readFile } from "node:fs/promises";
+import { createHash } from "node:crypto";
+import { isHardBudgetBreached } from "../../application/evaluation/budget.js";
+import { preflightBudgetOrBreach, recordFinishedNodeBudget, validateCumulativeBudgetSuccessor, writeBudgetLedgerArtifacts } from "./budget-enforcement.js";
+import { locateDagRun, readDagRunState } from "./lifecycle.js";
+import { markInterruptNeedsReconcile, mergeAbortSignals, readInterruptRequest, runnerIdentityFromState, settleDagInterrupt, startInterruptWatcher } from "./interrupt-request.js";
+import { moveToCompletedRunDir, moveToPausedRunDir, writeRunState } from "./run-store.js";
+import { evaluateNodeLiveness, resolveLivenessPolicy } from "./liveness-policy.js";
+import { createDagNodeExecutor } from "./executor-registry.js";
+import { executeDagPiNode } from "../../executors/dag-pi-executor.js";
+import { relocateConvergenceArtifactPaths, relocateRunArtifactPaths } from "./upstream-artifacts.js";
+import { WORKSPACE_CHECKPOINT_TERMINAL_REL, captureWorkspaceCheckpoint, writeWorkspaceCheckpoint } from "./workspace-checkpoint.js";
+import { executeDagNode } from "./node-execution.js";
+import { runConvergencePassController } from "./convergence/controller.js";
+import { executeDagRanksOnce } from "./scheduler.js";
+import { bindFrontendShapeStateToRunTopology, commitFrontendShapeTransitionCas, commitRuntimeFrontendShapeFact, isCanonicalFrontendShapeDag } from "./frontend-shape-facts.js";
+import { computeFrontendShapeSourceDigest } from "./frontend-shape.js";
+import { ensureFrontendShapeCapsuleRecord } from "./frontend-shape-capsule-store.js";
+import { executeDynamicCondition } from "./dynamic-runtime/condition.js";
+import { executeDynamicLoopUntil } from "./dynamic-runtime/loop-until.js";
+import { executeDynamicMapExpansion } from "./dynamic-runtime/map.js";
+import { executeDynamicReduction } from "./dynamic-runtime/reduction.js";
+import { installRunnerExitDiagnostics } from "./runner-exit-diagnostics.js";
+import { buildRecoveryResultForTrigger, createRecoveryOperation, finalizeTerminalRunStatus, } from "./terminal-status.js";
+export async function assertRecoveredFrontendLineage(input) {
+    const maxContinuations = Math.max(1, Math.min(5, input.maxContinuations));
+    let parentRunId = input.rootRunId;
+    let recovery = input.recovery;
+    const recoveryRootRunId = recovery.recoveryRootRunId;
+    const visited = new Set();
+    if (recoveryRootRunId !== input.rootRunId) {
+        throw new Error("recovery root lineage binding mismatch");
+    }
+    for (let depth = 0; depth < maxContinuations; depth += 1) {
+        // The persisted parent intent represents the edge about to be traversed,
+        // so its counters equal the number of already-completed continuation edges.
+        // Each completed child increments both counters exactly once; no edge may
+        // exceed the frozen DAG quota.
+        if (recovery.continuationCount !== depth ||
+            recovery.attemptIndex !== depth) {
+            throw new Error("recovery child lineage counters are not monotonic");
+        }
+        if (recovery.phase !== "child-running" ||
+            recovery.parentRunId !== parentRunId ||
+            recovery.recoveryRootRunId !== recoveryRootRunId ||
+            !recovery.childRunId ||
+            (depth === 0
+                ? recovery.requestId !== input.rootRequestId
+                : recovery.requestId !== parentRunId)) {
+            throw new Error("persisted recovery lineage is missing or unbound");
+        }
+        if (visited.has(recovery.childRunId)) {
+            throw new Error("recovery child lineage contains a cycle");
+        }
+        visited.add(recovery.childRunId);
+        const child = await locateDagRun(input.cwd, recovery.childRunId);
+        if (!child || child.lifecycle !== "completed") {
+            throw new Error("recovery child is not completed");
+        }
+        const childState = await readDagRunState(child.runDir);
+        if (childState.runId !== recovery.childRunId) {
+            throw new Error("recovery child state binding mismatch");
+        }
+        if (childState.status === "finished")
+            return;
+        if (childState.frontendRecoveryResult?.outcome !== "recovered" ||
+            !childState.frontendRecoveryState) {
+            throw new Error("recovery child terminal state is not successful");
+        }
+        parentRunId = childState.runId;
+        recovery = childState.frontendRecoveryState;
+    }
+    throw new Error("recovery child lineage exceeds the continuation quota");
+}
+export async function finalizeRunOwnedInterrupt(input) {
+    const request = await readInterruptRequest(input.runDir);
+    if (!request)
+        return;
+    if (request.status === "settled" || request.status === "needs-reconcile") {
+        return;
+    }
+    const hasUnconfirmed = Object.values(input.state.nodes).some((node) => node.failureCategory === "termination-unconfirmed");
+    if (hasUnconfirmed || input.paused) {
+        await markInterruptNeedsReconcile({
+            runDir: input.runDir,
+            outcome: hasUnconfirmed
+                ? "termination-unconfirmed"
+                : "paused-before-settle",
+        });
+        return;
+    }
+    if (input.interruptAborted &&
+        request.status === "acknowledged" &&
+        (input.state.status === "failed" ||
+            input.state.status === "partial_failed" ||
+            input.state.failureCategory === "controller-interrupted")) {
+        await settleDagInterrupt({
+            runDir: input.runDir,
+            outcome: "controller-interrupted",
+        });
+        return;
+    }
+    if (request.status === "requested") {
+        await markInterruptNeedsReconcile({
+            runDir: input.runDir,
+            outcome: "runner-did-not-acknowledge",
+        });
+        return;
+    }
+    await markInterruptNeedsReconcile({
+        runDir: input.runDir,
+        outcome: "run-exited-before-interrupt-settled",
+    });
+}
+export function buildRankAwareExecuteNode(input) {
+    if (input.customExecuteNode) {
+        return input.baseExecuteNode;
+    }
+    return async (nodeInput) => {
+        const isPiWriter = nodeInput.task.executor === "pi" &&
+            nodeInput.task.toolProfile === "write" &&
+            Boolean(nodeInput.model);
+        if (!isPiWriter) {
+            return input.baseExecuteNode(nodeInput);
+        }
+        const siblingWriteSets = nodeInput.task.writePolicy === "exclusive"
+            ? concurrentSiblingWriteSetsForNode(input.rankWriterNodeIds, nodeInput.task.id, input.tasksById)
+            : [];
+        return executeDagPiNode({ ...nodeInput, model: nodeInput.model }, {
+            ...input.meta,
+            concurrentSiblingWriteSets: siblingWriteSets.length > 0 ? siblingWriteSets : undefined,
+            writeGuardAttribution: siblingWriteSets.length > 0 ? "best-effort" : "per-node",
+        });
+    };
+}
+export async function executeDagCheckpoint(input) {
+    const { spec, state, ranks, runDir: initialRunDir, cwd, maxConcurrent, completedRunDir, pausedRunDir, } = input;
+    bindFrontendShapeStateToRunTopology(state);
+    let runDir = initialRunDir;
+    let stateWriteQueue = Promise.resolve();
+    let shapeTransitionBarrier;
+    const persistState = async (options) => {
+        if (shapeTransitionBarrier)
+            await shapeTransitionBarrier;
+        stateWriteQueue = stateWriteQueue.then(() => writeRunState(runDir, state, options));
+        await stateWriteQueue;
+    };
+    const runnerLivenessPolicy = resolveLivenessPolicy(spec.defaults?.livenessPolicy);
+    const heartbeatTimer = setInterval(() => {
+        if (!state.runner)
+            return;
+        // Runner lease only — never counts as meaningful Pi progress.
+        state.runner.heartbeatAt = new Date().toISOString();
+        const tasksByIdForPolicy = new Map(spec.tasks.map((task) => [task.id, task]));
+        for (const node of Object.values(state.nodes)) {
+            if (node.status !== "RUNNING")
+                continue;
+            // Lease clock is separate from meaningful activity.
+            node.lastLeaseAt = state.runner.heartbeatAt;
+            const task = tasksByIdForPolicy.get(node.id);
+            const policy = resolveLivenessPolicy(spec.defaults?.livenessPolicy, task?.livenessPolicy);
+            const evaluation = evaluateNodeLiveness({
+                node,
+                policy,
+                runnerHeartbeatAt: state.runner.heartbeatAt,
+            });
+            node.livenessStatus = evaluation.status;
+            // Intentionally do NOT refresh lastActivityAt / lastMeaningfulProgressAt
+            // from the runner lease timer.
+        }
+        void persistState().catch(() => { });
+    }, runnerLivenessPolicy.heartbeatIntervalMs);
+    heartbeatTimer.unref();
+    await persistState();
+    const interruptController = new AbortController();
+    const expectedRunnerIdentity = runnerIdentityFromState(state);
+    const interruptWatcher = expectedRunnerIdentity
+        ? startInterruptWatcher({
+            runDir,
+            runId: state.runId,
+            expectedRunnerIdentity,
+            controller: interruptController,
+        })
+        : undefined;
+    const abortSignal = mergeAbortSignals(input.abortSignal, interruptController.signal);
+    // Graceful terminal persistence: an outer SIGTERM/SIGINT (operator hard
+    // timeout, supervision layer, or shell wall-clock) must not leave the run
+    // orphaned in RUNNING with a dead heartbeat. Persist a terminal failed
+    // state with an explicit terminalReason before exiting so downstream
+    // liveness/doctor tooling sees a diagnosable terminal instead of an orphan.
+    let terminalSignal;
+    const shutdownOnSignal = (signal) => {
+        if (terminalSignal)
+            return;
+        terminalSignal = signal;
+        clearInterval(heartbeatTimer);
+        if (state.runner)
+            state.runner.heartbeatAt = new Date().toISOString();
+        state.status = "failed";
+        state.finishedAt = new Date().toISOString();
+        state.terminalReason = `runner terminated by ${signal} before run completion`;
+        console.warn(`[run-dag] received ${signal}; persisting terminal state and exiting`);
+        const exit = () => process.exit(signal === "SIGINT" ? 130 : 143);
+        // Best-effort atomic persist with a hard exit deadline so a stuck write
+        // queue cannot keep the process alive past the supervisor's kill window.
+        const hardExit = setTimeout(exit, 5000);
+        hardExit.unref();
+        void persistState()
+            .catch(() => { })
+            .finally(() => {
+            clearTimeout(hardExit);
+            exit();
+        });
+    };
+    const onSigTerm = () => shutdownOnSignal("SIGTERM");
+    const onSigInt = () => shutdownOnSignal("SIGINT");
+    // Keep listeners installed until persistence completes. Pi dependencies use
+    // signal-exit, which recounts listeners during signal dispatch and re-sends
+    // the signal if a once-listener has already removed itself. terminalSignal
+    // above makes repeated signals idempotent; normal cleanup removes listeners.
+    process.on("SIGTERM", onSigTerm);
+    process.on("SIGINT", onSigInt);
+    // Exit-time diagnostics (silent-death forensics): write an `armed` line at
+    // checkpoint start and append process-exit / uncaught-exception /
+    // unhandled-rejection lines with a live state snapshot so an abnormal death
+    // (supervisor hard kill, wedged event loop, crash with lost stderr) leaves
+    // a durable trace next to state.json. Removed on clean terminal return.
+    const exitDiagnostics = installRunnerExitDiagnostics({
+        runDir,
+        runId: state.runId,
+        title: state.title,
+        snapshot: () => {
+            const heartbeatAt = state.runner?.heartbeatAt;
+            return {
+                status: state.status,
+                terminalReason: state.terminalReason,
+                failureCategory: state.failureCategory,
+                heartbeatAt,
+                heartbeatAgeMs: heartbeatAt === undefined
+                    ? undefined
+                    : Math.max(0, Date.now() - Date.parse(heartbeatAt)),
+                running: Object.values(state.nodes)
+                    .filter((node) => node.status === "RUNNING")
+                    .map((node) => ({
+                    id: node.id,
+                    attempt: node.currentAttempt,
+                    startedAt: node.startedAt,
+                    lastMeaningfulProgressAt: node.lastMeaningfulProgressAt,
+                })),
+            };
+        },
+    });
+    try {
+        const tasksById = new Map(spec.tasks.map((task) => [task.id, task]));
+        const baseExecuteNode = input.executeNode ??
+            createDagNodeExecutor({
+                runDir,
+                runId: state.runId,
+                spec,
+            });
+        const executeDynamicNode = async (dynamicInput) => {
+            const { task } = dynamicInput;
+            if (task.dynamicExpansion) {
+                return executeDynamicMapExpansion({
+                    ...dynamicInput,
+                    expansion: task.dynamicExpansion,
+                    executeDynamicNode,
+                });
+            }
+            if (task.dynamicCondition) {
+                return executeDynamicCondition({
+                    task,
+                    condition: task.dynamicCondition,
+                    tasksById: dynamicInput.tasksById,
+                    state: dynamicInput.state,
+                });
+            }
+            if (task.dynamicLoopUntil) {
+                return executeDynamicLoopUntil({
+                    ...dynamicInput,
+                    loop: task.dynamicLoopUntil,
+                    executeDynamicNode,
+                });
+            }
+            return executeDynamicReduction({
+                task,
+                reduction: task.dynamicReduction,
+                state: dynamicInput.state,
+                runDir: dynamicInput.runDir,
+            });
+        };
+        const deferTerminalCloseout = state.convergence !== undefined && tasksById.has("closeout-pi");
+        const convergenceRanks = deferTerminalCloseout
+            ? ranks
+                .map((rank) => rank.filter((nodeId) => nodeId !== "closeout-pi"))
+                .filter((rank) => rank.length > 0)
+            : ranks;
+        const terminalCloseoutRanks = deferTerminalCloseout
+            ? ranks
+                .map((rank) => rank.filter((nodeId) => nodeId === "closeout-pi"))
+                .filter((rank) => rank.length > 0)
+            : [];
+        const canonicalFrontendSource = isCanonicalFrontendShapeDag(tasksById)
+            ? await loadCanonicalFrontendTaskSourceBinding(cwd, spec)
+            : undefined;
+        const executeRanks = (selectedRanks) => executeDagRanksOnce({
+            state,
+            ranks: selectedRanks,
+            tasksById,
+            maxConcurrent,
+            persistState,
+            commitFrontendShapeTransition: async (request) => {
+                let releaseBarrier;
+                shapeTransitionBarrier = new Promise((resolve) => {
+                    releaseBarrier = resolve;
+                });
+                try {
+                    if (!canonicalFrontendSource) {
+                        throw new Error("frontend shape transition requires canonical task source binding");
+                    }
+                    return await commitFrontendShapeTransitionCas({
+                        state,
+                        tasksById,
+                        request,
+                        taskSourceDigest: canonicalFrontendSource.sourceDigest,
+                        ensureRecord: async (capsule, terminalOutcome) => {
+                            await ensureFreshFrontendShapeCapsuleRecord({
+                                cwd,
+                                spec,
+                                canonicalSource: canonicalFrontendSource,
+                                parentRunId: state.runId,
+                                terminalOutcome,
+                                capsule,
+                            });
+                        },
+                        persist: async (nextState) => {
+                            stateWriteQueue = stateWriteQueue.then(() => writeRunState(runDir, nextState));
+                            await stateWriteQueue;
+                        },
+                    });
+                }
+                finally {
+                    releaseBarrier();
+                    shapeTransitionBarrier = undefined;
+                }
+            },
+            runDir,
+            abortSignal: input.abortSignal,
+            createExecuteNodeForRank: (rankWriterNodeIds) => buildRankAwareExecuteNode({
+                baseExecuteNode,
+                customExecuteNode: input.executeNode,
+                rankWriterNodeIds,
+                tasksById,
+                meta: { runDir, runId: state.runId, spec },
+            }),
+            executeScheduledNode: async (nodeId, executeNode, onPause) => {
+                if (abortSignal?.aborted)
+                    return;
+                if (isHardBudgetBreached(state.budgetLedger))
+                    return;
+                const preBreach = preflightBudgetOrBreach(state);
+                if (preBreach && isHardBudgetBreached(state.budgetLedger)) {
+                    await writeBudgetLedgerArtifacts(runDir, state.budgetLedger);
+                    await persistState();
+                    return;
+                }
+                await executeDagNode({
+                    nodeId,
+                    tasksById,
+                    state,
+                    spec,
+                    cwd,
+                    runDir,
+                    executeNode,
+                    executeDynamicNode,
+                    observer: input.observer,
+                    persistState,
+                    onPause,
+                });
+                const node = state.nodes[nodeId];
+                if (node && (node.status === "FINISHED" || node.status === "ERROR")) {
+                    if (node.status === "FINISHED") {
+                        if (canonicalFrontendSource) {
+                            await commitRuntimeFrontendShapeFact({
+                                state,
+                                tasksById,
+                                sourceNodeId: nodeId,
+                                taskId: canonicalFrontendSource.taskId,
+                                requirementMarkdown: canonicalFrontendSource.requirementMarkdown,
+                                constraintMarkdown: canonicalFrontendSource.constraintMarkdown,
+                            });
+                        }
+                    }
+                    recordFinishedNodeBudget(state, node);
+                    await writeBudgetLedgerArtifacts(runDir, state.budgetLedger);
+                    await persistState();
+                }
+            },
+        });
+        let pausedByNodeId;
+        while (true) {
+            pausedByNodeId = await executeRanks(convergenceRanks);
+            if (state.frontendShapeState?.terminalOutcome) {
+                break;
+            }
+            if (pausedByNodeId) {
+                break;
+            }
+            if (isHardBudgetBreached(state.budgetLedger)) {
+                break;
+            }
+            const convergenceDecision = await runConvergencePassController({
+                spec,
+                state,
+                tasksById,
+                runDir,
+                cwd,
+                persistState,
+            });
+            if (convergenceDecision.pausedByNodeId) {
+                pausedByNodeId = convergenceDecision.pausedByNodeId;
+                break;
+            }
+            if (convergenceDecision.retry) {
+                continue;
+            }
+            break;
+        }
+        if (!pausedByNodeId &&
+            terminalCloseoutRanks.length > 0 &&
+            state.convergence?.terminalReason &&
+            !isHardBudgetBreached(state.budgetLedger)) {
+            // Convergence closeout is a terminal phase, not an ordinary descendant
+            // rank. Running it only after the controller records terminalReason avoids
+            // stale success handoffs on intermediate request-revision passes while
+            // still producing a truthful failure handoff for max-passes/non-retry.
+            pausedByNodeId = await executeRanks(terminalCloseoutRanks);
+        }
+        interruptWatcher?.stop();
+        // Stop heartbeats before terminal archive. A late heartbeat writing
+        // state.json under completed/ trips the completed-facts write guard and
+        // makes run-dag exit non-zero after every node already finished.
+        clearInterval(heartbeatTimer);
+        await stateWriteQueue;
+        state.finishedAt = new Date().toISOString();
+        const runDirBeforeTransfer = runDir;
+        await writeBudgetLedgerArtifacts(runDir, state.budgetLedger);
+        if (pausedByNodeId) {
+            state.status = "paused";
+            await finalizeRunOwnedInterrupt({
+                runDir,
+                state,
+                paused: true,
+                interruptAborted: interruptController.signal.aborted,
+            });
+            await persistState();
+            await notifyRunObserver(input.observer, "onRunFinish", state);
+            runDir = await moveToPausedRunDir(runDir, pausedRunDir);
+        }
+        else {
+            const { recoveryPending, recovery: recoveryTrigger } = await finalizeTerminalRunStatus(state, spec.tasks.length, runDir, cwd, spec.frontendRecovery?.maxContinuations ?? 1);
+            await finalizeRunOwnedInterrupt({
+                runDir,
+                state,
+                paused: false,
+                interruptAborted: interruptController.signal.aborted,
+            });
+            await persistState();
+            if (recoveryPending && input.recovery && recoveryTrigger) {
+                // M6 recovery operation controller path: construct/reuse one recovery
+                // operation keyed by a deterministic clientRequestId and supervise it
+                // to terminal, instead of staging an in-process child run.
+                const created = await createRecoveryOperation({
+                    cwd,
+                    runDir,
+                    spec,
+                    state,
+                    trigger: recoveryTrigger,
+                    deps: input.recovery,
+                });
+                if (created.kind === "blocked") {
+                    state.status = "failed";
+                    state.terminalReason = created.reason;
+                    state.frontendRecoveryResult = buildRecoveryResultForTrigger(state, recoveryTrigger, "auto-recovery-blocked");
+                }
+                else if (created.kind === "conflict") {
+                    // Same clientRequestId with a different payload → fail closed; never
+                    // create a second recovery operation (AC-004).
+                    state.status = "failed";
+                    state.terminalReason =
+                        "frontend recovery request-id reuse conflict";
+                    state.frontendRecoveryResult = buildRecoveryResultForTrigger(state, recoveryTrigger, "auto-recovery-blocked");
+                }
+                else if (created.kind === "lease-held") {
+                    // A different recovery entrance has already durably claimed this
+                    // parent. Do not "help" by running its operation: that would reopen
+                    // the exact two-child-DAG race the lease prevents.
+                    state.status = "failed";
+                    state.terminalReason =
+                        "frontend recovery parent lease already held";
+                    state.frontendRecoveryResult = buildRecoveryResultForTrigger(state, recoveryTrigger, "auto-recovery-blocked");
+                }
+                else {
+                    const { runRecoveryOperationController } = await import("./frontend-recovery-controller.js");
+                    const result = await runRecoveryOperationController({
+                        operationId: created.operation.operationId,
+                        parentRunId: state.runId,
+                        observeOnly: created.kind === "observed",
+                        deps: {
+                            ...input.recovery,
+                            budget: input.recovery.budget ?? state.budget,
+                            readLedgers: input.recovery.readLedgers ?? (async () => {
+                                if (!state.budget)
+                                    return [];
+                                const location = await locateDagRun(cwd, state.runId);
+                                let current = await readDagRunState(location?.runDir ?? runDir);
+                                const visited = new Set([current.runId]);
+                                while (current.frontendRecoveryState?.parentRunId === current.runId && current.frontendRecoveryState.childRunId && current.frontendRecoveryState.childRunId !== current.runId) {
+                                    if (["child-staging", "child-activating"].includes(current.frontendRecoveryState.phase))
+                                        break;
+                                    const childId = current.frontendRecoveryState.childRunId;
+                                    if (visited.has(childId))
+                                        throw Error("FRONTEND_PROVIDER_BUDGET_LINEAGE_INVALID: cyclic recovery");
+                                    visited.add(childId);
+                                    const childLocation = await locateDagRun(cwd, childId);
+                                    if (!childLocation)
+                                        throw Error("FRONTEND_PROVIDER_BUDGET_LINEAGE_INVALID: missing recovery child");
+                                    const child = await readDagRunState(childLocation.runDir);
+                                    validateCumulativeBudgetSuccessor(current, child);
+                                    current = child;
+                                }
+                                if (!current.budgetLedger)
+                                    throw Error("FRONTEND_PROVIDER_BUDGET_INVALID: recovery ledger missing");
+                                return [current.budgetLedger];
+                            }),
+                            failureOwner: input.recovery.failureOwner ??
+                                recoveryTrigger.failureOwner,
+                            protocolFailureReason: input.recovery.protocolFailureReason ??
+                                recoveryTrigger.protocolFailureReason,
+                            recoveryMode: input.recovery.recoveryMode ?? recoveryTrigger.recoveryMode,
+                        },
+                    });
+                    // The recovery CLI stages its child by writing a fresh
+                    // `frontendRecoveryState` into the parent run on disk. A successful
+                    // operation without that bound lineage is not a successful recovery:
+                    // fail closed instead of archiving a false-positive `recovered` result.
+                    let recoveryLineageError;
+                    if (result.terminalReason === "succeeded") {
+                        try {
+                            const persistedParentState = await readDagRunState(runDir);
+                            const persistedRecovery = persistedParentState.frontendRecoveryState;
+                            if (!persistedRecovery ||
+                                persistedRecovery.phase !== "child-running" ||
+                                persistedRecovery.parentRunId !== state.runId ||
+                                persistedRecovery.requestId !== recoveryTrigger.requestId ||
+                                !persistedRecovery.childRunId) {
+                                throw new Error("persisted recovery lineage is missing or unbound");
+                            }
+                            await assertRecoveredFrontendLineage({
+                                cwd,
+                                rootRunId: state.runId,
+                                rootRequestId: recoveryTrigger.requestId,
+                                recovery: persistedRecovery,
+                                maxContinuations: spec.frontendRecovery?.maxContinuations ?? 1,
+                            });
+                            state.frontendRecoveryState = structuredClone(persistedRecovery);
+                        }
+                        catch (error) {
+                            recoveryLineageError =
+                                error instanceof Error ? error.message : String(error);
+                            state.status = "failed";
+                            state.terminalReason =
+                                `frontend recovery lineage invalid: ${recoveryLineageError}`;
+                        }
+                    }
+                    state.frontendRecoveryResult = buildRecoveryResultForTrigger(state, recoveryTrigger, result.terminalReason === "succeeded" && !recoveryLineageError
+                        ? "recovered"
+                        : "auto-recovery-blocked");
+                }
+            }
+            await notifyRunObserver(input.observer, "onRunFinish", state);
+            if (input.beforeTerminalArchive) {
+                await input.beforeTerminalArchive({ runDir, state });
+                await persistState();
+            }
+            // Parent is always archived to completed/ — no active/ retention for an
+            // in-process child CAS anymore (recovery runs as a separate operation).
+            try {
+                const terminalCheckpoint = await captureWorkspaceCheckpoint(cwd);
+                await writeWorkspaceCheckpoint(runDir, WORKSPACE_CHECKPOINT_TERMINAL_REL, terminalCheckpoint);
+            }
+            catch (error) {
+                console.warn(`[run-dag] warning: failed to write workspace terminal checkpoint: ${error instanceof Error ? error.message : String(error)}`);
+            }
+            runDir = await moveToCompletedRunDir(runDir, completedRunDir);
+        }
+        await relocateRunArtifactPaths({
+            runDir,
+            oldRunDir: runDirBeforeTransfer,
+            nodes: state.nodes,
+        });
+        if (state.convergence) {
+            relocateConvergenceArtifactPaths(state.convergence, runDirBeforeTransfer, runDir);
+        }
+        await writeBudgetLedgerArtifacts(runDir, state.budgetLedger);
+        await persistState({ allowCompletedFactsWrite: true });
+        return {
+            title: state.title,
+            runId: state.runId,
+            status: state.status,
+            ranks,
+            nodes: state.nodes,
+            runDir,
+        };
+    }
+    finally {
+        clearInterval(heartbeatTimer);
+        interruptWatcher?.stop();
+        exitDiagnostics.stop();
+        process.removeListener("SIGTERM", onSigTerm);
+        process.removeListener("SIGINT", onSigInt);
+    }
+}
+export function concurrentSiblingWriteSetsForNode(rankWriterNodeIds, nodeId, tasksById) {
+    if (rankWriterNodeIds.length <= 1)
+        return [];
+    return rankWriterNodeIds
+        .filter((id) => id !== nodeId)
+        .map((id) => tasksById.get(id)?.writeSet ?? [])
+        .filter((writeSet) => writeSet.length > 0);
+}
+export async function ensureFreshFrontendShapeCapsuleRecord(input) {
+    const freshSource = await loadCanonicalFrontendTaskSourceBinding(input.cwd, input.spec);
+    if (freshSource.taskId !== input.canonicalSource.taskId ||
+        freshSource.sourceDigest !== input.canonicalSource.sourceDigest) {
+        throw new Error("canonical frontend task source freshness binding drifted before transition commit");
+    }
+    await (input.ensureRecord ?? ensureFrontendShapeCapsuleRecord)({
+        repoRoot: input.cwd,
+        taskId: input.canonicalSource.taskId,
+        parentRunId: input.parentRunId,
+        terminalOutcome: input.terminalOutcome,
+        capsule: input.capsule,
+    });
+}
+export async function loadCanonicalFrontendTaskSourceBinding(cwd, spec) {
+    const binding = spec.sourceBinding;
+    if (!binding?.taskId)
+        throw new Error("canonical frontend DAG requires sourceBinding.taskId");
+    const requirement = binding.sources.find((source) => source.kind === "requirement");
+    if (!requirement)
+        throw new Error("canonical frontend DAG requires a bound requirement source");
+    const constraint = binding.sources.find((source) => source.kind === "constraint");
+    const readFreshSource = async (source) => {
+        let bytes;
+        try {
+            bytes = await readFile(path.resolve(cwd, source.path));
+        }
+        catch (error) {
+            throw new Error(`canonical frontend ${source.kind} source freshness read failed: ${error instanceof Error ? error.message : String(error)}`);
+        }
+        const actualSha256 = createHash("sha256").update(bytes).digest("hex");
+        if (actualSha256 !== source.sha256) {
+            throw new Error(`canonical frontend ${source.kind} source sha256 drifted: expected ${source.sha256}, received ${actualSha256}`);
+        }
+        return bytes.toString("utf8");
+    };
+    const requirementMarkdown = await readFreshSource(requirement);
+    const constraintMarkdown = constraint ? await readFreshSource(constraint) : "";
+    return {
+        taskId: binding.taskId,
+        requirementMarkdown,
+        constraintMarkdown,
+        sourceDigest: computeFrontendShapeSourceDigest({ requirementMarkdown, constraintMarkdown }),
+    };
+}
+export async function notifyRunObserver(observer, event, state) {
+    try {
+        await observer?.[event]?.(state);
+    }
+    catch {
+        // Observers are derived views; they must not affect canonical DAG execution.
+    }
+}