@tea-agent/loop-agent 0.1.0-fe-test.0

package/dist/workflows/dag/runner.js

@@ -0,0 +1,1069 @@
+import { readdir, readFile } from "node:fs/promises";
+import { randomUUID } from "node:crypto";
+import { writeJsonAtomic } from "../../infrastructure/harness/atomic-write.js";
+import { hostname } from "node:os";
+import path from "node:path";
+import { isHardBudgetBreached, resolveEffectiveMaxConcurrent, } from "../../application/evaluation/budget.js";
+import { readCandidateRecord } from "../../infrastructure/evaluation/candidate-store.js";
+import { CANONICAL_TASK_ID_PATTERN, formatLocalCompactDate, } from "../../task/runtime.js";
+import { assertFrozenBudget, initRunBudgetLedger, preflightBudgetOrBreach, recordFinishedNodeBudget, writeBudgetLedgerArtifacts, } from "./budget-enforcement.js";
+import { getDagRunDir, isTerminalDagRunStatus, locateDagRun, readDagRunSpec, readDagRunState, readHumanApprovalArtifact, requireActiveDagRun, } from "./lifecycle.js";
+import { markInterruptNeedsReconcile, mergeAbortSignals, readInterruptRequest, runnerIdentityFromState, settleDagInterrupt, startInterruptWatcher, } from "./interrupt-request.js";
+import { moveToCompletedRunDir, moveToPausedRunDir, prepareRunDir, writeRunSpec, writeRunState, } from "./run-store.js";
+import { evaluateNodeLiveness, resolveLivenessPolicy, } from "./liveness-policy.js";
+import { createDagNodeExecutor } from "./executor-registry.js";
+import { executeDagPiNode } from "../../executors/dag-pi-executor.js";
+import { assertValidDagSpec } from "./validate.js";
+import { assertRuntimeContractCompatible, DAG_CONTROLLER_CAPABILITIES, } from "./runtime-contract.js";
+import { captureControllerIdentity, verifyControllerIdentityForResume, } from "./controller-identity.js";
+import { resolveRunningControllerIdentity } from "../../shared/package-metadata.js";
+import { normalizeDagPromptSources } from "./prompt-source.js";
+import { relocateConvergenceArtifactPaths, relocateRunArtifactPaths, } from "./upstream-artifacts.js";
+import { createSkillSnapshot, prepareSkillSnapshotForContinuation, writeSkillSnapshot, } from "./skill-snapshot.js";
+import { captureWorkspaceCheckpoint, WORKSPACE_CHECKPOINT_START_REL, WORKSPACE_CHECKPOINT_TERMINAL_REL, writeWorkspaceCheckpoint, } from "./workspace-checkpoint.js";
+import { buildNodePrompt, buildNodePromptWithResolvedSkillInstructions, executeDagNode, } from "./node-execution.js";
+import { runConvergencePassController, } from "./convergence/controller.js";
+import { executeDagRanksOnce, FRONTEND_PREWRITE_RESULT_SOURCE_ARTIFACT, FRONTEND_WRITER_NODE_IDS, isConditionSkippedReason, readFrontendPrewriteResult, } from "./scheduler.js";
+import { topoSortToRanks } from "./topo.js";
+import { isFrontendWriterTransientPartialWrite } from "./frontend-writer-recovery.js";
+import { parseDagSpec, resolveModelForTask, } from "./types.js";
+import { executeDynamicCondition } from "./dynamic-runtime/condition.js";
+import { executeDynamicLoopUntil } from "./dynamic-runtime/loop-until.js";
+import { executeDynamicMapExpansion } from "./dynamic-runtime/map.js";
+import { executeDynamicReduction } from "./dynamic-runtime/reduction.js";
+export { buildNodePrompt, buildNodePromptWithResolvedSkillInstructions };
+export async function loadDagSpecFromFile(dagPath) {
+    const raw = JSON.parse(await readFile(dagPath, "utf-8"));
+    const normalized = await normalizeDagPromptSources(raw, path.dirname(dagPath));
+    return parseDagSpec(normalized);
+}
+export const CANONICAL_DAG_RUN_ID_PATTERN = /^\d{8}-[a-z0-9]+(-[a-z0-9]+)*$/;
+const TASK_ID_IN_TEXT_PATTERN = /\d{4}-\d{2}-\d{2}-[a-z0-9]+(?:-[a-z0-9]+)*/g;
+export function toKebabSlug(text, maxLen = 40) {
+    return (text
+        .toLowerCase()
+        .replace(/[^a-z0-9]+/g, "-")
+        .replace(/^-|-$/g, "")
+        .slice(0, maxLen) || "dag");
+}
+export function extractTaskIdFromDagSpec(spec) {
+    if (spec.objective) {
+        const match = spec.objective.match(TASK_ID_IN_TEXT_PATTERN);
+        if (match)
+            return match[0];
+    }
+    for (const line of spec.globalConstraints ?? []) {
+        const match = line.match(TASK_ID_IN_TEXT_PATTERN);
+        if (match)
+            return match[0];
+    }
+    return undefined;
+}
+export function deriveDagRunSlug(spec) {
+    const taskId = extractTaskIdFromDagSpec(spec);
+    if (taskId && CANONICAL_TASK_ID_PATTERN.test(taskId)) {
+        return taskId.slice(11);
+    }
+    return toKebabSlug(spec.title);
+}
+export function validateDagRunIdFormat(runId) {
+    const warnings = [];
+    if (!CANONICAL_DAG_RUN_ID_PATTERN.test(runId)) {
+        warnings.push(`run-id "${runId}" does not match canonical YYYYMMDD-<slug>; legacy run facts remain readable`);
+    }
+    return { warnings };
+}
+async function listExistingDagRunIds(cwd) {
+    const ids = new Set();
+    for (const lifecycle of ["active", "completed", "paused"]) {
+        const dir = path.join(cwd, ".harness", "dag-runs", lifecycle);
+        try {
+            const entries = await readdir(dir);
+            for (const entry of entries) {
+                if (entry !== ".gitkeep")
+                    ids.add(entry);
+            }
+        }
+        catch {
+            // lifecycle dir may not exist in minimal fixtures.
+        }
+    }
+    return ids;
+}
+export function collisionSafeDagRunId(baseId, existing, now = new Date()) {
+    if (!existing.has(baseId))
+        return baseId;
+    for (let round = 2; round <= 99; round += 1) {
+        const candidate = `${baseId}-r${round}`;
+        if (!existing.has(candidate))
+            return candidate;
+    }
+    const hh = String(now.getHours()).padStart(2, "0");
+    const mm = String(now.getMinutes()).padStart(2, "0");
+    const ss = String(now.getSeconds()).padStart(2, "0");
+    const timed = `${baseId}-${hh}${mm}${ss}`;
+    if (!existing.has(timed))
+        return timed;
+    return `${timed}-r2`;
+}
+export async function buildDagRunId(spec, options) {
+    const now = options?.now ?? new Date();
+    const datePrefix = formatLocalCompactDate(now);
+    const baseId = `${datePrefix}-${deriveDagRunSlug(spec)}`;
+    if (!options?.cwd)
+        return baseId;
+    const existing = await listExistingDagRunIds(options.cwd);
+    return collisionSafeDagRunId(baseId, existing, now);
+}
+/**
+ * Append a human-readable round label to a run title when the runId carries
+ * a `-rN` collision suffix (2026-08-16): the sidebar / Inspect list showed
+ * identical titles for reruns and only the id hinted at the round.
+ */
+export function titleWithRoundLabel(title, runId) {
+    const match = /-r(\d{1,2})$/.exec(runId);
+    if (!match)
+        return title;
+    const round = Number.parseInt(match[1], 10);
+    if (!Number.isFinite(round) || round < 2)
+        return title;
+    // Never stack labels on reruns of reruns.
+    const base = title.replace(/\s*（第\s*\d+\s*轮）$/u, "");
+    return `${base}（第 ${round} 轮）`;
+}
+export function createInitialRunState(spec, opts, ranks, runId = opts.runId ?? "dag") {
+    const nodes = {};
+    for (const task of spec.tasks) {
+        nodes[task.id] = {
+            id: task.id,
+            status: "PENDING",
+            executor: task.executor,
+            complexity: task.complexity,
+            ...(task.outputMode ? { outputMode: task.outputMode } : {}),
+            ...(resolveModelForTask(task, spec.executorModels)
+                ? { model: resolveModelForTask(task, spec.executorModels) }
+                : {}),
+        };
+    }
+    const executionMode = opts.dryRun
+        ? "dry-run"
+        : opts.initOnly
+            ? "init-only"
+            : "execute";
+    const state = {
+        version: 1,
+        title: titleWithRoundLabel(spec.title, runId),
+        runId,
+        cwd: opts.cwd,
+        startedAt: new Date().toISOString(),
+        executionMode,
+        runner: {
+            pid: process.pid,
+            hostname: hostname(),
+            startedAt: new Date().toISOString(),
+            heartbeatAt: new Date().toISOString(),
+        },
+        status: opts.initOnly || opts.dryRun ? "pending" : "running",
+        ranks,
+        nodes,
+        ...(spec.evaluation ? { evaluation: { ...spec.evaluation } } : {}),
+        ...(spec.convergence
+            ? {
+                convergence: {
+                    enabled: spec.convergence.enabled === true,
+                    maxPasses: Math.max(1, spec.convergence.maxPasses ?? 3),
+                    currentPass: 1,
+                    passHistory: [],
+                },
+            }
+            : {}),
+        ...(opts.workerAssociation
+            ? { workerAssociation: structuredClone(opts.workerAssociation) }
+            : {}),
+    };
+    initRunBudgetLedger(state, spec.budget);
+    return state;
+}
+export function assertFrozenEvaluationBinding(spec, state) {
+    const declared = spec.evaluation;
+    const frozen = state.evaluation;
+    if (!declared && !frozen)
+        return;
+    if (!declared ||
+        !frozen ||
+        JSON.stringify(declared) !== JSON.stringify(frozen)) {
+        throw new Error(`dag run ${state.runId} evaluation binding drifted between run.json and state.json; refuse to continue`);
+    }
+}
+export async function assertEvaluationBindingPreflight(cwd, binding) {
+    if (!binding)
+        return;
+    const record = await readCandidateRecord(cwd, binding.candidateId);
+    if (record.manifest.bundleHash !== binding.bundleHash) {
+        throw new Error(`evaluation bundleHash mismatch for ${binding.candidateId}: declared ${binding.bundleHash}, registry ${record.manifest.bundleHash}`);
+    }
+    if (!["eligible", "experimenting", "accepted"].includes(record.status)) {
+        throw new Error(`evaluation candidate ${binding.candidateId} lifecycle status ${record.status} is not executable`);
+    }
+}
+export async function runDag(spec, opts) {
+    assertValidDagSpec(spec);
+    // Candidate identity is validated before run-id allocation or directory creation.
+    await assertEvaluationBindingPreflight(opts.cwd, spec.evaluation);
+    // Phase 0.5: new writer execution requires DagSpec v4 + live taskContractBinding match.
+    const { assertDagSpecAllowsNewWriterExecution, assertTaskContractBindingConsistent, } = await import("./task-contract-binding.js");
+    assertDagSpecAllowsNewWriterExecution(spec);
+    await assertTaskContractBindingConsistent({ repoRoot: opts.cwd, spec });
+    const runningIdentity = resolveRunningControllerIdentity();
+    if (!runningIdentity) {
+        throw new Error("running controller identity could not be resolved; refuse to create an unpinned DAG run");
+    }
+    assertRuntimeContractCompatible(spec, {
+        ...DAG_CONTROLLER_CAPABILITIES,
+        controllerVersion: runningIdentity.packageVersion,
+    });
+    const { ranks } = topoSortToRanks(spec);
+    const { maxConcurrent } = resolveEffectiveMaxConcurrent(Math.max(1, opts.maxConcurrent ?? 4), spec.budget);
+    let runId = opts.runId;
+    if (runId) {
+        for (const warning of validateDagRunIdFormat(runId).warnings) {
+            console.warn(`[run-dag] warning: ${warning}`);
+        }
+    }
+    else {
+        runId = await buildDagRunId(spec, { cwd: opts.cwd });
+    }
+    const state = createInitialRunState(spec, opts, ranks, runId);
+    const activeRunDir = getDagRunDir(opts.cwd, "active", state.runId);
+    const completedRunDir = getDagRunDir(opts.cwd, "completed", state.runId);
+    const pausedRunDir = getDagRunDir(opts.cwd, "paused", state.runId);
+    // R-A2: non-execution modes use dedicated lifecycles; only execute enters active/.
+    const lifecycleDir = state.executionMode === "dry-run"
+        ? "dry-run"
+        : state.executionMode === "init-only"
+            ? "init-only"
+            : "active";
+    const runDir = getDagRunDir(opts.cwd, lifecycleDir, state.runId);
+    await prepareRunDir(runDir);
+    await writeRunSpec(runDir, spec);
+    // Freeze the running controller identity as a run-owned fact before the
+    // first canonical state write, reusing the identity resolved for preflight.
+    state.controllerIdentityRef = await captureControllerIdentity({
+        runDir,
+        identity: runningIdentity,
+    });
+    // Resolve and snapshot all skill profiles before any node executes.
+    try {
+        const snapshot = await createSkillSnapshot({
+            runId: state.runId,
+            spec,
+            cwd: opts.cwd,
+            mode: "run-start",
+        });
+        state.skillSnapshotRef = await writeSkillSnapshot(runDir, snapshot);
+    }
+    catch (error) {
+        throw new Error(`failed to create run-start skill snapshot: ${error instanceof Error ? error.message : String(error)}`);
+    }
+    await writeRunState(runDir, state);
+    if (opts.workerAssociation) {
+        await writeJsonAtomic(path.join(runDir, ".runtime", "worker-association.json"), opts.workerAssociation);
+    }
+    await notifyRunObserver(opts.observer, "onRunStart", state);
+    try {
+        const startCheckpoint = await captureWorkspaceCheckpoint(opts.cwd);
+        await writeWorkspaceCheckpoint(runDir, WORKSPACE_CHECKPOINT_START_REL, startCheckpoint);
+    }
+    catch (error) {
+        console.warn(`[run-dag] warning: failed to write workspace start checkpoint: ${error instanceof Error ? error.message : String(error)}`);
+    }
+    if (opts.dryRun || opts.initOnly) {
+        return {
+            title: state.title,
+            runId: state.runId,
+            status: state.status,
+            ranks,
+            nodes: state.nodes,
+            runDir,
+        };
+    }
+    try {
+        return await executeDagCheckpoint({
+            spec,
+            state,
+            ranks,
+            runDir,
+            cwd: opts.cwd,
+            maxConcurrent,
+            executeNode: opts.executeNode,
+            observer: opts.observer,
+            abortSignal: opts.abortSignal,
+            activeRunDir,
+            completedRunDir,
+            pausedRunDir,
+        });
+    }
+    finally {
+    }
+}
+export async function runDagContinuation(opts) {
+    const { maxConcurrent } = resolveEffectiveMaxConcurrent(Math.max(1, opts.maxConcurrent ?? 4), opts.spec.budget);
+    const state = opts.state;
+    state.status = "running";
+    const resumedAt = new Date().toISOString();
+    state.runner = {
+        pid: process.pid,
+        hostname: hostname(),
+        startedAt: resumedAt,
+        heartbeatAt: resumedAt,
+    };
+    const activeRunDir = getDagRunDir(opts.cwd, "active", state.runId);
+    const completedRunDir = getDagRunDir(opts.cwd, "completed", state.runId);
+    const pausedRunDir = getDagRunDir(opts.cwd, "paused", state.runId);
+    return executeDagCheckpoint({
+        spec: opts.spec,
+        state,
+        ranks: state.ranks,
+        runDir: opts.runDir,
+        cwd: opts.cwd,
+        maxConcurrent,
+        executeNode: opts.executeNode,
+        observer: opts.observer,
+        abortSignal: opts.abortSignal,
+        activeRunDir,
+        completedRunDir,
+        pausedRunDir,
+    });
+}
+export async function resumeDagRun(opts) {
+    const located = await locateDagRun(opts.cwd, opts.runId);
+    if (!located) {
+        throw new Error(`dag run not found: ${opts.runId}`);
+    }
+    if (located.lifecycle === "completed") {
+        throw new Error(`dag run ${opts.runId} is already completed`);
+    }
+    if (located.lifecycle === "paused") {
+        throw new Error(`dag run ${opts.runId} is paused; approve before resume`);
+    }
+    const { runDir, state, spec } = await requireActiveDagRun(opts.cwd, opts.runId);
+    if (state.status === "paused") {
+        throw new Error(`dag run ${opts.runId} is paused; approve before resume`);
+    }
+    if (isTerminalDagRunStatus(state.status)) {
+        throw new Error(`dag run ${opts.runId} is already terminal (status=${state.status})`);
+    }
+    assertFrozenEvaluationBinding(spec, state);
+    assertFrozenBudget(spec.budget, state.budget, state.runId);
+    // Phase 0.5: refuse resume of pre-v4 writer DAGs; revalidate live binding.
+    const { assertDagSpecAllowsNewWriterExecution, assertTaskContractBindingConsistent, } = await import("./task-contract-binding.js");
+    assertDagSpecAllowsNewWriterExecution(spec);
+    await assertTaskContractBindingConsistent({ repoRoot: opts.cwd, spec });
+    // Runtime contract + controller identity must be re-verified before executing
+    // any remaining node on resume; drift fails closed.
+    const runningIdentity = resolveRunningControllerIdentity();
+    if (!runningIdentity) {
+        throw new Error("running controller identity could not be resolved; refuse to resume an unpinned DAG run");
+    }
+    assertRuntimeContractCompatible(spec, {
+        ...DAG_CONTROLLER_CAPABILITIES,
+        controllerVersion: runningIdentity.packageVersion,
+    });
+    await verifyControllerIdentityForResume({
+        runDir,
+        state,
+    });
+    const decisionNodeId = state.humanDecisionNodeId;
+    if (!decisionNodeId) {
+        throw new Error(`dag run ${opts.runId} has no human decision checkpoint; was it approved?`);
+    }
+    try {
+        await readHumanApprovalArtifact(runDir, decisionNodeId);
+    }
+    catch {
+        throw new Error(`dag run ${opts.runId} is missing human-approval.json for node ${decisionNodeId}`);
+    }
+    // Validate or backfill before mutating runner/status for this resume attempt.
+    try {
+        const prepared = await prepareSkillSnapshotForContinuation({
+            runDir,
+            runId: state.runId,
+            spec,
+            state,
+            cwd: state.cwd,
+            ref: state.skillSnapshotRef,
+        });
+        if (prepared.backfilled) {
+            console.warn(`[dag resume] warning: run ${state.runId} has no skill snapshot ref; creating a legacy-resume-backfill that freezes remaining nodes only`);
+            state.skillSnapshotRef = prepared.ref;
+            await writeRunState(runDir, state);
+        }
+    }
+    catch (error) {
+        throw new Error(`skill snapshot validation failed on resume: ${error instanceof Error ? error.message : String(error)}`);
+    }
+    const { maxConcurrent } = resolveEffectiveMaxConcurrent(Math.max(1, opts.maxConcurrent ?? 4), spec.budget);
+    state.status = "running";
+    const resumedAt = new Date().toISOString();
+    state.runner = {
+        pid: process.pid,
+        hostname: hostname(),
+        startedAt: resumedAt,
+        heartbeatAt: resumedAt,
+    };
+    const activeRunDir = getDagRunDir(opts.cwd, "active", state.runId);
+    const completedRunDir = getDagRunDir(opts.cwd, "completed", state.runId);
+    const pausedRunDir = getDagRunDir(opts.cwd, "paused", state.runId);
+    try {
+        return await executeDagCheckpoint({
+            spec,
+            state,
+            ranks: state.ranks,
+            runDir,
+            cwd: state.cwd,
+            maxConcurrent,
+            executeNode: opts.executeNode,
+            observer: opts.observer,
+            // Resume path intentionally has no abortSignal field on ResumeDagRunOptions yet.
+            activeRunDir,
+            completedRunDir,
+            pausedRunDir,
+        });
+    }
+    finally {
+    }
+}
+async function executeDagCheckpoint(input) {
+    const { spec, state, ranks, runDir: initialRunDir, cwd, maxConcurrent, completedRunDir, pausedRunDir, } = input;
+    let runDir = initialRunDir;
+    let stateWriteQueue = Promise.resolve();
+    const persistState = async (options) => {
+        stateWriteQueue = stateWriteQueue.then(() => writeRunState(runDir, state, options));
+        await stateWriteQueue;
+    };
+    const runnerLivenessPolicy = resolveLivenessPolicy(spec.defaults?.livenessPolicy);
+    const heartbeatTimer = setInterval(() => {
+        if (!state.runner)
+            return;
+        // Runner lease only — never counts as meaningful Pi progress.
+        state.runner.heartbeatAt = new Date().toISOString();
+        const tasksByIdForPolicy = new Map(spec.tasks.map((task) => [task.id, task]));
+        for (const node of Object.values(state.nodes)) {
+            if (node.status !== "RUNNING")
+                continue;
+            // Lease clock is separate from meaningful activity.
+            node.lastLeaseAt = state.runner.heartbeatAt;
+            const task = tasksByIdForPolicy.get(node.id);
+            const policy = resolveLivenessPolicy(spec.defaults?.livenessPolicy, task?.livenessPolicy);
+            const evaluation = evaluateNodeLiveness({
+                node,
+                policy,
+                runnerHeartbeatAt: state.runner.heartbeatAt,
+            });
+            node.livenessStatus = evaluation.status;
+            // Intentionally do NOT refresh lastActivityAt / lastMeaningfulProgressAt
+            // from the runner lease timer.
+        }
+        void persistState().catch(() => { });
+    }, runnerLivenessPolicy.heartbeatIntervalMs);
+    heartbeatTimer.unref();
+    await persistState();
+    const interruptController = new AbortController();
+    const expectedRunnerIdentity = runnerIdentityFromState(state);
+    const interruptWatcher = expectedRunnerIdentity
+        ? startInterruptWatcher({
+            runDir,
+            runId: state.runId,
+            expectedRunnerIdentity,
+            controller: interruptController,
+        })
+        : undefined;
+    const abortSignal = mergeAbortSignals(input.abortSignal, interruptController.signal);
+    // Graceful terminal persistence: an outer SIGTERM/SIGINT (operator hard
+    // timeout, supervision layer, or shell wall-clock) must not leave the run
+    // orphaned in RUNNING with a dead heartbeat. Persist a terminal failed
+    // state with an explicit terminalReason before exiting so downstream
+    // liveness/doctor tooling sees a diagnosable terminal instead of an orphan.
+    let terminalSignal;
+    const shutdownOnSignal = (signal) => {
+        if (terminalSignal)
+            return;
+        terminalSignal = signal;
+        clearInterval(heartbeatTimer);
+        if (state.runner)
+            state.runner.heartbeatAt = new Date().toISOString();
+        state.status = "failed";
+        state.finishedAt = new Date().toISOString();
+        state.terminalReason = `runner terminated by ${signal} before run completion`;
+        console.warn(`[run-dag] received ${signal}; persisting terminal state and exiting`);
+        const exit = () => process.exit(signal === "SIGINT" ? 130 : 143);
+        // Best-effort atomic persist with a hard exit deadline so a stuck write
+        // queue cannot keep the process alive past the supervisor's kill window.
+        const hardExit = setTimeout(exit, 5000);
+        hardExit.unref();
+        void persistState()
+            .catch(() => { })
+            .finally(() => {
+            clearTimeout(hardExit);
+            exit();
+        });
+    };
+    const onSigTerm = () => shutdownOnSignal("SIGTERM");
+    const onSigInt = () => shutdownOnSignal("SIGINT");
+    process.once("SIGTERM", onSigTerm);
+    process.once("SIGINT", onSigInt);
+    try {
+        const tasksById = new Map(spec.tasks.map((task) => [task.id, task]));
+        const baseExecuteNode = input.executeNode ??
+            createDagNodeExecutor({
+                runDir,
+                runId: state.runId,
+                spec,
+            });
+        const executeDynamicNode = async (dynamicInput) => {
+            const { task } = dynamicInput;
+            if (task.dynamicExpansion) {
+                return executeDynamicMapExpansion({
+                    ...dynamicInput,
+                    expansion: task.dynamicExpansion,
+                    executeDynamicNode,
+                });
+            }
+            if (task.dynamicCondition) {
+                return executeDynamicCondition({
+                    task,
+                    condition: task.dynamicCondition,
+                    tasksById: dynamicInput.tasksById,
+                    state: dynamicInput.state,
+                });
+            }
+            if (task.dynamicLoopUntil) {
+                return executeDynamicLoopUntil({
+                    ...dynamicInput,
+                    loop: task.dynamicLoopUntil,
+                    executeDynamicNode,
+                });
+            }
+            return executeDynamicReduction({
+                task,
+                reduction: task.dynamicReduction,
+                state: dynamicInput.state,
+                runDir: dynamicInput.runDir,
+            });
+        };
+        const deferTerminalCloseout = state.convergence !== undefined && tasksById.has("closeout-pi");
+        const convergenceRanks = deferTerminalCloseout
+            ? ranks
+                .map((rank) => rank.filter((nodeId) => nodeId !== "closeout-pi"))
+                .filter((rank) => rank.length > 0)
+            : ranks;
+        const terminalCloseoutRanks = deferTerminalCloseout
+            ? ranks
+                .map((rank) => rank.filter((nodeId) => nodeId === "closeout-pi"))
+                .filter((rank) => rank.length > 0)
+            : [];
+        const executeRanks = (selectedRanks) => executeDagRanksOnce({
+            state,
+            ranks: selectedRanks,
+            tasksById,
+            maxConcurrent,
+            persistState,
+            runDir,
+            abortSignal: input.abortSignal,
+            createExecuteNodeForRank: (rankWriterNodeIds) => buildRankAwareExecuteNode({
+                baseExecuteNode,
+                customExecuteNode: input.executeNode,
+                rankWriterNodeIds,
+                tasksById,
+                meta: { runDir, runId: state.runId, spec },
+            }),
+            executeScheduledNode: async (nodeId, executeNode, onPause) => {
+                if (abortSignal?.aborted)
+                    return;
+                if (isHardBudgetBreached(state.budgetLedger))
+                    return;
+                const preBreach = preflightBudgetOrBreach(state);
+                if (preBreach && isHardBudgetBreached(state.budgetLedger)) {
+                    await writeBudgetLedgerArtifacts(runDir, state.budgetLedger);
+                    await persistState();
+                    return;
+                }
+                await executeDagNode({
+                    nodeId,
+                    tasksById,
+                    state,
+                    spec,
+                    cwd,
+                    runDir,
+                    executeNode,
+                    executeDynamicNode,
+                    observer: input.observer,
+                    persistState,
+                    onPause,
+                });
+                const node = state.nodes[nodeId];
+                if (node && (node.status === "FINISHED" || node.status === "ERROR")) {
+                    recordFinishedNodeBudget(state, node);
+                    await writeBudgetLedgerArtifacts(runDir, state.budgetLedger);
+                    await persistState();
+                }
+            },
+        });
+        let pausedByNodeId;
+        while (true) {
+            pausedByNodeId = await executeRanks(convergenceRanks);
+            if (pausedByNodeId) {
+                break;
+            }
+            if (isHardBudgetBreached(state.budgetLedger)) {
+                break;
+            }
+            const convergenceDecision = await runConvergencePassController({
+                spec,
+                state,
+                tasksById,
+                runDir,
+                cwd,
+                persistState,
+            });
+            if (convergenceDecision.pausedByNodeId) {
+                pausedByNodeId = convergenceDecision.pausedByNodeId;
+                break;
+            }
+            if (convergenceDecision.retry) {
+                continue;
+            }
+            break;
+        }
+        if (!pausedByNodeId &&
+            terminalCloseoutRanks.length > 0 &&
+            state.convergence?.terminalReason &&
+            !isHardBudgetBreached(state.budgetLedger)) {
+            // Convergence closeout is a terminal phase, not an ordinary descendant
+            // rank. Running it only after the controller records terminalReason avoids
+            // stale success handoffs on intermediate request-revision passes while
+            // still producing a truthful failure handoff for max-passes/non-retry.
+            pausedByNodeId = await executeRanks(terminalCloseoutRanks);
+        }
+        interruptWatcher?.stop();
+        // Stop heartbeats before terminal archive. A late heartbeat writing
+        // state.json under completed/ trips the completed-facts write guard and
+        // makes run-dag exit non-zero after every node already finished.
+        clearInterval(heartbeatTimer);
+        await stateWriteQueue;
+        state.finishedAt = new Date().toISOString();
+        const runDirBeforeTransfer = runDir;
+        await writeBudgetLedgerArtifacts(runDir, state.budgetLedger);
+        if (pausedByNodeId) {
+            state.status = "paused";
+            await finalizeRunOwnedInterrupt({
+                runDir,
+                state,
+                paused: true,
+                interruptAborted: interruptController.signal.aborted,
+            });
+            await persistState();
+            await notifyRunObserver(input.observer, "onRunFinish", state);
+            runDir = await moveToPausedRunDir(runDir, pausedRunDir);
+        }
+        else {
+            const { recoveryPending } = await finalizeTerminalRunStatus(state, spec.tasks.length, runDir, cwd);
+            await finalizeRunOwnedInterrupt({
+                runDir,
+                state,
+                paused: false,
+                interruptAborted: interruptController.signal.aborted,
+            });
+            await persistState();
+            if (recoveryPending) {
+                // Recovery coordinator (phase 3c AC-2/AC-3): materialize the reserved
+                // child and run it synchronously. Dynamic import avoids a static
+                // runner ↔ frontend-recovery-run module cycle. stageRecoveryChild
+                // locates the parent via the canonical active/ layout, so recovery
+                // only fires when this run is still the canonical active run dir
+                // (direct runDagContinuation callers using an arbitrary runDir keep
+                // the previous scheduler-layer skip semantics).
+                const canonicalActiveDir = getDagRunDir(cwd, "active", state.runId);
+                if (path.resolve(runDir) === path.resolve(canonicalActiveDir)) {
+                    const { stageRecoveryChild } = await import("./frontend-recovery-run.js");
+                    const staged = await stageRecoveryChild({
+                        cwd,
+                        parentRunId: state.runId,
+                    });
+                    // stageRecoveryChild CAS-advanced the parent's on-disk lineage
+                    // (child-staging → child-running + childRunId). Refresh the in-memory
+                    // copy so the terminal persistState below does not regress the parent
+                    // back to stale child-staging.
+                    const parentOnDisk = await readDagRunState(runDir);
+                    state.frontendRecoveryState = parentOnDisk.frontendRecoveryState;
+                    const childState = await readDagRunState(staged.childRunDir);
+                    const childSpec = await readDagRunSpec(staged.childRunDir);
+                    await runDagContinuation({
+                        cwd,
+                        spec: childSpec,
+                        state: childState,
+                        runDir: staged.childRunDir,
+                        maxConcurrent,
+                        executeNode: input.executeNode,
+                        observer: input.observer,
+                        abortSignal,
+                    });
+                }
+            }
+            await notifyRunObserver(input.observer, "onRunFinish", state);
+            if (!recoveryPending) {
+                try {
+                    const terminalCheckpoint = await captureWorkspaceCheckpoint(cwd);
+                    await writeWorkspaceCheckpoint(runDir, WORKSPACE_CHECKPOINT_TERMINAL_REL, terminalCheckpoint);
+                }
+                catch (error) {
+                    console.warn(`[run-dag] warning: failed to write workspace terminal checkpoint: ${error instanceof Error ? error.message : String(error)}`);
+                }
+                runDir = await moveToCompletedRunDir(runDir, completedRunDir);
+            }
+        }
+        await relocateRunArtifactPaths({
+            runDir,
+            oldRunDir: runDirBeforeTransfer,
+            nodes: state.nodes,
+        });
+        if (state.convergence) {
+            relocateConvergenceArtifactPaths(state.convergence, runDirBeforeTransfer, runDir);
+        }
+        await writeBudgetLedgerArtifacts(runDir, state.budgetLedger);
+        await persistState({ allowCompletedFactsWrite: true });
+        return {
+            title: state.title,
+            runId: state.runId,
+            status: state.status,
+            ranks,
+            nodes: state.nodes,
+            runDir,
+        };
+    }
+    finally {
+        clearInterval(heartbeatTimer);
+        interruptWatcher?.stop();
+        process.removeListener("SIGTERM", onSigTerm);
+        process.removeListener("SIGINT", onSigInt);
+    }
+}
+async function finalizeRunOwnedInterrupt(input) {
+    const request = await readInterruptRequest(input.runDir);
+    if (!request)
+        return;
+    if (request.status === "settled" || request.status === "needs-reconcile") {
+        return;
+    }
+    const hasUnconfirmed = Object.values(input.state.nodes).some((node) => node.failureCategory === "termination-unconfirmed");
+    if (hasUnconfirmed || input.paused) {
+        await markInterruptNeedsReconcile({
+            runDir: input.runDir,
+            outcome: hasUnconfirmed
+                ? "termination-unconfirmed"
+                : "paused-before-settle",
+        });
+        return;
+    }
+    if (input.interruptAborted &&
+        request.status === "acknowledged" &&
+        (input.state.status === "failed" ||
+            input.state.status === "partial_failed" ||
+            input.state.failureCategory === "controller-interrupted")) {
+        await settleDagInterrupt({
+            runDir: input.runDir,
+            outcome: "controller-interrupted",
+        });
+        return;
+    }
+    if (request.status === "requested") {
+        await markInterruptNeedsReconcile({
+            runDir: input.runDir,
+            outcome: "runner-did-not-acknowledge",
+        });
+        return;
+    }
+    await markInterruptNeedsReconcile({
+        runDir: input.runDir,
+        outcome: "run-exited-before-interrupt-settled",
+    });
+}
+async function notifyRunObserver(observer, event, state) {
+    try {
+        await observer?.[event]?.(state);
+    }
+    catch {
+        // Observers are derived views; they must not affect canonical DAG execution.
+    }
+}
+const SUCCESSFUL_CONVERGENCE_TERMINAL_REASONS = new Set([
+    "review-pass",
+    "hard-verify-pass",
+]);
+/**
+ * Intermediate soft-verify failures may be excluded from terminal error
+ * aggregation only when repair + final hard verify succeeded and, when a review
+ * gate is present, convergence ended in review-pass. Historical node status is
+ * never rewritten.
+ */
+function collectSupersededIntermediateFailures(state) {
+    const supersededIds = new Set();
+    const supersededFailures = [];
+    const soft = state.nodes["soft-verify-shell"];
+    if (!soft || soft.status !== "ERROR") {
+        return { supersededIds, supersededFailures };
+    }
+    const repair = state.nodes["repair-pi"];
+    const hard = state.nodes["hard-verify-shell"];
+    if (repair?.status !== "FINISHED" || hard?.status !== "FINISHED") {
+        return { supersededIds, supersededFailures };
+    }
+    const observesReview = Boolean(state.nodes["review-gate-shell"]);
+    const terminalReason = state.convergence?.terminalReason;
+    if (observesReview) {
+        if (terminalReason !== "review-pass") {
+            return { supersededIds, supersededFailures };
+        }
+    }
+    else if (terminalReason && terminalReason !== "hard-verify-pass") {
+        return { supersededIds, supersededFailures };
+    }
+    // Evidence: at least one successful repair pass after soft failure, with hard verify finished in terminal state.
+    const hasRepairEvidence = !state.convergence?.enabled ||
+        state.convergence.passHistory.some((pass) => pass.reason === "hard-verify-failed" ||
+            pass.reason === "review-request-revision" ||
+            pass.status === "retrying") ||
+        repair.status === "FINISHED";
+    if (!hasRepairEvidence) {
+        return { supersededIds, supersededFailures };
+    }
+    supersededIds.add("soft-verify-shell");
+    supersededFailures.push({
+        nodeId: "soft-verify-shell",
+        preservedStatus: "ERROR",
+        supersededBy: ["repair-pi", "hard-verify-shell"],
+        pass: state.convergence?.currentPass,
+        reason: "intermediate-soft-verify-covered-by-repair-and-final-hard",
+    });
+    return { supersededIds, supersededFailures };
+}
+function isSuccessfulConvergenceTerminal(state) {
+    const reason = state.convergence?.terminalReason;
+    if (!reason)
+        return true;
+    return SUCCESSFUL_CONVERGENCE_TERMINAL_REASONS.has(reason);
+}
+function buildFrontendRecoveryIntent(state, failureSource) {
+    const existing = state.frontendRecoveryState;
+    const requestId = existing?.requestId ?? randomUUID();
+    return {
+        schemaVersion: 1,
+        phase: "child-staging",
+        requestId,
+        recoveryRootRunId: existing?.recoveryRootRunId ?? state.runId,
+        parentRunId: state.runId,
+        ...(existing?.childRunId ? { childRunId: existing.childRunId } : {}),
+        attemptId: existing?.attemptId ?? `recovery-${requestId}`,
+        attemptIndex: existing?.attemptIndex ?? 0,
+        continuationCount: existing?.continuationCount ?? 0,
+        ...(failureSource ? { failureSource } : {}),
+        revision: (existing?.revision ?? 0) + 1,
+    };
+}
+function buildCandidateContractInvalidResult(state, result, artifactHash) {
+    const recovery = state.frontendRecoveryState;
+    return {
+        schemaVersion: 1,
+        outcome: "candidate-contract-invalid",
+        origin: {
+            kind: "frontend-prewrite-gate",
+            parentRunId: recovery?.parentRunId ?? state.runId,
+            ...(recovery?.childRunId ? { childRunId: recovery.childRunId } : {}),
+            requestId: recovery?.requestId ?? "",
+            failedNodeId: result.selectedPlanNodeId,
+        },
+        failureClass: {
+            code: "candidate-contract-invalid",
+            classification: result.classification,
+            reason: result.failureReason ?? "candidate contract invalid",
+        },
+        evidenceRefs: [
+            {
+                runId: state.runId,
+                relativePath: FRONTEND_PREWRITE_RESULT_SOURCE_ARTIFACT,
+                sha256: artifactHash,
+            },
+        ],
+    };
+}
+function buildWriterPartialWriteBlockedResult(state, failedNodeId, reason) {
+    const recovery = state.frontendRecoveryState;
+    return {
+        schemaVersion: 1,
+        outcome: "auto-recovery-blocked",
+        origin: {
+            kind: "frontend-writer",
+            parentRunId: recovery?.parentRunId ?? state.runId,
+            ...(recovery?.childRunId ? { childRunId: recovery.childRunId } : {}),
+            requestId: recovery?.requestId ?? "",
+            failedNodeId,
+        },
+        failureClass: {
+            code: "writer-transient-partial-write",
+            classification: "transient",
+            reason,
+        },
+        evidenceRefs: [],
+    };
+}
+function settleFrontendRecovery(state, outcome) {
+    const recovery = state.frontendRecoveryState;
+    if (!recovery)
+        return;
+    recovery.phase = "settled";
+    if (!state.frontendRecoveryResult) {
+        state.frontendRecoveryResult = {
+            schemaVersion: 1,
+            outcome,
+            origin: {
+                kind: "frontend-prewrite-gate",
+                parentRunId: recovery.parentRunId,
+                ...(recovery.childRunId ? { childRunId: recovery.childRunId } : {}),
+                requestId: recovery.requestId,
+                failedNodeId: "",
+            },
+            evidenceRefs: [],
+        };
+    }
+}
+export async function finalizeTerminalRunStatus(state, taskCount, runDir, cwd) {
+    const repoRoot = cwd ?? path.resolve(runDir, "..", "..", "..", "..");
+    // Terminal override: a denied frontend writer must never surface as a
+    // mechanical partial_failed just because prewrite FINISHED while the writer
+    // was SKIPPED. Fail closed on retryable-invalid/blocked before aggregation.
+    let recoveryPending = false;
+    const skipFrontendRecovery = state.failureCategory === "controller-interrupted";
+    const hasFrontendWriter = Object.keys(state.nodes).some((id) => FRONTEND_WRITER_NODE_IDS.includes(id));
+    if (hasFrontendWriter && !skipFrontendRecovery) {
+        const admission = await readFrontendPrewriteResult(runDir);
+        if (admission.ok) {
+            if (admission.result.classification === "blocked") {
+                state.status = "failed";
+                return { recoveryPending: false };
+            }
+            if (admission.result.classification === "retryable-invalid") {
+                const continuationCount = state.frontendRecoveryState?.continuationCount ?? 0;
+                if (continuationCount >= 1) {
+                    // Continuation quota exhausted: a second retryable-invalid candidate
+                    // is authoritative candidate-contract-invalid → forced failed.
+                    state.frontendRecoveryResult = buildCandidateContractInvalidResult(state, admission.result, admission.artifactHash);
+                    state.status = "failed";
+                    return { recoveryPending: false };
+                }
+                // Root continuation still available: record the recovery intent and let
+                // mechanical aggregation settle partial_failed/failed. The parent keeps
+                // phase != settled (recovery in progress) and no final root manifest.
+                state.frontendRecoveryState = buildFrontendRecoveryIntent(state);
+                recoveryPending = true;
+            }
+        }
+    }
+    // Phase 5: writer transient partial write → rollback + recovery intent. Only
+    // for frontend-implementation writers with a remaining root continuation, and
+    // only when the rollback journal (captured before the provider call) restores
+    // cleanly; otherwise the run stays failed with auto-recovery-blocked.
+    if (hasFrontendWriter && !recoveryPending && !skipFrontendRecovery) {
+        const writerNodeId = FRONTEND_WRITER_NODE_IDS.find((id) => {
+            const node = state.nodes[id];
+            return (node &&
+                node.status === "ERROR" &&
+                isFrontendWriterTransientPartialWrite({
+                    failureCategory: node.failureCategory,
+                }));
+        });
+        if (writerNodeId) {
+            const continuationCount = state.frontendRecoveryState?.continuationCount ?? 0;
+            if (continuationCount < 1) {
+                const { rollbackFrontendWriter } = await import("./frontend-writer-recovery.js");
+                const rollback = await rollbackFrontendWriter({
+                    cwd: repoRoot,
+                    parentRunId: state.runId,
+                });
+                if (rollback.ok) {
+                    state.frontendRecoveryState = buildFrontendRecoveryIntent(state, "frontend-implement-pi");
+                    recoveryPending = true;
+                }
+                else {
+                    state.frontendRecoveryResult = buildWriterPartialWriteBlockedResult(state, writerNodeId, rollback.reason);
+                    state.status = "failed";
+                    return { recoveryPending: false };
+                }
+            }
+        }
+    }
+    const finishedCount = Object.values(state.nodes).filter((n) => n.status === "FINISHED").length;
+    const { supersededIds, supersededFailures } = collectSupersededIntermediateFailures(state);
+    if (state.convergence && supersededFailures.length > 0) {
+        state.convergence.supersededFailures = supersededFailures;
+    }
+    // Prefer map keys: DagNodeRecord may not always mirror the map key on `.id`.
+    const effectiveErrorCount = Object.entries(state.nodes).filter(([nodeId, n]) => n.status === "ERROR" && !supersededIds.has(nodeId)).length;
+    const supersededErrorCount = Object.entries(state.nodes).filter(([nodeId, n]) => n.status === "ERROR" && supersededIds.has(nodeId)).length;
+    const conditionSkippedCount = Object.values(state.nodes).filter((n) => n.status === "SKIPPED" && isConditionSkippedReason(n.skippedReason)).length;
+    const successTerminal = isSuccessfulConvergenceTerminal(state);
+    // Superseded intermediate ERRORs remain ERROR in history but count as settled
+    // for terminal aggregation only (they do not inflate effectiveErrorCount).
+    if (effectiveErrorCount === 0 &&
+        successTerminal &&
+        finishedCount + conditionSkippedCount + supersededErrorCount === taskCount) {
+        state.status = "finished";
+    }
+    else if (finishedCount > 0) {
+        state.status = "partial_failed";
+    }
+    else {
+        state.status = "failed";
+    }
+    // Terminal recovery bookkeeping for a child attempt: a child that finished
+    // recovered settles its own lineage; any other child terminal settles as
+    // auto-recovery-blocked (phase 3 performs no further recovery).
+    const recovery = state.frontendRecoveryState;
+    if (recovery && recovery.attemptIndex >= 1) {
+        if (state.status === "finished") {
+            settleFrontendRecovery(state, "recovered");
+        }
+        else if (!state.frontendRecoveryResult) {
+            settleFrontendRecovery(state, "auto-recovery-blocked");
+        }
+    }
+    return { recoveryPending };
+}
+function concurrentSiblingWriteSetsForNode(rankWriterNodeIds, nodeId, tasksById) {
+    if (rankWriterNodeIds.length <= 1)
+        return [];
+    return rankWriterNodeIds
+        .filter((id) => id !== nodeId)
+        .map((id) => tasksById.get(id)?.writeSet ?? [])
+        .filter((writeSet) => writeSet.length > 0);
+}
+function buildRankAwareExecuteNode(input) {
+    if (input.customExecuteNode) {
+        return input.baseExecuteNode;
+    }
+    return async (nodeInput) => {
+        const isPiWriter = nodeInput.task.executor === "pi" &&
+            nodeInput.task.toolProfile === "write" &&
+            Boolean(nodeInput.model);
+        if (!isPiWriter) {
+            return input.baseExecuteNode(nodeInput);
+        }
+        const siblingWriteSets = nodeInput.task.writePolicy === "exclusive"
+            ? concurrentSiblingWriteSetsForNode(input.rankWriterNodeIds, nodeInput.task.id, input.tasksById)
+            : [];
+        return executeDagPiNode({ ...nodeInput, model: nodeInput.model }, {
+            ...input.meta,
+            concurrentSiblingWriteSets: siblingWriteSets.length > 0 ? siblingWriteSets : undefined,
+            writeGuardAttribution: siblingWriteSets.length > 0 ? "best-effort" : "per-node",
+        });
+    };
+}