@tea-agent/loop-agent 0.36.4-beta.0 → 0.37.1-beta.0

package/dist/workflows/dag/runner.js

@@ -1,5 +1,4 @@
 import { readdir, readFile } from "node:fs/promises";
-import { randomUUID } from "node:crypto";
 import { writeJsonAtomic } from "../../infrastructure/harness/atomic-write.js";
 import { hostname } from "node:os";
 import path from "node:path";
@@ -7,8 +6,7 @@ import { isHardBudgetBreached, resolveEffectiveMaxConcurrent, } from "../../appl
 import { readCandidateRecord } from "../../infrastructure/evaluation/candidate-store.js";
 import { CANONICAL_TASK_ID_PATTERN, formatLocalCompactDate, } from "../../task/runtime.js";
 import { assertFrozenBudget, initRunBudgetLedger, preflightBudgetOrBreach, recordFinishedNodeBudget, writeBudgetLedgerArtifacts, } from "./budget-enforcement.js";
-import { getDagRunDir, isTerminalDagRunStatus, locateDagRun, readDagRunSpec, readDagRunState, readHumanApprovalArtifact, requireActiveDagRun, } from "./lifecycle.js";
-import { markInterruptNeedsReconcile, mergeAbortSignals, readInterruptRequest, runnerIdentityFromState, settleDagInterrupt, startInterruptWatcher, } from "./interrupt-request.js";
+import { getDagRunDir, isTerminalDagRunStatus, locateDagRun, readHumanApprovalArtifact, requireActiveDagRun, } from "./lifecycle.js";
 import { moveToCompletedRunDir, moveToPausedRunDir, prepareRunDir, writeRunSpec, writeRunState, } from "./run-store.js";
 import { evaluateNodeLiveness, resolveLivenessPolicy, } from "./liveness-policy.js";
 import { createDagNodeExecutor } from "./executor-registry.js";
@@ -23,9 +21,8 @@ import { createSkillSnapshot, prepareSkillSnapshotForContinuation, writeSkillSna
 import { captureWorkspaceCheckpoint, WORKSPACE_CHECKPOINT_START_REL, WORKSPACE_CHECKPOINT_TERMINAL_REL, writeWorkspaceCheckpoint, } from "./workspace-checkpoint.js";
 import { buildNodePrompt, buildNodePromptWithResolvedSkillInstructions, executeDagNode, } from "./node-execution.js";
 import { runConvergencePassController, } from "./convergence/controller.js";
-import { executeDagRanksOnce, FRONTEND_PREWRITE_RESULT_SOURCE_ARTIFACT, FRONTEND_WRITER_NODE_IDS, isConditionSkippedReason, readFrontendPrewriteResult, } from "./scheduler.js";
+import { executeDagRanksOnce, isConditionSkippedReason } from "./scheduler.js";
 import { topoSortToRanks } from "./topo.js";
-import { isFrontendWriterTransientPartialWrite } from "./frontend-writer-recovery.js";
 import { parseDagSpec, resolveModelForTask, } from "./types.js";
 import { executeDynamicCondition } from "./dynamic-runtime/condition.js";
 import { executeDynamicLoopUntil } from "./dynamic-runtime/loop-until.js";
@@ -451,51 +448,6 @@ async function executeDagCheckpoint(input) {
         void persistState().catch(() => { });
     }, runnerLivenessPolicy.heartbeatIntervalMs);
     heartbeatTimer.unref();
-    await persistState();
-    const interruptController = new AbortController();
-    const expectedRunnerIdentity = runnerIdentityFromState(state);
-    const interruptWatcher = expectedRunnerIdentity
-        ? startInterruptWatcher({
-            runDir,
-            runId: state.runId,
-            expectedRunnerIdentity,
-            controller: interruptController,
-        })
-        : undefined;
-    const abortSignal = mergeAbortSignals(input.abortSignal, interruptController.signal);
-    // Graceful terminal persistence: an outer SIGTERM/SIGINT (operator hard
-    // timeout, supervision layer, or shell wall-clock) must not leave the run
-    // orphaned in RUNNING with a dead heartbeat. Persist a terminal failed
-    // state with an explicit terminalReason before exiting so downstream
-    // liveness/doctor tooling sees a diagnosable terminal instead of an orphan.
-    let terminalSignal;
-    const shutdownOnSignal = (signal) => {
-        if (terminalSignal)
-            return;
-        terminalSignal = signal;
-        clearInterval(heartbeatTimer);
-        if (state.runner)
-            state.runner.heartbeatAt = new Date().toISOString();
-        state.status = "failed";
-        state.finishedAt = new Date().toISOString();
-        state.terminalReason = `runner terminated by ${signal} before run completion`;
-        console.warn(`[run-dag] received ${signal}; persisting terminal state and exiting`);
-        const exit = () => process.exit(signal === "SIGINT" ? 130 : 143);
-        // Best-effort atomic persist with a hard exit deadline so a stuck write
-        // queue cannot keep the process alive past the supervisor's kill window.
-        const hardExit = setTimeout(exit, 5000);
-        hardExit.unref();
-        void persistState()
-            .catch(() => { })
-            .finally(() => {
-            clearTimeout(hardExit);
-            exit();
-        });
-    };
-    const onSigTerm = () => shutdownOnSignal("SIGTERM");
-    const onSigInt = () => shutdownOnSignal("SIGINT");
-    process.once("SIGTERM", onSigTerm);
-    process.once("SIGINT", onSigInt);
     try {
         const tasksById = new Map(spec.tasks.map((task) => [task.id, task]));
         const baseExecuteNode = input.executeNode ??
@@ -552,7 +504,6 @@ async function executeDagCheckpoint(input) {
             tasksById,
             maxConcurrent,
             persistState,
-            runDir,
             abortSignal: input.abortSignal,
             createExecuteNodeForRank: (rankWriterNodeIds) => buildRankAwareExecuteNode({
                 baseExecuteNode,
@@ -562,7 +513,7 @@ async function executeDagCheckpoint(input) {
                 meta: { runDir, runId: state.runId, spec },
             }),
             executeScheduledNode: async (nodeId, executeNode, onPause) => {
-                if (abortSignal?.aborted)
+                if (input.abortSignal?.aborted)
                     return;
                 if (isHardBudgetBreached(state.budgetLedger))
                     return;
@@ -629,7 +580,6 @@ async function executeDagCheckpoint(input) {
             // still producing a truthful failure handoff for max-passes/non-retry.
             pausedByNodeId = await executeRanks(terminalCloseoutRanks);
         }
-        interruptWatcher?.stop();
         // Stop heartbeats before terminal archive. A late heartbeat writing
         // state.json under completed/ trips the completed-facts write guard and
         // makes run-dag exit non-zero after every node already finished.
@@ -640,71 +590,22 @@ async function executeDagCheckpoint(input) {
         await writeBudgetLedgerArtifacts(runDir, state.budgetLedger);
         if (pausedByNodeId) {
             state.status = "paused";
-            await finalizeRunOwnedInterrupt({
-                runDir,
-                state,
-                paused: true,
-                interruptAborted: interruptController.signal.aborted,
-            });
             await persistState();
             await notifyRunObserver(input.observer, "onRunFinish", state);
             runDir = await moveToPausedRunDir(runDir, pausedRunDir);
         }
         else {
-            const { recoveryPending } = await finalizeTerminalRunStatus(state, spec.tasks.length, runDir, cwd);
-            await finalizeRunOwnedInterrupt({
-                runDir,
-                state,
-                paused: false,
-                interruptAborted: interruptController.signal.aborted,
-            });
+            finalizeTerminalRunStatus(state, spec.tasks.length);
             await persistState();
-            if (recoveryPending) {
-                // Recovery coordinator (phase 3c AC-2/AC-3): materialize the reserved
-                // child and run it synchronously. Dynamic import avoids a static
-                // runner ↔ frontend-recovery-run module cycle. stageRecoveryChild
-                // locates the parent via the canonical active/ layout, so recovery
-                // only fires when this run is still the canonical active run dir
-                // (direct runDagContinuation callers using an arbitrary runDir keep
-                // the previous scheduler-layer skip semantics).
-                const canonicalActiveDir = getDagRunDir(cwd, "active", state.runId);
-                if (path.resolve(runDir) === path.resolve(canonicalActiveDir)) {
-                    const { stageRecoveryChild } = await import("./frontend-recovery-run.js");
-                    const staged = await stageRecoveryChild({
-                        cwd,
-                        parentRunId: state.runId,
-                    });
-                    // stageRecoveryChild CAS-advanced the parent's on-disk lineage
-                    // (child-staging → child-running + childRunId). Refresh the in-memory
-                    // copy so the terminal persistState below does not regress the parent
-                    // back to stale child-staging.
-                    const parentOnDisk = await readDagRunState(runDir);
-                    state.frontendRecoveryState = parentOnDisk.frontendRecoveryState;
-                    const childState = await readDagRunState(staged.childRunDir);
-                    const childSpec = await readDagRunSpec(staged.childRunDir);
-                    await runDagContinuation({
-                        cwd,
-                        spec: childSpec,
-                        state: childState,
-                        runDir: staged.childRunDir,
-                        maxConcurrent,
-                        executeNode: input.executeNode,
-                        observer: input.observer,
-                        abortSignal,
-                    });
-                }
-            }
             await notifyRunObserver(input.observer, "onRunFinish", state);
-            if (!recoveryPending) {
-                try {
-                    const terminalCheckpoint = await captureWorkspaceCheckpoint(cwd);
-                    await writeWorkspaceCheckpoint(runDir, WORKSPACE_CHECKPOINT_TERMINAL_REL, terminalCheckpoint);
-                }
-                catch (error) {
-                    console.warn(`[run-dag] warning: failed to write workspace terminal checkpoint: ${error instanceof Error ? error.message : String(error)}`);
-                }
-                runDir = await moveToCompletedRunDir(runDir, completedRunDir);
+            try {
+                const terminalCheckpoint = await captureWorkspaceCheckpoint(cwd);
+                await writeWorkspaceCheckpoint(runDir, WORKSPACE_CHECKPOINT_TERMINAL_REL, terminalCheckpoint);
             }
+            catch (error) {
+                console.warn(`[run-dag] warning: failed to write workspace terminal checkpoint: ${error instanceof Error ? error.message : String(error)}`);
+            }
+            runDir = await moveToCompletedRunDir(runDir, completedRunDir);
         }
         await relocateRunArtifactPaths({
             runDir,
@@ -727,50 +628,7 @@ async function executeDagCheckpoint(input) {
     }
     finally {
         clearInterval(heartbeatTimer);
-        interruptWatcher?.stop();
-        process.removeListener("SIGTERM", onSigTerm);
-        process.removeListener("SIGINT", onSigInt);
-    }
-}
-async function finalizeRunOwnedInterrupt(input) {
-    const request = await readInterruptRequest(input.runDir);
-    if (!request)
-        return;
-    if (request.status === "settled" || request.status === "needs-reconcile") {
-        return;
-    }
-    const hasUnconfirmed = Object.values(input.state.nodes).some((node) => node.failureCategory === "termination-unconfirmed");
-    if (hasUnconfirmed || input.paused) {
-        await markInterruptNeedsReconcile({
-            runDir: input.runDir,
-            outcome: hasUnconfirmed
-                ? "termination-unconfirmed"
-                : "paused-before-settle",
-        });
-        return;
-    }
-    if (input.interruptAborted &&
-        request.status === "acknowledged" &&
-        (input.state.status === "failed" ||
-            input.state.status === "partial_failed" ||
-            input.state.failureCategory === "controller-interrupted")) {
-        await settleDagInterrupt({
-            runDir: input.runDir,
-            outcome: "controller-interrupted",
-        });
-        return;
-    }
-    if (request.status === "requested") {
-        await markInterruptNeedsReconcile({
-            runDir: input.runDir,
-            outcome: "runner-did-not-acknowledge",
-        });
-        return;
     }
-    await markInterruptNeedsReconcile({
-        runDir: input.runDir,
-        outcome: "run-exited-before-interrupt-settled",
-    });
 }
 async function notifyRunObserver(observer, event, state) {
     try {
@@ -837,154 +695,7 @@ function isSuccessfulConvergenceTerminal(state) {
         return true;
     return SUCCESSFUL_CONVERGENCE_TERMINAL_REASONS.has(reason);
 }
-function buildFrontendRecoveryIntent(state, failureSource) {
-    const existing = state.frontendRecoveryState;
-    const requestId = existing?.requestId ?? randomUUID();
-    return {
-        schemaVersion: 1,
-        phase: "child-staging",
-        requestId,
-        recoveryRootRunId: existing?.recoveryRootRunId ?? state.runId,
-        parentRunId: state.runId,
-        ...(existing?.childRunId ? { childRunId: existing.childRunId } : {}),
-        attemptId: existing?.attemptId ?? `recovery-${requestId}`,
-        attemptIndex: existing?.attemptIndex ?? 0,
-        continuationCount: existing?.continuationCount ?? 0,
-        ...(failureSource ? { failureSource } : {}),
-        revision: (existing?.revision ?? 0) + 1,
-    };
-}
-function buildCandidateContractInvalidResult(state, result, artifactHash) {
-    const recovery = state.frontendRecoveryState;
-    return {
-        schemaVersion: 1,
-        outcome: "candidate-contract-invalid",
-        origin: {
-            kind: "frontend-prewrite-gate",
-            parentRunId: recovery?.parentRunId ?? state.runId,
-            ...(recovery?.childRunId ? { childRunId: recovery.childRunId } : {}),
-            requestId: recovery?.requestId ?? "",
-            failedNodeId: result.selectedPlanNodeId,
-        },
-        failureClass: {
-            code: "candidate-contract-invalid",
-            classification: result.classification,
-            reason: result.failureReason ?? "candidate contract invalid",
-        },
-        evidenceRefs: [
-            {
-                runId: state.runId,
-                relativePath: FRONTEND_PREWRITE_RESULT_SOURCE_ARTIFACT,
-                sha256: artifactHash,
-            },
-        ],
-    };
-}
-function buildWriterPartialWriteBlockedResult(state, failedNodeId, reason) {
-    const recovery = state.frontendRecoveryState;
-    return {
-        schemaVersion: 1,
-        outcome: "auto-recovery-blocked",
-        origin: {
-            kind: "frontend-writer",
-            parentRunId: recovery?.parentRunId ?? state.runId,
-            ...(recovery?.childRunId ? { childRunId: recovery.childRunId } : {}),
-            requestId: recovery?.requestId ?? "",
-            failedNodeId,
-        },
-        failureClass: {
-            code: "writer-transient-partial-write",
-            classification: "transient",
-            reason,
-        },
-        evidenceRefs: [],
-    };
-}
-function settleFrontendRecovery(state, outcome) {
-    const recovery = state.frontendRecoveryState;
-    if (!recovery)
-        return;
-    recovery.phase = "settled";
-    if (!state.frontendRecoveryResult) {
-        state.frontendRecoveryResult = {
-            schemaVersion: 1,
-            outcome,
-            origin: {
-                kind: "frontend-prewrite-gate",
-                parentRunId: recovery.parentRunId,
-                ...(recovery.childRunId ? { childRunId: recovery.childRunId } : {}),
-                requestId: recovery.requestId,
-                failedNodeId: "",
-            },
-            evidenceRefs: [],
-        };
-    }
-}
-export async function finalizeTerminalRunStatus(state, taskCount, runDir, cwd) {
-    const repoRoot = cwd ?? path.resolve(runDir, "..", "..", "..", "..");
-    // Terminal override: a denied frontend writer must never surface as a
-    // mechanical partial_failed just because prewrite FINISHED while the writer
-    // was SKIPPED. Fail closed on retryable-invalid/blocked before aggregation.
-    let recoveryPending = false;
-    const skipFrontendRecovery = state.failureCategory === "controller-interrupted";
-    const hasFrontendWriter = Object.keys(state.nodes).some((id) => FRONTEND_WRITER_NODE_IDS.includes(id));
-    if (hasFrontendWriter && !skipFrontendRecovery) {
-        const admission = await readFrontendPrewriteResult(runDir);
-        if (admission.ok) {
-            if (admission.result.classification === "blocked") {
-                state.status = "failed";
-                return { recoveryPending: false };
-            }
-            if (admission.result.classification === "retryable-invalid") {
-                const continuationCount = state.frontendRecoveryState?.continuationCount ?? 0;
-                if (continuationCount >= 1) {
-                    // Continuation quota exhausted: a second retryable-invalid candidate
-                    // is authoritative candidate-contract-invalid → forced failed.
-                    state.frontendRecoveryResult = buildCandidateContractInvalidResult(state, admission.result, admission.artifactHash);
-                    state.status = "failed";
-                    return { recoveryPending: false };
-                }
-                // Root continuation still available: record the recovery intent and let
-                // mechanical aggregation settle partial_failed/failed. The parent keeps
-                // phase != settled (recovery in progress) and no final root manifest.
-                state.frontendRecoveryState = buildFrontendRecoveryIntent(state);
-                recoveryPending = true;
-            }
-        }
-    }
-    // Phase 5: writer transient partial write → rollback + recovery intent. Only
-    // for frontend-implementation writers with a remaining root continuation, and
-    // only when the rollback journal (captured before the provider call) restores
-    // cleanly; otherwise the run stays failed with auto-recovery-blocked.
-    if (hasFrontendWriter && !recoveryPending && !skipFrontendRecovery) {
-        const writerNodeId = FRONTEND_WRITER_NODE_IDS.find((id) => {
-            const node = state.nodes[id];
-            return (node &&
-                node.status === "ERROR" &&
-                isFrontendWriterTransientPartialWrite({
-                    failureCategory: node.failureCategory,
-                }));
-        });
-        if (writerNodeId) {
-            const continuationCount = state.frontendRecoveryState?.continuationCount ?? 0;
-            if (continuationCount < 1) {
-                const { rollbackFrontendWriter } = await import("./frontend-writer-recovery.js");
-                const rollback = await rollbackFrontendWriter({
-                    cwd: repoRoot,
-                    parentRunId: state.runId,
-                });
-                if (rollback.ok) {
-                    state.frontendRecoveryState = buildFrontendRecoveryIntent(state, "frontend-implement-pi");
-                    recoveryPending = true;
-                }
-                else {
-                    state.frontendRecoveryResult = buildWriterPartialWriteBlockedResult(state, writerNodeId, rollback.reason);
-                    state.status = "failed";
-                    return { recoveryPending: false };
-                }
-            }
-        }
-    }
+function finalizeTerminalRunStatus(state, taskCount) {
     const finishedCount = Object.values(state.nodes).filter((n) => n.status === "FINISHED").length;
     const { supersededIds, supersededFailures } = collectSupersededIntermediateFailures(state);
     if (state.convergence && supersededFailures.length > 0) {
@@ -1008,19 +719,6 @@ export async function finalizeTerminalRunStatus(state, taskCount, runDir, cwd) {
     else {
         state.status = "failed";
     }
-    // Terminal recovery bookkeeping for a child attempt: a child that finished
-    // recovered settles its own lineage; any other child terminal settles as
-    // auto-recovery-blocked (phase 3 performs no further recovery).
-    const recovery = state.frontendRecoveryState;
-    if (recovery && recovery.attemptIndex >= 1) {
-        if (state.status === "finished") {
-            settleFrontendRecovery(state, "recovered");
-        }
-        else if (!state.frontendRecoveryResult) {
-            settleFrontendRecovery(state, "auto-recovery-blocked");
-        }
-    }
-    return { recoveryPending };
 }
 function concurrentSiblingWriteSetsForNode(rankWriterNodeIds, nodeId, tasksById) {
     if (rankWriterNodeIds.length <= 1)