@tea-agent/loop-agent 0.44.0-next.10 → 0.44.0-next.11

package/dist/workflows/dag/terminal-status.js

@@ -0,0 +1,502 @@
+/** DAG runner 终态判定、前端恢复策略与恢复操作编排。含取证读取、lease 与 operation 创建；未拆成独立 policy/operation 文件。 */
+import path from "node:path";
+import { readFile } from "node:fs/promises";
+import { createHash } from "node:crypto";
+import { writeJsonAtomic } from "../../infrastructure/harness/atomic-write.js";
+import { FRONTEND_PREWRITE_RESULT_SOURCE_ARTIFACT, FRONTEND_WRITER_NODE_IDS, isConditionSkippedReason, readFrontendPrewriteResult } from "./scheduler.js";
+import { isFrontendWriterTransientPartialWrite } from "./frontend-writer-recovery.js";
+import { classifyReviewIssueCategoryFailureOwner, readCommittedDesignRequestFact, readCommittedReviewRequestFact, writeFrontendReviewFindingsPrd } from "./frontend-review-findings.js";
+import { computeActionPayloadHash } from "../../infrastructure/console/operation-store.js";
+import { classifyFailureOwner, failureOwnerSchema } from "./frontend-recovery-capsule.js";
+import { acquireDagRecoveryLease, bindDagRecoveryLeaseExecution, claimDagRecoveryLease } from "./recovery-lease.js";
+export function buildCandidateContractInvalidResult(state, result, artifactHash) {
+    return {
+        schemaVersion: 1,
+        outcome: "candidate-contract-invalid",
+        origin: {
+            kind: "frontend-prewrite-gate",
+            parentRunId: state.runId,
+            requestId: state.runId,
+            failedNodeId: result.failureSource ?? "frontend-plan-pi",
+        },
+        failureClass: {
+            code: "candidate-contract-invalid",
+            classification: result.classification,
+            reason: result.findings.map((finding) => `${finding.code}: ${finding.message}`).join("; ") ??
+                "candidate contract invalid",
+        },
+        evidenceRefs: [
+            {
+                runId: state.runId,
+                relativePath: FRONTEND_PREWRITE_RESULT_SOURCE_ARTIFACT,
+                sha256: artifactHash,
+            },
+        ],
+    };
+}
+export function buildWriterPartialWriteBlockedResult(state, failedNodeId, reason) {
+    return {
+        schemaVersion: 1,
+        outcome: "auto-recovery-blocked",
+        origin: {
+            kind: "frontend-writer",
+            parentRunId: state.runId,
+            requestId: state.runId,
+            failedNodeId,
+        },
+        failureClass: {
+            code: "writer-transient-partial-write",
+            classification: "transient",
+            reason,
+        },
+        evidenceRefs: [],
+    };
+}
+export function buildRecoveryResultForTrigger(state, trigger, outcome) {
+    return {
+        schemaVersion: 1,
+        outcome,
+        origin: {
+            kind: trigger.failureSource === "frontend-implement-pi"
+                ? "frontend-writer"
+                : "frontend-prewrite-gate",
+            parentRunId: state.runId,
+            requestId: trigger.requestId,
+            failedNodeId: trigger.failureSource,
+        },
+        evidenceRefs: [],
+    };
+}
+export function collectSupersededIntermediateFailures(state) {
+    const supersededIds = new Set();
+    const supersededFailures = [];
+    const soft = state.nodes["soft-verify-shell"];
+    if (!soft || soft.status !== "ERROR") {
+        return { supersededIds, supersededFailures };
+    }
+    const repair = state.nodes["repair-pi"];
+    const hard = state.nodes["hard-verify-shell"];
+    if (repair?.status !== "FINISHED" || hard?.status !== "FINISHED") {
+        return { supersededIds, supersededFailures };
+    }
+    const observesReview = Boolean(state.nodes["review-gate-shell"]);
+    const terminalReason = state.convergence?.terminalReason;
+    if (observesReview) {
+        if (terminalReason !== "review-pass") {
+            return { supersededIds, supersededFailures };
+        }
+    }
+    else if (terminalReason && terminalReason !== "hard-verify-pass") {
+        return { supersededIds, supersededFailures };
+    }
+    // Evidence: at least one successful repair pass after soft failure, with hard verify finished in terminal state.
+    const hasRepairEvidence = !state.convergence?.enabled ||
+        state.convergence.passHistory.some((pass) => pass.reason === "hard-verify-failed" ||
+            pass.reason === "review-request-revision" ||
+            pass.status === "retrying") ||
+        repair.status === "FINISHED";
+    if (!hasRepairEvidence) {
+        return { supersededIds, supersededFailures };
+    }
+    supersededIds.add("soft-verify-shell");
+    supersededFailures.push({
+        nodeId: "soft-verify-shell",
+        preservedStatus: "ERROR",
+        supersededBy: ["repair-pi", "hard-verify-shell"],
+        pass: state.convergence?.currentPass,
+        reason: "intermediate-soft-verify-covered-by-repair-and-final-hard",
+    });
+    return { supersededIds, supersededFailures };
+}
+export async function finalizeTerminalRunStatus(state, taskCount, runDir, cwd, frontendRecoveryQuota = 1) {
+    const repoRoot = cwd ?? path.resolve(runDir, "..", "..", "..", "..");
+    if (state.frontendShapeState?.terminalOutcome &&
+        state.frontendShapeState.transitionCapsule) {
+        state.status = "failed";
+        return { recoveryPending: false };
+    }
+    // Terminal override: a denied frontend writer must never surface as a
+    // mechanical partial_failed just because prewrite FINISHED while the writer
+    // was SKIPPED. Fail closed on blocked before aggregation.
+    let recoveryTrigger;
+    const skipFrontendRecovery = state.failureCategory === "controller-interrupted" ||
+        state.continuation?.writerTransaction !== undefined;
+    const hasFrontendWriter = Object.keys(state.nodes).some((id) => FRONTEND_WRITER_NODE_IDS.includes(id));
+    if (hasFrontendWriter && !skipFrontendRecovery) {
+        const admission = await readFrontendPrewriteResult(runDir);
+        if (admission.ok) {
+            // M8: admission uses the accepted/blocked state machine. A blocked admission whose
+            // failureSource is frontend-plan-pi is the contract-invalid analog of
+            // the old retryable-invalid path (candidate continuation may re-plan);
+            // every other blocked admission is terminal.
+            const planContractInvalid = admission.result.classification === "blocked" &&
+                admission.result.failureSource === "frontend-plan-pi";
+            // A blocked admission caused by the design review committing
+            // `request_design_changes` is a definite finding (not a protocol or
+            // environment error): the design was reviewed and rejected. It is the
+            // design-review analog of review-request-changes — not terminal, but
+            // re-enters the next plan round with the committed design findings.
+            // The failureSource may be frontend-writer-admission-shell or
+            // frontend-design-policy-shell; the committed design fact is the
+            // authoritative signal.
+            const designRequestFact = await readCommittedDesignRequestFact({
+                runDir,
+            });
+            const designRejected = admission.result.classification === "blocked" &&
+                designRequestFact !== undefined;
+            const terminalAdmissionDenial = (admission.result.classification === "blocked" ||
+                admission.result.classification === "stale") &&
+                !planContractInvalid &&
+                !designRejected;
+            if (terminalAdmissionDenial) {
+                state.status = "failed";
+                return { recoveryPending: false };
+            }
+            if (designRejected && designRequestFact) {
+                state.terminalReason = "design-request-changes";
+                try {
+                    await writeFrontendReviewFindingsPrd({
+                        runDir,
+                        parentRunId: state.runId,
+                        fact: designRequestFact,
+                    });
+                }
+                catch (error) {
+                    console.warn(`[run-dag] warning: failed to write frontend design findings PRD: ${error instanceof Error ? error.message : String(error)}`);
+                }
+                const continuationCount = state.frontendRecoveryState?.continuationCount ?? 0;
+                if (continuationCount < frontendRecoveryQuota) {
+                    recoveryTrigger = {
+                        requestId: state.runId,
+                        // The rejected design is a plan input defect. Reset from the
+                        // planner so the next child can incorporate the committed
+                        // findings before design review runs again.
+                        failureSource: "frontend-design-review-pi",
+                        failureOwner: classifyReviewIssueCategoryFailureOwner(designRequestFact.issueCategory),
+                        protocolFailureReason: `frontend design review request_design_changes (issueCategory=${designRequestFact.issueCategory})`,
+                        recoveryMode: "new-dag",
+                    };
+                }
+            }
+            if (planContractInvalid) {
+                const continuationCount = state.frontendRecoveryState?.continuationCount ?? 0;
+                if (continuationCount >= frontendRecoveryQuota) {
+                    // Continuation quota exhausted: a second contract-invalid candidate
+                    // is authoritative candidate-contract-invalid → forced failed.
+                    state.frontendRecoveryResult = buildCandidateContractInvalidResult(state, admission.result, admission.artifactHash);
+                    state.status = "failed";
+                    return { recoveryPending: false };
+                }
+                // Root continuation still available: emit a recovery trigger for the
+                // M6 operation controller. The parent no longer writes
+                // frontendRecoveryState or stays in active/ for a child CAS.
+                recoveryTrigger = {
+                    requestId: state.runId,
+                    failureSource: admission.result.failureSource ?? "frontend-plan-pi",
+                    failureOwner: "plan",
+                    protocolFailureReason: "frontend candidate contract invalid",
+                    recoveryMode: "new-dag",
+                };
+            }
+        }
+    }
+    // Phase 5: writer transient partial write → rollback + recovery trigger.
+    // Only for frontend-implementation writers with a remaining root
+    // continuation, and only when the rollback journal (captured before the
+    // provider call) restores cleanly; otherwise the run stays failed with
+    // auto-recovery-blocked.
+    if (hasFrontendWriter && !recoveryTrigger && !skipFrontendRecovery) {
+        const writerNodeId = FRONTEND_WRITER_NODE_IDS.find((id) => {
+            const node = state.nodes[id];
+            return (node &&
+                node.status === "ERROR" &&
+                isFrontendWriterTransientPartialWrite({
+                    failureCategory: node.failureCategory,
+                }));
+        });
+        if (writerNodeId) {
+            const continuationCount = state.frontendRecoveryState?.continuationCount ?? 0;
+            if (continuationCount < frontendRecoveryQuota) {
+                const { rollbackFrontendWriter } = await import("./frontend-writer-recovery.js");
+                const rollback = await rollbackFrontendWriter({
+                    cwd: repoRoot,
+                    parentRunId: state.runId,
+                });
+                if (rollback.ok) {
+                    recoveryTrigger = {
+                        requestId: state.runId,
+                        failureSource: "frontend-implement-pi",
+                        failureOwner: "implementation",
+                        protocolFailureReason: "frontend writer transient partial write",
+                        recoveryMode: "continuation",
+                    };
+                }
+                else {
+                    state.frontendRecoveryResult = buildWriterPartialWriteBlockedResult(state, writerNodeId, rollback.reason);
+                    state.status = "failed";
+                    return { recoveryPending: false };
+                }
+            }
+        }
+    }
+    // Phase 6: verification failure → automatic recovery (re-implement +
+    // re-verify). The verify shell is terminal by design so the failed run's
+    // evidence stays clean; the M6 recovery operation re-runs implement and
+    // verify in a fresh continuation, guarded by the same continuation quota,
+    // budget fold and failure fingerprint as the other recovery paths.
+    if (hasFrontendWriter && !recoveryTrigger && !skipFrontendRecovery) {
+        const verifyFailed = state.nodes["frontend-verify-shell"]?.status === "ERROR";
+        if (verifyFailed) {
+            const continuationCount = state.frontendRecoveryState?.continuationCount ?? 0;
+            if (continuationCount < frontendRecoveryQuota) {
+                let recoveryOwner = classifyFailureOwner({
+                    nodeId: "frontend-verify-shell",
+                    rawFailureCategory: state.nodes["frontend-verify-shell"]?.failureCategory,
+                });
+                try {
+                    const verifyFailure = JSON.parse(await readFile(path.join(runDir, "contracts/frontend-verify-failure.json"), "utf8"));
+                    const owner = failureOwnerSchema.safeParse(verifyFailure.failureOwner);
+                    if (verifyFailure.schemaVersion === 1 &&
+                        verifyFailure.schemaId === "frontend-verify-failure-v1" &&
+                        verifyFailure.nodeId === "frontend-verify-shell" &&
+                        verifyFailure.failureCategory === state.nodes["frontend-verify-shell"]?.failureCategory &&
+                        owner.success) {
+                        recoveryOwner = owner.data;
+                    }
+                    else {
+                        recoveryOwner = "unknown";
+                    }
+                }
+                catch {
+                    // The node's classified category remains the fallback; never turn
+                    // unknown/environment/configuration failures into writer work.
+                }
+                if (recoveryOwner === "implementation")
+                    recoveryTrigger = {
+                        requestId: state.runId,
+                        failureSource: "frontend-verify-shell",
+                        failureOwner: recoveryOwner,
+                        protocolFailureReason: "frontend verification failure",
+                        recoveryMode: "continuation",
+                    };
+                else
+                    state.terminalReason = recoveryOwner === "environment"
+                        ? "frontend-verification-wait-external"
+                        : `frontend-verification-${recoveryOwner}-requires-contract-review`;
+            }
+        }
+    }
+    // Phase 7: frontend review "request changes" → not passed. When the review
+    // node FINISHED but committed `request_review_changes` (typed terminal
+    // verdict), the implementation was reviewed and rejected. The run must not
+    // surface as finished: terminal aggregation below stays partial_failed, a
+    // terminal reason is recorded, the committed findings are rendered into a
+    // downstream-task PRD draft (run-owned), and a recovery trigger carries the
+    // issueCategory-derived owner so a later DAG round consumes the findings
+    // instead of repeating the previous flow.
+    const reviewRequestFact = await readCommittedReviewRequestFact({
+        runDir,
+    });
+    if (reviewRequestFact) {
+        state.terminalReason = "review-request-changes";
+        try {
+            await writeFrontendReviewFindingsPrd({
+                runDir,
+                parentRunId: state.runId,
+                fact: reviewRequestFact,
+            });
+        }
+        catch (error) {
+            // The PRD draft is additive; a write failure must not mask the
+            // not-passed verdict. The terminal aggregation below still applies.
+            console.warn(`[run-dag] warning: failed to write frontend review findings PRD: ${error instanceof Error ? error.message : String(error)}`);
+        }
+        if (!recoveryTrigger && !skipFrontendRecovery) {
+            const continuationCount = state.frontendRecoveryState?.continuationCount ?? 0;
+            if (continuationCount < frontendRecoveryQuota) {
+                recoveryTrigger = {
+                    requestId: state.runId,
+                    failureSource: "frontend-review-pi",
+                    failureOwner: classifyReviewIssueCategoryFailureOwner(reviewRequestFact.issueCategory),
+                    protocolFailureReason: `frontend review request_review_changes (issueCategory=${reviewRequestFact.issueCategory})`,
+                    recoveryMode: "new-dag",
+                };
+            }
+        }
+    }
+    const finishedCount = Object.values(state.nodes).filter((n) => n.status === "FINISHED").length;
+    const { supersededIds, supersededFailures } = collectSupersededIntermediateFailures(state);
+    if (state.convergence && supersededFailures.length > 0) {
+        state.convergence.supersededFailures = supersededFailures;
+    }
+    // Prefer map keys: DagNodeRecord may not always mirror the map key on `.id`.
+    const effectiveErrorCount = Object.entries(state.nodes).filter(([nodeId, n]) => n.status === "ERROR" && !supersededIds.has(nodeId)).length;
+    const supersededErrorCount = Object.entries(state.nodes).filter(([nodeId, n]) => n.status === "ERROR" && supersededIds.has(nodeId)).length;
+    const conditionSkippedCount = Object.values(state.nodes).filter((n) => n.status === "SKIPPED" && isConditionSkippedReason(n.skippedReason)).length;
+    const successTerminal = isSuccessfulConvergenceTerminal(state);
+    // Superseded intermediate ERRORs remain ERROR in history but count as settled
+    // for terminal aggregation only (they do not inflate effectiveErrorCount).
+    if (effectiveErrorCount === 0 &&
+        successTerminal &&
+        finishedCount + conditionSkippedCount + supersededErrorCount === taskCount) {
+        // Phase 7 guard: a committed `request_review_changes` means the review
+        // rejected the implementation; the run must never surface as finished
+        // even though every node technically FINISHED.
+        state.status =
+            state.terminalReason === "review-request-changes"
+                ? "partial_failed"
+                : "finished";
+    }
+    else if (finishedCount > 0) {
+        state.status = "partial_failed";
+    }
+    else {
+        state.status = "failed";
+    }
+    return recoveryTrigger
+        ? { recoveryPending: true, recovery: recoveryTrigger }
+        : { recoveryPending: false };
+}
+export function isSuccessfulConvergenceTerminal(state) {
+    const reason = state.convergence?.terminalReason;
+    if (!reason)
+        return true;
+    return SUCCESSFUL_CONVERGENCE_TERMINAL_REASONS.has(reason);
+}
+export async function createRecoveryOperation(input) {
+    const { cwd, runDir, spec, state, trigger } = input;
+    const { computeFrontendRecoveryPlanForSource } = await import("./frontend-recovery-plan.js");
+    let plan;
+    try {
+        plan = computeFrontendRecoveryPlanForSource({
+            spec,
+            parentRunId: state.runId,
+            requestId: trigger.requestId,
+            failureSource: trigger.failureSource,
+            failureOwner: trigger.failureOwner,
+        });
+    }
+    catch (error) {
+        return { kind: "blocked", reason: error instanceof Error ? error.message : String(error) };
+    }
+    const recoveryLease = await acquireDagRecoveryLease({
+        cwd,
+        parentRunId: state.runId,
+        requestId: trigger.requestId,
+        kind: "auto-frontend-recovery",
+    });
+    let effectiveLease = recoveryLease;
+    let observeOnly = false;
+    if (recoveryLease.kind === "held") {
+        if (recoveryLease.lease.kind !== "auto-frontend-recovery" ||
+            recoveryLease.lease.requestId !== trigger.requestId) {
+            return { kind: "lease-held", lease: recoveryLease.lease };
+        }
+        const claimed = await claimDagRecoveryLease({
+            leasePath: recoveryLease.leasePath,
+            lease: recoveryLease.lease,
+        });
+        effectiveLease = claimed;
+        if (claimed.kind === "held") {
+            if (!claimed.lease.executionRef) {
+                return { kind: "lease-held", lease: claimed.lease };
+            }
+            observeOnly = true;
+        }
+    }
+    const planHash = createHash("sha256")
+        .update(JSON.stringify(plan))
+        .digest("hex");
+    const continuationPath = path.join(runDir, ".runtime", "frontend-recovery", `${trigger.requestId}.json`);
+    await writeJsonAtomic(continuationPath, {
+        schemaVersion: 1,
+        parentRunId: state.runId,
+        requestId: trigger.requestId,
+        failureSource: trigger.failureSource,
+        failureOwner: plan.failureOwner,
+        resetRootNodeId: plan.resetRootNodeId,
+        recoveryMode: trigger.recoveryMode,
+        planHash,
+        plan,
+    });
+    const operationInput = {
+        clientRequestId: `dag-recovery-${state.runId}`,
+        action: "dagRunTask",
+        actionParams: {
+            parentRunId: state.runId,
+            requestId: trigger.requestId,
+            failureSource: trigger.failureSource,
+            failureOwner: plan.failureOwner,
+            resetRootNodeId: plan.resetRootNodeId,
+            protocolFailureReason: trigger.protocolFailureReason,
+            recoveryMode: trigger.recoveryMode,
+            planHash,
+        },
+        cliArgs: [
+            "dag",
+            "rerun-task",
+            "--run-id",
+            state.runId,
+            "--reason",
+            trigger.protocolFailureReason ?? "frontend automatic recovery",
+            "--request-id",
+            trigger.requestId,
+            "--frontend-recovery",
+            "--json",
+        ],
+    };
+    let created;
+    if (effectiveLease.lease.executionRef) {
+        // Replay the exact operation bound by the durable lease. A missing record
+        // cannot be replaced with a new operation id without violating the fence.
+        const existing = await input.deps.store.get(effectiveLease.lease.executionRef);
+        if (!existing) {
+            throw new Error(`frontend recovery lease operation not found: ${effectiveLease.lease.executionRef}`);
+        }
+        if (existing.clientRequestId !== operationInput.clientRequestId ||
+            existing.action !== operationInput.action ||
+            existing.payloadHash !==
+                computeActionPayloadHash(operationInput.action, operationInput.actionParams)) {
+            throw new Error("frontend recovery lease execution identity drift: referenced operation payload does not match the recovery request");
+        }
+        created = { kind: "idempotent", operation: existing };
+    }
+    else {
+        created = await input.deps.store.create(operationInput);
+    }
+    if (created.kind !== "conflict" && !observeOnly) {
+        // The durable lease must name the operation before controller execution.
+        // This also repairs a lease left behind by a crash before the original bind.
+        await bindDagRecoveryLeaseExecution({
+            leasePath: effectiveLease.leasePath,
+            lease: effectiveLease.lease,
+            executionRef: created.operation.operationId,
+        });
+        // Repair CLI args on both fresh and replayed operations. This closes the
+        // crash window after operation creation but before args were persisted.
+        await input.deps.store.update(created.operation.operationId, {
+            cliArgs: [
+                "dag",
+                "rerun-task",
+                "--run-id",
+                state.runId,
+                "--reason",
+                trigger.protocolFailureReason ?? "frontend automatic recovery",
+                "--request-id",
+                trigger.requestId,
+                "--frontend-recovery",
+                "--recovery-operation-id",
+                created.operation.operationId,
+                "--json",
+            ],
+        });
+    }
+    if (created.kind !== "conflict" && observeOnly) {
+        return { kind: "observed", operation: created.operation };
+    }
+    return created;
+}
+export const SUCCESSFUL_CONVERGENCE_TERMINAL_REASONS = new Set([
+    "review-pass",
+    "hard-verify-pass",
+]);

package/docs/architecture/dag-execution.md

@@ -55,7 +55,7 @@ src/commands/dag-validate.ts  runDagValidate
 | --- | --- | --- |
 | 拓扑排序 | `src/workflows/dag/topo.ts` `topoSortToRanks` | Kahn 算法，返回 `string[][]` ranks 并检测环 |
 | 单次 rank 执行 | `src/workflows/dag/scheduler.ts` `executeDagRanksOnce` | rank 间遍历、节点执行编排 |
-| run 主循环 | `src/workflows/dag/runner.ts` `runDag` / `executeDagCheckpoint` | 调用 scheduler + persistence + convergence |
+| run 主循环 | `src/workflows/dag/runner.ts` `runDag`；checkpoint 生命周期在 `src/workflows/dag/checkpoint.ts` `executeDagCheckpoint` | 调用 scheduler + persistence + convergence；终态判定在 `src/workflows/dag/terminal-status.ts` `finalizeTerminalRunStatus` |
 
 `scheduler.ts` `executeDagRanksOnce` 内每个 rank：
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tea-agent/loop-agent",
-  "version": "0.44.0-next.10",
+  "version": "0.44.0-next.11",
   "type": "module",
   "bin": {
     "loop-agent": "bin/loop-agent.js",