@tea-agent/loop-agent 0.28.1 → 0.28.2

package/dist/worker/scheduler/paths.js

@@ -0,0 +1,84 @@
+import path from "node:path";
+import { getTaskPoolRoot } from "../pool/run-store.js";
+import { SCHEDULER_ERROR_CODES, SchedulerError } from "./types.js";
+export const SCHEDULER_RELATIVE_ROOT = ".harness/task-pool/scheduler";
+const SAFE_SEGMENT = /^[A-Za-z0-9][A-Za-z0-9._-]*$/;
+export function assertSafeSchedulerId(value, label) {
+    if (typeof value !== "string" ||
+        value.length === 0 ||
+        !SAFE_SEGMENT.test(value)) {
+        throw new SchedulerError(SCHEDULER_ERROR_CODES.UNSAFE_ID, `${label} contains unsafe path characters: ${JSON.stringify(value)}`);
+    }
+}
+export function getSchedulerRoot(controlRepoRoot) {
+    return path.join(getTaskPoolRoot(controlRepoRoot), "scheduler");
+}
+export function getSchedulesDir(controlRepoRoot) {
+    return path.join(getSchedulerRoot(controlRepoRoot), "schedules");
+}
+export function getAdmissionsDir(controlRepoRoot) {
+    return path.join(getSchedulerRoot(controlRepoRoot), "admissions");
+}
+export function getExecutionsDir(controlRepoRoot) {
+    return path.join(getSchedulerRoot(controlRepoRoot), "executions");
+}
+export function getTransactionsDir(controlRepoRoot) {
+    return path.join(getSchedulerRoot(controlRepoRoot), "transactions");
+}
+export function getLeasesDir(controlRepoRoot) {
+    return path.join(getSchedulerRoot(controlRepoRoot), "leases");
+}
+export function getEvidenceDir(controlRepoRoot) {
+    return path.join(getSchedulerRoot(controlRepoRoot), "evidence");
+}
+export function getReportsDir(controlRepoRoot) {
+    return path.join(getSchedulerRoot(controlRepoRoot), "reports");
+}
+export function getLedgerPath(controlRepoRoot) {
+    return path.join(getSchedulerRoot(controlRepoRoot), "ledger.jsonl");
+}
+export function getMutationLockPath(controlRepoRoot) {
+    return path.join(getSchedulerRoot(controlRepoRoot), "mutation.lock");
+}
+export function getSchedulePath(controlRepoRoot, scheduleId) {
+    assertSafeSchedulerId(scheduleId, "scheduleId");
+    const schedulePath = path.join(getSchedulesDir(controlRepoRoot), `${scheduleId}.json`);
+    assertWithinSchedulerRoot(controlRepoRoot, schedulePath);
+    return schedulePath;
+}
+export function getAdmissionPath(controlRepoRoot, scheduleId) {
+    assertSafeSchedulerId(scheduleId, "scheduleId");
+    const admissionPath = path.join(getAdmissionsDir(controlRepoRoot), `${scheduleId}.json`);
+    assertWithinSchedulerRoot(controlRepoRoot, admissionPath);
+    return admissionPath;
+}
+export function getExecutionPath(controlRepoRoot, executionId) {
+    assertSafeSchedulerId(executionId, "executionId");
+    const executionPath = path.join(getExecutionsDir(controlRepoRoot), `${executionId}.json`);
+    assertWithinSchedulerRoot(controlRepoRoot, executionPath);
+    return executionPath;
+}
+export function getTransactionPath(controlRepoRoot, transitionId) {
+    assertSafeSchedulerId(transitionId, "transitionId");
+    const transactionPath = path.join(getTransactionsDir(controlRepoRoot), `${transitionId}.json`);
+    assertWithinSchedulerRoot(controlRepoRoot, transactionPath);
+    return transactionPath;
+}
+export function getLeasePath(controlRepoRoot, scheduleId) {
+    assertSafeSchedulerId(scheduleId, "scheduleId");
+    const leasePath = path.join(getLeasesDir(controlRepoRoot), `${scheduleId}.json`);
+    assertWithinSchedulerRoot(controlRepoRoot, leasePath);
+    return leasePath;
+}
+export function relativeSchedulerPath(controlRepoRoot, absolutePath) {
+    const relative = path.relative(path.resolve(controlRepoRoot), absolutePath);
+    return relative.split(path.sep).join("/");
+}
+function assertWithinSchedulerRoot(controlRepoRoot, candidatePath) {
+    const root = path.resolve(getSchedulerRoot(controlRepoRoot));
+    const resolved = path.resolve(candidatePath);
+    const relative = path.relative(root, resolved);
+    if (relative.startsWith("..") || path.isAbsolute(relative)) {
+        throw new SchedulerError(SCHEDULER_ERROR_CODES.UNSAFE_ID, `Scheduler path escapes scheduler root: ${candidatePath}`);
+    }
+}

package/dist/worker/scheduler/prepared-attempt-recovery.js

@@ -0,0 +1,471 @@
+import { spawn } from "node:child_process";
+import { readdir, readFile, stat } from "node:fs/promises";
+import path from "node:path";
+import { revalidateNightWorkspace } from "./workspace-adapter.js";
+const VERIFY_NODE_ID_RE = /(^|[-_])(hard-)?(soft-)?(re)?verify([-_]|$)|shell-verify|final-verify|frontend-.*verify|backend-.*verify|qa-.*verify/i;
+const WRITERISH_NODE_ID_RE = /(implement|repair|writer|codegen|scaffold|edit-pi|frontend-implement|backend-implement)/i;
+/**
+ * Pure: extract a candidate failed node id from attempt / report primaryFailure.
+ */
+export function extractFailedNodeId(attempt, extraPrimaryFailure) {
+    const primary = attempt?.result?.reportDecision?.primaryFailure ??
+        extraPrimaryFailure ??
+        attempt?.result
+            ?.primaryFailure;
+    if (primary && typeof primary === "object" && !Array.isArray(primary)) {
+        const rec = primary;
+        for (const key of ["nodeId", "id", "fromNodeId", "taskId"]) {
+            const value = rec[key];
+            if (typeof value === "string" && value.trim())
+                return value.trim();
+        }
+    }
+    const message = attempt?.error ?? attempt?.run?.error;
+    if (typeof message === "string") {
+        const m = /node[:\s]+([A-Za-z0-9._-]+)/i.exec(message);
+        if (m?.[1])
+            return m[1];
+    }
+    return undefined;
+}
+/**
+ * Scan a completed/active dag-run directory for the first ERROR verify-safe node.
+ */
+export async function extractFailedVerifyNodeFromDagRun(workspaceRoot, dagRunId) {
+    const candidates = [
+        path.join(workspaceRoot, ".harness", "dag-runs", "completed", dagRunId, "state.json"),
+        path.join(workspaceRoot, ".harness", "dag-runs", "completed", dagRunId, "run.json"),
+        path.join(workspaceRoot, ".harness", "dag-runs", "active", dagRunId, "state.json"),
+    ];
+    for (const filePath of candidates) {
+        try {
+            const raw = JSON.parse(await readFile(filePath, "utf8"));
+            const fromPrimary = extractFailedNodeId(undefined, raw.primaryFailure);
+            if (fromPrimary && isVerifySafeNodeId(fromPrimary))
+                return fromPrimary;
+            const nodes = Array.isArray(raw.nodes) ? raw.nodes : [];
+            for (const node of nodes) {
+                const status = typeof node.status === "string" ? node.status : "";
+                const id = (typeof node.nodeId === "string" && node.nodeId) ||
+                    (typeof node.id === "string" && node.id) ||
+                    "";
+                const failureCategory = typeof node.failureCategory === "string" ? node.failureCategory : "";
+                if (id &&
+                    isVerifySafeNodeId(id) &&
+                    (status === "ERROR" ||
+                        (failureCategory && failureCategory !== "success"))) {
+                    return id;
+                }
+            }
+        }
+        catch {
+            // try next
+        }
+    }
+    return undefined;
+}
+/**
+ * Pure heuristic: is this node id a verify/shell-style node that is safe to
+ * prefer for FlakyTest verify-only restart (not a business writer)?
+ */
+export function isVerifySafeNodeId(nodeId) {
+    const id = nodeId.trim();
+    if (!id)
+        return false;
+    if (WRITERISH_NODE_ID_RE.test(id))
+        return false;
+    return VERIFY_NODE_ID_RE.test(id);
+}
+/**
+ * Assess whether a terminal failed night attempt may re-enter waiting (full
+ * prepared re-attempt) or should try verify-only / stop.
+ *
+ * Does not re-generate DAG or expand writeSet. EnvFailure may full-reattempt
+ * only when admission/worktree are still bound. FlakyTest prefers verify-only.
+ */
+export async function assessPreparedAttemptRecovery(input) {
+    const category = input.failureCategory ?? "Unknown";
+    const reasons = [];
+    const terminationUnconfirmed = input.terminationUnconfirmed === true;
+    const maxAutoAttempts = input.schedule.policySnapshot?.maxAutoAttempts ?? 1;
+    if (terminationUnconfirmed) {
+        return {
+            ok: false,
+            mode: "stop",
+            category,
+            reasons: ["termination-unconfirmed"],
+            terminationUnconfirmed: true,
+            admissionReusable: false,
+        };
+    }
+    if (input.schedule.status !== "failed" &&
+        input.schedule.status !== "running") {
+        // running is allowed when assessing mid-dispatch before transition
+        if (input.schedule.status !== "dispatching") {
+            reasons.push(`status-not-retryable:${input.schedule.status}`);
+        }
+    }
+    if (input.attemptCount >= maxAutoAttempts) {
+        return {
+            ok: false,
+            mode: "stop",
+            category,
+            reasons: [
+                ...reasons,
+                `budget-exhausted:${input.attemptCount}/${maxAutoAttempts}`,
+            ],
+            terminationUnconfirmed: false,
+            admissionReusable: false,
+        };
+    }
+    const admission = input.admission;
+    if (!admission?.workspace ||
+        !admission.gateApproval?.token ||
+        !admission.dag) {
+        return {
+            ok: false,
+            mode: "stop",
+            category,
+            reasons: [...reasons, "admission-incomplete"],
+            terminationUnconfirmed: false,
+            admissionReusable: false,
+        };
+    }
+    let admissionReusable = true;
+    const revalidate = input.revalidateWorkspace ?? revalidateNightWorkspace;
+    try {
+        await revalidate({
+            controlRepoRoot: input.controlRepoRoot,
+            scheduleId: input.schedule.id,
+            worktreePath: admission.workspace.path,
+            branch: admission.workspace.branch,
+            baseCommit: admission.workspace.baseCommit,
+            ownershipToken: admission.workspace.ownershipToken,
+        });
+    }
+    catch (error) {
+        admissionReusable = false;
+        reasons.push(`worktree-not-reusable:${error instanceof Error ? error.message : String(error)}`);
+    }
+    // Fail-closed categories never auto.
+    if (category === "ProductBug" ||
+        category === "TestBug" ||
+        category === "SpecUnclear" ||
+        category === "ContractMismatch" ||
+        category === "RiskyChange" ||
+        category === "NeedsHuman" ||
+        category === "Unknown" ||
+        category === "DependencyFailure") {
+        return {
+            ok: false,
+            mode: "stop",
+            category,
+            reasons: [...reasons, `category-not-retryable:${category}`],
+            terminationUnconfirmed: false,
+            admissionReusable,
+        };
+    }
+    if (category === "FlakyTest") {
+        if (!admissionReusable) {
+            return {
+                ok: false,
+                mode: "stop",
+                category,
+                reasons: [...reasons, "admission-not-reusable-for-verify-only"],
+                terminationUnconfirmed: false,
+                admissionReusable: false,
+            };
+        }
+        const parentDagRunId = input.lastExecution?.dagRunId ??
+            (input.workspaceRoot
+                ? await resolveLatestCompletedDagRunId(input.workspaceRoot, admission.harnessTaskId)
+                : undefined);
+        let fromNodeId = extractFailedNodeId(input.attempt);
+        if ((!fromNodeId || !isVerifySafeNodeId(fromNodeId)) &&
+            parentDagRunId &&
+            input.workspaceRoot) {
+            fromNodeId =
+                (await extractFailedVerifyNodeFromDagRun(input.workspaceRoot, parentDagRunId)) ?? fromNodeId;
+        }
+        if (!fromNodeId || !isVerifySafeNodeId(fromNodeId)) {
+            return {
+                ok: false,
+                mode: "stop",
+                category,
+                reasons: [
+                    ...reasons,
+                    fromNodeId
+                        ? `flaky-node-not-verify-safe:${fromNodeId}`
+                        : "flaky-node-unresolved",
+                ],
+                terminationUnconfirmed: false,
+                admissionReusable,
+                ...(fromNodeId ? { fromNodeId } : {}),
+                ...(parentDagRunId ? { parentDagRunId } : {}),
+            };
+        }
+        if (!parentDagRunId) {
+            return {
+                ok: false,
+                mode: "stop",
+                category,
+                reasons: [...reasons, "parent-dag-run-unresolved"],
+                fromNodeId,
+                terminationUnconfirmed: false,
+                admissionReusable,
+            };
+        }
+        return {
+            ok: true,
+            mode: "verify-only-rerun",
+            category,
+            reasons: [...reasons, "flaky-verify-only-candidate"],
+            fromNodeId,
+            parentDagRunId,
+            terminationUnconfirmed: false,
+            admissionReusable,
+        };
+    }
+    if (category === "EnvFailure") {
+        if (!admissionReusable) {
+            return {
+                ok: false,
+                mode: "stop",
+                category,
+                reasons: [...reasons, "admission-not-reusable"],
+                terminationUnconfirmed: false,
+                admissionReusable: false,
+            };
+        }
+        if (!input.workspaceRoot) {
+            return {
+                ok: false,
+                mode: "stop",
+                category,
+                reasons: [...reasons, "workspace-root-required-for-env-retry"],
+                terminationUnconfirmed: false,
+                admissionReusable: false,
+            };
+        }
+        try {
+            const inspect = input.inspectWorkspaceChanges ?? inspectGitWorkspaceChanges;
+            const changes = await inspect(input.workspaceRoot);
+            if (changes.length > 0) {
+                return {
+                    ok: false,
+                    mode: "stop",
+                    category,
+                    reasons: [
+                        ...reasons,
+                        `writer-side-effects-present:${changes.slice(0, 5).join(",")}`,
+                    ],
+                    terminationUnconfirmed: false,
+                    admissionReusable: false,
+                };
+            }
+        }
+        catch (error) {
+            return {
+                ok: false,
+                mode: "stop",
+                category,
+                reasons: [
+                    ...reasons,
+                    `workspace-status-unavailable:${error instanceof Error ? error.message : String(error)}`,
+                ],
+                terminationUnconfirmed: false,
+                admissionReusable: false,
+            };
+        }
+        return {
+            ok: true,
+            mode: "full-prepared-reattempt",
+            category,
+            reasons: [...reasons, "env-failure-prepared-reattempt"],
+            terminationUnconfirmed: false,
+            admissionReusable: true,
+        };
+    }
+    return {
+        ok: false,
+        mode: "stop",
+        category,
+        reasons: [...reasons, `category-not-retryable:${category}`],
+        terminationUnconfirmed: false,
+        admissionReusable,
+    };
+}
+/**
+ * Plan + execute a verify-only dag rerun via published loop-agent CLI
+ * (cwd = workspace). Refuses when plan is ineligible or contains writers.
+ */
+async function inspectGitWorkspaceChanges(workspaceRoot) {
+    return new Promise((resolve, reject) => {
+        const child = spawn("git", ["-C", workspaceRoot, "status", "--porcelain=v1", "--untracked-files=all"], { stdio: ["ignore", "pipe", "pipe"] });
+        const stdout = [];
+        const stderr = [];
+        child.stdout?.on("data", (chunk) => stdout.push(chunk));
+        child.stderr?.on("data", (chunk) => stderr.push(chunk));
+        child.on("error", reject);
+        child.on("close", (code) => {
+            if (code !== 0) {
+                reject(new Error(`git status failed (exit ${code ?? "null"}): ${Buffer.concat(stderr).toString("utf-8").trim()}`));
+                return;
+            }
+            resolve(Buffer.concat(stdout)
+                .toString("utf-8")
+                .split(/\r?\n/)
+                .map((line) => line.trim())
+                .filter(Boolean));
+        });
+    });
+}
+export async function runVerifyOnlyRerun(input) {
+    const planArgs = [
+        "dag",
+        "rerun",
+        "--run-id",
+        input.parentDagRunId,
+        "--from-node",
+        input.fromNodeId,
+        "--plan",
+        "--json",
+    ];
+    const planResult = await input.client.run(planArgs, {
+        cwd: input.workspaceRoot,
+        artifactName: `night-verify-plan-${input.requestId}`,
+        expectJson: true,
+        timeoutMs: 120_000,
+    });
+    const planJson = asRecord(planResult.json);
+    const plan = asRecord(planJson?.plan) ?? planJson;
+    const eligible = plan?.eligible === true || planJson?.ok === true;
+    const reasonCodes = asStringArray(plan?.reasonCodes);
+    const blockedReasons = asStringArray(plan?.blockedReasons);
+    const planHash = typeof plan?.planHash === "string" ? plan.planHash : undefined;
+    const risk = typeof plan?.risk === "string" ? plan.risk : undefined;
+    if (!eligible || !planHash) {
+        return {
+            ok: false,
+            reason: "verify-plan-ineligible",
+            detail: [...reasonCodes, ...blockedReasons].join(",") || "no planHash",
+            parentDagRunId: input.parentDagRunId,
+            fromNodeId: input.fromNodeId,
+        };
+    }
+    if (reasonCodes.includes("restart-subgraph-contains-writer") ||
+        risk === "high") {
+        return {
+            ok: false,
+            reason: "verify-plan-contains-writer-or-high-risk",
+            detail: reasonCodes.join(",") || risk,
+            parentDagRunId: input.parentDagRunId,
+            fromNodeId: input.fromNodeId,
+        };
+    }
+    const execArgs = [
+        "dag",
+        "rerun",
+        "--run-id",
+        input.parentDagRunId,
+        "--from-node",
+        input.fromNodeId,
+        "--plan-hash",
+        planHash,
+        "--request-id",
+        input.requestId,
+        "--reason",
+        input.reason ?? "night-scheduler-flakytest-verify-only",
+        "--json",
+    ];
+    const execResult = await input.client.run(execArgs, {
+        cwd: input.workspaceRoot,
+        artifactName: `night-verify-exec-${input.requestId}`,
+        expectJson: true,
+        // Verify-only may still take a while; allow long wall clock.
+        timeoutMs: 0,
+    });
+    if (!execResult.ok && execResult.exitCode !== 0) {
+        return {
+            ok: false,
+            reason: "verify-exec-failed",
+            detail: execResult.stderr?.slice(0, 500) ||
+                `exit ${execResult.exitCode ?? "null"}`,
+            parentDagRunId: input.parentDagRunId,
+            fromNodeId: input.fromNodeId,
+        };
+    }
+    const execJson = asRecord(execResult.json);
+    const childRunId = (typeof execJson?.runId === "string" && execJson.runId) ||
+        (typeof execJson?.childRunId === "string" && execJson.childRunId) ||
+        undefined;
+    return {
+        ok: true,
+        parentDagRunId: input.parentDagRunId,
+        fromNodeId: input.fromNodeId,
+        planHash,
+        ...(childRunId ? { childRunId } : {}),
+    };
+}
+/**
+ * Resolve newest completed dag-run id under workspace for a harness task.
+ */
+export async function resolveLatestCompletedDagRunId(workspaceRoot, harnessTaskId) {
+    if (!harnessTaskId)
+        return undefined;
+    const completedDir = path.join(workspaceRoot, ".harness", "dag-runs", "completed");
+    let entries = [];
+    try {
+        entries = await readdir(completedDir);
+    }
+    catch {
+        return undefined;
+    }
+    const matches = [];
+    for (const name of entries) {
+        if (!name || name === ".gitkeep")
+            continue;
+        const runDir = path.join(completedDir, name);
+        let mtimeMs = 0;
+        try {
+            mtimeMs = (await stat(runDir)).mtimeMs;
+        }
+        catch {
+            continue;
+        }
+        const taskId = await readTaskIdFromRunDir(runDir);
+        if (taskId === harnessTaskId) {
+            matches.push({ id: name, mtimeMs });
+        }
+    }
+    matches.sort((a, b) => b.mtimeMs - a.mtimeMs);
+    return matches[0]?.id;
+}
+async function readTaskIdFromRunDir(runDir) {
+    try {
+        const raw = JSON.parse(await readFile(path.join(runDir, "run.json"), "utf8"));
+        const fromBinding = raw.taskContractBinding?.taskId?.trim() ||
+            raw.sourceBinding?.taskId?.trim();
+        if (fromBinding)
+            return fromBinding;
+        const title = raw.title?.trim() ?? "";
+        const hybrid = /^Hybrid DAG:\s*(.+)$/i.exec(title);
+        if (hybrid?.[1]?.trim())
+            return hybrid[1].trim();
+    }
+    catch {
+        // ignore
+    }
+    return undefined;
+}
+function asRecord(value) {
+    if (!value || typeof value !== "object" || Array.isArray(value)) {
+        return undefined;
+    }
+    return value;
+}
+function asStringArray(value) {
+    if (!Array.isArray(value))
+        return [];
+    return value.filter((item) => typeof item === "string");
+}