@tea-agent/loop-agent 0.1.0-fe-test.0

package/dist/workflows/dag/rerun-run.js

@@ -0,0 +1,556 @@
+import { createHash } from "node:crypto";
+import { access, copyFile, mkdir, readdir, readFile } from "node:fs/promises";
+import path from "node:path";
+import { writeJsonAtomic } from "../../infrastructure/harness/atomic-write.js";
+import { resolveRunningControllerIdentity } from "../../shared/package-metadata.js";
+import { readControllerIdentityArtifact, captureControllerIdentity } from "./controller-identity.js";
+import { DAG_RUNS_DIR, getDagRunDir, isDagNonExecutionLifecycle, locateDagRun, readDagRunSpec, readDagRunState, } from "./lifecycle.js";
+import { buildDagRunId, runDagContinuation, } from "./runner.js";
+import { computePlanHash, evaluateDagRerunPlan, evaluateLiveDagRerunBindings, } from "./rerun-plan.js";
+import { prepareActiveRunDir, writeNodeRecord, writeRunSpec, writeRunState, } from "./run-store.js";
+import { cloneParentSkillSnapshotForContinuation, readSkillSnapshot, } from "./skill-snapshot.js";
+import { topoSortToRanks } from "./topo.js";
+import { captureWorkspaceCheckpoint, readWorkspaceCheckpoint, WORKSPACE_CHECKPOINT_START_REL, WORKSPACE_CHECKPOINT_TERMINAL_REL, writeWorkspaceCheckpoint, } from "./workspace-checkpoint.js";
+export const RERUN_PLAN_REL_PATH = ".runtime/rerun-plan.json";
+export const IMPORT_MANIFEST_REL_PATH = ".runtime/import-manifest.json";
+export const WORKER_ASSOCIATION_REL_PATH = ".runtime/worker-association.json";
+function sha256Hex(content) {
+    return createHash("sha256").update(content).digest("hex");
+}
+async function fileExists(filePath) {
+    try {
+        await access(filePath);
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+function operatorRequestPath(cwd, requestId) {
+    return path.join(cwd, DAG_RUNS_DIR, ".operator-requests", `${requestId}.json`);
+}
+async function readExistingOperatorRequest(cwd, requestId) {
+    try {
+        const raw = JSON.parse(await readFile(operatorRequestPath(cwd, requestId), "utf-8"));
+        if (raw.result && typeof raw.result === "object") {
+            return { ...raw.result, idempotentReplay: true };
+        }
+    }
+    catch {
+        // not found or unreadable
+    }
+    return undefined;
+}
+async function persistOperatorRequest(cwd, result, requestId) {
+    await writeJsonAtomic(operatorRequestPath(cwd, requestId), {
+        schemaVersion: 1,
+        kind: "dag-rerun-from-node-request",
+        requestId,
+        parentRunId: result.plan.parentRunId,
+        newRunId: result.newRunId,
+        createdAt: new Date().toISOString(),
+        result,
+    }, { repoRoot: cwd });
+}
+function readOptionalString(record, key) {
+    const value = record[key];
+    return typeof value === "string" && value.trim() ? value.trim() : undefined;
+}
+async function detectWorkerManagedRun(parentRunDir, parentState, parentSpec) {
+    const association = parentState.workerAssociation;
+    if (association?.workerRunId) {
+        return {
+            workerManaged: true,
+            featureId: association.featureId,
+            taskId: association.taskId,
+            workerRunId: association.workerRunId,
+        };
+    }
+    const stateRecord = parentState;
+    const specRecord = parentSpec;
+    const workerRunId = readOptionalString(stateRecord, "workerRunId")
+        ?? readOptionalString(specRecord, "workerRunId");
+    const featureId = readOptionalString(stateRecord, "featureId")
+        ?? readOptionalString(specRecord, "featureId");
+    const taskId = readOptionalString(stateRecord, "taskId")
+        ?? readOptionalString(specRecord, "taskId")
+        ?? parentSpec.taskContractBinding?.taskId
+        ?? parentSpec.sourceBinding?.taskId;
+    if (workerRunId) {
+        return {
+            workerManaged: true,
+            ...(featureId ? { featureId } : {}),
+            ...(taskId ? { taskId } : {}),
+            workerRunId,
+        };
+    }
+    if (await fileExists(path.join(parentRunDir, WORKER_ASSOCIATION_REL_PATH))) {
+        return {
+            workerManaged: true,
+            ...(featureId ? { featureId } : {}),
+            ...(taskId ? { taskId } : {}),
+        };
+    }
+    return { workerManaged: false };
+}
+async function resolveControllerFingerprints(parentRunDir) {
+    const current = resolveRunningControllerIdentity();
+    const parentArtifact = await readControllerIdentityArtifact(parentRunDir);
+    return {
+        currentControllerFingerprint: current?.packageFingerprint.value,
+        parentControllerFingerprint: parentArtifact?.identity?.packageFingerprint.value,
+    };
+}
+async function verifyParentSkillSnapshot(parentRunDir, parentState, parentRunId) {
+    if (!parentState.skillSnapshotRef)
+        return false;
+    try {
+        await readSkillSnapshot(parentRunDir, parentState.skillSnapshotRef, {
+            expectedRunId: parentRunId,
+        });
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+export async function planDagRerun(input) {
+    const located = await locateDagRun(input.cwd, input.parentRunId);
+    if (!located) {
+        throw new Error(`dag run not found: ${input.parentRunId}`);
+    }
+    if (located.lifecycle !== "completed") {
+        const nonCompletedSpec = await readDagRunSpec(located.runDir);
+        const nonCompletedState = await readDagRunState(located.runDir);
+        const nonCompletedWorkerHints = await detectWorkerManagedRun(located.runDir, nonCompletedState, nonCompletedSpec);
+        // evaluateDagRerunPlan only accepts execution lifecycles. Audit parents must
+        // not be cast to "active" (reconcile) or "paused" (resume). Also avoid routing
+        // audit through evaluate: missing workspace checkpoint would surface as
+        // standalone-task-rerun instead of the required fail-closed manual action.
+        if (isDagNonExecutionLifecycle(located.lifecycle)) {
+            const bindingStatus = await evaluateLiveDagRerunBindings({
+                cwd: input.cwd,
+                parentSpec: nonCompletedSpec,
+            });
+            const workerAssociation = nonCompletedWorkerHints.workerManaged
+                ? {
+                    kind: "worker-managed",
+                    ...(nonCompletedWorkerHints.featureId
+                        ? { featureId: nonCompletedWorkerHints.featureId }
+                        : {}),
+                    ...(nonCompletedWorkerHints.taskId
+                        ? { taskId: nonCompletedWorkerHints.taskId }
+                        : {}),
+                    ...(nonCompletedWorkerHints.workerRunId
+                        ? { workerRunId: nonCompletedWorkerHints.workerRunId }
+                        : {}),
+                }
+                : { kind: "standalone" };
+            const planWithoutHash = {
+                schemaVersion: 1,
+                eligible: false,
+                parentRunId: input.parentRunId,
+                parentLifecycle: located.lifecycle,
+                selectedNodeId: input.selectedNodeId,
+                rewriteApplied: false,
+                resetNodeIds: [],
+                importedNodeIds: [],
+                blockingNodes: [],
+                reasonCodes: ["parent-lifecycle-ineligible"],
+                blockedReasons: ["parent-lifecycle-ineligible"],
+                risk: "high",
+                bindings: {
+                    sourceBindingMatched: bindingStatus.sourceBindingMatched,
+                    taskContractBindingMatched: bindingStatus.taskContractBindingMatched,
+                    controllerIdentityMatched: true,
+                    skillSnapshotVerified: true,
+                    workspaceMatched: false,
+                },
+                workerAssociation,
+                estimatedExecution: {
+                    nodeCount: 0,
+                    piCallCount: 0,
+                    shellNodeCount: 0,
+                    avoidedNodeCount: 0,
+                    avoidedPiCallCount: 0,
+                },
+                suggestedAction: "manual",
+            };
+            return {
+                ...planWithoutHash,
+                planHash: computePlanHash(planWithoutHash),
+            };
+        }
+        const parentLifecycle = located.lifecycle === "active" ? "active" : "paused";
+        return evaluateDagRerunPlan({
+            cwd: input.cwd,
+            parentRunId: input.parentRunId,
+            selectedNodeId: input.selectedNodeId,
+            parentRunDir: located.runDir,
+            parentSpec: nonCompletedSpec,
+            parentState: nonCompletedState,
+            parentLifecycle,
+            workerManaged: nonCompletedWorkerHints.workerManaged,
+            workerAssociation: nonCompletedWorkerHints.workerManaged
+                ? {
+                    kind: "worker-managed",
+                    ...(nonCompletedWorkerHints.featureId ? { featureId: nonCompletedWorkerHints.featureId } : {}),
+                    ...(nonCompletedWorkerHints.taskId ? { taskId: nonCompletedWorkerHints.taskId } : {}),
+                    ...(nonCompletedWorkerHints.workerRunId ? { workerRunId: nonCompletedWorkerHints.workerRunId } : {}),
+                }
+                : { kind: "standalone" },
+            bindingStatus: await evaluateLiveDagRerunBindings({
+                cwd: input.cwd,
+                parentSpec: nonCompletedSpec,
+            }),
+        });
+    }
+    const parentSpec = await readDagRunSpec(located.runDir);
+    const parentState = await readDagRunState(located.runDir);
+    const currentWorkspace = await captureWorkspaceCheckpoint(input.cwd).catch(() => undefined);
+    const parentTerminalWorkspace = await readWorkspaceCheckpoint(located.runDir, WORKSPACE_CHECKPOINT_TERMINAL_REL);
+    const fingerprints = await resolveControllerFingerprints(located.runDir);
+    const skillSnapshotOk = await verifyParentSkillSnapshot(located.runDir, parentState, input.parentRunId);
+    const workerHints = await detectWorkerManagedRun(located.runDir, parentState, parentSpec);
+    const bindingStatus = await evaluateLiveDagRerunBindings({
+        cwd: input.cwd,
+        parentSpec,
+    });
+    return evaluateDagRerunPlan({
+        cwd: input.cwd,
+        parentRunId: input.parentRunId,
+        selectedNodeId: input.selectedNodeId,
+        parentRunDir: located.runDir,
+        parentSpec,
+        parentState,
+        parentLifecycle: "completed",
+        currentControllerFingerprint: fingerprints.currentControllerFingerprint,
+        parentControllerFingerprint: fingerprints.parentControllerFingerprint,
+        parentTerminalWorkspace,
+        currentWorkspace,
+        skillSnapshotOk,
+        workerManaged: workerHints.workerManaged,
+        workerAssociation: workerHints.workerManaged
+            ? {
+                kind: "worker-managed",
+                ...(workerHints.featureId ? { featureId: workerHints.featureId } : {}),
+                ...(workerHints.taskId ? { taskId: workerHints.taskId } : {}),
+                ...(workerHints.workerRunId ? { workerRunId: workerHints.workerRunId } : {}),
+            }
+            : { kind: "standalone" },
+        bindingStatus,
+    });
+}
+function normalizeRunRelativePath(relPath) {
+    return relPath.replace(/\\/g, "/");
+}
+function assertArtifactPathWithinParentRun(parentRunDir, artifactPath) {
+    if (!path.isAbsolute(artifactPath)) {
+        return normalizeRunRelativePath(artifactPath);
+    }
+    const resolved = path.resolve(artifactPath);
+    const parentResolved = path.resolve(parentRunDir);
+    if (resolved !== parentResolved
+        && !resolved.startsWith(`${parentResolved}${path.sep}`)) {
+        throw new Error(`artifact path outside parent run directory: ${artifactPath}`);
+    }
+    return normalizeRunRelativePath(path.relative(parentResolved, resolved));
+}
+function collectNodeArtifactRelativePaths(record) {
+    const paths = new Set([`${record.id}.json`]);
+    const add = (value) => {
+        if (value?.trim())
+            paths.add(normalizeRunRelativePath(value.trim()));
+    };
+    add(record.stdoutArtifactPath);
+    add(record.assistantArtifactPath);
+    add(record.structuredArtifactPath);
+    add(record.nodeRecordPath);
+    add(record.escalationArtifactPath);
+    add(record.humanApprovalArtifactPath);
+    add(record.humanRejectionArtifactPath);
+    for (const attempt of record.attempts ?? []) {
+        add(attempt.artifactPath);
+    }
+    return [...paths];
+}
+async function listNodeDirectoryArtifacts(parentRunDir, nodeId) {
+    const nodeDir = path.join(parentRunDir, nodeId);
+    if (!(await fileExists(nodeDir)))
+        return [];
+    const entries = await readdir(nodeDir, { withFileTypes: true });
+    const artifacts = [];
+    for (const entry of entries) {
+        if (!entry.isFile())
+            continue;
+        artifacts.push(normalizeRunRelativePath(path.join(nodeId, entry.name)));
+    }
+    return artifacts;
+}
+async function copyArtifactIntoRun(input) {
+    const normalized = assertArtifactPathWithinParentRun(input.parentRunDir, input.relativePath);
+    const sourcePath = path.join(input.parentRunDir, ...normalized.split("/"));
+    if (!(await fileExists(sourcePath))) {
+        throw new Error(`missing imported artifact: ${normalized}`);
+    }
+    const destinationPath = path.join(input.newRunDir, ...normalized.split("/"));
+    await mkdir(path.dirname(destinationPath), { recursive: true });
+    await copyFile(sourcePath, destinationPath);
+    const bytes = await readFile(destinationPath);
+    return {
+        destinationRelativePath: normalized,
+        sha256: sha256Hex(bytes),
+    };
+}
+async function importNodeFacts(input) {
+    const nodeJsonRel = `${input.nodeId}.json`;
+    const parentNodeJsonPath = path.join(input.parentRunDir, nodeJsonRel);
+    const sourceNodeRecordSha256 = (await fileExists(parentNodeJsonPath))
+        ? sha256Hex(await readFile(parentNodeJsonPath))
+        : sha256Hex(`${JSON.stringify(input.parentRecord, null, 2)}\n`);
+    const importedRecord = buildImportedNodeRecord(input.parentRecord, input.parentRunId, sourceNodeRecordSha256, input.parentRunDir, input.newRunDir);
+    await writeNodeRecord(input.newRunDir, input.nodeId, importedRecord);
+    const relativePaths = [
+        ...new Set([
+            ...collectNodeArtifactRelativePaths(input.parentRecord).filter((relativePath) => relativePath !== nodeJsonRel),
+            ...(await listNodeDirectoryArtifacts(input.parentRunDir, input.nodeId)),
+        ]),
+    ];
+    const importedArtifacts = [
+        {
+            sourceRelativePath: nodeJsonRel,
+            destinationRelativePath: nodeJsonRel,
+            sha256: sha256Hex(await readFile(path.join(input.newRunDir, nodeJsonRel))),
+        },
+    ];
+    for (const relativePath of relativePaths.sort()) {
+        const copied = await copyArtifactIntoRun({
+            parentRunDir: input.parentRunDir,
+            newRunDir: input.newRunDir,
+            relativePath,
+        });
+        importedArtifacts.push({
+            sourceRelativePath: relativePath,
+            destinationRelativePath: copied.destinationRelativePath,
+            sha256: copied.sha256,
+        });
+    }
+    return {
+        manifestNode: {
+            nodeId: input.nodeId,
+            sourceNodeRecordSha256,
+            importedArtifacts,
+        },
+        importedRecord,
+    };
+}
+function normalizeImportedArtifactPath(parentRunDir, newRunDir, artifactPath) {
+    if (!artifactPath?.trim())
+        return artifactPath;
+    const relative = assertArtifactPathWithinParentRun(parentRunDir, artifactPath);
+    return path.join(newRunDir, ...relative.split("/"));
+}
+function buildImportedNodeRecord(parentRecord, parentRunId, sourceNodeRecordSha256, parentRunDir, newRunDir) {
+    const { origin: _origin, ...rest } = parentRecord;
+    void _origin;
+    const imported = structuredClone(rest);
+    imported.nodeRecordPath = path.join(newRunDir, `${parentRecord.id}.json`);
+    for (const key of [
+        "stdoutArtifactPath",
+        "assistantArtifactPath",
+        "structuredArtifactPath",
+        "escalationArtifactPath",
+        "humanApprovalArtifactPath",
+        "humanRejectionArtifactPath",
+    ]) {
+        imported[key] = normalizeImportedArtifactPath(parentRunDir, newRunDir, imported[key]);
+    }
+    return {
+        ...imported,
+        origin: {
+            kind: "imported",
+            parentRunId,
+            sourceNodeRecordSha256,
+        },
+    };
+}
+function buildPendingNodeRecord(task) {
+    return {
+        id: task.id,
+        status: "PENDING",
+        executor: task.executor,
+        complexity: task.complexity,
+        origin: { kind: "executed" },
+        ...(task.outputMode ? { outputMode: task.outputMode } : {}),
+    };
+}
+async function stageContinuationRun(input) {
+    const { ranks } = topoSortToRanks(input.parentSpec);
+    const runId = await buildDagRunId(input.parentSpec, { cwd: input.cwd });
+    const runDir = getDagRunDir(input.cwd, "active", runId);
+    await prepareActiveRunDir(runDir);
+    await writeRunSpec(runDir, input.parentSpec);
+    const resetSet = new Set(input.plan.resetNodeIds);
+    const importedSet = new Set(input.plan.importedNodeIds);
+    const tasksById = new Map(input.parentSpec.tasks.map((task) => [task.id, task]));
+    const importManifestNodes = [];
+    const nodes = {};
+    for (const nodeId of input.plan.importedNodeIds) {
+        const parentRecord = input.parentState.nodes[nodeId];
+        if (!parentRecord) {
+            throw new Error(`parent node record missing for imported node: ${nodeId}`);
+        }
+        const { manifestNode, importedRecord } = await importNodeFacts({
+            parentRunDir: input.parentRunDir,
+            newRunDir: runDir,
+            parentRunId: input.parentRunId,
+            nodeId,
+            parentRecord,
+        });
+        importManifestNodes.push(manifestNode);
+        nodes[nodeId] = importedRecord;
+    }
+    for (const nodeId of input.plan.resetNodeIds) {
+        const task = tasksById.get(nodeId);
+        if (!task) {
+            throw new Error(`reset node missing from spec: ${nodeId}`);
+        }
+        nodes[nodeId] = buildPendingNodeRecord(task);
+    }
+    for (const task of input.parentSpec.tasks) {
+        if (!resetSet.has(task.id) && !importedSet.has(task.id)) {
+            throw new Error(`node ${task.id} missing from rerun plan partition`);
+        }
+    }
+    const createdAt = new Date().toISOString();
+    const continuation = {
+        schemaVersion: 1,
+        kind: "rerun-from-node",
+        parentRunId: input.parentRunId,
+        operatorSelectedNodeId: input.plan.selectedNodeId,
+        effectiveFromNodeId: input.plan.effectiveFromNodeId,
+        rewriteApplied: input.plan.rewriteApplied,
+        reason: input.reason,
+        requestId: input.requestId,
+        planHash: input.plan.planHash,
+        createdAt,
+    };
+    const state = {
+        version: 1,
+        title: input.parentSpec.title,
+        runId,
+        cwd: input.cwd,
+        startedAt: createdAt,
+        status: "running",
+        ranks,
+        nodes,
+        continuation,
+        ...(input.parentState.evaluation
+            ? { evaluation: structuredClone(input.parentState.evaluation) }
+            : {}),
+        ...(input.parentState.budget
+            ? { budget: structuredClone(input.parentState.budget) }
+            : {}),
+    };
+    if (!input.parentState.skillSnapshotRef) {
+        throw new Error("parent run is missing skill snapshot ref");
+    }
+    state.skillSnapshotRef = await cloneParentSkillSnapshotForContinuation({
+        parentRunDir: input.parentRunDir,
+        parentRef: input.parentState.skillSnapshotRef,
+        parentRunId: input.parentRunId,
+        newRunId: runId,
+        newRunDir: runDir,
+    });
+    state.controllerIdentityRef = await captureControllerIdentity({ runDir });
+    try {
+        const startCheckpoint = await captureWorkspaceCheckpoint(input.cwd);
+        await writeWorkspaceCheckpoint(runDir, WORKSPACE_CHECKPOINT_START_REL, startCheckpoint);
+    }
+    catch (error) {
+        console.warn(`[dag rerun] warning: failed to write workspace start checkpoint: ${error instanceof Error ? error.message : String(error)}`);
+    }
+    const importManifest = {
+        schemaVersion: 1,
+        parentRunId: input.parentRunId,
+        createdAt,
+        nodes: importManifestNodes,
+    };
+    await writeJsonAtomic(path.join(runDir, IMPORT_MANIFEST_REL_PATH), importManifest);
+    await writeJsonAtomic(path.join(runDir, RERUN_PLAN_REL_PATH), input.plan);
+    await writeRunState(runDir, state);
+    return { runId, runDir, state };
+}
+export async function executeDagRerun(input) {
+    const existing = await readExistingOperatorRequest(input.cwd, input.requestId);
+    if (existing) {
+        return existing;
+    }
+    const plan = await planDagRerun({
+        cwd: input.cwd,
+        parentRunId: input.parentRunId,
+        selectedNodeId: input.selectedNodeId,
+    });
+    if (plan.planHash !== input.planHash) {
+        return {
+            ok: false,
+            plan,
+            errorCode: "PLAN_DRIFT",
+            message: "plan hash mismatch; re-run plan before executing",
+        };
+    }
+    if (!plan.eligible) {
+        return {
+            ok: false,
+            plan,
+            errorCode: "INELIGIBLE",
+            message: plan.blockedReasons.join(", ") || "rerun plan is not eligible",
+        };
+    }
+    const located = await locateDagRun(input.cwd, input.parentRunId);
+    if (!located || located.lifecycle !== "completed") {
+        return {
+            ok: false,
+            plan,
+            errorCode: "PARENT_LIFECYCLE",
+            message: `parent run must be completed: ${input.parentRunId}`,
+        };
+    }
+    const parentSpec = await readDagRunSpec(located.runDir);
+    const parentState = await readDagRunState(located.runDir);
+    const staged = await stageContinuationRun({
+        cwd: input.cwd,
+        parentRunDir: located.runDir,
+        parentRunId: input.parentRunId,
+        parentSpec,
+        parentState,
+        plan,
+        reason: input.reason,
+        requestId: input.requestId,
+    });
+    const continuationOptions = {
+        cwd: input.cwd,
+        spec: parentSpec,
+        state: staged.state,
+        runDir: staged.runDir,
+        ...(input.maxConcurrent !== undefined
+            ? { maxConcurrent: input.maxConcurrent }
+            : {}),
+        ...(input.executeNode ? { executeNode: input.executeNode } : {}),
+    };
+    const summary = await runDagContinuation(continuationOptions);
+    const result = {
+        ok: summary.status === "finished",
+        plan,
+        newRunId: summary.runId,
+        runDir: summary.runDir,
+        summary,
+    };
+    if (await fileExists(operatorRequestPath(input.cwd, input.requestId))) {
+        const replay = await readExistingOperatorRequest(input.cwd, input.requestId);
+        if (replay)
+            return replay;
+    }
+    await persistOperatorRequest(input.cwd, result, input.requestId);
+    return result;
+}