@dailephd/my-dev-kit-lab 0.8.1 → 0.9.0

package/dist/src/evaluation/agentSuccess/corpusTypes.js

@@ -0,0 +1,74 @@
+/**
+ * Contracts for the canonical agent-success implementation benchmark corpus. This is a separate contract family
+ * from the retrieval benchmark profiles (`benchmark-project-profiles.json`) and must not be merged into it.
+ */
+export const AGENT_SUCCESS_PROJECT_PROFILES_SCHEMA_VERSION = "my-dev-kit-lab-agent-success-project-profiles-v1";
+export const AGENT_SUCCESS_TASK_CATALOG_PATH = "benchmarks/contracts/agent-success-rate-tasks.json";
+export const AGENT_SUCCESS_PROJECT_PROFILES_PATH = "benchmarks/contracts/agent-success-rate-project-profiles.json";
+export const AGENT_SUCCESS_PROJECTS_DIRECTORY = "benchmarks/projects";
+export const AGENT_SUCCESS_CORPUS_PROJECT_IDS = ["agent-success-task-board-node", "agent-success-inventory-node"];
+/** The fixed catalog order. Three tasks per project: one localized, one cross-module, one broad-change. */
+export const AGENT_SUCCESS_CORPUS_TASKS = Object.freeze([
+    { id: "asr-board-title-normalization", project: "agent-success-task-board-node", locality: "localized" },
+    { id: "asr-board-import-idempotency", project: "agent-success-task-board-node", locality: "cross-module" },
+    { id: "asr-board-project-summary", project: "agent-success-task-board-node", locality: "broad-change" },
+    { id: "asr-inventory-quantity-boundary", project: "agent-success-inventory-node", locality: "localized" },
+    { id: "asr-inventory-reservation-atomicity", project: "agent-success-inventory-node", locality: "cross-module" },
+    { id: "asr-inventory-fulfillment-report", project: "agent-success-inventory-node", locality: "broad-change" }
+]);
+/** Files every project must contain, relative to the project root. */
+export const AGENT_SUCCESS_CORPUS_PROJECT_FILES = Object.freeze({
+    "agent-success-task-board-node": Object.freeze([
+        "README.md",
+        "package.json",
+        "src/validation.js",
+        "src/taskStore.js",
+        "src/taskService.js",
+        "src/projectSummary.js",
+        "src/index.js",
+        "tests/title-primary.check.mjs",
+        "tests/title-edge.check.mjs",
+        "tests/import-primary.check.mjs",
+        "tests/import-edge.check.mjs",
+        "tests/summary-primary.check.mjs",
+        "tests/summary-edge.check.mjs",
+        "tests/regression.check.mjs"
+    ]),
+    "agent-success-inventory-node": Object.freeze([
+        "README.md",
+        "package.json",
+        "src/quantity.js",
+        "src/inventoryStore.js",
+        "src/reservationService.js",
+        "src/fulfillmentReport.js",
+        "src/index.js",
+        "tests/quantity-primary.check.mjs",
+        "tests/quantity-edge.check.mjs",
+        "tests/reservation-primary.check.mjs",
+        "tests/reservation-edge.check.mjs",
+        "tests/report-primary.check.mjs",
+        "tests/report-edge.check.mjs",
+        "tests/regression.check.mjs"
+    ])
+});
+export const AGENT_SUCCESS_CORPUS_RAW_INCLUDE_GLOBS = Object.freeze(["src/**/*.js"]);
+/** Directory names that must never exist inside a canonical project. */
+export const AGENT_SUCCESS_CORPUS_FORBIDDEN_DIRECTORIES = Object.freeze([
+    ".git",
+    "node_modules",
+    "dist",
+    "build",
+    "coverage",
+    "lab-output"
+]);
+export class AgentSuccessCorpusError extends Error {
+    issues;
+    constructor(issues) {
+        super(`Invalid agent-success corpus: ${issues
+            .slice(0, 5)
+            .map((issue) => `${issue.path}: ${issue.code}`)
+            .join("; ")}${issues.length > 5 ? `; +${issues.length - 5} more` : ""}`);
+        this.name = "AgentSuccessCorpusError";
+        this.issues = [...issues];
+    }
+}

package/dist/src/evaluation/agentSuccess/extractPatchCandidate.js

@@ -0,0 +1,46 @@
+import { DEFAULT_PATCH_BOUNDS } from "./patchTypes.js";
+const FENCED_DIFF_BLOCK = /^```[ \t]*(?:diff|patch)[ \t]*\r?\n([\s\S]*?)^```[ \t]*$/gim;
+function looksLikeDiffStart(text) {
+    return text.startsWith("diff --git ") || text.startsWith("--- ");
+}
+/** Removes trailing line breaks only: a final context line that is a single space must survive. */
+function finalize(body) {
+    return `${body.replace(/(?:\r?\n)+$/, "")}\n`;
+}
+/**
+ * Extracts exactly one patch candidate from an agent's proposal: either one fenced `diff`/`patch` block, or
+ * the whole (trimmed) text when it is itself a unified diff. Ambiguity is rejected rather than guessed.
+ * Size and line bounds are enforced before any parsing.
+ */
+export function extractPatchCandidate(text, bounds = DEFAULT_PATCH_BOUNDS) {
+    if (typeof text !== "string" || text.trim() === "") {
+        return { ok: false, code: "EMPTY", message: "the proposal is empty." };
+    }
+    if (Buffer.byteLength(text, "utf8") > bounds.maxPatchBytes) {
+        return { ok: false, code: "PATCH_TOO_LARGE", message: `the proposal exceeds ${bounds.maxPatchBytes} bytes.` };
+    }
+    if (text.split("\n").length > bounds.maxPatchLines) {
+        return { ok: false, code: "PATCH_TOO_MANY_LINES", message: `the proposal exceeds ${bounds.maxPatchLines} lines.` };
+    }
+    const fenced = [...text.matchAll(FENCED_DIFF_BLOCK)];
+    if (fenced.length > 1) {
+        return { ok: false, code: "AMBIGUOUS_CANDIDATES", message: "more than one fenced diff/patch block was found." };
+    }
+    if (fenced.length === 1) {
+        const match = fenced[0];
+        const outside = text.slice(0, match.index) + text.slice((match.index ?? 0) + match[0].length);
+        if (/^diff --git /m.test(outside)) {
+            return { ok: false, code: "AMBIGUOUS_CANDIDATES", message: "a fenced block and a raw diff were both present." };
+        }
+        const body = match[1] ?? "";
+        if (body.trim() === "") {
+            return { ok: false, code: "EMPTY", message: "the fenced diff block is empty." };
+        }
+        return { ok: true, patch: finalize(body) };
+    }
+    const trimmed = text.trimStart();
+    if (looksLikeDiffStart(trimmed)) {
+        return { ok: true, patch: finalize(trimmed) };
+    }
+    return { ok: false, code: "NO_CANDIDATE", message: "no fenced diff/patch block and no raw unified diff was found." };
+}

package/dist/src/evaluation/agentSuccess/index.js

@@ -0,0 +1,14 @@
+export * from "./taskTypes.js";
+export { normalizeProjectRelativePath } from "./taskPaths.js";
+export { assertAgentSuccessTask, validateAgentSuccessTask } from "./validateAgentSuccessTask.js";
+export * from "./patchTypes.js";
+export { extractPatchCandidate } from "./extractPatchCandidate.js";
+export { parseUnifiedDiff } from "./parseUnifiedDiff.js";
+export { validatePatchPolicy } from "./validatePatchPolicy.js";
+export { applyPatchToSandbox } from "./applyPatchToSandbox.js";
+export * from "./verificationTypes.js";
+export { runVerificationCheck, runVerificationChecks } from "./runVerificationCheck.js";
+export { assessBaseline } from "./assessBaseline.js";
+export * from "./corpusTypes.js";
+export { assertAgentSuccessCorpus, validateAgentSuccessCorpus } from "./validateAgentSuccessCorpus.js";
+export { assertReadAgentSuccessCorpus, readAgentSuccessCorpus } from "./readAgentSuccessCorpus.js";

package/dist/src/evaluation/agentSuccess/parseUnifiedDiff.js

@@ -0,0 +1,210 @@
+const HUNK_HEADER = /^@@ -(\d+)(?:,(\d+))? \+(\d+)(?:,(\d+))? @@/;
+class PatchMalformed extends Error {
+    code;
+    constructor(code, message) {
+        super(message);
+        this.code = code;
+    }
+}
+function stripCr(line) {
+    return line.endsWith("\r") ? line.slice(0, -1) : line;
+}
+/** Mirrors `git apply -p1`: drop exactly one leading path component. */
+function stripOneComponent(rawPath) {
+    const slash = rawPath.indexOf("/");
+    if (slash < 0) {
+        throw new PatchMalformed("PATCH_MALFORMED", "a patch path has no directory component to strip.");
+    }
+    return rawPath.slice(slash + 1);
+}
+function parseHeaderPath(headerValue) {
+    const value = headerValue.split("\t")[0].trimEnd();
+    if (value.startsWith('"')) {
+        throw new PatchMalformed("UNSUPPORTED_PATH_QUOTING", "quoted patch paths are not supported.");
+    }
+    if (value === "/dev/null")
+        return null;
+    return stripOneComponent(value);
+}
+/** `diff --git <a> <b>`: returns the shared path when both sides agree after -p1 stripping, else null. */
+function parseGitHeaderPath(rest) {
+    if (rest.startsWith('"')) {
+        throw new PatchMalformed("UNSUPPORTED_PATH_QUOTING", "quoted patch paths are not supported.");
+    }
+    for (let index = rest.indexOf(" "); index >= 0; index = rest.indexOf(" ", index + 1)) {
+        const left = rest.slice(0, index);
+        const right = rest.slice(index + 1);
+        if (!left.includes("/") || !right.includes("/"))
+            continue;
+        if (stripOneComponent(left) === stripOneComponent(right))
+            return stripOneComponent(right);
+    }
+    return null;
+}
+function newFile(gitHeaderPath) {
+    return {
+        oldPath: null,
+        newPath: null,
+        gitHeaderPath,
+        isNew: false,
+        isDelete: false,
+        binary: false,
+        rename: false,
+        copy: false,
+        oldMode: null,
+        newMode: null,
+        subproject: false,
+        hunkCount: 0
+    };
+}
+/**
+ * Parses a Git/unified diff into per-file records with exact hunk accounting: every hunk must contain
+ * precisely the old/new line counts its header declares, so truncated or padded patches are rejected here
+ * instead of being half-understood. Only structure is recorded; policy decisions live in validatePatchPolicy.
+ */
+export function parseUnifiedDiff(patch) {
+    try {
+        const lines = patch.split("\n");
+        if (lines[lines.length - 1] === "")
+            lines.pop();
+        const files = [];
+        let index = 0;
+        const parseFileHeaders = (file) => {
+            const oldHeader = stripCr(lines[index]);
+            const plusLine = lines[index + 1];
+            if (plusLine === undefined || !stripCr(plusLine).startsWith("+++ ")) {
+                throw new PatchMalformed("PATCH_MALFORMED", 'a "---" header is not followed by a "+++" header.');
+            }
+            file.oldPath = parseHeaderPath(oldHeader.slice(4));
+            file.newPath = parseHeaderPath(stripCr(plusLine).slice(4));
+            if (file.oldPath === null)
+                file.isNew = true;
+            if (file.newPath === null)
+                file.isDelete = true;
+            index += 2;
+            while (index < lines.length && stripCr(lines[index]).startsWith("@@")) {
+                parseHunk(file);
+            }
+        };
+        const parseHunk = (file) => {
+            const match = HUNK_HEADER.exec(stripCr(lines[index]));
+            if (!match)
+                throw new PatchMalformed("PATCH_MALFORMED", "malformed hunk header.");
+            let oldRemaining = match[2] === undefined ? 1 : Number(match[2]);
+            let newRemaining = match[4] === undefined ? 1 : Number(match[4]);
+            index += 1;
+            while (oldRemaining > 0 || newRemaining > 0) {
+                const line = lines[index];
+                if (line === undefined)
+                    throw new PatchMalformed("PATCH_MALFORMED", "a hunk ends before its declared line counts.");
+                const marker = line.length === 0 ? " " : line[0];
+                if (marker === " ") {
+                    oldRemaining -= 1;
+                    newRemaining -= 1;
+                }
+                else if (marker === "-") {
+                    oldRemaining -= 1;
+                    if (line.startsWith("-Subproject commit "))
+                        file.subproject = true;
+                }
+                else if (marker === "+") {
+                    newRemaining -= 1;
+                    if (line.startsWith("+Subproject commit "))
+                        file.subproject = true;
+                }
+                else if (marker !== "\\") {
+                    throw new PatchMalformed("PATCH_MALFORMED", "unexpected line inside a hunk.");
+                }
+                if (oldRemaining < 0 || newRemaining < 0) {
+                    throw new PatchMalformed("PATCH_MALFORMED", "a hunk has more lines than its header declares.");
+                }
+                index += 1;
+            }
+            while (index < lines.length && lines[index].startsWith("\\"))
+                index += 1;
+            file.hunkCount += 1;
+        };
+        while (index < lines.length) {
+            const line = stripCr(lines[index]);
+            if (line.startsWith("diff --git ")) {
+                const file = newFile(parseGitHeaderPath(line.slice("diff --git ".length)));
+                index += 1;
+                while (index < lines.length) {
+                    const header = stripCr(lines[index]);
+                    if (header.startsWith("--- ") || header.startsWith("diff --git "))
+                        break;
+                    let match;
+                    if ((match = /^index [0-9a-f]+\.\.[0-9a-f]+(?: (\d+))?$/.exec(header))) {
+                        if (match[1]) {
+                            file.oldMode = match[1];
+                            file.newMode = match[1];
+                        }
+                    }
+                    else if ((match = /^old mode (\d+)$/.exec(header))) {
+                        file.oldMode = match[1];
+                    }
+                    else if ((match = /^new mode (\d+)$/.exec(header))) {
+                        file.newMode = match[1];
+                    }
+                    else if ((match = /^deleted file mode (\d+)$/.exec(header))) {
+                        file.isDelete = true;
+                        file.oldMode = match[1];
+                    }
+                    else if ((match = /^new file mode (\d+)$/.exec(header))) {
+                        file.isNew = true;
+                        file.newMode = match[1];
+                    }
+                    else if (/^(?:dis)?similarity index \d+%$/.test(header)) {
+                        // informational; rename/copy intent is carried by the from/to lines
+                    }
+                    else if (/^rename (?:from|to) /.test(header)) {
+                        file.rename = true;
+                    }
+                    else if (/^copy (?:from|to) /.test(header)) {
+                        file.copy = true;
+                    }
+                    else if (/^Binary files .* differ$/.test(header) || header === "GIT binary patch") {
+                        file.binary = true;
+                        index += 1;
+                        while (index < lines.length && !lines[index].startsWith("diff --git "))
+                            index += 1;
+                        break;
+                    }
+                    else {
+                        throw new PatchMalformed("PATCH_MALFORMED", "unrecognized extended header line.");
+                    }
+                    index += 1;
+                }
+                if (!file.binary && index < lines.length && stripCr(lines[index]).startsWith("--- ")) {
+                    parseFileHeaders(file);
+                }
+                files.push(file);
+            }
+            else if (line.startsWith("--- ") && lines[index + 1] !== undefined && stripCr(lines[index + 1]).startsWith("+++ ")) {
+                const file = newFile(null);
+                parseFileHeaders(file);
+                files.push(file);
+            }
+            else if (line.trim() === "") {
+                index += 1;
+            }
+            else {
+                throw new PatchMalformed("PATCH_MALFORMED", "unexpected content outside a file section.");
+            }
+        }
+        if (files.length === 0) {
+            throw new PatchMalformed("PATCH_MALFORMED", "the patch contains no file sections.");
+        }
+        for (const file of files) {
+            if ((file.oldPath ?? file.newPath ?? file.gitHeaderPath) === null && !file.rename && !file.copy) {
+                throw new PatchMalformed("PATCH_MALFORMED", "a file section has no determinable path.");
+            }
+        }
+        return { ok: true, files };
+    }
+    catch (error) {
+        if (error instanceof PatchMalformed)
+            return { ok: false, code: error.code, message: error.message };
+        throw error;
+    }
+}

package/dist/src/evaluation/agentSuccess/patchTypes.js

@@ -0,0 +1,5 @@
+export const DEFAULT_PATCH_BOUNDS = Object.freeze({
+    maxPatchBytes: 1_048_576,
+    maxChangedPaths: 64,
+    maxPatchLines: 20_000
+});

package/dist/src/evaluation/agentSuccess/readAgentSuccessCorpus.js

@@ -0,0 +1,183 @@
+import { lstatSync, readFileSync, readdirSync, realpathSync } from "node:fs";
+import path from "node:path";
+import { AGENT_SUCCESS_CORPUS_FORBIDDEN_DIRECTORIES, AGENT_SUCCESS_CORPUS_PROJECT_FILES, AGENT_SUCCESS_PROJECTS_DIRECTORY, AGENT_SUCCESS_PROJECT_PROFILES_PATH, AGENT_SUCCESS_TASK_CATALOG_PATH, AgentSuccessCorpusError } from "./corpusTypes.js";
+import { validateAgentSuccessCorpus } from "./validateAgentSuccessCorpus.js";
+const MAX_TREE_ENTRIES = 2_000;
+const MAX_ISSUES = 60;
+const MAX_CATALOG_BYTES = 4 * 1024 * 1024;
+const TEST_NAMING_PATTERN = /\.(test|spec)\.[cm]?[jt]sx?$/i;
+class Collector {
+    issues = [];
+    add(code, issuePath, message) {
+        if (this.issues.length < MAX_ISSUES)
+            this.issues.push({ code, path: issuePath, message });
+    }
+}
+function readCatalog(collector, rootDir, relativePath) {
+    const absolute = path.join(rootDir, ...relativePath.split("/"));
+    let stats;
+    try {
+        stats = lstatSync(absolute);
+    }
+    catch {
+        collector.add("CATALOG_MISSING", relativePath, "the catalog file is missing.");
+        return { found: false };
+    }
+    if (!stats.isFile() || stats.isSymbolicLink()) {
+        collector.add("CATALOG_NOT_FILE", relativePath, "the catalog must be a regular file, not a link or directory.");
+        return { found: false };
+    }
+    if (stats.size > MAX_CATALOG_BYTES) {
+        collector.add("CATALOG_UNREADABLE", relativePath, `the catalog exceeds ${MAX_CATALOG_BYTES} bytes.`);
+        return { found: false };
+    }
+    let text;
+    try {
+        text = readFileSync(absolute, "utf8");
+    }
+    catch {
+        collector.add("CATALOG_UNREADABLE", relativePath, "the catalog could not be read.");
+        return { found: false };
+    }
+    try {
+        return { found: true, value: JSON.parse(text) };
+    }
+    catch (error) {
+        collector.add("CATALOG_INVALID_JSON", relativePath, `the catalog is not valid JSON (${error instanceof Error ? error.name : "error"}).`);
+        return { found: false };
+    }
+}
+function realDirectory(absolute) {
+    try {
+        if (lstatSync(absolute).isSymbolicLink())
+            return null;
+        return realpathSync(absolute);
+    }
+    catch {
+        return null;
+    }
+}
+function isInside(root, candidate) {
+    const relative = path.relative(root, candidate);
+    return relative !== "" && !relative.startsWith("..") && !path.isAbsolute(relative);
+}
+/** Lists project-relative POSIX paths of every entry below `projectRoot`, using lstat so links are never followed. */
+function walkProject(collector, projectRoot, label) {
+    const files = new Set();
+    const directories = new Set();
+    let visited = 0;
+    const visit = (relative) => {
+        const directory = relative === "" ? projectRoot : path.join(projectRoot, ...relative.split("/"));
+        for (const name of readdirSync(directory).sort()) {
+            if (++visited > MAX_TREE_ENTRIES)
+                return;
+            const entryRelative = relative === "" ? name : `${relative}/${name}`;
+            const stats = lstatSync(path.join(directory, name));
+            if (stats.isSymbolicLink()) {
+                collector.add("PROJECT_SYMLINK", `${label}/${entryRelative}`, "links are not allowed in a canonical benchmark project.");
+                continue;
+            }
+            if (stats.isDirectory()) {
+                directories.add(entryRelative);
+                if (AGENT_SUCCESS_CORPUS_FORBIDDEN_DIRECTORIES.includes(name)) {
+                    collector.add("PROJECT_FORBIDDEN_OUTPUT", `${label}/${entryRelative}`, "generated or tool output is not allowed in a canonical benchmark project.");
+                    continue;
+                }
+                visit(entryRelative);
+            }
+            else if (stats.isFile()) {
+                files.add(entryRelative);
+                if (TEST_NAMING_PATTERN.test(name)) {
+                    collector.add("PROJECT_TEST_NAMING", `${label}/${entryRelative}`, "benchmark checks must be named *.check.mjs so root test discovery never runs them.");
+                }
+            }
+            else {
+                collector.add("PROJECT_SYMLINK", `${label}/${entryRelative}`, "only regular files and directories are allowed.");
+            }
+        }
+    };
+    visit("");
+    if (visited > MAX_TREE_ENTRIES)
+        collector.add("PROJECT_TREE_TOO_LARGE", label, `the project has more than ${MAX_TREE_ENTRIES} entries.`);
+    return { files, directories };
+}
+function checkProjectMetadata(collector, projectRoot, label) {
+    try {
+        const metadata = JSON.parse(readFileSync(path.join(projectRoot, "package.json"), "utf8"));
+        const engines = metadata.engines;
+        const problems = [];
+        if (metadata.private !== true)
+            problems.push("private must be true");
+        if (metadata.type !== "module")
+            problems.push('type must be "module"');
+        if (engines?.node !== ">=24")
+            problems.push('engines.node must be ">=24"');
+        if (metadata.dependencies !== undefined || metadata.devDependencies !== undefined)
+            problems.push("the project must be dependency-free");
+        if (problems.length > 0)
+            collector.add("PROJECT_METADATA_INVALID", `${label}/package.json`, `${problems.join("; ")}.`);
+    }
+    catch {
+        collector.add("PROJECT_METADATA_INVALID", `${label}/package.json`, "package.json is missing or not valid JSON.");
+    }
+}
+function checkProject(collector, projectsRoot, rootDir, profile, tasks) {
+    const label = profile.rootPath;
+    const absolute = path.join(rootDir, ...label.split("/"));
+    const physical = realDirectory(absolute);
+    if (projectsRoot === null || physical === null || !isInside(projectsRoot, physical)) {
+        collector.add("PROJECT_ROOT_UNSAFE", label, "the project root must be a real directory inside the controlled benchmark projects directory.");
+        return;
+    }
+    const { files, directories } = walkProject(collector, physical, label);
+    const required = AGENT_SUCCESS_CORPUS_PROJECT_FILES[profile.projectId] ?? [];
+    for (const file of required) {
+        if (!files.has(file))
+            collector.add("PROJECT_FILE_MISSING", `${label}/${file}`, "a required project file is missing.");
+    }
+    for (const root of [...profile.sourceRoots, ...profile.testRoots]) {
+        if (!directories.has(root))
+            collector.add("PROJECT_SCOPE_MISSING", `${label}/${root}`, "a declared source or test root is missing.");
+    }
+    checkProjectMetadata(collector, physical, label);
+    for (const task of tasks.filter((candidate) => candidate.benchmarkProject === profile.projectId)) {
+        const referenced = [
+            ...task.expectedEditFiles,
+            ...task.allowedEditFiles,
+            ...task.protectedFiles,
+            ...[...task.taskChecks, ...task.regressionChecks].map((check) => check.args[1] ?? "")
+        ];
+        for (const file of new Set(referenced)) {
+            if (!files.has(file))
+                collector.add("PROJECT_FILE_MISSING", `${task.id}:${file}`, "a file referenced by the task does not exist in the project.");
+        }
+    }
+}
+/**
+ * Reads the dedicated agent-success task and project-profile catalogs from `rootDir`, validates their structure,
+ * and checks that the referenced canonical projects exist on disk with the expected layout. Synchronous because
+ * `scripts/verify-benchmarks.ts` is synchronous. It never executes a check or a reference patch and never writes.
+ */
+export function readAgentSuccessCorpus(rootDir) {
+    const collector = new Collector();
+    const profiles = readCatalog(collector, rootDir, AGENT_SUCCESS_PROJECT_PROFILES_PATH);
+    const tasks = readCatalog(collector, rootDir, AGENT_SUCCESS_TASK_CATALOG_PATH);
+    if (!profiles.found || !tasks.found)
+        return { ok: false, issues: collector.issues };
+    const structural = validateAgentSuccessCorpus({ profiles: profiles.value, tasks: tasks.value });
+    if (!structural.ok)
+        return { ok: false, issues: structural.issues };
+    const projectsRoot = realDirectory(path.join(rootDir, ...AGENT_SUCCESS_PROJECTS_DIRECTORY.split("/")));
+    for (const profile of structural.corpus.profiles) {
+        checkProject(collector, projectsRoot, rootDir, profile, structural.corpus.tasks);
+    }
+    if (collector.issues.length > 0)
+        return { ok: false, issues: collector.issues };
+    return { ok: true, corpus: structural.corpus };
+}
+export function assertReadAgentSuccessCorpus(rootDir) {
+    const result = readAgentSuccessCorpus(rootDir);
+    if (!result.ok)
+        throw new AgentSuccessCorpusError(result.issues);
+    return result.corpus;
+}

package/dist/src/evaluation/agentSuccess/runVerificationCheck.js

@@ -0,0 +1,100 @@
+import path from "node:path";
+import { runMeasuredCommand } from "../../core/runMeasuredCommand.js";
+import { buildMinimalHostEnv } from "../benchmarkSandbox/index.js";
+import { VERIFICATION_OUTPUT_LIMIT_BYTES } from "./verificationTypes.js";
+function failureReasonFor(status, processError, exitCode) {
+    if (status === "passed")
+        return null;
+    if (status === "failed")
+        return `the check exited with code ${exitCode}.`;
+    return processError ?? "the check could not be evaluated.";
+}
+/**
+ * Runs one trusted check: `node` (the running Node binary) with the contract's structured arguments, in the
+ * sandbox project directory, with no shell, a clean environment (plus CI=1 and NO_COLOR=1), bounded
+ * stdout/stderr, a timeout, and whole-process-tree termination. Evidence files land beside the project copy.
+ */
+export async function runVerificationCheck(options) {
+    const { sandbox, check, checkClass, phase } = options;
+    const limit = options.outputLimitBytes ?? VERIFICATION_OUTPUT_LIMIT_BYTES;
+    const startedAt = new Date().toISOString();
+    const started = Date.now();
+    try {
+        const measured = await runMeasuredCommand({
+            commandId: `${checkClass}-${check.id}`,
+            executable: process.execPath,
+            args: check.args,
+            cwd: sandbox.projectRoot,
+            outDir: path.join(sandbox.evidenceRoot, "verification", phase),
+            env: buildMinimalHostEnv(process.env, { CI: "1", NO_COLOR: "1" }),
+            inheritParentEnv: false,
+            resolveCommand: false,
+            timeoutMs: check.timeoutMs,
+            stdoutMaxBytes: limit,
+            stderrMaxBytes: limit,
+            terminateProcessTree: true
+        });
+        const timedOut = measured.timedOut === true;
+        const outputLimit = measured.outputLimit ?? null;
+        const status = timedOut
+            ? "timeout"
+            : outputLimit !== null || (measured.exitCode === null && !measured.ok)
+                ? "error"
+                : measured.exitCode === 0
+                    ? "passed"
+                    : "failed";
+        const processError = status === "failed" || status === "passed" ? null : (measured.error ?? null);
+        return {
+            checkId: check.id,
+            class: checkClass,
+            executable: "node",
+            args: [...check.args],
+            startedAt: measured.startedAt,
+            endedAt: measured.endedAt,
+            durationMs: measured.durationMs,
+            status,
+            exitCode: measured.exitCode,
+            stdoutPath: measured.stdoutPath,
+            stderrPath: measured.stderrPath,
+            timedOut,
+            outputLimit,
+            processError,
+            failureReason: failureReasonFor(status, processError, measured.exitCode)
+        };
+    }
+    catch (error) {
+        const message = error instanceof Error ? error.message : String(error);
+        return {
+            checkId: check.id,
+            class: checkClass,
+            executable: "node",
+            args: [...check.args],
+            startedAt,
+            endedAt: new Date().toISOString(),
+            durationMs: Date.now() - started,
+            status: "error",
+            exitCode: null,
+            stdoutPath: "",
+            stderrPath: "",
+            timedOut: false,
+            outputLimit: null,
+            processError: message,
+            failureReason: message
+        };
+    }
+}
+/**
+ * Runs every task check and then every regression check in contract order. It never short-circuits and
+ * never derives a success flag: the caller receives separate raw evidence for each class.
+ */
+export async function runVerificationChecks(options) {
+    const taskResults = [];
+    for (const check of options.task.taskChecks) {
+        taskResults.push(await runVerificationCheck({ sandbox: options.sandbox, check, checkClass: "task", phase: options.phase, outputLimitBytes: options.outputLimitBytes }));
+    }
+    const regressionResults = [];
+    for (const check of options.task.regressionChecks) {
+        regressionResults.push(await runVerificationCheck({ sandbox: options.sandbox, check, checkClass: "regression", phase: options.phase, outputLimitBytes: options.outputLimitBytes }));
+    }
+    return { phase: options.phase, taskResults, regressionResults };
+}

package/dist/src/evaluation/agentSuccess/taskPaths.js

@@ -0,0 +1,34 @@
+import path from "node:path";
+/**
+ * Normalizes a project-relative file path to POSIX form, or reports why it is unsafe. Shared by the task
+ * contract and the patch policy so both apply exactly the same path rules.
+ *
+ * Rejected: non-strings, empty, NUL, absolute (POSIX or UNC style), drive-qualified, "." alone, any ".."
+ * segment (before or after normalization), any ".git" segment (case-insensitive), a trailing slash.
+ */
+export function normalizeProjectRelativePath(raw) {
+    if (typeof raw !== "string")
+        return { ok: false, code: "PATH_NOT_STRING" };
+    if (raw.trim() === "")
+        return { ok: false, code: "PATH_EMPTY" };
+    if (raw.includes("\0"))
+        return { ok: false, code: "PATH_NUL" };
+    const slashed = raw.replace(/\\/g, "/");
+    if (/^[A-Za-z]:/.test(slashed))
+        return { ok: false, code: "PATH_DRIVE" };
+    if (slashed.startsWith("/"))
+        return { ok: false, code: "PATH_ABSOLUTE" };
+    if (slashed.split("/").some((segment) => segment === ".."))
+        return { ok: false, code: "PATH_TRAVERSAL" };
+    if (slashed.endsWith("/"))
+        return { ok: false, code: "PATH_TRAILING_SLASH" };
+    const normalized = path.posix.normalize(slashed);
+    if (normalized === ".")
+        return { ok: false, code: "PATH_DOT" };
+    if (normalized === ".." || normalized.startsWith("../"))
+        return { ok: false, code: "PATH_TRAVERSAL" };
+    if (normalized.split("/").some((segment) => segment.toLowerCase() === ".git")) {
+        return { ok: false, code: "PATH_GIT_SEGMENT" };
+    }
+    return { ok: true, path: normalized };
+}