@dailephd/my-dev-kit-lab 0.8.2 → 0.9.0

package/dist/src/evaluation/agentSuccess/readAgentSuccessCorpus.js

@@ -0,0 +1,183 @@
+import { lstatSync, readFileSync, readdirSync, realpathSync } from "node:fs";
+import path from "node:path";
+import { AGENT_SUCCESS_CORPUS_FORBIDDEN_DIRECTORIES, AGENT_SUCCESS_CORPUS_PROJECT_FILES, AGENT_SUCCESS_PROJECTS_DIRECTORY, AGENT_SUCCESS_PROJECT_PROFILES_PATH, AGENT_SUCCESS_TASK_CATALOG_PATH, AgentSuccessCorpusError } from "./corpusTypes.js";
+import { validateAgentSuccessCorpus } from "./validateAgentSuccessCorpus.js";
+const MAX_TREE_ENTRIES = 2_000;
+const MAX_ISSUES = 60;
+const MAX_CATALOG_BYTES = 4 * 1024 * 1024;
+const TEST_NAMING_PATTERN = /\.(test|spec)\.[cm]?[jt]sx?$/i;
+class Collector {
+    issues = [];
+    add(code, issuePath, message) {
+        if (this.issues.length < MAX_ISSUES)
+            this.issues.push({ code, path: issuePath, message });
+    }
+}
+function readCatalog(collector, rootDir, relativePath) {
+    const absolute = path.join(rootDir, ...relativePath.split("/"));
+    let stats;
+    try {
+        stats = lstatSync(absolute);
+    }
+    catch {
+        collector.add("CATALOG_MISSING", relativePath, "the catalog file is missing.");
+        return { found: false };
+    }
+    if (!stats.isFile() || stats.isSymbolicLink()) {
+        collector.add("CATALOG_NOT_FILE", relativePath, "the catalog must be a regular file, not a link or directory.");
+        return { found: false };
+    }
+    if (stats.size > MAX_CATALOG_BYTES) {
+        collector.add("CATALOG_UNREADABLE", relativePath, `the catalog exceeds ${MAX_CATALOG_BYTES} bytes.`);
+        return { found: false };
+    }
+    let text;
+    try {
+        text = readFileSync(absolute, "utf8");
+    }
+    catch {
+        collector.add("CATALOG_UNREADABLE", relativePath, "the catalog could not be read.");
+        return { found: false };
+    }
+    try {
+        return { found: true, value: JSON.parse(text) };
+    }
+    catch (error) {
+        collector.add("CATALOG_INVALID_JSON", relativePath, `the catalog is not valid JSON (${error instanceof Error ? error.name : "error"}).`);
+        return { found: false };
+    }
+}
+function realDirectory(absolute) {
+    try {
+        if (lstatSync(absolute).isSymbolicLink())
+            return null;
+        return realpathSync(absolute);
+    }
+    catch {
+        return null;
+    }
+}
+function isInside(root, candidate) {
+    const relative = path.relative(root, candidate);
+    return relative !== "" && !relative.startsWith("..") && !path.isAbsolute(relative);
+}
+/** Lists project-relative POSIX paths of every entry below `projectRoot`, using lstat so links are never followed. */
+function walkProject(collector, projectRoot, label) {
+    const files = new Set();
+    const directories = new Set();
+    let visited = 0;
+    const visit = (relative) => {
+        const directory = relative === "" ? projectRoot : path.join(projectRoot, ...relative.split("/"));
+        for (const name of readdirSync(directory).sort()) {
+            if (++visited > MAX_TREE_ENTRIES)
+                return;
+            const entryRelative = relative === "" ? name : `${relative}/${name}`;
+            const stats = lstatSync(path.join(directory, name));
+            if (stats.isSymbolicLink()) {
+                collector.add("PROJECT_SYMLINK", `${label}/${entryRelative}`, "links are not allowed in a canonical benchmark project.");
+                continue;
+            }
+            if (stats.isDirectory()) {
+                directories.add(entryRelative);
+                if (AGENT_SUCCESS_CORPUS_FORBIDDEN_DIRECTORIES.includes(name)) {
+                    collector.add("PROJECT_FORBIDDEN_OUTPUT", `${label}/${entryRelative}`, "generated or tool output is not allowed in a canonical benchmark project.");
+                    continue;
+                }
+                visit(entryRelative);
+            }
+            else if (stats.isFile()) {
+                files.add(entryRelative);
+                if (TEST_NAMING_PATTERN.test(name)) {
+                    collector.add("PROJECT_TEST_NAMING", `${label}/${entryRelative}`, "benchmark checks must be named *.check.mjs so root test discovery never runs them.");
+                }
+            }
+            else {
+                collector.add("PROJECT_SYMLINK", `${label}/${entryRelative}`, "only regular files and directories are allowed.");
+            }
+        }
+    };
+    visit("");
+    if (visited > MAX_TREE_ENTRIES)
+        collector.add("PROJECT_TREE_TOO_LARGE", label, `the project has more than ${MAX_TREE_ENTRIES} entries.`);
+    return { files, directories };
+}
+function checkProjectMetadata(collector, projectRoot, label) {
+    try {
+        const metadata = JSON.parse(readFileSync(path.join(projectRoot, "package.json"), "utf8"));
+        const engines = metadata.engines;
+        const problems = [];
+        if (metadata.private !== true)
+            problems.push("private must be true");
+        if (metadata.type !== "module")
+            problems.push('type must be "module"');
+        if (engines?.node !== ">=24")
+            problems.push('engines.node must be ">=24"');
+        if (metadata.dependencies !== undefined || metadata.devDependencies !== undefined)
+            problems.push("the project must be dependency-free");
+        if (problems.length > 0)
+            collector.add("PROJECT_METADATA_INVALID", `${label}/package.json`, `${problems.join("; ")}.`);
+    }
+    catch {
+        collector.add("PROJECT_METADATA_INVALID", `${label}/package.json`, "package.json is missing or not valid JSON.");
+    }
+}
+function checkProject(collector, projectsRoot, rootDir, profile, tasks) {
+    const label = profile.rootPath;
+    const absolute = path.join(rootDir, ...label.split("/"));
+    const physical = realDirectory(absolute);
+    if (projectsRoot === null || physical === null || !isInside(projectsRoot, physical)) {
+        collector.add("PROJECT_ROOT_UNSAFE", label, "the project root must be a real directory inside the controlled benchmark projects directory.");
+        return;
+    }
+    const { files, directories } = walkProject(collector, physical, label);
+    const required = AGENT_SUCCESS_CORPUS_PROJECT_FILES[profile.projectId] ?? [];
+    for (const file of required) {
+        if (!files.has(file))
+            collector.add("PROJECT_FILE_MISSING", `${label}/${file}`, "a required project file is missing.");
+    }
+    for (const root of [...profile.sourceRoots, ...profile.testRoots]) {
+        if (!directories.has(root))
+            collector.add("PROJECT_SCOPE_MISSING", `${label}/${root}`, "a declared source or test root is missing.");
+    }
+    checkProjectMetadata(collector, physical, label);
+    for (const task of tasks.filter((candidate) => candidate.benchmarkProject === profile.projectId)) {
+        const referenced = [
+            ...task.expectedEditFiles,
+            ...task.allowedEditFiles,
+            ...task.protectedFiles,
+            ...[...task.taskChecks, ...task.regressionChecks].map((check) => check.args[1] ?? "")
+        ];
+        for (const file of new Set(referenced)) {
+            if (!files.has(file))
+                collector.add("PROJECT_FILE_MISSING", `${task.id}:${file}`, "a file referenced by the task does not exist in the project.");
+        }
+    }
+}
+/**
+ * Reads the dedicated agent-success task and project-profile catalogs from `rootDir`, validates their structure,
+ * and checks that the referenced canonical projects exist on disk with the expected layout. Synchronous because
+ * `scripts/verify-benchmarks.ts` is synchronous. It never executes a check or a reference patch and never writes.
+ */
+export function readAgentSuccessCorpus(rootDir) {
+    const collector = new Collector();
+    const profiles = readCatalog(collector, rootDir, AGENT_SUCCESS_PROJECT_PROFILES_PATH);
+    const tasks = readCatalog(collector, rootDir, AGENT_SUCCESS_TASK_CATALOG_PATH);
+    if (!profiles.found || !tasks.found)
+        return { ok: false, issues: collector.issues };
+    const structural = validateAgentSuccessCorpus({ profiles: profiles.value, tasks: tasks.value });
+    if (!structural.ok)
+        return { ok: false, issues: structural.issues };
+    const projectsRoot = realDirectory(path.join(rootDir, ...AGENT_SUCCESS_PROJECTS_DIRECTORY.split("/")));
+    for (const profile of structural.corpus.profiles) {
+        checkProject(collector, projectsRoot, rootDir, profile, structural.corpus.tasks);
+    }
+    if (collector.issues.length > 0)
+        return { ok: false, issues: collector.issues };
+    return { ok: true, corpus: structural.corpus };
+}
+export function assertReadAgentSuccessCorpus(rootDir) {
+    const result = readAgentSuccessCorpus(rootDir);
+    if (!result.ok)
+        throw new AgentSuccessCorpusError(result.issues);
+    return result.corpus;
+}

package/dist/src/evaluation/agentSuccess/runVerificationCheck.js

@@ -0,0 +1,100 @@
+import path from "node:path";
+import { runMeasuredCommand } from "../../core/runMeasuredCommand.js";
+import { buildMinimalHostEnv } from "../benchmarkSandbox/index.js";
+import { VERIFICATION_OUTPUT_LIMIT_BYTES } from "./verificationTypes.js";
+function failureReasonFor(status, processError, exitCode) {
+    if (status === "passed")
+        return null;
+    if (status === "failed")
+        return `the check exited with code ${exitCode}.`;
+    return processError ?? "the check could not be evaluated.";
+}
+/**
+ * Runs one trusted check: `node` (the running Node binary) with the contract's structured arguments, in the
+ * sandbox project directory, with no shell, a clean environment (plus CI=1 and NO_COLOR=1), bounded
+ * stdout/stderr, a timeout, and whole-process-tree termination. Evidence files land beside the project copy.
+ */
+export async function runVerificationCheck(options) {
+    const { sandbox, check, checkClass, phase } = options;
+    const limit = options.outputLimitBytes ?? VERIFICATION_OUTPUT_LIMIT_BYTES;
+    const startedAt = new Date().toISOString();
+    const started = Date.now();
+    try {
+        const measured = await runMeasuredCommand({
+            commandId: `${checkClass}-${check.id}`,
+            executable: process.execPath,
+            args: check.args,
+            cwd: sandbox.projectRoot,
+            outDir: path.join(sandbox.evidenceRoot, "verification", phase),
+            env: buildMinimalHostEnv(process.env, { CI: "1", NO_COLOR: "1" }),
+            inheritParentEnv: false,
+            resolveCommand: false,
+            timeoutMs: check.timeoutMs,
+            stdoutMaxBytes: limit,
+            stderrMaxBytes: limit,
+            terminateProcessTree: true
+        });
+        const timedOut = measured.timedOut === true;
+        const outputLimit = measured.outputLimit ?? null;
+        const status = timedOut
+            ? "timeout"
+            : outputLimit !== null || (measured.exitCode === null && !measured.ok)
+                ? "error"
+                : measured.exitCode === 0
+                    ? "passed"
+                    : "failed";
+        const processError = status === "failed" || status === "passed" ? null : (measured.error ?? null);
+        return {
+            checkId: check.id,
+            class: checkClass,
+            executable: "node",
+            args: [...check.args],
+            startedAt: measured.startedAt,
+            endedAt: measured.endedAt,
+            durationMs: measured.durationMs,
+            status,
+            exitCode: measured.exitCode,
+            stdoutPath: measured.stdoutPath,
+            stderrPath: measured.stderrPath,
+            timedOut,
+            outputLimit,
+            processError,
+            failureReason: failureReasonFor(status, processError, measured.exitCode)
+        };
+    }
+    catch (error) {
+        const message = error instanceof Error ? error.message : String(error);
+        return {
+            checkId: check.id,
+            class: checkClass,
+            executable: "node",
+            args: [...check.args],
+            startedAt,
+            endedAt: new Date().toISOString(),
+            durationMs: Date.now() - started,
+            status: "error",
+            exitCode: null,
+            stdoutPath: "",
+            stderrPath: "",
+            timedOut: false,
+            outputLimit: null,
+            processError: message,
+            failureReason: message
+        };
+    }
+}
+/**
+ * Runs every task check and then every regression check in contract order. It never short-circuits and
+ * never derives a success flag: the caller receives separate raw evidence for each class.
+ */
+export async function runVerificationChecks(options) {
+    const taskResults = [];
+    for (const check of options.task.taskChecks) {
+        taskResults.push(await runVerificationCheck({ sandbox: options.sandbox, check, checkClass: "task", phase: options.phase, outputLimitBytes: options.outputLimitBytes }));
+    }
+    const regressionResults = [];
+    for (const check of options.task.regressionChecks) {
+        regressionResults.push(await runVerificationCheck({ sandbox: options.sandbox, check, checkClass: "regression", phase: options.phase, outputLimitBytes: options.outputLimitBytes }));
+    }
+    return { phase: options.phase, taskResults, regressionResults };
+}

package/dist/src/evaluation/agentSuccess/taskPaths.js

@@ -0,0 +1,34 @@
+import path from "node:path";
+/**
+ * Normalizes a project-relative file path to POSIX form, or reports why it is unsafe. Shared by the task
+ * contract and the patch policy so both apply exactly the same path rules.
+ *
+ * Rejected: non-strings, empty, NUL, absolute (POSIX or UNC style), drive-qualified, "." alone, any ".."
+ * segment (before or after normalization), any ".git" segment (case-insensitive), a trailing slash.
+ */
+export function normalizeProjectRelativePath(raw) {
+    if (typeof raw !== "string")
+        return { ok: false, code: "PATH_NOT_STRING" };
+    if (raw.trim() === "")
+        return { ok: false, code: "PATH_EMPTY" };
+    if (raw.includes("\0"))
+        return { ok: false, code: "PATH_NUL" };
+    const slashed = raw.replace(/\\/g, "/");
+    if (/^[A-Za-z]:/.test(slashed))
+        return { ok: false, code: "PATH_DRIVE" };
+    if (slashed.startsWith("/"))
+        return { ok: false, code: "PATH_ABSOLUTE" };
+    if (slashed.split("/").some((segment) => segment === ".."))
+        return { ok: false, code: "PATH_TRAVERSAL" };
+    if (slashed.endsWith("/"))
+        return { ok: false, code: "PATH_TRAILING_SLASH" };
+    const normalized = path.posix.normalize(slashed);
+    if (normalized === ".")
+        return { ok: false, code: "PATH_DOT" };
+    if (normalized === ".." || normalized.startsWith("../"))
+        return { ok: false, code: "PATH_TRAVERSAL" };
+    if (normalized.split("/").some((segment) => segment.toLowerCase() === ".git")) {
+        return { ok: false, code: "PATH_GIT_SEGMENT" };
+    }
+    return { ok: true, path: normalized };
+}

package/dist/src/evaluation/agentSuccess/taskTypes.js

@@ -0,0 +1,14 @@
+export const AGENT_SUCCESS_TASK_SCHEMA_VERSION = "my-dev-kit-lab-agent-success-task-v1";
+/** Upper bound for one verification check; bounded so a hung check cannot stall an evaluation indefinitely. */
+export const MAX_VERIFICATION_CHECK_TIMEOUT_MS = 300_000;
+export class AgentSuccessTaskValidationError extends Error {
+    issues;
+    constructor(issues) {
+        super(`Invalid agent-success task: ${issues
+            .slice(0, 5)
+            .map((issue) => `${issue.path}: ${issue.code}`)
+            .join("; ")}${issues.length > 5 ? `; +${issues.length - 5} more` : ""}`);
+        this.name = "AgentSuccessTaskValidationError";
+        this.issues = [...issues];
+    }
+}

package/dist/src/evaluation/agentSuccess/validateAgentSuccessCorpus.js

@@ -0,0 +1,307 @@
+import { AGENT_SUCCESS_CORPUS_PROJECT_IDS, AGENT_SUCCESS_CORPUS_RAW_INCLUDE_GLOBS, AGENT_SUCCESS_CORPUS_TASKS, AGENT_SUCCESS_PROJECTS_DIRECTORY, AGENT_SUCCESS_PROJECT_PROFILES_SCHEMA_VERSION, AgentSuccessCorpusError } from "./corpusTypes.js";
+import { extractPatchCandidate } from "./extractPatchCandidate.js";
+import { parseUnifiedDiff } from "./parseUnifiedDiff.js";
+import { normalizeProjectRelativePath } from "./taskPaths.js";
+import { validateAgentSuccessTask } from "./validateAgentSuccessTask.js";
+import { validatePatchPolicy } from "./validatePatchPolicy.js";
+const MAX_ISSUES = 60;
+const PROJECT_FIELDS = ["projectId", "displayName", "description", "rootPath", "sourceRoots", "testRoots", "primaryLanguage", "runtime"];
+const PROFILES_FIELDS = ["schemaVersion", "projects"];
+const CHECK_FILE_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]*\.check\.mjs$/;
+const MIN_LEAK_LINE_LENGTH = 12;
+/** Files every task must protect in addition to its check files. */
+const ALWAYS_PROTECTED = ["README.md", "package.json", "tests/regression.check.mjs"];
+class Collector {
+    issues = [];
+    add(code, path, message) {
+        if (this.issues.length < MAX_ISSUES)
+            this.issues.push({ code, path, message });
+    }
+}
+function isRecord(value) {
+    return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+function sameList(left, right) {
+    return left.length === right.length && left.every((value, index) => value === right[index]);
+}
+function isStringList(value) {
+    return Array.isArray(value) && value.every((entry) => typeof entry === "string");
+}
+function readProfiles(collector, input) {
+    if (!isRecord(input)) {
+        collector.add("PROFILES_INVALID", "profiles", "the project-profile catalog must be an object.");
+        return [];
+    }
+    for (const key of Object.keys(input)) {
+        if (!PROFILES_FIELDS.includes(key))
+            collector.add("PROFILES_INVALID", `profiles.${key}`, `unknown field "${key}".`);
+    }
+    if (input.schemaVersion !== AGENT_SUCCESS_PROJECT_PROFILES_SCHEMA_VERSION) {
+        collector.add("PROFILES_INVALID", "profiles.schemaVersion", `must equal ${AGENT_SUCCESS_PROJECT_PROFILES_SCHEMA_VERSION}.`);
+    }
+    if (!Array.isArray(input.projects)) {
+        collector.add("PROFILES_INVALID", "profiles.projects", "must be an array.");
+        return [];
+    }
+    const profiles = [];
+    input.projects.forEach((entry, index) => {
+        const base = `profiles.projects[${index}]`;
+        if (!isRecord(entry)) {
+            collector.add("PROFILE_INVALID", base, "must be an object.");
+            return;
+        }
+        for (const key of Object.keys(entry)) {
+            if (!PROJECT_FIELDS.includes(key))
+                collector.add("PROFILE_INVALID", `${base}.${key}`, `unknown field "${key}".`);
+        }
+        for (const key of ["projectId", "displayName", "description", "rootPath"]) {
+            if (typeof entry[key] !== "string" || entry[key].trim() === "") {
+                collector.add("PROFILE_INVALID", `${base}.${key}`, "must be a non-empty string.");
+            }
+        }
+        if (!isStringList(entry.sourceRoots) || !sameList(entry.sourceRoots, ["src"])) {
+            collector.add("PROFILE_INVALID", `${base}.sourceRoots`, 'must equal ["src"].');
+        }
+        if (!isStringList(entry.testRoots) || !sameList(entry.testRoots, ["tests"])) {
+            collector.add("PROFILE_INVALID", `${base}.testRoots`, 'must equal ["tests"].');
+        }
+        if (entry.primaryLanguage !== "javascript")
+            collector.add("PROFILE_INVALID", `${base}.primaryLanguage`, 'must equal "javascript".');
+        if (entry.runtime !== "node>=24")
+            collector.add("PROFILE_INVALID", `${base}.runtime`, 'must equal "node>=24".');
+        if (typeof entry.projectId === "string" && typeof entry.rootPath === "string") {
+            const normalized = normalizeProjectRelativePath(entry.rootPath);
+            const expected = `${AGENT_SUCCESS_PROJECTS_DIRECTORY}/${entry.projectId}`;
+            if (!normalized.ok || normalized.path !== expected) {
+                collector.add("PROFILE_ROOT_INVALID", `${base}.rootPath`, `must equal ${expected} (inside the controlled benchmark projects directory).`);
+            }
+        }
+        profiles.push({
+            projectId: String(entry.projectId ?? ""),
+            displayName: String(entry.displayName ?? ""),
+            description: String(entry.description ?? ""),
+            rootPath: String(entry.rootPath ?? ""),
+            sourceRoots: isStringList(entry.sourceRoots) ? [...entry.sourceRoots] : [],
+            testRoots: isStringList(entry.testRoots) ? [...entry.testRoots] : [],
+            primaryLanguage: "javascript",
+            runtime: "node>=24"
+        });
+    });
+    return profiles;
+}
+function validateProfileSet(collector, profiles) {
+    const ids = profiles.map((profile) => profile.projectId);
+    if (new Set(ids).size !== ids.length)
+        collector.add("PROJECT_SET_MISMATCH", "profiles.projects", "project ids must be unique.");
+    if (!sameList(ids, AGENT_SUCCESS_CORPUS_PROJECT_IDS)) {
+        collector.add("PROJECT_SET_MISMATCH", "profiles.projects", `expected exactly these project ids in order: ${AGENT_SUCCESS_CORPUS_PROJECT_IDS.join(", ")}.`);
+    }
+}
+function checkFileOf(args, testRoot) {
+    if (args.length !== 2 || args[0] !== "--test")
+        return null;
+    const normalized = normalizeProjectRelativePath(args[1]);
+    if (!normalized.ok)
+        return null;
+    const segments = normalized.path.split("/");
+    if (segments.length !== 2 || segments[0] !== testRoot)
+        return null;
+    return CHECK_FILE_PATTERN.test(segments[1]) ? normalized.path : null;
+}
+function validateChecks(collector, task, index, profile) {
+    const base = `tasks[${index}]`;
+    if (task.taskChecks.length !== 2)
+        collector.add("CHECK_COUNT_INVALID", `${base}.taskChecks`, "exactly two task checks are required.");
+    if (task.regressionChecks.length < 1)
+        collector.add("CHECK_COUNT_INVALID", `${base}.regressionChecks`, "at least one regression check is required.");
+    const files = [];
+    const testRoot = profile?.testRoots[0] ?? "tests";
+    const groups = [
+        ["taskChecks", task.taskChecks],
+        ["regressionChecks", task.regressionChecks]
+    ];
+    for (const [key, checks] of groups) {
+        checks.forEach((check, checkIndex) => {
+            const path = `${base}.${key}[${checkIndex}].args`;
+            const unsafe = check.args.some((arg) => normalizeProjectRelativePath(arg).ok === false && arg !== "--test");
+            const file = checkFileOf(check.args, testRoot);
+            if (file === null) {
+                collector.add(unsafe ? "CHECK_PATH_UNSAFE" : "CHECK_COMMAND_INVALID", path, `must be exactly ["--test", "${testRoot}/<name>.check.mjs"].`);
+            }
+            else {
+                files.push(file);
+            }
+        });
+    }
+    if (new Set(files).size !== files.length)
+        collector.add("CHECK_COMMAND_INVALID", `${base}.taskChecks`, "check files must be unique within a task.");
+    return files;
+}
+function validateScopes(collector, task, index, checkFiles) {
+    const base = `tasks[${index}]`;
+    if (task.expectedEditFiles.length === 0)
+        collector.add("EDIT_SCOPE_INVALID", `${base}.expectedEditFiles`, "must not be empty.");
+    for (const key of ["expectedEditFiles", "allowedEditFiles"]) {
+        for (const file of task[key]) {
+            if (!file.startsWith("src/"))
+                collector.add("EDIT_SCOPE_INVALID", `${base}.${key}`, `${file} is outside the project source root.`);
+        }
+    }
+    const protectedSet = new Set(task.protectedFiles);
+    for (const required of [...ALWAYS_PROTECTED, ...checkFiles]) {
+        if (!protectedSet.has(required))
+            collector.add("PROTECTED_SCOPE_INVALID", `${base}.protectedFiles`, `${required} must be protected.`);
+    }
+    for (const file of task.protectedFiles) {
+        if (file.startsWith("src/"))
+            collector.add("PROTECTED_SCOPE_INVALID", `${base}.protectedFiles`, `${file} is source and must not be protected.`);
+    }
+}
+function validateFacts(collector, task, index) {
+    const base = `tasks[${index}].behaviorFacts`;
+    if (!task.behaviorFacts.some((fact) => fact.required))
+        collector.add("FACT_COVERAGE_INVALID", base, "at least one required fact is needed.");
+    if (!task.behaviorFacts.some((fact) => !fact.required))
+        collector.add("FACT_COVERAGE_INVALID", base, "at least one optional fact is needed.");
+    task.behaviorFacts.forEach((fact, factIndex) => {
+        if (fact.verificationCheckIds.length === 0) {
+            collector.add("FACT_COVERAGE_INVALID", `${base}[${factIndex}].verificationCheckIds`, "every fact must reference at least one check.");
+        }
+    });
+}
+function changedLines(patch) {
+    return patch
+        .split("\n")
+        .filter((line) => (line.startsWith("+") && !line.startsWith("+++")) || (line.startsWith("-") && !line.startsWith("---")))
+        .map((line) => line.slice(1).trim())
+        .filter((line) => line.length >= MIN_LEAK_LINE_LENGTH);
+}
+function validateFixture(collector, task, index) {
+    const base = `tasks[${index}].deterministicFixture`;
+    const fixture = task.deterministicFixture;
+    if (fixture === undefined) {
+        collector.add("FIXTURE_MISSING", base, "a deterministic reference fixture is required.");
+        return;
+    }
+    const extracted = extractPatchCandidate(fixture.patch);
+    if (!extracted.ok) {
+        collector.add("FIXTURE_INVALID", base, `the reference patch is not an acceptable patch (${extracted.code}).`);
+        return;
+    }
+    const parsed = parseUnifiedDiff(extracted.patch);
+    if (!parsed.ok) {
+        collector.add("FIXTURE_INVALID", base, `the reference patch does not parse (${parsed.code}).`);
+        return;
+    }
+    const policy = validatePatchPolicy(parsed.files, { protectedFiles: task.protectedFiles });
+    if (!policy.ok) {
+        collector.add("FIXTURE_INVALID", base, `the reference patch violates the patch policy (${policy.rejections.map((r) => r.code).join(", ")}).`);
+        return;
+    }
+    const paths = policy.files.map((file) => file.path);
+    if (!sameList(paths, task.expectedEditFiles) || policy.files.some((file) => file.status !== "modified")) {
+        collector.add("FIXTURE_SCOPE_MISMATCH", base, "the reference patch must modify exactly the task's expectedEditFiles.");
+    }
+    const visibleText = `${task.query}\n${task.instruction}\n${task.title}`;
+    if (/(^|\n)(diff --git |@@ -)/.test(visibleText) || changedLines(fixture.patch).some((line) => visibleText.includes(line))) {
+        collector.add("FIXTURE_LEAKED", `tasks[${index}]`, "the task query, title or instruction reveals reference-patch content.");
+    }
+}
+function validateTasks(collector, input, profiles) {
+    if (!Array.isArray(input)) {
+        collector.add("TASKS_NOT_ARRAY", "tasks", "the task catalog must be an array.");
+        return [];
+    }
+    const tasks = [];
+    input.forEach((entry, index) => {
+        const result = validateAgentSuccessTask(entry);
+        if (!result.ok) {
+            for (const issue of result.issues.slice(0, 5))
+                collector.add("TASK_INVALID", `tasks[${index}].${issue.path}`.replace(/\.$/, ""), `${issue.code}: ${issue.message}`);
+            return;
+        }
+        tasks.push(result.task);
+    });
+    if (tasks.length !== input.length)
+        return tasks;
+    const ids = tasks.map((task) => task.id);
+    const seen = new Set();
+    ids.forEach((id, index) => {
+        if (seen.has(id))
+            collector.add("TASK_ID_DUPLICATE", `tasks[${index}].id`, `duplicate task id: ${id}.`);
+        seen.add(id);
+    });
+    const expectedIds = AGENT_SUCCESS_CORPUS_TASKS.map((task) => task.id);
+    if (tasks.length !== expectedIds.length) {
+        collector.add("TASK_COUNT_MISMATCH", "tasks", `expected exactly ${expectedIds.length} tasks, found ${tasks.length}.`);
+    }
+    else if (!sameList(ids, expectedIds) && new Set(ids).size === ids.length) {
+        collector.add("TASK_ORDER_MISMATCH", "tasks", `expected task ids in this order: ${expectedIds.join(", ")}.`);
+    }
+    const profileById = new Map(profiles.map((profile) => [profile.projectId, profile]));
+    const fixtureIds = new Set();
+    const fixturePatches = new Set();
+    const perProject = new Map();
+    tasks.forEach((task, index) => {
+        const base = `tasks[${index}]`;
+        const expectation = AGENT_SUCCESS_CORPUS_TASKS.find((candidate) => candidate.id === task.id);
+        const profile = profileById.get(task.projectProfileRef);
+        if (task.projectProfileRef !== task.benchmarkProject || profile === undefined) {
+            collector.add("PROJECT_REF_INVALID", `${base}.projectProfileRef`, "must equal benchmarkProject and resolve to a known project profile.");
+        }
+        else if (expectation !== undefined && expectation.project !== task.benchmarkProject) {
+            collector.add("PROJECT_REF_INVALID", `${base}.benchmarkProject`, `task ${task.id} belongs to ${expectation.project}.`);
+        }
+        if (expectation !== undefined && expectation.locality !== task.taskLocality) {
+            collector.add("LOCALITY_MISMATCH", `${base}.taskLocality`, `task ${task.id} must be ${expectation.locality}.`);
+        }
+        perProject.set(task.benchmarkProject, [...(perProject.get(task.benchmarkProject) ?? []), task.taskLocality]);
+        const sourceRoots = profile?.sourceRoots ?? ["src"];
+        if (!sameList(task.sourceRoots, sourceRoots)) {
+            collector.add("SOURCE_SCOPE_INVALID", `${base}.sourceRoots`, `must equal the project's source roots (${sourceRoots.join(", ")}).`);
+        }
+        if (!sameList(task.rawIncludeGlobs, AGENT_SUCCESS_CORPUS_RAW_INCLUDE_GLOBS)) {
+            collector.add("SOURCE_SCOPE_INVALID", `${base}.rawIncludeGlobs`, `must equal ${AGENT_SUCCESS_CORPUS_RAW_INCLUDE_GLOBS.join(", ")} so tests and contracts stay out of agent context.`);
+        }
+        const checkFiles = validateChecks(collector, task, index, profile);
+        validateScopes(collector, task, index, checkFiles);
+        validateFacts(collector, task, index);
+        validateFixture(collector, task, index);
+        if (task.deterministicFixture !== undefined) {
+            if (fixtureIds.has(task.deterministicFixture.id))
+                collector.add("FIXTURE_DUPLICATE", `${base}.deterministicFixture.id`, "fixture ids must be unique.");
+            if (fixturePatches.has(task.deterministicFixture.patch))
+                collector.add("FIXTURE_DUPLICATE", `${base}.deterministicFixture.patch`, "tasks must not share a reference patch.");
+            fixtureIds.add(task.deterministicFixture.id);
+            fixturePatches.add(task.deterministicFixture.patch);
+        }
+    });
+    for (const [project, localities] of perProject) {
+        const sorted = [...localities].sort();
+        if (!sameList(sorted, ["broad-change", "cross-module", "localized"])) {
+            collector.add("LOCALITY_DISTRIBUTION", "tasks", `project ${project} must have exactly one localized, one cross-module and one broad-change task.`);
+        }
+    }
+    return tasks;
+}
+/**
+ * Structural validation of the agent-success implementation corpus. Pure: it only inspects the two parsed
+ * catalogs. It never touches the filesystem and never applies or executes a reference patch; that is dynamic
+ * benchmark acceptance and is tested separately. The task shape is validated by validateAgentSuccessTask.
+ */
+export function validateAgentSuccessCorpus(input) {
+    const collector = new Collector();
+    const profiles = readProfiles(collector, input.profiles);
+    validateProfileSet(collector, profiles);
+    const tasks = validateTasks(collector, input.tasks, profiles);
+    if (collector.issues.length > 0)
+        return { ok: false, issues: collector.issues };
+    const corpus = { profiles, tasks };
+    return { ok: true, corpus };
+}
+export function assertAgentSuccessCorpus(input) {
+    const result = validateAgentSuccessCorpus(input);
+    if (!result.ok)
+        throw new AgentSuccessCorpusError(result.issues);
+    return result.corpus;
+}