@mcoda/codali 0.1.134 → 0.1.135-codali.4

package/dist/eval/TerminalCorpusRunner.d.ts

@@ -0,0 +1,36 @@
+import { type EffortLevel } from "../config/OrchestrationConfig.js";
+import { type TerminalCorpusTask } from "./TerminalTaskCorpus.js";
+export interface TerminalGrade {
+    passed: boolean;
+    contractPassed: boolean;
+    publicTestsPassed: boolean;
+    publicTestCount: number;
+    protectedFilesPreserved: boolean;
+    scopePreserved: boolean;
+    testsExtended: boolean;
+    changedPaths: string[];
+    failures: string[];
+}
+/** Detect edits to the executed build between cases. Use an isolated package install for dependency pinning. */
+export declare function terminalClientBuildHash(root?: string): Promise<string>;
+export declare function materializeTerminalTask(task: TerminalCorpusTask, workspace: string): Promise<void>;
+export declare function gradeTerminalTask(task: TerminalCorpusTask, workspace: string): Promise<TerminalGrade>;
+export interface TerminalCorpusRunOptions {
+    configPath: string;
+    outputRoot: string;
+    effort?: EffortLevel;
+    taskIds?: string[];
+    timeoutMs?: number;
+    signal?: AbortSignal;
+    onProgress?: (record: Record<string, unknown>) => void;
+    /** Trusted, versioned harness input, not model-supplied configuration. */
+    corpus?: {
+        version: string;
+        hash: string;
+        tasks: TerminalCorpusTask[];
+        limitations: string[];
+    };
+}
+/** Serial admission avoids confounding model comparisons with self-induced queue contention. */
+export declare function runTerminalCorpus(options: TerminalCorpusRunOptions): Promise<void>;
+//# sourceMappingURL=TerminalCorpusRunner.d.ts.map

package/dist/eval/TerminalCorpusRunner.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"TerminalCorpusRunner.d.ts","sourceRoot":"","sources":["../../src/eval/TerminalCorpusRunner.ts"],"names":[],"mappings":"AASA,OAAO,EAA2B,KAAK,WAAW,EAAE,MAAM,kCAAkC,CAAC;AAC7F,OAAO,EAAmE,KAAK,kBAAkB,EAAE,MAAM,yBAAyB,CAAC;AAEnI,MAAM,WAAW,aAAa;IAC5B,MAAM,EAAE,OAAO,CAAC;IAChB,cAAc,EAAE,OAAO,CAAC;IACxB,iBAAiB,EAAE,OAAO,CAAC;IAC3B,eAAe,EAAE,MAAM,CAAC;IACxB,uBAAuB,EAAE,OAAO,CAAC;IACjC,cAAc,EAAE,OAAO,CAAC;IACxB,aAAa,EAAE,OAAO,CAAC;IACvB,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,QAAQ,EAAE,MAAM,EAAE,CAAC;CACpB;AAGD,gHAAgH;AAChH,wBAAsB,uBAAuB,CAAC,IAAI,SAAiD,GAAG,OAAO,CAAC,MAAM,CAAC,CAYpH;AACD,wBAAsB,uBAAuB,CAAC,IAAI,EAAE,kBAAkB,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAMxG;AAiBD,wBAAsB,iBAAiB,CAAC,IAAI,EAAE,kBAAkB,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,CAAC,CAyC3G;AAED,MAAM,WAAW,wBAAwB;IACvC,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,UAAU,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,IAAI,CAAC;IACvD,0EAA0E;IAC1E,MAAM,CAAC,EAAE;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,kBAAkB,EAAE,CAAC;QAAC,WAAW,EAAE,MAAM,EAAE,CAAA;KAAE,CAAC;CAChG;AAED,gGAAgG;AAChG,wBAAsB,iBAAiB,CAAC,OAAO,EAAE,wBAAwB,GAAG,OAAO,CAAC,IAAI,CAAC,CAkExF"}

package/dist/eval/TerminalCorpusRunner.js

@@ -0,0 +1,222 @@
+import { createHash, randomUUID } from "node:crypto";
+import { spawn } from "node:child_process";
+import { promises as fs, createWriteStream } from "node:fs";
+import { finished } from "node:stream/promises";
+import os from "node:os";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+import { createIsolatedExecutor } from "../tools/shell/IsolatedExecutor.js";
+import { runProcess } from "../tools/shell/AsyncProcess.js";
+import { loadOrchestrationConfig } from "../config/OrchestrationConfig.js";
+import { terminalTaskCorpus, terminalCorpusHash, TERMINAL_CORPUS_VERSION } from "./TerminalTaskCorpus.js";
+const hash = (text) => createHash("sha256").update(text).digest("hex");
+/** Detect edits to the executed build between cases. Use an isolated package install for dependency pinning. */
+export async function terminalClientBuildHash(root = fileURLToPath(new URL("../", import.meta.url))) {
+    const digest = createHash("sha256");
+    const visit = async (relative) => {
+        const entries = await fs.readdir(path.join(root, relative), { withFileTypes: true });
+        for (const entry of entries.sort((a, b) => a.name.localeCompare(b.name))) {
+            const file = path.join(relative, entry.name);
+            if (entry.isSymbolicLink())
+                throw new Error(`Evaluation build contains a symlink: ${file}`);
+            if (entry.isDirectory()) {
+                if (entry.name !== "__tests__")
+                    await visit(file);
+            }
+            else if (/\.(?:js|json|node)$/.test(entry.name))
+                digest.update(file).update("\0").update(await fs.readFile(path.join(root, file))).update("\0");
+        }
+    };
+    await visit("");
+    return digest.digest("hex");
+}
+export async function materializeTerminalTask(task, workspace) {
+    await fs.mkdir(workspace, { mode: 0o700 });
+    for (const [file, content] of Object.entries(task.files)) {
+        await fs.mkdir(path.dirname(path.join(workspace, file)), { recursive: true });
+        await fs.writeFile(path.join(workspace, file), content, { flag: "wx", mode: 0o600 });
+    }
+}
+/** Only regular, bounded files enter the grading copy; it never executes agent code on the host. */
+async function workspaceFiles(root, relative = "", output = new Map()) {
+    for (const entry of await fs.readdir(path.join(root, relative), { withFileTypes: true })) {
+        if (!relative && [".git", ".mcoda"].includes(entry.name))
+            continue;
+        const file = path.posix.join(relative, entry.name);
+        if (entry.isDirectory())
+            await workspaceFiles(root, file, output);
+        else {
+            if (!entry.isFile())
+                throw new Error(`Nonregular evaluation file: ${file}`);
+            if (output.size >= 200 || (await fs.stat(path.join(root, file))).size > 1048576)
+                throw new Error("Evaluation workspace exceeds file bounds");
+            output.set(file, await fs.readFile(path.join(root, file), "utf8"));
+        }
+    }
+    return output;
+}
+export async function gradeTerminalTask(task, workspace) {
+    const result = { passed: false, contractPassed: false, publicTestsPassed: false, publicTestCount: 0,
+        protectedFilesPreserved: false, scopePreserved: false, testsExtended: false, changedPaths: [], failures: [] };
+    const grading = await fs.mkdtemp(path.join(os.tmpdir(), "codali-corpus-grade-"));
+    try {
+        const files = await workspaceFiles(workspace);
+        result.changedPaths = [...new Set([...files.keys(), ...Object.keys(task.files)])]
+            .filter(file => files.get(file) !== task.files[file]).sort();
+        result.scopePreserved = result.changedPaths.every(file => task.allowedChanges.includes(file));
+        result.protectedFilesPreserved = Object.entries(task.files).filter(([file]) => !task.allowedChanges.includes(file)).every(([file, content]) => files.get(file) === content);
+        const requiredExtensions = task.grading?.testFilesToExtend ?? ["test/task.test.mjs"];
+        result.testsExtended = requiredExtensions.every(file => files.has(file) && files.get(file) !== task.files[file]);
+        for (const [file, content] of files) {
+            await fs.mkdir(path.dirname(path.join(grading, file)), { recursive: true });
+            await fs.writeFile(path.join(grading, file), content, { flag: "wx", mode: 0o600 });
+        }
+        const nonce = randomUUID();
+        const hidden = `.grade-${nonce}.mjs`;
+        const entrypoint = task.grading?.entrypoint === undefined ? "src/task.mjs" : task.grading.entrypoint;
+        const implementationImport = entrypoint === null ? "" : `import * as m from ${JSON.stringify(`./${entrypoint}`)};\n`;
+        await fs.writeFile(path.join(grading, hidden), `import assert from 'node:assert/strict';\n${implementationImport}${task.hiddenChecks}\nconsole.log(${JSON.stringify(nonce)});\n`, { flag: "wx", mode: 0o600 });
+        const execute = createIsolatedExecutor({ network: false });
+        const context = { workspaceRoot: grading, allowedWritePaths: [], shellTimeoutMs: 15000, shellMaxOutputBytes: 131072 };
+        try {
+            const outcome = await execute(process.execPath, [hidden], context);
+            result.contractPassed = outcome.output.trim() === nonce;
+            if (!result.contractPassed)
+                result.failures.push("Hidden contract did not finish with its completion receipt");
+        }
+        catch (error) {
+            result.failures.push(`contract: ${String(error).slice(0, 2000)}`);
+        }
+        try {
+            const outcome = await execute(process.execPath, ["--test", ...(task.grading?.publicTestFiles ?? ["test/task.test.mjs"])], context);
+            result.publicTestCount = Number(outcome.output.match(/# tests (\d+)/)?.[1] ?? 0);
+            const minimumTests = task.grading?.minimumPublicTests ?? 2;
+            result.publicTestsPassed = result.publicTestCount >= minimumTests;
+            if (!result.publicTestsPassed)
+                result.failures.push(`Public suite must contain at least ${minimumTests} passing tests`);
+        }
+        catch (error) {
+            result.failures.push(`public tests: ${String(error).slice(0, 2000)}`);
+        }
+        if (!result.scopePreserved)
+            result.failures.push("Changes escaped the permitted paths");
+        if (!result.protectedFilesPreserved)
+            result.failures.push("User notes or package manifest changed");
+        if (!result.testsExtended)
+            result.failures.push("Public tests were not extended");
+        result.passed = result.contractPassed && result.publicTestsPassed && result.scopePreserved && result.protectedFilesPreserved && result.testsExtended;
+    }
+    catch (error) {
+        result.failures.push(String(error).slice(0, 2000));
+    }
+    finally {
+        await fs.rm(grading, { recursive: true, force: true });
+    }
+    return result;
+}
+/** Serial admission avoids confounding model comparisons with self-induced queue contention. */
+export async function runTerminalCorpus(options) {
+    const resolved = await loadOrchestrationConfig(options.configPath);
+    const clientBuildHash = await terminalClientBuildHash();
+    const all = options.corpus?.tasks ?? terminalTaskCorpus();
+    if (options.taskIds?.some(id => !all.some(task => task.id === id)))
+        throw new Error("Unknown terminal corpus task ID");
+    const selected = options.taskIds ? all.filter(task => options.taskIds.includes(task.id)) : all.filter(task => task.split === "held-out");
+    await fs.mkdir(options.outputRoot, { mode: 0o700 });
+    const manifest = { schemaVersion: 1, corpusVersion: options.corpus?.version ?? TERMINAL_CORPUS_VERSION, corpusHash: options.corpus?.hash ?? terminalCorpusHash(),
+        configHash: resolved.hash, clientBuildHash, nodeVersion: process.version, platform: process.platform, architecture: process.arch,
+        effort: options.effort ?? "high", taskIds: selected.map(task => task.id), splits: [...new Set(selected.map(task => task.split))], startedAt: new Date().toISOString(),
+        limitations: options.corpus?.limitations ?? ["60 seeded cases across 12 correlated coding families; report families separately. Does not qualify long-session planning, media or production coexistence."] };
+    await fs.writeFile(path.join(options.outputRoot, "manifest.json"), JSON.stringify(manifest, null, 2) + "\n", { flag: "wx", mode: 0o600 });
+    const cli = fileURLToPath(new URL("../cli.js", import.meta.url));
+    for (const task of selected) {
+        options.signal?.throwIfAborted();
+        const current = await loadOrchestrationConfig(options.configPath);
+        if (current.hash !== resolved.hash)
+            throw new Error("Evaluation configuration changed between tasks; start a separately identified run");
+        if (await terminalClientBuildHash() !== clientBuildHash)
+            throw new Error("Evaluation client build changed between tasks; use a frozen package install");
+        const workspace = path.join(options.outputRoot, task.id);
+        await materializeTerminalTask(task, workspace);
+        await runProcess("git", ["-c", "core.hooksPath=/dev/null", "init", "--quiet"], { workspaceRoot: workspace, shellTimeoutMs: 10000 });
+        await runProcess("git", ["-c", "core.hooksPath=/dev/null", "add", "--", ...Object.keys(task.files)], { workspaceRoot: workspace, shellTimeoutMs: 10000 });
+        await runProcess("git", ["-c", "core.hooksPath=/dev/null", "-c", "user.name=Codali evaluation", "-c", "user.email=codali-evaluation@localhost", "-c", "commit.gpgSign=false", "commit", "--quiet", "-m", "Evaluation fixture baseline"], { workspaceRoot: workspace, shellTimeoutMs: 10000 });
+        const startedAt = new Date().toISOString();
+        const logPath = path.join(options.outputRoot, `${task.id}.jsonl`);
+        options.onProgress?.({ type: "task_start", taskId: task.id, workspace, startedAt });
+        const log = createWriteStream(logPath, { flags: "wx", mode: 0o600 });
+        let bytes = 0;
+        let overflow = false;
+        let timedOut = false;
+        let launchError;
+        const child = spawn(process.execPath, [cli, "exec", task.prompt, "--workspace-root", workspace, "--config", path.resolve(options.configPath),
+            "--permissions", "workspace", "--effort", options.effort ?? "high", "--json"], {
+            cwd: workspace, env: { ...process.env }, detached: process.platform !== "win32", stdio: ["ignore", "pipe", "pipe"],
+        });
+        let escalation;
+        const kill = (signal) => {
+            if (!child.pid)
+                return;
+            try {
+                if (process.platform !== "win32")
+                    process.kill(-child.pid, signal);
+                else
+                    child.kill(signal);
+            }
+            catch (error) {
+                if (error.code !== "ESRCH")
+                    throw error;
+            }
+        };
+        const stop = () => { kill("SIGTERM"); escalation ?? (escalation = setTimeout(() => kill("SIGKILL"), 3000)); escalation.unref(); };
+        const timer = setTimeout(() => { timedOut = true; stop(); }, options.timeoutMs ?? 900000);
+        const capture = (chunk) => {
+            bytes += chunk.length;
+            if (bytes <= 50 * 1024 * 1024) {
+                if (!log.write(chunk)) {
+                    child.stdout.pause();
+                    child.stderr.pause();
+                    log.once("drain", () => { child.stdout.resume(); child.stderr.resume(); });
+                }
+            }
+            else {
+                overflow = true;
+                stop();
+            }
+        };
+        child.stdout.on("data", capture);
+        child.stderr.on("data", capture);
+        child.once("error", error => { launchError = error.message; });
+        options.signal?.addEventListener("abort", stop, { once: true });
+        if (options.signal?.aborted)
+            stop();
+        const exitCode = await new Promise(resolve => { child.once("close", resolve); });
+        clearTimeout(timer);
+        if (escalation)
+            clearTimeout(escalation);
+        options.signal?.removeEventListener("abort", stop);
+        log.end();
+        await finished(log);
+        const events = [];
+        const contents = await fs.readFile(logPath, "utf8");
+        for (const line of contents.split("\n")) {
+            try {
+                const value = JSON.parse(line);
+                if (value && typeof value === "object")
+                    events.push(value);
+            }
+            catch { /* stderr diagnostics are not events */ }
+        }
+        const grade = await gradeTerminalTask(task, workspace);
+        const result = { taskId: task.id, family: task.family, variant: task.variant, split: task.split, startedAt, endedAt: new Date().toISOString(), exitCode,
+            timedOut, overflow, launchError, passed: grade.passed && exitCode === 0 && !timedOut && !overflow && !launchError,
+            grade, logSha256: hash(contents), configHash: resolved.hash, clientBuildHash,
+            turnResult: [...events].reverse().find(event => event.type === "turn_result")?.data,
+            usage: [...events].reverse().find(event => event.type === "usage")?.data,
+            routes: events.filter(event => event.type === "model_route").map(event => event.data),
+        };
+        await fs.writeFile(path.join(options.outputRoot, `${task.id}.result.json`), JSON.stringify(result, null, 2) + "\n", { flag: "wx", mode: 0o600 });
+        await fs.appendFile(path.join(options.outputRoot, "results.jsonl"), JSON.stringify(result) + "\n", { mode: 0o600 });
+        options.onProgress?.({ type: "task_result", taskId: task.id, passed: result.passed, exitCode, failures: grade.failures, endedAt: result.endedAt });
+    }
+}

package/dist/eval/TerminalTaskCorpus.d.ts

@@ -0,0 +1,26 @@
+export interface TerminalCorpusTask {
+    id: string;
+    family: string;
+    variant: number;
+    split: "development" | "held-out";
+    prompt: string;
+    files: Record<string, string>;
+    allowedChanges: string[];
+    /** Never materialized in the agent workspace. Used only to qualify the harness. */
+    reference: Record<string, string>;
+    /** Evaluator-owned checks, materialized only in an isolated grading copy after execution. */
+    hiddenChecks: string;
+    /** Evaluator-owned configuration; never exposed as an editable workspace file. */
+    grading?: {
+        entrypoint?: string | null;
+        publicTestFiles?: string[];
+        minimumPublicTests?: number;
+        testFilesToExtend?: string[];
+    };
+}
+export declare const TERMINAL_CORPUS_VERSION = "terminal-coding-v3";
+/** One development case plus five held-out seeds per family. These are correlated variants,
+ * not 60 independent problem domains. Report family results and retain the long-session corpus. */
+export declare const terminalTaskCorpus: () => TerminalCorpusTask[];
+export declare const terminalCorpusHash: () => string;
+//# sourceMappingURL=TerminalTaskCorpus.d.ts.map

package/dist/eval/TerminalTaskCorpus.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"TerminalTaskCorpus.d.ts","sourceRoot":"","sources":["../../src/eval/TerminalTaskCorpus.ts"],"names":[],"mappings":"AAEA,MAAM,WAAW,kBAAkB;IACjC,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,aAAa,GAAG,UAAU,CAAC;IAClC,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC9B,cAAc,EAAE,MAAM,EAAE,CAAC;IACzB,mFAAmF;IACnF,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAClC,6FAA6F;IAC7F,YAAY,EAAE,MAAM,CAAC;IACrB,kFAAkF;IAClF,OAAO,CAAC,EAAE;QACR,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;QAC3B,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;QAC3B,kBAAkB,CAAC,EAAE,MAAM,CAAC;QAC5B,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAC;KAC9B,CAAC;CACH;AAiGD,eAAO,MAAM,uBAAuB,uBAAuB,CAAC;AAC5D;mGACmG;AACnG,eAAO,MAAM,kBAAkB,QAAO,kBAAkB,EAYrD,CAAC;AACJ,eAAO,MAAM,kBAAkB,QAAO,MAAsI,CAAC"}

package/dist/eval/TerminalTaskCorpus.js

@@ -0,0 +1,105 @@
+import { createHash } from "node:crypto";
+const families = [
+    {
+        id: "clamp", contract: "Export clamp(value, minimum, maximum). Require finite numbers, reject minimum > maximum with RangeError, reject other invalid inputs with TypeError, preserve fractional values and clamp inclusively. Do not coerce strings.",
+        starter: "export const clamp = (value, minimum, maximum) => Math.max(minimum, value);",
+        reference: "export function clamp(value, minimum, maximum) { if (![value,minimum,maximum].every(Number.isFinite)) throw new TypeError('finite numbers required'); if (minimum > maximum) throw new RangeError('reversed range'); return Math.min(maximum, Math.max(minimum,value)); }",
+        smoke: "assert.equal(m.clamp(20, 0, 10), 10);",
+        checks: n => `for (const [v,lo,hi,want] of [[${n * 4},0,${n},${n}],[-${n},0,${n},0],[0.25,0,1,0.25],[${n},${n},${n},${n}]]) assert.equal(m.clamp(v,lo,hi),want); for (const value of [NaN,Infinity,-Infinity,'2',null,undefined]) assert.throws(()=>m.clamp(value,0,3),TypeError); assert.throws(()=>m.clamp(1,4,2),RangeError);`,
+    },
+    {
+        id: "chunk", contract: "Export chunk(items, size). Return new arrays of consecutive items, including a shorter final chunk. Empty input returns []. Reject non-array items with TypeError and nonpositive/noninteger sizes with RangeError. Never mutate the input.",
+        starter: "export const chunk = (items, size) => [items.slice(0,size)];",
+        reference: "export function chunk(items,size) { if (!Array.isArray(items)) throw new TypeError('array required'); if (!Number.isInteger(size)||size<1) throw new RangeError('positive integer required'); const out=[]; for(let i=0;i<items.length;i+=size) out.push(items.slice(i,i+size)); return out; }",
+        smoke: "assert.deepEqual(m.chunk([1,2,3],2),[[1,2],[3]]);",
+        checks: n => `const items=Object.freeze(Array.from({length:${n * 3 + 1}},(_,i)=>i)); const out=m.chunk(items,${n}); assert.deepEqual(out.flat(),items); assert.equal(out.length,4); assert.equal(out.at(-1).length,1); assert.notEqual(out[0],items); assert.deepEqual(m.chunk([],2),[]); for(const size of [0,-1,1.5,NaN,Infinity,'2']) assert.throws(()=>m.chunk([],size),RangeError); assert.throws(()=>m.chunk(null,2),TypeError);`,
+    },
+    {
+        id: "stable-unique", contract: "Export uniqueBy(items, key). key is a property name. Keep the first item for each key value using Map equality (including NaN and object identity), preserve order and references, and do not mutate the input. Missing keys are one undefined group. Reject non-array input.",
+        starter: "export const uniqueBy = (items,key) => [...new Set(items)];",
+        reference: "export function uniqueBy(items,key) { if(!Array.isArray(items)) throw new TypeError('array required'); const seen=new Set(); return items.filter(item=>{ const value=item[key]; if(seen.has(value)) return false; seen.add(value); return true; }); }",
+        smoke: "assert.deepEqual(m.uniqueBy([{id:1,v:'a'},{id:1,v:'b'}],'id'),[{id:1,v:'a'}]);",
+        checks: n => `const key='key${n}', obj={}; const values=[${n},${n},NaN,NaN,obj,obj,{},undefined,undefined]; const items=Object.freeze(values.map((v,i)=>Object.freeze({[key]:v,i}))); const out=m.uniqueBy(items,key); assert.deepEqual(out.map(v=>v.i),[0,2,4,6,7]); assert.equal(out[0],items[0]); assert.deepEqual(m.uniqueBy([{x:1},{x:2}],'missing'),[{x:1}]); assert.throws(()=>m.uniqueBy(null,key));`,
+    },
+    {
+        id: "lower-bound", contract: "Export lowerBound(sorted, value), returning the first index whose number is >= value or the array length if absent. Sorted input is ascending and finite. Use logarithmic indexed reads, handle duplicates/empty input, and do not mutate the input.",
+        starter: "export const lowerBound = (sorted,value) => sorted.findIndex(x=>x>=value);",
+        reference: "export function lowerBound(sorted,value) { let lo=0,hi=sorted.length; while(lo<hi){const mid=lo+Math.floor((hi-lo)/2); if(sorted[mid]<value)lo=mid+1;else hi=mid;} return lo; }",
+        smoke: "assert.equal(m.lowerBound([1,2,2,5],8),4);",
+        checks: n => `assert.equal(m.lowerBound([],${n}),0); const a=Object.freeze([-${n},0,${n},${n},${n + 1}]); assert.equal(m.lowerBound(a,${n}),2); assert.equal(m.lowerBound(a,-100),0); assert.equal(m.lowerBound(a,100),5); let reads=0; const big=new Proxy(Array.from({length:65536},(_,i)=>i*2),{get(t,k){if(typeof k==='string'&&/^[0-9]+$/.test(k))reads++;return Reflect.get(t,k);}}); assert.equal(m.lowerBound(big,65000+${n}),Math.ceil((65000+${n})/2)); assert.ok(reads>0&&reads<=20,'must use logarithmic indexed reads');`,
+    },
+    {
+        id: "task-dag", contract: "Export orderTasks(tasks), where each task has unique string id and dependsOn string[]. Return IDs in topological order, choosing the lexicographically smallest currently-ready ID each time. Reject duplicate IDs, unknown prerequisites and cycles. Do not mutate tasks or dependency arrays.",
+        starter: "export const orderTasks = tasks => tasks.map(t=>t.id).sort();",
+        reference: "export function orderTasks(tasks) { const by=new Map(); for(const t of tasks){if(by.has(t.id))throw new Error('duplicate');by.set(t.id,t);} for(const t of tasks)for(const d of t.dependsOn)if(!by.has(d))throw new Error('missing'); const done=new Set(),out=[]; while(out.length<tasks.length){const ready=tasks.filter(t=>!done.has(t.id)&&t.dependsOn.every(d=>done.has(d))).map(t=>t.id).sort();if(!ready.length)throw new Error('cycle');out.push(ready[0]);done.add(ready[0]);}return out; }",
+        smoke: "assert.deepEqual(m.orderTasks([{id:'a',dependsOn:['z']},{id:'z',dependsOn:[]}]),['z','a']);",
+        checks: n => `const p='p${n}';const tasks=Object.freeze([{id:'a',dependsOn:Object.freeze([p])},{id:'b',dependsOn:Object.freeze([])},{id:p,dependsOn:Object.freeze([])}].map(Object.freeze));assert.deepEqual(m.orderTasks(tasks),['b',p,'a']);assert.deepEqual(m.orderTasks([]),[]);assert.throws(()=>m.orderTasks([{id:'a',dependsOn:['missing']}]));assert.throws(()=>m.orderTasks([{id:'a',dependsOn:[]},{id:'a',dependsOn:[]}]));assert.throws(()=>m.orderTasks([{id:'a',dependsOn:['b']},{id:'b',dependsOn:['a']}]));assert.throws(()=>m.orderTasks([{id:'a',dependsOn:['a']}]));`,
+    },
+    {
+        id: "relative-path", contract: "Export safePath(value). Accept only a nonempty POSIX relative path whose slash-separated components are nonempty and are neither the exact component '.' nor '..'. Reject absolute paths, backslashes, control characters U+0000-U+001F and U+007F-U+009F inclusive, drive prefixes and percent escapes (no decoding). Return valid paths unchanged; all invalid values throw TypeError. Unicode names and ordinary dots/hyphens within names are allowed.",
+        starter: "export const safePath = value => value.replace('../','');",
+        reference: "export function safePath(value) { if(typeof value!=='string'||!value||/[\\\\%\\x00-\\x1f\\x7f-\\x9f]/.test(value)||/^[A-Za-z]:/.test(value)||value.split('/').some(x=>!x||x==='.'||x==='..'))throw new TypeError('unsafe path');return value; }",
+        smoke: "assert.throws(()=>m.safePath('../secret'),TypeError);",
+        checks: n => `for(const value of ['src/file${n}.mjs','a.b/c-d','ü/文.txt','.hidden','..hidden','...','a\\u00a0b'])assert.equal(m.safePath(value),value);for(const value of ['',null,42,'/etc/passwd','a//b','a/./b','a/../b','a/','C:/x','a\\\\b','%2e%2e/x','x\\u0000y','x\\u001fy','x\\u007fy','x\\u0080y','x\\u009fy','a\\nb'])assert.throws(()=>m.safePath(value),TypeError);`,
+    },
+    {
+        id: "journal-tail", contract: "Export readJournal(text). Parse newline-terminated JSON object records with contiguous positive integer seq starting at 1. Ignore any non-newline-terminated final fragment, even if valid JSON. Skip blank complete lines. Reject malformed complete JSON, arrays/null/nonobjects and missing/duplicate/gapped seq. Return records without changing their fields.",
+        starter: "export const readJournal = text => text.trim().split('\\n').filter(Boolean).map(JSON.parse);",
+        reference: "export function readJournal(text){const lines=text.split('\\n');lines.pop();const out=[];for(const line of lines){if(!line.trim())continue;const value=JSON.parse(line);if(!value||Array.isArray(value)||typeof value!=='object'||value.seq!==out.length+1)throw new Error('invalid sequence');out.push(value);}return out;}",
+        smoke: "assert.deepEqual(m.readJournal('{\"seq\":1}\\n{\"seq\":'),[{seq:1}]);",
+        checks: n => `const items=Array.from({length:${n}},(_,i)=>({seq:i+1,value:i}));const full=items.map(JSON.stringify).join('\\n')+'\\n';assert.deepEqual(m.readJournal(full+'{'),items);assert.deepEqual(m.readJournal(full+JSON.stringify({seq:${n + 1}})),items);assert.deepEqual(m.readJournal(''),[]);assert.deepEqual(m.readJournal('  \\n'+full),items);for(const text of ['{bad}\\n','null\\n','[]\\n','{}\\n','{\"seq\":2}\\n','{\"seq\":1}\\n{\"seq\":1}\\n'])assert.throws(()=>m.readJournal(text));`,
+    },
+    {
+        id: "ttl-lru", contract: "Export createCache(capacity, now), with get(key), set(key,value,ttlMs) and size(). Capacity is a positive integer; ttlMs must be positive finite. Expiry is inclusive (now >= expires means absent). Reads refresh LRU order but not expiry. On set, remove expired items and evict least recently used to capacity. size() excludes expired entries. Use injected now(), never the wall clock.",
+        starter: "export function createCache(capacity,now){const map=new Map();return{get:key=>map.get(key),set:(key,value,ttl)=>map.set(key,value),size:()=>map.size};}",
+        reference: "export function createCache(capacity,now){if(!Number.isInteger(capacity)||capacity<1)throw new RangeError('capacity');const map=new Map();const prune=()=>{const at=now();for(const[k,v]of map)if(at>=v.expires)map.delete(k);};return{get(key){prune();const v=map.get(key);if(!v)return undefined;map.delete(key);map.set(key,v);return v.value;},set(key,value,ttl){if(!Number.isFinite(ttl)||ttl<=0)throw new RangeError('ttl');prune();map.delete(key);map.set(key,{value,expires:now()+ttl});while(map.size>capacity)map.delete(map.keys().next().value);},size(){prune();return map.size;}};}",
+        smoke: "let now=0;const cache=m.createCache(1,()=>now);cache.set('a',1,10);now=10;assert.equal(cache.get('a'),undefined);",
+        checks: n => `let at=${n};const c=m.createCache(2,()=>at);c.set('a',1,10);c.set('b',2,20);assert.equal(c.get('a'),1);c.set('c',3,30);assert.equal(c.get('b'),undefined);at+=10;assert.equal(c.get('a'),undefined);assert.equal(c.size(),1);c.set('c',4,5);at+=5;assert.equal(c.size(),0);assert.throws(()=>m.createCache(0,()=>0));for(const ttl of [0,-1,Infinity,NaN])assert.throws(()=>c.set('x',1,ttl));`,
+    },
+    {
+        id: "budget-receipts", contract: "Export createBudget(maxCalls,maxTokens) returning reserve(id,tokens), settle(id,tokens), snapshot(). Positive safe-integer caps required. Nonnegative safe-integer token amounts required. reserve rejects reused IDs and cap breaches without mutation. Reservations increment calls and tokens immediately. calls is cumulative and is never decremented by settlement. settle only once for a pending ID replaces its token reservation with actual token usage, even if reported usage exceeds the cap (then future admission fails). snapshot is a fresh {calls,tokens,pending} object. A lost response retains its reservation.",
+        starter: "export function createBudget(){return{reserve(){},settle(){},snapshot(){return{calls:0,tokens:0,pending:0};}};}",
+        reference: "export function createBudget(maxCalls,maxTokens){if(![maxCalls,maxTokens].every(x=>Number.isSafeInteger(x)&&x>0))throw new RangeError('caps');let calls=0,tokens=0;const all=new Set(),pending=new Map();const check=n=>{if(!Number.isSafeInteger(n)||n<0)throw new RangeError('tokens');};return{reserve(id,n){check(n);if(all.has(id)||calls>=maxCalls||tokens+n>maxTokens)throw new Error('admission');all.add(id);pending.set(id,n);calls++;tokens+=n;},settle(id,n){check(n);if(!pending.has(id))throw new Error('receipt');tokens+=n-pending.get(id);pending.delete(id);},snapshot(){return{calls,tokens,pending:pending.size};}};}",
+        smoke: "const b=m.createBudget(1,10);b.reserve('a',5);assert.throws(()=>b.reserve('b',1));",
+        checks: n => `const b=m.createBudget(3,${n * 10});b.reserve('a',${n});b.reserve('lost',${n});assert.deepEqual(b.snapshot(),{calls:2,tokens:${2 * n},pending:2});assert.throws(()=>b.reserve('a',1));assert.throws(()=>b.settle('missing',1));b.settle('a',${n * 11});assert.deepEqual(b.snapshot(),{calls:2,tokens:${12 * n},pending:1});assert.throws(()=>b.reserve('next',0));assert.throws(()=>b.settle('a',0));const view=b.snapshot();view.tokens=0;assert.equal(b.snapshot().tokens,${12 * n});for(const value of [-1,NaN,Infinity,1.5])assert.throws(()=>b.settle('lost',value));`,
+    },
+    {
+        id: "binding-resolution", contract: "Export resolveBinding(config,name). config has models and bindings object maps. A binding has either nonempty candidates of model IDs or useBinding naming another binding. Resolve aliases; reject unknown bindings/models, cycles, mixed candidates/useBinding and duplicate candidate IDs. Preserve candidate order. Return a new array of model objects deeply cloned from config, so caller mutation cannot change config. Do not mutate config.",
+        starter: "export const resolveBinding = (config,name) => config.bindings[name].candidates.map(id=>config.models[id]);",
+        reference: "export function resolveBinding(config,name){const seen=new Set();while(true){if(seen.has(name)||!Object.hasOwn(config.bindings,name))throw new Error('binding');seen.add(name);const b=config.bindings[name];if(b.useBinding!==undefined){if(b.candidates!==undefined)throw new Error('mixed');name=b.useBinding;continue;}if(!Array.isArray(b.candidates)||!b.candidates.length||new Set(b.candidates).size!==b.candidates.length)throw new Error('candidates');return b.candidates.map(id=>{if(!Object.hasOwn(config.models,id))throw new Error('model');return structuredClone(config.models[id]);});}}",
+        smoke: "assert.deepEqual(m.resolveBinding({models:{a:{id:'a'}},bindings:{lead:{useBinding:'base'},base:{candidates:['a']}}},'lead'),[{id:'a'}]);",
+        checks: n => `const c={models:{a:{id:'a',limits:{n:${n}}},b:{id:'b'}},bindings:{lead:{useBinding:'base'},base:{candidates:['b','a']}}};const r=m.resolveBinding(c,'lead');assert.deepEqual(r.map(x=>x.id),['b','a']);r[1].limits.n=0;assert.equal(c.models.a.limits.n,${n});assert.throws(()=>m.resolveBinding(c,'toString'));for(const bindings of [{a:{useBinding:'b'},b:{useBinding:'a'}},{a:{candidates:['missing']}},{a:{candidates:['a','a']}},{a:{useBinding:'b',candidates:['a']}},{a:{candidates:[]}}])assert.throws(()=>m.resolveBinding({models:c.models,bindings},'a'));`,
+    },
+    {
+        id: "bounded-retry", contract: "Export async retry(operation,{attempts,delay,signal}). Call operation with a 1-based attempt. Retry only errors whose retryable is exactly true; attempts is a positive integer total-attempt cap. Await injected delay(attempt,signal) between attempts only. Before every operation and after every delay check signal and throw signal.reason if aborted. Never retry cancellation or a delay failure. Return first successful result; exhausted retries throw the last operation error.",
+        starter: "export const retry = async operation => operation(1);",
+        reference: "export async function retry(operation,{attempts,delay,signal}){if(!Number.isInteger(attempts)||attempts<1)throw new RangeError('attempts');for(let attempt=1;attempt<=attempts;attempt++){signal?.throwIfAborted();try{return await operation(attempt);}catch(error){signal?.throwIfAborted();if(error?.retryable!==true||attempt===attempts)throw error;await delay(attempt,signal);signal?.throwIfAborted();}}}",
+        smoke: "let calls=0;assert.equal(await m.retry(async()=>{if(++calls<2)throw Object.assign(new Error('retry'),{retryable:true});return 7;},{attempts:2,delay:async()=>{}}),7);",
+        checks: n => `let calls=0;const delays=[];const e=Object.assign(new Error('temporary'),{retryable:true});assert.equal(await m.retry(async attempt=>{calls++;if(attempt<${n})throw e;return attempt;},{attempts:${n},delay:async attempt=>{delays.push(attempt);}}),${n});assert.equal(calls,${n});assert.deepEqual(delays,Array.from({length:${n - 1}},(_,i)=>i+1));const ac=new AbortController();const stop=new Error('stop');calls=0;await assert.rejects(m.retry(async()=>{calls++;throw e;},{attempts:3,signal:ac.signal,delay:async()=>ac.abort(stop)}),x=>x===stop);assert.equal(calls,1);calls=0;await assert.rejects(m.retry(async()=>{calls++;throw new Error('permanent');},{attempts:3,delay:async()=>{throw new Error('wrong');}}),/permanent/);assert.equal(calls,1);`,
+    },
+    {
+        id: "phase-coverage", contract: "Export validatePlan(requirements,tasks). Requirements is a list of unique string IDs. Each task has unique id, requirementIds and dependsOn arrays. Reject duplicate requirement/task IDs, missing requirement coverage, unknown requirement refs, unknown task dependencies, and dependency cycles. Empty requirements/tasks is valid. Return {covered: requirements.length, tasks: tasks.length}. Do not mutate input. A requirement may be covered by several tasks.",
+        starter: "export const validatePlan = (requirements,tasks) => ({covered:requirements.length,tasks:tasks.length});",
+        reference: "export function validatePlan(requirements,tasks){const req=new Set(requirements),ids=new Set(tasks.map(t=>t.id)),covered=new Set();if(req.size!==requirements.length||ids.size!==tasks.length)throw new Error('duplicate');for(const t of tasks){for(const r of t.requirementIds){if(!req.has(r))throw new Error('requirement');covered.add(r);}for(const d of t.dependsOn)if(!ids.has(d))throw new Error('dependency');}if(covered.size!==req.size)throw new Error('coverage');const done=new Set();while(done.size<tasks.length){const ready=tasks.filter(t=>!done.has(t.id)&&t.dependsOn.every(d=>done.has(d)));if(!ready.length)throw new Error('cycle');for(const t of ready)done.add(t.id);}return{covered:req.size,tasks:tasks.length};}",
+        smoke: "assert.throws(()=>m.validatePlan(['R1'],[]));",
+        checks: n => `const req=Array.from({length:${n}},(_,i)=>'R'+i);const task={id:'all',requirementIds:req,dependsOn:[]};assert.deepEqual(m.validatePlan(req,[task]),{covered:${n},tasks:1});assert.deepEqual(m.validatePlan([],[]),{covered:0,tasks:0});assert.throws(()=>m.validatePlan([...req,req[0]],[task]));assert.throws(()=>m.validatePlan(req,[task,task]));assert.throws(()=>m.validatePlan(req,[{...task,requirementIds:req.slice(1)}]));assert.throws(()=>m.validatePlan(req,[{...task,requirementIds:[...req,'unknown']} ]));assert.throws(()=>m.validatePlan(req,[{...task,dependsOn:['missing']} ]));assert.throws(()=>m.validatePlan(req,[{...task,dependsOn:['all']} ]));`,
+    },
+];
+export const TERMINAL_CORPUS_VERSION = "terminal-coding-v3";
+/** One development case plus five held-out seeds per family. These are correlated variants,
+ * not 60 independent problem domains. Report family results and retain the long-session corpus. */
+export const terminalTaskCorpus = () => families.flatMap(family => Array.from({ length: 6 }, (_, variant) => {
+    const id = `${family.id}-${variant + 1}`;
+    const seed = variant + 2;
+    const contract = `${family.contract}\nQualification case: ${id}.`;
+    return { id, family: family.id, variant: variant + 1, split: variant === 0 ? "development" : "held-out",
+        prompt: `Complete the implementation in src/task.mjs for this repository.\n${contract}\nRequirements: implement the full documented contract, add meaningful public tests in test/task.test.mjs, run the tests, and preserve USER_NOTES.txt and package.json exactly. Only src/task.mjs and test/task.test.mjs may change. Do not add dependencies or unrelated features. Keep the original objective and constraints through any planning, helper calls or compaction.`,
+        files: { "package.json": JSON.stringify({ name: `codali-eval-${id}`, private: true, type: "module", scripts: { test: "node --test test/task.test.mjs" } }, null, 2) + "\n",
+            "USER_NOTES.txt": `User-owned note for ${id}: keep this file exactly as written.\n`,
+            "src/task.mjs": `// ${contract.replace(/\n/g, "\n// ")}\n${family.starter}\n`,
+            "test/task.test.mjs": `import test from 'node:test';\nimport assert from 'node:assert/strict';\nimport * as m from '../src/task.mjs';\ntest('public contract example', async()=>{${family.smoke}});\n` },
+        allowedChanges: ["src/task.mjs", "test/task.test.mjs"], reference: { "src/task.mjs": family.reference + "\n" },
+        hiddenChecks: family.checks(seed),
+    };
+}));
+export const terminalCorpusHash = () => createHash("sha256").update(JSON.stringify({ version: TERMINAL_CORPUS_VERSION, tasks: terminalTaskCorpus() })).digest("hex");

package/dist/eval/TerminalWorkflowCorpus.d.ts

@@ -0,0 +1,11 @@
+import type { TerminalCorpusTask } from "./TerminalTaskCorpus.js";
+export declare const TERMINAL_WORKFLOW_CORPUS_VERSION = "terminal-workflows-v2";
+export declare function terminalWorkflowCorpus(): TerminalCorpusTask[];
+export declare const terminalWorkflowCorpusHash: () => string;
+export declare const terminalWorkflowCorpusDefinition: () => {
+    version: string;
+    hash: string;
+    tasks: TerminalCorpusTask[];
+    limitations: string[];
+};
+//# sourceMappingURL=TerminalWorkflowCorpus.d.ts.map

package/dist/eval/TerminalWorkflowCorpus.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"TerminalWorkflowCorpus.d.ts","sourceRoot":"","sources":["../../src/eval/TerminalWorkflowCorpus.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,yBAAyB,CAAC;AAgGlE,eAAO,MAAM,gCAAgC,0BAA0B,CAAC;AACxE,wBAAgB,sBAAsB,IAAI,kBAAkB,EAAE,CA0B7D;AAED,eAAO,MAAM,0BAA0B,QAAO,MAAmJ,CAAC;AAClM,eAAO,MAAM,gCAAgC;;;;;CAG+D,CAAC"}

package/dist/eval/TerminalWorkflowCorpus.js

@@ -0,0 +1,116 @@
+import { createHash } from "node:crypto";
+// Independent contracts, not reskinned seeds of TerminalTaskCorpus. References
+// and hidden checks are evaluator-owned and never materialized for the agent.
+const coding = [
+    {
+        id: "invoice-cents",
+        contract: "Implement parsePrice(text) in src/money.mjs and invoice(lines) in src/task.mjs. A price is an ASCII decimal string with one or more digits, optionally a dot and exactly one or two decimal digits; no signs, whitespace or exponent. parsePrice returns BigInt cents and throws TypeError otherwise. Each invoice line has price and a positive safe-integer quantity. invoice returns the exact total cents as a decimal string, including 0 for no lines. Invalid quantity throws RangeError. Never use floating-point money arithmetic or mutate lines. Keep the money helper as the single parser used by invoice.",
+        files: { "src/money.mjs": "export const parsePrice = text => BigInt(Math.round(Number(text)*100));\n", "src/task.mjs": "import {parsePrice} from './money.mjs';\nexport const invoice = lines => String(lines.reduce((n,l)=>n+parsePrice(l.price),0n));\n" },
+        reference: { "src/money.mjs": "export function parsePrice(text){if(typeof text!=='string'||!/^\\d+(?:\\.\\d{1,2})?$/.test(text))throw new TypeError('price');const [whole,fraction='']=text.split('.');return BigInt(whole)*100n+BigInt(fraction.padEnd(2,'0'));}\n", "src/task.mjs": "import {parsePrice} from './money.mjs';\nexport function invoice(lines){let total=0n;for(const l of lines){if(!Number.isSafeInteger(l.quantity)||l.quantity<1)throw new RangeError('quantity');total+=parsePrice(l.price)*BigInt(l.quantity);}return String(total);}\n" },
+        smoke: "assert.equal(m.invoice([{price:'1.25',quantity:3}]),'375');",
+        checks: "const {parsePrice}=await import('./src/money.mjs');assert.equal(parsePrice('0001.2'),120n);assert.equal(m.invoice([]),'0');assert.equal(m.invoice(Object.freeze([Object.freeze({price:'9007199254740993.01',quantity:3})])),'2702159776422297903');for(const p of ['1.234',' 1','1.','-1','1e2','',1,null])assert.throws(()=>parsePrice(p),TypeError);for(const quantity of [0,-1,1.5,NaN,Infinity,'2'])assert.throws(()=>m.invoice([{price:'1',quantity}]),RangeError);",
+    },
+    {
+        id: "jsonl-import",
+        contract: "Fix parseRecords(text) in src/records.mjs and summarize(text) in src/task.mjs. Parse LF/CRLF JSONL, skip whitespace-only lines, require each record to have string id and nonnegative safe-integer count. Reject malformed JSON or invalid records with Error.message exactly 'Invalid record at line N', using the original 1-based line including blanks. summarize deduplicates by id keeping the LAST count, sorts ids in ascending JS string order, returns {ids,total}; reject unsafe total with RangeError. Empty input produces empty ids and total 0. Preserve records not provided by callers; no global state between calls.",
+        files: { "src/records.mjs": "export const parseRecords = text => text.split('\\n').filter(Boolean).map(JSON.parse);\n", "src/task.mjs": "import {parseRecords} from './records.mjs';\nexport const summarize = text => ({ids:parseRecords(text).map(x=>x.id),total:0});\n" },
+        reference: { "src/records.mjs": "export function parseRecords(text){const out=[];for(const [i,line]of text.split('\\n').entries()){if(!line.trim())continue;try{const r=JSON.parse(line);if(!r||typeof r.id!=='string'||!Number.isSafeInteger(r.count)||r.count<0)throw new Error();out.push(r);}catch{throw new Error('Invalid record at line '+(i+1));}}return out;}\n", "src/task.mjs": "import {parseRecords} from './records.mjs';\nexport function summarize(text){const by=new Map(parseRecords(text).map(r=>[r.id,r.count]));const ids=[...by.keys()].sort();const total=ids.reduce((n,id)=>n+by.get(id),0);if(!Number.isSafeInteger(total))throw new RangeError('total');return{ids,total};}\n" },
+        smoke: "assert.deepEqual(m.summarize('{\"id\":\"b\",\"count\":2}\\n{\"id\":\"a\",\"count\":3}'),{ids:['a','b'],total:5});",
+        checks: "const {parseRecords}=await import('./src/records.mjs');assert.deepEqual(m.summarize(' \\r\\n'+JSON.stringify({id:'__proto__',count:9})+'\\r\\n'+JSON.stringify({id:'__proto__',count:2})),{ids:['__proto__'],total:2});assert.deepEqual(m.summarize(''),{ids:[],total:0});for(const line of ['{','null','{}','{\"id\":\"x\",\"count\":-1}'])assert.throws(()=>parseRecords('\\n \\n'+line),{message:'Invalid record at line 3'});assert.throws(()=>m.summarize(JSON.stringify({id:'a',count:Number.MAX_SAFE_INTEGER})+'\\n'+JSON.stringify({id:'b',count:1})),RangeError);",
+    },
+    {
+        id: "locale-fallback",
+        contract: "Implement lookup(catalogs,locales,key) in src/catalog.mjs and translate(catalogs,locales,key,vars) in src/task.mjs. lookup scans locales in supplied order, accepting only own locale and own key properties with string values; an empty string is a valid translation. If none, return key. translate replaces tokens of exactly the form {ASCII letters/digits/underscore} only when vars owns that name; substitute String(value) literally, including dollar signs. Leave unknown tokens untouched. Do not replace replacement text recursively or mutate catalogs/vars. src/task.mjs must use the lookup helper.",
+        files: { "src/catalog.mjs": "export const lookup=(catalogs,locales,key)=>catalogs[locales[0]]?.[key]||key;\n", "src/task.mjs": "import {lookup} from './catalog.mjs';\nexport const translate=(c,l,k,v)=>lookup(c,l,k);\n" },
+        reference: { "src/catalog.mjs": "export function lookup(catalogs,locales,key){for(const locale of locales){if(!Object.hasOwn(catalogs,locale))continue;const c=catalogs[locale];if(c&&Object.hasOwn(c,key)&&typeof c[key]==='string')return c[key];}return key;}\n", "src/task.mjs": "import {lookup} from './catalog.mjs';\nexport function translate(c,l,k,v){return lookup(c,l,k).replace(/\\{([A-Za-z0-9_]+)\\}/g,(all,name)=>Object.hasOwn(v,name)?String(v[name]):all);}\n" },
+        smoke: "assert.equal(m.translate({en:{hello:'Hi {name}'}},['tr','en'],'hello',{name:'Ada'}),'Hi Ada');",
+        checks: "const {lookup}=await import('./src/catalog.mjs');const c=Object.assign(Object.create({evil:{x:'bad'}}),{en:Object.assign(Object.create({hidden:'bad'}),{blank:'',line:'{x} {missing} {toString}'})});assert.equal(lookup(c,['evil'],'x'),'x');assert.equal(lookup(c,['en'],'hidden'),'hidden');assert.equal(lookup(c,['en'],'blank'),'');assert.equal(m.translate(c,['en'],'line',{x:'$&{x}'}),'$&{x} {missing} {toString}');assert.equal(m.translate(c,['en'],'{x} {x}',{x:0}),'0 0');",
+    },
+    {
+        id: "header-normalization",
+        contract: "Refactor src/task.mjs to use src/headers.mjs normalizeHeaders(entries), where entries is an array of [name,value] strings. Normalize names to lower-case after trimming ASCII space/tab only, reject empty names or any non-token ASCII character (valid token chars: letters, digits, !#$%&'*+-.^_`|~). Trim ASCII space/tab from values; reject any CR/LF anywhere in a value. Join duplicate normalized names in encounter order with ', '. Return a fresh null-prototype object so __proto__ is an ordinary key. buildRequest(entries,body) returns {headers,body}, preserving the exact body reference; no input mutation.",
+        files: { "src/headers.mjs": "export const normalizeHeaders = entries => Object.fromEntries(entries);\n", "src/task.mjs": "export const buildRequest=(entries,body)=>({headers:Object.fromEntries(entries),body});\n" },
+        reference: { "src/headers.mjs": "export function normalizeHeaders(entries){const out=Object.create(null);for(const [raw,value]of entries){if(typeof raw!=='string'||typeof value!=='string')throw new TypeError('strings');const name=raw.replace(/^[ \\t]+|[ \\t]+$/g,'').toLowerCase();if(!/^[!#$%&'*+.^_`|~0-9a-z-]+$/.test(name)||/[\\r\\n]/.test(value))throw new TypeError('header');const v=value.replace(/^[ \\t]+|[ \\t]+$/g,'');out[name]=Object.hasOwn(out,name)?out[name]+', '+v:v;}return out;}\n", "src/task.mjs": "import {normalizeHeaders} from './headers.mjs';\nexport const buildRequest=(entries,body)=>({headers:normalizeHeaders(entries),body});\n" },
+        smoke: "assert.equal(m.buildRequest([['X-Id','a'],[' x-ID ','b']],null).headers['x-id'],'a, b');",
+        checks: "const {normalizeHeaders}=await import('./src/headers.mjs');const body={};const h=m.buildRequest(Object.freeze([Object.freeze(['__proto__','ok'])]),body);assert.equal(Object.getPrototypeOf(h.headers),null);assert.equal(h.headers.__proto__,'ok');assert.equal(h.body,body);assert.equal(normalizeHeaders([['X','  a\\t'],['x','']]).x,'a, ');for(const name of ['','a:b','a b','é','\\nx'])assert.throws(()=>normalizeHeaders([[name,'x']]),TypeError);for(const v of ['a\\nb','a\\rb'])assert.throws(()=>normalizeHeaders([['x',v]]),TypeError);",
+    },
+    {
+        id: "async-worker-pool",
+        contract: "Implement mapPool(items,limit,worker) in src/pool.mjs and doubleAll(items,limit) in src/task.mjs using that helper. mapPool invokes async worker(item,index) at most limit concurrently, preserves result order, accepts empty arrays, rejects nonpositive/noninteger limits with RangeError. On any worker rejection, stop dispatching new items, await all already-started workers, then reject with the FIRST observed error. A worker throwing synchronously follows the same rule. Do not mutate items. doubleAll uses mapPool to multiply each number by 2; do not implement a second pool.",
+        files: { "src/pool.mjs": "export const mapPool=async(items,limit,worker)=>Promise.all(items.map(worker));\n", "src/task.mjs": "export const doubleAll=async items=>items.map(x=>x*2);\n" },
+        reference: { "src/pool.mjs": "export async function mapPool(items,limit,worker){if(!Number.isInteger(limit)||limit<1)throw new RangeError('limit');const out=new Array(items.length);let next=0,failed=false,first;async function lane(){while(!failed&&next<items.length){const i=next++;try{out[i]=await worker(items[i],i);}catch(e){if(!failed){failed=true;first=e;}}}}await Promise.all(Array.from({length:Math.min(limit,items.length)},lane));if(failed)throw first;return out;}\n", "src/task.mjs": "import {mapPool} from './pool.mjs';\nexport const doubleAll=(items,limit)=>mapPool(items,limit,async x=>x*2);\n" },
+        smoke: "assert.deepEqual(await m.doubleAll([2,4],1),[4,8]);",
+        checks: "const {mapPool}=await import('./src/pool.mjs');let active=0,peak=0;assert.deepEqual(await mapPool(Object.freeze([1,2,3,4]),2,async(x,i)=>{active++;peak=Math.max(peak,active);await new Promise(r=>setTimeout(r,2));active--;return x+i;}),[1,3,5,7]);assert.equal(peak,2);let release;const gate=new Promise(r=>release=r);const started=[];const boom=new Error('first');let settled=false;const p=mapPool([0,1,2,3],2,async x=>{started.push(x);if(x===0)throw boom;await gate;return x;}).then(()=>{settled=true;throw new Error('unexpected');},e=>{settled=true;return e;});await new Promise(r=>setTimeout(r,5));assert.equal(settled,false);assert.deepEqual(started,[0,1]);release();assert.equal(await p,boom);for(const n of [0,-1,1.5,NaN])await assert.rejects(async()=>await mapPool([],n,async()=>0),RangeError);assert.deepEqual(await mapPool([],2,async()=>0),[]);",
+    },
+    {
+        id: "interval-reservation",
+        contract: "Implement overlaps(a,b) in src/interval.mjs and reserve(existing,candidate) in src/task.mjs. Intervals are half-open [start,end) finite-number bounds with start < end; invalid intervals throw RangeError even when the other list is empty. Endpoint touching is not overlap. reserve validates every interval before changing anything, rejects any overlap between candidate and existing with Error('conflict'), otherwise returns a new start-sorted array of fresh {start,end} objects, preserving equal-start order. Existing intervals may overlap each other; do not reject that. Ignore extra properties, never mutate inputs, do not share interval objects in output.",
+        files: { "src/interval.mjs": "export const overlaps=(a,b)=>a.start<=b.end&&b.start<=a.end;\n", "src/task.mjs": "export const reserve=(existing,candidate)=>[...existing,candidate];\n" },
+        reference: { "src/interval.mjs": "export function validate(x){if(!x||!Number.isFinite(x.start)||!Number.isFinite(x.end)||x.start>=x.end)throw new RangeError('interval');}\nexport function overlaps(a,b){validate(a);validate(b);return a.start<b.end&&b.start<a.end;}\n", "src/task.mjs": "import {overlaps,validate} from './interval.mjs';\nexport function reserve(existing,candidate){validate(candidate);for(const x of existing)validate(x);if(existing.some(x=>overlaps(x,candidate)))throw new Error('conflict');return [...existing,candidate].map(({start,end})=>({start,end})).sort((a,b)=>a.start-b.start);}\n" },
+        smoke: "assert.deepEqual(m.reserve([{start:0,end:1}],{start:1,end:2}),[{start:0,end:1},{start:1,end:2}]);",
+        checks: "const {overlaps}=await import('./src/interval.mjs');assert.equal(overlaps({start:0,end:2},{start:2,end:3}),false);assert.throws(()=>m.reserve([],{start:1,end:1}),RangeError);assert.throws(()=>m.reserve([{start:NaN,end:3}],{start:5,end:6}),RangeError);assert.throws(()=>m.reserve([{start:0,end:2}],{start:1,end:3}),{message:'conflict'});const a=Object.freeze({start:0,end:3,extra:1});const b=Object.freeze({start:1,end:2});const out=m.reserve(Object.freeze([a,b]),Object.freeze({start:4,end:5}));assert.notEqual(out[0],a);assert.deepEqual(out[0],{start:0,end:3});",
+    },
+    {
+        id: "csv-export",
+        contract: "Implement cell(value) in src/csv.mjs and exportRows(rows) in src/task.mjs. Cells accept string, number, boolean, null or undefined; null/undefined become empty, others String(value). Quote a cell exactly when it contains comma, double quote, CR or LF; double every internal quote. exportRows joins cells with commas and rows with CRLF, with a final CRLF for a nonempty list and empty string for no rows. Preserve empty rows as CRLF; never trim data or mutate input. Use the shared cell helper.",
+        files: { "src/csv.mjs": "export const cell=value=>String(value??'');\n", "src/task.mjs": "export const exportRows=rows=>rows.map(row=>row.join(',')).join('\\n');\n" },
+        reference: { "src/csv.mjs": "export function cell(value){const s=value==null?'':String(value);return /[,\"\\r\\n]/.test(s)?'\"'+s.replace(/\"/g,'\"\"')+'\"':s;}\n", "src/task.mjs": "import {cell} from './csv.mjs';\nexport const exportRows=rows=>rows.length?rows.map(row=>row.map(cell).join(',')).join('\\r\\n')+'\\r\\n':'';\n" },
+        smoke: "assert.equal(m.exportRows([['a,b','c']]),'\"a,b\",c\\r\\n');",
+        checks: "const {cell}=await import('./src/csv.mjs');assert.equal(cell('a\"b'),'\"a\"\"b\"');assert.equal(cell(' x '),' x ');assert.equal(cell(false),'false');assert.equal(m.exportRows([]),'');assert.equal(m.exportRows([[]]),'\\r\\n');assert.equal(m.exportRows(Object.freeze([Object.freeze([null,undefined,0,'a\\nb'])])),',,0,\"a\\nb\"\\r\\n');",
+    },
+    {
+        id: "additive-record-migration",
+        contract: "Implement migrate(document) in src/task.mjs using validateV1 in src/schema.mjs. Input schemaVersion must be exactly 1; records must be an array with unique string ids and string names. Reject invalid documents with TypeError. Output is a deep clone preserving ALL existing document/record fields, with schemaVersion 2 and a displayName field set to name only when displayName was not already an own property (preserve empty string/null). Do not alter source objects, reorder records or delete any published field. Already-v2 input must be rejected instead of silently reprocessed.",
+        files: { "src/schema.mjs": "export const validateV1=document=>document;\n", "src/task.mjs": "export const migrate=document=>({schemaVersion:2,records:document.records.map(r=>({id:r.id,displayName:r.name}))});\n" },
+        reference: { "src/schema.mjs": "export function validateV1(d){if(!d||d.schemaVersion!==1||!Array.isArray(d.records))throw new TypeError('document');const ids=new Set();for(const r of d.records){if(!r||typeof r.id!=='string'||typeof r.name!=='string'||ids.has(r.id))throw new TypeError('record');ids.add(r.id);}return d;}\n", "src/task.mjs": "import {validateV1} from './schema.mjs';\nexport function migrate(document){validateV1(document);const out=structuredClone(document);out.schemaVersion=2;for(const r of out.records)if(!Object.hasOwn(r,'displayName'))r.displayName=r.name;return out;}\n" },
+        smoke: "assert.deepEqual(m.migrate({schemaVersion:1,records:[{id:'a',name:'A',published:true}]}).records,[{id:'a',name:'A',published:true,displayName:'A'}]);",
+        checks: "const d={schemaVersion:1,metadata:{keep:[1]},records:[{id:'a',name:'A',displayName:''},{id:'b',name:'B',displayName:null},{id:'c',name:'C',history:[1]}]};const before=structuredClone(d);const out=m.migrate(d);assert.deepEqual(d,before);assert.equal(out.records[0].displayName,'');assert.equal(out.records[1].displayName,null);assert.equal(out.records[2].displayName,'C');out.metadata.keep.push(2);out.records[2].history.push(2);assert.deepEqual(d,before);for(const bad of [null,{schemaVersion:2,records:[]},{schemaVersion:1,records:[{id:'x',name:'x'},{id:'x',name:'y'}]},{schemaVersion:1,records:[{id:1,name:'x'}]}])assert.throws(()=>m.migrate(bad),TypeError);",
+    },
+];
+const reports = [
+    { id: "investigate-cache-expiry", question: "Investigate why a cached value disappears at its expiry timestamp. Do not change implementation. Write docs/report.json with exactly these keys: entrypoint (path), owner (path), unit ('milliseconds' or 'seconds'), boundary ('inclusive' or 'exclusive'), repair (one of 'change-greater-than', 'change-greater-or-equal', 'no-change'), and evidence (array of source paths). The required behavior is: expired when now >= expiresAt. Report the minimal correction needed.",
+        files: { "src/api.mjs": "import {readCached} from './cache.mjs';\nexport const getItem=(store,key,now)=>readCached(store,key,now);\n", "src/cache.mjs": "export function readCached(store,key,now){const entry=store.get(key);if(!entry)return undefined;if(now>=entry.expiresAt){store.delete(key);return undefined;}return entry.value;}\n", "docs/clock.md": "now and expiresAt are epoch milliseconds. Expiry is inclusive.\n" },
+        expected: { entrypoint: "src/api.mjs", owner: "src/cache.mjs", unit: "milliseconds", boundary: "inclusive", repair: "no-change", evidence: ["docs/clock.md", "src/api.mjs", "src/cache.mjs"] } },
+    { id: "investigate-duplicate-submit", question: "Investigate duplicate billing after a retry using only this fixture. Do not repair code. Write docs/report.json with exactly: client (path), server (path), cause (one of 'new-id-per-attempt', 'missing-dedup', 'shared-tenant-key'), remedy (one of 'reuse-operation-id', 'disable-retries', 'remove-dedup'), and evidence (sorted array of source paths). Identify the client/server interaction, not an unsupported infrastructure theory.",
+        files: { "src/client.mjs": "export async function charge(send,amount){for(let attempt=0;attempt<2;attempt++){try{return await send({operationId:crypto.randomUUID(),amount});}catch(e){if(attempt===1)throw e;}}}\n", "src/server.mjs": "const seen=new Map();\nexport async function handle(tenant,request,bill){const key=tenant+':'+request.operationId;if(seen.has(key))return seen.get(key);const result=await bill(request.amount);seen.set(key,result);return result;}\n", "docs/incident.md": "The first bill completed and its result was stored, but the response was lost. The retry then billed again. No concurrent requests occurred.\n" },
+        expected: { client: "src/client.mjs", server: "src/server.mjs", cause: "new-id-per-attempt", remedy: "reuse-operation-id", evidence: ["docs/incident.md", "src/client.mjs", "src/server.mjs"] } },
+    { id: "research-pinned-upgrade", question: "Research a fictional service upgrade using the pinned vendor documents in vendor-docs only; no internet claims are needed. Write docs/report.json with exactly: version ('1' or '2'), submitPath (string), pollPath (string with {id}), cancelMethod (string), retrySubmit (boolean), citations (sorted source-path array). Choose the version that supports polling and cancellation. A lost submit response must not be automatically retried unless the chosen version documents idempotency. Do not change the vendor documents or application.",
+        files: { "vendor-docs/v1.md": "Fictional service v1. POST /v1/render synchronously returns a render. No status endpoint. No cancellation. Requests may be retried safely with Idempotency-Key.\n", "vendor-docs/v2.md": "Fictional service v2. POST /v2/jobs returns {id}. GET /v2/jobs/{id} reports status. DELETE /v2/jobs/{id} requests cancellation. Idempotency keys are not supported; retrying a submit can create a second job.\n", "src/app.mjs": "export const currentVersion='1';\n" },
+        expected: { version: "2", submitPath: "/v2/jobs", pollPath: "/v2/jobs/{id}", cancelMethod: "DELETE", retrySubmit: false, citations: ["vendor-docs/v2.md"] } },
+    { id: "investigate-tenant-key", question: "Inspect the cache and callers. Write docs/report.json with exactly: owner (path), callers (sorted path array), isolation ('safe' or 'unsafe'), cause (one of 'tenant-omitted', 'user-omitted', 'none'), remedy (one of 'include-tenant-and-user', 'increase-ttl', 'none'). Users can have the same id in different tenants. Do not modify code or invent runtime observations.",
+        files: { "src/cache.mjs": "const values=new Map();\nexport const get=(tenant,user)=>values.get(user);\nexport const set=(tenant,user,value)=>values.set(user,value);\n", "src/profile.mjs": "import {get} from './cache.mjs';\nexport const profile=(ctx,user)=>get(ctx.tenant,user);\n", "src/update.mjs": "import {set} from './cache.mjs';\nexport const update=(ctx,user,value)=>set(ctx.tenant,user,value);\n" },
+        expected: { owner: "src/cache.mjs", callers: ["src/profile.mjs", "src/update.mjs"], isolation: "unsafe", cause: "tenant-omitted", remedy: "include-tenant-and-user" } },
+];
+export const TERMINAL_WORKFLOW_CORPUS_VERSION = "terminal-workflows-v2";
+export function terminalWorkflowCorpus() {
+    const common = (id) => ({ "package.json": JSON.stringify({ name: `workflow-${id}`, private: true, type: "module", scripts: { test: "node --test test/task.test.mjs" } }, null, 2) + "\n", "USER_NOTES.txt": `User-owned ${id} notes. Preserve byte for byte.\n` });
+    const tasks = coding.map(item => ({ id: item.id, family: "multi-file", variant: 1, split: "held-out",
+        prompt: `${item.contract}\nExtend test/task.test.mjs with meaningful regression tests and run the suite. Only ${Object.keys(item.files).join(', ')} and test/task.test.mjs may change. Preserve all other files exactly. No new dependencies or unrelated features.`,
+        files: { ...common(item.id), ...item.files, "test/task.test.mjs": `import test from 'node:test';\nimport assert from 'node:assert/strict';\nimport * as m from '../src/task.mjs';\ntest('public example',async()=>{${item.smoke}});\n` },
+        allowedChanges: [...Object.keys(item.files), "test/task.test.mjs"], reference: item.reference, hiddenChecks: item.checks,
+    }));
+    for (const item of reports) {
+        const expected = JSON.stringify(item.expected);
+        // v2 corrects unstated v1 oracle assumptions without changing task prompts.
+        // Only the unordered evidence field is normalized. Research may also cite
+        // v1 for the comparison, while v2 remains mandatory and citations sorted.
+        const normalize = item.id === "investigate-cache-expiry"
+            ? "assert.ok(Array.isArray(report.evidence));report.evidence.sort();"
+            : item.id === "research-pinned-upgrade"
+                ? "assert.ok(Array.isArray(report.citations));assert.deepEqual(report.citations,[...new Set(report.citations)].sort());assert.ok(report.citations.includes('vendor-docs/v2.md'));assert.ok(report.citations.every(p=>['vendor-docs/v1.md','vendor-docs/v2.md'].includes(p)));report.citations=['vendor-docs/v2.md'];"
+                : "";
+        tasks.push({ id: item.id, family: item.id.startsWith("research") ? "pinned-document-research" : "repository-investigation", variant: 1, split: "held-out",
+            prompt: `${item.question}\nOnly docs/report.json may be created or changed. Preserve every existing file, including tests and user notes, exactly. Run the existing tests; do not add tests or dependencies. This is investigation/report delivery, not implementation.`,
+            files: { ...common(item.id), ...item.files, "test/task.test.mjs": `import test from 'node:test';\nimport assert from 'node:assert/strict';\nimport {readFile} from 'node:fs/promises';\nconst read=async()=>JSON.parse(await readFile(new URL('../docs/report.json',import.meta.url),'utf8'));\ntest('report is an object',async()=>assert.equal(typeof await read(),'object'));\ntest('report is nonempty',async()=>assert.ok(Object.keys(await read()).length>0));\n` },
+            allowedChanges: ["docs/report.json"], reference: { "docs/report.json": JSON.stringify(item.expected, null, 2) + "\n" },
+            hiddenChecks: `const {readFile}=await import('node:fs/promises');\nconst report=JSON.parse(await readFile(new URL('./docs/report.json',import.meta.url),'utf8'));${normalize}assert.deepEqual(report,${expected});`,
+            grading: { entrypoint: null, testFilesToExtend: [] },
+        });
+    }
+    return tasks;
+}
+export const terminalWorkflowCorpusHash = () => createHash("sha256").update(JSON.stringify({ version: TERMINAL_WORKFLOW_CORPUS_VERSION, tasks: terminalWorkflowCorpus() })).digest("hex");
+export const terminalWorkflowCorpusDefinition = () => ({ version: TERMINAL_WORKFLOW_CORPUS_VERSION, hash: terminalWorkflowCorpusHash(), tasks: terminalWorkflowCorpus(),
+    limitations: ["12 distinct small workflows: eight multi-file implementations, three repository investigations, one research task over pinned fictional documents",
+        "Pinned-document research does not qualify live web retrieval or citation quality", "Single-run outcomes are descriptive, not statistically powered model comparisons",
+        "Does not qualify multi-hour recovery, media, physical resource admission or production coexistence"] });

package/dist/eval/fixtures/TerminalRecoveryJourney.d.ts

@@ -0,0 +1,4 @@
+/** A scripted fault fixture, deliberately separate from live-model evaluation.
+ * Each stage is launched in a fresh OS process by the parent test. */
+export declare function runTerminalRecoveryStage(workspaceRoot: string, storageRoot: string, stage: number): Promise<void>;
+//# sourceMappingURL=TerminalRecoveryJourney.d.ts.map

package/dist/eval/fixtures/TerminalRecoveryJourney.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"TerminalRecoveryJourney.d.ts","sourceRoot":"","sources":["../../../src/eval/fixtures/TerminalRecoveryJourney.ts"],"names":[],"mappings":"AAsBA;sEACsE;AACtE,wBAAsB,wBAAwB,CAAC,aAAa,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CA0EvH"}