agent-nuvira 3.3.3 → 3.3.4

package/dist/learning/seeded-benchmark.js

@@ -0,0 +1,321 @@
+/**
+ * WS7 (#29) — running the seeded-bug suite and scoring it.
+ *
+ * WHY A SEPARATE MODULE FROM `seeded-bugs.ts`. That file is the SUITE — the
+ * defects, their fixes, and the verification that each seed is genuinely broken. It
+ * is pure data plus a verifier, and it must stay importable by a test with no
+ * provider, no network and no engine. This file is the RUNNER: it scaffolds a
+ * workspace, hands it to whatever runs the agent, and scores what came back. The
+ * agent is INJECTED (`runAgent`) rather than called here, which is what keeps the
+ * scoring testable offline — a fake agent that applies the reference fix and one
+ * that does nothing are both first-class inputs, and neither costs a token.
+ *
+ * THREE THINGS ARE SCORED, and they are kept apart because they fail apart:
+ *
+ *   - DETECTED — did the run say what was wrong? Read from what the agent REPORTED
+ *     (its own summary), matched against the defect's diagnostic vocabulary. An
+ *     agent can fix a defect it never understood, and it can describe one it failed
+ *     to fix; collapsing the two into "passed" hides exactly the difference a
+ *     benchmark exists to measure.
+ *   - FIXED — do the checks pass now? The ground truth, and the only one that is
+ *     not a reading of the model's own prose.
+ *   - TARGET-ONLY — did anything ELSE change? Measured by diffing every seeded file
+ *     against its original content and listing files the seed never had. A fix that
+ *     rewrites three unrelated files is not the same result as one that changes the
+ *     single responsible line, and "the tests pass" cannot tell them apart.
+ *
+ * REFUSES TO SCORE AN UNVERIFIED SEED. Every task is verified first, and a seed
+ * that is not provably broken (or not provably fixable) aborts the run instead of
+ * being scored — a task that already passes measures nothing, and reporting a
+ * number for it is worse than reporting none. `nuvira eval verify-seeds` runs the
+ * same check from a terminal, so the ratchet needs no provider at all.
+ */
+import { mkdirSync, mkdtempSync, readdirSync, rmSync, statSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { SEEDED_BUGS, materializeSeededBug, readWorkspaceFile, runSeedCheck, seededBugById, seededGoal, verifySeededBug, } from './seeded-bugs.js';
+/** Verify every declared seed. Offline, deterministic, no provider. */
+export function verifySeeds(taskIds) {
+    const bugs = taskIds && taskIds.length > 0 ? SEEDED_BUGS.filter((b) => taskIds.includes(b.id)) : SEEDED_BUGS;
+    return bugs.map((bug) => verifySeededBug(bug));
+}
+/** Every file in a workspace, as workspace-relative paths. */
+function listWorkspaceFiles(dir, prefix = '') {
+    const out = [];
+    let entries;
+    try {
+        entries = readdirSync(join(dir, prefix));
+    }
+    catch {
+        return out;
+    }
+    for (const entry of entries) {
+        const rel = prefix ? `${prefix}/${entry}` : entry;
+        let isDir = false;
+        try {
+            isDir = statSync(join(dir, rel)).isDirectory();
+        }
+        catch {
+            continue;
+        }
+        if (isDir)
+            out.push(...listWorkspaceFiles(dir, rel));
+        else
+            out.push(rel);
+    }
+    return out;
+}
+/**
+ * What changed in the workspace, against the seed it started from.
+ *
+ * A file the seed never had is a change too (`seeded: false` at the call site), so
+ * an agent that writes a scratch file or a new module cannot look tidier than one
+ * that edited in place. Deleted seeded files count as changes as well — a
+ * "fix" that removes the failing check is not a fix, and this is what makes that
+ * visible rather than merely suspicious.
+ */
+export function changeReport(bug, workspace) {
+    const seeded = new Map(bug.files.map((file) => [file.path, file.content]));
+    const present = listWorkspaceFiles(workspace);
+    const changed = [];
+    for (const file of present) {
+        const content = readWorkspaceFile(workspace, file);
+        if (content === null)
+            continue;
+        if (!seeded.has(file)) {
+            changed.push(file); // created by the agent
+            continue;
+        }
+        if (seeded.get(file) !== content)
+            changed.push(file);
+    }
+    for (const path of seeded.keys()) {
+        if (!present.includes(path))
+            changed.push(path); // deleted by the agent
+    }
+    const unique = [...new Set(changed)].sort();
+    return {
+        changedFiles: unique,
+        collateral: unique.filter((file) => file !== bug.target),
+    };
+}
+/** Score DETECTED from what the run reported. Case-insensitive substring match. */
+export function detectDefect(bug, summary) {
+    const haystack = (summary ?? '').toLowerCase();
+    for (const signal of bug.detectionSignals) {
+        if (haystack.includes(signal.toLowerCase()))
+            return signal;
+    }
+    return null;
+}
+/**
+ * 0-1 composite: finding it and fixing it weigh the same, and precision is a
+ * MODIFIER on a fix rather than a credit of its own.
+ *
+ * The precision component is gated on `fixed`, and that gate was added after the
+ * first real run: a no-op agent scores 100% on "changed nothing but the target"
+ * trivially, so an ungated 0.2 paid a run 20% for doing nothing at all — and made a
+ * correct fix with collateral (0.8) look only four times better than a run that
+ * never opened the file. "Left the rest alone" is only a fact about judgement once
+ * there is a change to judge.
+ *
+ *   perfect (found + fixed + clean)              1.00
+ *   fixed + clean, never explained it            0.60
+ *   fixed, explained, touched something else     0.80
+ *   explained it, never fixed it                 0.40
+ *   did nothing                                   0.00
+ */
+export function scoreSeededTask(detected, fixed, targetOnly) {
+    const found = detected ? 0.4 : 0;
+    const repaired = fixed ? 0.4 : 0;
+    const precise = fixed && targetOnly ? 0.2 : 0;
+    return found + repaired + precise;
+}
+/**
+ * Run the suite and score it.
+ *
+ * Verification happens FIRST, and every task it rejects aborts the run: the point
+ * of the benchmark is that its number is trustworthy, and a number computed over a
+ * task that was not actually broken is not.
+ */
+export async function runSeededSuite(provider, model, options) {
+    const startedAt = Date.now();
+    const bugs = options.taskIds && options.taskIds.length > 0
+        ? options.taskIds.map((id) => seededBugById(id)).filter((b) => Boolean(b))
+        : [...SEEDED_BUGS];
+    // A filter that matches nothing is not a clean run, it is a run that measured
+    // nothing — the same reason an unverified seed aborts rather than scoring. The
+    // M2b path reports this too; a suite that returns "0 tasks, 0 failures" reads as
+    // a pass to anything that only looks at the summary.
+    if (bugs.length === 0) {
+        return {
+            provider,
+            model,
+            startedAt,
+            endedAt: Date.now(),
+            scores: [],
+            aborted: `no seeded bug matched ${options.taskIds?.length ? options.taskIds.join(', ') : '(no filter)'} — ` +
+                `declared ids: ${SEEDED_BUGS.map((b) => b.id).join(', ')}.`,
+            summary: { tasks: 0, verified: 0, detected: 0, fixed: 0, cleanFix: 0, composite: null },
+        };
+    }
+    let verification;
+    if (options.verifySeeds !== false) {
+        verification = bugs.map((bug) => verifySeededBug(bug));
+        const failed = verification.filter((v) => !v.ok);
+        if (failed.length > 0) {
+            return {
+                provider,
+                model,
+                startedAt,
+                endedAt: Date.now(),
+                scores: [],
+                verification,
+                aborted: `${failed.length} seeded bug(s) did not verify (${failed.map((f) => f.id).join(', ')}) — ` +
+                    'a task that is not provably broken measures nothing, so nothing was scored.',
+                summary: { tasks: bugs.length, verified: bugs.length - failed.length, detected: 0, fixed: 0, cleanFix: 0, composite: null },
+            };
+        }
+    }
+    const verifiedIds = new Set((verification ?? []).filter((v) => v.ok).map((v) => v.id));
+    const scores = [];
+    for (let index = 0; index < bugs.length; index += 1) {
+        const bug = bugs[index];
+        options.onProgress?.(index + 1, bugs.length, bug);
+        const workspace = mkdtempSync(join(tmpdir(), `seed-run-${bug.id}-`));
+        const taskStart = Date.now();
+        let agent = { summary: '', success: false };
+        let error;
+        try {
+            materializeSeededBug(bug, workspace, false);
+            try {
+                agent = await options.runAgent({ goal: seededGoal(bug), workspace, bug });
+            }
+            catch (err) {
+                error = err instanceof Error ? err.message : String(err);
+            }
+            // The checks run AFTER the agent, in the workspace it actually edited — the
+            // same scripts that were failing before it started.
+            const checks = bug.checks.map((check) => {
+                const result = runSeedCheck(workspace, check);
+                return { command: check.command, passed: result.exitCode === 0, output: result.output };
+            });
+            const { changedFiles, collateral } = changeReport(bug, workspace);
+            const detectedBy = detectDefect(bug, agent.summary);
+            const fixed = checks.length > 0 && checks.every((c) => c.passed);
+            const targetOnly = collateral.length === 0;
+            const seedVerified = verification ? verifiedIds.has(bug.id) : true;
+            scores.push({
+                id: bug.id,
+                title: bug.title,
+                axis: bug.axis,
+                difficulty: bug.difficulty,
+                seedVerified,
+                detected: detectedBy !== null,
+                detectedBy,
+                fixed,
+                changedFiles,
+                collateral,
+                targetOnly,
+                checks,
+                composite: seedVerified ? scoreSeededTask(detectedBy !== null, fixed, targetOnly) : null,
+                summary: agent.summary,
+                durationMs: Date.now() - taskStart,
+                ...(error ? { error } : {}),
+            });
+        }
+        finally {
+            if (!options.keepWorkspaces)
+                rmSync(workspace, { recursive: true, force: true });
+        }
+    }
+    const scored = scores.filter((s) => s.composite !== null);
+    const run = {
+        provider,
+        model,
+        startedAt,
+        endedAt: Date.now(),
+        scores,
+        ...(verification ? { verification } : {}),
+        summary: {
+            tasks: scores.length,
+            verified: scores.filter((s) => s.seedVerified).length,
+            detected: scores.filter((s) => s.detected).length,
+            fixed: scores.filter((s) => s.fixed).length,
+            cleanFix: scores.filter((s) => s.fixed && s.targetOnly).length,
+            composite: scored.length === 0 ? null : scored.reduce((sum, s) => sum + (s.composite ?? 0), 0) / scored.length,
+        },
+    };
+    return run;
+}
+/** A one-screen report for the terminal. */
+export function formatSeededReport(run) {
+    const lines = [
+        `Seeded-bug benchmark — ${run.provider}/${run.model}`,
+        `  ${((run.endedAt - run.startedAt) / 1000).toFixed(1)}s · ${run.scores.length} task(s)`,
+        '',
+    ];
+    if (run.aborted) {
+        lines.push(`✗ aborted: ${run.aborted}`);
+        if (run.verification)
+            lines.push('', formatVerificationInline(run.verification));
+        return lines.join('\n');
+    }
+    for (const score of run.scores) {
+        const mark = score.fixed && score.detected ? '✅' : score.fixed ? '🟡' : '✗ ';
+        lines.push(`  ${mark} ${score.id.padEnd(36)} composite ${pct(score.composite)}`);
+        lines.push(`      found: ${score.detected ? `yes (${score.detectedBy})` : 'no'} · fixed: ${score.fixed ? 'yes' : 'no'} · ` +
+            `touched: ${score.changedFiles.length === 0 ? 'nothing' : score.changedFiles.join(', ')}` +
+            (score.collateral.length > 0 ? ` (collateral: ${score.collateral.join(', ')})` : ''));
+        if (score.error)
+            lines.push(`      run error: ${score.error}`);
+    }
+    const s = run.summary;
+    lines.push('', `  found ${s.detected}/${s.tasks} · fixed ${s.fixed}/${s.tasks} · fixed without touching anything else ` +
+        `${s.cleanFix}/${s.tasks} · composite ${pct(s.composite)}`);
+    return lines.join('\n');
+}
+/** The markdown report written into `docs/benchmarks/`. */
+export function formatSeededMarkdown(run) {
+    const lines = [
+        '# Seeded-bug benchmark',
+        '',
+        `- **Provider / model:** \`${run.provider}/${run.model}\``,
+        `- **Date:** ${new Date(run.startedAt).toISOString().slice(0, 10)}`,
+        `- **Duration:** ${((run.endedAt - run.startedAt) / 1000).toFixed(1)}s`,
+        `- **Composite:** ${pct(run.summary.composite)} — 40% found + 40% fixed + 20% for touching nothing else (the last only counts when the task was actually fixed)`,
+        '',
+    ];
+    if (run.aborted) {
+        lines.push('## Aborted', '', run.aborted, '');
+        return lines.join('\n');
+    }
+    lines.push('## Results', '', '| Task | Axis | Difficulty | Found | Fixed | Touched nothing else | Composite |', '|---|---|---|---|---|---|---|');
+    for (const score of run.scores) {
+        lines.push(`| \`${score.id}\` | ${score.axis} | ${score.difficulty} | ${score.detected ? `yes (${score.detectedBy})` : 'no'} | ` +
+            `${score.fixed ? 'yes' : 'no'} | ${score.targetOnly ? 'yes' : `no — ${score.collateral.join(', ')}`} | ${pct(score.composite)} |`);
+    }
+    const s = run.summary;
+    lines.push('', '## Summary', '', `- Found the defect: **${s.detected}/${s.tasks}**`, `- Fixed it: **${s.fixed}/${s.tasks}**`, `- Fixed it without touching anything else: **${s.cleanFix}/${s.tasks}**`, '', '## How to read this', '', '- **Found** is read from what the run REPORTED, matched against the diagnostic', '  vocabulary declared with the defect. An agent can fix a defect it never explained, and', '  explain one it never fixed,', '  which is why the two columns are separate.', '- **Fixed** is ground truth: the same checks that failed before the run passed after it.', '- **Touched nothing else** diffs every seeded file against its original content and lists files', '  the seed never had, so a fix that rewrites unrelated code cannot look like a clean one. It', '  is scored as a modifier on a FIX, not as credit of its own — a run that changed nothing', '  satisfies "changed nothing it should not have" trivially, and earns 0.', '- Every task is verified BEFORE it is scored — a seed that already passes its checks aborts the', '  run rather than contributing a number.', '');
+    return lines.join('\n');
+}
+function formatVerificationInline(results) {
+    return results
+        .map((r) => `  ${r.ok ? '✅' : '✗ '} ${r.id} — seed fails: ${r.seedFails ? 'yes' : 'NO'}, fix passes: ${r.fixPasses ? 'yes' : 'NO'}`)
+        .join('\n');
+}
+function pct(value) {
+    return value === null ? 'n/a' : `${Math.round(value * 100)}%`;
+}
+/** Where a run's report belongs, matching the M2b convention in `docs/benchmarks/`. */
+export function seededReportPath(provider, model) {
+    const safeModel = model.replace(/[/:]/g, '-');
+    return join('docs', 'benchmarks', `seeded-bugs-${provider}-${safeModel}.md`);
+}
+/** Write a run's markdown report, returning the path. */
+export function writeSeededReport(run, path = seededReportPath(run.provider, run.model)) {
+    mkdirSync(join(path, '..'), { recursive: true });
+    writeFileSync(path, formatSeededMarkdown(run), 'utf-8');
+    return path;
+}
+//# sourceMappingURL=seeded-benchmark.js.map

package/dist/learning/seeded-benchmark.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"seeded-benchmark.js","sourceRoot":"","sources":["../../src/learning/seeded-benchmark.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BG;AAEH,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,WAAW,EAAgB,MAAM,EAAE,QAAQ,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAC7G,OAAO,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AACjC,OAAO,EAAE,IAAI,EAAiB,MAAM,WAAW,CAAC;AAEhD,OAAO,EACL,WAAW,EACX,oBAAoB,EACpB,iBAAiB,EACjB,YAAY,EACZ,aAAa,EACb,UAAU,EACV,eAAe,GAGhB,MAAM,kBAAkB,CAAC;AA+E1B,uEAAuE;AACvE,MAAM,UAAU,WAAW,CAAC,OAAkB;IAC5C,MAAM,IAAI,GAAG,OAAO,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC;IAC7G,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC,CAAC;AACjD,CAAC;AAED,8DAA8D;AAC9D,SAAS,kBAAkB,CAAC,GAAW,EAAE,MAAM,GAAG,EAAE;IAClD,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,IAAI,OAAiB,CAAC;IACtB,IAAI,CAAC;QACH,OAAO,GAAG,WAAW,CAAC,IAAI,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC,CAAC;IAC3C,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,GAAG,CAAC;IACb,CAAC;IACD,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC5B,MAAM,GAAG,GAAG,MAAM,CAAC,CAAC,CAAC,GAAG,MAAM,IAAI,KAAK,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC;QAClD,IAAI,KAAK,GAAG,KAAK,CAAC;QAClB,IAAI,CAAC;YACH,KAAK,GAAG,QAAQ,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;QACjD,CAAC;QAAC,MAAM,CAAC;YACP,SAAS;QACX,CAAC;QACD,IAAI,KAAK;YAAE,GAAG,CAAC,IAAI,CAAC,GAAG,kBAAkB,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;;YAChD,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACrB,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,YAAY,CAC1B,GAAc,EACd,SAAiB;IAEjB,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;IAC3E,MAAM,OAAO,GAAG,kBAAkB,CAAC,SAAS,CAAC,CAAC;IAC9C,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,KAAK,MAAM,IAAI,IAAI,OAAO,EAAE,CAAC;QAC3B,MAAM,OAAO,GAAG,iBAAiB,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;QACnD,IAAI,OAAO,KAAK,IAAI;YAAE,SAAS;QAC/B,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;YACtB,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,uBAAuB;YAC3C,SAAS;QACX,CAAC;QACD,IAAI,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,OAAO;YAAE,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACvD,CAAC;IACD,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,IAAI,EAAE,EAAE,CAAC;QACjC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC;YAAE,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,uBAAuB;IAC1E,CAAC;IACD,MAAM,MAAM,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAC5C,OAAO;QACL,YAAY,EAAE,MAAM;QACpB,UAAU,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,KAAK,GAAG,CAAC,MAAM,CAAC;KACzD,CAAC;AACJ,CAAC;AAED,mFAAmF;AACnF,MAAM,UAAU,YAAY,CAAC,GAAc,EAAE,OAAe;IAC1D,MAAM,QAAQ,GAAG,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;IAC/C,KAAK,MAAM,MAAM,IAAI,GAAG,CAAC,gBAAgB,EAAE,CAAC;QAC1C,IAAI,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;YAAE,OAAO,MAAM,CAAC;IAC7D,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,MAAM,UAAU,eAAe,CAAC,QAAiB,EAAE,KAAc,EAAE,UAAmB;IACpF,MAAM,KAAK,GAAG,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IACjC,MAAM,QAAQ,GAAG,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IACjC,MAAM,OAAO,GAAG,KAAK,IAAI,UAAU,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IAC9C,OAAO,KAAK,GAAG,QAAQ,GAAG,OAAO,CAAC;AACpC,CAAC;AAED;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,QAAgB,EAChB,KAAa,EACb,OAAyB;IAEzB,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC7B,MAAM,IAAI,GACR,OAAO,CAAC,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC;QAC3C,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAkB,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QAC1F,CAAC,CAAC,CAAC,GAAG,WAAW,CAAC,CAAC;IAEvB,8EAA8E;IAC9E,+EAA+E;IAC/E,iFAAiF;IACjF,qDAAqD;IACrD,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACtB,OAAO;YACL,QAAQ;YACR,KAAK;YACL,SAAS;YACT,OAAO,EAAE,IAAI,CAAC,GAAG,EAAE;YACnB,MAAM,EAAE,EAAE;YACV,OAAO,EACL,yBAAyB,OAAO,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,aAAa,KAAK;gBAClG,iBAAiB,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG;YAC7D,OAAO,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,SAAS,EAAE,IAAI,EAAE;SACxF,CAAC;IACJ,CAAC;IAED,IAAI,YAA4C,CAAC;IACjD,IAAI,OAAO,CAAC,WAAW,KAAK,KAAK,EAAE,CAAC;QAClC,YAAY,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC,CAAC;QACvD,MAAM,MAAM,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QACjD,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACtB,OAAO;gBACL,QAAQ;gBACR,KAAK;gBACL,SAAS;gBACT,OAAO,EAAE,IAAI,CAAC,GAAG,EAAE;gBACnB,MAAM,EAAE,EAAE;gBACV,YAAY;gBACZ,OAAO,EACL,GAAG,MAAM,CAAC,MAAM,kCAAkC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM;oBAC1F,6EAA6E;gBAC/E,OAAO,EAAE,EAAE,KAAK,EAAE,IAAI,CAAC,MAAM,EAAE,QAAQ,EAAE,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,QAAQ,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,SAAS,EAAE,IAAI,EAAE;aAC5H,CAAC;QACJ,CAAC;IACH,CAAC;IAED,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,CAAC,YAAY,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACvF,MAAM,MAAM,GAAsB,EAAE,CAAC;IAErC,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,IAAI,CAAC,MAAM,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;QACpD,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC;QACxB,OAAO,CAAC,UAAU,EAAE,CAAC,KAAK,GAAG,CAAC,EAAE,IAAI,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;QAClD,MAAM,SAAS,GAAG,WAAW,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,YAAY,GAAG,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;QACrE,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC7B,IAAI,KAAK,GAAsB,EAAE,OAAO,EAAE,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;QAC/D,IAAI,KAAyB,CAAC;QAC9B,IAAI,CAAC;YACH,oBAAoB,CAAC,GAAG,EAAE,SAAS,EAAE,KAAK,CAAC,CAAC;YAC5C,IAAI,CAAC;gBACH,KAAK,GAAG,MAAM,OAAO,CAAC,QAAQ,CAAC,EAAE,IAAI,EAAE,UAAU,CAAC,GAAG,CAAC,EAAE,SAAS,EAAE,GAAG,EAAE,CAAC,CAAC;YAC5E,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,KAAK,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAC3D,CAAC;YACD,4EAA4E;YAC5E,oDAAoD;YACpD,MAAM,MAAM,GAAyB,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;gBAC5D,MAAM,MAAM,GAAG,YAAY,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;gBAC9C,OAAO,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,QAAQ,KAAK,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC;YAC1F,CAAC,CAAC,CAAC;YACH,MAAM,EAAE,YAAY,EAAE,UAAU,EAAE,GAAG,YAAY,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC;YAClE,MAAM,UAAU,GAAG,YAAY,CAAC,GAAG,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;YACpD,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,GAAG,CAAC,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;YACjE,MAAM,UAAU,GAAG,UAAU,CAAC,MAAM,KAAK,CAAC,CAAC;YAC3C,MAAM,YAAY,GAAG,YAAY,CAAC,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;YACnE,MAAM,CAAC,IAAI,CAAC;gBACV,EAAE,EAAE,GAAG,CAAC,EAAE;gBACV,KAAK,EAAE,GAAG,CAAC,KAAK;gBAChB,IAAI,EAAE,GAAG,CAAC,IAAI;gBACd,UAAU,EAAE,GAAG,CAAC,UAAU;gBAC1B,YAAY;gBACZ,QAAQ,EAAE,UAAU,KAAK,IAAI;gBAC7B,UAAU;gBACV,KAAK;gBACL,YAAY;gBACZ,UAAU;gBACV,UAAU;gBACV,MAAM;gBACN,SAAS,EAAE,YAAY,CAAC,CAAC,CAAC,eAAe,CAAC,UAAU,KAAK,IAAI,EAAE,KAAK,EAAE,UAAU,CAAC,CAAC,CAAC,CAAC,IAAI;gBACxF,OAAO,EAAE,KAAK,CAAC,OAAO;gBACtB,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;gBAClC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aAC5B,CAAC,CAAC;QACL,CAAC;gBAAS,CAAC;YACT,IAAI,CAAC,OAAO,CAAC,cAAc;gBAAE,MAAM,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACnF,CAAC;IACH,CAAC;IAED,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,KAAK,IAAI,CAAC,CAAC;IAC1D,MAAM,GAAG,GAAc;QACrB,QAAQ;QACR,KAAK;QACL,SAAS;QACT,OAAO,EAAE,IAAI,CAAC,GAAG,EAAE;QACnB,MAAM;QACN,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACzC,OAAO,EAAE;YACP,KAAK,EAAE,MAAM,CAAC,MAAM;YACpB,QAAQ,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,MAAM;YACrD,QAAQ,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,MAAM;YACjD,KAAK,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,MAAM;YAC3C,QAAQ,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,UAAU,CAAC,CAAC,MAAM;YAC9D,SAAS,EACP,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,SAAS,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM;SACtG;KACF,CAAC;IACF,OAAO,GAAG,CAAC;AACb,CAAC;AAED,4CAA4C;AAC5C,MAAM,UAAU,kBAAkB,CAAC,GAAc;IAC/C,MAAM,KAAK,GAAG;QACZ,0BAA0B,GAAG,CAAC,QAAQ,IAAI,GAAG,CAAC,KAAK,EAAE;QACrD,KAAK,CAAC,CAAC,GAAG,CAAC,OAAO,GAAG,GAAG,CAAC,SAAS,CAAC,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,GAAG,CAAC,MAAM,CAAC,MAAM,UAAU;QACxF,EAAE;KACH,CAAC;IACF,IAAI,GAAG,CAAC,OAAO,EAAE,CAAC;QAChB,KAAK,CAAC,IAAI,CAAC,cAAc,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;QACxC,IAAI,GAAG,CAAC,YAAY;YAAE,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,wBAAwB,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC;QACjF,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC1B,CAAC;IACD,KAAK,MAAM,KAAK,IAAI,GAAG,CAAC,MAAM,EAAE,CAAC;QAC/B,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,IAAI,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;QAC7E,KAAK,CAAC,IAAI,CAAC,KAAK,IAAI,IAAI,KAAK,CAAC,EAAE,CAAC,MAAM,CAAC,EAAE,CAAC,cAAc,GAAG,CAAC,KAAK,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QACjF,KAAK,CAAC,IAAI,CACR,gBAAgB,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,KAAK,CAAC,UAAU,GAAG,CAAC,CAAC,CAAC,IAAI,aAAa,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK;YAC7G,YAAY,KAAK,CAAC,YAAY,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;YACzF,CAAC,KAAK,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,iBAAiB,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CACvF,CAAC;QACF,IAAI,KAAK,CAAC,KAAK;YAAE,KAAK,CAAC,IAAI,CAAC,oBAAoB,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC;IACjE,CAAC;IACD,MAAM,CAAC,GAAG,GAAG,CAAC,OAAO,CAAC;IACtB,KAAK,CAAC,IAAI,CACR,EAAE,EACF,WAAW,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,KAAK,YAAY,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,KAAK,0CAA0C;QACtG,GAAG,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,KAAK,gBAAgB,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,EAAE,CAC7D,CAAC;IACF,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED,2DAA2D;AAC3D,MAAM,UAAU,oBAAoB,CAAC,GAAc;IACjD,MAAM,KAAK,GAAG;QACZ,wBAAwB;QACxB,EAAE;QACF,6BAA6B,GAAG,CAAC,QAAQ,IAAI,GAAG,CAAC,KAAK,IAAI;QAC1D,eAAe,IAAI,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE;QACnE,mBAAmB,CAAC,CAAC,GAAG,CAAC,OAAO,GAAG,GAAG,CAAC,SAAS,CAAC,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG;QACvE,oBAAoB,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,SAAS,CAAC,kHAAkH;QAChK,EAAE;KACH,CAAC;IACF,IAAI,GAAG,CAAC,OAAO,EAAE,CAAC;QAChB,KAAK,CAAC,IAAI,CAAC,YAAY,EAAE,EAAE,EAAE,GAAG,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;QAC9C,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC1B,CAAC;IACD,KAAK,CAAC,IAAI,CACR,YAAY,EACZ,EAAE,EACF,iFAAiF,EACjF,+BAA+B,CAChC,CAAC;IACF,KAAK,MAAM,KAAK,IAAI,GAAG,CAAC,MAAM,EAAE,CAAC;QAC/B,KAAK,CAAC,IAAI,CACR,OAAO,KAAK,CAAC,EAAE,QAAQ,KAAK,CAAC,IAAI,MAAM,KAAK,CAAC,UAAU,MAAM,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,KAAK,CAAC,UAAU,GAAG,CAAC,CAAC,CAAC,IAAI,KAAK;YACnH,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,MAAM,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,QAAQ,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,MAAM,GAAG,CAAC,KAAK,CAAC,SAAS,CAAC,IAAI,CACpI,CAAC;IACJ,CAAC;IACD,MAAM,CAAC,GAAG,GAAG,CAAC,OAAO,CAAC;IACtB,KAAK,CAAC,IAAI,CACR,EAAE,EACF,YAAY,EACZ,EAAE,EACF,yBAAyB,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,KAAK,IAAI,EAClD,iBAAiB,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,KAAK,IAAI,EACvC,gDAAgD,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,KAAK,IAAI,EACzE,EAAE,EACF,qBAAqB,EACrB,EAAE,EACF,gFAAgF,EAChF,0FAA0F,EAC1F,+BAA+B,EAC/B,8CAA8C,EAC9C,0FAA0F,EAC1F,iGAAiG,EACjG,8FAA8F,EAC9F,2FAA2F,EAC3F,0EAA0E,EAC1E,iGAAiG,EACjG,0CAA0C,EAC1C,EAAE,CACH,CAAC;IACF,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED,SAAS,wBAAwB,CAAC,OAAoC;IACpE,OAAO,OAAO;SACX,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,EAAE,kBAAkB,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,iBAAiB,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;SACnI,IAAI,CAAC,IAAI,CAAC,CAAC;AAChB,CAAC;AAED,SAAS,GAAG,CAAC,KAAoB;IAC/B,OAAO,KAAK,KAAK,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,GAAG,CAAC,GAAG,CAAC;AAChE,CAAC;AAED,uFAAuF;AACvF,MAAM,UAAU,gBAAgB,CAAC,QAAgB,EAAE,KAAa;IAC9D,MAAM,SAAS,GAAG,KAAK,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;IAC9C,OAAO,IAAI,CAAC,MAAM,EAAE,YAAY,EAAE,eAAe,QAAQ,IAAI,SAAS,KAAK,CAAC,CAAC;AAC/E,CAAC;AAED,yDAAyD;AACzD,MAAM,UAAU,iBAAiB,CAAC,GAAc,EAAE,IAAI,GAAG,gBAAgB,CAAC,GAAG,CAAC,QAAQ,EAAE,GAAG,CAAC,KAAK,CAAC;IAChG,SAAS,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACjD,aAAa,CAAC,IAAI,EAAE,oBAAoB,CAAC,GAAG,CAAC,EAAE,OAAO,CAAC,CAAC;IACxD,OAAO,IAAI,CAAC;AACd,CAAC"}

package/dist/learning/seeded-bugs.d.ts

@@ -0,0 +1,142 @@
+/**
+ * WS7 (#29) — the seeded-bug benchmark.
+ *
+ * WHY THIS IS TEST INFRASTRUCTURE AND NOT A MATRIX ROW. `src/parity/matrix.ts`
+ * says it in one line: the seeded-bug benchmark is not a capability a surface can
+ * have, so it has no cells and no parity verdict of its own. What it has is a job —
+ * measure whether the agent can FIND and FIX a defect it was not told about, which
+ * is the thing a coding agent is for and the thing the M2b suite could only
+ * approximate: M2b tasks are written by hand and graded by hidden tests, but
+ * nothing proves the task was BROKEN before the agent ran, so a task that already
+ * passed measured nothing.
+ *
+ * THE PROPERTY THAT MAKES THIS A BENCHMARK RATHER THAN A TASK LIST. Every seeded
+ * bug is declared TWICE — the broken workspace the agent receives, and the same
+ * workspace after a reference fix — and `verifySeededBug` runs the checks against
+ * both. The checks must FAIL on the seed (and say so in their own words) and PASS
+ * after the fix. A seed that already passes its checks, or a check that cannot
+ * pass even when fixed, fails verification and is NOT scored: `runSeededSuite`
+ * verifies every task before it runs a single model call. `nuvira eval
+ * verify-seeds` exposes the same verification directly, so the ratchet is
+ * runnable from a terminal and in CI without a provider.
+ *
+ * WHAT 'FAILING' HAS TO MEAN. A check that exits non-zero because it crashed —
+ * a syntax error, a missing module — would satisfy a naive verification while
+ * proving nothing about the bug. So a seed counts as genuinely broken only when
+ * its check exits non-zero AND prints the failure marker its own assertions
+ * print (`FAIL`). Combined with "the same check passes once the fix is applied",
+ * that pins both ends: the check is well-formed, and it is the DEFECT that makes
+ * it fail.
+ *
+ * PLAIN NODE, ON PURPOSE. Every seed is a CommonJS `.js` file with a `.js` test
+ * script and no dependencies, so verification is deterministic, offline, and
+ * needs nothing but the `node` already running the suite. (A future Python seed
+ * would have to be gated on an interpreter actually being present rather than
+ * assumed — the honest version of "we also test Python".)
+ *
+ * Deliberately NOT measured here: collateral damage to files the fix should not
+ * touch is scored from a diff of the workspace against the seed
+ * (`src/learning/seeded-benchmark.ts`), not from reference patterns, because
+ * "this file still contains line X" is a weaker claim than "this file is
+ * unchanged" and the benchmark should not make the weak one.
+ */
+/** One file in a seeded workspace. */
+export interface SeedFile {
+    path: string;
+    content: string;
+}
+/** One check that must fail on the seed and pass after the fix. */
+export interface SeedCheck {
+    /** The script to run, relative to the workspace root. */
+    file: string;
+    /** The command that runs it (cwd = workspace root). */
+    command: string;
+}
+/** A deliberately seeded defect, with the workspace on both sides of the fix. */
+export interface SeededBug {
+    id: string;
+    title: string;
+    /** What kind of defect it is — the axis the benchmark reports on. */
+    axis: string;
+    difficulty: 'easy' | 'medium';
+    /**
+     * The defect, as GROUND TRUTH. Never shown to the agent: the whole measurement
+     * is whether it can find this from the failing check alone.
+     */
+    bug: string;
+    /** The workspace as the agent receives it: broken, and provably so. */
+    files: readonly SeedFile[];
+    /**
+     * The files the reference fix rewrites. Every path here must also appear in
+     * `files` — a fix that adds a file is a different shape and would make the
+     * before/after diff meaningless.
+     */
+    fixedFiles: readonly SeedFile[];
+    /** The file the fix belongs in. Everything else is collateral if it changes. */
+    target: string;
+    /** Checks that fail on the seed and pass after the reference fix. */
+    checks: readonly SeedCheck[];
+    /**
+     * Substrings (case-insensitive) that count as the agent having IDENTIFIED the
+     * defect in what it reported. Diagnostic vocabulary, not the fix: an agent that
+     * rewrites the function correctly but reports "done" has fixed it without
+     * finding it, and the two are scored apart.
+     */
+    detectionSignals: readonly string[];
+    /** Token budget for the run, so cost is scored alongside correctness. */
+    tokenBudget: number;
+}
+/** The exact request every seeded task makes — uniform, so the score compares competence. */
+export declare function seededGoal(bug: SeededBug): string;
+/**
+ * The suite.
+ *
+ * Each entry is a real defect with a real fix, small enough that a strong model
+ * can find it from the failing check and that the reference fix is unambiguous.
+ * The axes are the ones that actually bite in production code: an inclusive bound,
+ * a comparator that was never given, an async callback in a synchronous-looking
+ * call, a falsy guard that eats a legitimate zero, a copy that is only one level
+ * deep, a helper that mutates its caller's data, and a denominator that was not
+ * updated with the filter beside it.
+ */
+export declare const SEEDED_BUGS: readonly SeededBug[];
+/** A bug by id, or undefined. */
+export declare function seededBugById(id: string): SeededBug | undefined;
+/** Write a workspace to `dir`. `fixed` swaps in the reference fix. */
+export declare function materializeSeededBug(bug: SeededBug, dir: string, fixed?: boolean): string;
+/** One check's outcome in a workspace. */
+export interface SeedCheckResult {
+    command: string;
+    exitCode: number;
+    output: string;
+    durationMs: number;
+}
+/** Run one check in a workspace. Never throws — a failure IS the result. */
+export declare function runSeedCheck(workspace: string, check: SeedCheck, timeoutMs?: number): SeedCheckResult;
+/** What verification found for one seed. */
+export interface SeedVerification {
+    id: string;
+    title: string;
+    /** The check FAILED on the seeded workspace, and said so in its own words. */
+    seedFails: boolean;
+    /** The same check PASSED once the reference fix was applied. */
+    fixPasses: boolean;
+    ok: boolean;
+    /** Human-readable evidence, one line per side. */
+    detail: string;
+}
+/**
+ * Prove a seed is genuinely broken and genuinely fixable.
+ *
+ * Both halves are required, and the marker requirement is what stops a check that
+ * merely CRASHED from counting as evidence of a bug: a syntax error also exits
+ * non-zero, so "it failed" alone would verify nothing. The fix half is what proves
+ * the check is well-formed; the marker half is what proves the failure is the
+ * defect's.
+ */
+export declare function verifySeededBug(bug: SeededBug): SeedVerification;
+/** A table of verifications, ready to print. */
+export declare function formatSeedVerification(results: readonly SeedVerification[]): string;
+/** Read a file from a workspace, or null. */
+export declare function readWorkspaceFile(workspace: string, path: string): string | null;
+//# sourceMappingURL=seeded-bugs.d.ts.map

package/dist/learning/seeded-bugs.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"seeded-bugs.d.ts","sourceRoot":"","sources":["../../src/learning/seeded-bugs.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAyCG;AAOH,sCAAsC;AACtC,MAAM,WAAW,QAAQ;IACvB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,mEAAmE;AACnE,MAAM,WAAW,SAAS;IACxB,yDAAyD;IACzD,IAAI,EAAE,MAAM,CAAC;IACb,uDAAuD;IACvD,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,iFAAiF;AACjF,MAAM,WAAW,SAAS;IACxB,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,qEAAqE;IACrE,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,GAAG,QAAQ,CAAC;IAC9B;;;OAGG;IACH,GAAG,EAAE,MAAM,CAAC;IACZ,uEAAuE;IACvE,KAAK,EAAE,SAAS,QAAQ,EAAE,CAAC;IAC3B;;;;OAIG;IACH,UAAU,EAAE,SAAS,QAAQ,EAAE,CAAC;IAChC,gFAAgF;IAChF,MAAM,EAAE,MAAM,CAAC;IACf,qEAAqE;IACrE,MAAM,EAAE,SAAS,SAAS,EAAE,CAAC;IAC7B;;;;;OAKG;IACH,gBAAgB,EAAE,SAAS,MAAM,EAAE,CAAC;IACpC,yEAAyE;IACzE,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,6FAA6F;AAC7F,wBAAgB,UAAU,CAAC,GAAG,EAAE,SAAS,GAAG,MAAM,CAOjD;AA8FD;;;;;;;;;;GAUG;AACH,eAAO,MAAM,WAAW,EAAE,SAAS,SAAS,EAoQ3C,CAAC;AAEF,iCAAiC;AACjC,wBAAgB,aAAa,CAAC,EAAE,EAAE,MAAM,GAAG,SAAS,GAAG,SAAS,CAE/D;AAED,sEAAsE;AACtE,wBAAgB,oBAAoB,CAAC,GAAG,EAAE,SAAS,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,UAAQ,GAAG,MAAM,CAavF;AAED,0CAA0C;AAC1C,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,4EAA4E;AAC5E,wBAAgB,YAAY,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,SAAS,SAAS,GAAG,eAAe,CAyBrG;AAED,4CAA4C;AAC5C,MAAM,WAAW,gBAAgB;IAC/B,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,8EAA8E;IAC9E,SAAS,EAAE,OAAO,CAAC;IACnB,gEAAgE;IAChE,SAAS,EAAE,OAAO,CAAC;IACnB,EAAE,EAAE,OAAO,CAAC;IACZ,kDAAkD;IAClD,MAAM,EAAE,MAAM,CAAC;CAChB;AAED;;;;;;;;GAQG;AACH,wBAAgB,eAAe,CAAC,GAAG,EAAE,SAAS,GAAG,gBAAgB,CAsChE;AAOD,gDAAgD;AAChD,wBAAgB,sBAAsB,CAAC,OAAO,EAAE,SAAS,gBAAgB,EAAE,GAAG,MAAM,CAmBnF;AAED,6CAA6C;AAC7C,wBAAgB,iBAAiB,CAAC,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAMhF"}