@vercel/agent-eval 0.2.0 → 0.3.1

package/dist/lib/classifier.js

@@ -0,0 +1,268 @@
+/**
+ * Failure classification for eval results.
+ *
+ * Classifies failed eval runs as:
+ * - "model" — the model tried but wrote incorrect code
+ * - "infra" — infrastructure broke (API errors, rate limits, crashes)
+ * - "timeout" — the run hit its time limit
+ *
+ * Uses AI classification via the Vercel AI Gateway. Requires AI_GATEWAY_API_KEY.
+ */
+import { readFileSync, readdirSync, statSync, writeFileSync } from 'fs';
+import { join, resolve } from 'path';
+import { tool } from 'ai';
+import { z } from 'zod';
+const CLASSIFIER_SYSTEM_PROMPT = `You are a failure classifier for an AI coding agent benchmark.
+
+Your job: figure out WHY a failed eval run failed. Each eval tests whether an AI model can complete a coding task (e.g. migrate to App Router, add a Next.js feature). You have tools to explore the result files.
+
+Classify into one of:
+- "model" — the model tried but wrote incorrect code
+- "infra" — infrastructure broke (API errors, rate limits, crashes) and the model never got to do real work
+- "timeout" — the run hit its time limit
+
+The eval result directory contains run-1/ through run-N/ subdirectories (one per attempt, N depends on config), plus a summary.json. Each run directory has:
+- result.json — status, error, duration
+- transcript.json or transcript-raw.jsonl (or older results may have transcript.jsonl) — the agent's conversation log
+- outputs/eval.txt — EVAL.ts test output
+- outputs/scripts/*.txt — npm script outputs (e.g. build.txt), if the experiment configured scripts
+
+IMPORTANT: The eval harness always runs EVAL.ts tests after the agent finishes, plus any npm scripts configured in the experiment's \`scripts\` array (e.g. \`["build"]\`). These run even if the model produced nothing — tests just run against unmodified scaffold code (TODO placeholders). So test/script failures alone do NOT mean the model wrote code.
+
+The transcript is the key evidence. It records every action the model took. If there is no transcript file, or the transcript only shows errors (no tool calls or text output from the model), the model never actually ran — that's "infra". Only classify as "model" if you see evidence in the transcript that the model actually generated code.`;
+/**
+ * Validates and resolves a path, ensuring it stays within the allowed root.
+ */
+function safePath(root, relativePath) {
+    const resolved = resolve(root, relativePath);
+    if (!resolved.startsWith(root))
+        return null;
+    return resolved;
+}
+/**
+ * Creates sandboxed read-only tools for the AI classifier.
+ */
+export function createClassifierTools(evalResultDir) {
+    return {
+        list_files: tool({
+            description: 'List files and directories at a path relative to the eval result root. Use "." for the root.',
+            inputSchema: z.object({
+                path: z
+                    .string()
+                    .describe('Relative path to list, e.g. "." or "run-1" or "run-1/outputs"'),
+            }),
+            execute: async ({ path: relPath }) => {
+                const target = safePath(evalResultDir, relPath);
+                if (!target)
+                    return { error: 'Path outside allowed directory' };
+                try {
+                    const entries = readdirSync(target);
+                    const results = [];
+                    for (const entry of entries.sort()) {
+                        const info = statSync(join(target, entry));
+                        results.push({ name: entry, type: info.isDirectory() ? 'dir' : 'file' });
+                    }
+                    return { entries: results };
+                }
+                catch {
+                    return { error: `Cannot list: ${relPath}` };
+                }
+            },
+        }),
+        read_file: tool({
+            description: 'Read a file relative to the eval result root. For large files, use offset/limit to paginate.',
+            inputSchema: z.object({
+                path: z
+                    .string()
+                    .describe('Relative path to the file, e.g. "run-1/result.json"'),
+                offset: z
+                    .number()
+                    .describe('Line offset to start reading from (0-based)')
+                    .optional(),
+                limit: z
+                    .number()
+                    .describe('Max number of lines to return')
+                    .optional(),
+            }),
+            execute: async ({ path: relPath, offset: rawOffset, limit: rawLimit }) => {
+                const offset = rawOffset ?? 0;
+                const limit = rawLimit ?? 200;
+                const target = safePath(evalResultDir, relPath);
+                if (!target)
+                    return { error: 'Path outside allowed directory' };
+                try {
+                    const content = readFileSync(target, 'utf-8');
+                    const lines = content.split('\n');
+                    const sliced = lines.slice(offset, offset + limit);
+                    return {
+                        content: sliced.join('\n'),
+                        totalLines: lines.length,
+                        showing: `lines ${offset}-${Math.min(offset + limit, lines.length)} of ${lines.length}`,
+                    };
+                }
+                catch {
+                    return { error: `Cannot read: ${relPath}` };
+                }
+            },
+        }),
+        grep: tool({
+            description: 'Search for a pattern in files under a directory. Returns matching lines with context.',
+            inputSchema: z.object({
+                pattern: z.string().describe('Text or regex pattern to search for'),
+                path: z
+                    .string()
+                    .describe('Relative directory or file to search in, e.g. "." or "run-1"'),
+                maxResults: z
+                    .number()
+                    .describe('Max number of matches to return')
+                    .optional(),
+            }),
+            execute: async ({ pattern, path: relPath, maxResults: rawMax }) => {
+                const maxResults = rawMax ?? 20;
+                const target = safePath(evalResultDir, relPath);
+                if (!target)
+                    return { error: 'Path outside allowed directory' };
+                const regex = new RegExp(pattern, 'i');
+                const matches = [];
+                async function searchFile(filePath, relName) {
+                    try {
+                        const content = readFileSync(filePath, 'utf-8');
+                        const lines = content.split('\n');
+                        for (let i = 0; i < lines.length && matches.length < maxResults; i++) {
+                            if (regex.test(lines[i])) {
+                                matches.push({
+                                    file: relName,
+                                    line: i + 1,
+                                    text: lines[i].slice(0, 500),
+                                });
+                            }
+                        }
+                    }
+                    catch {
+                        // Skip unreadable files
+                    }
+                }
+                async function searchDir(dirPath, prefix) {
+                    try {
+                        const entries = readdirSync(dirPath);
+                        for (const entry of entries) {
+                            if (matches.length >= maxResults)
+                                break;
+                            const full = join(dirPath, entry);
+                            const rel = prefix ? `${prefix}/${entry}` : entry;
+                            const info = statSync(full);
+                            if (info.isDirectory()) {
+                                await searchDir(full, rel);
+                            }
+                            else {
+                                await searchFile(full, rel);
+                            }
+                        }
+                    }
+                    catch {
+                        // Skip unreadable dirs
+                    }
+                }
+                try {
+                    const info = statSync(target);
+                    if (info.isDirectory()) {
+                        await searchDir(target, relPath === '.' ? '' : relPath);
+                    }
+                    else {
+                        await searchFile(target, relPath);
+                    }
+                }
+                catch {
+                    return { error: `Path not found: ${relPath}` };
+                }
+                return {
+                    matches,
+                    totalFound: matches.length,
+                    truncated: matches.length >= maxResults,
+                };
+            },
+        }),
+    };
+}
+/**
+ * Classify a failure using AI via the Vercel AI Gateway.
+ * Requires AI_GATEWAY_API_KEY in the environment.
+ */
+export async function classifyWithAI(evalResultDir, evalName, experimentName) {
+    const { generateText, hasToolCall, createGateway } = await import('ai');
+    const gateway = createGateway({ apiKey: process.env.AI_GATEWAY_API_KEY ?? '' });
+    let classification = null;
+    const explorationTools = createClassifierTools(evalResultDir);
+    const allTools = {
+        ...explorationTools,
+        classify: tool({
+            description: 'Submit your final classification. Call this once you have enough evidence.',
+            inputSchema: z.object({
+                failureType: z
+                    .enum(['model', 'infra', 'timeout'])
+                    .describe('The failure category'),
+                failureReason: z
+                    .string()
+                    .describe('Brief 1-2 sentence explanation of why'),
+            }),
+            execute: async ({ failureType, failureReason }) => {
+                classification = { failureType: failureType, failureReason };
+                return { ok: true };
+            },
+        }),
+    };
+    try {
+        await generateText({
+            model: gateway('anthropic/claude-sonnet-4-5'),
+            system: CLASSIFIER_SYSTEM_PROMPT,
+            prompt: `Classify the failure for eval "${evalName}" (experiment: ${experimentName}). Use the exploration tools to investigate, then call classify() with your verdict.`,
+            tools: allTools,
+            stopWhen: hasToolCall('classify'),
+        });
+        return classification;
+    }
+    catch {
+        return null;
+    }
+}
+/**
+ * Classify a failed eval result using AI.
+ * Requires AI_GATEWAY_API_KEY in the environment.
+ *
+ * Caches results in classification.json within the eval result directory.
+ */
+export async function classifyFailure(evalResultDir, evalName, experimentName) {
+    // Check for cached classification
+    const cachedPath = join(evalResultDir, 'classification.json');
+    try {
+        const cached = JSON.parse(readFileSync(cachedPath, 'utf-8'));
+        if (cached.failureType && cached.failureReason) {
+            return { failureType: cached.failureType, failureReason: cached.failureReason };
+        }
+    }
+    catch {
+        // No cache
+    }
+    // Classify with AI
+    const classification = await classifyWithAI(evalResultDir, evalName, experimentName);
+    // Cache the result
+    if (classification) {
+        try {
+            writeFileSync(cachedPath, JSON.stringify(classification, null, 2));
+        }
+        catch {
+            // Non-fatal: caching failed
+        }
+    }
+    return classification;
+}
+/**
+ * Check if all runs of an eval failed with non-model failures.
+ * Used to determine if auto-retry is appropriate.
+ */
+export function shouldRetry(classifications) {
+    if (classifications.length === 0)
+        return false;
+    return classifications.every((c) => c.failureType !== 'model');
+}
+//# sourceMappingURL=classifier.js.map

package/dist/lib/classifier.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"classifier.js","sourceRoot":"","sources":["../../src/lib/classifier.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,EAAE,YAAY,EAAE,WAAW,EAAE,QAAQ,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AACxE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,MAAM,CAAC;AACrC,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC1B,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAGxB,MAAM,wBAAwB,GAAG;;;;;;;;;;;;;;;;;qVAiBoT,CAAC;AAEtV;;GAEG;AACH,SAAS,QAAQ,CAAC,IAAY,EAAE,YAAoB;IAClD,MAAM,QAAQ,GAAG,OAAO,CAAC,IAAI,EAAE,YAAY,CAAC,CAAC;IAC7C,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,IAAI,CAAC;QAAE,OAAO,IAAI,CAAC;IAC5C,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,qBAAqB,CAAC,aAAqB;IACzD,OAAO;QACL,UAAU,EAAE,IAAI,CAAC;YACf,WAAW,EACT,8FAA8F;YAChG,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;gBACpB,IAAI,EAAE,CAAC;qBACJ,MAAM,EAAE;qBACR,QAAQ,CAAC,+DAA+D,CAAC;aAC7E,CAAC;YACF,OAAO,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,EAAE,EAAE;gBACnC,MAAM,MAAM,GAAG,QAAQ,CAAC,aAAa,EAAE,OAAO,CAAC,CAAC;gBAChD,IAAI,CAAC,MAAM;oBAAE,OAAO,EAAE,KAAK,EAAE,gCAAgC,EAAE,CAAC;gBAChE,IAAI,CAAC;oBACH,MAAM,OAAO,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC;oBACpC,MAAM,OAAO,GAAkD,EAAE,CAAC;oBAClE,KAAK,MAAM,KAAK,IAAI,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC;wBACnC,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC;wBAC3C,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC;oBAC3E,CAAC;oBACD,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC;gBAC9B,CAAC;gBAAC,MAAM,CAAC;oBACP,OAAO,EAAE,KAAK,EAAE,gBAAgB,OAAO,EAAE,EAAE,CAAC;gBAC9C,CAAC;YACH,CAAC;SACF,CAAC;QAEF,SAAS,EAAE,IAAI,CAAC;YACd,WAAW,EACT,8FAA8F;YAChG,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;gBACpB,IAAI,EAAE,CAAC;qBACJ,MAAM,EAAE;qBACR,QAAQ,CAAC,qDAAqD,CAAC;gBAClE,MAAM,EAAE,CAAC;qBACN,MAAM,EAAE;qBACR,QAAQ,CAAC,6CAA6C,CAAC;qBACvD,QAAQ,EAAE;gBACb,KAAK,EAAE,CAAC;qBACL,MAAM,EAAE;qBACR,QAAQ,CAAC,+BAA+B,CAAC;qBACzC,QAAQ,EAAE;aACd,CAAC;YACF,OAAO,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,KAAK,EAAE,QAAQ,EAAE,EAAE,EAAE;gBACvE,MAAM,MAAM,GAAG,SAAS,IAAI,CAAC,CAAC;gBAC9B,MAAM,KAAK,GAAG,QAAQ,IAAI,GAAG,CAAC;gBAC9B,MAAM,MAAM,GAAG,QAAQ,CAAC,aAAa,EAAE,OAAO,CAAC,CAAC;gBAChD,IAAI,CAAC,MAAM;oBAAE,OAAO,EAAE,KAAK,EAAE,gCAAgC,EAAE,CAAC;gBAChE,IAAI,CAAC;oBACH,MAAM,OAAO,GAAG,YAAY,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;oBAC9C,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;oBAClC,MAAM,MAAM,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,EAAE,MAAM,GAAG,KAAK,CAAC,CAAC;oBACnD,OAAO;wBACL,OAAO,EAAE,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC;wBAC1B,UAAU,EAAE,KAAK,CAAC,MAAM;wBACxB,OAAO,EAAE,SAAS,MAAM,IAAI,IAAI,CAAC,GAAG,CAAC,MAAM,GAAG,KAAK,EAAE,KAAK,CAAC,MAAM,CAAC,OAAO,KAAK,CAAC,MAAM,EAAE;qBACxF,CAAC;gBACJ,CAAC;gBAAC,MAAM,CAAC;oBACP,OAAO,EAAE,KAAK,EAAE,gBAAgB,OAAO,EAAE,EAAE,CAAC;gBAC9C,CAAC;YACH,CAAC;SACF,CAAC;QAEF,IAAI,EAAE,IAAI,CAAC;YACT,WAAW,EACT,uFAAuF;YACzF,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;gBACpB,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,qCAAqC,CAAC;gBACnE,IAAI,EAAE,CAAC;qBACJ,MAAM,EAAE;qBACR,QAAQ,CAAC,8DAA8D,CAAC;gBAC3E,UAAU,EAAE,CAAC;qBACV,MAAM,EAAE;qBACR,QAAQ,CAAC,iCAAiC,CAAC;qBAC3C,QAAQ,EAAE;aACd,CAAC;YACF,OAAO,EAAE,KAAK,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,UAAU,EAAE,MAAM,EAAE,EAAE,EAAE;gBAChE,MAAM,UAAU,GAAG,MAAM,IAAI,EAAE,CAAC;gBAChC,MAAM,MAAM,GAAG,QAAQ,CAAC,aAAa,EAAE,OAAO,CAAC,CAAC;gBAChD,IAAI,CAAC,MAAM;oBAAE,OAAO,EAAE,KAAK,EAAE,gCAAgC,EAAE,CAAC;gBAChE,MAAM,KAAK,GAAG,IAAI,MAAM,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;gBACvC,MAAM,OAAO,GAAwD,EAAE,CAAC;gBAExE,KAAK,UAAU,UAAU,CAAC,QAAgB,EAAE,OAAe;oBACzD,IAAI,CAAC;wBACH,MAAM,OAAO,GAAG,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;wBAChD,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;wBAClC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,IAAI,OAAO,CAAC,MAAM,GAAG,UAAU,EAAE,CAAC,EAAE,EAAE,CAAC;4BACrE,IAAI,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;gCACzB,OAAO,CAAC,IAAI,CAAC;oCACX,IAAI,EAAE,OAAO;oCACb,IAAI,EAAE,CAAC,GAAG,CAAC;oCACX,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC;iCAC7B,CAAC,CAAC;4BACL,CAAC;wBACH,CAAC;oBACH,CAAC;oBAAC,MAAM,CAAC;wBACP,wBAAwB;oBAC1B,CAAC;gBACH,CAAC;gBAED,KAAK,UAAU,SAAS,CAAC,OAAe,EAAE,MAAc;oBACtD,IAAI,CAAC;wBACH,MAAM,OAAO,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC;wBACrC,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;4BAC5B,IAAI,OAAO,CAAC,MAAM,IAAI,UAAU;gCAAE,MAAM;4BACxC,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;4BAClC,MAAM,GAAG,GAAG,MAAM,CAAC,CAAC,CAAC,GAAG,MAAM,IAAI,KAAK,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC;4BAClD,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC;4BAC5B,IAAI,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC;gCACvB,MAAM,SAAS,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;4BAC7B,CAAC;iCAAM,CAAC;gCACN,MAAM,UAAU,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;4BAC9B,CAAC;wBACH,CAAC;oBACH,CAAC;oBAAC,MAAM,CAAC;wBACP,uBAAuB;oBACzB,CAAC;gBACH,CAAC;gBAED,IAAI,CAAC;oBACH,MAAM,IAAI,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC;oBAC9B,IAAI,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC;wBACvB,MAAM,SAAS,CAAC,MAAM,EAAE,OAAO,KAAK,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;oBAC1D,CAAC;yBAAM,CAAC;wBACN,MAAM,UAAU,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;oBACpC,CAAC;gBACH,CAAC;gBAAC,MAAM,CAAC;oBACP,OAAO,EAAE,KAAK,EAAE,mBAAmB,OAAO,EAAE,EAAE,CAAC;gBACjD,CAAC;gBAED,OAAO;oBACL,OAAO;oBACP,UAAU,EAAE,OAAO,CAAC,MAAM;oBAC1B,SAAS,EAAE,OAAO,CAAC,MAAM,IAAI,UAAU;iBACxC,CAAC;YACJ,CAAC;SACF,CAAC;KACH,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,aAAqB,EACrB,QAAgB,EAChB,cAAsB;IAEtB,MAAM,EAAE,YAAY,EAAE,WAAW,EAAE,aAAa,EAAE,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,CAAC;IAExE,MAAM,OAAO,GAAG,aAAa,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,GAAG,CAAC,kBAAkB,IAAI,EAAE,EAAE,CAAC,CAAC;IAEhF,IAAI,cAAc,GAA0B,IAAI,CAAC;IAEjD,MAAM,gBAAgB,GAAG,qBAAqB,CAAC,aAAa,CAAC,CAAC;IAC9D,MAAM,QAAQ,GAAG;QACf,GAAG,gBAAgB;QACnB,QAAQ,EAAE,IAAI,CAAC;YACb,WAAW,EAAE,4EAA4E;YACzF,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;gBACpB,WAAW,EAAE,CAAC;qBACX,IAAI,CAAC,CAAC,OAAO,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC;qBACnC,QAAQ,CAAC,sBAAsB,CAAC;gBACnC,aAAa,EAAE,CAAC;qBACb,MAAM,EAAE;qBACR,QAAQ,CAAC,uCAAuC,CAAC;aACrD,CAAC;YACF,OAAO,EAAE,KAAK,EAAE,EAAE,WAAW,EAAE,aAAa,EAAE,EAAE,EAAE;gBAChD,cAAc,GAAG,EAAE,WAAW,EAAE,WAA0B,EAAE,aAAa,EAAE,CAAC;gBAC5E,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC;YACtB,CAAC;SACF,CAAC;KACH,CAAC;IAEF,IAAI,CAAC;QACH,MAAM,YAAY,CAAC;YACjB,KAAK,EAAE,OAAO,CAAC,6BAA6B,CAAC;YAC7C,MAAM,EAAE,wBAAwB;YAChC,MAAM,EAAE,kCAAkC,QAAQ,kBAAkB,cAAc,sFAAsF;YACxK,KAAK,EAAE,QAAQ;YACf,QAAQ,EAAE,WAAW,CAAC,UAAU,CAAC;SAClC,CAAC,CAAC;QAEH,OAAO,cAAc,CAAC;IACxB,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,aAAqB,EACrB,QAAgB,EAChB,cAAsB;IAEtB,kCAAkC;IAClC,MAAM,UAAU,GAAG,IAAI,CAAC,aAAa,EAAE,qBAAqB,CAAC,CAAC;IAC9D,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC,CAAC;QAC7D,IAAI,MAAM,CAAC,WAAW,IAAI,MAAM,CAAC,aAAa,EAAE,CAAC;YAC/C,OAAO,EAAE,WAAW,EAAE,MAAM,CAAC,WAAW,EAAE,aAAa,EAAE,MAAM,CAAC,aAAa,EAAE,CAAC;QAClF,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,WAAW;IACb,CAAC;IAED,mBAAmB;IACnB,MAAM,cAAc,GAAG,MAAM,cAAc,CAAC,aAAa,EAAE,QAAQ,EAAE,cAAc,CAAC,CAAC;IAErF,mBAAmB;IACnB,IAAI,cAAc,EAAE,CAAC;QACnB,IAAI,CAAC;YACH,aAAa,CAAC,UAAU,EAAE,IAAI,CAAC,SAAS,CAAC,cAAc,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QACrE,CAAC;QAAC,MAAM,CAAC;YACP,4BAA4B;QAC9B,CAAC;IACH,CAAC;IAED,OAAO,cAAc,CAAC;AACxB,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,WAAW,CAAC,eAAiC;IAC3D,IAAI,eAAe,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IAC/C,OAAO,eAAe,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,KAAK,OAAO,CAAC,CAAC;AACjE,CAAC"}

package/dist/lib/fingerprint.d.ts

@@ -0,0 +1,15 @@
+/**
+ * Content fingerprinting for eval result reuse.
+ *
+ * A fingerprint captures the eval files + config fields that affect results.
+ * If the fingerprint matches and the result is valid, the eval can be skipped.
+ */
+import type { RunnableExperimentConfig } from './types.js';
+/**
+ * Compute a fingerprint for an (eval, config) pair.
+ *
+ * Hashes: all eval directory files + config fields that affect results.
+ * Returns a hex SHA-256 digest.
+ */
+export declare function computeFingerprint(evalPath: string, config: RunnableExperimentConfig): string;
+//# sourceMappingURL=fingerprint.d.ts.map

package/dist/lib/fingerprint.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"fingerprint.d.ts","sourceRoot":"","sources":["../../src/lib/fingerprint.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAKH,OAAO,KAAK,EAAE,wBAAwB,EAAE,MAAM,YAAY,CAAC;AAuC3D;;;;;GAKG;AACH,wBAAgB,kBAAkB,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,wBAAwB,GAAG,MAAM,CAuB7F"}

package/dist/lib/fingerprint.js

@@ -0,0 +1,59 @@
+/**
+ * Content fingerprinting for eval result reuse.
+ *
+ * A fingerprint captures the eval files + config fields that affect results.
+ * If the fingerprint matches and the result is valid, the eval can be skipped.
+ */
+import { createHash } from 'crypto';
+import { readFileSync, readdirSync, statSync } from 'fs';
+import { join } from 'path';
+/**
+ * Recursively collects all files in a directory, sorted for deterministic hashing.
+ * Skips node_modules and .git.
+ */
+function collectFiles(dir, basePath = '') {
+    const files = [];
+    const entries = readdirSync(dir).sort();
+    for (const entry of entries) {
+        if (entry === 'node_modules' || entry === '.git')
+            continue;
+        const fullPath = join(dir, entry);
+        const relativePath = basePath ? `${basePath}/${entry}` : entry;
+        const stat = statSync(fullPath);
+        if (stat.isDirectory()) {
+            files.push(...collectFiles(fullPath, relativePath));
+        }
+        else {
+            files.push({ relativePath, content: readFileSync(fullPath, 'utf-8') });
+        }
+    }
+    return files;
+}
+/**
+ * Compute a fingerprint for an (eval, config) pair.
+ *
+ * Hashes: all eval directory files + config fields that affect results.
+ * Returns a hex SHA-256 digest.
+ */
+export function computeFingerprint(evalPath, config) {
+    const hash = createHash('sha256');
+    // Hash all files in the eval directory (sorted for determinism)
+    const files = collectFiles(evalPath);
+    for (const file of files) {
+        hash.update(`file:${file.relativePath}\n`);
+        hash.update(file.content);
+        hash.update('\0');
+    }
+    // Hash config fields that affect results
+    const configForHash = {
+        agent: config.agent,
+        model: config.model,
+        scripts: [...config.scripts].sort(),
+        timeout: config.timeout,
+        earlyExit: config.earlyExit,
+        runs: config.runs,
+    };
+    hash.update(`config:${JSON.stringify(configForHash)}`);
+    return hash.digest('hex');
+}
+//# sourceMappingURL=fingerprint.js.map

package/dist/lib/fingerprint.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"fingerprint.js","sourceRoot":"","sources":["../../src/lib/fingerprint.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AACpC,OAAO,EAAE,YAAY,EAAE,WAAW,EAAE,QAAQ,EAAE,MAAM,IAAI,CAAC;AACzD,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAgB5B;;;GAGG;AACH,SAAS,YAAY,CAAC,GAAW,EAAE,WAAmB,EAAE;IACtD,MAAM,KAAK,GAAqD,EAAE,CAAC;IACnE,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;IAExC,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC5B,IAAI,KAAK,KAAK,cAAc,IAAI,KAAK,KAAK,MAAM;YAAE,SAAS;QAC3D,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QAClC,MAAM,YAAY,GAAG,QAAQ,CAAC,CAAC,CAAC,GAAG,QAAQ,IAAI,KAAK,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC;QAC/D,MAAM,IAAI,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAC;QAEhC,IAAI,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC;YACvB,KAAK,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC,CAAC;QACtD,CAAC;aAAM,CAAC;YACN,KAAK,CAAC,IAAI,CAAC,EAAE,YAAY,EAAE,OAAO,EAAE,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,EAAE,CAAC,CAAC;QACzE,CAAC;IACH,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,kBAAkB,CAAC,QAAgB,EAAE,MAAgC;IACnF,MAAM,IAAI,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC;IAElC,gEAAgE;IAChE,MAAM,KAAK,GAAG,YAAY,CAAC,QAAQ,CAAC,CAAC;IACrC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,CAAC,MAAM,CAAC,QAAQ,IAAI,CAAC,YAAY,IAAI,CAAC,CAAC;QAC3C,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAC1B,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACpB,CAAC;IAED,yCAAyC;IACzC,MAAM,aAAa,GAA0B;QAC3C,KAAK,EAAE,MAAM,CAAC,KAAK;QACnB,KAAK,EAAE,MAAM,CAAC,KAAK;QACnB,OAAO,EAAE,CAAC,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE;QACnC,OAAO,EAAE,MAAM,CAAC,OAAO;QACvB,SAAS,EAAE,MAAM,CAAC,SAAS;QAC3B,IAAI,EAAE,MAAM,CAAC,IAAI;KAClB,CAAC;IACF,IAAI,CAAC,MAAM,CAAC,UAAU,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC;IAEvD,OAAO,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAC5B,CAAC"}

package/dist/lib/housekeeping.d.ts

@@ -0,0 +1,25 @@
+/**
+ * Housekeeping for eval results.
+ *
+ * After experiments complete, consolidate results:
+ * - For each (experiment, eval) pair: keep only the latest valid result
+ * - Remove older duplicates and dangling/incomplete results
+ * - Remove empty timestamp directories
+ */
+interface HousekeepingStats {
+    removedDuplicates: number;
+    removedIncomplete: number;
+    removedEmptyDirs: number;
+}
+/**
+ * Run housekeeping on a single experiment's results directory.
+ *
+ * For each eval: keeps the newest complete result (has summary.json and
+ * at least one transcript), removes older duplicates and incomplete results.
+ * Removes empty timestamp directories afterward.
+ */
+export declare function housekeep(resultsDir: string, experimentName: string, options?: {
+    dry?: boolean;
+}): HousekeepingStats;
+export {};
+//# sourceMappingURL=housekeeping.d.ts.map

package/dist/lib/housekeeping.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"housekeeping.d.ts","sourceRoot":"","sources":["../../src/lib/housekeeping.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAKH,UAAU,iBAAiB;IACzB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,iBAAiB,EAAE,MAAM,CAAC;IAC1B,gBAAgB,EAAE,MAAM,CAAC;CAC1B;AAED;;;;;;GAMG;AACH,wBAAgB,SAAS,CACvB,UAAU,EAAE,MAAM,EAClB,cAAc,EAAE,MAAM,EACtB,OAAO,CAAC,EAAE;IAAE,GAAG,CAAC,EAAE,OAAO,CAAA;CAAE,GAC1B,iBAAiB,CAwFnB"}

package/dist/lib/housekeeping.js

@@ -0,0 +1,148 @@
+/**
+ * Housekeeping for eval results.
+ *
+ * After experiments complete, consolidate results:
+ * - For each (experiment, eval) pair: keep only the latest valid result
+ * - Remove older duplicates and dangling/incomplete results
+ * - Remove empty timestamp directories
+ */
+import { readdirSync, rmSync, existsSync, readFileSync, statSync } from 'fs';
+import { join } from 'path';
+/**
+ * Run housekeeping on a single experiment's results directory.
+ *
+ * For each eval: keeps the newest complete result (has summary.json and
+ * at least one transcript), removes older duplicates and incomplete results.
+ * Removes empty timestamp directories afterward.
+ */
+export function housekeep(resultsDir, experimentName, options) {
+    const stats = {
+        removedDuplicates: 0,
+        removedIncomplete: 0,
+        removedEmptyDirs: 0,
+    };
+    const experimentDir = join(resultsDir, experimentName);
+    if (!existsSync(experimentDir))
+        return stats;
+    // Get all timestamps sorted newest first
+    let timestamps;
+    try {
+        timestamps = readdirSync(experimentDir)
+            .filter((t) => !t.startsWith('.'))
+            .filter((t) => {
+            try {
+                return statSync(join(experimentDir, t)).isDirectory();
+            }
+            catch {
+                return false;
+            }
+        })
+            .sort()
+            .reverse();
+    }
+    catch {
+        return stats;
+    }
+    // Track which (eval, fingerprint) pairs we've already seen (newest wins).
+    // Results with different fingerprints (e.g. smoke vs full run) are not
+    // duplicates of each other and should coexist.
+    const seenEvals = new Set();
+    for (const timestamp of timestamps) {
+        const tsDir = join(experimentDir, timestamp);
+        let evalDirs;
+        try {
+            evalDirs = readdirSync(tsDir).filter((d) => !d.startsWith('.'));
+        }
+        catch {
+            continue;
+        }
+        for (const evalDir of evalDirs) {
+            const evalResultDir = join(tsDir, evalDir);
+            if (!statSync(evalResultDir).isDirectory())
+                continue;
+            // Read fingerprint to distinguish different configs (e.g. smoke vs full)
+            const fingerprint = readFingerprint(evalResultDir);
+            const dedupeKey = fingerprint ? `${evalDir}:${fingerprint}` : evalDir;
+            if (seenEvals.has(dedupeKey)) {
+                // Older duplicate with same fingerprint — remove
+                if (!options?.dry) {
+                    rmSync(evalResultDir, { recursive: true });
+                }
+                stats.removedDuplicates++;
+                continue;
+            }
+            // Check if this result is complete
+            if (isComplete(evalResultDir)) {
+                seenEvals.add(dedupeKey);
+            }
+            else {
+                // Incomplete — remove
+                if (!options?.dry) {
+                    rmSync(evalResultDir, { recursive: true });
+                }
+                stats.removedIncomplete++;
+            }
+        }
+        // Check if timestamp dir is now empty
+        try {
+            const remaining = readdirSync(tsDir).filter((d) => !d.startsWith('.'));
+            if (remaining.length === 0) {
+                if (!options?.dry) {
+                    rmSync(tsDir, { recursive: true });
+                }
+                stats.removedEmptyDirs++;
+            }
+        }
+        catch {
+            // Directory already removed or inaccessible
+        }
+    }
+    return stats;
+}
+/**
+ * Read the fingerprint from an eval result's summary.json, if present.
+ */
+function readFingerprint(evalResultDir) {
+    try {
+        const summary = JSON.parse(readFileSync(join(evalResultDir, 'summary.json'), 'utf-8'));
+        return summary.fingerprint;
+    }
+    catch {
+        return undefined;
+    }
+}
+/**
+ * Check if an eval result directory is complete.
+ * Complete means: has summary.json and at least one run with a transcript.
+ */
+function isComplete(evalResultDir) {
+    const summaryPath = join(evalResultDir, 'summary.json');
+    if (!existsSync(summaryPath))
+        return false;
+    // Check for at least one transcript
+    try {
+        const entries = readdirSync(evalResultDir);
+        for (const entry of entries) {
+            if (!entry.startsWith('run-'))
+                continue;
+            const runDir = join(evalResultDir, entry);
+            if (existsSync(join(runDir, 'transcript-raw.jsonl')) ||
+                existsSync(join(runDir, 'transcript.json'))) {
+                return true;
+            }
+        }
+    }
+    catch {
+        return false;
+    }
+    // No transcript found — but summary.json exists.
+    // Still consider complete if summary shows 0% (model produced nothing, which is valid).
+    try {
+        const summary = JSON.parse(readFileSync(summaryPath, 'utf-8'));
+        return summary.totalRuns > 0;
+    }
+    catch {
+        return false;
+    }
+}
+//# sourceMappingURL=housekeeping.js.map

package/dist/lib/housekeeping.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"housekeeping.js","sourceRoot":"","sources":["../../src/lib/housekeeping.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EAAE,WAAW,EAAE,MAAM,EAAE,UAAU,EAAE,YAAY,EAAE,QAAQ,EAAE,MAAM,IAAI,CAAC;AAC7E,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAQ5B;;;;;;GAMG;AACH,MAAM,UAAU,SAAS,CACvB,UAAkB,EAClB,cAAsB,EACtB,OAA2B;IAE3B,MAAM,KAAK,GAAsB;QAC/B,iBAAiB,EAAE,CAAC;QACpB,iBAAiB,EAAE,CAAC;QACpB,gBAAgB,EAAE,CAAC;KACpB,CAAC;IAEF,MAAM,aAAa,GAAG,IAAI,CAAC,UAAU,EAAE,cAAc,CAAC,CAAC;IACvD,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC;QAAE,OAAO,KAAK,CAAC;IAE7C,yCAAyC;IACzC,IAAI,UAAoB,CAAC;IACzB,IAAI,CAAC;QACH,UAAU,GAAG,WAAW,CAAC,aAAa,CAAC;aACpC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;aACjC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE;YACZ,IAAI,CAAC;gBACH,OAAO,QAAQ,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;YACxD,CAAC;YAAC,MAAM,CAAC;gBACP,OAAO,KAAK,CAAC;YACf,CAAC;QACH,CAAC,CAAC;aACD,IAAI,EAAE;aACN,OAAO,EAAE,CAAC;IACf,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;IAED,0EAA0E;IAC1E,uEAAuE;IACvE,+CAA+C;IAC/C,MAAM,SAAS,GAAG,IAAI,GAAG,EAAU,CAAC;IAEpC,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;QACnC,MAAM,KAAK,GAAG,IAAI,CAAC,aAAa,EAAE,SAAS,CAAC,CAAC;QAE7C,IAAI,QAAkB,CAAC;QACvB,IAAI,CAAC;YACH,QAAQ,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC;QAClE,CAAC;QAAC,MAAM,CAAC;YACP,SAAS;QACX,CAAC;QAED,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC/B,MAAM,aAAa,GAAG,IAAI,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;YAE3C,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC,WAAW,EAAE;gBAAE,SAAS;YAErD,yEAAyE;YACzE,MAAM,WAAW,GAAG,eAAe,CAAC,aAAa,CAAC,CAAC;YACnD,MAAM,SAAS,GAAG,WAAW,CAAC,CAAC,CAAC,GAAG,OAAO,IAAI,WAAW,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC;YAEtE,IAAI,SAAS,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC;gBAC7B,iDAAiD;gBACjD,IAAI,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC;oBAClB,MAAM,CAAC,aAAa,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;gBAC7C,CAAC;gBACD,KAAK,CAAC,iBAAiB,EAAE,CAAC;gBAC1B,SAAS;YACX,CAAC;YAED,mCAAmC;YACnC,IAAI,UAAU,CAAC,aAAa,CAAC,EAAE,CAAC;gBAC9B,SAAS,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;YAC3B,CAAC;iBAAM,CAAC;gBACN,sBAAsB;gBACtB,IAAI,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC;oBAClB,MAAM,CAAC,aAAa,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;gBAC7C,CAAC;gBACD,KAAK,CAAC,iBAAiB,EAAE,CAAC;YAC5B,CAAC;QACH,CAAC;QAED,sCAAsC;QACtC,IAAI,CAAC;YACH,MAAM,SAAS,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC;YACvE,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC3B,IAAI,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC;oBAClB,MAAM,CAAC,KAAK,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;gBACrC,CAAC;gBACD,KAAK,CAAC,gBAAgB,EAAE,CAAC;YAC3B,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,4CAA4C;QAC9C,CAAC;IACH,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;GAEG;AACH,SAAS,eAAe,CAAC,aAAqB;IAC5C,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,IAAI,CAAC,aAAa,EAAE,cAAc,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC;QACvF,OAAO,OAAO,CAAC,WAAW,CAAC;IAC7B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,SAAS,CAAC;IACnB,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,SAAS,UAAU,CAAC,aAAqB;IACvC,MAAM,WAAW,GAAG,IAAI,CAAC,aAAa,EAAE,cAAc,CAAC,CAAC;IACxD,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC;QAAE,OAAO,KAAK,CAAC;IAE3C,oCAAoC;IACpC,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,WAAW,CAAC,aAAa,CAAC,CAAC;QAC3C,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC5B,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,MAAM,CAAC;gBAAE,SAAS;YACxC,MAAM,MAAM,GAAG,IAAI,CAAC,aAAa,EAAE,KAAK,CAAC,CAAC;YAC1C,IACE,UAAU,CAAC,IAAI,CAAC,MAAM,EAAE,sBAAsB,CAAC,CAAC;gBAChD,UAAU,CAAC,IAAI,CAAC,MAAM,EAAE,iBAAiB,CAAC,CAAC,EAC3C,CAAC;gBACD,OAAO,IAAI,CAAC;YACd,CAAC;QACH,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;IAED,iDAAiD;IACjD,wFAAwF;IACxF,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC,CAAC;QAC/D,OAAO,OAAO,CAAC,SAAS,GAAG,CAAC,CAAC;IAC/B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC"}

package/dist/lib/results.d.ts

@@ -23,6 +23,15 @@ export interface SaveResultsOptions {
     resultsDir: string;
     /** Experiment name (used for subdirectory) */
     experimentName: string;
+    /** Per-eval fingerprints (eval name -> fingerprint hash) */
+    fingerprints?: Record<string, string>;
+    /** Per-eval classification results (eval name -> classification) */
+    classifications?: Record<string, {
+        failureType: string;
+        failureReason: string;
+    }>;
+    /** Per-eval validity flags (eval name -> valid). Defaults to true. */
+    validity?: Record<string, boolean>;
 }
 /**
  * Save experiment results to disk.
@@ -52,4 +61,24 @@ export declare function formatRunResult(evalName: string, runNumber: number, tot
  * Create a progress indicator for running evals.
  */
 export declare function createProgressDisplay(evalName: string, runNumber: number, totalRuns: number): string;
+/**
+ * A reusable result found by the scanner.
+ */
+export interface ReusableResult {
+    evalName: string;
+    fingerprint: string;
+    passRate: string;
+    timestamp: string;
+}
+/**
+ * Scan existing results for an experiment to find reusable eval results.
+ *
+ * A result is reusable if:
+ * 1. Its fingerprint matches the current fingerprint
+ * 2. It is "valid" (not marked as invalid by the classifier)
+ * 3. It has passedRuns > 0 (successful result worth reusing)
+ *
+ * Scans all timestamps newest-first and returns the latest match per eval.
+ */
+export declare function scanReusableResults(resultsDir: string, experimentName: string, fingerprints: Record<string, string>): Map<string, ReusableResult>;
 //# sourceMappingURL=results.d.ts.map

package/dist/lib/results.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"results.d.ts","sourceRoot":"","sources":["../../src/lib/results.ts"],"names":[],"mappings":"AAAA;;GAEG;AAKH,OAAO,KAAK,EACV,aAAa,EACb,WAAW,EACX,WAAW,EACX,iBAAiB,EACjB,wBAAwB,EACzB,MAAM,YAAY,CAAC;AACpB,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAGxD;;GAEG;AACH,wBAAgB,wBAAwB,CAAC,WAAW,EAAE,cAAc,GAAG,WAAW,CA4BjF;AAED;;GAEG;AACH,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,WAAW,EAAE,GAAG,WAAW,CAanF;AAED;;GAEG;AACH,wBAAgB,uBAAuB,CACrC,MAAM,EAAE,wBAAwB,EAChC,KAAK,EAAE,WAAW,EAAE,EACpB,SAAS,EAAE,IAAI,EACf,WAAW,EAAE,IAAI,GAChB,iBAAiB,CAOnB;AAED;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC,iCAAiC;IACjC,UAAU,EAAE,MAAM,CAAC;IACnB,8CAA8C;IAC9C,cAAc,EAAE,MAAM,CAAC;CACxB;AAED;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,WAAW,CACzB,OAAO,EAAE,iBAAiB,EAC1B,OAAO,EAAE,kBAAkB,GAC1B,MAAM,CAoGR;AAED;;GAEG;AACH,wBAAgB,kBAAkB,CAAC,OAAO,EAAE,iBAAiB,GAAG,MAAM,CAsCrE;AAED;;GAEG;AACH,wBAAgB,eAAe,CAC7B,QAAQ,EAAE,MAAM,EAChB,SAAS,EAAE,MAAM,EACjB,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,aAAa,GACpB,MAAM,CAYR;AAED;;GAEG;AACH,wBAAgB,qBAAqB,CACnC,QAAQ,EAAE,MAAM,EAChB,SAAS,EAAE,MAAM,EACjB,SAAS,EAAE,MAAM,GAChB,MAAM,CAER"}
+{"version":3,"file":"results.d.ts","sourceRoot":"","sources":["../../src/lib/results.ts"],"names":[],"mappings":"AAAA;;GAEG;AAKH,OAAO,KAAK,EACV,aAAa,EACb,WAAW,EACX,WAAW,EACX,iBAAiB,EACjB,wBAAwB,EACzB,MAAM,YAAY,CAAC;AACpB,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAGxD;;GAEG;AACH,wBAAgB,wBAAwB,CAAC,WAAW,EAAE,cAAc,GAAG,WAAW,CA4BjF;AAED;;GAEG;AACH,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,WAAW,EAAE,GAAG,WAAW,CAanF;AAED;;GAEG;AACH,wBAAgB,uBAAuB,CACrC,MAAM,EAAE,wBAAwB,EAChC,KAAK,EAAE,WAAW,EAAE,EACpB,SAAS,EAAE,IAAI,EACf,WAAW,EAAE,IAAI,GAChB,iBAAiB,CAOnB;AAED;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC,iCAAiC;IACjC,UAAU,EAAE,MAAM,CAAC;IACnB,8CAA8C;IAC9C,cAAc,EAAE,MAAM,CAAC;IACvB,4DAA4D;IAC5D,YAAY,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACtC,oEAAoE;IACpE,eAAe,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE;QAAE,WAAW,EAAE,MAAM,CAAC;QAAC,aAAa,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACjF,sEAAsE;IACtE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,WAAW,CACzB,OAAO,EAAE,iBAAiB,EAC1B,OAAO,EAAE,kBAAkB,GAC1B,MAAM,CAkHR;AAED;;GAEG;AACH,wBAAgB,kBAAkB,CAAC,OAAO,EAAE,iBAAiB,GAAG,MAAM,CAsCrE;AAED;;GAEG;AACH,wBAAgB,eAAe,CAC7B,QAAQ,EAAE,MAAM,EAChB,SAAS,EAAE,MAAM,EACjB,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,aAAa,GACpB,MAAM,CAYR;AAED;;GAEG;AACH,wBAAgB,qBAAqB,CACnC,QAAQ,EAAE,MAAM,EAChB,SAAS,EAAE,MAAM,EACjB,SAAS,EAAE,MAAM,GAChB,MAAM,CAER;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED;;;;;;;;;GASG;AACH,wBAAgB,mBAAmB,CACjC,UAAU,EAAE,MAAM,EAClB,cAAc,EAAE,MAAM,EACtB,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GACnC,GAAG,CAAC,MAAM,EAAE,cAAc,CAAC,CA8D7B"}