agent-eval-opencode 0.10.0

package/dist/lib/fixture.js

@@ -0,0 +1,215 @@
+/**
+ * Eval fixture discovery and validation.
+ */
+import { readFileSync, readdirSync, statSync, existsSync } from 'fs';
+import { join, resolve } from 'path';
+import { REQUIRED_EVAL_FILES, EXCLUDED_FILES } from './types.js';
+/**
+ * Error thrown when an eval fixture is invalid.
+ */
+export class FixtureValidationError extends Error {
+    fixtureName;
+    constructor(fixtureName, message) {
+        super(`Eval "${fixtureName}": ${message}`);
+        this.fixtureName = fixtureName;
+        this.name = 'FixtureValidationError';
+    }
+}
+/**
+ * Check if a file exists with exact case match (case-sensitive even on Mac/Windows).
+ * Returns true only if the file exists AND the case matches exactly.
+ */
+function existsWithExactCase(dirPath, fileName) {
+    try {
+        const files = readdirSync(dirPath);
+        return files.includes(fileName);
+    }
+    catch {
+        return false;
+    }
+}
+/**
+ * Discovers all eval fixtures in a directory.
+ * Recursively searches for directories containing PROMPT.md.
+ * Supports nested organization like "vercel-cli/deploy" or "flags/create-flag".
+ */
+export function discoverFixtures(evalsDir) {
+    const absolutePath = resolve(evalsDir);
+    if (!existsSync(absolutePath)) {
+        throw new Error(`Evals directory not found: ${absolutePath}`);
+    }
+    const fixtures = [];
+    function walk(dir, basePath = '') {
+        const entries = readdirSync(dir);
+        for (const entry of entries) {
+            // Skip hidden directories and files
+            if (entry.startsWith('.')) {
+                continue;
+            }
+            const entryPath = join(dir, entry);
+            // Only consider directories
+            if (!statSync(entryPath).isDirectory()) {
+                continue;
+            }
+            const relativePath = basePath ? `${basePath}/${entry}` : entry;
+            // Check if this directory contains PROMPT.md (potential fixture)
+            if (existsWithExactCase(entryPath, 'PROMPT.md')) {
+                fixtures.push(relativePath);
+            }
+            else {
+                // Not a fixture directory, recurse into it
+                walk(entryPath, relativePath);
+            }
+        }
+    }
+    walk(absolutePath);
+    return fixtures.sort();
+}
+/**
+ * Validates that a fixture has all required files with correct case.
+ * Returns an array of missing file names, or empty array if valid.
+ * Note: Accepts either EVAL.ts or EVAL.tsx for the eval file.
+ * Case-sensitive: 'prompt.md' will fail even on Mac/Windows.
+ */
+export function validateFixtureFiles(fixturePath) {
+    const missing = [];
+    for (const file of REQUIRED_EVAL_FILES) {
+        // Special case: Accept either EVAL.ts or EVAL.tsx (both case-sensitive)
+        if (file === 'EVAL.ts') {
+            const hasEvalTs = existsWithExactCase(fixturePath, 'EVAL.ts');
+            const hasEvalTsx = existsWithExactCase(fixturePath, 'EVAL.tsx');
+            if (!hasEvalTs && !hasEvalTsx) {
+                missing.push('EVAL.ts or EVAL.tsx');
+            }
+        }
+        else {
+            if (!existsWithExactCase(fixturePath, file)) {
+                missing.push(file);
+            }
+        }
+    }
+    return missing;
+}
+/**
+ * Validates the package.json of a fixture.
+ * Ensures it has "type": "module".
+ */
+export function validatePackageJson(fixturePath) {
+    const packageJsonPath = join(fixturePath, 'package.json');
+    try {
+        const content = readFileSync(packageJsonPath, 'utf-8');
+        const pkg = JSON.parse(content);
+        if (pkg.type !== 'module') {
+            return {
+                isModule: false,
+                error: 'package.json must have "type": "module"',
+            };
+        }
+        return { isModule: true };
+    }
+    catch (error) {
+        if (error instanceof SyntaxError) {
+            return {
+                isModule: false,
+                error: 'package.json is not valid JSON',
+            };
+        }
+        throw error;
+    }
+}
+/**
+ * Loads a single eval fixture with full validation.
+ */
+export function loadFixture(evalsDir, name) {
+    const fixturePath = resolve(evalsDir, name);
+    if (!existsSync(fixturePath)) {
+        throw new FixtureValidationError(name, `Directory not found: ${fixturePath}`);
+    }
+    // Validate required files
+    const missingFiles = validateFixtureFiles(fixturePath);
+    if (missingFiles.length > 0) {
+        throw new FixtureValidationError(name, `Missing required files: ${missingFiles.join(', ')}`);
+    }
+    // Validate package.json
+    const pkgValidation = validatePackageJson(fixturePath);
+    if (pkgValidation.error) {
+        throw new FixtureValidationError(name, pkgValidation.error);
+    }
+    // Read prompt (case-sensitive check)
+    if (!existsWithExactCase(fixturePath, 'PROMPT.md')) {
+        throw new FixtureValidationError(name, 'PROMPT.md not found (case-sensitive: must be uppercase)');
+    }
+    const promptPath = join(fixturePath, 'PROMPT.md');
+    const prompt = readFileSync(promptPath, 'utf-8');
+    return {
+        name,
+        path: fixturePath,
+        prompt,
+        isModule: pkgValidation.isModule,
+    };
+}
+/**
+ * Discovers and loads all valid eval fixtures from a directory.
+ * Returns both valid fixtures and any validation errors encountered.
+ */
+export function loadAllFixtures(evalsDir) {
+    const fixtureNames = discoverFixtures(evalsDir);
+    const fixtures = [];
+    const errors = [];
+    for (const name of fixtureNames) {
+        try {
+            const fixture = loadFixture(evalsDir, name);
+            fixtures.push(fixture);
+        }
+        catch (error) {
+            if (error instanceof FixtureValidationError) {
+                errors.push(error);
+            }
+            else {
+                throw error;
+            }
+        }
+    }
+    return { fixtures, errors };
+}
+/**
+ * Gets a list of all files in a fixture directory.
+ * Excludes PROMPT.md, EVAL.ts, node_modules, and .git.
+ */
+export function getFixtureFiles(fixturePath, excludePatterns = EXCLUDED_FILES) {
+    const files = [];
+    function walk(dir, basePath = '') {
+        const entries = readdirSync(dir);
+        for (const entry of entries) {
+            const relativePath = basePath ? `${basePath}/${entry}` : entry;
+            // Check if should be excluded
+            if (excludePatterns.some((pattern) => relativePath === pattern || entry === pattern)) {
+                continue;
+            }
+            const fullPath = join(dir, entry);
+            const stat = statSync(fullPath);
+            if (stat.isDirectory()) {
+                walk(fullPath, relativePath);
+            }
+            else {
+                files.push(relativePath);
+            }
+        }
+    }
+    walk(fixturePath);
+    return files.sort();
+}
+/**
+ * Reads all fixture files into a map.
+ * Keys are relative paths, values are file contents.
+ */
+export function readFixtureFiles(fixturePath, excludePatterns) {
+    const files = getFixtureFiles(fixturePath, excludePatterns);
+    const contents = new Map();
+    for (const file of files) {
+        const fullPath = join(fixturePath, file);
+        contents.set(file, readFileSync(fullPath, 'utf-8'));
+    }
+    return contents;
+}
+//# sourceMappingURL=fixture.js.map

package/dist/lib/fixture.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"fixture.js","sourceRoot":"","sources":["../../src/lib/fixture.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,YAAY,EAAE,WAAW,EAAE,QAAQ,EAAE,UAAU,EAAE,MAAM,IAAI,CAAC;AACrE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,MAAM,CAAC;AAErC,OAAO,EAAE,mBAAmB,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAEjE;;GAEG;AACH,MAAM,OAAO,sBAAuB,SAAQ,KAAK;IAEtC;IADT,YACS,WAAmB,EAC1B,OAAe;QAEf,KAAK,CAAC,SAAS,WAAW,MAAM,OAAO,EAAE,CAAC,CAAC;QAHpC,gBAAW,GAAX,WAAW,CAAQ;QAI1B,IAAI,CAAC,IAAI,GAAG,wBAAwB,CAAC;IACvC,CAAC;CACF;AAED;;;GAGG;AACH,SAAS,mBAAmB,CAAC,OAAe,EAAE,QAAgB;IAC5D,IAAI,CAAC;QACH,MAAM,KAAK,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC;QACnC,OAAO,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IAClC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,gBAAgB,CAAC,QAAgB;IAC/C,MAAM,YAAY,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC;IAEvC,IAAI,CAAC,UAAU,CAAC,YAAY,CAAC,EAAE,CAAC;QAC9B,MAAM,IAAI,KAAK,CAAC,8BAA8B,YAAY,EAAE,CAAC,CAAC;IAChE,CAAC;IAED,MAAM,QAAQ,GAAa,EAAE,CAAC;IAE9B,SAAS,IAAI,CAAC,GAAW,EAAE,WAAmB,EAAE;QAC9C,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC;QAEjC,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC5B,oCAAoC;YACpC,IAAI,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;gBAC1B,SAAS;YACX,CAAC;YAED,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;YAEnC,4BAA4B;YAC5B,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,WAAW,EAAE,EAAE,CAAC;gBACvC,SAAS;YACX,CAAC;YAED,MAAM,YAAY,GAAG,QAAQ,CAAC,CAAC,CAAC,GAAG,QAAQ,IAAI,KAAK,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC;YAE/D,iEAAiE;YACjE,IAAI,mBAAmB,CAAC,SAAS,EAAE,WAAW,CAAC,EAAE,CAAC;gBAChD,QAAQ,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;YAC9B,CAAC;iBAAM,CAAC;gBACN,2CAA2C;gBAC3C,IAAI,CAAC,SAAS,EAAE,YAAY,CAAC,CAAC;YAChC,CAAC;QACH,CAAC;IACH,CAAC;IAED,IAAI,CAAC,YAAY,CAAC,CAAC;IACnB,OAAO,QAAQ,CAAC,IAAI,EAAE,CAAC;AACzB,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,oBAAoB,CAAC,WAAmB;IACtD,MAAM,OAAO,GAAa,EAAE,CAAC;IAE7B,KAAK,MAAM,IAAI,IAAI,mBAAmB,EAAE,CAAC;QACvC,wEAAwE;QACxE,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;YACvB,MAAM,SAAS,GAAG,mBAAmB,CAAC,WAAW,EAAE,SAAS,CAAC,CAAC;YAC9D,MAAM,UAAU,GAAG,mBAAmB,CAAC,WAAW,EAAE,UAAU,CAAC,CAAC;YAChE,IAAI,CAAC,SAAS,IAAI,CAAC,UAAU,EAAE,CAAC;gBAC9B,OAAO,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;YACtC,CAAC;QACH,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,mBAAmB,CAAC,WAAW,EAAE,IAAI,CAAC,EAAE,CAAC;gBAC5C,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACrB,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,mBAAmB,CAAC,WAAmB;IACrD,MAAM,eAAe,GAAG,IAAI,CAAC,WAAW,EAAE,cAAc,CAAC,CAAC;IAE1D,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,YAAY,CAAC,eAAe,EAAE,OAAO,CAAC,CAAC;QACvD,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAEhC,IAAI,GAAG,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YAC1B,OAAO;gBACL,QAAQ,EAAE,KAAK;gBACf,KAAK,EAAE,yCAAyC;aACjD,CAAC;QACJ,CAAC;QAED,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;IAC5B,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,KAAK,YAAY,WAAW,EAAE,CAAC;YACjC,OAAO;gBACL,QAAQ,EAAE,KAAK;gBACf,KAAK,EAAE,gCAAgC;aACxC,CAAC;QACJ,CAAC;QACD,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,WAAW,CAAC,QAAgB,EAAE,IAAY;IACxD,MAAM,WAAW,GAAG,OAAO,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;IAE5C,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC,EAAE,CAAC;QAC7B,MAAM,IAAI,sBAAsB,CAAC,IAAI,EAAE,wBAAwB,WAAW,EAAE,CAAC,CAAC;IAChF,CAAC;IAED,0BAA0B;IAC1B,MAAM,YAAY,GAAG,oBAAoB,CAAC,WAAW,CAAC,CAAC;IACvD,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC5B,MAAM,IAAI,sBAAsB,CAC9B,IAAI,EACJ,2BAA2B,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CACrD,CAAC;IACJ,CAAC;IAED,wBAAwB;IACxB,MAAM,aAAa,GAAG,mBAAmB,CAAC,WAAW,CAAC,CAAC;IACvD,IAAI,aAAa,CAAC,KAAK,EAAE,CAAC;QACxB,MAAM,IAAI,sBAAsB,CAAC,IAAI,EAAE,aAAa,CAAC,KAAK,CAAC,CAAC;IAC9D,CAAC;IAED,qCAAqC;IACrC,IAAI,CAAC,mBAAmB,CAAC,WAAW,EAAE,WAAW,CAAC,EAAE,CAAC;QACnD,MAAM,IAAI,sBAAsB,CAAC,IAAI,EAAE,yDAAyD,CAAC,CAAC;IACpG,CAAC;IACD,MAAM,UAAU,GAAG,IAAI,CAAC,WAAW,EAAE,WAAW,CAAC,CAAC;IAClD,MAAM,MAAM,GAAG,YAAY,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;IAEjD,OAAO;QACL,IAAI;QACJ,IAAI,EAAE,WAAW;QACjB,MAAM;QACN,QAAQ,EAAE,aAAa,CAAC,QAAQ;KACjC,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,eAAe,CAAC,QAAgB;IAI9C,MAAM,YAAY,GAAG,gBAAgB,CAAC,QAAQ,CAAC,CAAC;IAChD,MAAM,QAAQ,GAAkB,EAAE,CAAC;IACnC,MAAM,MAAM,GAA6B,EAAE,CAAC;IAE5C,KAAK,MAAM,IAAI,IAAI,YAAY,EAAE,CAAC;QAChC,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,WAAW,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;YAC5C,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACzB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,KAAK,YAAY,sBAAsB,EAAE,CAAC;gBAC5C,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACrB,CAAC;iBAAM,CAAC;gBACN,MAAM,KAAK,CAAC;YACd,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC;AAC9B,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,eAAe,CAC7B,WAAmB,EACnB,kBAAqC,cAAc;IAEnD,MAAM,KAAK,GAAa,EAAE,CAAC;IAE3B,SAAS,IAAI,CAAC,GAAW,EAAE,WAAmB,EAAE;QAC9C,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC;QAEjC,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC5B,MAAM,YAAY,GAAG,QAAQ,CAAC,CAAC,CAAC,GAAG,QAAQ,IAAI,KAAK,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC;YAE/D,8BAA8B;YAC9B,IAAI,eAAe,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,YAAY,KAAK,OAAO,IAAI,KAAK,KAAK,OAAO,CAAC,EAAE,CAAC;gBACrF,SAAS;YACX,CAAC;YAED,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;YAClC,MAAM,IAAI,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAC;YAEhC,IAAI,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC;gBACvB,IAAI,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;YAC/B,CAAC;iBAAM,CAAC;gBACN,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;YAC3B,CAAC;QACH,CAAC;IACH,CAAC;IAED,IAAI,CAAC,WAAW,CAAC,CAAC;IAClB,OAAO,KAAK,CAAC,IAAI,EAAE,CAAC;AACtB,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,gBAAgB,CAC9B,WAAmB,EACnB,eAAmC;IAEnC,MAAM,KAAK,GAAG,eAAe,CAAC,WAAW,EAAE,eAAe,CAAC,CAAC;IAC5D,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAkB,CAAC;IAE3C,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,QAAQ,GAAG,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,CAAC;QACzC,QAAQ,CAAC,GAAG,CAAC,IAAI,EAAE,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC;IACtD,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC"}

package/dist/lib/housekeeping.d.ts

@@ -0,0 +1,26 @@
+/**
+ * Housekeeping for eval results.
+ *
+ * After experiments complete, consolidate results:
+ * - For each (experiment, eval) pair: keep only the latest valid result
+ * - Remove older duplicates and dangling/incomplete results
+ * - Remove empty timestamp directories
+ */
+interface HousekeepingStats {
+    removedDuplicates: number;
+    removedIncomplete: number;
+    removedNonModelFailures: number;
+    removedEmptyDirs: number;
+}
+/**
+ * Run housekeeping on a single experiment's results directory.
+ *
+ * For each eval: keeps the newest complete result (has summary.json and
+ * at least one transcript), removes older duplicates and incomplete results.
+ * Removes empty timestamp directories afterward.
+ */
+export declare function housekeep(resultsDir: string, experimentName: string, options?: {
+    dry?: boolean;
+}): HousekeepingStats;
+export {};
+//# sourceMappingURL=housekeeping.d.ts.map

package/dist/lib/housekeeping.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"housekeeping.d.ts","sourceRoot":"","sources":["../../src/lib/housekeeping.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAMH,UAAU,iBAAiB;IACzB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,iBAAiB,EAAE,MAAM,CAAC;IAC1B,uBAAuB,EAAE,MAAM,CAAC;IAChC,gBAAgB,EAAE,MAAM,CAAC;CAC1B;AAED;;;;;;GAMG;AACH,wBAAgB,SAAS,CACvB,UAAU,EAAE,MAAM,EAClB,cAAc,EAAE,MAAM,EACtB,OAAO,CAAC,EAAE;IAAE,GAAG,CAAC,EAAE,OAAO,CAAA;CAAE,GAC1B,iBAAiB,CAgGnB"}

package/dist/lib/housekeeping.js

@@ -0,0 +1,170 @@
+/**
+ * Housekeeping for eval results.
+ *
+ * After experiments complete, consolidate results:
+ * - For each (experiment, eval) pair: keep only the latest valid result
+ * - Remove older duplicates and dangling/incomplete results
+ * - Remove empty timestamp directories
+ */
+import { readdirSync, rmSync, existsSync, readFileSync, statSync } from 'fs';
+import { join } from 'path';
+import { isClassifierEnabled, isNonModelFailure } from './classifier.js';
+/**
+ * Run housekeeping on a single experiment's results directory.
+ *
+ * For each eval: keeps the newest complete result (has summary.json and
+ * at least one transcript), removes older duplicates and incomplete results.
+ * Removes empty timestamp directories afterward.
+ */
+export function housekeep(resultsDir, experimentName, options) {
+    const stats = {
+        removedDuplicates: 0,
+        removedIncomplete: 0,
+        removedNonModelFailures: 0,
+        removedEmptyDirs: 0,
+    };
+    const experimentDir = join(resultsDir, experimentName);
+    if (!existsSync(experimentDir))
+        return stats;
+    // Get all timestamps sorted newest first
+    let timestamps;
+    try {
+        timestamps = readdirSync(experimentDir)
+            .filter((t) => !t.startsWith('.'))
+            .filter((t) => {
+            try {
+                return statSync(join(experimentDir, t)).isDirectory();
+            }
+            catch {
+                return false;
+            }
+        })
+            .sort()
+            .reverse();
+    }
+    catch {
+        return stats;
+    }
+    // Track which (eval, fingerprint) pairs we've already seen (newest wins).
+    // Results with different fingerprints (e.g. smoke vs full run) are not
+    // duplicates of each other and should coexist.
+    const seenEvals = new Set();
+    for (const timestamp of timestamps) {
+        const tsDir = join(experimentDir, timestamp);
+        let evalDirs;
+        try {
+            evalDirs = readdirSync(tsDir).filter((d) => !d.startsWith('.'));
+        }
+        catch {
+            continue;
+        }
+        for (const evalDir of evalDirs) {
+            const evalResultDir = join(tsDir, evalDir);
+            if (!statSync(evalResultDir).isDirectory())
+                continue;
+            // Read fingerprint to distinguish different configs (e.g. smoke vs full)
+            const fingerprint = readFingerprint(evalResultDir);
+            const dedupeKey = fingerprint ? `${evalDir}:${fingerprint}` : evalDir;
+            if (seenEvals.has(dedupeKey)) {
+                // Older duplicate with same fingerprint — remove
+                if (!options?.dry) {
+                    rmSync(evalResultDir, { recursive: true });
+                }
+                stats.removedDuplicates++;
+                continue;
+            }
+            // Check if this result is complete
+            // Note: non-model failures are only cleaned up if the classifier is enabled
+            const isNonModel = isClassifierEnabled() && isNonModelFailure(evalResultDir);
+            if (isComplete(evalResultDir) && !isSmoke(evalResultDir) && !isNonModel) {
+                seenEvals.add(dedupeKey);
+            }
+            else if (isNonModel) {
+                if (!options?.dry) {
+                    rmSync(evalResultDir, { recursive: true });
+                }
+                stats.removedNonModelFailures++;
+            }
+            else {
+                // Incomplete or smoke — remove
+                if (!options?.dry) {
+                    rmSync(evalResultDir, { recursive: true });
+                }
+                stats.removedIncomplete++;
+            }
+        }
+        // Check if timestamp dir is now empty
+        try {
+            const remaining = readdirSync(tsDir).filter((d) => !d.startsWith('.'));
+            if (remaining.length === 0) {
+                if (!options?.dry) {
+                    rmSync(tsDir, { recursive: true });
+                }
+                stats.removedEmptyDirs++;
+            }
+        }
+        catch {
+            // Directory already removed or inaccessible
+        }
+    }
+    return stats;
+}
+/**
+ * Check if an eval result is from a smoke test.
+ */
+function isSmoke(evalResultDir) {
+    try {
+        const summary = JSON.parse(readFileSync(join(evalResultDir, 'summary.json'), 'utf-8'));
+        return summary.smoke === true;
+    }
+    catch {
+        return false;
+    }
+}
+/**
+ * Read the fingerprint from an eval result's summary.json, if present.
+ */
+function readFingerprint(evalResultDir) {
+    try {
+        const summary = JSON.parse(readFileSync(join(evalResultDir, 'summary.json'), 'utf-8'));
+        return summary.fingerprint;
+    }
+    catch {
+        return undefined;
+    }
+}
+/**
+ * Check if an eval result directory is complete.
+ * Complete means: has summary.json and at least one run with a transcript.
+ */
+function isComplete(evalResultDir) {
+    const summaryPath = join(evalResultDir, 'summary.json');
+    if (!existsSync(summaryPath))
+        return false;
+    // Check for at least one transcript
+    try {
+        const entries = readdirSync(evalResultDir);
+        for (const entry of entries) {
+            if (!entry.startsWith('run-'))
+                continue;
+            const runDir = join(evalResultDir, entry);
+            if (existsSync(join(runDir, 'transcript-raw.jsonl')) ||
+                existsSync(join(runDir, 'transcript.json'))) {
+                return true;
+            }
+        }
+    }
+    catch {
+        return false;
+    }
+    // No transcript found — but summary.json exists.
+    // Still consider complete if summary shows 0% (model produced nothing, which is valid).
+    try {
+        const summary = JSON.parse(readFileSync(summaryPath, 'utf-8'));
+        return summary.totalRuns > 0;
+    }
+    catch {
+        return false;
+    }
+}
+//# sourceMappingURL=housekeeping.js.map

package/dist/lib/housekeeping.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"housekeeping.js","sourceRoot":"","sources":["../../src/lib/housekeeping.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EAAE,WAAW,EAAE,MAAM,EAAE,UAAU,EAAE,YAAY,EAAE,QAAQ,EAAE,MAAM,IAAI,CAAC;AAC7E,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAC5B,OAAO,EAAE,mBAAmB,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AASzE;;;;;;GAMG;AACH,MAAM,UAAU,SAAS,CACvB,UAAkB,EAClB,cAAsB,EACtB,OAA2B;IAE3B,MAAM,KAAK,GAAsB;QAC/B,iBAAiB,EAAE,CAAC;QACpB,iBAAiB,EAAE,CAAC;QACpB,uBAAuB,EAAE,CAAC;QAC1B,gBAAgB,EAAE,CAAC;KACpB,CAAC;IAEF,MAAM,aAAa,GAAG,IAAI,CAAC,UAAU,EAAE,cAAc,CAAC,CAAC;IACvD,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC;QAAE,OAAO,KAAK,CAAC;IAE7C,yCAAyC;IACzC,IAAI,UAAoB,CAAC;IACzB,IAAI,CAAC;QACH,UAAU,GAAG,WAAW,CAAC,aAAa,CAAC;aACpC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;aACjC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE;YACZ,IAAI,CAAC;gBACH,OAAO,QAAQ,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;YACxD,CAAC;YAAC,MAAM,CAAC;gBACP,OAAO,KAAK,CAAC;YACf,CAAC;QACH,CAAC,CAAC;aACD,IAAI,EAAE;aACN,OAAO,EAAE,CAAC;IACf,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;IAED,0EAA0E;IAC1E,uEAAuE;IACvE,+CAA+C;IAC/C,MAAM,SAAS,GAAG,IAAI,GAAG,EAAU,CAAC;IAEpC,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;QACnC,MAAM,KAAK,GAAG,IAAI,CAAC,aAAa,EAAE,SAAS,CAAC,CAAC;QAE7C,IAAI,QAAkB,CAAC;QACvB,IAAI,CAAC;YACH,QAAQ,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC;QAClE,CAAC;QAAC,MAAM,CAAC;YACP,SAAS;QACX,CAAC;QAED,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC/B,MAAM,aAAa,GAAG,IAAI,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;YAE3C,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC,WAAW,EAAE;gBAAE,SAAS;YAErD,yEAAyE;YACzE,MAAM,WAAW,GAAG,eAAe,CAAC,aAAa,CAAC,CAAC;YACnD,MAAM,SAAS,GAAG,WAAW,CAAC,CAAC,CAAC,GAAG,OAAO,IAAI,WAAW,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC;YAEtE,IAAI,SAAS,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC;gBAC7B,iDAAiD;gBACjD,IAAI,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC;oBAClB,MAAM,CAAC,aAAa,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;gBAC7C,CAAC;gBACD,KAAK,CAAC,iBAAiB,EAAE,CAAC;gBAC1B,SAAS;YACX,CAAC;YAED,mCAAmC;YACnC,4EAA4E;YAC5E,MAAM,UAAU,GAAG,mBAAmB,EAAE,IAAI,iBAAiB,CAAC,aAAa,CAAC,CAAC;YAC7E,IAAI,UAAU,CAAC,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC;gBACxE,SAAS,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;YAC3B,CAAC;iBAAM,IAAI,UAAU,EAAE,CAAC;gBACtB,IAAI,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC;oBAClB,MAAM,CAAC,aAAa,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;gBAC7C,CAAC;gBACD,KAAK,CAAC,uBAAuB,EAAE,CAAC;YAClC,CAAC;iBAAM,CAAC;gBACN,+BAA+B;gBAC/B,IAAI,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC;oBAClB,MAAM,CAAC,aAAa,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;gBAC7C,CAAC;gBACD,KAAK,CAAC,iBAAiB,EAAE,CAAC;YAC5B,CAAC;QACH,CAAC;QAED,sCAAsC;QACtC,IAAI,CAAC;YACH,MAAM,SAAS,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC;YACvE,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC3B,IAAI,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC;oBAClB,MAAM,CAAC,KAAK,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;gBACrC,CAAC;gBACD,KAAK,CAAC,gBAAgB,EAAE,CAAC;YAC3B,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,4CAA4C;QAC9C,CAAC;IACH,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;GAEG;AACH,SAAS,OAAO,CAAC,aAAqB;IACpC,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,IAAI,CAAC,aAAa,EAAE,cAAc,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC;QACvF,OAAO,OAAO,CAAC,KAAK,KAAK,IAAI,CAAC;IAChC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED;;GAEG;AACH,SAAS,eAAe,CAAC,aAAqB;IAC5C,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,IAAI,CAAC,aAAa,EAAE,cAAc,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC;QACvF,OAAO,OAAO,CAAC,WAAW,CAAC;IAC7B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,SAAS,CAAC;IACnB,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,SAAS,UAAU,CAAC,aAAqB;IACvC,MAAM,WAAW,GAAG,IAAI,CAAC,aAAa,EAAE,cAAc,CAAC,CAAC;IACxD,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC;QAAE,OAAO,KAAK,CAAC;IAE3C,oCAAoC;IACpC,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,WAAW,CAAC,aAAa,CAAC,CAAC;QAC3C,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC5B,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,MAAM,CAAC;gBAAE,SAAS;YACxC,MAAM,MAAM,GAAG,IAAI,CAAC,aAAa,EAAE,KAAK,CAAC,CAAC;YAC1C,IACE,UAAU,CAAC,IAAI,CAAC,MAAM,EAAE,sBAAsB,CAAC,CAAC;gBAChD,UAAU,CAAC,IAAI,CAAC,MAAM,EAAE,iBAAiB,CAAC,CAAC,EAC3C,CAAC;gBACD,OAAO,IAAI,CAAC;YACd,CAAC;QACH,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;IAED,iDAAiD;IACjD,wFAAwF;IACxF,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC,CAAC;QAC/D,OAAO,OAAO,CAAC,SAAS,GAAG,CAAC,CAAC;IAC/B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC"}

package/dist/lib/init.d.ts

@@ -0,0 +1,21 @@
+/**
+ * Project initialization - create new eval projects.
+ */
+/**
+ * Options for initializing a new project.
+ */
+export interface InitOptions {
+    /** Project name */
+    name: string;
+    /** Target directory (defaults to current working directory) */
+    targetDir?: string;
+}
+/**
+ * Initialize a new eval project.
+ */
+export declare function initProject(options: InitOptions): string;
+/**
+ * Get instructions for after project creation.
+ */
+export declare function getPostInitInstructions(projectDir: string, projectName: string): string;
+//# sourceMappingURL=init.d.ts.map

package/dist/lib/init.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"init.d.ts","sourceRoot":"","sources":["../../src/lib/init.ts"],"names":[],"mappings":"AAAA;;GAEG;AAMH;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,mBAAmB;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,+DAA+D;IAC/D,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAyQD;;GAEG;AACH,wBAAgB,WAAW,CAAC,OAAO,EAAE,WAAW,GAAG,MAAM,CA0BxD;AAED;;GAEG;AACH,wBAAgB,uBAAuB,CAAC,UAAU,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,GAAG,MAAM,CAavF"}