agent-eval-opencode 0.10.0

package/dist/lib/agents/types.d.ts

@@ -0,0 +1,73 @@
+/**
+ * Agent interface and common types for all agents.
+ */
+import type { ModelTier, SetupFunction, SandboxBackend } from '../types.js';
+/**
+ * Common options for all agents.
+ */
+export interface AgentRunOptions {
+    /** The prompt/task for the agent */
+    prompt: string;
+    /** Model to use (agent-specific) */
+    model: ModelTier;
+    /** Timeout in milliseconds */
+    timeout: number;
+    /** API key for the agent */
+    apiKey: string;
+    /** Optional setup function to run before agent */
+    setup?: SetupFunction;
+    /** npm scripts to run after agent completes */
+    scripts?: string[];
+    /** Abort signal to cancel the run */
+    signal?: AbortSignal;
+    /** Sandbox backend to use */
+    sandbox?: SandboxBackend | 'auto';
+}
+/**
+ * Result from running a script.
+ */
+export interface ScriptResult {
+    success: boolean;
+    output: string;
+}
+/**
+ * Result of an agent run.
+ */
+export interface AgentRunResult {
+    /** Whether the run was successful */
+    success: boolean;
+    /** Raw output from the agent (stdout/stderr) */
+    output: string;
+    /** Structured transcript (JSONL format for Claude, JSON for Codex) */
+    transcript?: string;
+    /** Error message if failed */
+    error?: string;
+    /** Duration in milliseconds */
+    duration: number;
+    /** Test result (EVAL.ts always runs) */
+    testResult?: ScriptResult;
+    /** Results from configured scripts */
+    scriptsResults?: Record<string, ScriptResult>;
+    /** Sandbox ID for debugging */
+    sandboxId?: string;
+    /** Files generated by the agent */
+    generatedFiles?: Record<string, string>;
+    /** Files deleted by the agent */
+    deletedFiles?: string[];
+}
+/**
+ * Agent interface that all agents must implement.
+ */
+export interface Agent {
+    /** Unique identifier for the agent */
+    name: string;
+    /** Human-readable display name */
+    displayName: string;
+    /** Run the agent on a fixture */
+    run(fixturePath: string, options: AgentRunOptions): Promise<AgentRunResult>;
+    /** Get agent-specific environment variable name for API key */
+    getApiKeyEnvVar(): string;
+    /** Get the default model for this agent */
+    getDefaultModel(): ModelTier;
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/lib/agents/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/lib/agents/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAAE,SAAS,EAAE,aAAa,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAE5E;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,oCAAoC;IACpC,MAAM,EAAE,MAAM,CAAC;IACf,oCAAoC;IACpC,KAAK,EAAE,SAAS,CAAC;IACjB,8BAA8B;IAC9B,OAAO,EAAE,MAAM,CAAC;IAChB,4BAA4B;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,kDAAkD;IAClD,KAAK,CAAC,EAAE,aAAa,CAAC;IACtB,+CAA+C;IAC/C,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,qCAAqC;IACrC,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,6BAA6B;IAC7B,OAAO,CAAC,EAAE,cAAc,GAAG,MAAM,CAAC;CACnC;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,OAAO,EAAE,OAAO,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;CAChB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,qCAAqC;IACrC,OAAO,EAAE,OAAO,CAAC;IACjB,gDAAgD;IAChD,MAAM,EAAE,MAAM,CAAC;IACf,sEAAsE;IACtE,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,8BAA8B;IAC9B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,+BAA+B;IAC/B,QAAQ,EAAE,MAAM,CAAC;IACjB,wCAAwC;IACxC,UAAU,CAAC,EAAE,YAAY,CAAC;IAC1B,sCAAsC;IACtC,cAAc,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;IAC9C,+BAA+B;IAC/B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,mCAAmC;IACnC,cAAc,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACxC,iCAAiC;IACjC,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;CACzB;AAED;;GAEG;AACH,MAAM,WAAW,KAAK;IACpB,sCAAsC;IACtC,IAAI,EAAE,MAAM,CAAC;IAEb,kCAAkC;IAClC,WAAW,EAAE,MAAM,CAAC;IAEpB,iCAAiC;IACjC,GAAG,CAAC,WAAW,EAAE,MAAM,EAAE,OAAO,EAAE,eAAe,GAAG,OAAO,CAAC,cAAc,CAAC,CAAC;IAE5E,+DAA+D;IAC/D,eAAe,IAAI,MAAM,CAAC;IAE1B,2CAA2C;IAC3C,eAAe,IAAI,SAAS,CAAC;CAC9B"}

package/dist/lib/agents/types.js

@@ -0,0 +1,5 @@
+/**
+ * Agent interface and common types for all agents.
+ */
+export {};
+//# sourceMappingURL=types.js.map

package/dist/lib/agents/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../../src/lib/agents/types.ts"],"names":[],"mappings":"AAAA;;GAEG"}

package/dist/lib/classifier.d.ts

@@ -0,0 +1,89 @@
+/**
+ * Failure classification for eval results.
+ *
+ * Classifies failed eval runs as:
+ * - "model" — the model tried but wrote incorrect code
+ * - "infra" — infrastructure broke (API errors, rate limits, crashes)
+ * - "timeout" — the run hit its time limit
+ *
+ * Uses AI classification via the Vercel AI Gateway. Requires AI_GATEWAY_API_KEY or VERCEL_OIDC_TOKEN.
+ */
+import type { Classification } from './types.js';
+/**
+ * Check if the classifier feature is enabled.
+ * The classifier requires either AI_GATEWAY_API_KEY or VERCEL_OIDC_TOKEN to be set.
+ * If neither is available, classification is disabled and housekeeping won't clean up non-model failures.
+ */
+export declare function isClassifierEnabled(): boolean;
+/**
+ * Creates sandboxed read-only tools for the AI classifier.
+ */
+export declare function createClassifierTools(evalResultDir: string): {
+    list_files: import("ai").Tool<{
+        path: string;
+    }, {
+        error: string;
+        entries?: undefined;
+    } | {
+        entries: {
+            name: string;
+            type: "file" | "dir";
+        }[];
+        error?: undefined;
+    }>;
+    read_file: import("ai").Tool<{
+        path: string;
+        offset?: number | undefined;
+        limit?: number | undefined;
+    }, {
+        error: string;
+        content?: undefined;
+        totalLines?: undefined;
+        showing?: undefined;
+    } | {
+        content: string;
+        totalLines: number;
+        showing: string;
+        error?: undefined;
+    }>;
+    grep: import("ai").Tool<{
+        path: string;
+        pattern: string;
+        maxResults?: number | undefined;
+    }, {
+        error: string;
+        matches?: undefined;
+        totalFound?: undefined;
+        truncated?: undefined;
+    } | {
+        matches: {
+            file: string;
+            line: number;
+            text: string;
+        }[];
+        totalFound: number;
+        truncated: boolean;
+        error?: undefined;
+    }>;
+};
+/**
+ * Classify a failure using AI via the Vercel AI Gateway.
+ * Requires AI_GATEWAY_API_KEY in the environment.
+ */
+export declare function classifyWithAI(evalResultDir: string, evalName: string, experimentName: string): Promise<Classification | null>;
+/**
+ * Classify a failed eval result using AI.
+ * Requires AI_GATEWAY_API_KEY in the environment.
+ *
+ * Caches results in classification.json within the eval result directory.
+ */
+export declare function classifyFailure(evalResultDir: string, evalName: string, experimentName: string): Promise<Classification | null>;
+/**
+ * Check if an eval result was classified as a non-model failure (infra or timeout).
+ * Reads classification.json — the single source of truth for classification data.
+ *
+ * Returns false for acknowledged failures (--ack-failures), since those are
+ * intentionally kept as final results.
+ */
+export declare function isNonModelFailure(evalResultDir: string): boolean;
+//# sourceMappingURL=classifier.d.ts.map

package/dist/lib/classifier.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"classifier.d.ts","sourceRoot":"","sources":["../../src/lib/classifier.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAMH,OAAO,KAAK,EAAE,cAAc,EAAe,MAAM,YAAY,CAAC;AAE9D;;;;GAIG;AACH,wBAAgB,mBAAmB,IAAI,OAAO,CAE7C;AA8BD;;GAEG;AACH,wBAAgB,qBAAqB,CAAC,aAAa,EAAE,MAAM;;;;;;;;kBAepB,MAAM;kBAAQ,MAAM,GAAG,KAAK;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAkE9B,MAAM;kBAAQ,MAAM;kBAAQ,MAAM;;;;;;EA0DtE;AAED;;;GAGG;AACH,wBAAsB,cAAc,CAClC,aAAa,EAAE,MAAM,EACrB,QAAQ,EAAE,MAAM,EAChB,cAAc,EAAE,MAAM,GACrB,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC,CAwChC;AAED;;;;;GAKG;AACH,wBAAsB,eAAe,CACnC,aAAa,EAAE,MAAM,EACrB,QAAQ,EAAE,MAAM,EAChB,cAAc,EAAE,MAAM,GACrB,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC,CAyBhC;AAED;;;;;;GAMG;AACH,wBAAgB,iBAAiB,CAAC,aAAa,EAAE,MAAM,GAAG,OAAO,CAQhE"}

package/dist/lib/classifier.js

@@ -0,0 +1,285 @@
+/**
+ * Failure classification for eval results.
+ *
+ * Classifies failed eval runs as:
+ * - "model" — the model tried but wrote incorrect code
+ * - "infra" — infrastructure broke (API errors, rate limits, crashes)
+ * - "timeout" — the run hit its time limit
+ *
+ * Uses AI classification via the Vercel AI Gateway. Requires AI_GATEWAY_API_KEY or VERCEL_OIDC_TOKEN.
+ */
+import { readFileSync, readdirSync, statSync, writeFileSync } from 'fs';
+import { join, resolve } from 'path';
+import { tool } from 'ai';
+import { z } from 'zod';
+/**
+ * Check if the classifier feature is enabled.
+ * The classifier requires either AI_GATEWAY_API_KEY or VERCEL_OIDC_TOKEN to be set.
+ * If neither is available, classification is disabled and housekeeping won't clean up non-model failures.
+ */
+export function isClassifierEnabled() {
+    return !!(process.env.AI_GATEWAY_API_KEY || process.env.VERCEL_OIDC_TOKEN);
+}
+const CLASSIFIER_SYSTEM_PROMPT = `You are a failure classifier for an AI coding agent benchmark.
+
+Your job: figure out WHY a failed eval run failed. Each eval tests whether an AI model can complete a coding task (e.g. migrate to App Router, add a Next.js feature). You have tools to explore the result files.
+
+Classify into one of:
+- "model" — the model tried but wrote incorrect code
+- "infra" — infrastructure broke (API errors, rate limits, crashes) and the model never got to do real work
+- "timeout" — the run hit its time limit
+
+The eval result directory contains run-1/ through run-N/ subdirectories (one per attempt, N depends on config), plus a summary.json. Each run directory has:
+- result.json — status, error, duration
+- transcript.json or transcript-raw.jsonl (or older results may have transcript.jsonl) — the agent's conversation log
+- outputs/eval.txt — EVAL.ts test output
+- outputs/scripts/*.txt — npm script outputs (e.g. build.txt), if the experiment configured scripts
+
+IMPORTANT: The eval harness always runs EVAL.ts tests after the agent finishes, plus any npm scripts configured in the experiment's \`scripts\` array (e.g. \`["build"]\`). These run even if the model produced nothing — tests just run against unmodified scaffold code (TODO placeholders). So test/script failures alone do NOT mean the model wrote code.
+
+The transcript is the key evidence. It records every action the model took. If there is no transcript file, or the transcript only shows errors (no tool calls or text output from the model), the model never actually ran — that's "infra". Only classify as "model" if you see evidence in the transcript that the model actually generated code.`;
+/**
+ * Validates and resolves a path, ensuring it stays within the allowed root.
+ */
+function safePath(root, relativePath) {
+    const resolved = resolve(root, relativePath);
+    if (!resolved.startsWith(root))
+        return null;
+    return resolved;
+}
+/**
+ * Creates sandboxed read-only tools for the AI classifier.
+ */
+export function createClassifierTools(evalResultDir) {
+    return {
+        list_files: tool({
+            description: 'List files and directories at a path relative to the eval result root. Use "." for the root.',
+            inputSchema: z.object({
+                path: z
+                    .string()
+                    .describe('Relative path to list, e.g. "." or "run-1" or "run-1/outputs"'),
+            }),
+            execute: async ({ path: relPath }) => {
+                const target = safePath(evalResultDir, relPath);
+                if (!target)
+                    return { error: 'Path outside allowed directory' };
+                try {
+                    const entries = readdirSync(target);
+                    const results = [];
+                    for (const entry of entries.sort()) {
+                        const info = statSync(join(target, entry));
+                        results.push({ name: entry, type: info.isDirectory() ? 'dir' : 'file' });
+                    }
+                    return { entries: results };
+                }
+                catch {
+                    return { error: `Cannot list: ${relPath}` };
+                }
+            },
+        }),
+        read_file: tool({
+            description: 'Read a file relative to the eval result root. For large files, use offset/limit to paginate.',
+            inputSchema: z.object({
+                path: z
+                    .string()
+                    .describe('Relative path to the file, e.g. "run-1/result.json"'),
+                offset: z
+                    .number()
+                    .describe('Line offset to start reading from (0-based)')
+                    .optional(),
+                limit: z
+                    .number()
+                    .describe('Max number of lines to return')
+                    .optional(),
+            }),
+            execute: async ({ path: relPath, offset: rawOffset, limit: rawLimit }) => {
+                const offset = rawOffset ?? 0;
+                const limit = rawLimit ?? 200;
+                const target = safePath(evalResultDir, relPath);
+                if (!target)
+                    return { error: 'Path outside allowed directory' };
+                try {
+                    const content = readFileSync(target, 'utf-8');
+                    const lines = content.split('\n');
+                    const sliced = lines.slice(offset, offset + limit);
+                    return {
+                        content: sliced.join('\n'),
+                        totalLines: lines.length,
+                        showing: `lines ${offset}-${Math.min(offset + limit, lines.length)} of ${lines.length}`,
+                    };
+                }
+                catch {
+                    return { error: `Cannot read: ${relPath}` };
+                }
+            },
+        }),
+        grep: tool({
+            description: 'Search for a pattern in files under a directory. Returns matching lines with context.',
+            inputSchema: z.object({
+                pattern: z.string().describe('Text or regex pattern to search for'),
+                path: z
+                    .string()
+                    .describe('Relative directory or file to search in, e.g. "." or "run-1"'),
+                maxResults: z
+                    .number()
+                    .describe('Max number of matches to return')
+                    .optional(),
+            }),
+            execute: async ({ pattern, path: relPath, maxResults: rawMax }) => {
+                const maxResults = rawMax ?? 20;
+                const target = safePath(evalResultDir, relPath);
+                if (!target)
+                    return { error: 'Path outside allowed directory' };
+                const regex = new RegExp(pattern, 'i');
+                const matches = [];
+                async function searchFile(filePath, relName) {
+                    try {
+                        const content = readFileSync(filePath, 'utf-8');
+                        const lines = content.split('\n');
+                        for (let i = 0; i < lines.length && matches.length < maxResults; i++) {
+                            if (regex.test(lines[i])) {
+                                matches.push({
+                                    file: relName,
+                                    line: i + 1,
+                                    text: lines[i].slice(0, 500),
+                                });
+                            }
+                        }
+                    }
+                    catch {
+                        // Skip unreadable files
+                    }
+                }
+                async function searchDir(dirPath, prefix) {
+                    try {
+                        const entries = readdirSync(dirPath);
+                        for (const entry of entries) {
+                            if (matches.length >= maxResults)
+                                break;
+                            const full = join(dirPath, entry);
+                            const rel = prefix ? `${prefix}/${entry}` : entry;
+                            const info = statSync(full);
+                            if (info.isDirectory()) {
+                                await searchDir(full, rel);
+                            }
+                            else {
+                                await searchFile(full, rel);
+                            }
+                        }
+                    }
+                    catch {
+                        // Skip unreadable dirs
+                    }
+                }
+                try {
+                    const info = statSync(target);
+                    if (info.isDirectory()) {
+                        await searchDir(target, relPath === '.' ? '' : relPath);
+                    }
+                    else {
+                        await searchFile(target, relPath);
+                    }
+                }
+                catch {
+                    return { error: `Path not found: ${relPath}` };
+                }
+                return {
+                    matches,
+                    totalFound: matches.length,
+                    truncated: matches.length >= maxResults,
+                };
+            },
+        }),
+    };
+}
+/**
+ * Classify a failure using AI via the Vercel AI Gateway.
+ * Requires AI_GATEWAY_API_KEY in the environment.
+ */
+export async function classifyWithAI(evalResultDir, evalName, experimentName) {
+    const { generateText, hasToolCall, createGateway } = await import('ai');
+    const gateway = createGateway({ apiKey: process.env.AI_GATEWAY_API_KEY ?? process.env.VERCEL_OIDC_TOKEN ?? '' });
+    let classification = null;
+    const explorationTools = createClassifierTools(evalResultDir);
+    const allTools = {
+        ...explorationTools,
+        classify: tool({
+            description: 'Submit your final classification. Call this once you have enough evidence.',
+            inputSchema: z.object({
+                failureType: z
+                    .enum(['model', 'infra', 'timeout'])
+                    .describe('The failure category'),
+                failureReason: z
+                    .string()
+                    .describe('Brief 1-2 sentence explanation of why'),
+            }),
+            execute: async ({ failureType, failureReason }) => {
+                classification = { failureType: failureType, failureReason };
+                return { ok: true };
+            },
+        }),
+    };
+    try {
+        await generateText({
+            model: gateway('anthropic/claude-haiku-4-5-20251001'),
+            system: CLASSIFIER_SYSTEM_PROMPT,
+            prompt: `Classify the failure for eval "${evalName}" (experiment: ${experimentName}). Use the exploration tools to investigate, then call classify() with your verdict.`,
+            tools: allTools,
+            stopWhen: hasToolCall('classify'),
+        });
+        return classification;
+    }
+    catch {
+        return null;
+    }
+}
+/**
+ * Classify a failed eval result using AI.
+ * Requires AI_GATEWAY_API_KEY in the environment.
+ *
+ * Caches results in classification.json within the eval result directory.
+ */
+export async function classifyFailure(evalResultDir, evalName, experimentName) {
+    // Check for cached classification
+    const cachedPath = join(evalResultDir, 'classification.json');
+    try {
+        const cached = JSON.parse(readFileSync(cachedPath, 'utf-8'));
+        if (cached.failureType && cached.failureReason) {
+            return { failureType: cached.failureType, failureReason: cached.failureReason };
+        }
+    }
+    catch {
+        // No cache
+    }
+    // Classify with AI
+    const classification = await classifyWithAI(evalResultDir, evalName, experimentName);
+    // Cache the result
+    if (classification) {
+        try {
+            writeFileSync(cachedPath, JSON.stringify(classification, null, 2));
+        }
+        catch {
+            // Non-fatal: caching failed
+        }
+    }
+    return classification;
+}
+/**
+ * Check if an eval result was classified as a non-model failure (infra or timeout).
+ * Reads classification.json — the single source of truth for classification data.
+ *
+ * Returns false for acknowledged failures (--ack-failures), since those are
+ * intentionally kept as final results.
+ */
+export function isNonModelFailure(evalResultDir) {
+    try {
+        const classification = JSON.parse(readFileSync(join(evalResultDir, 'classification.json'), 'utf-8'));
+        if (classification.acknowledged)
+            return false;
+        return classification.failureType != null && classification.failureType !== 'model';
+    }
+    catch {
+        return false;
+    }
+}
+//# sourceMappingURL=classifier.js.map

package/dist/lib/classifier.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"classifier.js","sourceRoot":"","sources":["../../src/lib/classifier.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,EAAE,YAAY,EAAE,WAAW,EAAE,QAAQ,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AACxE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,MAAM,CAAC;AACrC,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC1B,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAGxB;;;;GAIG;AACH,MAAM,UAAU,mBAAmB;IACjC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,kBAAkB,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC,CAAC;AAC7E,CAAC;AAED,MAAM,wBAAwB,GAAG;;;;;;;;;;;;;;;;;qVAiBoT,CAAC;AAEtV;;GAEG;AACH,SAAS,QAAQ,CAAC,IAAY,EAAE,YAAoB;IAClD,MAAM,QAAQ,GAAG,OAAO,CAAC,IAAI,EAAE,YAAY,CAAC,CAAC;IAC7C,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,IAAI,CAAC;QAAE,OAAO,IAAI,CAAC;IAC5C,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,qBAAqB,CAAC,aAAqB;IACzD,OAAO;QACL,UAAU,EAAE,IAAI,CAAC;YACf,WAAW,EACT,8FAA8F;YAChG,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;gBACpB,IAAI,EAAE,CAAC;qBACJ,MAAM,EAAE;qBACR,QAAQ,CAAC,+DAA+D,CAAC;aAC7E,CAAC;YACF,OAAO,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,EAAE,EAAE;gBACnC,MAAM,MAAM,GAAG,QAAQ,CAAC,aAAa,EAAE,OAAO,CAAC,CAAC;gBAChD,IAAI,CAAC,MAAM;oBAAE,OAAO,EAAE,KAAK,EAAE,gCAAgC,EAAE,CAAC;gBAChE,IAAI,CAAC;oBACH,MAAM,OAAO,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC;oBACpC,MAAM,OAAO,GAAkD,EAAE,CAAC;oBAClE,KAAK,MAAM,KAAK,IAAI,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC;wBACnC,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC;wBAC3C,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC;oBAC3E,CAAC;oBACD,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC;gBAC9B,CAAC;gBAAC,MAAM,CAAC;oBACP,OAAO,EAAE,KAAK,EAAE,gBAAgB,OAAO,EAAE,EAAE,CAAC;gBAC9C,CAAC;YACH,CAAC;SACF,CAAC;QAEF,SAAS,EAAE,IAAI,CAAC;YACd,WAAW,EACT,8FAA8F;YAChG,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;gBACpB,IAAI,EAAE,CAAC;qBACJ,MAAM,EAAE;qBACR,QAAQ,CAAC,qDAAqD,CAAC;gBAClE,MAAM,EAAE,CAAC;qBACN,MAAM,EAAE;qBACR,QAAQ,CAAC,6CAA6C,CAAC;qBACvD,QAAQ,EAAE;gBACb,KAAK,EAAE,CAAC;qBACL,MAAM,EAAE;qBACR,QAAQ,CAAC,+BAA+B,CAAC;qBACzC,QAAQ,EAAE;aACd,CAAC;YACF,OAAO,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,KAAK,EAAE,QAAQ,EAAE,EAAE,EAAE;gBACvE,MAAM,MAAM,GAAG,SAAS,IAAI,CAAC,CAAC;gBAC9B,MAAM,KAAK,GAAG,QAAQ,IAAI,GAAG,CAAC;gBAC9B,MAAM,MAAM,GAAG,QAAQ,CAAC,aAAa,EAAE,OAAO,CAAC,CAAC;gBAChD,IAAI,CAAC,MAAM;oBAAE,OAAO,EAAE,KAAK,EAAE,gCAAgC,EAAE,CAAC;gBAChE,IAAI,CAAC;oBACH,MAAM,OAAO,GAAG,YAAY,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;oBAC9C,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;oBAClC,MAAM,MAAM,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,EAAE,MAAM,GAAG,KAAK,CAAC,CAAC;oBACnD,OAAO;wBACL,OAAO,EAAE,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC;wBAC1B,UAAU,EAAE,KAAK,CAAC,MAAM;wBACxB,OAAO,EAAE,SAAS,MAAM,IAAI,IAAI,CAAC,GAAG,CAAC,MAAM,GAAG,KAAK,EAAE,KAAK,CAAC,MAAM,CAAC,OAAO,KAAK,CAAC,MAAM,EAAE;qBACxF,CAAC;gBACJ,CAAC;gBAAC,MAAM,CAAC;oBACP,OAAO,EAAE,KAAK,EAAE,gBAAgB,OAAO,EAAE,EAAE,CAAC;gBAC9C,CAAC;YACH,CAAC;SACF,CAAC;QAEF,IAAI,EAAE,IAAI,CAAC;YACT,WAAW,EACT,uFAAuF;YACzF,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;gBACpB,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,qCAAqC,CAAC;gBACnE,IAAI,EAAE,CAAC;qBACJ,MAAM,EAAE;qBACR,QAAQ,CAAC,8DAA8D,CAAC;gBAC3E,UAAU,EAAE,CAAC;qBACV,MAAM,EAAE;qBACR,QAAQ,CAAC,iCAAiC,CAAC;qBAC3C,QAAQ,EAAE;aACd,CAAC;YACF,OAAO,EAAE,KAAK,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,UAAU,EAAE,MAAM,EAAE,EAAE,EAAE;gBAChE,MAAM,UAAU,GAAG,MAAM,IAAI,EAAE,CAAC;gBAChC,MAAM,MAAM,GAAG,QAAQ,CAAC,aAAa,EAAE,OAAO,CAAC,CAAC;gBAChD,IAAI,CAAC,MAAM;oBAAE,OAAO,EAAE,KAAK,EAAE,gCAAgC,EAAE,CAAC;gBAChE,MAAM,KAAK,GAAG,IAAI,MAAM,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;gBACvC,MAAM,OAAO,GAAwD,EAAE,CAAC;gBAExE,KAAK,UAAU,UAAU,CAAC,QAAgB,EAAE,OAAe;oBACzD,IAAI,CAAC;wBACH,MAAM,OAAO,GAAG,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;wBAChD,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;wBAClC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,IAAI,OAAO,CAAC,MAAM,GAAG,UAAU,EAAE,CAAC,EAAE,EAAE,CAAC;4BACrE,IAAI,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;gCACzB,OAAO,CAAC,IAAI,CAAC;oCACX,IAAI,EAAE,OAAO;oCACb,IAAI,EAAE,CAAC,GAAG,CAAC;oCACX,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC;iCAC7B,CAAC,CAAC;4BACL,CAAC;wBACH,CAAC;oBACH,CAAC;oBAAC,MAAM,CAAC;wBACP,wBAAwB;oBAC1B,CAAC;gBACH,CAAC;gBAED,KAAK,UAAU,SAAS,CAAC,OAAe,EAAE,MAAc;oBACtD,IAAI,CAAC;wBACH,MAAM,OAAO,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC;wBACrC,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;4BAC5B,IAAI,OAAO,CAAC,MAAM,IAAI,UAAU;gCAAE,MAAM;4BACxC,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;4BAClC,MAAM,GAAG,GAAG,MAAM,CAAC,CAAC,CAAC,GAAG,MAAM,IAAI,KAAK,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC;4BAClD,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC;4BAC5B,IAAI,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC;gCACvB,MAAM,SAAS,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;4BAC7B,CAAC;iCAAM,CAAC;gCACN,MAAM,UAAU,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;4BAC9B,CAAC;wBACH,CAAC;oBACH,CAAC;oBAAC,MAAM,CAAC;wBACP,uBAAuB;oBACzB,CAAC;gBACH,CAAC;gBAED,IAAI,CAAC;oBACH,MAAM,IAAI,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC;oBAC9B,IAAI,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC;wBACvB,MAAM,SAAS,CAAC,MAAM,EAAE,OAAO,KAAK,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;oBAC1D,CAAC;yBAAM,CAAC;wBACN,MAAM,UAAU,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;oBACpC,CAAC;gBACH,CAAC;gBAAC,MAAM,CAAC;oBACP,OAAO,EAAE,KAAK,EAAE,mBAAmB,OAAO,EAAE,EAAE,CAAC;gBACjD,CAAC;gBAED,OAAO;oBACL,OAAO;oBACP,UAAU,EAAE,OAAO,CAAC,MAAM;oBAC1B,SAAS,EAAE,OAAO,CAAC,MAAM,IAAI,UAAU;iBACxC,CAAC;YACJ,CAAC;SACF,CAAC;KACH,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,aAAqB,EACrB,QAAgB,EAChB,cAAsB;IAEtB,MAAM,EAAE,YAAY,EAAE,WAAW,EAAE,aAAa,EAAE,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,CAAC;IAExE,MAAM,OAAO,GAAG,aAAa,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,GAAG,CAAC,kBAAkB,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,EAAE,EAAE,CAAC,CAAC;IAEjH,IAAI,cAAc,GAA0B,IAAI,CAAC;IAEjD,MAAM,gBAAgB,GAAG,qBAAqB,CAAC,aAAa,CAAC,CAAC;IAC9D,MAAM,QAAQ,GAAG;QACf,GAAG,gBAAgB;QACnB,QAAQ,EAAE,IAAI,CAAC;YACb,WAAW,EAAE,4EAA4E;YACzF,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;gBACpB,WAAW,EAAE,CAAC;qBACX,IAAI,CAAC,CAAC,OAAO,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC;qBACnC,QAAQ,CAAC,sBAAsB,CAAC;gBACnC,aAAa,EAAE,CAAC;qBACb,MAAM,EAAE;qBACR,QAAQ,CAAC,uCAAuC,CAAC;aACrD,CAAC;YACF,OAAO,EAAE,KAAK,EAAE,EAAE,WAAW,EAAE,aAAa,EAAE,EAAE,EAAE;gBAChD,cAAc,GAAG,EAAE,WAAW,EAAE,WAA0B,EAAE,aAAa,EAAE,CAAC;gBAC5E,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC;YACtB,CAAC;SACF,CAAC;KACH,CAAC;IAEF,IAAI,CAAC;QACH,MAAM,YAAY,CAAC;YACjB,KAAK,EAAE,OAAO,CAAC,qCAAqC,CAAC;YACrD,MAAM,EAAE,wBAAwB;YAChC,MAAM,EAAE,kCAAkC,QAAQ,kBAAkB,cAAc,sFAAsF;YACxK,KAAK,EAAE,QAAQ;YACf,QAAQ,EAAE,WAAW,CAAC,UAAU,CAAC;SAClC,CAAC,CAAC;QAEH,OAAO,cAAc,CAAC;IACxB,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,aAAqB,EACrB,QAAgB,EAChB,cAAsB;IAEtB,kCAAkC;IAClC,MAAM,UAAU,GAAG,IAAI,CAAC,aAAa,EAAE,qBAAqB,CAAC,CAAC;IAC9D,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC,CAAC;QAC7D,IAAI,MAAM,CAAC,WAAW,IAAI,MAAM,CAAC,aAAa,EAAE,CAAC;YAC/C,OAAO,EAAE,WAAW,EAAE,MAAM,CAAC,WAAW,EAAE,aAAa,EAAE,MAAM,CAAC,aAAa,EAAE,CAAC;QAClF,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,WAAW;IACb,CAAC;IAED,mBAAmB;IACnB,MAAM,cAAc,GAAG,MAAM,cAAc,CAAC,aAAa,EAAE,QAAQ,EAAE,cAAc,CAAC,CAAC;IAErF,mBAAmB;IACnB,IAAI,cAAc,EAAE,CAAC;QACnB,IAAI,CAAC;YACH,aAAa,CAAC,UAAU,EAAE,IAAI,CAAC,SAAS,CAAC,cAAc,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QACrE,CAAC;QAAC,MAAM,CAAC;YACP,4BAA4B;QAC9B,CAAC;IACH,CAAC;IAED,OAAO,cAAc,CAAC;AACxB,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,iBAAiB,CAAC,aAAqB;IACrD,IAAI,CAAC;QACH,MAAM,cAAc,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,IAAI,CAAC,aAAa,EAAE,qBAAqB,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC;QACrG,IAAI,cAAc,CAAC,YAAY;YAAE,OAAO,KAAK,CAAC;QAC9C,OAAO,cAAc,CAAC,WAAW,IAAI,IAAI,IAAI,cAAc,CAAC,WAAW,KAAK,OAAO,CAAC;IACtF,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC"}

package/dist/lib/config.d.ts

@@ -0,0 +1,37 @@
+/**
+ * Experiment configuration validation and defaults.
+ */
+import type { ExperimentConfig, ResolvedExperimentConfig, EvalFilter } from './types.js';
+/**
+ * Default configuration values.
+ */
+export declare const CONFIG_DEFAULTS: {
+    model: "opus";
+    evals: "*";
+    runs: number;
+    earlyExit: boolean;
+    scripts: string[];
+    timeout: number;
+    sandbox: "auto";
+    copyFiles: "none";
+};
+/**
+ * Validates an experiment configuration object.
+ * Throws a descriptive error if validation fails.
+ */
+export declare function validateConfig(config: unknown): ExperimentConfig;
+/**
+ * Resolves an experiment configuration by applying defaults.
+ */
+export declare function resolveConfig(config: ExperimentConfig): ResolvedExperimentConfig;
+/**
+ * Loads an experiment configuration from a file path.
+ * Supports TypeScript and JavaScript files with default exports.
+ */
+export declare function loadConfig(configPath: string): Promise<ResolvedExperimentConfig>;
+/**
+ * Resolves the evals filter to a list of eval names.
+ * Supports glob patterns like "vercel-cli/*" for nested directories.
+ */
+export declare function resolveEvalNames(filter: string | string[] | EvalFilter, availableEvals: string[]): string[];
+//# sourceMappingURL=config.d.ts.map

package/dist/lib/config.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../../src/lib/config.ts"],"names":[],"mappings":"AAAA;;GAEG;AAIH,OAAO,KAAK,EACV,gBAAgB,EAChB,wBAAwB,EACxB,UAAU,EACX,MAAM,YAAY,CAAC;AAGpB;;GAEG;AACH,eAAO,MAAM,eAAe;;;;;aAKX,MAAM,EAAE;;;;CAIxB,CAAC;AA6BF;;;GAGG;AACH,wBAAgB,cAAc,CAAC,MAAM,EAAE,OAAO,GAAG,gBAAgB,CAWhE;AAED;;GAEG;AACH,wBAAgB,aAAa,CAAC,MAAM,EAAE,gBAAgB,GAAG,wBAAwB,CAoBhF;AAED;;;GAGG;AACH,wBAAsB,UAAU,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,wBAAwB,CAAC,CA8BtF;AAgBD;;;GAGG;AACH,wBAAgB,gBAAgB,CAC9B,MAAM,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,UAAU,EACtC,cAAc,EAAE,MAAM,EAAE,GACvB,MAAM,EAAE,CA0DV"}