agent-eval-opencode 0.10.0

package/dist/lib/sandbox.js

@@ -0,0 +1,337 @@
+/**
+ * Sandbox integration for isolated eval execution.
+ * Supports both Vercel Sandbox and Docker backends.
+ */
+import { Sandbox as VercelSandbox } from '@vercel/sandbox';
+import { readFileSync } from 'fs';
+import { join } from 'path';
+import { DockerSandboxManager } from './docker-sandbox.js';
+/**
+ * Default timeout for sandbox operations (10 minutes).
+ */
+export const DEFAULT_SANDBOX_TIMEOUT = 600000;
+/**
+ * Files to ignore when copying to sandbox.
+ * These are build artifacts and dependencies that shouldn't be uploaded.
+ * Note: This is a general-purpose pattern list used by collectLocalFiles().
+ * For eval-specific exclusions (PROMPT.md, EVAL.ts), see TEST_FILE_PATTERNS.
+ */
+export const IGNORED_PATTERNS = [
+    '.git',
+    '.next',
+    'node_modules',
+    '.DS_Store',
+    '*.log',
+    'build',
+    'dist',
+    'pnpm-lock.yaml',
+    'package-lock.json',
+];
+/**
+ * Test/eval file patterns to withhold from agent during task execution.
+ * These files are uploaded AFTER the agent completes for validation.
+ * - PROMPT.md: Contains the task - agent receives this via CLI argument, not as a file
+ * - EVAL.ts/tsx: Validation tests - must be hidden so agent can't "cheat"
+ */
+export const TEST_FILE_PATTERNS = ['EVAL.ts', 'EVAL.tsx', 'PROMPT.md'];
+/**
+ * Wrapper around Vercel Sandbox providing a cleaner API.
+ */
+export class SandboxManager {
+    sandbox;
+    _workingDirectory = '/vercel/sandbox';
+    constructor(sandbox) {
+        this.sandbox = sandbox;
+    }
+    /**
+     * Create a new sandbox instance.
+     */
+    static async create(options = {}) {
+        const timeout = options.timeout ?? DEFAULT_SANDBOX_TIMEOUT;
+        const runtime = options.runtime ?? 'node24';
+        const credentials = resolveVercelSandboxCredentials(options);
+        const sandbox = await VercelSandbox.create({
+            runtime,
+            timeout,
+            ...(credentials ?? {}),
+        });
+        return new SandboxManager(sandbox);
+    }
+    /**
+     * Get the sandbox ID.
+     */
+    get sandboxId() {
+        return this.sandbox.sandboxId;
+    }
+    /**
+     * Run a command in the sandbox.
+     */
+    async runCommand(command, args = [], options = {}) {
+        const result = await this.sandbox.runCommand({
+            cmd: command,
+            args,
+            env: options.env,
+        });
+        return {
+            stdout: await result.stdout(),
+            stderr: await result.stderr(),
+            exitCode: result.exitCode,
+        };
+    }
+    /**
+     * Run a shell command (through bash).
+     */
+    async runShell(command, env) {
+        const result = await this.sandbox.runCommand({
+            cmd: 'bash',
+            args: ['-c', command],
+            env,
+        });
+        return {
+            stdout: await result.stdout(),
+            stderr: await result.stderr(),
+            exitCode: result.exitCode,
+        };
+    }
+    /**
+     * Read a file from the sandbox.
+     */
+    async readFile(path) {
+        const result = await this.runCommand('cat', [path]);
+        if (result.exitCode !== 0) {
+            throw new Error(`Failed to read file ${path}: ${result.stderr}`);
+        }
+        return result.stdout;
+    }
+    /**
+     * Check if a file exists in the sandbox.
+     */
+    async fileExists(path) {
+        const result = await this.runCommand('test', ['-f', path]);
+        return result.exitCode === 0;
+    }
+    /**
+     * Write files to the sandbox.
+     */
+    async writeFiles(files) {
+        const sandboxFiles = [];
+        for (const [path, content] of Object.entries(files)) {
+            sandboxFiles.push({
+                path,
+                content: Buffer.from(content, 'utf-8'),
+            });
+        }
+        await this.sandbox.writeFiles(sandboxFiles);
+    }
+    /**
+     * Upload files from local filesystem to sandbox.
+     */
+    async uploadFiles(files) {
+        const sandboxFiles = files.map((f) => ({
+            path: f.path,
+            content: typeof f.content === 'string' ? Buffer.from(f.content, 'utf-8') : f.content,
+        }));
+        await this.sandbox.writeFiles(sandboxFiles);
+    }
+    /**
+     * Get the working directory.
+     */
+    getWorkingDirectory() {
+        return this._workingDirectory;
+    }
+    /**
+     * Stop and clean up the sandbox.
+     */
+    async stop() {
+        await this.sandbox.stop();
+    }
+}
+function resolveVercelSandboxCredentials(options) {
+    const token = options.token ?? process.env.VERCEL_TOKEN;
+    const teamId = options.teamId ?? process.env.VERCEL_TEAM_ID;
+    const projectId = options.projectId ?? process.env.VERCEL_PROJECT_ID;
+    if (token && teamId && projectId) {
+        return { token, teamId, projectId };
+    }
+    return null;
+}
+/**
+ * Resolve which sandbox backend to use based on options.
+ *
+ * Priority:
+ * 1. Explicit backend in options (if not 'auto')
+ * 2. Auto-detect: Vercel if token present, else Docker
+ */
+export function resolveBackend(options) {
+    // Explicit backend in options
+    if (options?.backend && options.backend !== 'auto') {
+        return options.backend;
+    }
+    // Auto-detect: Vercel if token present, else Docker
+    if (process.env.VERCEL_TOKEN || process.env.VERCEL_OIDC_TOKEN) {
+        return 'vercel';
+    }
+    return 'docker';
+}
+/**
+ * Get information about the sandbox backend that will be used.
+ * Useful for displaying to users.
+ */
+export function getSandboxBackendInfo(options) {
+    const backend = resolveBackend(options);
+    // Determine the reason
+    let reason;
+    let description;
+    const hasExplicitOption = options?.backend && options.backend !== 'auto';
+    if (hasExplicitOption) {
+        reason = 'explicit';
+        description = `${backend} (explicit)`;
+    }
+    else {
+        reason = 'auto-detected';
+        if (backend === 'vercel') {
+            description = `${backend} (auto-detected: VERCEL_TOKEN found)`;
+        }
+        else {
+            description = `${backend} (auto-detected: no VERCEL_TOKEN, using Docker)`;
+        }
+    }
+    return { backend, reason, description };
+}
+/**
+ * Create a sandbox using the appropriate backend.
+ *
+ * By default, uses Vercel Sandbox if VERCEL_TOKEN is present,
+ * otherwise falls back to Docker.
+ *
+ * @example
+ * ```typescript
+ * // Auto-detect backend
+ * const sandbox = await createSandbox();
+ *
+ * // Explicit Docker
+ * const sandbox = await createSandbox({ backend: 'docker' });
+ *
+ * // Explicit Vercel
+ * const sandbox = await createSandbox({ backend: 'vercel' });
+ * ```
+ */
+export async function createSandbox(options = {}) {
+    const backend = resolveBackend(options);
+    if (backend === 'docker') {
+        return DockerSandboxManager.create({
+            timeout: options.timeout,
+            runtime: options.runtime,
+        });
+    }
+    return SandboxManager.create({
+        timeout: options.timeout,
+        runtime: options.runtime,
+    });
+}
+/**
+ * Collect files from a local directory for uploading to sandbox.
+ */
+export async function collectLocalFiles(dir, options = {}) {
+    const { readdirSync, statSync } = await import('fs');
+    const excludePatterns = options.excludePatterns ?? IGNORED_PATTERNS;
+    const includePatterns = options.includePatterns;
+    const files = [];
+    function shouldExclude(name, relativePath) {
+        for (const pattern of excludePatterns) {
+            if (pattern.startsWith('*.')) {
+                // Wildcard pattern
+                const ext = pattern.slice(1);
+                if (name.endsWith(ext)) {
+                    return true;
+                }
+            }
+            else if (name === pattern || relativePath === pattern) {
+                return true;
+            }
+        }
+        return false;
+    }
+    function shouldInclude(name) {
+        if (!includePatterns) {
+            return true;
+        }
+        for (const pattern of includePatterns) {
+            if (pattern.startsWith('*.')) {
+                const ext = pattern.slice(1);
+                if (name.endsWith(ext)) {
+                    return true;
+                }
+            }
+            else if (name === pattern) {
+                return true;
+            }
+        }
+        return false;
+    }
+    function walk(currentDir, relativePath = '') {
+        const entries = readdirSync(currentDir);
+        for (const entry of entries) {
+            const entryRelativePath = relativePath ? `${relativePath}/${entry}` : entry;
+            const fullPath = join(currentDir, entry);
+            if (shouldExclude(entry, entryRelativePath)) {
+                continue;
+            }
+            const stat = statSync(fullPath);
+            if (stat.isDirectory()) {
+                walk(fullPath, entryRelativePath);
+            }
+            else if (shouldInclude(entry)) {
+                const content = readFileSync(fullPath);
+                files.push({ path: entryRelativePath, content });
+            }
+        }
+    }
+    walk(dir);
+    return files;
+}
+/**
+ * Check if a filename matches any of the test file patterns.
+ */
+function isTestFilePattern(filename) {
+    for (const pattern of TEST_FILE_PATTERNS) {
+        if (pattern.startsWith('*.')) {
+            const ext = pattern.slice(1);
+            if (filename.endsWith(ext)) {
+                return true;
+            }
+        }
+        else if (filename === pattern) {
+            return true;
+        }
+    }
+    return false;
+}
+/**
+ * Split files into workspace files (visible to agent) and test files (hidden until validation).
+ */
+export function splitTestFiles(files) {
+    const workspaceFiles = [];
+    const testFiles = [];
+    for (const file of files) {
+        const name = file.path.split('/').pop() ?? file.path;
+        if (isTestFilePattern(name)) {
+            testFiles.push(file);
+        }
+        else {
+            workspaceFiles.push(file);
+        }
+    }
+    return { workspaceFiles, testFiles };
+}
+/**
+ * Verify that no test files exist in the sandbox.
+ */
+export async function verifyNoTestFiles(sandbox) {
+    const result = await sandbox.runShell("find . -path './node_modules' -prune  -o -name 'EVAL.ts' -print");
+    const foundTests = result.stdout.trim();
+    if (foundTests) {
+        throw new Error(`Test files found in sandbox before agent run: ${foundTests}`);
+    }
+}
+//# sourceMappingURL=sandbox.js.map

package/dist/lib/sandbox.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"sandbox.js","sourceRoot":"","sources":["../../src/lib/sandbox.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,OAAO,IAAI,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAE3D,OAAO,EAAE,YAAY,EAAE,MAAM,IAAI,CAAC;AAClC,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAC5B,OAAO,EAAE,oBAAoB,EAAE,MAAM,qBAAqB,CAAC;AAE3D;;GAEG;AACH,MAAM,CAAC,MAAM,uBAAuB,GAAG,MAAM,CAAC;AAmB9C;;;;;GAKG;AACH,MAAM,CAAC,MAAM,gBAAgB,GAAG;IAC9B,MAAM;IACN,OAAO;IACP,cAAc;IACd,WAAW;IACX,OAAO;IACP,OAAO;IACP,MAAM;IACN,gBAAgB;IAChB,mBAAmB;CACpB,CAAC;AAEF;;;;;GAKG;AACH,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAAC,SAAS,EAAE,UAAU,EAAE,WAAW,CAAC,CAAC;AAqCvE;;GAEG;AACH,MAAM,OAAO,cAAc;IACjB,OAAO,CAAgB;IACvB,iBAAiB,GAAW,iBAAiB,CAAC;IAEtD,YAAY,OAAsB;QAChC,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;IACzB,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,UAA0B,EAAE;QAC9C,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,uBAAuB,CAAC;QAC3D,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,QAAQ,CAAC;QAC5C,MAAM,WAAW,GAAG,+BAA+B,CAAC,OAAO,CAAC,CAAC;QAE7D,MAAM,OAAO,GAAG,MAAM,aAAa,CAAC,MAAM,CAAC;YACzC,OAAO;YACP,OAAO;YACP,GAAG,CAAC,WAAW,IAAI,EAAE,CAAC;SACvB,CAAC,CAAC;QACH,OAAO,IAAI,cAAc,CAAC,OAAO,CAAC,CAAC;IACrC,CAAC;IAED;;OAEG;IACH,IAAI,SAAS;QACX,OAAO,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC;IAChC,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,UAAU,CACd,OAAe,EACf,OAAiB,EAAE,EACnB,UAA4C,EAAE;QAE9C,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC;YAC3C,GAAG,EAAE,OAAO;YACZ,IAAI;YACJ,GAAG,EAAE,OAAO,CAAC,GAAG;SACjB,CAAC,CAAC;QAEH,OAAO;YACL,MAAM,EAAE,MAAM,MAAM,CAAC,MAAM,EAAE;YAC7B,MAAM,EAAE,MAAM,MAAM,CAAC,MAAM,EAAE;YAC7B,QAAQ,EAAE,MAAM,CAAC,QAAQ;SAC1B,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,QAAQ,CAAC,OAAe,EAAE,GAA4B;QAC1D,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC;YAC3C,GAAG,EAAE,MAAM;YACX,IAAI,EAAE,CAAC,IAAI,EAAE,OAAO,CAAC;YACrB,GAAG;SACJ,CAAC,CAAC;QAEH,OAAO;YACL,MAAM,EAAE,MAAM,MAAM,CAAC,MAAM,EAAE;YAC7B,MAAM,EAAE,MAAM,MAAM,CAAC,MAAM,EAAE;YAC7B,QAAQ,EAAE,MAAM,CAAC,QAAQ;SAC1B,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,QAAQ,CAAC,IAAY;QACzB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC;QACpD,IAAI,MAAM,CAAC,QAAQ,KAAK,CAAC,EAAE,CAAC;YAC1B,MAAM,IAAI,KAAK,CAAC,uBAAuB,IAAI,KAAK,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;QACnE,CAAC;QACD,OAAO,MAAM,CAAC,MAAM,CAAC;IACvB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,UAAU,CAAC,IAAY;QAC3B,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC;QAC3D,OAAO,MAAM,CAAC,QAAQ,KAAK,CAAC,CAAC;IAC/B,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,UAAU,CAAC,KAA6B;QAC5C,MAAM,YAAY,GAA6C,EAAE,CAAC;QAElE,KAAK,MAAM,CAAC,IAAI,EAAE,OAAO,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;YACpD,YAAY,CAAC,IAAI,CAAC;gBAChB,IAAI;gBACJ,OAAO,EAAE,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC;aACvC,CAAC,CAAC;QACL,CAAC;QAED,MAAM,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC;IAC9C,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,WAAW,CAAC,KAAoB;QACpC,MAAM,YAAY,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YACrC,IAAI,EAAE,CAAC,CAAC,IAAI;YACZ,OAAO,EAAE,OAAO,CAAC,CAAC,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO;SACrF,CAAC,CAAC,CAAC;QAEJ,MAAM,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC;IAC9C,CAAC;IAED;;OAEG;IACH,mBAAmB;QACjB,OAAO,IAAI,CAAC,iBAAiB,CAAC;IAChC,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,IAAI;QACR,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;IAC5B,CAAC;CACF;AAED,SAAS,+BAA+B,CAAC,OAAuB;IAK9D,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC;IACxD,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC;IAC5D,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC;IAErE,IAAI,KAAK,IAAI,MAAM,IAAI,SAAS,EAAE,CAAC;QACjC,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC;IACtC,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,cAAc,CAAC,OAAwB;IACrD,8BAA8B;IAC9B,IAAI,OAAO,EAAE,OAAO,IAAI,OAAO,CAAC,OAAO,KAAK,MAAM,EAAE,CAAC;QACnD,OAAO,OAAO,CAAC,OAAO,CAAC;IACzB,CAAC;IAED,oDAAoD;IACpD,IAAI,OAAO,CAAC,GAAG,CAAC,YAAY,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB,EAAE,CAAC;QAC9D,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,qBAAqB,CAAC,OAAwB;IAC5D,MAAM,OAAO,GAAG,cAAc,CAAC,OAAO,CAAC,CAAC;IAExC,uBAAuB;IACvB,IAAI,MAAoC,CAAC;IACzC,IAAI,WAAmB,CAAC;IAExB,MAAM,iBAAiB,GAAG,OAAO,EAAE,OAAO,IAAI,OAAO,CAAC,OAAO,KAAK,MAAM,CAAC;IAEzE,IAAI,iBAAiB,EAAE,CAAC;QACtB,MAAM,GAAG,UAAU,CAAC;QACpB,WAAW,GAAG,GAAG,OAAO,aAAa,CAAC;IACxC,CAAC;SAAM,CAAC;QACN,MAAM,GAAG,eAAe,CAAC;QACzB,IAAI,OAAO,KAAK,QAAQ,EAAE,CAAC;YACzB,WAAW,GAAG,GAAG,OAAO,sCAAsC,CAAC;QACjE,CAAC;aAAM,CAAC;YACN,WAAW,GAAG,GAAG,OAAO,iDAAiD,CAAC;QAC5E,CAAC;IACH,CAAC;IAED,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC;AAC1C,CAAC;AAED;;;;;;;;;;;;;;;;;GAiBG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,UAA0B,EAAE;IAE5B,MAAM,OAAO,GAAG,cAAc,CAAC,OAAO,CAAC,CAAC;IAExC,IAAI,OAAO,KAAK,QAAQ,EAAE,CAAC;QACzB,OAAO,oBAAoB,CAAC,MAAM,CAAC;YACjC,OAAO,EAAE,OAAO,CAAC,OAAO;YACxB,OAAO,EAAE,OAAO,CAAC,OAAO;SACzB,CAAC,CAAC;IACL,CAAC;IAED,OAAO,cAAc,CAAC,MAAM,CAAC;QAC3B,OAAO,EAAE,OAAO,CAAC,OAAO;QACxB,OAAO,EAAE,OAAO,CAAC,OAAO;KACzB,CAAC,CAAC;AACL,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,GAAW,EACX,UAGI,EAAE;IAEN,MAAM,EAAE,WAAW,EAAE,QAAQ,EAAE,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,CAAC;IAErD,MAAM,eAAe,GAAG,OAAO,CAAC,eAAe,IAAI,gBAAgB,CAAC;IACpE,MAAM,eAAe,GAAG,OAAO,CAAC,eAAe,CAAC;IAChD,MAAM,KAAK,GAAkB,EAAE,CAAC;IAEhC,SAAS,aAAa,CAAC,IAAY,EAAE,YAAoB;QACvD,KAAK,MAAM,OAAO,IAAI,eAAe,EAAE,CAAC;YACtC,IAAI,OAAO,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC7B,mBAAmB;gBACnB,MAAM,GAAG,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;gBAC7B,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;oBACvB,OAAO,IAAI,CAAC;gBACd,CAAC;YACH,CAAC;iBAAM,IAAI,IAAI,KAAK,OAAO,IAAI,YAAY,KAAK,OAAO,EAAE,CAAC;gBACxD,OAAO,IAAI,CAAC;YACd,CAAC;QACH,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;IAED,SAAS,aAAa,CAAC,IAAY;QACjC,IAAI,CAAC,eAAe,EAAE,CAAC;YACrB,OAAO,IAAI,CAAC;QACd,CAAC;QACD,KAAK,MAAM,OAAO,IAAI,eAAe,EAAE,CAAC;YACtC,IAAI,OAAO,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC7B,MAAM,GAAG,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;gBAC7B,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;oBACvB,OAAO,IAAI,CAAC;gBACd,CAAC;YACH,CAAC;iBAAM,IAAI,IAAI,KAAK,OAAO,EAAE,CAAC;gBAC5B,OAAO,IAAI,CAAC;YACd,CAAC;QACH,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;IAED,SAAS,IAAI,CAAC,UAAkB,EAAE,eAAuB,EAAE;QACzD,MAAM,OAAO,GAAG,WAAW,CAAC,UAAU,CAAC,CAAC;QAExC,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC5B,MAAM,iBAAiB,GAAG,YAAY,CAAC,CAAC,CAAC,GAAG,YAAY,IAAI,KAAK,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC;YAC5E,MAAM,QAAQ,GAAG,IAAI,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC;YAEzC,IAAI,aAAa,CAAC,KAAK,EAAE,iBAAiB,CAAC,EAAE,CAAC;gBAC5C,SAAS;YACX,CAAC;YAED,MAAM,IAAI,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAC;YAEhC,IAAI,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC;gBACvB,IAAI,CAAC,QAAQ,EAAE,iBAAiB,CAAC,CAAC;YACpC,CAAC;iBAAM,IAAI,aAAa,CAAC,KAAK,CAAC,EAAE,CAAC;gBAChC,MAAM,OAAO,GAAG,YAAY,CAAC,QAAQ,CAAC,CAAC;gBACvC,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,iBAAiB,EAAE,OAAO,EAAE,CAAC,CAAC;YACnD,CAAC;QACH,CAAC;IACH,CAAC;IAED,IAAI,CAAC,GAAG,CAAC,CAAC;IACV,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;GAEG;AACH,SAAS,iBAAiB,CAAC,QAAgB;IACzC,KAAK,MAAM,OAAO,IAAI,kBAAkB,EAAE,CAAC;QACzC,IAAI,OAAO,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;YAC7B,MAAM,GAAG,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;YAC7B,IAAI,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;gBAC3B,OAAO,IAAI,CAAC;YACd,CAAC;QACH,CAAC;aAAM,IAAI,QAAQ,KAAK,OAAO,EAAE,CAAC;YAChC,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,cAAc,CAAC,KAAoB;IAIjD,MAAM,cAAc,GAAkB,EAAE,CAAC;IACzC,MAAM,SAAS,GAAkB,EAAE,CAAC;IAEpC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,IAAI,IAAI,CAAC,IAAI,CAAC;QAErD,IAAI,iBAAiB,CAAC,IAAI,CAAC,EAAE,CAAC;YAC5B,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACvB,CAAC;aAAM,CAAC;YACN,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC5B,CAAC;IACH,CAAC;IAED,OAAO,EAAE,cAAc,EAAE,SAAS,EAAE,CAAC;AACvC,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,OAA8C;IAE9C,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,QAAQ,CACnC,iEAAiE,CAClE,CAAC;IAEF,MAAM,UAAU,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;IACxC,IAAI,UAAU,EAAE,CAAC;QACf,MAAM,IAAI,KAAK,CAAC,iDAAiD,UAAU,EAAE,CAAC,CAAC;IACjF,CAAC;AACH,CAAC"}

package/dist/lib/types.d.ts

@@ -0,0 +1,258 @@
+/**
+ * Core types for the eval framework.
+ */
+/**
+ * Supported AI agent types.
+ */
+export type AgentType = 'vercel-ai-gateway/claude-code' | 'claude-code' | 'vercel-ai-gateway/codex' | 'codex' | 'opencode' | 'gemini' | 'cursor';
+/**
+ * Model identifier - any string accepted.
+ * Each agent validates its own models at runtime.
+ */
+export type ModelTier = string;
+/**
+ * Function type for filtering evals.
+ */
+export type EvalFilter = (name: string) => boolean;
+/**
+ * Sandbox interface for setup functions.
+ * Provides methods to interact with the isolated VM.
+ */
+export interface Sandbox {
+    /** Run a command in the sandbox */
+    runCommand(command: string, args?: string[], options?: {
+        env?: Record<string, string>;
+    }): Promise<{
+        stdout: string;
+        stderr: string;
+        exitCode: number;
+    }>;
+    /** Read a file from the sandbox */
+    readFile(path: string): Promise<string>;
+    /** Write files to the sandbox */
+    writeFiles(files: Record<string, string>): Promise<void>;
+    /** Get the sandbox working directory */
+    getWorkingDirectory(): string;
+}
+/**
+ * Setup function that runs before the agent starts.
+ * Receives a sandbox instance for pre-configuration.
+ */
+export type SetupFunction = (sandbox: Sandbox) => Promise<void>;
+/**
+ * Sandbox backend type.
+ */
+export type SandboxBackend = 'vercel' | 'docker';
+/**
+ * Experiment configuration.
+ * Defines what to test and how.
+ */
+export interface ExperimentConfig {
+    /** Which AI agent to use */
+    agent: AgentType;
+    /** Which AI model the agent should use. Can be a single model or array of models to test.
+     * If an array is provided, the experiment will run on each model.
+     * Default is agent-specific: 'opus' for claude-code, 'openai/gpt-5.2-codex' for codex */
+    model?: ModelTier | ModelTier[];
+    /** Which evals to run. Can be a string, array, or filter function. @default '*' (all evals) */
+    evals?: string | string[] | EvalFilter;
+    /** How many times to run each eval. @default 1 */
+    runs?: number;
+    /** Stop after first successful run? @default true */
+    earlyExit?: boolean;
+    /** npm scripts that must pass after agent finishes. @default [] */
+    scripts?: string[];
+    /** Maximum time in seconds for agent to complete. @default 300 (5 minutes) */
+    timeout?: number;
+    /** Setup function that runs before agent starts. @default undefined */
+    setup?: SetupFunction;
+    /** Sandbox backend to use. @default 'auto' (Vercel if token present, else Docker) */
+    sandbox?: SandboxBackend | 'auto';
+    /** Optional function to modify the prompt before running the experiment. @default undefined */
+    editPrompt?: (prompt: string) => string;
+    /** Whether to copy project files into the result directory.
+     * - 'none': No files are copied (default)
+     * - 'changed': Only files the agent changed/created
+     * - 'all': Original project files + agent changes overlaid on top
+     * @default 'none' */
+    copyFiles?: 'none' | 'changed' | 'all';
+}
+/**
+ * Resolved experiment config with all defaults applied.
+ */
+export interface ResolvedExperimentConfig {
+    agent: AgentType;
+    model: ModelTier | ModelTier[];
+    evals: string | string[] | EvalFilter;
+    runs: number;
+    earlyExit: boolean;
+    scripts: string[];
+    timeout: number;
+    setup?: SetupFunction;
+    sandbox: SandboxBackend | 'auto';
+    editPrompt?: (prompt: string) => string;
+    copyFiles: 'none' | 'changed' | 'all';
+}
+/**
+ * Resolved experiment config with all defaults applied.
+ */
+export interface RunnableExperimentConfig {
+    agent: AgentType;
+    model: ModelTier;
+    evals: string | string[] | EvalFilter;
+    runs: number;
+    earlyExit: boolean;
+    scripts: string[];
+    timeout: number;
+    setup?: SetupFunction;
+    sandbox: SandboxBackend | 'auto';
+    editPrompt?: (prompt: string) => string;
+    copyFiles: 'none' | 'changed' | 'all';
+}
+/**
+ * Required files for a valid eval fixture.
+ * Note: Either EVAL.ts or EVAL.tsx is required (not both).
+ */
+export declare const REQUIRED_EVAL_FILES: readonly ["PROMPT.md", "EVAL.ts", "package.json"];
+/**
+ * Files excluded when listing fixture files (used by getFixtureFiles in fixture.ts).
+ * This is for local fixture introspection, NOT for sandbox uploads.
+ * For sandbox file filtering, see TEST_FILE_PATTERNS in sandbox.ts.
+ */
+export declare const EXCLUDED_FILES: readonly ["PROMPT.md", "EVAL.ts", "EVAL.tsx", "node_modules", ".git"];
+/**
+ * Represents a discovered eval fixture.
+ */
+export interface EvalFixture {
+    /** Name of the eval (folder name) */
+    name: string;
+    /** Absolute path to the eval folder */
+    path: string;
+    /** Contents of PROMPT.md */
+    prompt: string;
+    /** Whether package.json has "type": "module" */
+    isModule: boolean;
+}
+/**
+ * Result of a single eval run.
+ */
+export interface EvalRunResult {
+    /** Pass or fail status */
+    status: 'passed' | 'failed';
+    /** Error message if failed */
+    error?: string;
+    /** Duration in seconds */
+    duration: number;
+    /** Model used for this run */
+    model?: string;
+    /** Path to parsed transcript file (relative to run directory) */
+    transcriptPath?: string;
+    /** Path to raw transcript file (relative to run directory) */
+    transcriptRawPath?: string;
+    /** Paths to output files (relative to run directory) */
+    outputPaths?: {
+        /** Path to EVAL.ts test output */
+        eval?: string;
+        /** Paths to npm script outputs (nested to avoid collision) */
+        scripts?: Record<string, string>;
+    };
+}
+/**
+ * Internal run data including transcript and outputs (content, not paths).
+ */
+export interface EvalRunData {
+    /** The eval result (will have paths added when saving) */
+    result: EvalRunResult;
+    /** Structured transcript from Claude Code (saved to transcript.jsonl) */
+    transcript?: string;
+    /** Script/test output content (saved to outputs/) */
+    outputContent?: {
+        /** EVAL.ts test output */
+        eval?: string;
+        /** npm script outputs (nested to avoid collision) */
+        scripts?: Record<string, string>;
+    };
+    /** Files generated/modified by the agent (path -> content). Used for copyFiles option. */
+    generatedFiles?: Record<string, string>;
+    /** Files deleted by the agent. Used for copyFiles option. */
+    deletedFiles?: string[];
+}
+/**
+ * Summary of multiple runs for a single eval.
+ */
+export interface EvalSummary {
+    /** Name of the eval */
+    name: string;
+    /** Total number of runs */
+    totalRuns: number;
+    /** Number of passed runs */
+    passedRuns: number;
+    /** Pass rate as a percentage */
+    passRate: number;
+    /** Mean duration across all runs */
+    meanDuration: number;
+    /** Individual run data (internal, not all fields saved to summary.json) */
+    runs: EvalRunData[];
+}
+/**
+ * Failure classification for a failed eval run.
+ */
+export type FailureType = 'model' | 'infra' | 'timeout';
+/**
+ * Classification result for a failed eval.
+ */
+export interface Classification {
+    failureType: FailureType;
+    failureReason: string;
+    /** When true, the user has acknowledged this non-model failure as a final result via --ack-failures. */
+    acknowledged?: boolean;
+}
+/**
+ * Structured progress events emitted by the runner.
+ * The CLI decides how to render these (dashboard, console.log, etc.).
+ */
+export type ProgressEvent = {
+    type: 'experiment:start';
+    totalAttempts: number;
+    totalEvals: number;
+    totalRuns: number;
+} | {
+    type: 'eval:start';
+    evalName: string;
+    runNumber: number;
+    totalRuns: number;
+} | {
+    type: 'eval:complete';
+    evalName: string;
+    runNumber: number;
+    totalRuns: number;
+    result: EvalRunResult;
+} | {
+    type: 'eval:abort';
+    evalName: string;
+    runNumber: number;
+} | {
+    type: 'experiment:earlyExit';
+    evalName: string;
+    runNumber: number;
+} | {
+    type: 'experiment:saved';
+    outputDir: string;
+} | {
+    type: 'experiment:summary';
+    results: ExperimentResults;
+};
+/**
+ * Complete experiment results.
+ */
+export interface ExperimentResults {
+    /** Timestamp when experiment started */
+    startedAt: string;
+    /** Timestamp when experiment completed */
+    completedAt: string;
+    /** Experiment configuration used */
+    config: RunnableExperimentConfig;
+    /** Results for each eval */
+    evals: EvalSummary[];
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/lib/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/lib/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH;;GAEG;AACH,MAAM,MAAM,SAAS,GACjB,+BAA+B,GAC/B,aAAa,GACb,yBAAyB,GACzB,OAAO,GACP,UAAU,GACV,QAAQ,GACR,QAAQ,CAAC;AAEb;;;GAGG;AACH,MAAM,MAAM,SAAS,GAAG,MAAM,CAAC;AAE/B;;GAEG;AACH,MAAM,MAAM,UAAU,GAAG,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC;AAEnD;;;GAGG;AACH,MAAM,WAAW,OAAO;IACtB,mCAAmC;IACnC,UAAU,CACR,OAAO,EAAE,MAAM,EACf,IAAI,CAAC,EAAE,MAAM,EAAE,EACf,OAAO,CAAC,EAAE;QAAE,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;KAAE,GACzC,OAAO,CAAC;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACjE,mCAAmC;IACnC,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IACxC,iCAAiC;IACjC,UAAU,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACzD,wCAAwC;IACxC,mBAAmB,IAAI,MAAM,CAAC;CAC/B;AAED;;;GAGG;AACH,MAAM,MAAM,aAAa,GAAG,CAAC,OAAO,EAAE,OAAO,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;AAEhE;;GAEG;AACH,MAAM,MAAM,cAAc,GAAG,QAAQ,GAAG,QAAQ,CAAC;AAEjD;;;GAGG;AACH,MAAM,WAAW,gBAAgB;IAC/B,4BAA4B;IAC5B,KAAK,EAAE,SAAS,CAAC;IAEjB;;6FAEyF;IACzF,KAAK,CAAC,EAAE,SAAS,GAAG,SAAS,EAAE,CAAC;IAEhC,+FAA+F;IAC/F,KAAK,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,UAAU,CAAC;IAEvC,kDAAkD;IAClD,IAAI,CAAC,EAAE,MAAM,CAAC;IAEd,qDAAqD;IACrD,SAAS,CAAC,EAAE,OAAO,CAAC;IAEpB,mEAAmE;IACnE,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IAEnB,8EAA8E;IAC9E,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB,uEAAuE;IACvE,KAAK,CAAC,EAAE,aAAa,CAAC;IAEtB,qFAAqF;IACrF,OAAO,CAAC,EAAE,cAAc,GAAG,MAAM,CAAC;IAElC,+FAA+F;IAC/F,UAAU,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,MAAM,CAAC;IAExC;;;;yBAIqB;IACrB,SAAS,CAAC,EAAE,MAAM,GAAG,SAAS,GAAG,KAAK,CAAC;CACxC;AAED;;GAEG;AACH,MAAM,WAAW,wBAAwB;IACvC,KAAK,EAAE,SAAS,CAAC;IACjB,KAAK,EAAE,SAAS,GAAG,SAAS,EAAE,CAAC;IAC/B,KAAK,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,UAAU,CAAC;IACtC,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,OAAO,CAAC;IACnB,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,aAAa,CAAC;IACtB,OAAO,EAAE,cAAc,GAAG,MAAM,CAAC;IACjC,UAAU,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,MAAM,CAAC;IACxC,SAAS,EAAE,MAAM,GAAG,SAAS,GAAG,KAAK,CAAC;CACvC;AAED;;GAEG;AACH,MAAM,WAAW,wBAAwB;IACvC,KAAK,EAAE,SAAS,CAAC;IACjB,KAAK,EAAE,SAAS,CAAC;IACjB,KAAK,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,UAAU,CAAC;IACtC,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,OAAO,CAAC;IACnB,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,aAAa,CAAC;IACtB,OAAO,EAAE,cAAc,GAAG,MAAM,CAAC;IACjC,UAAU,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,MAAM,CAAC;IACxC,SAAS,EAAE,MAAM,GAAG,SAAS,GAAG,KAAK,CAAC;CACvC;AAED;;;GAGG;AACH,eAAO,MAAM,mBAAmB,mDAAoD,CAAC;AAErF;;;;GAIG;AACH,eAAO,MAAM,cAAc,uEAAwE,CAAC;AAEpG;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,qCAAqC;IACrC,IAAI,EAAE,MAAM,CAAC;IACb,uCAAuC;IACvC,IAAI,EAAE,MAAM,CAAC;IACb,4BAA4B;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,gDAAgD;IAChD,QAAQ,EAAE,OAAO,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,0BAA0B;IAC1B,MAAM,EAAE,QAAQ,GAAG,QAAQ,CAAC;IAC5B,8BAA8B;IAC9B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,0BAA0B;IAC1B,QAAQ,EAAE,MAAM,CAAC;IACjB,8BAA8B;IAC9B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,iEAAiE;IACjE,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,8DAA8D;IAC9D,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,wDAAwD;IACxD,WAAW,CAAC,EAAE;QACZ,kCAAkC;QAClC,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,8DAA8D;QAC9D,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;KAClC,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,0DAA0D;IAC1D,MAAM,EAAE,aAAa,CAAC;IACtB,yEAAyE;IACzE,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,qDAAqD;IACrD,aAAa,CAAC,EAAE;QACd,0BAA0B;QAC1B,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,qDAAqD;QACrD,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;KAClC,CAAC;IACF,0FAA0F;IAC1F,cAAc,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACxC,6DAA6D;IAC7D,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;CACzB;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,uBAAuB;IACvB,IAAI,EAAE,MAAM,CAAC;IACb,2BAA2B;IAC3B,SAAS,EAAE,MAAM,CAAC;IAClB,4BAA4B;IAC5B,UAAU,EAAE,MAAM,CAAC;IACnB,gCAAgC;IAChC,QAAQ,EAAE,MAAM,CAAC;IACjB,oCAAoC;IACpC,YAAY,EAAE,MAAM,CAAC;IACrB,2EAA2E;IAC3E,IAAI,EAAE,WAAW,EAAE,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,MAAM,WAAW,GAAG,OAAO,GAAG,OAAO,GAAG,SAAS,CAAC;AAExD;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,WAAW,EAAE,WAAW,CAAC;IACzB,aAAa,EAAE,MAAM,CAAC;IACtB,wGAAwG;IACxG,YAAY,CAAC,EAAE,OAAO,CAAC;CACxB;AAED;;;GAGG;AACH,MAAM,MAAM,aAAa,GACrB;IAAE,IAAI,EAAE,kBAAkB,CAAC;IAAC,aAAa,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE,GAC1F;IAAE,IAAI,EAAE,YAAY,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE,GAC9E;IAAE,IAAI,EAAE,eAAe,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,aAAa,CAAA;CAAE,GACxG;IAAE,IAAI,EAAE,YAAY,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE,GAC3D;IAAE,IAAI,EAAE,sBAAsB,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE,GACrE;IAAE,IAAI,EAAE,kBAAkB,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE,GAC/C;IAAE,IAAI,EAAE,oBAAoB,CAAC;IAAC,OAAO,EAAE,iBAAiB,CAAA;CAAE,CAAC;AAE/D;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,wCAAwC;IACxC,SAAS,EAAE,MAAM,CAAC;IAClB,0CAA0C;IAC1C,WAAW,EAAE,MAAM,CAAC;IACpB,oCAAoC;IACpC,MAAM,EAAE,wBAAwB,CAAC;IACjC,4BAA4B;IAC5B,KAAK,EAAE,WAAW,EAAE,CAAC;CACtB"}

package/dist/lib/types.js

@@ -0,0 +1,15 @@
+/**
+ * Core types for the eval framework.
+ */
+/**
+ * Required files for a valid eval fixture.
+ * Note: Either EVAL.ts or EVAL.tsx is required (not both).
+ */
+export const REQUIRED_EVAL_FILES = ['PROMPT.md', 'EVAL.ts', 'package.json'];
+/**
+ * Files excluded when listing fixture files (used by getFixtureFiles in fixture.ts).
+ * This is for local fixture introspection, NOT for sandbox uploads.
+ * For sandbox file filtering, see TEST_FILE_PATTERNS in sandbox.ts.
+ */
+export const EXCLUDED_FILES = ['PROMPT.md', 'EVAL.ts', 'EVAL.tsx', 'node_modules', '.git'];
+//# sourceMappingURL=types.js.map

package/dist/lib/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/lib/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAsIH;;;GAGG;AACH,MAAM,CAAC,MAAM,mBAAmB,GAAG,CAAC,WAAW,EAAE,SAAS,EAAE,cAAc,CAAU,CAAC;AAErF;;;;GAIG;AACH,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,WAAW,EAAE,SAAS,EAAE,UAAU,EAAE,cAAc,EAAE,MAAM,CAAU,CAAC"}

package/dist/test-setup.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=test-setup.d.ts.map

package/dist/test-setup.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"test-setup.d.ts","sourceRoot":"","sources":["../src/test-setup.ts"],"names":[],"mappings":""}

package/dist/test-setup.js

@@ -0,0 +1,6 @@
+/**
+ * Test setup file - loads environment variables for tests.
+ */
+import { config as dotenvConfig } from 'dotenv';
+dotenvConfig();
+//# sourceMappingURL=test-setup.js.map

package/dist/test-setup.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"test-setup.js","sourceRoot":"","sources":["../src/test-setup.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,OAAO,EAAE,MAAM,IAAI,YAAY,EAAE,MAAM,QAAQ,CAAC;AAEhD,YAAY,EAAE,CAAC"}