@vercel/agent-eval 0.0.1

package/dist/lib/runner.js

@@ -0,0 +1,142 @@
+/**
+ * Experiment runner - orchestrates running evals against agent.
+ * All evals and attempts run concurrently for maximum throughput.
+ * With earlyExit, in-flight attempts are aborted when one passes.
+ */
+import { getAgent } from './agents/index.js';
+import { agentResultToEvalRunData, createEvalSummary, createExperimentResults, saveResults, formatResultsTable, formatRunResult, createProgressDisplay, } from './results.js';
+/**
+ * Run an experiment - execute all evals with configured runs concurrently.
+ * With earlyExit enabled, remaining attempts for a fixture are aborted once one passes.
+ */
+export async function runExperiment(options) {
+    const { config, fixtures, apiKey, resultsDir, experimentName, onProgress, verbose } = options;
+    const startedAt = new Date();
+    // Get the agent from registry
+    const agent = getAgent(config.agent);
+    const log = (msg) => {
+        if (onProgress) {
+            onProgress(msg);
+        }
+        else if (verbose) {
+            console.log(msg);
+        }
+    };
+    // Create AbortController per fixture for earlyExit
+    const abortControllers = new Map();
+    for (const fixture of fixtures) {
+        abortControllers.set(fixture.name, new AbortController());
+    }
+    // Build list of all attempts to run
+    const attempts = [];
+    for (const fixture of fixtures) {
+        for (let i = 0; i < config.runs; i++) {
+            attempts.push({ fixture, runIndex: i });
+        }
+    }
+    log(`Starting ${attempts.length} eval attempts concurrently (${fixtures.length} evals × ${config.runs} runs)`);
+    // Run a single attempt
+    const runAttempt = async (attempt) => {
+        const { fixture, runIndex } = attempt;
+        const controller = abortControllers.get(fixture.name);
+        // Check if already aborted before starting
+        if (controller.signal.aborted) {
+            return {
+                fixtureName: fixture.name,
+                runIndex,
+                runData: {
+                    result: { status: 'failed', error: 'Aborted', duration: 0 },
+                },
+                aborted: true,
+            };
+        }
+        log(createProgressDisplay(fixture.name, runIndex + 1, config.runs));
+        const agentResult = await agent.run(fixture.path, {
+            prompt: fixture.prompt,
+            model: config.model,
+            timeout: config.timeout * 1000,
+            apiKey,
+            setup: config.setup,
+            scripts: config.scripts,
+            signal: config.earlyExit ? controller.signal : undefined,
+        });
+        // Check if this was aborted
+        if (agentResult.error === 'Aborted' || agentResult.error === 'Aborted before start') {
+            return {
+                fixtureName: fixture.name,
+                runIndex,
+                runData: {
+                    result: { status: 'failed', error: 'Aborted', duration: agentResult.duration / 1000 },
+                },
+                aborted: true,
+            };
+        }
+        const runData = agentResultToEvalRunData(agentResult);
+        log(formatRunResult(fixture.name, runIndex + 1, config.runs, runData.result));
+        // If this attempt passed and earlyExit is enabled, abort remaining attempts
+        if (config.earlyExit && runData.result.status === 'passed') {
+            log(`Early exit: ${fixture.name} passed on run ${runIndex + 1}, aborting remaining attempts`);
+            controller.abort();
+        }
+        return {
+            fixtureName: fixture.name,
+            runIndex,
+            runData,
+        };
+    };
+    // Run all attempts concurrently
+    const results = await Promise.all(attempts.map(runAttempt));
+    // Group results by fixture, excluding aborted results
+    const resultsByFixture = new Map();
+    for (const fixture of fixtures) {
+        resultsByFixture.set(fixture.name, []);
+    }
+    for (const result of results) {
+        if (!result.aborted) {
+            resultsByFixture.get(result.fixtureName).push(result);
+        }
+    }
+    // Build eval summaries, respecting earlyExit
+    const evalSummaries = [];
+    for (const fixture of fixtures) {
+        const fixtureResults = resultsByFixture.get(fixture.name);
+        // Sort by run index to process in order
+        fixtureResults.sort((a, b) => a.runIndex - b.runIndex);
+        const runDataList = [];
+        for (const result of fixtureResults) {
+            runDataList.push(result.runData);
+            // With earlyExit, stop counting after first pass
+            if (config.earlyExit && result.runData.result.status === 'passed') {
+                break;
+            }
+        }
+        const summary = createEvalSummary(fixture.name, runDataList);
+        evalSummaries.push(summary);
+    }
+    const completedAt = new Date();
+    const experimentResults = createExperimentResults(config, evalSummaries, startedAt, completedAt);
+    // Save results to disk
+    const outputDir = saveResults(experimentResults, {
+        resultsDir,
+        experimentName,
+    });
+    log(`\nResults saved to: ${outputDir}`);
+    log(formatResultsTable(experimentResults));
+    return experimentResults;
+}
+/**
+ * Run a single eval (for testing/debugging).
+ */
+export async function runSingleEval(fixture, options) {
+    const agent = getAgent(options.agent ?? 'vercel-ai-gateway/claude-code');
+    const agentResult = await agent.run(fixture.path, {
+        prompt: fixture.prompt,
+        model: options.model,
+        timeout: options.timeout * 1000,
+        apiKey: options.apiKey,
+        setup: options.setup,
+        scripts: options.scripts,
+    });
+    return agentResultToEvalRunData(agentResult);
+}
+//# sourceMappingURL=runner.js.map

package/dist/lib/runner.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"runner.js","sourceRoot":"","sources":["../../src/lib/runner.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AASH,OAAO,EAAE,QAAQ,EAAE,MAAM,mBAAmB,CAAC;AAC7C,OAAO,EACL,wBAAwB,EACxB,iBAAiB,EACjB,uBAAuB,EACvB,WAAW,EACX,kBAAkB,EAClB,eAAe,EACf,qBAAqB,GACtB,MAAM,cAAc,CAAC;AAwCtB;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,OAA6B;IAE7B,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,UAAU,EAAE,cAAc,EAAE,UAAU,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC;IAC9F,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC;IAE7B,8BAA8B;IAC9B,MAAM,KAAK,GAAG,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAErC,MAAM,GAAG,GAAG,CAAC,GAAW,EAAE,EAAE;QAC1B,IAAI,UAAU,EAAE,CAAC;YACf,UAAU,CAAC,GAAG,CAAC,CAAC;QAClB,CAAC;aAAM,IAAI,OAAO,EAAE,CAAC;YACnB,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACnB,CAAC;IACH,CAAC,CAAC;IAEF,mDAAmD;IACnD,MAAM,gBAAgB,GAAG,IAAI,GAAG,EAA2B,CAAC;IAC5D,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,gBAAgB,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,eAAe,EAAE,CAAC,CAAC;IAC5D,CAAC;IAED,oCAAoC;IACpC,MAAM,QAAQ,GAAkB,EAAE,CAAC;IACnC,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC;YACrC,QAAQ,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC;QAC1C,CAAC;IACH,CAAC;IAED,GAAG,CAAC,YAAY,QAAQ,CAAC,MAAM,gCAAgC,QAAQ,CAAC,MAAM,YAAY,MAAM,CAAC,IAAI,QAAQ,CAAC,CAAC;IAE/G,uBAAuB;IACvB,MAAM,UAAU,GAAG,KAAK,EAAE,OAAoB,EAA0B,EAAE;QACxE,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC;QACtC,MAAM,UAAU,GAAG,gBAAgB,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,CAAE,CAAC;QAEvD,2CAA2C;QAC3C,IAAI,UAAU,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YAC9B,OAAO;gBACL,WAAW,EAAE,OAAO,CAAC,IAAI;gBACzB,QAAQ;gBACR,OAAO,EAAE;oBACP,MAAM,EAAE,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC,EAAE;iBAC5D;gBACD,OAAO,EAAE,IAAI;aACd,CAAC;QACJ,CAAC;QAED,GAAG,CAAC,qBAAqB,CAAC,OAAO,CAAC,IAAI,EAAE,QAAQ,GAAG,CAAC,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;QAEpE,MAAM,WAAW,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE;YAChD,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,OAAO,EAAE,MAAM,CAAC,OAAO,GAAG,IAAI;YAC9B,MAAM;YACN,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,OAAO,EAAE,MAAM,CAAC,OAAO;YACvB,MAAM,EAAE,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS;SACzD,CAAC,CAAC;QAEH,4BAA4B;QAC5B,IAAI,WAAW,CAAC,KAAK,KAAK,SAAS,IAAI,WAAW,CAAC,KAAK,KAAK,sBAAsB,EAAE,CAAC;YACpF,OAAO;gBACL,WAAW,EAAE,OAAO,CAAC,IAAI;gBACzB,QAAQ;gBACR,OAAO,EAAE;oBACP,MAAM,EAAE,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,SAAS,EAAE,QAAQ,EAAE,WAAW,CAAC,QAAQ,GAAG,IAAI,EAAE;iBACtF;gBACD,OAAO,EAAE,IAAI;aACd,CAAC;QACJ,CAAC;QAED,MAAM,OAAO,GAAG,wBAAwB,CAAC,WAAW,CAAC,CAAC;QAEtD,GAAG,CAAC,eAAe,CAAC,OAAO,CAAC,IAAI,EAAE,QAAQ,GAAG,CAAC,EAAE,MAAM,CAAC,IAAI,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;QAE9E,4EAA4E;QAC5E,IAAI,MAAM,CAAC,SAAS,IAAI,OAAO,CAAC,MAAM,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;YAC3D,GAAG,CAAC,eAAe,OAAO,CAAC,IAAI,kBAAkB,QAAQ,GAAG,CAAC,+BAA+B,CAAC,CAAC;YAC9F,UAAU,CAAC,KAAK,EAAE,CAAC;QACrB,CAAC;QAED,OAAO;YACL,WAAW,EAAE,OAAO,CAAC,IAAI;YACzB,QAAQ;YACR,OAAO;SACR,CAAC;IACJ,CAAC,CAAC;IAEF,gCAAgC;IAChC,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC;IAE5D,sDAAsD;IACtD,MAAM,gBAAgB,GAAG,IAAI,GAAG,EAA2B,CAAC;IAC5D,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,gBAAgB,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;IACzC,CAAC;IAED,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YACpB,gBAAgB,CAAC,GAAG,CAAC,MAAM,CAAC,WAAW,CAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACzD,CAAC;IACH,CAAC;IAED,6CAA6C;IAC7C,MAAM,aAAa,GAAkB,EAAE,CAAC;IACxC,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,MAAM,cAAc,GAAG,gBAAgB,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,CAAE,CAAC;QAE3D,wCAAwC;QACxC,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC;QAEvD,MAAM,WAAW,GAAkB,EAAE,CAAC;QACtC,KAAK,MAAM,MAAM,IAAI,cAAc,EAAE,CAAC;YACpC,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YAEjC,iDAAiD;YACjD,IAAI,MAAM,CAAC,SAAS,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;gBAClE,MAAM;YACR,CAAC;QACH,CAAC;QAED,MAAM,OAAO,GAAG,iBAAiB,CAAC,OAAO,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC;QAC7D,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAC9B,CAAC;IAED,MAAM,WAAW,GAAG,IAAI,IAAI,EAAE,CAAC;IAC/B,MAAM,iBAAiB,GAAG,uBAAuB,CAAC,MAAM,EAAE,aAAa,EAAE,SAAS,EAAE,WAAW,CAAC,CAAC;IAEjG,uBAAuB;IACvB,MAAM,SAAS,GAAG,WAAW,CAAC,iBAAiB,EAAE;QAC/C,UAAU;QACV,cAAc;KACf,CAAC,CAAC;IAEH,GAAG,CAAC,uBAAuB,SAAS,EAAE,CAAC,CAAC;IACxC,GAAG,CAAC,kBAAkB,CAAC,iBAAiB,CAAC,CAAC,CAAC;IAE3C,OAAO,iBAAiB,CAAC;AAC3B,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,OAAoB,EACpB,OAQC;IAED,MAAM,KAAK,GAAG,QAAQ,CAAC,OAAO,CAAC,KAAK,IAAI,+BAA+B,CAAC,CAAC;IAEzE,MAAM,WAAW,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE;QAChD,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,OAAO,EAAE,OAAO,CAAC,OAAO,GAAG,IAAI;QAC/B,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,OAAO,EAAE,OAAO,CAAC,OAAO;KACzB,CAAC,CAAC;IAEH,OAAO,wBAAwB,CAAC,WAAW,CAAC,CAAC;AAC/C,CAAC"}

package/dist/lib/sandbox.d.ts

@@ -0,0 +1,117 @@
+/**
+ * Vercel Sandbox integration for isolated eval execution.
+ */
+import { Sandbox as VercelSandbox } from '@vercel/sandbox';
+import type { Sandbox } from './types.js';
+/**
+ * Default timeout for sandbox operations (10 minutes).
+ */
+export declare const DEFAULT_SANDBOX_TIMEOUT = 600000;
+/**
+ * Files to ignore when copying to sandbox.
+ * These are build artifacts and dependencies that shouldn't be uploaded.
+ * Note: This is a general-purpose pattern list used by collectLocalFiles().
+ * For eval-specific exclusions (PROMPT.md, EVAL.ts), see TEST_FILE_PATTERNS.
+ */
+export declare const IGNORED_PATTERNS: string[];
+/**
+ * Test/eval file patterns to withhold from agent during task execution.
+ * These files are uploaded AFTER the agent completes for validation.
+ * - PROMPT.md: Contains the task - agent receives this via CLI argument, not as a file
+ * - EVAL.ts: Validation tests - must be hidden so agent can't "cheat"
+ * - *.test.ts/tsx: Additional test files that shouldn't influence agent
+ */
+export declare const TEST_FILE_PATTERNS: string[];
+/**
+ * Options for creating a sandbox.
+ */
+export interface SandboxOptions {
+    /** Timeout in milliseconds */
+    timeout?: number;
+    /** Runtime environment */
+    runtime?: 'node20' | 'node24';
+}
+/**
+ * Result of running a command in the sandbox.
+ */
+export interface CommandResult {
+    stdout: string;
+    stderr: string;
+    exitCode: number;
+}
+/**
+ * File to upload to sandbox.
+ */
+export interface SandboxFile {
+    path: string;
+    content: Buffer | string;
+}
+/**
+ * Wrapper around Vercel Sandbox providing a cleaner API.
+ */
+export declare class SandboxManager implements Sandbox {
+    private sandbox;
+    private _workingDirectory;
+    constructor(sandbox: VercelSandbox);
+    /**
+     * Create a new sandbox instance.
+     */
+    static create(options?: SandboxOptions): Promise<SandboxManager>;
+    /**
+     * Get the sandbox ID.
+     */
+    get sandboxId(): string;
+    /**
+     * Run a command in the sandbox.
+     */
+    runCommand(command: string, args?: string[], options?: {
+        env?: Record<string, string>;
+    }): Promise<CommandResult>;
+    /**
+     * Run a shell command (through bash).
+     */
+    runShell(command: string, env?: Record<string, string>): Promise<CommandResult>;
+    /**
+     * Read a file from the sandbox.
+     */
+    readFile(path: string): Promise<string>;
+    /**
+     * Check if a file exists in the sandbox.
+     */
+    fileExists(path: string): Promise<boolean>;
+    /**
+     * Write files to the sandbox.
+     */
+    writeFiles(files: Record<string, string>): Promise<void>;
+    /**
+     * Upload files from local filesystem to sandbox.
+     */
+    uploadFiles(files: SandboxFile[]): Promise<void>;
+    /**
+     * Get the working directory.
+     */
+    getWorkingDirectory(): string;
+    /**
+     * Stop and clean up the sandbox.
+     */
+    stop(): Promise<void>;
+}
+/**
+ * Collect files from a local directory for uploading to sandbox.
+ */
+export declare function collectLocalFiles(dir: string, options?: {
+    excludePatterns?: string[];
+    includePatterns?: string[];
+}): Promise<SandboxFile[]>;
+/**
+ * Split files into workspace files (visible to agent) and test files (hidden until validation).
+ */
+export declare function splitTestFiles(files: SandboxFile[]): {
+    workspaceFiles: SandboxFile[];
+    testFiles: SandboxFile[];
+};
+/**
+ * Verify that no test files exist in the sandbox.
+ */
+export declare function verifyNoTestFiles(sandbox: SandboxManager): Promise<void>;
+//# sourceMappingURL=sandbox.d.ts.map

package/dist/lib/sandbox.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"sandbox.d.ts","sourceRoot":"","sources":["../../src/lib/sandbox.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,OAAO,IAAI,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAC3D,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAI1C;;GAEG;AACH,eAAO,MAAM,uBAAuB,SAAS,CAAC;AAE9C;;;;;GAKG;AACH,eAAO,MAAM,gBAAgB,UAU5B,CAAC;AAEF;;;;;;GAMG;AACH,eAAO,MAAM,kBAAkB,UAAsD,CAAC;AAEtF;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,8BAA8B;IAC9B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,0BAA0B;IAC1B,OAAO,CAAC,EAAE,QAAQ,GAAG,QAAQ,CAAC;CAC/B;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,GAAG,MAAM,CAAC;CAC1B;AAED;;GAEG;AACH,qBAAa,cAAe,YAAW,OAAO;IAC5C,OAAO,CAAC,OAAO,CAAgB;IAC/B,OAAO,CAAC,iBAAiB,CAA6B;gBAE1C,OAAO,EAAE,aAAa;IAIlC;;OAEG;WACU,MAAM,CAAC,OAAO,GAAE,cAAmB,GAAG,OAAO,CAAC,cAAc,CAAC;IAQ1E;;OAEG;IACH,IAAI,SAAS,IAAI,MAAM,CAEtB;IAED;;OAEG;IACG,UAAU,CACd,OAAO,EAAE,MAAM,EACf,IAAI,GAAE,MAAM,EAAO,EACnB,OAAO,GAAE;QAAE,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;KAAO,GAC7C,OAAO,CAAC,aAAa,CAAC;IAczB;;OAEG;IACG,QAAQ,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,OAAO,CAAC,aAAa,CAAC;IAcrF;;OAEG;IACG,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAQ7C;;OAEG;IACG,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAKhD;;OAEG;IACG,UAAU,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IAa9D;;OAEG;IACG,WAAW,CAAC,KAAK,EAAE,WAAW,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAStD;;OAEG;IACH,mBAAmB,IAAI,MAAM;IAI7B;;OAEG;IACG,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;CAG5B;AAED;;GAEG;AACH,wBAAsB,iBAAiB,CACrC,GAAG,EAAE,MAAM,EACX,OAAO,GAAE;IACP,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAC3B,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;CACvB,GACL,OAAO,CAAC,WAAW,EAAE,CAAC,CA+DxB;AAmBD;;GAEG;AACH,wBAAgB,cAAc,CAAC,KAAK,EAAE,WAAW,EAAE,GAAG;IACpD,cAAc,EAAE,WAAW,EAAE,CAAC;IAC9B,SAAS,EAAE,WAAW,EAAE,CAAC;CAC1B,CAeA;AAED;;GAEG;AACH,wBAAsB,iBAAiB,CAAC,OAAO,EAAE,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC,CAS9E"}

package/dist/lib/sandbox.js

@@ -0,0 +1,248 @@
+/**
+ * Vercel Sandbox integration for isolated eval execution.
+ */
+import { Sandbox as VercelSandbox } from '@vercel/sandbox';
+import { readFileSync } from 'fs';
+import { join } from 'path';
+/**
+ * Default timeout for sandbox operations (10 minutes).
+ */
+export const DEFAULT_SANDBOX_TIMEOUT = 600000;
+/**
+ * Files to ignore when copying to sandbox.
+ * These are build artifacts and dependencies that shouldn't be uploaded.
+ * Note: This is a general-purpose pattern list used by collectLocalFiles().
+ * For eval-specific exclusions (PROMPT.md, EVAL.ts), see TEST_FILE_PATTERNS.
+ */
+export const IGNORED_PATTERNS = [
+    '.git',
+    '.next',
+    'node_modules',
+    '.DS_Store',
+    '*.log',
+    'build',
+    'dist',
+    'pnpm-lock.yaml',
+    'package-lock.json',
+];
+/**
+ * Test/eval file patterns to withhold from agent during task execution.
+ * These files are uploaded AFTER the agent completes for validation.
+ * - PROMPT.md: Contains the task - agent receives this via CLI argument, not as a file
+ * - EVAL.ts: Validation tests - must be hidden so agent can't "cheat"
+ * - *.test.ts/tsx: Additional test files that shouldn't influence agent
+ */
+export const TEST_FILE_PATTERNS = ['*.test.tsx', '*.test.ts', 'EVAL.ts', 'PROMPT.md'];
+/**
+ * Wrapper around Vercel Sandbox providing a cleaner API.
+ */
+export class SandboxManager {
+    sandbox;
+    _workingDirectory = '/vercel/sandbox';
+    constructor(sandbox) {
+        this.sandbox = sandbox;
+    }
+    /**
+     * Create a new sandbox instance.
+     */
+    static async create(options = {}) {
+        const timeout = options.timeout ?? DEFAULT_SANDBOX_TIMEOUT;
+        const runtime = options.runtime ?? 'node24';
+        const sandbox = await VercelSandbox.create({ runtime, timeout });
+        return new SandboxManager(sandbox);
+    }
+    /**
+     * Get the sandbox ID.
+     */
+    get sandboxId() {
+        return this.sandbox.sandboxId;
+    }
+    /**
+     * Run a command in the sandbox.
+     */
+    async runCommand(command, args = [], options = {}) {
+        const result = await this.sandbox.runCommand({
+            cmd: command,
+            args,
+            env: options.env,
+        });
+        return {
+            stdout: await result.stdout(),
+            stderr: await result.stderr(),
+            exitCode: result.exitCode,
+        };
+    }
+    /**
+     * Run a shell command (through bash).
+     */
+    async runShell(command, env) {
+        const result = await this.sandbox.runCommand({
+            cmd: 'bash',
+            args: ['-c', command],
+            env,
+        });
+        return {
+            stdout: await result.stdout(),
+            stderr: await result.stderr(),
+            exitCode: result.exitCode,
+        };
+    }
+    /**
+     * Read a file from the sandbox.
+     */
+    async readFile(path) {
+        const result = await this.runCommand('cat', [path]);
+        if (result.exitCode !== 0) {
+            throw new Error(`Failed to read file ${path}: ${result.stderr}`);
+        }
+        return result.stdout;
+    }
+    /**
+     * Check if a file exists in the sandbox.
+     */
+    async fileExists(path) {
+        const result = await this.runCommand('test', ['-f', path]);
+        return result.exitCode === 0;
+    }
+    /**
+     * Write files to the sandbox.
+     */
+    async writeFiles(files) {
+        const sandboxFiles = [];
+        for (const [path, content] of Object.entries(files)) {
+            sandboxFiles.push({
+                path,
+                content: Buffer.from(content, 'utf-8'),
+            });
+        }
+        await this.sandbox.writeFiles(sandboxFiles);
+    }
+    /**
+     * Upload files from local filesystem to sandbox.
+     */
+    async uploadFiles(files) {
+        const sandboxFiles = files.map((f) => ({
+            path: f.path,
+            content: typeof f.content === 'string' ? Buffer.from(f.content, 'utf-8') : f.content,
+        }));
+        await this.sandbox.writeFiles(sandboxFiles);
+    }
+    /**
+     * Get the working directory.
+     */
+    getWorkingDirectory() {
+        return this._workingDirectory;
+    }
+    /**
+     * Stop and clean up the sandbox.
+     */
+    async stop() {
+        await this.sandbox.stop();
+    }
+}
+/**
+ * Collect files from a local directory for uploading to sandbox.
+ */
+export async function collectLocalFiles(dir, options = {}) {
+    const { readdirSync, statSync } = await import('fs');
+    const excludePatterns = options.excludePatterns ?? IGNORED_PATTERNS;
+    const includePatterns = options.includePatterns;
+    const files = [];
+    function shouldExclude(name, relativePath) {
+        for (const pattern of excludePatterns) {
+            if (pattern.startsWith('*.')) {
+                // Wildcard pattern
+                const ext = pattern.slice(1);
+                if (name.endsWith(ext)) {
+                    return true;
+                }
+            }
+            else if (name === pattern || relativePath === pattern) {
+                return true;
+            }
+        }
+        return false;
+    }
+    function shouldInclude(name) {
+        if (!includePatterns) {
+            return true;
+        }
+        for (const pattern of includePatterns) {
+            if (pattern.startsWith('*.')) {
+                const ext = pattern.slice(1);
+                if (name.endsWith(ext)) {
+                    return true;
+                }
+            }
+            else if (name === pattern) {
+                return true;
+            }
+        }
+        return false;
+    }
+    function walk(currentDir, relativePath = '') {
+        const entries = readdirSync(currentDir);
+        for (const entry of entries) {
+            const entryRelativePath = relativePath ? `${relativePath}/${entry}` : entry;
+            const fullPath = join(currentDir, entry);
+            if (shouldExclude(entry, entryRelativePath)) {
+                continue;
+            }
+            const stat = statSync(fullPath);
+            if (stat.isDirectory()) {
+                walk(fullPath, entryRelativePath);
+            }
+            else if (shouldInclude(entry)) {
+                const content = readFileSync(fullPath);
+                files.push({ path: entryRelativePath, content });
+            }
+        }
+    }
+    walk(dir);
+    return files;
+}
+/**
+ * Check if a filename matches any of the test file patterns.
+ */
+function isTestFilePattern(filename) {
+    for (const pattern of TEST_FILE_PATTERNS) {
+        if (pattern.startsWith('*.')) {
+            const ext = pattern.slice(1);
+            if (filename.endsWith(ext)) {
+                return true;
+            }
+        }
+        else if (filename === pattern) {
+            return true;
+        }
+    }
+    return false;
+}
+/**
+ * Split files into workspace files (visible to agent) and test files (hidden until validation).
+ */
+export function splitTestFiles(files) {
+    const workspaceFiles = [];
+    const testFiles = [];
+    for (const file of files) {
+        const name = file.path.split('/').pop() ?? file.path;
+        if (isTestFilePattern(name)) {
+            testFiles.push(file);
+        }
+        else {
+            workspaceFiles.push(file);
+        }
+    }
+    return { workspaceFiles, testFiles };
+}
+/**
+ * Verify that no test files exist in the sandbox.
+ */
+export async function verifyNoTestFiles(sandbox) {
+    const result = await sandbox.runShell("find . -path './node_modules' -prune -o \\( -name '*.test.tsx' -o -name '*.test.ts' -o -name 'EVAL.ts' \\) -print");
+    const foundTests = result.stdout.trim();
+    if (foundTests) {
+        throw new Error(`Test files found in sandbox before agent run: ${foundTests}`);
+    }
+}
+//# sourceMappingURL=sandbox.js.map

package/dist/lib/sandbox.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"sandbox.js","sourceRoot":"","sources":["../../src/lib/sandbox.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,OAAO,IAAI,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAE3D,OAAO,EAAE,YAAY,EAAE,MAAM,IAAI,CAAC;AAClC,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAE5B;;GAEG;AACH,MAAM,CAAC,MAAM,uBAAuB,GAAG,MAAM,CAAC;AAE9C;;;;;GAKG;AACH,MAAM,CAAC,MAAM,gBAAgB,GAAG;IAC9B,MAAM;IACN,OAAO;IACP,cAAc;IACd,WAAW;IACX,OAAO;IACP,OAAO;IACP,MAAM;IACN,gBAAgB;IAChB,mBAAmB;CACpB,CAAC;AAEF;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAAC,YAAY,EAAE,WAAW,EAAE,SAAS,EAAE,WAAW,CAAC,CAAC;AA6BtF;;GAEG;AACH,MAAM,OAAO,cAAc;IACjB,OAAO,CAAgB;IACvB,iBAAiB,GAAW,iBAAiB,CAAC;IAEtD,YAAY,OAAsB;QAChC,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;IACzB,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,UAA0B,EAAE;QAC9C,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,uBAAuB,CAAC;QAC3D,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,QAAQ,CAAC;QAE5C,MAAM,OAAO,GAAG,MAAM,aAAa,CAAC,MAAM,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC;QACjE,OAAO,IAAI,cAAc,CAAC,OAAO,CAAC,CAAC;IACrC,CAAC;IAED;;OAEG;IACH,IAAI,SAAS;QACX,OAAO,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC;IAChC,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,UAAU,CACd,OAAe,EACf,OAAiB,EAAE,EACnB,UAA4C,EAAE;QAE9C,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC;YAC3C,GAAG,EAAE,OAAO;YACZ,IAAI;YACJ,GAAG,EAAE,OAAO,CAAC,GAAG;SACjB,CAAC,CAAC;QAEH,OAAO;YACL,MAAM,EAAE,MAAM,MAAM,CAAC,MAAM,EAAE;YAC7B,MAAM,EAAE,MAAM,MAAM,CAAC,MAAM,EAAE;YAC7B,QAAQ,EAAE,MAAM,CAAC,QAAQ;SAC1B,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,QAAQ,CAAC,OAAe,EAAE,GAA4B;QAC1D,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC;YAC3C,GAAG,EAAE,MAAM;YACX,IAAI,EAAE,CAAC,IAAI,EAAE,OAAO,CAAC;YACrB,GAAG;SACJ,CAAC,CAAC;QAEH,OAAO;YACL,MAAM,EAAE,MAAM,MAAM,CAAC,MAAM,EAAE;YAC7B,MAAM,EAAE,MAAM,MAAM,CAAC,MAAM,EAAE;YAC7B,QAAQ,EAAE,MAAM,CAAC,QAAQ;SAC1B,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,QAAQ,CAAC,IAAY;QACzB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC;QACpD,IAAI,MAAM,CAAC,QAAQ,KAAK,CAAC,EAAE,CAAC;YAC1B,MAAM,IAAI,KAAK,CAAC,uBAAuB,IAAI,KAAK,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;QACnE,CAAC;QACD,OAAO,MAAM,CAAC,MAAM,CAAC;IACvB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,UAAU,CAAC,IAAY;QAC3B,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC;QAC3D,OAAO,MAAM,CAAC,QAAQ,KAAK,CAAC,CAAC;IAC/B,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,UAAU,CAAC,KAA6B;QAC5C,MAAM,YAAY,GAA6C,EAAE,CAAC;QAElE,KAAK,MAAM,CAAC,IAAI,EAAE,OAAO,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;YACpD,YAAY,CAAC,IAAI,CAAC;gBAChB,IAAI;gBACJ,OAAO,EAAE,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC;aACvC,CAAC,CAAC;QACL,CAAC;QAED,MAAM,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC;IAC9C,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,WAAW,CAAC,KAAoB;QACpC,MAAM,YAAY,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YACrC,IAAI,EAAE,CAAC,CAAC,IAAI;YACZ,OAAO,EAAE,OAAO,CAAC,CAAC,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO;SACrF,CAAC,CAAC,CAAC;QAEJ,MAAM,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC;IAC9C,CAAC;IAED;;OAEG;IACH,mBAAmB;QACjB,OAAO,IAAI,CAAC,iBAAiB,CAAC;IAChC,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,IAAI;QACR,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;IAC5B,CAAC;CACF;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,GAAW,EACX,UAGI,EAAE;IAEN,MAAM,EAAE,WAAW,EAAE,QAAQ,EAAE,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,CAAC;IAErD,MAAM,eAAe,GAAG,OAAO,CAAC,eAAe,IAAI,gBAAgB,CAAC;IACpE,MAAM,eAAe,GAAG,OAAO,CAAC,eAAe,CAAC;IAChD,MAAM,KAAK,GAAkB,EAAE,CAAC;IAEhC,SAAS,aAAa,CAAC,IAAY,EAAE,YAAoB;QACvD,KAAK,MAAM,OAAO,IAAI,eAAe,EAAE,CAAC;YACtC,IAAI,OAAO,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC7B,mBAAmB;gBACnB,MAAM,GAAG,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;gBAC7B,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;oBACvB,OAAO,IAAI,CAAC;gBACd,CAAC;YACH,CAAC;iBAAM,IAAI,IAAI,KAAK,OAAO,IAAI,YAAY,KAAK,OAAO,EAAE,CAAC;gBACxD,OAAO,IAAI,CAAC;YACd,CAAC;QACH,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;IAED,SAAS,aAAa,CAAC,IAAY;QACjC,IAAI,CAAC,eAAe,EAAE,CAAC;YACrB,OAAO,IAAI,CAAC;QACd,CAAC;QACD,KAAK,MAAM,OAAO,IAAI,eAAe,EAAE,CAAC;YACtC,IAAI,OAAO,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC7B,MAAM,GAAG,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;gBAC7B,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;oBACvB,OAAO,IAAI,CAAC;gBACd,CAAC;YACH,CAAC;iBAAM,IAAI,IAAI,KAAK,OAAO,EAAE,CAAC;gBAC5B,OAAO,IAAI,CAAC;YACd,CAAC;QACH,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;IAED,SAAS,IAAI,CAAC,UAAkB,EAAE,eAAuB,EAAE;QACzD,MAAM,OAAO,GAAG,WAAW,CAAC,UAAU,CAAC,CAAC;QAExC,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC5B,MAAM,iBAAiB,GAAG,YAAY,CAAC,CAAC,CAAC,GAAG,YAAY,IAAI,KAAK,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC;YAC5E,MAAM,QAAQ,GAAG,IAAI,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC;YAEzC,IAAI,aAAa,CAAC,KAAK,EAAE,iBAAiB,CAAC,EAAE,CAAC;gBAC5C,SAAS;YACX,CAAC;YAED,MAAM,IAAI,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAC;YAEhC,IAAI,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC;gBACvB,IAAI,CAAC,QAAQ,EAAE,iBAAiB,CAAC,CAAC;YACpC,CAAC;iBAAM,IAAI,aAAa,CAAC,KAAK,CAAC,EAAE,CAAC;gBAChC,MAAM,OAAO,GAAG,YAAY,CAAC,QAAQ,CAAC,CAAC;gBACvC,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,iBAAiB,EAAE,OAAO,EAAE,CAAC,CAAC;YACnD,CAAC;QACH,CAAC;IACH,CAAC;IAED,IAAI,CAAC,GAAG,CAAC,CAAC;IACV,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;GAEG;AACH,SAAS,iBAAiB,CAAC,QAAgB;IACzC,KAAK,MAAM,OAAO,IAAI,kBAAkB,EAAE,CAAC;QACzC,IAAI,OAAO,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;YAC7B,MAAM,GAAG,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;YAC7B,IAAI,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;gBAC3B,OAAO,IAAI,CAAC;YACd,CAAC;QACH,CAAC;aAAM,IAAI,QAAQ,KAAK,OAAO,EAAE,CAAC;YAChC,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,cAAc,CAAC,KAAoB;IAIjD,MAAM,cAAc,GAAkB,EAAE,CAAC;IACzC,MAAM,SAAS,GAAkB,EAAE,CAAC;IAEpC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,IAAI,IAAI,CAAC,IAAI,CAAC;QAErD,IAAI,iBAAiB,CAAC,IAAI,CAAC,EAAE,CAAC;YAC5B,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACvB,CAAC;aAAM,CAAC;YACN,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC5B,CAAC;IACH,CAAC;IAED,OAAO,EAAE,cAAc,EAAE,SAAS,EAAE,CAAC;AACvC,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,OAAuB;IAC7D,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,QAAQ,CACnC,mHAAmH,CACpH,CAAC;IAEF,MAAM,UAAU,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;IACxC,IAAI,UAAU,EAAE,CAAC;QACf,MAAM,IAAI,KAAK,CAAC,iDAAiD,UAAU,EAAE,CAAC,CAAC;IACjF,CAAC;AACH,CAAC"}