agent-eval-opencode 0.10.0

package/dist/lib/runner.js

@@ -0,0 +1,267 @@
+/**
+ * Experiment runner - orchestrates running evals against agent.
+ * Concurrency is controlled via an optional ConcurrencyLimiter shared across experiments.
+ * 429 rate-limit errors are retried with exponential backoff.
+ * With earlyExit, in-flight attempts are aborted when one passes.
+ */
+import { getAgent } from './agents/index.js';
+import { agentResultToEvalRunData, createEvalSummary, createExperimentResults, saveResults, } from './results.js';
+/**
+ * Rate-limits how many operations can START within a time window.
+ * Once started, operations run freely with no concurrency limit.
+ * Create one instance and share it across experiments to control global start rate.
+ */
+export class StartRateLimiter {
+    startsPerWindow;
+    windowMs;
+    queue = [];
+    started = 0;
+    timer = null;
+    constructor(startsPerWindow, windowMs) {
+        this.startsPerWindow = startsPerWindow;
+        this.windowMs = windowMs;
+    }
+    /**
+     * Wait for permission to start, then return immediately.
+     * The operation runs freely after this resolves.
+     */
+    async waitToStart() {
+        if (this.started < this.startsPerWindow) {
+            this.started++;
+            this.ensureTimer();
+            return;
+        }
+        return new Promise((resolve) => {
+            this.queue.push(resolve);
+        });
+    }
+    ensureTimer() {
+        if (this.timer)
+            return;
+        this.timer = setInterval(() => {
+            this.started = 0;
+            // Drain as many queued starters as the window allows
+            while (this.queue.length > 0 && this.started < this.startsPerWindow) {
+                this.started++;
+                this.queue.shift()();
+            }
+            // Stop the timer if nothing is waiting
+            if (this.queue.length === 0) {
+                clearInterval(this.timer);
+                this.timer = null;
+            }
+        }, this.windowMs);
+    }
+}
+/**
+ * Run an experiment - execute all evals with configured runs concurrently.
+ * With earlyExit enabled, remaining attempts for a fixture are aborted once one passes.
+ */
+export async function runExperiment(options) {
+    const { config, fixtures, apiKey, resultsDir, experimentName, fingerprints, onProgress, smoke, rateLimiter } = options;
+    const startedAt = new Date();
+    // Get the agent from registry
+    const agent = getAgent(config.agent);
+    const emit = (event) => {
+        if (onProgress) {
+            onProgress(event);
+        }
+    };
+    // Create AbortController per fixture for earlyExit
+    const abortControllers = new Map();
+    for (const fixture of fixtures) {
+        abortControllers.set(fixture.name, new AbortController());
+    }
+    // Build list of all attempts to run
+    const attempts = [];
+    for (const fixture of fixtures) {
+        for (let i = 0; i < config.runs; i++) {
+            attempts.push({ fixture, runIndex: i });
+        }
+    }
+    emit({ type: 'experiment:start', totalAttempts: attempts.length, totalEvals: fixtures.length, totalRuns: config.runs });
+    // Run a single attempt
+    const runAttempt = async (attempt) => {
+        const { fixture, runIndex } = attempt;
+        const controller = abortControllers.get(fixture.name);
+        // Check if already aborted before starting
+        if (controller.signal.aborted) {
+            return {
+                fixtureName: fixture.name,
+                runIndex,
+                runData: {
+                    result: { status: 'failed', error: 'Aborted', duration: 0 },
+                },
+                aborted: true,
+            };
+        }
+        const timeoutMs = config.timeout * 1000;
+        const startTime = Date.now();
+        // Create per-attempt controller for timeout cleanup
+        const attemptController = new AbortController();
+        // Propagate earlyExit abort to this attempt's controller
+        if (config.earlyExit) {
+            controller.signal.addEventListener('abort', () => attemptController.abort(), { once: true });
+        }
+        let timeoutId;
+        const agentResult = await Promise.race([
+            agent.run(fixture.path, {
+                prompt: config.editPrompt ? config.editPrompt(fixture.prompt) : fixture.prompt,
+                model: config.model,
+                timeout: timeoutMs,
+                apiKey,
+                setup: config.setup,
+                scripts: config.scripts,
+                signal: attemptController.signal,
+                sandbox: config.sandbox,
+            }),
+            new Promise((_, reject) => {
+                timeoutId = setTimeout(() => {
+                    attemptController.abort(); // Signal agent to clean up sandbox
+                    reject(new Error(`Eval timed out after ${config.timeout}s`));
+                }, timeoutMs);
+            }),
+        ]).catch((error) => {
+            // Convert timeout error to AgentResult format
+            if (error instanceof Error && error.message.includes('timed out')) {
+                return {
+                    success: false,
+                    output: '',
+                    error: error.message,
+                    duration: Date.now() - startTime,
+                };
+            }
+            throw error;
+        });
+        // Clear timeout if agent completed before timeout
+        if (timeoutId)
+            clearTimeout(timeoutId);
+        // Check if this was aborted
+        if (agentResult.error === 'Aborted' || agentResult.error === 'Aborted before start') {
+            return {
+                fixtureName: fixture.name,
+                runIndex,
+                runData: {
+                    result: { status: 'failed', error: 'Aborted', duration: agentResult.duration / 1000 },
+                },
+                aborted: true,
+            };
+        }
+        const runData = agentResultToEvalRunData(agentResult);
+        return {
+            fixtureName: fixture.name,
+            runIndex,
+            runData,
+        };
+    };
+    // Retry wrapper: if an attempt fails suspiciously fast (< 5s), it's likely an infra issue (429, etc.)
+    // Real evals take minutes, so a sub-5s failure is always anomalous.
+    const MAX_RETRIES = 5;
+    const INITIAL_BACKOFF_MS = 5_000;
+    const ANOMALY_THRESHOLD_S = 5;
+    const runAttemptWithRetry = async (attempt) => {
+        for (let retry = 0;; retry++) {
+            const result = await runAttempt(attempt);
+            const isSuspiciouslyFast = !result.aborted &&
+                result.runData.result.status === 'failed' &&
+                result.runData.result.duration < ANOMALY_THRESHOLD_S &&
+                !result.runData.result.error?.includes('timed out');
+            if (!isSuspiciouslyFast || retry >= MAX_RETRIES) {
+                return result;
+            }
+            const backoff = INITIAL_BACKOFF_MS * Math.pow(2, retry);
+            const jitter = Math.random() * backoff * 0.5;
+            await new Promise((resolve) => setTimeout(resolve, backoff + jitter));
+        }
+    };
+    // Run all attempts with rate-limited starts, 429 retry, and progress events
+    const runOne = async (attempt) => {
+        // Wait for rate limiter before starting (if provided)
+        if (rateLimiter) {
+            await rateLimiter.waitToStart();
+        }
+        emit({ type: 'eval:start', evalName: attempt.fixture.name, runNumber: attempt.runIndex + 1, totalRuns: config.runs });
+        const result = await runAttemptWithRetry(attempt);
+        if (!result.aborted) {
+            emit({ type: 'eval:complete', evalName: attempt.fixture.name, runNumber: attempt.runIndex + 1, totalRuns: config.runs, result: result.runData.result });
+            // If this attempt passed and earlyExit is enabled, abort remaining attempts
+            if (config.earlyExit && result.runData.result.status === 'passed') {
+                emit({ type: 'experiment:earlyExit', evalName: attempt.fixture.name, runNumber: attempt.runIndex + 1 });
+                abortControllers.get(attempt.fixture.name).abort();
+            }
+        }
+        return result;
+    };
+    const results = await Promise.all(attempts.map(runOne));
+    // Group results by fixture, excluding aborted results
+    const resultsByFixture = new Map();
+    for (const fixture of fixtures) {
+        resultsByFixture.set(fixture.name, []);
+    }
+    for (const result of results) {
+        if (!result.aborted) {
+            resultsByFixture.get(result.fixtureName).push(result);
+        }
+    }
+    // Build eval summaries, respecting earlyExit
+    const evalSummaries = [];
+    for (const fixture of fixtures) {
+        const fixtureResults = resultsByFixture.get(fixture.name);
+        // Sort by run index to process in order
+        fixtureResults.sort((a, b) => a.runIndex - b.runIndex);
+        const runDataList = [];
+        for (const result of fixtureResults) {
+            runDataList.push(result.runData);
+            // With earlyExit, stop counting after first pass
+            if (config.earlyExit && result.runData.result.status === 'passed') {
+                break;
+            }
+        }
+        const summary = createEvalSummary(fixture.name, runDataList);
+        evalSummaries.push(summary);
+    }
+    const completedAt = new Date();
+    const experimentResults = createExperimentResults(config, evalSummaries, startedAt, completedAt);
+    // Save results to disk
+    const fixturePaths = {};
+    for (const fixture of fixtures) {
+        fixturePaths[fixture.name] = fixture.path;
+    }
+    const outputDir = saveResults(experimentResults, {
+        resultsDir,
+        experimentName,
+        fingerprints,
+        smoke,
+        fixturePaths,
+    });
+    emit({ type: 'experiment:saved', outputDir });
+    emit({ type: 'experiment:summary', results: experimentResults });
+    return experimentResults;
+}
+/**
+ * Run a single eval (for testing/debugging).
+ */
+export async function runSingleEval(fixture, options) {
+    const agent = getAgent(options.agent ?? 'vercel-ai-gateway/claude-code');
+    const models = Array.isArray(options.model) ? options.model : [options.model];
+    const prompt = options.editPrompt ? options.editPrompt(fixture.prompt) : fixture.prompt;
+    const results = [];
+    for (const model of models) {
+        const agentResult = await agent.run(fixture.path, {
+            prompt,
+            model,
+            timeout: options.timeout * 1000,
+            apiKey: options.apiKey,
+            setup: options.setup,
+            scripts: options.scripts,
+            sandbox: options.sandbox,
+        });
+        results.push(agentResultToEvalRunData(agentResult));
+    }
+    // TODO: remove this on the next major and return an array directly...it's just here to prevent breaking changes
+    if (!Array.isArray(options.model)) {
+        return results[0];
+    }
+    return results;
+}
+//# sourceMappingURL=runner.js.map

package/dist/lib/runner.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"runner.js","sourceRoot":"","sources":["../../src/lib/runner.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAWH,OAAO,EAAE,QAAQ,EAAE,MAAM,mBAAmB,CAAC;AAC7C,OAAO,EACL,wBAAwB,EACxB,iBAAiB,EACjB,uBAAuB,EACvB,WAAW,GACZ,MAAM,cAAc,CAAC;AAEtB;;;;GAIG;AACH,MAAM,OAAO,gBAAgB;IAMR;IACA;IANX,KAAK,GAAmB,EAAE,CAAC;IAC3B,OAAO,GAAG,CAAC,CAAC;IACZ,KAAK,GAA0C,IAAI,CAAC;IAE5D,YACmB,eAAuB,EACvB,QAAgB;QADhB,oBAAe,GAAf,eAAe,CAAQ;QACvB,aAAQ,GAAR,QAAQ,CAAQ;IAChC,CAAC;IAEJ;;;OAGG;IACH,KAAK,CAAC,WAAW;QACf,IAAI,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,eAAe,EAAE,CAAC;YACxC,IAAI,CAAC,OAAO,EAAE,CAAC;YACf,IAAI,CAAC,WAAW,EAAE,CAAC;YACnB,OAAO;QACT,CAAC;QACD,OAAO,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;YACnC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAC3B,CAAC,CAAC,CAAC;IACL,CAAC;IAEO,WAAW;QACjB,IAAI,IAAI,CAAC,KAAK;YAAE,OAAO;QACvB,IAAI,CAAC,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE;YAC5B,IAAI,CAAC,OAAO,GAAG,CAAC,CAAC;YACjB,qDAAqD;YACrD,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,eAAe,EAAE,CAAC;gBACpE,IAAI,CAAC,OAAO,EAAE,CAAC;gBACf,IAAI,CAAC,KAAK,CAAC,KAAK,EAAG,EAAE,CAAC;YACxB,CAAC;YACD,uCAAuC;YACvC,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC5B,aAAa,CAAC,IAAI,CAAC,KAAM,CAAC,CAAC;gBAC3B,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;YACpB,CAAC;QACH,CAAC,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;IACpB,CAAC;CACF;AA8CD;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,OAA6B;IAE7B,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,UAAU,EAAE,cAAc,EAAE,YAAY,EAAE,UAAU,EAAE,KAAK,EAAE,WAAW,EAAE,GAAG,OAAO,CAAC;IACvH,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC;IAE7B,8BAA8B;IAC9B,MAAM,KAAK,GAAG,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAErC,MAAM,IAAI,GAAG,CAAC,KAAoB,EAAE,EAAE;QACpC,IAAI,UAAU,EAAE,CAAC;YACf,UAAU,CAAC,KAAK,CAAC,CAAC;QACpB,CAAC;IACH,CAAC,CAAC;IAEF,mDAAmD;IACnD,MAAM,gBAAgB,GAAG,IAAI,GAAG,EAA2B,CAAC;IAC5D,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,gBAAgB,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,eAAe,EAAE,CAAC,CAAC;IAC5D,CAAC;IAED,oCAAoC;IACpC,MAAM,QAAQ,GAAkB,EAAE,CAAC;IACnC,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC;YACrC,QAAQ,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC;QAC1C,CAAC;IACH,CAAC;IAED,IAAI,CAAC,EAAE,IAAI,EAAE,kBAAkB,EAAE,aAAa,EAAE,QAAQ,CAAC,MAAM,EAAE,UAAU,EAAE,QAAQ,CAAC,MAAM,EAAE,SAAS,EAAE,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;IAExH,uBAAuB;IACvB,MAAM,UAAU,GAAG,KAAK,EAAE,OAAoB,EAA0B,EAAE;QACxE,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC;QACtC,MAAM,UAAU,GAAG,gBAAgB,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,CAAE,CAAC;QAEvD,2CAA2C;QAC3C,IAAI,UAAU,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YAC9B,OAAO;gBACL,WAAW,EAAE,OAAO,CAAC,IAAI;gBACzB,QAAQ;gBACR,OAAO,EAAE;oBACP,MAAM,EAAE,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC,EAAE;iBAC5D;gBACD,OAAO,EAAE,IAAI;aACd,CAAC;QACJ,CAAC;QAED,MAAM,SAAS,GAAG,MAAM,CAAC,OAAO,GAAG,IAAI,CAAC;QACxC,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAE7B,oDAAoD;QACpD,MAAM,iBAAiB,GAAG,IAAI,eAAe,EAAE,CAAC;QAEhD,yDAAyD;QACzD,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;YACrB,UAAU,CAAC,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,iBAAiB,CAAC,KAAK,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;QAC/F,CAAC;QAED,IAAI,SAAoD,CAAC;QAEzD,MAAM,WAAW,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC;YACrC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE;gBACtB,MAAM,EAAE,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM;gBAC9E,KAAK,EAAE,MAAM,CAAC,KAAK;gBACnB,OAAO,EAAE,SAAS;gBAClB,MAAM;gBACN,KAAK,EAAE,MAAM,CAAC,KAAK;gBACnB,OAAO,EAAE,MAAM,CAAC,OAAO;gBACvB,MAAM,EAAE,iBAAiB,CAAC,MAAM;gBAChC,OAAO,EAAE,MAAM,CAAC,OAAO;aACxB,CAAC;YACF,IAAI,OAAO,CAAQ,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE;gBAC/B,SAAS,GAAG,UAAU,CAAC,GAAG,EAAE;oBAC1B,iBAAiB,CAAC,KAAK,EAAE,CAAC,CAAC,mCAAmC;oBAC9D,MAAM,CAAC,IAAI,KAAK,CAAC,wBAAwB,MAAM,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC;gBAC/D,CAAC,EAAE,SAAS,CAAC,CAAC;YAChB,CAAC,CAAC;SACH,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;YACjB,8CAA8C;YAC9C,IAAI,KAAK,YAAY,KAAK,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;gBAClE,OAAO;oBACL,OAAO,EAAE,KAAK;oBACd,MAAM,EAAE,EAAE;oBACV,KAAK,EAAE,KAAK,CAAC,OAAO;oBACpB,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;iBACjC,CAAC;YACJ,CAAC;YACD,MAAM,KAAK,CAAC;QACd,CAAC,CAAC,CAAC;QAEH,kDAAkD;QAClD,IAAI,SAAS;YAAE,YAAY,CAAC,SAAS,CAAC,CAAC;QAEvC,4BAA4B;QAC5B,IAAI,WAAW,CAAC,KAAK,KAAK,SAAS,IAAI,WAAW,CAAC,KAAK,KAAK,sBAAsB,EAAE,CAAC;YACpF,OAAO;gBACL,WAAW,EAAE,OAAO,CAAC,IAAI;gBACzB,QAAQ;gBACR,OAAO,EAAE;oBACP,MAAM,EAAE,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,SAAS,EAAE,QAAQ,EAAE,WAAW,CAAC,QAAQ,GAAG,IAAI,EAAE;iBACtF;gBACD,OAAO,EAAE,IAAI;aACd,CAAC;QACJ,CAAC;QAED,MAAM,OAAO,GAAG,wBAAwB,CAAC,WAAW,CAAC,CAAC;QAEtD,OAAO;YACL,WAAW,EAAE,OAAO,CAAC,IAAI;YACzB,QAAQ;YACR,OAAO;SACR,CAAC;IACJ,CAAC,CAAC;IAEF,sGAAsG;IACtG,oEAAoE;IACpE,MAAM,WAAW,GAAG,CAAC,CAAC;IACtB,MAAM,kBAAkB,GAAG,KAAK,CAAC;IACjC,MAAM,mBAAmB,GAAG,CAAC,CAAC;IAE9B,MAAM,mBAAmB,GAAG,KAAK,EAAE,OAAoB,EAA0B,EAAE;QACjF,KAAK,IAAI,KAAK,GAAG,CAAC,GAAI,KAAK,EAAE,EAAE,CAAC;YAC9B,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,OAAO,CAAC,CAAC;YAEzC,MAAM,kBAAkB,GACtB,CAAC,MAAM,CAAC,OAAO;gBACf,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,KAAK,QAAQ;gBACzC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,QAAQ,GAAG,mBAAmB;gBACpD,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,EAAE,QAAQ,CAAC,WAAW,CAAC,CAAC;YAEtD,IAAI,CAAC,kBAAkB,IAAI,KAAK,IAAI,WAAW,EAAE,CAAC;gBAChD,OAAO,MAAM,CAAC;YAChB,CAAC;YAED,MAAM,OAAO,GAAG,kBAAkB,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;YACxD,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,GAAG,OAAO,GAAG,GAAG,CAAC;YAC7C,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,OAAO,GAAG,MAAM,CAAC,CAAC,CAAC;QACxE,CAAC;IACH,CAAC,CAAC;IAEF,4EAA4E;IAC5E,MAAM,MAAM,GAAG,KAAK,EAAE,OAAoB,EAA0B,EAAE;QACpE,sDAAsD;QACtD,IAAI,WAAW,EAAE,CAAC;YAChB,MAAM,WAAW,CAAC,WAAW,EAAE,CAAC;QAClC,CAAC;QAED,IAAI,CAAC,EAAE,IAAI,EAAE,YAAY,EAAE,QAAQ,EAAE,OAAO,CAAC,OAAO,CAAC,IAAI,EAAE,SAAS,EAAE,OAAO,CAAC,QAAQ,GAAG,CAAC,EAAE,SAAS,EAAE,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;QAEtH,MAAM,MAAM,GAAG,MAAM,mBAAmB,CAAC,OAAO,CAAC,CAAC;QAElD,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YACpB,IAAI,CAAC,EAAE,IAAI,EAAE,eAAe,EAAE,QAAQ,EAAE,OAAO,CAAC,OAAO,CAAC,IAAI,EAAE,SAAS,EAAE,OAAO,CAAC,QAAQ,GAAG,CAAC,EAAE,SAAS,EAAE,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;YAExJ,4EAA4E;YAC5E,IAAI,MAAM,CAAC,SAAS,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;gBAClE,IAAI,CAAC,EAAE,IAAI,EAAE,sBAAsB,EAAE,QAAQ,EAAE,OAAO,CAAC,OAAO,CAAC,IAAI,EAAE,SAAS,EAAE,OAAO,CAAC,QAAQ,GAAG,CAAC,EAAE,CAAC,CAAC;gBACxG,gBAAgB,CAAC,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAE,CAAC,KAAK,EAAE,CAAC;YACtD,CAAC;QACH,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC,CAAC;IAEF,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC;IAExD,sDAAsD;IACtD,MAAM,gBAAgB,GAAG,IAAI,GAAG,EAA2B,CAAC;IAC5D,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,gBAAgB,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;IACzC,CAAC;IAED,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YACpB,gBAAgB,CAAC,GAAG,CAAC,MAAM,CAAC,WAAW,CAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACzD,CAAC;IACH,CAAC;IAED,6CAA6C;IAC7C,MAAM,aAAa,GAAkB,EAAE,CAAC;IACxC,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,MAAM,cAAc,GAAG,gBAAgB,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,CAAE,CAAC;QAE3D,wCAAwC;QACxC,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC;QAEvD,MAAM,WAAW,GAAkB,EAAE,CAAC;QACtC,KAAK,MAAM,MAAM,IAAI,cAAc,EAAE,CAAC;YACpC,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YAEjC,iDAAiD;YACjD,IAAI,MAAM,CAAC,SAAS,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;gBAClE,MAAM;YACR,CAAC;QACH,CAAC;QAED,MAAM,OAAO,GAAG,iBAAiB,CAAC,OAAO,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC;QAC7D,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAC9B,CAAC;IAED,MAAM,WAAW,GAAG,IAAI,IAAI,EAAE,CAAC;IAC/B,MAAM,iBAAiB,GAAG,uBAAuB,CAAC,MAAM,EAAE,aAAa,EAAE,SAAS,EAAE,WAAW,CAAC,CAAC;IAEjG,uBAAuB;IACvB,MAAM,YAAY,GAA2B,EAAE,CAAC;IAChD,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,YAAY,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IAC5C,CAAC;IACD,MAAM,SAAS,GAAG,WAAW,CAAC,iBAAiB,EAAE;QAC/C,UAAU;QACV,cAAc;QACd,YAAY;QACZ,KAAK;QACL,YAAY;KACb,CAAC,CAAC;IAEH,IAAI,CAAC,EAAE,IAAI,EAAE,kBAAkB,EAAE,SAAS,EAAE,CAAC,CAAC;IAC9C,IAAI,CAAC,EAAE,IAAI,EAAE,oBAAoB,EAAE,OAAO,EAAE,iBAAiB,EAAE,CAAC,CAAC;IAEjE,OAAO,iBAAiB,CAAC;AAC3B,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,OAAoB,EACpB,OAUC;IAED,MAAM,KAAK,GAAG,QAAQ,CAAC,OAAO,CAAC,KAAK,IAAI,+BAA+B,CAAC,CAAC;IAEzE,MAAM,MAAM,GAAa,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IACxF,MAAM,MAAM,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC;IAExF,MAAM,OAAO,GAAkB,EAAE,CAAC;IAElC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAE9B,MAAM,WAAW,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE;YACjD,MAAM;YACN,KAAK;YACL,OAAO,EAAE,OAAO,CAAC,OAAO,GAAG,IAAI;YAC/B,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,OAAO,EAAE,OAAO,CAAC,OAAO;YACxB,OAAO,EAAE,OAAO,CAAC,OAAO;SACxB,CAAC,CAAC;QAEA,OAAO,CAAC,IAAI,CAAC,wBAAwB,CAAC,WAAW,CAAC,CAAC,CAAC;IACtD,CAAC;IAED,gHAAgH;IAChH,IAAG,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACpC,OAAO,OAAO,CAAC,CAAC,CAA2D,CAAC;IAC3E,CAAC;IAED,OAAO,OAAiE,CAAC;AAC3E,CAAC"}

package/dist/lib/sandbox.d.ts

@@ -0,0 +1,173 @@
+/**
+ * Sandbox integration for isolated eval execution.
+ * Supports both Vercel Sandbox and Docker backends.
+ */
+import { Sandbox as VercelSandbox } from '@vercel/sandbox';
+import type { Sandbox } from './types.js';
+import { DockerSandboxManager } from './docker-sandbox.js';
+/**
+ * Default timeout for sandbox operations (10 minutes).
+ */
+export declare const DEFAULT_SANDBOX_TIMEOUT = 600000;
+/**
+ * Supported sandbox backends.
+ */
+export type SandboxBackend = 'vercel' | 'docker';
+/**
+ * Information about the resolved sandbox backend.
+ */
+export interface SandboxBackendInfo {
+    /** Which backend will be used */
+    backend: SandboxBackend;
+    /** How it was determined */
+    reason: 'explicit' | 'auto-detected';
+    /** Human-readable description */
+    description: string;
+}
+/**
+ * Files to ignore when copying to sandbox.
+ * These are build artifacts and dependencies that shouldn't be uploaded.
+ * Note: This is a general-purpose pattern list used by collectLocalFiles().
+ * For eval-specific exclusions (PROMPT.md, EVAL.ts), see TEST_FILE_PATTERNS.
+ */
+export declare const IGNORED_PATTERNS: string[];
+/**
+ * Test/eval file patterns to withhold from agent during task execution.
+ * These files are uploaded AFTER the agent completes for validation.
+ * - PROMPT.md: Contains the task - agent receives this via CLI argument, not as a file
+ * - EVAL.ts/tsx: Validation tests - must be hidden so agent can't "cheat"
+ */
+export declare const TEST_FILE_PATTERNS: string[];
+/**
+ * Options for creating a sandbox.
+ */
+export interface SandboxOptions {
+    /** Timeout in milliseconds */
+    timeout?: number;
+    /** Runtime environment */
+    runtime?: 'node20' | 'node24';
+    /** Sandbox backend to use. 'auto' will use Vercel if token present, else Docker. @default 'auto' */
+    backend?: SandboxBackend | 'auto';
+    /** Optional explicit Vercel auth token for sandbox API auth */
+    token?: string;
+    /** Optional explicit Vercel team ID for sandbox API auth */
+    teamId?: string;
+    /** Optional explicit Vercel project ID for sandbox API auth */
+    projectId?: string;
+}
+/**
+ * Result of running a command in the sandbox.
+ */
+export interface CommandResult {
+    stdout: string;
+    stderr: string;
+    exitCode: number;
+}
+/**
+ * File to upload to sandbox.
+ */
+export interface SandboxFile {
+    path: string;
+    content: Buffer | string;
+}
+/**
+ * Wrapper around Vercel Sandbox providing a cleaner API.
+ */
+export declare class SandboxManager implements Sandbox {
+    private sandbox;
+    private _workingDirectory;
+    constructor(sandbox: VercelSandbox);
+    /**
+     * Create a new sandbox instance.
+     */
+    static create(options?: SandboxOptions): Promise<SandboxManager>;
+    /**
+     * Get the sandbox ID.
+     */
+    get sandboxId(): string;
+    /**
+     * Run a command in the sandbox.
+     */
+    runCommand(command: string, args?: string[], options?: {
+        env?: Record<string, string>;
+    }): Promise<CommandResult>;
+    /**
+     * Run a shell command (through bash).
+     */
+    runShell(command: string, env?: Record<string, string>): Promise<CommandResult>;
+    /**
+     * Read a file from the sandbox.
+     */
+    readFile(path: string): Promise<string>;
+    /**
+     * Check if a file exists in the sandbox.
+     */
+    fileExists(path: string): Promise<boolean>;
+    /**
+     * Write files to the sandbox.
+     */
+    writeFiles(files: Record<string, string>): Promise<void>;
+    /**
+     * Upload files from local filesystem to sandbox.
+     */
+    uploadFiles(files: SandboxFile[]): Promise<void>;
+    /**
+     * Get the working directory.
+     */
+    getWorkingDirectory(): string;
+    /**
+     * Stop and clean up the sandbox.
+     */
+    stop(): Promise<void>;
+}
+/**
+ * Resolve which sandbox backend to use based on options.
+ *
+ * Priority:
+ * 1. Explicit backend in options (if not 'auto')
+ * 2. Auto-detect: Vercel if token present, else Docker
+ */
+export declare function resolveBackend(options?: SandboxOptions): SandboxBackend;
+/**
+ * Get information about the sandbox backend that will be used.
+ * Useful for displaying to users.
+ */
+export declare function getSandboxBackendInfo(options?: SandboxOptions): SandboxBackendInfo;
+/**
+ * Create a sandbox using the appropriate backend.
+ *
+ * By default, uses Vercel Sandbox if VERCEL_TOKEN is present,
+ * otherwise falls back to Docker.
+ *
+ * @example
+ * ```typescript
+ * // Auto-detect backend
+ * const sandbox = await createSandbox();
+ *
+ * // Explicit Docker
+ * const sandbox = await createSandbox({ backend: 'docker' });
+ *
+ * // Explicit Vercel
+ * const sandbox = await createSandbox({ backend: 'vercel' });
+ * ```
+ */
+export declare function createSandbox(options?: SandboxOptions): Promise<SandboxManager | DockerSandboxManager>;
+/**
+ * Collect files from a local directory for uploading to sandbox.
+ */
+export declare function collectLocalFiles(dir: string, options?: {
+    excludePatterns?: string[];
+    includePatterns?: string[];
+}): Promise<SandboxFile[]>;
+/**
+ * Split files into workspace files (visible to agent) and test files (hidden until validation).
+ */
+export declare function splitTestFiles(files: SandboxFile[]): {
+    workspaceFiles: SandboxFile[];
+    testFiles: SandboxFile[];
+};
+/**
+ * Verify that no test files exist in the sandbox.
+ */
+export declare function verifyNoTestFiles(sandbox: SandboxManager | DockerSandboxManager): Promise<void>;
+//# sourceMappingURL=sandbox.d.ts.map

package/dist/lib/sandbox.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"sandbox.d.ts","sourceRoot":"","sources":["../../src/lib/sandbox.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,OAAO,IAAI,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAC3D,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAG1C,OAAO,EAAE,oBAAoB,EAAE,MAAM,qBAAqB,CAAC;AAE3D;;GAEG;AACH,eAAO,MAAM,uBAAuB,SAAS,CAAC;AAE9C;;GAEG;AACH,MAAM,MAAM,cAAc,GAAG,QAAQ,GAAG,QAAQ,CAAC;AAEjD;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC,iCAAiC;IACjC,OAAO,EAAE,cAAc,CAAC;IACxB,4BAA4B;IAC5B,MAAM,EAAE,UAAU,GAAG,eAAe,CAAC;IACrC,iCAAiC;IACjC,WAAW,EAAE,MAAM,CAAC;CACrB;AAED;;;;;GAKG;AACH,eAAO,MAAM,gBAAgB,UAU5B,CAAC;AAEF;;;;;GAKG;AACH,eAAO,MAAM,kBAAkB,UAAuC,CAAC;AAEvE;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,8BAA8B;IAC9B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,0BAA0B;IAC1B,OAAO,CAAC,EAAE,QAAQ,GAAG,QAAQ,CAAC;IAC9B,oGAAoG;IACpG,OAAO,CAAC,EAAE,cAAc,GAAG,MAAM,CAAC;IAClC,+DAA+D;IAC/D,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,4DAA4D;IAC5D,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,+DAA+D;IAC/D,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,GAAG,MAAM,CAAC;CAC1B;AAED;;GAEG;AACH,qBAAa,cAAe,YAAW,OAAO;IAC5C,OAAO,CAAC,OAAO,CAAgB;IAC/B,OAAO,CAAC,iBAAiB,CAA6B;gBAE1C,OAAO,EAAE,aAAa;IAIlC;;OAEG;WACU,MAAM,CAAC,OAAO,GAAE,cAAmB,GAAG,OAAO,CAAC,cAAc,CAAC;IAa1E;;OAEG;IACH,IAAI,SAAS,IAAI,MAAM,CAEtB;IAED;;OAEG;IACG,UAAU,CACd,OAAO,EAAE,MAAM,EACf,IAAI,GAAE,MAAM,EAAO,EACnB,OAAO,GAAE;QAAE,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;KAAO,GAC7C,OAAO,CAAC,aAAa,CAAC;IAczB;;OAEG;IACG,QAAQ,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,OAAO,CAAC,aAAa,CAAC;IAcrF;;OAEG;IACG,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAQ7C;;OAEG;IACG,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAKhD;;OAEG;IACG,UAAU,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IAa9D;;OAEG;IACG,WAAW,CAAC,KAAK,EAAE,WAAW,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAStD;;OAEG;IACH,mBAAmB,IAAI,MAAM;IAI7B;;OAEG;IACG,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;CAG5B;AAkBD;;;;;;GAMG;AACH,wBAAgB,cAAc,CAAC,OAAO,CAAC,EAAE,cAAc,GAAG,cAAc,CAYvE;AAED;;;GAGG;AACH,wBAAgB,qBAAqB,CAAC,OAAO,CAAC,EAAE,cAAc,GAAG,kBAAkB,CAsBlF;AAED;;;;;;;;;;;;;;;;;GAiBG;AACH,wBAAsB,aAAa,CACjC,OAAO,GAAE,cAAmB,GAC3B,OAAO,CAAC,cAAc,GAAG,oBAAoB,CAAC,CAchD;AAED;;GAEG;AACH,wBAAsB,iBAAiB,CACrC,GAAG,EAAE,MAAM,EACX,OAAO,GAAE;IACP,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAC3B,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;CACvB,GACL,OAAO,CAAC,WAAW,EAAE,CAAC,CA+DxB;AAmBD;;GAEG;AACH,wBAAgB,cAAc,CAAC,KAAK,EAAE,WAAW,EAAE,GAAG;IACpD,cAAc,EAAE,WAAW,EAAE,CAAC;IAC9B,SAAS,EAAE,WAAW,EAAE,CAAC;CAC1B,CAeA;AAED;;GAEG;AACH,wBAAsB,iBAAiB,CACrC,OAAO,EAAE,cAAc,GAAG,oBAAoB,GAC7C,OAAO,CAAC,IAAI,CAAC,CASf"}