@vercel/agent-eval 0.0.1

package/README.md

@@ -0,0 +1,370 @@
+# @vercel/agent-eval
+
+Test AI coding agents on your framework. Measure what actually works.
+
+## Why?
+
+You're building a frontend framework and want AI agents to work well with it. But how do you know if:
+- Your documentation helps agents write correct code?
+- Adding an MCP server improves agent success rates?
+- Sonnet performs as well as Opus for your use cases?
+- Your latest API changes broke agent compatibility?
+
+**This framework gives you answers.** Run controlled experiments, measure pass rates, compare techniques.
+
+## Quick Start
+
+```bash
+# Create a new eval project
+npx @vercel/agent-eval init my-framework-evals
+cd my-framework-evals
+
+# Install dependencies
+npm install
+
+# Add your API keys
+cp .env.example .env
+# Edit .env with your AI_GATEWAY_API_KEY and VERCEL_TOKEN
+
+# Preview what will run (no API calls, no cost)
+npx @vercel/agent-eval cc --dry
+
+# Run the evals
+npx @vercel/agent-eval cc
+```
+
+## A/B Testing AI Techniques
+
+The real power is comparing different approaches. Create multiple experiment configs:
+
+### Control: Baseline Agent
+
+```typescript
+// experiments/control.ts
+import type { ExperimentConfig } from 'agent-eval';
+
+const config: ExperimentConfig = {
+  agent: 'vercel-ai-gateway/claude-code',
+  model: 'opus',
+  runs: 10,        // Multiple runs for statistical significance
+  earlyExit: false, // Run all attempts to measure reliability
+};
+
+export default config;
+```
+
+### Treatment: Agent with MCP Server
+
+```typescript
+// experiments/with-mcp.ts
+import type { ExperimentConfig } from 'agent-eval';
+
+const config: ExperimentConfig = {
+  agent: 'vercel-ai-gateway/claude-code',
+  model: 'opus',
+  runs: 10,
+  earlyExit: false,
+
+  setup: async (sandbox) => {
+    // Install your framework's MCP server
+    await sandbox.runCommand('npm', ['install', '-g', '@myframework/mcp-server']);
+
+    // Configure Claude to use it
+    await sandbox.writeFiles({
+      '.claude/settings.json': JSON.stringify({
+        mcpServers: {
+          myframework: { command: 'myframework-mcp' }
+        }
+      })
+    });
+  },
+};
+
+export default config;
+```
+
+### Run Both & Compare
+
+```bash
+# Preview first
+npx @vercel/agent-eval control --dry
+npx @vercel/agent-eval with-mcp --dry
+
+# Run experiments
+npx @vercel/agent-eval control
+npx @vercel/agent-eval with-mcp
+```
+
+**Compare results:**
+```
+Control (baseline):     7/10 passed (70%)
+With MCP:              9/10 passed (90%)
+```
+
+## Creating Evals for Your Framework
+
+Each eval tests one specific task an agent should be able to do with your framework.
+
+### Example: Testing Component Creation
+
+```
+evals/
+  create-button-component/
+    PROMPT.md           # Task for the agent
+    EVAL.ts             # Tests to verify success
+    package.json        # Your framework as a dependency
+    src/                # Starter code
+```
+
+**PROMPT.md** - What you want the agent to do:
+```markdown
+Create a Button component using MyFramework.
+
+Requirements:
+- Export a Button component from src/components/Button.tsx
+- Accept `label` and `onClick` props
+- Use the framework's styling system for hover states
+```
+
+**EVAL.ts** - How you verify it worked:
+```typescript
+import { test, expect } from 'vitest';
+import { readFileSync, existsSync } from 'fs';
+import { execSync } from 'child_process';
+
+test('Button component exists', () => {
+  expect(existsSync('src/components/Button.tsx')).toBe(true);
+});
+
+test('has required props', () => {
+  const content = readFileSync('src/components/Button.tsx', 'utf-8');
+  expect(content).toContain('label');
+  expect(content).toContain('onClick');
+});
+
+test('project builds', () => {
+  execSync('npm run build', { stdio: 'pipe' });
+});
+```
+
+**package.json** - Include your framework:
+```json
+{
+  "name": "create-button-component",
+  "type": "module",
+  "scripts": { "build": "tsc" },
+  "dependencies": {
+    "myframework": "^2.0.0"
+  }
+}
+```
+
+## Experiment Ideas
+
+| Experiment | Control | Treatment |
+|------------|---------|-----------|
+| MCP impact | No MCP | With MCP server |
+| Model comparison | Haiku | Sonnet / Opus |
+| Documentation | Minimal docs | Rich examples |
+| System prompt | Default | Framework-specific |
+| Tool availability | Read/write only | + custom tools |
+
+## Configuration Reference
+
+### Agent Selection
+
+Choose your agent and authentication method:
+
+```typescript
+// Vercel AI Gateway (recommended - unified billing & observability)
+agent: 'vercel-ai-gateway/claude-code'  // or 'vercel-ai-gateway/codex'
+
+// Direct API (uses provider keys directly)
+agent: 'claude-code'  // requires ANTHROPIC_API_KEY
+agent: 'codex'        // requires OPENAI_API_KEY
+```
+
+See the Environment Variables section below for setup instructions.
+
+### Full Configuration
+
+```typescript
+import type { ExperimentConfig } from 'agent-eval';
+
+const config: ExperimentConfig = {
+  // Required: which agent and authentication to use
+  agent: 'vercel-ai-gateway/claude-code',
+
+  // Model to use (defaults: 'opus' for claude-code, 'openai/gpt-5.2-codex' for codex)
+  model: 'opus',
+
+  // How many times to run each eval
+  runs: 10,
+
+  // Stop after first success? (false for reliability measurement)
+  earlyExit: false,
+
+  // npm scripts that must pass after agent finishes
+  scripts: ['build', 'lint'],
+
+  // Timeout per run in seconds
+  timeout: 300,
+
+  // Filter which evals to run (pick one)
+  evals: '*',                                // all (default)
+  // evals: ['specific-eval'],              // by name
+  // evals: (name) => name.startsWith('api-'), // by function
+
+  // Setup function for environment configuration
+  setup: async (sandbox) => {
+    await sandbox.writeFiles({ '.env': 'API_KEY=test' });
+    await sandbox.runCommand('npm', ['run', 'setup']);
+  },
+};
+
+export default config;
+```
+
+## CLI Commands
+
+### `init <name>`
+
+Create a new eval project:
+```bash
+npx @vercel/agent-eval init my-evals
+```
+
+### `<experiment>`
+
+Run an experiment:
+```bash
+npx @vercel/agent-eval cc
+```
+
+**Dry run** - preview without executing (no API calls, no cost):
+```bash
+npx @vercel/agent-eval cc --dry
+
+# Output:
+# Found 5 valid fixture(s), will run 5:
+#   - create-button
+#   - add-routing
+#   - setup-state
+#   - ...
+# Running 5 eval(s) x 10 run(s) = 50 total runs
+# Agent: claude-code, Model: opus, Timeout: 300s
+# [DRY RUN] Would execute evals here
+```
+
+## Results
+
+Results are saved to `results/<experiment>/<timestamp>/`:
+
+```
+results/
+  with-mcp/
+    2026-01-27T10-30-00Z/
+      experiment.json       # Config and summary
+      create-button/
+        summary.json        # { totalRuns: 10, passedRuns: 9, passRate: "90%" }
+        run-1/
+          result.json       # Individual run result
+          transcript.jsonl  # Agent conversation
+          outputs/          # Test/script output
+```
+
+### Analyzing Results
+
+```bash
+# Quick comparison
+cat results/control/*/experiment.json | jq '.evals[] | {name, passRate}'
+cat results/with-mcp/*/experiment.json | jq '.evals[] | {name, passRate}'
+```
+
+| Pass Rate | Interpretation |
+|-----------|----------------|
+| 90-100%   | Agent handles this reliably |
+| 70-89%    | Usually works, room for improvement |
+| 50-69%    | Unreliable, needs investigation |
+| < 50%     | Task too hard or prompt needs work |
+
+## Environment Variables
+
+Every run requires **two things**: an API key for the agent and a token for the Vercel sandbox. The exact variables depend on which authentication mode you use.
+
+| Variable | Required when | Description |
+|---|---|---|
+| `AI_GATEWAY_API_KEY` | `agent: 'vercel-ai-gateway/...'` | Vercel AI Gateway key — works for all agents |
+| `ANTHROPIC_API_KEY` | `agent: 'claude-code'` | Direct Anthropic API key (`sk-ant-...`) |
+| `OPENAI_API_KEY` | `agent: 'codex'` | Direct OpenAI API key (`sk-proj-...`) |
+| `VERCEL_TOKEN` | Always (pick one) | Vercel personal access token — for local dev |
+| `VERCEL_OIDC_TOKEN` | Always (pick one) | Vercel OIDC token — for CI/CD pipelines |
+
+> You always need **one agent key** + **one sandbox token**.
+
+### Vercel AI Gateway (Recommended)
+
+Use `vercel-ai-gateway/` prefixed agents. One key for all models.
+
+```bash
+# Agent access — get yours at https://vercel.com/dashboard -> AI Gateway
+AI_GATEWAY_API_KEY=your-ai-gateway-api-key
+
+# Sandbox access — create at https://vercel.com/account/tokens
+VERCEL_TOKEN=your-vercel-token
+# OR for CI/CD:
+# VERCEL_OIDC_TOKEN=your-oidc-token
+```
+
+### Direct API Keys (Alternative)
+
+Remove the `vercel-ai-gateway/` prefix and use provider keys directly:
+
+```bash
+# For agent: 'claude-code'
+ANTHROPIC_API_KEY=sk-ant-...
+
+# For agent: 'codex'
+OPENAI_API_KEY=sk-proj-...
+
+# Sandbox access is still required
+VERCEL_TOKEN=your-vercel-token
+```
+
+### `.env` Setup
+
+The `init` command generates a `.env.example` file. Copy it and fill in your keys:
+
+```bash
+cp .env.example .env
+```
+
+The framework loads `.env` automatically via [dotenv](https://github.com/motdotla/dotenv).
+
+### Vercel Employees
+
+**To get the environment variables, link to `vercel-labs/agent-eval` on Vercel:**
+
+```bash
+# Link to the vercel-labs/agent-eval project
+vc link vercel-labs/agent-eval
+
+# Pull environment variables
+vc env pull
+```
+
+This writes a `.env.local` file with all the required environment variables (AI_GATEWAY_API_KEY, ANTHROPIC_API_KEY, OPENAI_API_KEY, VERCEL_OIDC_TOKEN) — no manual key setup needed. The framework automatically loads from both `.env` and `.env.local`.
+
+## Tips
+
+**Start with `--dry`**: Always preview before running to verify your config and avoid unexpected costs.
+
+**Use multiple runs**: Single runs don't tell you reliability. Use `runs: 10` and `earlyExit: false` for meaningful data.
+
+**Isolate variables**: Change one thing at a time between experiments. Don't compare "Opus with MCP" to "Haiku without MCP".
+
+**Test incrementally**: Start with simple tasks, add complexity as you learn what works.
+
+## License
+
+MIT

package/dist/cli.d.ts

@@ -0,0 +1,6 @@
+#!/usr/bin/env node
+/**
+ * CLI entry point for the eval framework.
+ */
+export {};
+//# sourceMappingURL=cli.d.ts.map

package/dist/cli.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cli.d.ts","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":";AAEA;;GAEG"}

package/dist/cli.js

@@ -0,0 +1,166 @@
+#!/usr/bin/env node
+/**
+ * CLI entry point for the eval framework.
+ */
+import { Command } from 'commander';
+import { config as dotenvConfig } from 'dotenv';
+import { resolve, dirname, basename } from 'path';
+import { existsSync } from 'fs';
+import chalk from 'chalk';
+import { loadConfig, resolveEvalNames } from './lib/config.js';
+import { loadAllFixtures } from './lib/fixture.js';
+import { runExperiment } from './lib/runner.js';
+import { initProject, getPostInitInstructions } from './lib/init.js';
+import { getAgent } from './lib/agents/index.js';
+// Load environment variables
+dotenvConfig();
+const program = new Command();
+program
+    .name('agent-eval')
+    .description('Framework for testing AI coding agents in isolated sandboxes')
+    .version('0.0.1');
+/**
+ * Resolve config path shorthand.
+ * - "cc" -> "experiments/cc.ts"
+ * - "experiments/cc.ts" -> "experiments/cc.ts" (unchanged)
+ */
+function resolveConfigPath(input) {
+    // If it already has a path separator or extension, use as-is
+    if (input.includes('/') || input.includes('\\') || input.endsWith('.ts') || input.endsWith('.js')) {
+        return input;
+    }
+    // Otherwise, treat as shorthand: "cc" -> "experiments/cc.ts"
+    return `experiments/${input}.ts`;
+}
+/**
+ * Run experiment command handler
+ */
+async function runExperimentCommand(configInput, options) {
+    try {
+        const configPath = resolveConfigPath(configInput);
+        const absoluteConfigPath = resolve(process.cwd(), configPath);
+        if (!existsSync(absoluteConfigPath)) {
+            console.error(chalk.red(`Config file not found: ${absoluteConfigPath}`));
+            process.exit(1);
+        }
+        console.log(chalk.blue(`Loading config from ${configPath}...`));
+        const config = await loadConfig(absoluteConfigPath);
+        // Discover evals - infer from config file location
+        // Config at project/experiments/foo.ts -> evals at project/evals/
+        const projectDir = dirname(dirname(absoluteConfigPath));
+        const evalsDir = resolve(projectDir, 'evals');
+        if (!existsSync(evalsDir)) {
+            console.error(chalk.red(`Evals directory not found: ${evalsDir}`));
+            console.error(chalk.gray(`Expected evals/ to be sibling to experiments/ directory`));
+            process.exit(1);
+        }
+        console.log(chalk.blue(`Discovering evals in ${evalsDir}...`));
+        const { fixtures, errors } = loadAllFixtures(evalsDir);
+        if (errors.length > 0) {
+            console.log(chalk.yellow(`\nWarning: ${errors.length} invalid fixture(s):`));
+            for (const error of errors) {
+                console.log(chalk.yellow(`  - ${error.fixtureName}: ${error.message}`));
+            }
+        }
+        if (fixtures.length === 0) {
+            console.error(chalk.red('No valid eval fixtures found'));
+            process.exit(1);
+        }
+        // Resolve which evals to run
+        const availableNames = fixtures.map((f) => f.name);
+        const evalNames = resolveEvalNames(config.evals, availableNames);
+        if (evalNames.length === 0) {
+            console.error(chalk.red('No evals matched the filter'));
+            process.exit(1);
+        }
+        console.log(chalk.green(`\nFound ${fixtures.length} valid fixture(s), will run ${evalNames.length}:`));
+        for (const name of evalNames) {
+            console.log(chalk.green(`  - ${name}`));
+        }
+        console.log(chalk.blue(`\nRunning ${evalNames.length} eval(s) x ${config.runs} run(s) = ${evalNames.length * config.runs} total runs`));
+        console.log(chalk.blue(`Agent: ${config.agent}, Model: ${config.model}, Timeout: ${config.timeout}s, Early Exit: ${config.earlyExit}`));
+        if (options.dry) {
+            console.log(chalk.yellow('\n[DRY RUN] Would execute evals here'));
+            return;
+        }
+        // Get the agent to check for required API key
+        const agent = getAgent(config.agent);
+        const apiKeyEnvVar = agent.getApiKeyEnvVar();
+        const apiKey = process.env[apiKeyEnvVar];
+        if (!apiKey) {
+            console.error(chalk.red(`${apiKeyEnvVar} environment variable is required`));
+            console.error(chalk.gray(`Get your API key at: https://vercel.com/dashboard -> AI Gateway`));
+            process.exit(1);
+        }
+        // Filter fixtures to only the ones we want to run
+        const selectedFixtures = fixtures.filter((f) => evalNames.includes(f.name));
+        // Get experiment name from config file
+        const experimentName = basename(configPath, '.ts').replace(/\.js$/, '');
+        const resultsDir = resolve(process.cwd(), 'results');
+        console.log(chalk.blue('\nStarting experiment...'));
+        // Run the experiment
+        const results = await runExperiment({
+            config,
+            fixtures: selectedFixtures,
+            apiKey,
+            resultsDir,
+            experimentName,
+            onProgress: (msg) => console.log(msg),
+        });
+        // Exit with appropriate code
+        const allPassed = results.evals.every((e) => e.passedRuns === e.totalRuns);
+        process.exit(allPassed ? 0 : 1);
+    }
+    catch (error) {
+        if (error instanceof Error) {
+            console.error(chalk.red(`Error: ${error.message}`));
+        }
+        else {
+            console.error(chalk.red('An unknown error occurred'));
+        }
+        process.exit(1);
+    }
+}
+/**
+ * init command - Create a new eval project
+ */
+program
+    .command('init')
+    .argument('<name>', 'Name of the project to create')
+    .description('Create a new eval project with example fixtures')
+    .action(async (name) => {
+    try {
+        console.log(chalk.blue(`Creating new eval project: ${name}`));
+        const projectDir = initProject({
+            name,
+            targetDir: process.cwd(),
+        });
+        console.log(chalk.green('Project created successfully!'));
+        console.log(getPostInitInstructions(projectDir, name));
+    }
+    catch (error) {
+        if (error instanceof Error) {
+            console.error(chalk.red(`Error: ${error.message}`));
+        }
+        else {
+            console.error(chalk.red('An unknown error occurred'));
+        }
+        process.exit(1);
+    }
+});
+/**
+ * Default command - run experiment (no subcommand needed)
+ * Usage: agent-eval cc --dry
+ */
+program
+    .argument('[config]', 'Experiment name (e.g., "cc") or path')
+    .option('--dry', 'Preview what would run without executing')
+    .action(async (configInput, options) => {
+    if (!configInput) {
+        program.help();
+        return;
+    }
+    await runExperimentCommand(configInput, options);
+});
+program.parse();
+//# sourceMappingURL=cli.js.map

package/dist/cli.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cli.js","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":";AAEA;;GAEG;AAEH,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,MAAM,IAAI,YAAY,EAAE,MAAM,QAAQ,CAAC;AAChD,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,MAAM,CAAC;AAClD,OAAO,EAAE,UAAU,EAAE,MAAM,IAAI,CAAC;AAChC,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAE,UAAU,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AAC/D,OAAO,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AACnD,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAChD,OAAO,EAAE,WAAW,EAAE,uBAAuB,EAAE,MAAM,eAAe,CAAC;AACrE,OAAO,EAAE,QAAQ,EAAE,MAAM,uBAAuB,CAAC;AAEjD,6BAA6B;AAC7B,YAAY,EAAE,CAAC;AAEf,MAAM,OAAO,GAAG,IAAI,OAAO,EAAE,CAAC;AAE9B,OAAO;KACJ,IAAI,CAAC,YAAY,CAAC;KAClB,WAAW,CAAC,8DAA8D,CAAC;KAC3E,OAAO,CAAC,OAAO,CAAC,CAAC;AAEpB;;;;GAIG;AACH,SAAS,iBAAiB,CAAC,KAAa;IACtC,6DAA6D;IAC7D,IAAI,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;QAClG,OAAO,KAAK,CAAC;IACf,CAAC;IACD,6DAA6D;IAC7D,OAAO,eAAe,KAAK,KAAK,CAAC;AACnC,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,oBAAoB,CAAC,WAAmB,EAAE,OAA0B;IACjF,IAAI,CAAC;QACH,MAAM,UAAU,GAAG,iBAAiB,CAAC,WAAW,CAAC,CAAC;QAClD,MAAM,kBAAkB,GAAG,OAAO,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,UAAU,CAAC,CAAC;QAE9D,IAAI,CAAC,UAAU,CAAC,kBAAkB,CAAC,EAAE,CAAC;YACpC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,0BAA0B,kBAAkB,EAAE,CAAC,CAAC,CAAC;YACzE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,uBAAuB,UAAU,KAAK,CAAC,CAAC,CAAC;QAChE,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,kBAAkB,CAAC,CAAC;QAEpD,mDAAmD;QACnD,kEAAkE;QAClE,MAAM,UAAU,GAAG,OAAO,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC,CAAC;QACxD,MAAM,QAAQ,GAAG,OAAO,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;QAC9C,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC1B,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,8BAA8B,QAAQ,EAAE,CAAC,CAAC,CAAC;YACnE,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,yDAAyD,CAAC,CAAC,CAAC;YACrF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,wBAAwB,QAAQ,KAAK,CAAC,CAAC,CAAC;QAC/D,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,eAAe,CAAC,QAAQ,CAAC,CAAC;QAEvD,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACtB,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,cAAc,MAAM,CAAC,MAAM,sBAAsB,CAAC,CAAC,CAAC;YAC7E,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;gBAC3B,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO,KAAK,CAAC,WAAW,KAAK,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;YAC1E,CAAC;QACH,CAAC;QAED,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC1B,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,8BAA8B,CAAC,CAAC,CAAC;YACzD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;QAED,6BAA6B;QAC7B,MAAM,cAAc,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QACnD,MAAM,SAAS,GAAG,gBAAgB,CAAC,MAAM,CAAC,KAAK,EAAE,cAAc,CAAC,CAAC;QAEjE,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC3B,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,6BAA6B,CAAC,CAAC,CAAC;YACxD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,WAAW,QAAQ,CAAC,MAAM,+BAA+B,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QACvG,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;YAC7B,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,CAAC;QAC1C,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,aAAa,SAAS,CAAC,MAAM,cAAc,MAAM,CAAC,IAAI,aAAa,SAAS,CAAC,MAAM,GAAG,MAAM,CAAC,IAAI,aAAa,CAAC,CAAC,CAAC;QACxI,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,UAAU,MAAM,CAAC,KAAK,YAAY,MAAM,CAAC,KAAK,cAAc,MAAM,CAAC,OAAO,kBAAkB,MAAM,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC;QAExI,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC;YAChB,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,sCAAsC,CAAC,CAAC,CAAC;YAClE,OAAO;QACT,CAAC;QAED,8CAA8C;QAC9C,MAAM,KAAK,GAAG,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACrC,MAAM,YAAY,GAAG,KAAK,CAAC,eAAe,EAAE,CAAC;QAC7C,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;QACzC,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,YAAY,mCAAmC,CAAC,CAAC,CAAC;YAC7E,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,iEAAiE,CAAC,CAAC,CAAC;YAC7F,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;QAED,kDAAkD;QAClD,MAAM,gBAAgB,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;QAE5E,uCAAuC;QACvC,MAAM,cAAc,GAAG,QAAQ,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;QACxE,MAAM,UAAU,GAAG,OAAO,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,SAAS,CAAC,CAAC;QAErD,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC,CAAC;QAEpD,qBAAqB;QACrB,MAAM,OAAO,GAAG,MAAM,aAAa,CAAC;YAClC,MAAM;YACN,QAAQ,EAAE,gBAAgB;YAC1B,MAAM;YACN,UAAU;YACV,cAAc;YACd,UAAU,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC;SACtC,CAAC,CAAC;QAEH,6BAA6B;QAC7B,MAAM,SAAS,GAAG,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,KAAK,CAAC,CAAC,SAAS,CAAC,CAAC;QAC3E,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAClC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,KAAK,YAAY,KAAK,EAAE,CAAC;YAC3B,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,UAAU,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;QACtD,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,2BAA2B,CAAC,CAAC,CAAC;QACxD,CAAC;QACD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC;AAED;;GAEG;AACH,OAAO;KACJ,OAAO,CAAC,MAAM,CAAC;KACf,QAAQ,CAAC,QAAQ,EAAE,+BAA+B,CAAC;KACnD,WAAW,CAAC,iDAAiD,CAAC;KAC9D,MAAM,CAAC,KAAK,EAAE,IAAY,EAAE,EAAE;IAC7B,IAAI,CAAC;QACH,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,8BAA8B,IAAI,EAAE,CAAC,CAAC,CAAC;QAE9D,MAAM,UAAU,GAAG,WAAW,CAAC;YAC7B,IAAI;YACJ,SAAS,EAAE,OAAO,CAAC,GAAG,EAAE;SACzB,CAAC,CAAC;QAEH,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,+BAA+B,CAAC,CAAC,CAAC;QAC1D,OAAO,CAAC,GAAG,CAAC,uBAAuB,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC,CAAC;IACzD,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,KAAK,YAAY,KAAK,EAAE,CAAC;YAC3B,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,UAAU,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;QACtD,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,2BAA2B,CAAC,CAAC,CAAC;QACxD,CAAC;QACD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC,CAAC,CAAC;AAEL;;;GAGG;AACH,OAAO;KACJ,QAAQ,CAAC,UAAU,EAAE,sCAAsC,CAAC;KAC5D,MAAM,CAAC,OAAO,EAAE,0CAA0C,CAAC;KAC3D,MAAM,CAAC,KAAK,EAAE,WAA+B,EAAE,OAA0B,EAAE,EAAE;IAC5E,IAAI,CAAC,WAAW,EAAE,CAAC;QACjB,OAAO,CAAC,IAAI,EAAE,CAAC;QACf,OAAO;IACT,CAAC;IACD,MAAM,oBAAoB,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC;AACnD,CAAC,CAAC,CAAC;AAEL,OAAO,CAAC,KAAK,EAAE,CAAC"}

package/dist/index.d.ts

@@ -0,0 +1,21 @@
+/**
+ * agent-eval
+ *
+ * Framework for testing AI coding agents in isolated sandboxes.
+ */
+export type { AgentType, ModelTier, EvalFilter, Sandbox, SetupFunction, ExperimentConfig, ResolvedExperimentConfig, EvalFixture, EvalRunResult, EvalRunData, EvalSummary, ExperimentResults, } from './lib/types.js';
+export { REQUIRED_EVAL_FILES, EXCLUDED_FILES } from './lib/types.js';
+export { CONFIG_DEFAULTS, validateConfig, resolveConfig, loadConfig, resolveEvalNames, } from './lib/config.js';
+export { FixtureValidationError, discoverFixtures, validateFixtureFiles, validatePackageJson, loadFixture, loadAllFixtures, getFixtureFiles, readFixtureFiles, } from './lib/fixture.js';
+export type { SandboxOptions, CommandResult, SandboxFile } from './lib/sandbox.js';
+export { SandboxManager, DEFAULT_SANDBOX_TIMEOUT, IGNORED_PATTERNS, TEST_FILE_PATTERNS, collectLocalFiles, splitTestFiles, verifyNoTestFiles, } from './lib/sandbox.js';
+export type { AgentRunOptions, AgentRunResult } from './lib/agents/types.js';
+export type { Agent, ScriptResult } from './lib/agents/types.js';
+export { getAgent, listAgents, registerAgent } from './lib/agents/index.js';
+export type { SaveResultsOptions } from './lib/results.js';
+export { agentResultToEvalRunData, createEvalSummary, createExperimentResults, saveResults, formatResultsTable, formatRunResult, createProgressDisplay, } from './lib/results.js';
+export type { RunExperimentOptions } from './lib/runner.js';
+export { runExperiment, runSingleEval } from './lib/runner.js';
+export type { InitOptions } from './lib/init.js';
+export { initProject, getPostInitInstructions } from './lib/init.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAGH,YAAY,EACV,SAAS,EACT,SAAS,EACT,UAAU,EACV,OAAO,EACP,aAAa,EACb,gBAAgB,EAChB,wBAAwB,EACxB,WAAW,EACX,aAAa,EACb,WAAW,EACX,WAAW,EACX,iBAAiB,GAClB,MAAM,gBAAgB,CAAC;AAGxB,OAAO,EAAE,mBAAmB,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAGrE,OAAO,EACL,eAAe,EACf,cAAc,EACd,aAAa,EACb,UAAU,EACV,gBAAgB,GACjB,MAAM,iBAAiB,CAAC;AAGzB,OAAO,EACL,sBAAsB,EACtB,gBAAgB,EAChB,oBAAoB,EACpB,mBAAmB,EACnB,WAAW,EACX,eAAe,EACf,eAAe,EACf,gBAAgB,GACjB,MAAM,kBAAkB,CAAC;AAG1B,YAAY,EAAE,cAAc,EAAE,aAAa,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AACnF,OAAO,EACL,cAAc,EACd,uBAAuB,EACvB,gBAAgB,EAChB,kBAAkB,EAClB,iBAAiB,EACjB,cAAc,EACd,iBAAiB,GAClB,MAAM,kBAAkB,CAAC;AAG1B,YAAY,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AAG7E,YAAY,EAAE,KAAK,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AACjE,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AAG5E,YAAY,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AAC3D,OAAO,EACL,wBAAwB,EACxB,iBAAiB,EACjB,uBAAuB,EACvB,WAAW,EACX,kBAAkB,EAClB,eAAe,EACf,qBAAqB,GACtB,MAAM,kBAAkB,CAAC;AAG1B,YAAY,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AAC5D,OAAO,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAG/D,YAAY,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AACjD,OAAO,EAAE,WAAW,EAAE,uBAAuB,EAAE,MAAM,eAAe,CAAC"}

package/dist/index.js

@@ -0,0 +1,17 @@
+/**
+ * agent-eval
+ *
+ * Framework for testing AI coding agents in isolated sandboxes.
+ */
+// Re-export constants
+export { REQUIRED_EVAL_FILES, EXCLUDED_FILES } from './lib/types.js';
+// Re-export config utilities
+export { CONFIG_DEFAULTS, validateConfig, resolveConfig, loadConfig, resolveEvalNames, } from './lib/config.js';
+// Re-export fixture utilities
+export { FixtureValidationError, discoverFixtures, validateFixtureFiles, validatePackageJson, loadFixture, loadAllFixtures, getFixtureFiles, readFixtureFiles, } from './lib/fixture.js';
+export { SandboxManager, DEFAULT_SANDBOX_TIMEOUT, IGNORED_PATTERNS, TEST_FILE_PATTERNS, collectLocalFiles, splitTestFiles, verifyNoTestFiles, } from './lib/sandbox.js';
+export { getAgent, listAgents, registerAgent } from './lib/agents/index.js';
+export { agentResultToEvalRunData, createEvalSummary, createExperimentResults, saveResults, formatResultsTable, formatRunResult, createProgressDisplay, } from './lib/results.js';
+export { runExperiment, runSingleEval } from './lib/runner.js';
+export { initProject, getPostInitInstructions } from './lib/init.js';
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAkBH,sBAAsB;AACtB,OAAO,EAAE,mBAAmB,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAErE,6BAA6B;AAC7B,OAAO,EACL,eAAe,EACf,cAAc,EACd,aAAa,EACb,UAAU,EACV,gBAAgB,GACjB,MAAM,iBAAiB,CAAC;AAEzB,8BAA8B;AAC9B,OAAO,EACL,sBAAsB,EACtB,gBAAgB,EAChB,oBAAoB,EACpB,mBAAmB,EACnB,WAAW,EACX,eAAe,EACf,eAAe,EACf,gBAAgB,GACjB,MAAM,kBAAkB,CAAC;AAI1B,OAAO,EACL,cAAc,EACd,uBAAuB,EACvB,gBAAgB,EAChB,kBAAkB,EAClB,iBAAiB,EACjB,cAAc,EACd,iBAAiB,GAClB,MAAM,kBAAkB,CAAC;AAO1B,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AAI5E,OAAO,EACL,wBAAwB,EACxB,iBAAiB,EACjB,uBAAuB,EACvB,WAAW,EACX,kBAAkB,EAClB,eAAe,EACf,qBAAqB,GACtB,MAAM,kBAAkB,CAAC;AAI1B,OAAO,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAI/D,OAAO,EAAE,WAAW,EAAE,uBAAuB,EAAE,MAAM,eAAe,CAAC"}

package/dist/lib/agents/claude-code.d.ts

@@ -0,0 +1,12 @@
+/**
+ * Claude Code agent implementation.
+ * Uses Vercel AI Gateway for model access.
+ */
+import type { Agent } from './types.js';
+/**
+ * Create Claude Code agent with specified authentication method.
+ */
+export declare function createClaudeCodeAgent({ useVercelAiGateway }: {
+    useVercelAiGateway: boolean;
+}): Agent;
+//# sourceMappingURL=claude-code.d.ts.map

package/dist/lib/agents/claude-code.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"claude-code.d.ts","sourceRoot":"","sources":["../../../src/lib/agents/claude-code.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EAAE,KAAK,EAAmC,MAAM,YAAY,CAAC;AA8CzE;;GAEG;AACH,wBAAgB,qBAAqB,CAAC,EAAE,kBAAkB,EAAE,EAAE;IAAE,kBAAkB,EAAE,OAAO,CAAA;CAAE,GAAG,KAAK,CAiMpG"}