@vercel/agent-eval 0.0.1

package/dist/lib/types.d.ts

@@ -0,0 +1,166 @@
+/**
+ * Core types for the eval framework.
+ */
+/**
+ * Supported AI agent types.
+ */
+export type AgentType = 'vercel-ai-gateway/claude-code' | 'claude-code' | 'vercel-ai-gateway/codex' | 'codex';
+/**
+ * Model identifier - any string accepted.
+ * Each agent validates its own models at runtime.
+ */
+export type ModelTier = string;
+/**
+ * Function type for filtering evals.
+ */
+export type EvalFilter = (name: string) => boolean;
+/**
+ * Sandbox interface for setup functions.
+ * Provides methods to interact with the isolated VM.
+ */
+export interface Sandbox {
+    /** Run a command in the sandbox */
+    runCommand(command: string, args?: string[], options?: {
+        env?: Record<string, string>;
+    }): Promise<{
+        stdout: string;
+        stderr: string;
+        exitCode: number;
+    }>;
+    /** Read a file from the sandbox */
+    readFile(path: string): Promise<string>;
+    /** Write files to the sandbox */
+    writeFiles(files: Record<string, string>): Promise<void>;
+    /** Get the sandbox working directory */
+    getWorkingDirectory(): string;
+}
+/**
+ * Setup function that runs before the agent starts.
+ * Receives a sandbox instance for pre-configuration.
+ */
+export type SetupFunction = (sandbox: Sandbox) => Promise<void>;
+/**
+ * Experiment configuration.
+ * Defines what to test and how.
+ */
+export interface ExperimentConfig {
+    /** Which AI agent to use */
+    agent: AgentType;
+    /** Which AI model the agent should use. Default is agent-specific: 'opus' for claude-code, 'openai/gpt-5.2-codex' for codex */
+    model?: ModelTier;
+    /** Which evals to run. Can be a string, array, or filter function. @default '*' (all evals) */
+    evals?: string | string[] | EvalFilter;
+    /** How many times to run each eval. @default 1 */
+    runs?: number;
+    /** Stop after first successful run? @default true */
+    earlyExit?: boolean;
+    /** npm scripts that must pass after agent finishes. @default [] */
+    scripts?: string[];
+    /** Maximum time in seconds for agent to complete. @default 300 (5 minutes) */
+    timeout?: number;
+    /** Setup function that runs before agent starts. @default undefined */
+    setup?: SetupFunction;
+}
+/**
+ * Resolved experiment config with all defaults applied.
+ */
+export interface ResolvedExperimentConfig {
+    agent: AgentType;
+    model: ModelTier;
+    evals: string | string[] | EvalFilter;
+    runs: number;
+    earlyExit: boolean;
+    scripts: string[];
+    timeout: number;
+    setup?: SetupFunction;
+}
+/**
+ * Required files for a valid eval fixture.
+ */
+export declare const REQUIRED_EVAL_FILES: readonly ["PROMPT.md", "EVAL.ts", "package.json"];
+/**
+ * Files excluded when listing fixture files (used by getFixtureFiles in fixture.ts).
+ * This is for local fixture introspection, NOT for sandbox uploads.
+ * For sandbox file filtering, see TEST_FILE_PATTERNS in sandbox.ts.
+ */
+export declare const EXCLUDED_FILES: readonly ["PROMPT.md", "EVAL.ts", "node_modules", ".git"];
+/**
+ * Represents a discovered eval fixture.
+ */
+export interface EvalFixture {
+    /** Name of the eval (folder name) */
+    name: string;
+    /** Absolute path to the eval folder */
+    path: string;
+    /** Contents of PROMPT.md */
+    prompt: string;
+    /** Whether package.json has "type": "module" */
+    isModule: boolean;
+}
+/**
+ * Result of a single eval run.
+ */
+export interface EvalRunResult {
+    /** Pass or fail status */
+    status: 'passed' | 'failed';
+    /** Error message if failed */
+    error?: string;
+    /** Duration in seconds */
+    duration: number;
+    /** Path to transcript file (relative to run directory) */
+    transcriptPath?: string;
+    /** Paths to output files (relative to run directory) */
+    outputPaths?: {
+        /** Path to EVAL.ts test output */
+        eval?: string;
+        /** Paths to npm script outputs (nested to avoid collision) */
+        scripts?: Record<string, string>;
+    };
+}
+/**
+ * Internal run data including transcript and outputs (content, not paths).
+ */
+export interface EvalRunData {
+    /** The eval result (will have paths added when saving) */
+    result: EvalRunResult;
+    /** Structured transcript from Claude Code (saved to transcript.jsonl) */
+    transcript?: string;
+    /** Script/test output content (saved to outputs/) */
+    outputContent?: {
+        /** EVAL.ts test output */
+        eval?: string;
+        /** npm script outputs (nested to avoid collision) */
+        scripts?: Record<string, string>;
+    };
+}
+/**
+ * Summary of multiple runs for a single eval.
+ */
+export interface EvalSummary {
+    /** Name of the eval */
+    name: string;
+    /** Total number of runs */
+    totalRuns: number;
+    /** Number of passed runs */
+    passedRuns: number;
+    /** Pass rate as a percentage */
+    passRate: number;
+    /** Mean duration across all runs */
+    meanDuration: number;
+    /** Individual run data (internal, not all fields saved to summary.json) */
+    runs: EvalRunData[];
+}
+/**
+ * Complete experiment results.
+ */
+export interface ExperimentResults {
+    /** Timestamp when experiment started */
+    startedAt: string;
+    /** Timestamp when experiment completed */
+    completedAt: string;
+    /** Experiment configuration used */
+    config: ResolvedExperimentConfig;
+    /** Results for each eval */
+    evals: EvalSummary[];
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/lib/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/lib/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH;;GAEG;AACH,MAAM,MAAM,SAAS,GACjB,+BAA+B,GAC/B,aAAa,GACb,yBAAyB,GACzB,OAAO,CAAC;AAEZ;;;GAGG;AACH,MAAM,MAAM,SAAS,GAAG,MAAM,CAAC;AAE/B;;GAEG;AACH,MAAM,MAAM,UAAU,GAAG,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC;AAEnD;;;GAGG;AACH,MAAM,WAAW,OAAO;IACtB,mCAAmC;IACnC,UAAU,CACR,OAAO,EAAE,MAAM,EACf,IAAI,CAAC,EAAE,MAAM,EAAE,EACf,OAAO,CAAC,EAAE;QAAE,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;KAAE,GACzC,OAAO,CAAC;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACjE,mCAAmC;IACnC,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IACxC,iCAAiC;IACjC,UAAU,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACzD,wCAAwC;IACxC,mBAAmB,IAAI,MAAM,CAAC;CAC/B;AAED;;;GAGG;AACH,MAAM,MAAM,aAAa,GAAG,CAAC,OAAO,EAAE,OAAO,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;AAEhE;;;GAGG;AACH,MAAM,WAAW,gBAAgB;IAC/B,4BAA4B;IAC5B,KAAK,EAAE,SAAS,CAAC;IAEjB,+HAA+H;IAC/H,KAAK,CAAC,EAAE,SAAS,CAAC;IAElB,+FAA+F;IAC/F,KAAK,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,UAAU,CAAC;IAEvC,kDAAkD;IAClD,IAAI,CAAC,EAAE,MAAM,CAAC;IAEd,qDAAqD;IACrD,SAAS,CAAC,EAAE,OAAO,CAAC;IAEpB,mEAAmE;IACnE,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IAEnB,8EAA8E;IAC9E,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB,uEAAuE;IACvE,KAAK,CAAC,EAAE,aAAa,CAAC;CACvB;AAED;;GAEG;AACH,MAAM,WAAW,wBAAwB;IACvC,KAAK,EAAE,SAAS,CAAC;IACjB,KAAK,EAAE,SAAS,CAAC;IACjB,KAAK,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,UAAU,CAAC;IACtC,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,OAAO,CAAC;IACnB,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,aAAa,CAAC;CACvB;AAED;;GAEG;AACH,eAAO,MAAM,mBAAmB,mDAAoD,CAAC;AAErF;;;;GAIG;AACH,eAAO,MAAM,cAAc,2DAA4D,CAAC;AAExF;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,qCAAqC;IACrC,IAAI,EAAE,MAAM,CAAC;IACb,uCAAuC;IACvC,IAAI,EAAE,MAAM,CAAC;IACb,4BAA4B;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,gDAAgD;IAChD,QAAQ,EAAE,OAAO,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,0BAA0B;IAC1B,MAAM,EAAE,QAAQ,GAAG,QAAQ,CAAC;IAC5B,8BAA8B;IAC9B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,0BAA0B;IAC1B,QAAQ,EAAE,MAAM,CAAC;IACjB,0DAA0D;IAC1D,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,wDAAwD;IACxD,WAAW,CAAC,EAAE;QACZ,kCAAkC;QAClC,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,8DAA8D;QAC9D,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;KAClC,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,0DAA0D;IAC1D,MAAM,EAAE,aAAa,CAAC;IACtB,yEAAyE;IACzE,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,qDAAqD;IACrD,aAAa,CAAC,EAAE;QACd,0BAA0B;QAC1B,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,qDAAqD;QACrD,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;KAClC,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,uBAAuB;IACvB,IAAI,EAAE,MAAM,CAAC;IACb,2BAA2B;IAC3B,SAAS,EAAE,MAAM,CAAC;IAClB,4BAA4B;IAC5B,UAAU,EAAE,MAAM,CAAC;IACnB,gCAAgC;IAChC,QAAQ,EAAE,MAAM,CAAC;IACjB,oCAAoC;IACpC,YAAY,EAAE,MAAM,CAAC;IACrB,2EAA2E;IAC3E,IAAI,EAAE,WAAW,EAAE,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,wCAAwC;IACxC,SAAS,EAAE,MAAM,CAAC;IAClB,0CAA0C;IAC1C,WAAW,EAAE,MAAM,CAAC;IACpB,oCAAoC;IACpC,MAAM,EAAE,wBAAwB,CAAC;IACjC,4BAA4B;IAC5B,KAAK,EAAE,WAAW,EAAE,CAAC;CACtB"}

package/dist/lib/types.js

@@ -0,0 +1,14 @@
+/**
+ * Core types for the eval framework.
+ */
+/**
+ * Required files for a valid eval fixture.
+ */
+export const REQUIRED_EVAL_FILES = ['PROMPT.md', 'EVAL.ts', 'package.json'];
+/**
+ * Files excluded when listing fixture files (used by getFixtureFiles in fixture.ts).
+ * This is for local fixture introspection, NOT for sandbox uploads.
+ * For sandbox file filtering, see TEST_FILE_PATTERNS in sandbox.ts.
+ */
+export const EXCLUDED_FILES = ['PROMPT.md', 'EVAL.ts', 'node_modules', '.git'];
+//# sourceMappingURL=types.js.map

package/dist/lib/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/lib/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AA2FH;;GAEG;AACH,MAAM,CAAC,MAAM,mBAAmB,GAAG,CAAC,WAAW,EAAE,SAAS,EAAE,cAAc,CAAU,CAAC;AAErF;;;;GAIG;AACH,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,WAAW,EAAE,SAAS,EAAE,cAAc,EAAE,MAAM,CAAU,CAAC"}

package/dist/test-setup.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=test-setup.d.ts.map

package/dist/test-setup.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"test-setup.d.ts","sourceRoot":"","sources":["../src/test-setup.ts"],"names":[],"mappings":""}

package/dist/test-setup.js

@@ -0,0 +1,6 @@
+/**
+ * Test setup file - loads environment variables for tests.
+ */
+import { config as dotenvConfig } from 'dotenv';
+dotenvConfig();
+//# sourceMappingURL=test-setup.js.map

package/dist/test-setup.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"test-setup.js","sourceRoot":"","sources":["../src/test-setup.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,OAAO,EAAE,MAAM,IAAI,YAAY,EAAE,MAAM,QAAQ,CAAC;AAEhD,YAAY,EAAE,CAAC"}

package/package.json

@@ -0,0 +1,58 @@
+{
+  "name": "@vercel/agent-eval",
+  "version": "0.0.1",
+  "description": "Framework for testing AI coding agents in isolated sandboxes",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/vercel-labs/agent-eval.git"
+  },
+  "type": "module",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "bin": {
+    "agent-eval": "dist/cli.js"
+  },
+  "files": [
+    "dist"
+  ],
+  "scripts": {
+    "build": "tsc",
+    "test": "vitest run",
+    "test:watch": "vitest",
+    "lint": "eslint src/",
+    "prepublishOnly": "npm run build"
+  },
+  "dependencies": {
+    "@ai-sdk/anthropic": "^1.2.12",
+    "@vercel/sandbox": "^1.2.0",
+    "ai": "^5.0.11",
+    "chalk": "^5.3.0",
+    "commander": "^12.1.0",
+    "dotenv": "^16.4.5",
+    "glob": "^11.0.0",
+    "zod": "^3.23.8"
+  },
+  "devDependencies": {
+    "@types/node": "^22.0.0",
+    "eslint": "^9.0.0",
+    "tsx": "^4.21.0",
+    "typescript": "^5.6.0",
+    "typescript-eslint": "^8.54.0",
+    "vitest": "^2.1.0"
+  },
+  "engines": {
+    "node": ">=18.0.0"
+  },
+  "keywords": [
+    "ai",
+    "eval",
+    "testing",
+    "claude",
+    "agent",
+    "sandbox"
+  ],
+  "license": "MIT",
+  "publishConfig": {
+    "access": "public"
+  }
+}