@vercel/agent-eval 0.0.1

package/dist/lib/init.js

@@ -0,0 +1,250 @@
+/**
+ * Project initialization - create new eval projects.
+ */
+import { mkdirSync, writeFileSync, existsSync } from 'fs';
+import { join, dirname } from 'path';
+/**
+ * Get the package.json template.
+ */
+function getPackageJson(projectName) {
+    return JSON.stringify({
+        name: projectName,
+        version: '0.0.1',
+        private: true,
+        type: 'module',
+        devDependencies: {
+            'agent-eval': '^0.0.1',
+            '@types/node': '^22.0.0',
+            typescript: '^5.6.0',
+            vitest: '^2.1.0',
+        },
+    }, null, 2);
+}
+/**
+ * Get the .env.example template.
+ */
+function getEnvExample() {
+    return `# Required - Vercel AI Gateway API key (works for all agents)
+# Get yours at: https://vercel.com/dashboard -> AI Gateway
+AI_GATEWAY_API_KEY=your-ai-gateway-api-key
+
+# Required - Vercel token for sandbox access (choose ONE of the options below)
+# The @vercel/sandbox package automatically detects either token.
+
+# Option 1: Personal Access Token (for local development)
+# Create at: https://vercel.com/account/tokens
+VERCEL_TOKEN=your-vercel-token
+
+# Option 2: OIDC Token (for CI/CD pipelines like GitHub Actions)
+# Automatically provided by Vercel's CI integration
+# VERCEL_OIDC_TOKEN=your-oidc-token
+`;
+}
+/**
+ * Get the .gitignore template.
+ */
+function getGitignore() {
+    return `node_modules/
+dist/
+.env
+.env.local
+results/
+*.log
+.DS_Store
+`;
+}
+/**
+ * Get the default experiment configuration template (Claude Code).
+ */
+function getCCExperiment() {
+    return `import type { ExperimentConfig } from 'agent-eval';
+
+const config: ExperimentConfig = {
+  agent: 'vercel-ai-gateway/claude-code',
+  runs: 1,
+  earlyExit: true,
+  scripts: ['build'],
+  timeout: 300,
+};
+
+export default config;
+`;
+}
+/**
+ * Get the Codex experiment configuration template.
+ */
+function getCodexExperiment() {
+    return `import type { ExperimentConfig } from 'agent-eval';
+
+const config: ExperimentConfig = {
+  agent: 'vercel-ai-gateway/codex',
+  runs: 1,
+  earlyExit: true,
+  scripts: ['build'],
+  timeout: 300,
+};
+
+export default config;
+`;
+}
+/**
+ * Get the example eval fixture PROMPT.md.
+ */
+function getExamplePrompt() {
+    return `Add a greeting message below the heading that says "Welcome, user!"
+
+Requirements:
+- Add a paragraph element below the h1
+- The text should be exactly "Welcome, user!"
+- Keep the existing heading unchanged
+`;
+}
+/**
+ * Get the example eval fixture EVAL.ts.
+ */
+function getExampleEval() {
+    return `import { readFileSync } from 'fs';
+import { execSync } from 'child_process';
+import { test, expect } from 'vitest';
+
+test('greeting message exists in source', () => {
+  const content = readFileSync('src/App.tsx', 'utf-8');
+  expect(content).toContain('Welcome, user!');
+});
+
+test('app still builds', () => {
+  // This throws if the build fails
+  execSync('npm run build', { stdio: 'pipe' });
+});
+`;
+}
+/**
+ * Get the example eval fixture package.json.
+ */
+function getExamplePackageJson() {
+    return JSON.stringify({
+        name: 'add-greeting',
+        type: 'module',
+        scripts: {
+            build: 'tsc',
+        },
+        dependencies: {
+            react: '^18.0.0',
+        },
+        devDependencies: {
+            '@types/react': '^18.0.0',
+            typescript: '^5.0.0',
+            vitest: '^2.1.0',
+        },
+    }, null, 2);
+}
+/**
+ * Get the root tsconfig.json for the project.
+ */
+function getRootTsconfig() {
+    return JSON.stringify({
+        compilerOptions: {
+            target: 'ES2022',
+            module: 'NodeNext',
+            moduleResolution: 'NodeNext',
+            strict: true,
+            skipLibCheck: true,
+            noEmit: true,
+            lib: ['ES2022'],
+        },
+        include: ['experiments'],
+    }, null, 2);
+}
+/**
+ * Get the example eval fixture tsconfig.json.
+ */
+function getExampleTsconfig() {
+    return JSON.stringify({
+        compilerOptions: {
+            target: 'ES2020',
+            module: 'ESNext',
+            moduleResolution: 'bundler',
+            jsx: 'react-jsx',
+            strict: true,
+            outDir: 'dist',
+            skipLibCheck: true,
+        },
+        include: ['src'],
+    }, null, 2);
+}
+/**
+ * Get the example eval fixture App.tsx.
+ */
+function getExampleApp() {
+    return `export function App() {
+  return (
+    <div>
+      <h1>Hello World</h1>
+      {/* TODO: Add greeting message here */}
+    </div>
+  );
+}
+
+export default App;
+`;
+}
+/**
+ * Get all template files for a new project.
+ */
+function getTemplateFiles(projectName) {
+    return [
+        { path: 'package.json', content: getPackageJson(projectName) },
+        { path: 'tsconfig.json', content: getRootTsconfig() },
+        { path: '.env.example', content: getEnvExample() },
+        { path: '.gitignore', content: getGitignore() },
+        { path: 'experiments/cc.ts', content: getCCExperiment() },
+        { path: 'experiments/codex.ts', content: getCodexExperiment() },
+        { path: 'evals/add-greeting/PROMPT.md', content: getExamplePrompt() },
+        { path: 'evals/add-greeting/EVAL.ts', content: getExampleEval() },
+        { path: 'evals/add-greeting/package.json', content: getExamplePackageJson() },
+        { path: 'evals/add-greeting/tsconfig.json', content: getExampleTsconfig() },
+        { path: 'evals/add-greeting/src/App.tsx', content: getExampleApp() },
+    ];
+}
+/**
+ * Initialize a new eval project.
+ */
+export function initProject(options) {
+    const targetDir = options.targetDir ?? process.cwd();
+    const projectDir = join(targetDir, options.name);
+    // Check if directory already exists
+    if (existsSync(projectDir)) {
+        throw new Error(`Directory already exists: ${projectDir}`);
+    }
+    // Create project directory
+    mkdirSync(projectDir, { recursive: true });
+    // Write all template files
+    const files = getTemplateFiles(options.name);
+    for (const file of files) {
+        const filePath = join(projectDir, file.path);
+        const fileDir = dirname(filePath);
+        // Create parent directories
+        mkdirSync(fileDir, { recursive: true });
+        // Write file
+        writeFileSync(filePath, file.content);
+    }
+    return projectDir;
+}
+/**
+ * Get instructions for after project creation.
+ */
+export function getPostInitInstructions(projectDir, projectName) {
+    return `
+Project created at: ${projectDir}
+
+Next steps:
+  1. cd ${projectName}
+  2. npm install
+  3. Copy .env.example to .env and add your API keys
+  4. npx agent-eval
+
+For more information, see the documentation at:
+  https://github.com/vercel-labs/agent-eval
+`;
+}
+//# sourceMappingURL=init.js.map

package/dist/lib/init.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"init.js","sourceRoot":"","sources":["../../src/lib/init.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,SAAS,EAAE,aAAa,EAAE,UAAU,EAAE,MAAM,IAAI,CAAC;AAC1D,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,MAAM,CAAC;AAoBrC;;GAEG;AACH,SAAS,cAAc,CAAC,WAAmB;IACzC,OAAO,IAAI,CAAC,SAAS,CACnB;QACE,IAAI,EAAE,WAAW;QACjB,OAAO,EAAE,OAAO;QAChB,OAAO,EAAE,IAAI;QACb,IAAI,EAAE,QAAQ;QACd,eAAe,EAAE;YACf,YAAY,EAAE,QAAQ;YACtB,aAAa,EAAE,SAAS;YACxB,UAAU,EAAE,QAAQ;YACpB,MAAM,EAAE,QAAQ;SACjB;KACF,EACD,IAAI,EACJ,CAAC,CACF,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,SAAS,aAAa;IACpB,OAAO;;;;;;;;;;;;;;CAcR,CAAC;AACF,CAAC;AAED;;GAEG;AACH,SAAS,YAAY;IACnB,OAAO;;;;;;;CAOR,CAAC;AACF,CAAC;AAED;;GAEG;AACH,SAAS,eAAe;IACtB,OAAO;;;;;;;;;;;CAWR,CAAC;AACF,CAAC;AAED;;GAEG;AACH,SAAS,kBAAkB;IACzB,OAAO;;;;;;;;;;;CAWR,CAAC;AACF,CAAC;AAED;;GAEG;AACH,SAAS,gBAAgB;IACvB,OAAO;;;;;;CAMR,CAAC;AACF,CAAC;AAED;;GAEG;AACH,SAAS,cAAc;IACrB,OAAO;;;;;;;;;;;;;CAaR,CAAC;AACF,CAAC;AAED;;GAEG;AACH,SAAS,qBAAqB;IAC5B,OAAO,IAAI,CAAC,SAAS,CACnB;QACE,IAAI,EAAE,cAAc;QACpB,IAAI,EAAE,QAAQ;QACd,OAAO,EAAE;YACP,KAAK,EAAE,KAAK;SACb;QACD,YAAY,EAAE;YACZ,KAAK,EAAE,SAAS;SACjB;QACD,eAAe,EAAE;YACf,cAAc,EAAE,SAAS;YACzB,UAAU,EAAE,QAAQ;YACpB,MAAM,EAAE,QAAQ;SACjB;KACF,EACD,IAAI,EACJ,CAAC,CACF,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,SAAS,eAAe;IACtB,OAAO,IAAI,CAAC,SAAS,CACnB;QACE,eAAe,EAAE;YACf,MAAM,EAAE,QAAQ;YAChB,MAAM,EAAE,UAAU;YAClB,gBAAgB,EAAE,UAAU;YAC5B,MAAM,EAAE,IAAI;YACZ,YAAY,EAAE,IAAI;YAClB,MAAM,EAAE,IAAI;YACZ,GAAG,EAAE,CAAC,QAAQ,CAAC;SAChB;QACD,OAAO,EAAE,CAAC,aAAa,CAAC;KACzB,EACD,IAAI,EACJ,CAAC,CACF,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,SAAS,kBAAkB;IACzB,OAAO,IAAI,CAAC,SAAS,CACnB;QACE,eAAe,EAAE;YACf,MAAM,EAAE,QAAQ;YAChB,MAAM,EAAE,QAAQ;YAChB,gBAAgB,EAAE,SAAS;YAC3B,GAAG,EAAE,WAAW;YAChB,MAAM,EAAE,IAAI;YACZ,MAAM,EAAE,MAAM;YACd,YAAY,EAAE,IAAI;SACnB;QACD,OAAO,EAAE,CAAC,KAAK,CAAC;KACjB,EACD,IAAI,EACJ,CAAC,CACF,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,SAAS,aAAa;IACpB,OAAO;;;;;;;;;;CAUR,CAAC;AACF,CAAC;AAED;;GAEG;AACH,SAAS,gBAAgB,CAAC,WAAmB;IAC3C,OAAO;QACL,EAAE,IAAI,EAAE,cAAc,EAAE,OAAO,EAAE,cAAc,CAAC,WAAW,CAAC,EAAE;QAC9D,EAAE,IAAI,EAAE,eAAe,EAAE,OAAO,EAAE,eAAe,EAAE,EAAE;QACrD,EAAE,IAAI,EAAE,cAAc,EAAE,OAAO,EAAE,aAAa,EAAE,EAAE;QAClD,EAAE,IAAI,EAAE,YAAY,EAAE,OAAO,EAAE,YAAY,EAAE,EAAE;QAC/C,EAAE,IAAI,EAAE,mBAAmB,EAAE,OAAO,EAAE,eAAe,EAAE,EAAE;QACzD,EAAE,IAAI,EAAE,sBAAsB,EAAE,OAAO,EAAE,kBAAkB,EAAE,EAAE;QAC/D,EAAE,IAAI,EAAE,8BAA8B,EAAE,OAAO,EAAE,gBAAgB,EAAE,EAAE;QACrE,EAAE,IAAI,EAAE,4BAA4B,EAAE,OAAO,EAAE,cAAc,EAAE,EAAE;QACjE,EAAE,IAAI,EAAE,iCAAiC,EAAE,OAAO,EAAE,qBAAqB,EAAE,EAAE;QAC7E,EAAE,IAAI,EAAE,kCAAkC,EAAE,OAAO,EAAE,kBAAkB,EAAE,EAAE;QAC3E,EAAE,IAAI,EAAE,gCAAgC,EAAE,OAAO,EAAE,aAAa,EAAE,EAAE;KACrE,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,WAAW,CAAC,OAAoB;IAC9C,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC;IACrD,MAAM,UAAU,GAAG,IAAI,CAAC,SAAS,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;IAEjD,oCAAoC;IACpC,IAAI,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;QAC3B,MAAM,IAAI,KAAK,CAAC,6BAA6B,UAAU,EAAE,CAAC,CAAC;IAC7D,CAAC;IAED,2BAA2B;IAC3B,SAAS,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAE3C,2BAA2B;IAC3B,MAAM,KAAK,GAAG,gBAAgB,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IAC7C,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,QAAQ,GAAG,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;QAC7C,MAAM,OAAO,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC;QAElC,4BAA4B;QAC5B,SAAS,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAExC,aAAa;QACb,aAAa,CAAC,QAAQ,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;IACxC,CAAC;IAED,OAAO,UAAU,CAAC;AACpB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,uBAAuB,CAAC,UAAkB,EAAE,WAAmB;IAC7E,OAAO;sBACa,UAAU;;;UAGtB,WAAW;;;;;;;CAOpB,CAAC;AACF,CAAC"}

package/dist/lib/results.d.ts

@@ -0,0 +1,54 @@
+/**
+ * Results storage and reporting for eval experiments.
+ */
+import type { EvalRunResult, EvalRunData, EvalSummary, ExperimentResults, ResolvedExperimentConfig } from './types.js';
+import type { AgentRunResult } from './agents/types.js';
+/**
+ * Convert AgentRunResult to EvalRunData (result + transcript).
+ */
+export declare function agentResultToEvalRunData(agentResult: AgentRunResult): EvalRunData;
+/**
+ * Create a summary from multiple run data.
+ */
+export declare function createEvalSummary(name: string, runData: EvalRunData[]): EvalSummary;
+/**
+ * Create experiment results from eval summaries.
+ */
+export declare function createExperimentResults(config: ResolvedExperimentConfig, evals: EvalSummary[], startedAt: Date, completedAt: Date): ExperimentResults;
+/**
+ * Options for saving results.
+ */
+export interface SaveResultsOptions {
+    /** Base directory for results */
+    resultsDir: string;
+    /** Experiment name (used for subdirectory) */
+    experimentName: string;
+}
+/**
+ * Save experiment results to disk.
+ *
+ * Creates a directory structure per design:
+ * results/
+ *   experiment-name/
+ *     2024-01-26T12-00-00Z/
+ *       eval-1/
+ *         run-1/
+ *           result.json
+ *           transcript.jsonl
+ *           outputs/
+ *         summary.json
+ */
+export declare function saveResults(results: ExperimentResults, options: SaveResultsOptions): string;
+/**
+ * Format results for terminal display.
+ */
+export declare function formatResultsTable(results: ExperimentResults): string;
+/**
+ * Format a single eval result for terminal display (used during progress).
+ */
+export declare function formatRunResult(evalName: string, runNumber: number, totalRuns: number, result: EvalRunResult): string;
+/**
+ * Create a progress indicator for running evals.
+ */
+export declare function createProgressDisplay(evalName: string, runNumber: number, totalRuns: number): string;
+//# sourceMappingURL=results.d.ts.map

package/dist/lib/results.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"results.d.ts","sourceRoot":"","sources":["../../src/lib/results.ts"],"names":[],"mappings":"AAAA;;GAEG;AAKH,OAAO,KAAK,EACV,aAAa,EACb,WAAW,EACX,WAAW,EACX,iBAAiB,EACjB,wBAAwB,EACzB,MAAM,YAAY,CAAC;AACpB,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAExD;;GAEG;AACH,wBAAgB,wBAAwB,CAAC,WAAW,EAAE,cAAc,GAAG,WAAW,CA4BjF;AAED;;GAEG;AACH,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,WAAW,EAAE,GAAG,WAAW,CAanF;AAED;;GAEG;AACH,wBAAgB,uBAAuB,CACrC,MAAM,EAAE,wBAAwB,EAChC,KAAK,EAAE,WAAW,EAAE,EACpB,SAAS,EAAE,IAAI,EACf,WAAW,EAAE,IAAI,GAChB,iBAAiB,CAOnB;AAED;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC,iCAAiC;IACjC,UAAU,EAAE,MAAM,CAAC;IACnB,8CAA8C;IAC9C,cAAc,EAAE,MAAM,CAAC;CACxB;AAED;;;;;;;;;;;;;GAaG;AACH,wBAAgB,WAAW,CACzB,OAAO,EAAE,iBAAiB,EAC1B,OAAO,EAAE,kBAAkB,GAC1B,MAAM,CA8ER;AAED;;GAEG;AACH,wBAAgB,kBAAkB,CAAC,OAAO,EAAE,iBAAiB,GAAG,MAAM,CAsCrE;AAED;;GAEG;AACH,wBAAgB,eAAe,CAC7B,QAAQ,EAAE,MAAM,EAChB,SAAS,EAAE,MAAM,EACjB,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,aAAa,GACpB,MAAM,CAYR;AAED;;GAEG;AACH,wBAAgB,qBAAqB,CACnC,QAAQ,EAAE,MAAM,EAChB,SAAS,EAAE,MAAM,EACjB,SAAS,EAAE,MAAM,GAChB,MAAM,CAER"}

package/dist/lib/results.js

@@ -0,0 +1,186 @@
+/**
+ * Results storage and reporting for eval experiments.
+ */
+import { mkdirSync, writeFileSync } from 'fs';
+import { join } from 'path';
+import chalk from 'chalk';
+/**
+ * Convert AgentRunResult to EvalRunData (result + transcript).
+ */
+export function agentResultToEvalRunData(agentResult) {
+    // Collect output content from scripts and tests
+    const outputContent = {};
+    // Add EVAL.ts test output
+    if (agentResult.testResult?.output) {
+        outputContent.eval = agentResult.testResult.output;
+    }
+    // Add all script outputs (nested under 'scripts' to avoid collision)
+    if (agentResult.scriptsResults && Object.keys(agentResult.scriptsResults).length > 0) {
+        outputContent.scripts = {};
+        for (const [name, result] of Object.entries(agentResult.scriptsResults)) {
+            if (result.output) {
+                outputContent.scripts[name] = result.output;
+            }
+        }
+    }
+    return {
+        result: {
+            status: agentResult.success ? 'passed' : 'failed',
+            error: agentResult.error,
+            duration: agentResult.duration / 1000, // Convert to seconds
+        },
+        transcript: agentResult.transcript,
+        outputContent: Object.keys(outputContent).length > 0 ? outputContent : undefined,
+    };
+}
+/**
+ * Create a summary from multiple run data.
+ */
+export function createEvalSummary(name, runData) {
+    const runs = runData.map((r) => r.result);
+    const passedRuns = runs.filter((r) => r.status === 'passed').length;
+    const totalDuration = runs.reduce((sum, r) => sum + r.duration, 0);
+    return {
+        name,
+        totalRuns: runs.length,
+        passedRuns,
+        passRate: runs.length > 0 ? (passedRuns / runs.length) * 100 : 0,
+        meanDuration: runs.length > 0 ? totalDuration / runs.length : 0,
+        runs: runData,
+    };
+}
+/**
+ * Create experiment results from eval summaries.
+ */
+export function createExperimentResults(config, evals, startedAt, completedAt) {
+    return {
+        startedAt: startedAt.toISOString(),
+        completedAt: completedAt.toISOString(),
+        config,
+        evals,
+    };
+}
+/**
+ * Save experiment results to disk.
+ *
+ * Creates a directory structure per design:
+ * results/
+ *   experiment-name/
+ *     2024-01-26T12-00-00Z/
+ *       eval-1/
+ *         run-1/
+ *           result.json
+ *           transcript.jsonl
+ *           outputs/
+ *         summary.json
+ */
+export function saveResults(results, options) {
+    const timestamp = results.startedAt.replace(/:/g, '-');
+    const experimentDir = join(options.resultsDir, options.experimentName, timestamp);
+    // Create experiment directory
+    mkdirSync(experimentDir, { recursive: true });
+    // Save per-eval results
+    for (const evalSummary of results.evals) {
+        const evalDir = join(experimentDir, evalSummary.name);
+        mkdirSync(evalDir, { recursive: true });
+        // Save summary (simplified format per design)
+        const summaryForFile = {
+            totalRuns: evalSummary.totalRuns,
+            passedRuns: evalSummary.passedRuns,
+            passRate: `${evalSummary.passRate.toFixed(0)}%`,
+            meanDuration: evalSummary.meanDuration,
+        };
+        writeFileSync(join(evalDir, 'summary.json'), JSON.stringify(summaryForFile, null, 2));
+        // Save individual run results
+        for (let i = 0; i < evalSummary.runs.length; i++) {
+            const runData = evalSummary.runs[i];
+            const runDir = join(evalDir, `run-${i + 1}`);
+            mkdirSync(runDir, { recursive: true });
+            // Build the result with paths
+            const resultWithPaths = { ...runData.result };
+            // Save transcript.jsonl if available
+            if (runData.transcript) {
+                writeFileSync(join(runDir, 'transcript.jsonl'), runData.transcript);
+                resultWithPaths.transcriptPath = './transcript.jsonl';
+            }
+            // Save script/test outputs to outputs/
+            const outputsDir = join(runDir, 'outputs');
+            mkdirSync(outputsDir, { recursive: true });
+            if (runData.outputContent) {
+                const outputPaths = {};
+                // Save EVAL.ts test output
+                if (runData.outputContent.eval) {
+                    writeFileSync(join(outputsDir, 'eval.txt'), runData.outputContent.eval);
+                    outputPaths.eval = './outputs/eval.txt';
+                }
+                // Save npm script outputs (nested to avoid collision)
+                if (runData.outputContent.scripts) {
+                    outputPaths.scripts = {};
+                    for (const [name, content] of Object.entries(runData.outputContent.scripts)) {
+                        if (content) {
+                            const fileName = `${name}.txt`;
+                            writeFileSync(join(outputsDir, fileName), content);
+                            outputPaths.scripts[name] = `./outputs/${fileName}`;
+                        }
+                    }
+                }
+                if (outputPaths.eval || (outputPaths.scripts && Object.keys(outputPaths.scripts).length > 0)) {
+                    resultWithPaths.outputPaths = outputPaths;
+                }
+            }
+            // Save result.json with paths
+            writeFileSync(join(runDir, 'result.json'), JSON.stringify(resultWithPaths, null, 2));
+        }
+    }
+    return experimentDir;
+}
+/**
+ * Format results for terminal display.
+ */
+export function formatResultsTable(results) {
+    const lines = [];
+    const separator = '─'.repeat(60);
+    lines.push('');
+    lines.push(chalk.bold('Experiment Results'));
+    lines.push(chalk.gray(separator));
+    lines.push('');
+    // Calculate overall stats
+    const totalRuns = results.evals.reduce((sum, e) => sum + e.totalRuns, 0);
+    const totalPassed = results.evals.reduce((sum, e) => sum + e.passedRuns, 0);
+    const overallPassRate = totalRuns > 0 ? (totalPassed / totalRuns) * 100 : 0;
+    for (const evalSummary of results.evals) {
+        const passIcon = evalSummary.passedRuns === evalSummary.totalRuns ? '✓' : '✗';
+        const passColor = evalSummary.passedRuns === evalSummary.totalRuns ? chalk.green : chalk.red;
+        lines.push(passColor(`${passIcon} ${evalSummary.name}: ${evalSummary.passedRuns}/${evalSummary.totalRuns} passed (${evalSummary.passRate.toFixed(0)}%)`));
+        lines.push(chalk.gray(`  Mean duration: ${evalSummary.meanDuration.toFixed(1)}s`));
+        lines.push('');
+    }
+    lines.push(chalk.gray(separator));
+    lines.push('');
+    const overallColor = overallPassRate === 100 ? chalk.green : overallPassRate >= 50 ? chalk.yellow : chalk.red;
+    lines.push(overallColor(`Overall: ${totalPassed}/${totalRuns} passed (${overallPassRate.toFixed(0)}%)`));
+    const duration = (new Date(results.completedAt).getTime() - new Date(results.startedAt).getTime()) / 1000;
+    lines.push(chalk.gray(`Total time: ${duration.toFixed(1)}s`));
+    lines.push('');
+    return lines.join('\n');
+}
+/**
+ * Format a single eval result for terminal display (used during progress).
+ */
+export function formatRunResult(evalName, runNumber, totalRuns, result) {
+    const icon = result.status === 'passed' ? '✓' : '✗';
+    const color = result.status === 'passed' ? chalk.green : chalk.red;
+    let line = color(`${icon} ${evalName} [${runNumber}/${totalRuns}]`);
+    line += chalk.gray(` (${result.duration.toFixed(1)}s)`);
+    if (result.error) {
+        line += chalk.red(` - ${result.error.slice(0, 50)}${result.error.length > 50 ? '...' : ''}`);
+    }
+    return line;
+}
+/**
+ * Create a progress indicator for running evals.
+ */
+export function createProgressDisplay(evalName, runNumber, totalRuns) {
+    return chalk.blue(`Running ${evalName} [${runNumber}/${totalRuns}]...`);
+}
+//# sourceMappingURL=results.js.map

package/dist/lib/results.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"results.js","sourceRoot":"","sources":["../../src/lib/results.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AAC9C,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAC5B,OAAO,KAAK,MAAM,OAAO,CAAC;AAU1B;;GAEG;AACH,MAAM,UAAU,wBAAwB,CAAC,WAA2B;IAClE,gDAAgD;IAChD,MAAM,aAAa,GAAiC,EAAE,CAAC;IAEvD,0BAA0B;IAC1B,IAAI,WAAW,CAAC,UAAU,EAAE,MAAM,EAAE,CAAC;QACnC,aAAa,CAAC,IAAI,GAAG,WAAW,CAAC,UAAU,CAAC,MAAM,CAAC;IACrD,CAAC;IAED,qEAAqE;IACrE,IAAI,WAAW,CAAC,cAAc,IAAI,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,cAAc,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACrF,aAAa,CAAC,OAAO,GAAG,EAAE,CAAC;QAC3B,KAAK,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC,cAAc,CAAC,EAAE,CAAC;YACxE,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;gBAClB,aAAa,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;YAC9C,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO;QACL,MAAM,EAAE;YACN,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ;YACjD,KAAK,EAAE,WAAW,CAAC,KAAK;YACxB,QAAQ,EAAE,WAAW,CAAC,QAAQ,GAAG,IAAI,EAAE,qBAAqB;SAC7D;QACD,UAAU,EAAE,WAAW,CAAC,UAAU;QAClC,aAAa,EAAE,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,SAAS;KACjF,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,iBAAiB,CAAC,IAAY,EAAE,OAAsB;IACpE,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;IAC1C,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,MAAM,CAAC;IACpE,MAAM,aAAa,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC;IAEnE,OAAO;QACL,IAAI;QACJ,SAAS,EAAE,IAAI,CAAC,MAAM;QACtB,UAAU;QACV,QAAQ,EAAE,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;QAChE,YAAY,EAAE,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,aAAa,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAC/D,IAAI,EAAE,OAAO;KACd,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,uBAAuB,CACrC,MAAgC,EAChC,KAAoB,EACpB,SAAe,EACf,WAAiB;IAEjB,OAAO;QACL,SAAS,EAAE,SAAS,CAAC,WAAW,EAAE;QAClC,WAAW,EAAE,WAAW,CAAC,WAAW,EAAE;QACtC,MAAM;QACN,KAAK;KACN,CAAC;AACJ,CAAC;AAYD;;;;;;;;;;;;;GAaG;AACH,MAAM,UAAU,WAAW,CACzB,OAA0B,EAC1B,OAA2B;IAE3B,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;IACvD,MAAM,aAAa,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,OAAO,CAAC,cAAc,EAAE,SAAS,CAAC,CAAC;IAElF,8BAA8B;IAC9B,SAAS,CAAC,aAAa,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAE9C,wBAAwB;IACxB,KAAK,MAAM,WAAW,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;QACxC,MAAM,OAAO,GAAG,IAAI,CAAC,aAAa,EAAE,WAAW,CAAC,IAAI,CAAC,CAAC;QACtD,SAAS,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAExC,8CAA8C;QAC9C,MAAM,cAAc,GAAG;YACrB,SAAS,EAAE,WAAW,CAAC,SAAS;YAChC,UAAU,EAAE,WAAW,CAAC,UAAU;YAClC,QAAQ,EAAE,GAAG,WAAW,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG;YAC/C,YAAY,EAAE,WAAW,CAAC,YAAY;SACvC,CAAC;QACF,aAAa,CACX,IAAI,CAAC,OAAO,EAAE,cAAc,CAAC,EAC7B,IAAI,CAAC,SAAS,CAAC,cAAc,EAAE,IAAI,EAAE,CAAC,CAAC,CACxC,CAAC;QAEF,8BAA8B;QAC9B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,WAAW,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACjD,MAAM,OAAO,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACpC,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YAC7C,SAAS,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YAEvC,8BAA8B;YAC9B,MAAM,eAAe,GAAG,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;YAE9C,qCAAqC;YACrC,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC;gBACvB,aAAa,CAAC,IAAI,CAAC,MAAM,EAAE,kBAAkB,CAAC,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC;gBACpE,eAAe,CAAC,cAAc,GAAG,oBAAoB,CAAC;YACxD,CAAC;YAED,uCAAuC;YACvC,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;YAC3C,SAAS,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YAE3C,IAAI,OAAO,CAAC,aAAa,EAAE,CAAC;gBAC1B,MAAM,WAAW,GAAiC,EAAE,CAAC;gBAErD,2BAA2B;gBAC3B,IAAI,OAAO,CAAC,aAAa,CAAC,IAAI,EAAE,CAAC;oBAC/B,aAAa,CAAC,IAAI,CAAC,UAAU,EAAE,UAAU,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;oBACxE,WAAW,CAAC,IAAI,GAAG,oBAAoB,CAAC;gBAC1C,CAAC;gBAED,sDAAsD;gBACtD,IAAI,OAAO,CAAC,aAAa,CAAC,OAAO,EAAE,CAAC;oBAClC,WAAW,CAAC,OAAO,GAAG,EAAE,CAAC;oBACzB,KAAK,MAAM,CAAC,IAAI,EAAE,OAAO,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,aAAa,CAAC,OAAO,CAAC,EAAE,CAAC;wBAC5E,IAAI,OAAO,EAAE,CAAC;4BACZ,MAAM,QAAQ,GAAG,GAAG,IAAI,MAAM,CAAC;4BAC/B,aAAa,CAAC,IAAI,CAAC,UAAU,EAAE,QAAQ,CAAC,EAAE,OAAO,CAAC,CAAC;4BACnD,WAAW,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,aAAa,QAAQ,EAAE,CAAC;wBACtD,CAAC;oBACH,CAAC;gBACH,CAAC;gBAED,IAAI,WAAW,CAAC,IAAI,IAAI,CAAC,WAAW,CAAC,OAAO,IAAI,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,CAAC;oBAC7F,eAAe,CAAC,WAAW,GAAG,WAAW,CAAC;gBAC5C,CAAC;YACH,CAAC;YAED,8BAA8B;YAC9B,aAAa,CACX,IAAI,CAAC,MAAM,EAAE,aAAa,CAAC,EAC3B,IAAI,CAAC,SAAS,CAAC,eAAe,EAAE,IAAI,EAAE,CAAC,CAAC,CACzC,CAAC;QACJ,CAAC;IACH,CAAC;IAED,OAAO,aAAa,CAAC;AACvB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,kBAAkB,CAAC,OAA0B;IAC3D,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,MAAM,SAAS,GAAG,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IAEjC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC,CAAC;IAC7C,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC;IAClC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEf,0BAA0B;IAC1B,MAAM,SAAS,GAAG,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC;IACzE,MAAM,WAAW,GAAG,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;IAC5E,MAAM,eAAe,GAAG,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,WAAW,GAAG,SAAS,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IAE5E,KAAK,MAAM,WAAW,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;QACxC,MAAM,QAAQ,GAAG,WAAW,CAAC,UAAU,KAAK,WAAW,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;QAC9E,MAAM,SAAS,GAAG,WAAW,CAAC,UAAU,KAAK,WAAW,CAAC,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC;QAE7F,KAAK,CAAC,IAAI,CACR,SAAS,CACP,GAAG,QAAQ,IAAI,WAAW,CAAC,IAAI,KAAK,WAAW,CAAC,UAAU,IAAI,WAAW,CAAC,SAAS,YAAY,WAAW,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CACnI,CACF,CAAC;QACF,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,oBAAoB,WAAW,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QACnF,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjB,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC;IAClC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEf,MAAM,YAAY,GAAG,eAAe,KAAK,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,eAAe,IAAI,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC;IAC9G,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,YAAY,WAAW,IAAI,SAAS,YAAY,eAAe,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IAEzG,MAAM,QAAQ,GAAG,CAAC,IAAI,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,OAAO,EAAE,GAAG,IAAI,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE,CAAC,GAAG,IAAI,CAAC;IAC1G,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,eAAe,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC9D,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEf,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,eAAe,CAC7B,QAAgB,EAChB,SAAiB,EACjB,SAAiB,EACjB,MAAqB;IAErB,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;IACpD,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC;IAEnE,IAAI,IAAI,GAAG,KAAK,CAAC,GAAG,IAAI,IAAI,QAAQ,KAAK,SAAS,IAAI,SAAS,GAAG,CAAC,CAAC;IACpE,IAAI,IAAI,KAAK,CAAC,IAAI,CAAC,KAAK,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IAExD,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;QACjB,IAAI,IAAI,KAAK,CAAC,GAAG,CAAC,MAAM,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,GAAG,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAC/F,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,qBAAqB,CACnC,QAAgB,EAChB,SAAiB,EACjB,SAAiB;IAEjB,OAAO,KAAK,CAAC,IAAI,CAAC,WAAW,QAAQ,KAAK,SAAS,IAAI,SAAS,MAAM,CAAC,CAAC;AAC1E,CAAC"}

package/dist/lib/runner.d.ts

@@ -0,0 +1,43 @@
+/**
+ * Experiment runner - orchestrates running evals against agent.
+ * All evals and attempts run concurrently for maximum throughput.
+ * With earlyExit, in-flight attempts are aborted when one passes.
+ */
+import type { ResolvedExperimentConfig, EvalFixture, EvalRunData, ExperimentResults } from './types.js';
+/**
+ * Options for running an experiment.
+ */
+export interface RunExperimentOptions {
+    /** Resolved experiment configuration */
+    config: ResolvedExperimentConfig;
+    /** Fixtures to run */
+    fixtures: EvalFixture[];
+    /** API key for the agent */
+    apiKey: string;
+    /** Directory to save results */
+    resultsDir: string;
+    /** Experiment name */
+    experimentName: string;
+    /** Callback for progress updates */
+    onProgress?: (message: string) => void;
+    /** Whether to run in verbose mode */
+    verbose?: boolean;
+}
+/**
+ * Run an experiment - execute all evals with configured runs concurrently.
+ * With earlyExit enabled, remaining attempts for a fixture are aborted once one passes.
+ */
+export declare function runExperiment(options: RunExperimentOptions): Promise<ExperimentResults>;
+/**
+ * Run a single eval (for testing/debugging).
+ */
+export declare function runSingleEval(fixture: EvalFixture, options: {
+    agent?: ResolvedExperimentConfig['agent'];
+    model: ResolvedExperimentConfig['model'];
+    timeout: number;
+    apiKey: string;
+    setup?: ResolvedExperimentConfig['setup'];
+    scripts?: string[];
+    verbose?: boolean;
+}): Promise<EvalRunData>;
+//# sourceMappingURL=runner.d.ts.map

package/dist/lib/runner.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../src/lib/runner.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,KAAK,EACV,wBAAwB,EACxB,WAAW,EACX,WAAW,EAEX,iBAAiB,EAClB,MAAM,YAAY,CAAC;AAYpB;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,wCAAwC;IACxC,MAAM,EAAE,wBAAwB,CAAC;IACjC,sBAAsB;IACtB,QAAQ,EAAE,WAAW,EAAE,CAAC;IACxB,4BAA4B;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,gCAAgC;IAChC,UAAU,EAAE,MAAM,CAAC;IACnB,sBAAsB;IACtB,cAAc,EAAE,MAAM,CAAC;IACvB,oCAAoC;IACpC,UAAU,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC;IACvC,qCAAqC;IACrC,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB;AAoBD;;;GAGG;AACH,wBAAsB,aAAa,CACjC,OAAO,EAAE,oBAAoB,GAC5B,OAAO,CAAC,iBAAiB,CAAC,CA2I5B;AAED;;GAEG;AACH,wBAAsB,aAAa,CACjC,OAAO,EAAE,WAAW,EACpB,OAAO,EAAE;IACP,KAAK,CAAC,EAAE,wBAAwB,CAAC,OAAO,CAAC,CAAC;IAC1C,KAAK,EAAE,wBAAwB,CAAC,OAAO,CAAC,CAAC;IACzC,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,wBAAwB,CAAC,OAAO,CAAC,CAAC;IAC1C,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB,GACA,OAAO,CAAC,WAAW,CAAC,CAatB"}