agent-eval-opencode 0.10.0

package/dist/lib/config.js

@@ -0,0 +1,187 @@
+/**
+ * Experiment configuration validation and defaults.
+ */
+import { z } from 'zod';
+import { minimatch } from 'minimatch';
+import { getAgent } from './agents/index.js';
+/**
+ * Default configuration values.
+ */
+export const CONFIG_DEFAULTS = {
+    model: 'opus',
+    evals: '*',
+    runs: 1,
+    earlyExit: true,
+    scripts: [],
+    timeout: 600, // 10 minutes
+    sandbox: 'auto',
+    copyFiles: 'none',
+};
+/**
+ * Zod schema for validating experiment configuration.
+ */
+const experimentConfigSchema = z.object({
+    agent: z.enum([
+        'vercel-ai-gateway/claude-code',
+        'claude-code',
+        'vercel-ai-gateway/codex',
+        'codex',
+        'opencode',
+        'gemini',
+        'cursor',
+    ]),
+    model: z.union([z.string(), z.array(z.string())]).optional(),
+    evals: z
+        .union([z.string(), z.array(z.string()), z.function().args(z.string()).returns(z.boolean())])
+        .optional(),
+    runs: z.number().int().positive().optional(),
+    earlyExit: z.boolean().optional(),
+    scripts: z.array(z.string()).optional(),
+    timeout: z.number().positive().optional(),
+    setup: z.function().optional(),
+    sandbox: z.enum(['vercel', 'docker', 'auto']).optional(),
+    editPrompt: z.function().args(z.string()).returns(z.string()).optional(),
+    copyFiles: z.enum(['none', 'changed', 'all']).optional(),
+});
+/**
+ * Validates an experiment configuration object.
+ * Throws a descriptive error if validation fails.
+ */
+export function validateConfig(config) {
+    const result = experimentConfigSchema.safeParse(config);
+    if (!result.success) {
+        const errors = result.error.issues
+            .map((issue) => `  - ${issue.path.join('.')}: ${issue.message}`)
+            .join('\n');
+        throw new Error(`Invalid experiment configuration:\n${errors}`);
+    }
+    return result.data;
+}
+/**
+ * Resolves an experiment configuration by applying defaults.
+ */
+export function resolveConfig(config) {
+    // Validate agent exists
+    const agent = getAgent(config.agent);
+    // Get the default model based on the agent type
+    const defaultModel = config.model ?? agent.getDefaultModel();
+    return {
+        agent: config.agent,
+        model: defaultModel,
+        evals: config.evals ?? '*',
+        runs: config.runs ?? CONFIG_DEFAULTS.runs,
+        earlyExit: config.earlyExit ?? CONFIG_DEFAULTS.earlyExit,
+        scripts: config.scripts ?? CONFIG_DEFAULTS.scripts,
+        timeout: config.timeout ?? CONFIG_DEFAULTS.timeout,
+        setup: config.setup,
+        sandbox: config.sandbox ?? CONFIG_DEFAULTS.sandbox,
+        editPrompt: config.editPrompt,
+        copyFiles: config.copyFiles ?? CONFIG_DEFAULTS.copyFiles,
+    };
+}
+/**
+ * Loads an experiment configuration from a file path.
+ * Supports TypeScript and JavaScript files with default exports.
+ */
+export async function loadConfig(configPath) {
+    try {
+        let rawConfig;
+        // Use jiti for TypeScript files
+        if (configPath.endsWith('.ts')) {
+            const { createJiti } = await import('jiti');
+            const jiti = createJiti(import.meta.url, {
+                interopDefault: true,
+                moduleCache: false,
+            });
+            rawConfig = await jiti.import(configPath);
+        }
+        else {
+            // Dynamic import for JavaScript files
+            const module = await import(configPath);
+            rawConfig = module.default;
+        }
+        if (!rawConfig) {
+            throw new Error(`Config file must have a default export`);
+        }
+        const config = validateConfig(rawConfig);
+        return resolveConfig(config);
+    }
+    catch (error) {
+        if (error instanceof Error) {
+            throw new Error(`Failed to load config from ${configPath}: ${error.message}`);
+        }
+        throw error;
+    }
+}
+/**
+ * Check if a name matches a glob-style pattern.
+ * Supports:
+ * - "*" matches everything
+ * - "vercel-cli/*" matches all under vercel-cli/
+ * - "* /deploy" matches any deploy in any folder
+ * - "vercel-cli/deploy" exact match
+ *
+ * Uses minimatch for robust glob matching.
+ */
+function matchesPattern(name, pattern) {
+    return minimatch(name, pattern);
+}
+/**
+ * Resolves the evals filter to a list of eval names.
+ * Supports glob patterns like "vercel-cli/*" for nested directories.
+ */
+export function resolveEvalNames(filter, availableEvals) {
+    // Single eval name or pattern
+    if (typeof filter === 'string') {
+        if (filter === '*') {
+            return availableEvals;
+        }
+        // Check if it's a glob pattern
+        if (filter.includes('*')) {
+            const matched = availableEvals.filter((name) => matchesPattern(name, filter));
+            if (matched.length === 0) {
+                throw new Error(`No evals matched pattern "${filter}". Available evals: ${availableEvals.join(', ')}`);
+            }
+            return matched;
+        }
+        // Exact match
+        if (!availableEvals.includes(filter)) {
+            throw new Error(`Eval "${filter}" not found. Available evals: ${availableEvals.join(', ')}`);
+        }
+        return [filter];
+    }
+    // Array of eval names or patterns
+    if (Array.isArray(filter)) {
+        const result = [];
+        const seen = new Set();
+        for (const item of filter) {
+            // Handle glob patterns in arrays
+            if (item.includes('*')) {
+                const matched = availableEvals.filter((name) => matchesPattern(name, item));
+                if (matched.length === 0) {
+                    throw new Error(`No evals matched pattern "${item}". Available evals: ${availableEvals.join(', ')}`);
+                }
+                for (const name of matched) {
+                    if (!seen.has(name)) {
+                        result.push(name);
+                        seen.add(name);
+                    }
+                }
+            }
+            else {
+                // Exact match
+                if (!availableEvals.includes(item)) {
+                    throw new Error(`Eval "${item}" not found. Available evals: ${availableEvals.join(', ')}`);
+                }
+                if (!seen.has(item)) {
+                    result.push(item);
+                    seen.add(item);
+                }
+            }
+        }
+        return result;
+    }
+    // Filter function
+    return availableEvals.filter(filter);
+}
+//# sourceMappingURL=config.js.map

package/dist/lib/config.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"config.js","sourceRoot":"","sources":["../../src/lib/config.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAMtC,OAAO,EAAE,QAAQ,EAAE,MAAM,mBAAmB,CAAC;AAE7C;;GAEG;AACH,MAAM,CAAC,MAAM,eAAe,GAAG;IAC7B,KAAK,EAAE,MAAe;IACtB,KAAK,EAAE,GAAY;IACnB,IAAI,EAAE,CAAC;IACP,SAAS,EAAE,IAAI;IACf,OAAO,EAAE,EAAc;IACvB,OAAO,EAAE,GAAG,EAAE,aAAa;IAC3B,OAAO,EAAE,MAAe;IACxB,SAAS,EAAE,MAAe;CAC3B,CAAC;AAEF;;GAEG;AACH,MAAM,sBAAsB,GAAG,CAAC,CAAC,MAAM,CAAC;IACtC,KAAK,EAAE,CAAC,CAAC,IAAI,CAAC;QACZ,+BAA+B;QAC/B,aAAa;QACb,yBAAyB;QACzB,OAAO;QACP,UAAU;QACV,QAAQ;QACR,QAAQ;KACT,CAAC;IACF,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;IAC5D,KAAK,EAAE,CAAC;SACL,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;SAC5F,QAAQ,EAAE;IACb,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;IAC5C,SAAS,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;IACjC,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;IACvC,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;IACzC,KAAK,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;IAC9B,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC,QAAQ,EAAE;IACxD,UAAU,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;IACxE,SAAS,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,SAAS,EAAE,KAAK,CAAC,CAAC,CAAC,QAAQ,EAAE;CACzD,CAAC,CAAC;AAEH;;;GAGG;AACH,MAAM,UAAU,cAAc,CAAC,MAAe;IAC5C,MAAM,MAAM,GAAG,sBAAsB,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;IAExD,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QACpB,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM;aAC/B,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,KAAK,CAAC,OAAO,EAAE,CAAC;aAC/D,IAAI,CAAC,IAAI,CAAC,CAAC;QACd,MAAM,IAAI,KAAK,CAAC,sCAAsC,MAAM,EAAE,CAAC,CAAC;IAClE,CAAC;IAED,OAAO,MAAM,CAAC,IAAwB,CAAC;AACzC,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,aAAa,CAAC,MAAwB;IACpD,wBAAwB;IACxB,MAAM,KAAK,GAAG,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAErC,gDAAgD;IAChD,MAAM,YAAY,GAAG,MAAM,CAAC,KAAK,IAAI,KAAK,CAAC,eAAe,EAAE,CAAC;IAE7D,OAAO;QACL,KAAK,EAAE,MAAM,CAAC,KAAK;QACnB,KAAK,EAAE,YAAY;QACnB,KAAK,EAAE,MAAM,CAAC,KAAK,IAAI,GAAG;QAC1B,IAAI,EAAE,MAAM,CAAC,IAAI,IAAI,eAAe,CAAC,IAAI;QACzC,SAAS,EAAE,MAAM,CAAC,SAAS,IAAI,eAAe,CAAC,SAAS;QACxD,OAAO,EAAE,MAAM,CAAC,OAAO,IAAI,eAAe,CAAC,OAAO;QAClD,OAAO,EAAE,MAAM,CAAC,OAAO,IAAI,eAAe,CAAC,OAAO;QAClD,KAAK,EAAE,MAAM,CAAC,KAAK;QACnB,OAAO,EAAE,MAAM,CAAC,OAAO,IAAI,eAAe,CAAC,OAAO;QAClD,UAAU,EAAE,MAAM,CAAC,UAAU;QAC7B,SAAS,EAAE,MAAM,CAAC,SAAS,IAAI,eAAe,CAAC,SAAS;KACzD,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,UAAkB;IACjD,IAAI,CAAC;QACH,IAAI,SAAkB,CAAC;QAEvB,gCAAgC;QAChC,IAAI,UAAU,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;YAC/B,MAAM,EAAE,UAAU,EAAE,GAAG,MAAM,MAAM,CAAC,MAAM,CAAC,CAAC;YAC5C,MAAM,IAAI,GAAG,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE;gBACvC,cAAc,EAAE,IAAI;gBACpB,WAAW,EAAE,KAAK;aACnB,CAAC,CAAC;YACH,SAAS,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;QAC5C,CAAC;aAAM,CAAC;YACN,sCAAsC;YACtC,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,UAAU,CAAC,CAAC;YACxC,SAAS,GAAG,MAAM,CAAC,OAAO,CAAC;QAC7B,CAAC;QAED,IAAI,CAAC,SAAS,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CAAC,wCAAwC,CAAC,CAAC;QAC5D,CAAC;QAED,MAAM,MAAM,GAAG,cAAc,CAAC,SAAS,CAAC,CAAC;QACzC,OAAO,aAAa,CAAC,MAAM,CAAC,CAAC;IAC/B,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,KAAK,YAAY,KAAK,EAAE,CAAC;YAC3B,MAAM,IAAI,KAAK,CAAC,8BAA8B,UAAU,KAAK,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;QAChF,CAAC;QACD,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC;AAED;;;;;;;;;GASG;AACH,SAAS,cAAc,CAAC,IAAY,EAAE,OAAe;IACnD,OAAO,SAAS,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;AAClC,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,gBAAgB,CAC9B,MAAsC,EACtC,cAAwB;IAExB,8BAA8B;IAC9B,IAAI,OAAO,MAAM,KAAK,QAAQ,EAAE,CAAC;QAC/B,IAAI,MAAM,KAAK,GAAG,EAAE,CAAC;YACnB,OAAO,cAAc,CAAC;QACxB,CAAC;QAED,+BAA+B;QAC/B,IAAI,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;YACzB,MAAM,OAAO,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,cAAc,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC;YAC9E,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACzB,MAAM,IAAI,KAAK,CAAC,6BAA6B,MAAM,uBAAuB,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACzG,CAAC;YACD,OAAO,OAAO,CAAC;QACjB,CAAC;QAED,cAAc;QACd,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;YACrC,MAAM,IAAI,KAAK,CAAC,SAAS,MAAM,iCAAiC,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC/F,CAAC;QACD,OAAO,CAAC,MAAM,CAAC,CAAC;IAClB,CAAC;IAED,kCAAkC;IAClC,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;QAC1B,MAAM,MAAM,GAAa,EAAE,CAAC;QAC5B,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;QAE/B,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;YAC1B,iCAAiC;YACjC,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;gBACvB,MAAM,OAAO,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,cAAc,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC;gBAC5E,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;oBACzB,MAAM,IAAI,KAAK,CAAC,6BAA6B,IAAI,uBAAuB,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;gBACvG,CAAC;gBACD,KAAK,MAAM,IAAI,IAAI,OAAO,EAAE,CAAC;oBAC3B,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;wBACpB,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;wBAClB,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;oBACjB,CAAC;gBACH,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,cAAc;gBACd,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;oBACnC,MAAM,IAAI,KAAK,CAAC,SAAS,IAAI,iCAAiC,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;gBAC7F,CAAC;gBACD,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;oBACpB,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;oBAClB,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;gBACjB,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,kBAAkB;IAClB,OAAO,cAAc,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;AACvC,CAAC"}

package/dist/lib/dashboard.d.ts

@@ -0,0 +1,65 @@
+/**
+ * Live terminal dashboard for parallel experiment runs.
+ * Uses log-update for in-place terminal rendering.
+ * Falls back to console.log for non-TTY environments.
+ */
+import type { ProgressEvent, ExperimentResults, Classification } from './types.js';
+/**
+ * Tracked state for a single experiment in the dashboard.
+ */
+interface ExperimentState {
+    name: string;
+    agent: string;
+    model: string;
+    totalEvals: number;
+    completedEvals: number;
+    passed: number;
+    failed: number;
+    runStartTime: number | null;
+    phase: 'waiting' | 'running' | 'classifying' | 'done';
+    /** Currently in-flight eval names */
+    activeEvals: Set<string>;
+    /** Recently completed eval results (kept for display, max 3) */
+    recentResults: Array<{
+        name: string;
+        status: 'passed' | 'failed';
+    }>;
+    /** Classification progress counters */
+    classifyingDone: number;
+    classifyingTotal: number;
+}
+/**
+ * Live terminal dashboard that renders experiment progress in-place.
+ */
+export declare class Dashboard {
+    private experiments;
+    private intervalId;
+    private experimentOrder;
+    private tick;
+    addExperiment(name: string, meta: {
+        agent: string;
+        model: string;
+        totalEvals: number;
+    }): void;
+    handleEvent(experimentName: string, event: ProgressEvent): void;
+    setPhase(experimentName: string, phase: 'classifying' | 'done'): void;
+    setClassifyingProgress(experimentName: string, done: number, total: number): void;
+    completeExperiment(experimentName: string, results: ExperimentResults, classifications: Map<string, Classification>): void;
+    start(): void;
+    stop(): void;
+    private render;
+}
+/**
+ * Render a permanent completed-experiment block that scrolls above the dashboard.
+ */
+export declare function renderCompletedBlock(experimentName: string, _state: ExperimentState, results: ExperimentResults, classifications: Map<string, Classification>): string;
+/**
+ * Console-based progress handler for non-TTY / single experiment mode.
+ */
+export declare function createConsoleProgressHandler(context: {
+    experimentName: string;
+    model: string;
+    agent: string;
+}): (event: ProgressEvent) => void;
+export {};
+//# sourceMappingURL=dashboard.d.ts.map

package/dist/lib/dashboard.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"dashboard.d.ts","sourceRoot":"","sources":["../../src/lib/dashboard.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAIH,OAAO,KAAK,EACV,aAAa,EACb,iBAAiB,EACjB,cAAc,EACf,MAAM,YAAY,CAAC;AASpB;;GAEG;AACH,UAAU,eAAe;IACvB,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,CAAC;IACnB,cAAc,EAAE,MAAM,CAAC;IACvB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,KAAK,EAAE,SAAS,GAAG,SAAS,GAAG,aAAa,GAAG,MAAM,CAAC;IACtD,qCAAqC;IACrC,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IACzB,gEAAgE;IAChE,aAAa,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,QAAQ,GAAG,QAAQ,CAAA;KAAE,CAAC,CAAC;IACpE,uCAAuC;IACvC,eAAe,EAAE,MAAM,CAAC;IACxB,gBAAgB,EAAE,MAAM,CAAC;CAC1B;AAED;;GAEG;AACH,qBAAa,SAAS;IACpB,OAAO,CAAC,WAAW,CAAsC;IACzD,OAAO,CAAC,UAAU,CAA6C;IAC/D,OAAO,CAAC,eAAe,CAAgB;IACvC,OAAO,CAAC,IAAI,CAAK;IAEjB,aAAa,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE;IAoBtF,WAAW,CAAC,cAAc,EAAE,MAAM,EAAE,KAAK,EAAE,aAAa;IA2BxD,QAAQ,CAAC,cAAc,EAAE,MAAM,EAAE,KAAK,EAAE,aAAa,GAAG,MAAM;IAO9D,sBAAsB,CAAC,cAAc,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM;IAQ1E,kBAAkB,CAChB,cAAc,EAAE,MAAM,EACtB,OAAO,EAAE,iBAAiB,EAC1B,eAAe,EAAE,GAAG,CAAC,MAAM,EAAE,cAAc,CAAC;IAY9C,KAAK;IAOL,IAAI;IAQJ,OAAO,CAAC,MAAM;CAiCf;AA0DD;;GAEG;AACH,wBAAgB,oBAAoB,CAClC,cAAc,EAAE,MAAM,EACtB,MAAM,EAAE,eAAe,EACvB,OAAO,EAAE,iBAAiB,EAC1B,eAAe,EAAE,GAAG,CAAC,MAAM,EAAE,cAAc,CAAC,GAC3C,MAAM,CA2CR;AAED;;GAEG;AACH,wBAAgB,4BAA4B,CAAC,OAAO,EAAE;IACpD,cAAc,EAAE,MAAM,CAAC;IACvB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;CACf,GAAG,CAAC,KAAK,EAAE,aAAa,KAAK,IAAI,CA+BjC"}

package/dist/lib/dashboard.js

@@ -0,0 +1,237 @@
+/**
+ * Live terminal dashboard for parallel experiment runs.
+ * Uses log-update for in-place terminal rendering.
+ * Falls back to console.log for non-TTY environments.
+ */
+import logUpdate from 'log-update';
+import chalk from 'chalk';
+import { formatRunResult, createProgressDisplay, formatResultsTable, } from './results.js';
+const SPINNER = ['\u280b', '\u2819', '\u2839', '\u2838', '\u283c', '\u2834', '\u2826', '\u2827', '\u2807', '\u280f'];
+/**
+ * Live terminal dashboard that renders experiment progress in-place.
+ */
+export class Dashboard {
+    experiments = new Map();
+    intervalId;
+    experimentOrder = [];
+    tick = 0;
+    addExperiment(name, meta) {
+        const state = {
+            name,
+            agent: meta.agent,
+            model: meta.model,
+            totalEvals: meta.totalEvals,
+            completedEvals: 0,
+            passed: 0,
+            failed: 0,
+            runStartTime: null,
+            phase: 'waiting',
+            activeEvals: new Set(),
+            recentResults: [],
+            classifyingDone: 0,
+            classifyingTotal: 0,
+        };
+        this.experiments.set(name, state);
+        this.experimentOrder.push(name);
+    }
+    handleEvent(experimentName, event) {
+        const state = this.experiments.get(experimentName);
+        if (!state)
+            return;
+        switch (event.type) {
+            case 'experiment:start':
+                state.totalEvals = event.totalAttempts;
+                state.phase = 'running';
+                state.runStartTime = Date.now();
+                break;
+            case 'eval:start':
+                state.activeEvals.add(event.evalName);
+                break;
+            case 'eval:complete':
+                state.activeEvals.delete(event.evalName);
+                state.completedEvals++;
+                if (event.result.status === 'passed') {
+                    state.passed++;
+                }
+                else {
+                    state.failed++;
+                }
+                state.recentResults.push({ name: event.evalName, status: event.result.status });
+                if (state.recentResults.length > 3)
+                    state.recentResults.shift();
+                break;
+        }
+    }
+    setPhase(experimentName, phase) {
+        const state = this.experiments.get(experimentName);
+        if (state) {
+            state.phase = phase;
+        }
+    }
+    setClassifyingProgress(experimentName, done, total) {
+        const state = this.experiments.get(experimentName);
+        if (state) {
+            state.classifyingDone = done;
+            state.classifyingTotal = total;
+        }
+    }
+    completeExperiment(experimentName, results, classifications) {
+        const state = this.experiments.get(experimentName);
+        if (!state)
+            return;
+        state.phase = 'done';
+        logUpdate.clear();
+        console.log(renderCompletedBlock(experimentName, state, results, classifications));
+        this.render();
+    }
+    start() {
+        this.intervalId = setInterval(() => {
+            this.tick++;
+            this.render();
+        }, 100);
+    }
+    stop() {
+        if (this.intervalId) {
+            clearInterval(this.intervalId);
+            this.intervalId = undefined;
+        }
+        logUpdate.clear();
+    }
+    render() {
+        const active = this.experimentOrder.filter((name) => this.experiments.get(name).phase !== 'done');
+        if (active.length === 0) {
+            logUpdate.clear();
+            return;
+        }
+        const totalExperiments = this.experimentOrder.length;
+        const completedExperiments = totalExperiments - active.length;
+        const maxNameLen = Math.max(...active.map((n) => n.length));
+        const nameWidth = Math.max(maxNameLen + 2, 20);
+        const spinner = SPINNER[this.tick % SPINNER.length];
+        const lines = [];
+        lines.push('');
+        lines.push(chalk.bold(` ${spinner} ${completedExperiments}/${totalExperiments} experiments complete`));
+        lines.push('');
+        for (const name of active) {
+            const state = this.experiments.get(name);
+            lines.push(renderExperimentLine(state, nameWidth));
+        }
+        lines.push('');
+        logUpdate(lines.join('\n'));
+    }
+}
+/**
+ * Render a single experiment's progress bar line.
+ */
+function renderExperimentLine(state, nameWidth) {
+    const nameCol = state.name.padEnd(nameWidth);
+    const elapsed = state.runStartTime ? Math.round((Date.now() - state.runStartTime) / 1000) : 0;
+    if (state.phase === 'waiting') {
+        const bar = renderBar(0, 1);
+        return ` ${chalk.gray(nameCol)} ${bar}  ${chalk.gray('waiting\u2026')}`;
+    }
+    if (state.phase === 'classifying') {
+        const bar = renderBar(state.totalEvals, state.totalEvals);
+        const stats = renderStats(state);
+        const classifyLabel = state.classifyingTotal > 0
+            ? `classifying ${state.classifyingDone}/${state.classifyingTotal}\u2026`
+            : 'classifying\u2026';
+        return ` ${chalk.cyan(nameCol)} ${bar}  ${stats} ${chalk.cyan('\u00b7')} ${chalk.cyan(classifyLabel)}`;
+    }
+    const bar = renderBar(state.completedEvals, state.totalEvals);
+    const stats = renderStats(state);
+    const time = chalk.gray(formatElapsed(elapsed));
+    return ` ${chalk.white(nameCol)} ${bar}  ${stats} ${chalk.gray('\u00b7')} ${time}`;
+}
+function renderStats(state) {
+    const parts = [];
+    parts.push(chalk.white(`${state.completedEvals}/${state.totalEvals}`));
+    if (state.passed > 0)
+        parts.push(chalk.green(`${state.passed}\u2713`));
+    if (state.failed > 0)
+        parts.push(chalk.red(`${state.failed}\u2717`));
+    return parts.join(' ');
+}
+/**
+ * Format elapsed seconds as "Xm Ys" for >= 60s, otherwise "Xs".
+ */
+function formatElapsed(seconds) {
+    if (seconds < 60)
+        return `${seconds}s`;
+    const m = Math.floor(seconds / 60);
+    const s = seconds % 60;
+    return `${m}m${s > 0 ? ` ${s}s` : ''}`;
+}
+/**
+ * Render a progress bar.
+ */
+function renderBar(completed, total) {
+    const width = 20;
+    const filled = total > 0 ? Math.min(width, Math.round((completed / total) * width)) : 0;
+    const empty = width - filled;
+    return chalk.green('\u2501'.repeat(filled)) + chalk.gray('\u2500'.repeat(empty));
+}
+/**
+ * Render a permanent completed-experiment block that scrolls above the dashboard.
+ */
+export function renderCompletedBlock(experimentName, _state, results, classifications) {
+    const lines = [];
+    const width = Math.min(process.stdout.columns || 80, 80);
+    const separator = '\u2500'.repeat(width);
+    lines.push(chalk.gray(separator));
+    const totalEvals = results.evals.length;
+    const passedEvals = results.evals.filter((e) => e.passedRuns > 0).length;
+    const elapsed = Math.round((new Date(results.completedAt).getTime() - new Date(results.startedAt).getTime()) / 1000);
+    const passRate = totalEvals > 0 ? Math.round((passedEvals / totalEvals) * 100) : 0;
+    const passColor = passRate === 100 ? chalk.green : passRate >= 50 ? chalk.yellow : chalk.red;
+    lines.push(` ${chalk.bold(experimentName)}  ${passColor(`${passedEvals}/${totalEvals} passed (${passRate}%)`)}  ${chalk.gray(formatElapsed(elapsed))}`);
+    const passed = results.evals.filter((e) => e.passedRuns > 0);
+    const failed = results.evals.filter((e) => e.passedRuns === 0);
+    if (passed.length > 0) {
+        lines.push(` ${passed.map((e) => chalk.green(`\u2713 ${e.name}`)).join('  ')}`);
+    }
+    if (failed.length > 0) {
+        lines.push(` ${failed.map((e) => chalk.red(`\u2717 ${e.name}`)).join('  ')}`);
+    }
+    if (failed.length > 0 && classifications.size > 0) {
+        for (const evalSummary of failed) {
+            const c = classifications.get(evalSummary.name);
+            if (c) {
+                const suffix = c.failureType !== 'model'
+                    ? chalk.gray(c.acknowledged ? ' (kept)' : ' (removed)')
+                    : '';
+                lines.push(chalk.gray(`   ${evalSummary.name}: ${c.failureType} \u2014 ${c.failureReason}${suffix}`));
+            }
+        }
+    }
+    lines.push(chalk.gray(separator));
+    return lines.join('\n');
+}
+/**
+ * Console-based progress handler for non-TTY / single experiment mode.
+ */
+export function createConsoleProgressHandler(context) {
+    return (event) => {
+        switch (event.type) {
+            case 'experiment:start':
+                console.log(`Starting ${event.totalAttempts} eval attempts concurrently (${event.totalEvals} evals \u00d7 ${event.totalRuns} runs)`);
+                break;
+            case 'eval:start':
+                console.log(createProgressDisplay(event.evalName, event.runNumber, event.totalRuns, context));
+                break;
+            case 'eval:complete':
+                console.log(formatRunResult(event.evalName, event.runNumber, event.totalRuns, event.result, context));
+                break;
+            case 'experiment:earlyExit':
+                console.log(`Early exit: ${event.evalName} passed on run ${event.runNumber}, aborting remaining attempts`);
+                break;
+            case 'experiment:saved':
+                console.log(`\nResults saved to: ${event.outputDir}`);
+                break;
+            case 'experiment:summary':
+                console.log(formatResultsTable(event.results));
+                break;
+        }
+    };
+}
+//# sourceMappingURL=dashboard.js.map

package/dist/lib/dashboard.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"dashboard.js","sourceRoot":"","sources":["../../src/lib/dashboard.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,SAAS,MAAM,YAAY,CAAC;AACnC,OAAO,KAAK,MAAM,OAAO,CAAC;AAM1B,OAAO,EACL,eAAe,EACf,qBAAqB,EACrB,kBAAkB,GACnB,MAAM,cAAc,CAAC;AAEtB,MAAM,OAAO,GAAG,CAAC,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC;AAwBrH;;GAEG;AACH,MAAM,OAAO,SAAS;IACZ,WAAW,GAAG,IAAI,GAAG,EAA2B,CAAC;IACjD,UAAU,CAA6C;IACvD,eAAe,GAAa,EAAE,CAAC;IAC/B,IAAI,GAAG,CAAC,CAAC;IAEjB,aAAa,CAAC,IAAY,EAAE,IAA0D;QACpF,MAAM,KAAK,GAAoB;YAC7B,IAAI;YACJ,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,cAAc,EAAE,CAAC;YACjB,MAAM,EAAE,CAAC;YACT,MAAM,EAAE,CAAC;YACT,YAAY,EAAE,IAAI;YAClB,KAAK,EAAE,SAAS;YAChB,WAAW,EAAE,IAAI,GAAG,EAAE;YACtB,aAAa,EAAE,EAAE;YACjB,eAAe,EAAE,CAAC;YAClB,gBAAgB,EAAE,CAAC;SACpB,CAAC;QACF,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;QAClC,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAClC,CAAC;IAED,WAAW,CAAC,cAAsB,EAAE,KAAoB;QACtD,MAAM,KAAK,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;QACnD,IAAI,CAAC,KAAK;YAAE,OAAO;QAEnB,QAAQ,KAAK,CAAC,IAAI,EAAE,CAAC;YACnB,KAAK,kBAAkB;gBACrB,KAAK,CAAC,UAAU,GAAG,KAAK,CAAC,aAAa,CAAC;gBACvC,KAAK,CAAC,KAAK,GAAG,SAAS,CAAC;gBACxB,KAAK,CAAC,YAAY,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;gBAChC,MAAM;YACR,KAAK,YAAY;gBACf,KAAK,CAAC,WAAW,CAAC,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;gBACtC,MAAM;YACR,KAAK,eAAe;gBAClB,KAAK,CAAC,WAAW,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;gBACzC,KAAK,CAAC,cAAc,EAAE,CAAC;gBACvB,IAAI,KAAK,CAAC,MAAM,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;oBACrC,KAAK,CAAC,MAAM,EAAE,CAAC;gBACjB,CAAC;qBAAM,CAAC;oBACN,KAAK,CAAC,MAAM,EAAE,CAAC;gBACjB,CAAC;gBACD,KAAK,CAAC,aAAa,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;gBAChF,IAAI,KAAK,CAAC,aAAa,CAAC,MAAM,GAAG,CAAC;oBAAE,KAAK,CAAC,aAAa,CAAC,KAAK,EAAE,CAAC;gBAChE,MAAM;QACV,CAAC;IACH,CAAC;IAED,QAAQ,CAAC,cAAsB,EAAE,KAA6B;QAC5D,MAAM,KAAK,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;QACnD,IAAI,KAAK,EAAE,CAAC;YACV,KAAK,CAAC,KAAK,GAAG,KAAK,CAAC;QACtB,CAAC;IACH,CAAC;IAED,sBAAsB,CAAC,cAAsB,EAAE,IAAY,EAAE,KAAa;QACxE,MAAM,KAAK,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;QACnD,IAAI,KAAK,EAAE,CAAC;YACV,KAAK,CAAC,eAAe,GAAG,IAAI,CAAC;YAC7B,KAAK,CAAC,gBAAgB,GAAG,KAAK,CAAC;QACjC,CAAC;IACH,CAAC;IAED,kBAAkB,CAChB,cAAsB,EACtB,OAA0B,EAC1B,eAA4C;QAE5C,MAAM,KAAK,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;QACnD,IAAI,CAAC,KAAK;YAAE,OAAO;QAEnB,KAAK,CAAC,KAAK,GAAG,MAAM,CAAC;QAErB,SAAS,CAAC,KAAK,EAAE,CAAC;QAClB,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,cAAc,EAAE,KAAK,EAAE,OAAO,EAAE,eAAe,CAAC,CAAC,CAAC;QACnF,IAAI,CAAC,MAAM,EAAE,CAAC;IAChB,CAAC;IAED,KAAK;QACH,IAAI,CAAC,UAAU,GAAG,WAAW,CAAC,GAAG,EAAE;YACjC,IAAI,CAAC,IAAI,EAAE,CAAC;YACZ,IAAI,CAAC,MAAM,EAAE,CAAC;QAChB,CAAC,EAAE,GAAG,CAAC,CAAC;IACV,CAAC;IAED,IAAI;QACF,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACpB,aAAa,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YAC/B,IAAI,CAAC,UAAU,GAAG,SAAS,CAAC;QAC9B,CAAC;QACD,SAAS,CAAC,KAAK,EAAE,CAAC;IACpB,CAAC;IAEO,MAAM;QACZ,MAAM,MAAM,GAAG,IAAI,CAAC,eAAe,CAAC,MAAM,CACxC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,IAAI,CAAE,CAAC,KAAK,KAAK,MAAM,CACvD,CAAC;QAEF,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACxB,SAAS,CAAC,KAAK,EAAE,CAAC;YAClB,OAAO;QACT,CAAC;QAED,MAAM,gBAAgB,GAAG,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC;QACrD,MAAM,oBAAoB,GAAG,gBAAgB,GAAG,MAAM,CAAC,MAAM,CAAC;QAE9D,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;QAC5D,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,UAAU,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC;QAE/C,MAAM,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;QAEpD,MAAM,KAAK,GAAa,EAAE,CAAC;QAC3B,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,CAAC,IAAI,CACR,KAAK,CAAC,IAAI,CAAC,IAAI,OAAO,IAAI,oBAAoB,IAAI,gBAAgB,uBAAuB,CAAC,CAC3F,CAAC;QACF,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAEf,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;YAC1B,MAAM,KAAK,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,IAAI,CAAE,CAAC;YAC1C,KAAK,CAAC,IAAI,CAAC,oBAAoB,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC,CAAC;QACrD,CAAC;QAED,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;IAC9B,CAAC;CACF;AAED;;GAEG;AACH,SAAS,oBAAoB,CAAC,KAAsB,EAAE,SAAiB;IACrE,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IAC7C,MAAM,OAAO,GAAG,KAAK,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,YAAY,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAE9F,IAAI,KAAK,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;QAC9B,MAAM,GAAG,GAAG,SAAS,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QAC5B,OAAO,IAAI,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,GAAG,KAAK,KAAK,CAAC,IAAI,CAAC,eAAe,CAAC,EAAE,CAAC;IAC1E,CAAC;IAED,IAAI,KAAK,CAAC,KAAK,KAAK,aAAa,EAAE,CAAC;QAClC,MAAM,GAAG,GAAG,SAAS,CAAC,KAAK,CAAC,UAAU,EAAE,KAAK,CAAC,UAAU,CAAC,CAAC;QAC1D,MAAM,KAAK,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;QACjC,MAAM,aAAa,GAAG,KAAK,CAAC,gBAAgB,GAAG,CAAC;YAC9C,CAAC,CAAC,eAAe,KAAK,CAAC,eAAe,IAAI,KAAK,CAAC,gBAAgB,QAAQ;YACxE,CAAC,CAAC,mBAAmB,CAAC;QACxB,OAAO,IAAI,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,GAAG,KAAK,KAAK,IAAI,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,aAAa,CAAC,EAAE,CAAC;IACzG,CAAC;IAED,MAAM,GAAG,GAAG,SAAS,CAAC,KAAK,CAAC,cAAc,EAAE,KAAK,CAAC,UAAU,CAAC,CAAC;IAC9D,MAAM,KAAK,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;IACjC,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC;IAEhD,OAAO,IAAI,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,GAAG,KAAK,KAAK,IAAI,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,IAAI,EAAE,CAAC;AACrF,CAAC;AAED,SAAS,WAAW,CAAC,KAAsB;IACzC,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,KAAK,CAAC,cAAc,IAAI,KAAK,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;IACvE,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;QAAE,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,KAAK,CAAC,MAAM,QAAQ,CAAC,CAAC,CAAC;IACvE,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;QAAE,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,MAAM,QAAQ,CAAC,CAAC,CAAC;IACrE,OAAO,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AACzB,CAAC;AAED;;GAEG;AACH,SAAS,aAAa,CAAC,OAAe;IACpC,IAAI,OAAO,GAAG,EAAE;QAAE,OAAO,GAAG,OAAO,GAAG,CAAC;IACvC,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,GAAG,EAAE,CAAC,CAAC;IACnC,MAAM,CAAC,GAAG,OAAO,GAAG,EAAE,CAAC;IACvB,OAAO,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;AACzC,CAAC;AAED;;GAEG;AACH,SAAS,SAAS,CAAC,SAAiB,EAAE,KAAa;IACjD,MAAM,KAAK,GAAG,EAAE,CAAC;IACjB,MAAM,MAAM,GAAG,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,SAAS,GAAG,KAAK,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACxF,MAAM,KAAK,GAAG,KAAK,GAAG,MAAM,CAAC;IAC7B,OAAO,KAAK,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;AACnF,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,oBAAoB,CAClC,cAAsB,EACtB,MAAuB,EACvB,OAA0B,EAC1B,eAA4C;IAE5C,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC;IACzD,MAAM,SAAS,GAAG,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAEzC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC;IAElC,MAAM,UAAU,GAAG,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC;IACxC,MAAM,WAAW,GAAG,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC;IACzE,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CACxB,CAAC,IAAI,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,OAAO,EAAE,GAAG,IAAI,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE,CAAC,GAAG,IAAI,CACzF,CAAC;IACF,MAAM,QAAQ,GAAG,UAAU,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,WAAW,GAAG,UAAU,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACnF,MAAM,SAAS,GAAG,QAAQ,KAAK,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC;IAE7F,KAAK,CAAC,IAAI,CACR,IAAI,KAAK,CAAC,IAAI,CAAC,cAAc,CAAC,KAAK,SAAS,CAAC,GAAG,WAAW,IAAI,UAAU,YAAY,QAAQ,IAAI,CAAC,KAAK,KAAK,CAAC,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC,EAAE,CAC5I,CAAC;IAEF,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,GAAG,CAAC,CAAC,CAAC;IAC7D,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,KAAK,CAAC,CAAC,CAAC;IAE/D,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACtB,KAAK,CAAC,IAAI,CAAC,IAAI,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAClF,CAAC;IACD,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACtB,KAAK,CAAC,IAAI,CAAC,IAAI,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAChF,CAAC;IAED,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,IAAI,eAAe,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;QAClD,KAAK,MAAM,WAAW,IAAI,MAAM,EAAE,CAAC;YACjC,MAAM,CAAC,GAAG,eAAe,CAAC,GAAG,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;YAChD,IAAI,CAAC,EAAE,CAAC;gBACN,MAAM,MAAM,GAAG,CAAC,CAAC,WAAW,KAAK,OAAO;oBACtC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,YAAY,CAAC;oBACvD,CAAC,CAAC,EAAE,CAAC;gBACP,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,WAAW,CAAC,IAAI,KAAK,CAAC,CAAC,WAAW,WAAW,CAAC,CAAC,aAAa,GAAG,MAAM,EAAE,CAAC,CAAC,CAAC;YACxG,CAAC;QACH,CAAC;IACH,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC;IAClC,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,4BAA4B,CAAC,OAI5C;IACC,OAAO,CAAC,KAAoB,EAAE,EAAE;QAC9B,QAAQ,KAAK,CAAC,IAAI,EAAE,CAAC;YACnB,KAAK,kBAAkB;gBACrB,OAAO,CAAC,GAAG,CACT,YAAY,KAAK,CAAC,aAAa,gCAAgC,KAAK,CAAC,UAAU,iBAAiB,KAAK,CAAC,SAAS,QAAQ,CACxH,CAAC;gBACF,MAAM;YACR,KAAK,YAAY;gBACf,OAAO,CAAC,GAAG,CACT,qBAAqB,CAAC,KAAK,CAAC,QAAQ,EAAE,KAAK,CAAC,SAAS,EAAE,KAAK,CAAC,SAAS,EAAE,OAAO,CAAC,CACjF,CAAC;gBACF,MAAM;YACR,KAAK,eAAe;gBAClB,OAAO,CAAC,GAAG,CACT,eAAe,CAAC,KAAK,CAAC,QAAQ,EAAE,KAAK,CAAC,SAAS,EAAE,KAAK,CAAC,SAAS,EAAE,KAAK,CAAC,MAAM,EAAE,OAAO,CAAC,CACzF,CAAC;gBACF,MAAM;YACR,KAAK,sBAAsB;gBACzB,OAAO,CAAC,GAAG,CACT,eAAe,KAAK,CAAC,QAAQ,kBAAkB,KAAK,CAAC,SAAS,+BAA+B,CAC9F,CAAC;gBACF,MAAM;YACR,KAAK,kBAAkB;gBACrB,OAAO,CAAC,GAAG,CAAC,uBAAuB,KAAK,CAAC,SAAS,EAAE,CAAC,CAAC;gBACtD,MAAM;YACR,KAAK,oBAAoB;gBACvB,OAAO,CAAC,GAAG,CAAC,kBAAkB,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;gBAC/C,MAAM;QACV,CAAC;IACH,CAAC,CAAC;AACJ,CAAC"}

package/dist/lib/docker-sandbox.d.ts

@@ -0,0 +1,92 @@
+/**
+ * Docker-based sandbox implementation for isolated eval execution.
+ * Uses dockerode to manage Docker containers as sandboxes.
+ */
+import Docker from 'dockerode';
+import type { Sandbox } from './types.js';
+import type { CommandResult, SandboxFile } from './sandbox.js';
+export declare function resolveDockerOptions(): Docker.DockerOptions;
+export interface DockerSandboxOptions {
+    /** Timeout in milliseconds */
+    timeout?: number;
+    /** Runtime environment */
+    runtime?: 'node20' | 'node24';
+}
+/**
+ * Docker-based sandbox manager.
+ * Creates isolated containers for running evals.
+ */
+export declare class DockerSandboxManager implements Sandbox {
+    private docker;
+    private container;
+    private _containerId;
+    private timeout;
+    private runtime;
+    constructor(options?: DockerSandboxOptions);
+    /**
+     * Create and start a new Docker sandbox.
+     */
+    static create(options?: DockerSandboxOptions): Promise<DockerSandboxManager>;
+    /**
+     * Initialize the sandbox by pulling image and creating container.
+     */
+    private initialize;
+    /**
+     * Ensure the Docker image is available locally, pulling if needed.
+     */
+    private ensureImage;
+    /**
+     * Pull a Docker image with progress output.
+     */
+    private pullImage;
+    /**
+     * Get the container ID.
+     */
+    get sandboxId(): string;
+    /**
+     * Run a command in the container as the sandbox (non-root) user.
+     */
+    runCommand(command: string, args?: string[], options?: {
+        env?: Record<string, string>;
+    }): Promise<CommandResult>;
+    /**
+     * Run a command in the container as root.
+     * Used internally for setup tasks.
+     */
+    private runCommandAsRoot;
+    /**
+     * Execute a command in the container.
+     */
+    private execCommand;
+    /**
+     * Run a shell command (through bash).
+     */
+    runShell(command: string, env?: Record<string, string>): Promise<CommandResult>;
+    /**
+     * Read a file from the container.
+     */
+    readFile(path: string): Promise<string>;
+    /**
+     * Check if a file exists in the container.
+     */
+    fileExists(path: string): Promise<boolean>;
+    /**
+     * Write files to the container.
+     */
+    writeFiles(files: Record<string, string>): Promise<void>;
+    /**
+     * Upload files to the container using tar archive.
+     */
+    uploadFiles(files: SandboxFile[]): Promise<void>;
+    uploadFilesToPath(containerDir: string, files: SandboxFile[]): Promise<void>;
+    ensureUserOwnership(containerDir: string): Promise<void>;
+    /**
+     * Get the working directory.
+     */
+    getWorkingDirectory(): string;
+    /**
+     * Stop and clean up the container.
+     */
+    stop(): Promise<void>;
+}
+//# sourceMappingURL=docker-sandbox.d.ts.map

package/dist/lib/docker-sandbox.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"docker-sandbox.d.ts","sourceRoot":"","sources":["../../src/lib/docker-sandbox.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,MAAM,MAAM,WAAW,CAAC;AAG/B,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAC1C,OAAO,KAAK,EAAE,aAAa,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAmD/D,wBAAgB,oBAAoB,IAAI,MAAM,CAAC,aAAa,CAS3D;AAED,MAAM,WAAW,oBAAoB;IACnC,8BAA8B;IAC9B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,0BAA0B;IAC1B,OAAO,CAAC,EAAE,QAAQ,GAAG,QAAQ,CAAC;CAC/B;AAED;;;GAGG;AACH,qBAAa,oBAAqB,YAAW,OAAO;IAClD,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,SAAS,CAAiC;IAClD,OAAO,CAAC,YAAY,CAAc;IAClC,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,OAAO,CAAS;gBAEZ,OAAO,GAAE,oBAAyB;IAM9C;;OAEG;WACU,MAAM,CAAC,OAAO,GAAE,oBAAyB,GAAG,OAAO,CAAC,oBAAoB,CAAC;IAMtF;;OAEG;YACW,UAAU;IAyCxB;;OAEG;YACW,WAAW;IAazB;;OAEG;YACW,SAAS;IAyBvB;;OAEG;IACH,IAAI,SAAS,IAAI,MAAM,CAEtB;IAED;;OAEG;IACG,UAAU,CACd,OAAO,EAAE,MAAM,EACf,IAAI,GAAE,MAAM,EAAO,EACnB,OAAO,GAAE;QAAE,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;KAAO,GAC7C,OAAO,CAAC,aAAa,CAAC;IAazB;;;OAGG;YACW,gBAAgB;IAW9B;;OAEG;YACW,WAAW;IAiGzB;;OAEG;IACG,QAAQ,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,OAAO,CAAC,aAAa,CAAC;IAIrF;;OAEG;IACG,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAQ7C;;OAEG;IACG,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAKhD;;OAEG;IACG,UAAU,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IAS9D;;OAEG;IACG,WAAW,CAAC,KAAK,EAAE,WAAW,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IA8BhD,iBAAiB,CAAC,YAAY,EAAE,MAAM,EAAE,KAAK,EAAE,WAAW,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAyB5E,mBAAmB,CAAC,YAAY,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAI9D;;OAEG;IACH,mBAAmB,IAAI,MAAM;IAI7B;;OAEG;IACG,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;CAU5B"}