agent-eval-opencode 0.10.0

package/dist/lib/docker-sandbox.js

@@ -0,0 +1,375 @@
+/**
+ * Docker-based sandbox implementation for isolated eval execution.
+ * Uses dockerode to manage Docker containers as sandboxes.
+ */
+import Docker from 'dockerode';
+import { execSync } from 'child_process';
+import * as tar from 'tar-stream';
+/**
+ * Docker images for each Node.js runtime.
+ * Using -slim variants for faster downloads while maintaining compatibility.
+ */
+const DOCKER_IMAGES = {
+    node20: 'node:20-slim',
+    node24: 'node:24-slim',
+};
+/**
+ * Default timeout for container operations (10 minutes).
+ */
+const DEFAULT_TIMEOUT = 600000;
+/**
+ * Working directory inside the container.
+ */
+const CONTAINER_WORKDIR = '/home/sandbox/workspace';
+/**
+ * Non-root user configuration.
+ * Running as non-root is important for security and compatibility
+ * (e.g., Claude Code refuses --dangerously-skip-permissions as root).
+ * Node.js images already have a 'node' user with UID/GID 1000.
+ */
+const SANDBOX_UID = 1000;
+const SANDBOX_GID = 1000;
+/**
+ * Directory for npm global packages (non-root install location).
+ */
+const NPM_GLOBAL_DIR = '/home/node/.npm-global';
+function resolveDockerHost() {
+    if (process.env.DOCKER_HOST) {
+        return process.env.DOCKER_HOST;
+    }
+    try {
+        return execSync('docker context inspect --format {{.Endpoints.docker.Host}}', {
+            stdio: ['ignore', 'pipe', 'ignore'],
+            timeout: 5000,
+        })
+            .toString()
+            .trim();
+    }
+    catch {
+        return undefined;
+    }
+}
+export function resolveDockerOptions() {
+    const host = resolveDockerHost();
+    if (host?.startsWith('unix://')) {
+        return { socketPath: host.slice('unix://'.length) };
+    }
+    if (host) {
+        return { host };
+    }
+    return {};
+}
+/**
+ * Docker-based sandbox manager.
+ * Creates isolated containers for running evals.
+ */
+export class DockerSandboxManager {
+    docker;
+    container = null;
+    _containerId = '';
+    timeout;
+    runtime;
+    constructor(options = {}) {
+        this.docker = new Docker(resolveDockerOptions());
+        this.timeout = options.timeout ?? DEFAULT_TIMEOUT;
+        this.runtime = options.runtime ?? 'node24';
+    }
+    /**
+     * Create and start a new Docker sandbox.
+     */
+    static async create(options = {}) {
+        const manager = new DockerSandboxManager(options);
+        await manager.initialize();
+        return manager;
+    }
+    /**
+     * Initialize the sandbox by pulling image and creating container.
+     */
+    async initialize() {
+        const imageName = DOCKER_IMAGES[this.runtime];
+        if (!imageName) {
+            throw new Error(`Unsupported runtime: ${this.runtime}`);
+        }
+        // Ensure the image is available
+        await this.ensureImage(imageName);
+        this.container = await this.docker.createContainer({
+            Image: imageName,
+            Cmd: ['sleep', 'infinity'],
+            WorkingDir: CONTAINER_WORKDIR,
+            Tty: true,
+            HostConfig: {
+                AutoRemove: true,
+            },
+        });
+        this._containerId = this.container.id;
+        // Start the container
+        await this.container.start();
+        // Install CA certificates and git (slim images may not include them)
+        await this.runCommandAsRoot('bash', ['-c', 'apt-get update -qq && apt-get install -y -qq ca-certificates git > /dev/null 2>&1']);
+        // Create workspace directory owned by the non-root user (node:node in Node.js images)
+        // The node user (UID 1000) already exists in node:*-slim images
+        await this.runCommandAsRoot('mkdir', ['-p', CONTAINER_WORKDIR]);
+        await this.runCommandAsRoot('chown', ['-R', `${SANDBOX_UID}:${SANDBOX_GID}`, CONTAINER_WORKDIR]);
+        // Configure npm for non-root global installs
+        // Create a user-local directory for global packages
+        await this.runCommandAsRoot('mkdir', ['-p', NPM_GLOBAL_DIR]);
+        await this.runCommandAsRoot('chown', ['-R', `${SANDBOX_UID}:${SANDBOX_GID}`, NPM_GLOBAL_DIR]);
+        // Configure npm to use this directory
+        await this.runCommand('npm', ['config', 'set', 'prefix', NPM_GLOBAL_DIR]);
+    }
+    /**
+     * Ensure the Docker image is available locally, pulling if needed.
+     */
+    async ensureImage(imageName) {
+        try {
+            // Check if image exists
+            const image = this.docker.getImage(imageName);
+            await image.inspect();
+        }
+        catch {
+            // Image doesn't exist, pull it
+            console.log(`Pulling Docker image: ${imageName}...`);
+            await this.pullImage(imageName);
+            console.log(`Docker image ready: ${imageName}`);
+        }
+    }
+    /**
+     * Pull a Docker image with progress output.
+     */
+    async pullImage(imageName) {
+        return new Promise((resolve, reject) => {
+            this.docker.pull(imageName, (err, stream) => {
+                if (err) {
+                    reject(err);
+                    return;
+                }
+                // Follow the pull progress
+                this.docker.modem.followProgress(stream, (err) => {
+                    if (err) {
+                        reject(err);
+                    }
+                    else {
+                        resolve();
+                    }
+                }, 
+                // Progress callback (optional, could add progress bar here)
+                () => { });
+            });
+        });
+    }
+    /**
+     * Get the container ID.
+     */
+    get sandboxId() {
+        return this._containerId.slice(0, 12); // Short ID like Docker CLI
+    }
+    /**
+     * Run a command in the container as the sandbox (non-root) user.
+     */
+    async runCommand(command, args = [], options = {}) {
+        // Ensure npm global binaries are in PATH
+        const env = {
+            ...options.env,
+            PATH: `${NPM_GLOBAL_DIR}/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin`,
+        };
+        return this.execCommand(command, args, {
+            env,
+            user: `${SANDBOX_UID}:${SANDBOX_GID}`,
+        });
+    }
+    /**
+     * Run a command in the container as root.
+     * Used internally for setup tasks.
+     */
+    async runCommandAsRoot(command, args = [], options = {}) {
+        return this.execCommand(command, args, {
+            ...options,
+            user: 'root',
+        });
+    }
+    /**
+     * Execute a command in the container.
+     */
+    async execCommand(command, args = [], options = {}) {
+        if (!this.container) {
+            throw new Error('Container not initialized');
+        }
+        const cmd = [command, ...args];
+        const env = options.env
+            ? Object.entries(options.env).map(([k, v]) => `${k}=${v}`)
+            : undefined;
+        const exec = await this.container.exec({
+            Cmd: cmd,
+            AttachStdout: true,
+            AttachStderr: true,
+            WorkingDir: CONTAINER_WORKDIR,
+            Env: env,
+            User: options.user,
+        });
+        const stream = await exec.start({ hijack: true, stdin: false });
+        return new Promise((resolve, reject) => {
+            let stdout = '';
+            let stderr = '';
+            // Docker multiplexes stdout/stderr in the stream
+            // We need to demux it
+            const stdoutChunks = [];
+            const stderrChunks = [];
+            stream.on('data', (chunk) => {
+                // Docker stream format: 8-byte header + payload
+                // Header: [stream_type (1 byte), 0, 0, 0, size (4 bytes)]
+                // stream_type: 1 = stdout, 2 = stderr
+                let offset = 0;
+                while (offset < chunk.length) {
+                    if (offset + 8 > chunk.length) {
+                        // Incomplete header, treat rest as stdout
+                        stdoutChunks.push(chunk.slice(offset));
+                        break;
+                    }
+                    const streamType = chunk[offset];
+                    const size = chunk.readUInt32BE(offset + 4);
+                    if (offset + 8 + size > chunk.length) {
+                        // Incomplete payload, treat rest as stdout
+                        stdoutChunks.push(chunk.slice(offset + 8));
+                        break;
+                    }
+                    const payload = chunk.slice(offset + 8, offset + 8 + size);
+                    if (streamType === 1) {
+                        stdoutChunks.push(payload);
+                    }
+                    else if (streamType === 2) {
+                        stderrChunks.push(payload);
+                    }
+                    else {
+                        // Unknown type, assume stdout
+                        stdoutChunks.push(payload);
+                    }
+                    offset += 8 + size;
+                }
+            });
+            stream.on('end', async () => {
+                stdout = Buffer.concat(stdoutChunks).toString('utf-8');
+                stderr = Buffer.concat(stderrChunks).toString('utf-8');
+                try {
+                    const inspection = await exec.inspect();
+                    resolve({
+                        stdout,
+                        stderr,
+                        exitCode: inspection.ExitCode ?? 0,
+                    });
+                }
+                catch (error) {
+                    reject(error);
+                }
+            });
+            stream.on('error', reject);
+            // Timeout handling
+            const timeoutId = setTimeout(() => {
+                stream.destroy();
+                reject(new Error(`Command timed out after ${this.timeout}ms`));
+            }, this.timeout);
+            stream.on('end', () => clearTimeout(timeoutId));
+        });
+    }
+    /**
+     * Run a shell command (through bash).
+     */
+    async runShell(command, env) {
+        return this.runCommand('bash', ['-c', command], { env });
+    }
+    /**
+     * Read a file from the container.
+     */
+    async readFile(path) {
+        const result = await this.runCommand('cat', [path]);
+        if (result.exitCode !== 0) {
+            throw new Error(`Failed to read file ${path}: ${result.stderr}`);
+        }
+        return result.stdout;
+    }
+    /**
+     * Check if a file exists in the container.
+     */
+    async fileExists(path) {
+        const result = await this.runCommand('test', ['-f', path]);
+        return result.exitCode === 0;
+    }
+    /**
+     * Write files to the container.
+     */
+    async writeFiles(files) {
+        const sandboxFiles = Object.entries(files).map(([path, content]) => ({
+            path,
+            content: Buffer.from(content, 'utf-8'),
+        }));
+        await this.uploadFiles(sandboxFiles);
+    }
+    /**
+     * Upload files to the container using tar archive.
+     */
+    async uploadFiles(files) {
+        if (!this.container) {
+            throw new Error('Container not initialized');
+        }
+        if (files.length === 0) {
+            return;
+        }
+        // Create a tar archive
+        const pack = tar.pack();
+        for (const file of files) {
+            const content = typeof file.content === 'string'
+                ? Buffer.from(file.content, 'utf-8')
+                : file.content;
+            pack.entry({ name: file.path }, content);
+        }
+        pack.finalize();
+        // Upload to container
+        await this.container.putArchive(pack, { path: CONTAINER_WORKDIR });
+        // Fix ownership - putArchive uploads as root, but we need files owned by node user
+        // so that OpenCode and other agents can edit them
+        await this.runCommandAsRoot('chown', ['-R', `${SANDBOX_UID}:${SANDBOX_GID}`, CONTAINER_WORKDIR]);
+    }
+    async uploadFilesToPath(containerDir, files) {
+        if (!this.container) {
+            throw new Error('Container not initialized');
+        }
+        await this.runCommandAsRoot('mkdir', ['-p', containerDir]);
+        await this.runCommandAsRoot('chown', ['-R', `${SANDBOX_UID}:${SANDBOX_GID}`, containerDir]);
+        if (files.length === 0) {
+            return;
+        }
+        const pack = tar.pack();
+        for (const file of files) {
+            const content = typeof file.content === 'string'
+                ? Buffer.from(file.content, 'utf-8')
+                : file.content;
+            pack.entry({ name: file.path }, content);
+        }
+        pack.finalize();
+        await this.container.putArchive(pack, { path: containerDir });
+        await this.runCommandAsRoot('chown', ['-R', `${SANDBOX_UID}:${SANDBOX_GID}`, containerDir]);
+    }
+    async ensureUserOwnership(containerDir) {
+        await this.runCommandAsRoot('chown', ['-R', `${SANDBOX_UID}:${SANDBOX_GID}`, containerDir]);
+    }
+    /**
+     * Get the working directory.
+     */
+    getWorkingDirectory() {
+        return CONTAINER_WORKDIR;
+    }
+    /**
+     * Stop and clean up the container.
+     */
+    async stop() {
+        if (this.container) {
+            try {
+                await this.container.stop({ t: 0 }); // Immediate stop
+            }
+            catch {
+                // Container may already be stopped or removed
+            }
+            this.container = null;
+        }
+    }
+}
+//# sourceMappingURL=docker-sandbox.js.map

package/dist/lib/docker-sandbox.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"docker-sandbox.js","sourceRoot":"","sources":["../../src/lib/docker-sandbox.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,MAAM,MAAM,WAAW,CAAC;AAC/B,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AACzC,OAAO,KAAK,GAAG,MAAM,YAAY,CAAC;AAIlC;;;GAGG;AACH,MAAM,aAAa,GAA2B;IAC5C,MAAM,EAAE,cAAc;IACtB,MAAM,EAAE,cAAc;CACvB,CAAC;AAEF;;GAEG;AACH,MAAM,eAAe,GAAG,MAAM,CAAC;AAE/B;;GAEG;AACH,MAAM,iBAAiB,GAAG,yBAAyB,CAAC;AAEpD;;;;;GAKG;AACH,MAAM,WAAW,GAAG,IAAI,CAAC;AACzB,MAAM,WAAW,GAAG,IAAI,CAAC;AAEzB;;GAEG;AACH,MAAM,cAAc,GAAG,wBAAwB,CAAC;AAEhD,SAAS,iBAAiB;IACxB,IAAI,OAAO,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC;QAC5B,OAAO,OAAO,CAAC,GAAG,CAAC,WAAW,CAAC;IACjC,CAAC;IACD,IAAI,CAAC;QACH,OAAO,QAAQ,CAAC,4DAA4D,EAAE;YAC5E,KAAK,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,QAAQ,CAAC;YACnC,OAAO,EAAE,IAAI;SACd,CAAC;aACC,QAAQ,EAAE;aACV,IAAI,EAAE,CAAC;IACZ,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,SAAS,CAAC;IACnB,CAAC;AACH,CAAC;AAED,MAAM,UAAU,oBAAoB;IAClC,MAAM,IAAI,GAAG,iBAAiB,EAAE,CAAC;IACjC,IAAI,IAAI,EAAE,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;QAChC,OAAO,EAAE,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;IACtD,CAAC;IACD,IAAI,IAAI,EAAE,CAAC;QACT,OAAO,EAAE,IAAI,EAAE,CAAC;IAClB,CAAC;IACD,OAAO,EAAE,CAAC;AACZ,CAAC;AASD;;;GAGG;AACH,MAAM,OAAO,oBAAoB;IACvB,MAAM,CAAS;IACf,SAAS,GAA4B,IAAI,CAAC;IAC1C,YAAY,GAAW,EAAE,CAAC;IAC1B,OAAO,CAAS;IAChB,OAAO,CAAS;IAExB,YAAY,UAAgC,EAAE;QAC5C,IAAI,CAAC,MAAM,GAAG,IAAI,MAAM,CAAC,oBAAoB,EAAE,CAAC,CAAC;QACjD,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,eAAe,CAAC;QAClD,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,QAAQ,CAAC;IAC7C,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,UAAgC,EAAE;QACpD,MAAM,OAAO,GAAG,IAAI,oBAAoB,CAAC,OAAO,CAAC,CAAC;QAClD,MAAM,OAAO,CAAC,UAAU,EAAE,CAAC;QAC3B,OAAO,OAAO,CAAC;IACjB,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,UAAU;QACtB,MAAM,SAAS,GAAG,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAC9C,IAAI,CAAC,SAAS,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CAAC,wBAAwB,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;QAC1D,CAAC;QAED,gCAAgC;QAChC,MAAM,IAAI,CAAC,WAAW,CAAC,SAAS,CAAC,CAAC;QAElC,IAAI,CAAC,SAAS,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,eAAe,CAAC;YACjD,KAAK,EAAE,SAAS;YAChB,GAAG,EAAE,CAAC,OAAO,EAAE,UAAU,CAAC;YAC1B,UAAU,EAAE,iBAAiB;YAC7B,GAAG,EAAE,IAAI;YACT,UAAU,EAAE;gBACV,UAAU,EAAE,IAAI;aACjB;SACF,CAAC,CAAC;QAEH,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;QAEtC,sBAAsB;QACtB,MAAM,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC;QAE7B,qEAAqE;QACrE,MAAM,IAAI,CAAC,gBAAgB,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,mFAAmF,CAAC,CAAC,CAAC;QAEjI,sFAAsF;QACtF,gEAAgE;QAChE,MAAM,IAAI,CAAC,gBAAgB,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,iBAAiB,CAAC,CAAC,CAAC;QAChE,MAAM,IAAI,CAAC,gBAAgB,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,GAAG,WAAW,IAAI,WAAW,EAAE,EAAE,iBAAiB,CAAC,CAAC,CAAC;QAEjG,6CAA6C;QAC7C,oDAAoD;QACpD,MAAM,IAAI,CAAC,gBAAgB,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,cAAc,CAAC,CAAC,CAAC;QAC7D,MAAM,IAAI,CAAC,gBAAgB,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,GAAG,WAAW,IAAI,WAAW,EAAE,EAAE,cAAc,CAAC,CAAC,CAAC;QAE9F,sCAAsC;QACtC,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,cAAc,CAAC,CAAC,CAAC;IAC5E,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,WAAW,CAAC,SAAiB;QACzC,IAAI,CAAC;YACH,wBAAwB;YACxB,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;YAC9C,MAAM,KAAK,CAAC,OAAO,EAAE,CAAC;QACxB,CAAC;QAAC,MAAM,CAAC;YACP,+BAA+B;YAC/B,OAAO,CAAC,GAAG,CAAC,yBAAyB,SAAS,KAAK,CAAC,CAAC;YACrD,MAAM,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC;YAChC,OAAO,CAAC,GAAG,CAAC,uBAAuB,SAAS,EAAE,CAAC,CAAC;QAClD,CAAC;IACH,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,SAAS,CAAC,SAAiB;QACvC,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACrC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC,GAAiB,EAAE,MAA6B,EAAE,EAAE;gBAC/E,IAAI,GAAG,EAAE,CAAC;oBACR,MAAM,CAAC,GAAG,CAAC,CAAC;oBACZ,OAAO;gBACT,CAAC;gBAED,2BAA2B;gBAC3B,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,cAAc,CAC9B,MAAM,EACN,CAAC,GAAiB,EAAE,EAAE;oBACpB,IAAI,GAAG,EAAE,CAAC;wBACR,MAAM,CAAC,GAAG,CAAC,CAAC;oBACd,CAAC;yBAAM,CAAC;wBACN,OAAO,EAAE,CAAC;oBACZ,CAAC;gBACH,CAAC;gBACD,4DAA4D;gBAC5D,GAAG,EAAE,GAAE,CAAC,CACT,CAAC;YACJ,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACH,IAAI,SAAS;QACX,OAAO,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,2BAA2B;IACpE,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,UAAU,CACd,OAAe,EACf,OAAiB,EAAE,EACnB,UAA4C,EAAE;QAE9C,yCAAyC;QACzC,MAAM,GAAG,GAAG;YACV,GAAG,OAAO,CAAC,GAAG;YACd,IAAI,EAAE,GAAG,cAAc,mEAAmE;SAC3F,CAAC;QAEF,OAAO,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,IAAI,EAAE;YACrC,GAAG;YACH,IAAI,EAAE,GAAG,WAAW,IAAI,WAAW,EAAE;SACtC,CAAC,CAAC;IACL,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,gBAAgB,CAC5B,OAAe,EACf,OAAiB,EAAE,EACnB,UAA4C,EAAE;QAE9C,OAAO,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,IAAI,EAAE;YACrC,GAAG,OAAO;YACV,IAAI,EAAE,MAAM;SACb,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,WAAW,CACvB,OAAe,EACf,OAAiB,EAAE,EACnB,UAA2D,EAAE;QAE7D,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;YACpB,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC;QAC/C,CAAC;QAED,MAAM,GAAG,GAAG,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,CAAC;QAC/B,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG;YACrB,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;YAC1D,CAAC,CAAC,SAAS,CAAC;QAEd,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;YACrC,GAAG,EAAE,GAAG;YACR,YAAY,EAAE,IAAI;YAClB,YAAY,EAAE,IAAI;YAClB,UAAU,EAAE,iBAAiB;YAC7B,GAAG,EAAE,GAAG;YACR,IAAI,EAAE,OAAO,CAAC,IAAI;SACnB,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC;QAEhE,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACrC,IAAI,MAAM,GAAG,EAAE,CAAC;YAChB,IAAI,MAAM,GAAG,EAAE,CAAC;YAEhB,iDAAiD;YACjD,sBAAsB;YACtB,MAAM,YAAY,GAAa,EAAE,CAAC;YAClC,MAAM,YAAY,GAAa,EAAE,CAAC;YAElC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,KAAa,EAAE,EAAE;gBAClC,gDAAgD;gBAChD,0DAA0D;gBAC1D,sCAAsC;gBACtC,IAAI,MAAM,GAAG,CAAC,CAAC;gBACf,OAAO,MAAM,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;oBAC7B,IAAI,MAAM,GAAG,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;wBAC9B,0CAA0C;wBAC1C,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC;wBACvC,MAAM;oBACR,CAAC;oBAED,MAAM,UAAU,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC;oBACjC,MAAM,IAAI,GAAG,KAAK,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;oBAE5C,IAAI,MAAM,GAAG,CAAC,GAAG,IAAI,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;wBACrC,2CAA2C;wBAC3C,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC;wBAC3C,MAAM;oBACR,CAAC;oBAED,MAAM,OAAO,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,MAAM,GAAG,CAAC,GAAG,IAAI,CAAC,CAAC;oBAC3D,IAAI,UAAU,KAAK,CAAC,EAAE,CAAC;wBACrB,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;oBAC7B,CAAC;yBAAM,IAAI,UAAU,KAAK,CAAC,EAAE,CAAC;wBAC5B,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;oBAC7B,CAAC;yBAAM,CAAC;wBACN,8BAA8B;wBAC9B,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;oBAC7B,CAAC;oBAED,MAAM,IAAI,CAAC,GAAG,IAAI,CAAC;gBACrB,CAAC;YACH,CAAC,CAAC,CAAC;YAEH,MAAM,CAAC,EAAE,CAAC,KAAK,EAAE,KAAK,IAAI,EAAE;gBAC1B,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;gBACvD,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;gBAEvD,IAAI,CAAC;oBACH,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;oBACxC,OAAO,CAAC;wBACN,MAAM;wBACN,MAAM;wBACN,QAAQ,EAAE,UAAU,CAAC,QAAQ,IAAI,CAAC;qBACnC,CAAC,CAAC;gBACL,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBACf,MAAM,CAAC,KAAK,CAAC,CAAC;gBAChB,CAAC;YACH,CAAC,CAAC,CAAC;YAEH,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;YAE3B,mBAAmB;YACnB,MAAM,SAAS,GAAG,UAAU,CAAC,GAAG,EAAE;gBAChC,MAAM,CAAC,OAAO,EAAE,CAAC;gBACjB,MAAM,CAAC,IAAI,KAAK,CAAC,2BAA2B,IAAI,CAAC,OAAO,IAAI,CAAC,CAAC,CAAC;YACjE,CAAC,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;YAEjB,MAAM,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC,CAAC;QAClD,CAAC,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,QAAQ,CAAC,OAAe,EAAE,GAA4B;QAC1D,OAAO,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,OAAO,CAAC,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC;IAC3D,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,QAAQ,CAAC,IAAY;QACzB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC;QACpD,IAAI,MAAM,CAAC,QAAQ,KAAK,CAAC,EAAE,CAAC;YAC1B,MAAM,IAAI,KAAK,CAAC,uBAAuB,IAAI,KAAK,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;QACnE,CAAC;QACD,OAAO,MAAM,CAAC,MAAM,CAAC;IACvB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,UAAU,CAAC,IAAY;QAC3B,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC;QAC3D,OAAO,MAAM,CAAC,QAAQ,KAAK,CAAC,CAAC;IAC/B,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,UAAU,CAAC,KAA6B;QAC5C,MAAM,YAAY,GAAkB,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,OAAO,CAAC,EAAE,EAAE,CAAC,CAAC;YAClF,IAAI;YACJ,OAAO,EAAE,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC;SACvC,CAAC,CAAC,CAAC;QAEJ,MAAM,IAAI,CAAC,WAAW,CAAC,YAAY,CAAC,CAAC;IACvC,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,WAAW,CAAC,KAAoB;QACpC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;YACpB,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC;QAC/C,CAAC;QAED,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,OAAO;QACT,CAAC;QAED,uBAAuB;QACvB,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC;QAExB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,MAAM,OAAO,GAAG,OAAO,IAAI,CAAC,OAAO,KAAK,QAAQ;gBAC9C,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC;gBACpC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC;YAEjB,IAAI,CAAC,KAAK,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,EAAE,OAAO,CAAC,CAAC;QAC3C,CAAC;QAED,IAAI,CAAC,QAAQ,EAAE,CAAC;QAEhB,sBAAsB;QACtB,MAAM,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,IAAI,EAAE,EAAE,IAAI,EAAE,iBAAiB,EAAE,CAAC,CAAC;QAEnE,mFAAmF;QACnF,kDAAkD;QAClD,MAAM,IAAI,CAAC,gBAAgB,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,GAAG,WAAW,IAAI,WAAW,EAAE,EAAE,iBAAiB,CAAC,CAAC,CAAC;IACnG,CAAC;IAED,KAAK,CAAC,iBAAiB,CAAC,YAAoB,EAAE,KAAoB;QAChE,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;YACpB,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC;QAC/C,CAAC;QAED,MAAM,IAAI,CAAC,gBAAgB,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,YAAY,CAAC,CAAC,CAAC;QAC3D,MAAM,IAAI,CAAC,gBAAgB,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,GAAG,WAAW,IAAI,WAAW,EAAE,EAAE,YAAY,CAAC,CAAC,CAAC;QAE5F,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,OAAO;QACT,CAAC;QAED,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC;QACxB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,MAAM,OAAO,GAAG,OAAO,IAAI,CAAC,OAAO,KAAK,QAAQ;gBAC9C,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC;gBACpC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC;YACjB,IAAI,CAAC,KAAK,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,EAAE,OAAO,CAAC,CAAC;QAC3C,CAAC;QACD,IAAI,CAAC,QAAQ,EAAE,CAAC;QAEhB,MAAM,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,IAAI,EAAE,EAAE,IAAI,EAAE,YAAY,EAAE,CAAC,CAAC;QAC9D,MAAM,IAAI,CAAC,gBAAgB,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,GAAG,WAAW,IAAI,WAAW,EAAE,EAAE,YAAY,CAAC,CAAC,CAAC;IAC9F,CAAC;IAED,KAAK,CAAC,mBAAmB,CAAC,YAAoB;QAC5C,MAAM,IAAI,CAAC,gBAAgB,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,GAAG,WAAW,IAAI,WAAW,EAAE,EAAE,YAAY,CAAC,CAAC,CAAC;IAC9F,CAAC;IAED;;OAEG;IACH,mBAAmB;QACjB,OAAO,iBAAiB,CAAC;IAC3B,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,IAAI;QACR,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACnB,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,iBAAiB;YACxD,CAAC;YAAC,MAAM,CAAC;gBACP,8CAA8C;YAChD,CAAC;YACD,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QACxB,CAAC;IACH,CAAC;CACF"}

package/dist/lib/fingerprint.d.ts

@@ -0,0 +1,15 @@
+/**
+ * Content fingerprinting for eval result reuse.
+ *
+ * A fingerprint captures the eval files + config fields that affect results.
+ * If the fingerprint matches and the result is valid, the eval can be skipped.
+ */
+import type { RunnableExperimentConfig } from './types.js';
+/**
+ * Compute a fingerprint for an (eval, config) pair.
+ *
+ * Hashes: all eval directory files + config fields that affect results.
+ * Returns a hex SHA-256 digest.
+ */
+export declare function computeFingerprint(evalPath: string, config: RunnableExperimentConfig): string;
+//# sourceMappingURL=fingerprint.d.ts.map

package/dist/lib/fingerprint.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"fingerprint.d.ts","sourceRoot":"","sources":["../../src/lib/fingerprint.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAKH,OAAO,KAAK,EAAE,wBAAwB,EAAE,MAAM,YAAY,CAAC;AAuC3D;;;;;GAKG;AACH,wBAAgB,kBAAkB,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,wBAAwB,GAAG,MAAM,CAuB7F"}

package/dist/lib/fingerprint.js

@@ -0,0 +1,59 @@
+/**
+ * Content fingerprinting for eval result reuse.
+ *
+ * A fingerprint captures the eval files + config fields that affect results.
+ * If the fingerprint matches and the result is valid, the eval can be skipped.
+ */
+import { createHash } from 'crypto';
+import { readFileSync, readdirSync, statSync } from 'fs';
+import { join } from 'path';
+/**
+ * Recursively collects all files in a directory, sorted for deterministic hashing.
+ * Skips node_modules and .git.
+ */
+function collectFiles(dir, basePath = '') {
+    const files = [];
+    const entries = readdirSync(dir).sort();
+    for (const entry of entries) {
+        if (entry === 'node_modules' || entry === '.git')
+            continue;
+        const fullPath = join(dir, entry);
+        const relativePath = basePath ? `${basePath}/${entry}` : entry;
+        const stat = statSync(fullPath);
+        if (stat.isDirectory()) {
+            files.push(...collectFiles(fullPath, relativePath));
+        }
+        else {
+            files.push({ relativePath, content: readFileSync(fullPath, 'utf-8') });
+        }
+    }
+    return files;
+}
+/**
+ * Compute a fingerprint for an (eval, config) pair.
+ *
+ * Hashes: all eval directory files + config fields that affect results.
+ * Returns a hex SHA-256 digest.
+ */
+export function computeFingerprint(evalPath, config) {
+    const hash = createHash('sha256');
+    // Hash all files in the eval directory (sorted for determinism)
+    const files = collectFiles(evalPath);
+    for (const file of files) {
+        hash.update(`file:${file.relativePath}\n`);
+        hash.update(file.content);
+        hash.update('\0');
+    }
+    // Hash config fields that affect results
+    const configForHash = {
+        agent: config.agent,
+        model: config.model,
+        scripts: [...config.scripts].sort(),
+        timeout: config.timeout,
+        earlyExit: config.earlyExit,
+        runs: config.runs,
+    };
+    hash.update(`config:${JSON.stringify(configForHash)}`);
+    return hash.digest('hex');
+}
+//# sourceMappingURL=fingerprint.js.map

package/dist/lib/fingerprint.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"fingerprint.js","sourceRoot":"","sources":["../../src/lib/fingerprint.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AACpC,OAAO,EAAE,YAAY,EAAE,WAAW,EAAE,QAAQ,EAAE,MAAM,IAAI,CAAC;AACzD,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAgB5B;;;GAGG;AACH,SAAS,YAAY,CAAC,GAAW,EAAE,WAAmB,EAAE;IACtD,MAAM,KAAK,GAAqD,EAAE,CAAC;IACnE,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;IAExC,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC5B,IAAI,KAAK,KAAK,cAAc,IAAI,KAAK,KAAK,MAAM;YAAE,SAAS;QAC3D,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QAClC,MAAM,YAAY,GAAG,QAAQ,CAAC,CAAC,CAAC,GAAG,QAAQ,IAAI,KAAK,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC;QAC/D,MAAM,IAAI,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAC;QAEhC,IAAI,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC;YACvB,KAAK,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC,CAAC;QACtD,CAAC;aAAM,CAAC;YACN,KAAK,CAAC,IAAI,CAAC,EAAE,YAAY,EAAE,OAAO,EAAE,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,EAAE,CAAC,CAAC;QACzE,CAAC;IACH,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,kBAAkB,CAAC,QAAgB,EAAE,MAAgC;IACnF,MAAM,IAAI,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC;IAElC,gEAAgE;IAChE,MAAM,KAAK,GAAG,YAAY,CAAC,QAAQ,CAAC,CAAC;IACrC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,CAAC,MAAM,CAAC,QAAQ,IAAI,CAAC,YAAY,IAAI,CAAC,CAAC;QAC3C,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAC1B,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACpB,CAAC;IAED,yCAAyC;IACzC,MAAM,aAAa,GAA0B;QAC3C,KAAK,EAAE,MAAM,CAAC,KAAK;QACnB,KAAK,EAAE,MAAM,CAAC,KAAK;QACnB,OAAO,EAAE,CAAC,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE;QACnC,OAAO,EAAE,MAAM,CAAC,OAAO;QACvB,SAAS,EAAE,MAAM,CAAC,SAAS;QAC3B,IAAI,EAAE,MAAM,CAAC,IAAI;KAClB,CAAC;IACF,IAAI,CAAC,MAAM,CAAC,UAAU,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC;IAEvD,OAAO,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAC5B,CAAC"}

package/dist/lib/fixture.d.ts

@@ -0,0 +1,55 @@
+/**
+ * Eval fixture discovery and validation.
+ */
+import type { EvalFixture } from './types.js';
+/**
+ * Error thrown when an eval fixture is invalid.
+ */
+export declare class FixtureValidationError extends Error {
+    fixtureName: string;
+    constructor(fixtureName: string, message: string);
+}
+/**
+ * Discovers all eval fixtures in a directory.
+ * Recursively searches for directories containing PROMPT.md.
+ * Supports nested organization like "vercel-cli/deploy" or "flags/create-flag".
+ */
+export declare function discoverFixtures(evalsDir: string): string[];
+/**
+ * Validates that a fixture has all required files with correct case.
+ * Returns an array of missing file names, or empty array if valid.
+ * Note: Accepts either EVAL.ts or EVAL.tsx for the eval file.
+ * Case-sensitive: 'prompt.md' will fail even on Mac/Windows.
+ */
+export declare function validateFixtureFiles(fixturePath: string): string[];
+/**
+ * Validates the package.json of a fixture.
+ * Ensures it has "type": "module".
+ */
+export declare function validatePackageJson(fixturePath: string): {
+    isModule: boolean;
+    error?: string;
+};
+/**
+ * Loads a single eval fixture with full validation.
+ */
+export declare function loadFixture(evalsDir: string, name: string): EvalFixture;
+/**
+ * Discovers and loads all valid eval fixtures from a directory.
+ * Returns both valid fixtures and any validation errors encountered.
+ */
+export declare function loadAllFixtures(evalsDir: string): {
+    fixtures: EvalFixture[];
+    errors: FixtureValidationError[];
+};
+/**
+ * Gets a list of all files in a fixture directory.
+ * Excludes PROMPT.md, EVAL.ts, node_modules, and .git.
+ */
+export declare function getFixtureFiles(fixturePath: string, excludePatterns?: readonly string[]): string[];
+/**
+ * Reads all fixture files into a map.
+ * Keys are relative paths, values are file contents.
+ */
+export declare function readFixtureFiles(fixturePath: string, excludePatterns?: readonly string[]): Map<string, string>;
+//# sourceMappingURL=fixture.d.ts.map

package/dist/lib/fixture.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"fixture.d.ts","sourceRoot":"","sources":["../../src/lib/fixture.ts"],"names":[],"mappings":"AAAA;;GAEG;AAIH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAG9C;;GAEG;AACH,qBAAa,sBAAuB,SAAQ,KAAK;IAEtC,WAAW,EAAE,MAAM;gBAAnB,WAAW,EAAE,MAAM,EAC1B,OAAO,EAAE,MAAM;CAKlB;AAeD;;;;GAIG;AACH,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,EAAE,CAuC3D;AAED;;;;;GAKG;AACH,wBAAgB,oBAAoB,CAAC,WAAW,EAAE,MAAM,GAAG,MAAM,EAAE,CAmBlE;AAED;;;GAGG;AACH,wBAAgB,mBAAmB,CAAC,WAAW,EAAE,MAAM,GAAG;IAAE,QAAQ,EAAE,OAAO,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAA;CAAE,CAwB9F;AAED;;GAEG;AACH,wBAAgB,WAAW,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,WAAW,CAmCvE;AAED;;;GAGG;AACH,wBAAgB,eAAe,CAAC,QAAQ,EAAE,MAAM,GAAG;IACjD,QAAQ,EAAE,WAAW,EAAE,CAAC;IACxB,MAAM,EAAE,sBAAsB,EAAE,CAAC;CAClC,CAmBA;AAED;;;GAGG;AACH,wBAAgB,eAAe,CAC7B,WAAW,EAAE,MAAM,EACnB,eAAe,GAAE,SAAS,MAAM,EAAmB,GAClD,MAAM,EAAE,CA2BV;AAED;;;GAGG;AACH,wBAAgB,gBAAgB,CAC9B,WAAW,EAAE,MAAM,EACnB,eAAe,CAAC,EAAE,SAAS,MAAM,EAAE,GAClC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAUrB"}