@aether-agent/evals 0.1.0

package/LICENSE

@@ -0,0 +1,7 @@
+Copyright (c) 2026 Joshua Carver
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

package/README.md

@@ -0,0 +1,137 @@
+<!-- START doctoc generated TOC please keep comment here to allow auto update -->
+<!-- DON'T EDIT THIS SECTION, INSTEAD RE-RUN doctoc TO UPDATE -->
+
+- [@aether-agent/evals](#aether-agentevals)
+  - [Install](#install)
+  - [Writing evals with vitest](#writing-evals-with-vitest)
+    - [Grading a run with an LLM judge](#grading-a-run-with-an-llm-judge)
+
+<!-- END doctoc generated TOC please keep comment here to allow auto update -->
+
+# @aether-agent/evals
+
+This package facilitates writing evals for Aether agents. 
+
+## Install
+
+```sh
+npm install --save-dev @aether-agent/evals
+```
+
+## Writing evals with vitest
+
+`@aether-agent/evals` runs a single Dockerized eval per test:
+
+```ts
+import { test, expect } from "vitest";
+import {
+  Transcript,
+  DockerAgent,
+  Container,
+  Image,
+  Task,
+  Workspace,
+} from "@aether-agent/evals";
+import { readFile } from "node:fs/promises";
+import { join } from "node:path";
+
+test("agent edits notes.txt", async () => {
+  await using workspace = await Workspace.fromFiles({ "notes.txt": "alpha\nalpha\n" });
+  const task = new Task(
+    "Change the first line of notes.txt from alpha to beta"
+  );
+  
+  await using container = await Container.builder(
+    Image.parse("aether-sandbox:latest")
+  ).start(workspace);
+  
+  const agent = new DockerAgent({
+    container,
+    command: ["node", "/app/eval-agent.js"]
+  });
+
+  const trace = await Transcript.fromStream(agent.run(task));
+  expect(trace.messages.at(-1)?.type).toBe("done");
+  
+  // Assert against the retained workspace and the recorded tool calls.
+  expect(await readFile(join(workspace.path, "notes.txt"), "utf8")).toBe(
+    "beta\nalpha\n",
+  );
+  
+  const tests = await container.exec({ command: ["pnpm", "test"] });
+  expect(tests.exitCode).toBe(0);
+  
+  // Workspace is removed when `workspace` goes out of scope; run container exec assertions first.
+});
+```
+
+### Grading a run with an LLM judge
+
+```ts
+import { readFile } from "node:fs/promises";
+import { join } from "node:path";
+import { test, expect } from "vitest";
+import {
+  Transcript,
+  DockerAgent,
+  Container,
+  Image,
+  generate,
+  judge,
+  Task,
+  Workspace,
+} from "@aether-agent/evals";
+
+test("agent makes a maintainer-quality edit", async () => {
+  await using workspace = await Workspace.fromFiles({ "notes.txt": "alpha\nalpha\n" });
+  const task = new Task(
+    "Change the first line of notes.txt from alpha to beta",
+  );
+  
+  await using container = await Container.builder(
+    Image.parse("aether-sandbox:latest"),
+  ).start(workspace);
+  
+  const agent = new DockerAgent({
+    container,
+    command: ["node", "/app/eval-agent.js"],
+  });
+  
+  const trace = await Transcript.fromStream(agent.run(task));
+  const grader = judge({
+    instructions: "Grade strictly using only the provided transcript and files.",
+    task: "Change the first line of notes.txt from alpha to beta",
+    context: {
+      transcript: trace.messages,
+      files: {
+        "notes.txt": await readFile(join(workspace.path, "notes.txt"), "utf8"),
+      },
+    },
+    criteria: [
+      {
+        id: "correct-edit",
+        description: "The final notes.txt content is exactly 'beta\\nalpha\\n'.",
+        blocking: true,
+        threshold: 1,
+        weight: 3,
+      },
+      {
+        id: "minimal-change",
+        description: "No unrelated files or lines were changed.",
+        blocking: true,
+        threshold: 0.8,
+        weight: 2,
+      },
+    ],
+  });
+
+  const response = await generate(grader.prompt, {
+    model: "anthropic:claude-sonnet-4-5", // or set AETHER_LLM_MODEL
+    temperature: 0,
+    schema: grader.schema,
+  });
+  
+  const summary = grader.summarize(response);
+  expect(summary.passed, summary.reason).toBe(true);
+});
+```

package/dist/Agent.d.ts

@@ -0,0 +1,6 @@
+import type { AgentMessage } from "@aether-agent/sdk";
+import type { Task } from "./task.js";
+export interface Agent {
+    run(task: Task): AsyncIterable<AgentMessage>;
+}
+//# sourceMappingURL=Agent.d.ts.map

package/dist/Agent.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"Agent.d.ts","sourceRoot":"","sources":["../src/Agent.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AACtD,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAEtC,MAAM,WAAW,KAAK;IACpB,GAAG,CAAC,IAAI,EAAE,IAAI,GAAG,aAAa,CAAC,YAAY,CAAC,CAAC;CAC9C"}

package/dist/Agent.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=Agent.js.map

package/dist/Agent.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"Agent.js","sourceRoot":"","sources":["../src/Agent.ts"],"names":[],"mappings":""}

package/dist/DockerAgent.d.ts

@@ -0,0 +1,24 @@
+import type { AgentMessage } from "@aether-agent/sdk";
+import { Container } from "./containers/container.js";
+import type { Agent } from "./Agent.js";
+import type { Task } from "./task.js";
+export declare const CONTAINER_AETHER_HOME = "/root/.aether";
+export declare const AETHER_EVAL_TASK_PROMPT_ENV = "AETHER_EVAL_TASK_PROMPT";
+export declare const AETHER_EVAL_WORKSPACE_ROOT_ENV = "AETHER_EVAL_WORKSPACE_ROOT";
+export declare const AETHER_EVAL_CWD_ENV = "AETHER_EVAL_CWD";
+export interface DockerAgentOptions {
+    container: Container;
+    command: string[];
+    env?: Record<string, string>;
+}
+export declare class DockerAgent implements Agent {
+    readonly container: Container;
+    readonly command: string[];
+    readonly env: Record<string, string>;
+    constructor(options: DockerAgentOptions);
+    withEnvVar(key: string, value: string): DockerAgent;
+    withEnvVars(env: Record<string, string>): DockerAgent;
+    run(task: Task): AsyncIterable<AgentMessage>;
+}
+export declare function defaultEvalEnvVars(hostEnv?: Record<string, string | undefined>): Record<string, string>;
+//# sourceMappingURL=DockerAgent.d.ts.map

package/dist/DockerAgent.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"DockerAgent.d.ts","sourceRoot":"","sources":["../src/DockerAgent.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AACtD,OAAO,EAAE,SAAS,EAAE,MAAM,2BAA2B,CAAC;AAEtD,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AACxC,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAGtC,eAAO,MAAM,qBAAqB,kBAAkB,CAAC;AACrD,eAAO,MAAM,2BAA2B,4BAA4B,CAAC;AACrE,eAAO,MAAM,8BAA8B,+BAA+B,CAAC;AAC3E,eAAO,MAAM,mBAAmB,oBAAoB,CAAC;AAWrD,MAAM,WAAW,kBAAkB;IACjC,SAAS,EAAE,SAAS,CAAC;IACrB,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAC9B;AAED,qBAAa,WAAY,YAAW,KAAK;IACvC,QAAQ,CAAC,SAAS,EAAE,SAAS,CAAC;IAC9B,QAAQ,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC;IAC3B,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;gBAEzB,OAAO,EAAE,kBAAkB;IAevC,UAAU,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,WAAW;IAQnD,WAAW,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,WAAW;IAQ9C,GAAG,CAAC,IAAI,EAAE,IAAI,GAAG,aAAa,CAAC,YAAY,CAAC;CAiCpD;AAED,wBAAgB,kBAAkB,CAChC,OAAO,GAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAc,GACvD,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAexB"}

package/dist/DockerAgent.js

@@ -0,0 +1,101 @@
+import { env as processEnv } from "node:process";
+import { AetherEvalError } from "./errors.js";
+import { isTerminalMessage } from "./transcript.js";
+export const CONTAINER_AETHER_HOME = "/root/.aether";
+export const AETHER_EVAL_TASK_PROMPT_ENV = "AETHER_EVAL_TASK_PROMPT";
+export const AETHER_EVAL_WORKSPACE_ROOT_ENV = "AETHER_EVAL_WORKSPACE_ROOT";
+export const AETHER_EVAL_CWD_ENV = "AETHER_EVAL_CWD";
+const PROVIDER_ENV_VARS = new Set([
+    "ANTHROPIC_API_KEY",
+    "DEEPSEEK_API_KEY",
+    "GEMINI_API_KEY",
+    "MOONSHOT_API_KEY",
+    "OPENAI_API_KEY",
+    "OPENROUTER_API_KEY",
+    "ZAI_API_KEY",
+]);
+export class DockerAgent {
+    container;
+    command;
+    env;
+    constructor(options) {
+        if (options.command.length === 0) {
+            throw new AetherEvalError("configuration_error", "DockerAgent command must not be empty");
+        }
+        this.container = options.container;
+        this.command = [...options.command];
+        this.env = {
+            ...defaultEvalEnvVars(processEnv),
+            ...(options.env ?? {}),
+        };
+    }
+    withEnvVar(key, value) {
+        return new DockerAgent({
+            container: this.container,
+            command: this.command,
+            env: { ...this.env, [key]: value },
+        });
+    }
+    withEnvVars(env) {
+        return new DockerAgent({
+            container: this.container,
+            command: this.command,
+            env: { ...this.env, ...env },
+        });
+    }
+    async *run(task) {
+        const env = {
+            ...this.env,
+            [AETHER_EVAL_WORKSPACE_ROOT_ENV]: this.container.workspaceRoot,
+            [AETHER_EVAL_CWD_ENV]: this.container.cwd,
+            [AETHER_EVAL_TASK_PROMPT_ENV]: task.prompt,
+        };
+        let stderr = "";
+        let finished = false;
+        for await (const line of this.container.execStreaming({
+            command: this.command,
+            cwd: this.container.cwd,
+            env,
+            onStderr: (chunk) => {
+                stderr += chunk;
+            },
+        })) {
+            const trimmed = line.trim();
+            if (!trimmed)
+                continue;
+            const message = parseAgentMessage(trimmed);
+            finished = isTerminalMessage(message);
+            yield message;
+            if (finished)
+                break;
+        }
+        if (!finished) {
+            throw new AetherEvalError("command_exit_without_terminal", `agent command exited without emitting a terminal AgentMessage.\nstderr:\n${stderr}`);
+        }
+    }
+}
+export function defaultEvalEnvVars(hostEnv = processEnv) {
+    const env = {};
+    for (const [key, value] of Object.entries(hostEnv)) {
+        if (value === undefined)
+            continue;
+        if (key === "AETHER_HOME")
+            continue;
+        if (PROVIDER_ENV_VARS.has(key) ||
+            key === "OLLAMA_HOST" ||
+            key.startsWith("AETHER_")) {
+            env[key] = value;
+        }
+    }
+    env.AETHER_HOME = CONTAINER_AETHER_HOME;
+    return env;
+}
+function parseAgentMessage(line) {
+    try {
+        return JSON.parse(line);
+    }
+    catch (err) {
+        throw new AetherEvalError("agent_message_json_line", `agent emitted an invalid AgentMessage JSON line: ${line}`, err);
+    }
+}
+//# sourceMappingURL=DockerAgent.js.map

package/dist/DockerAgent.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"DockerAgent.js","sourceRoot":"","sources":["../src/DockerAgent.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,IAAI,UAAU,EAAE,MAAM,cAAc,CAAC;AAGjD,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAG9C,OAAO,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AAEpD,MAAM,CAAC,MAAM,qBAAqB,GAAG,eAAe,CAAC;AACrD,MAAM,CAAC,MAAM,2BAA2B,GAAG,yBAAyB,CAAC;AACrE,MAAM,CAAC,MAAM,8BAA8B,GAAG,4BAA4B,CAAC;AAC3E,MAAM,CAAC,MAAM,mBAAmB,GAAG,iBAAiB,CAAC;AACrD,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC;IAChC,mBAAmB;IACnB,kBAAkB;IAClB,gBAAgB;IAChB,kBAAkB;IAClB,gBAAgB;IAChB,oBAAoB;IACpB,aAAa;CACd,CAAC,CAAC;AAQH,MAAM,OAAO,WAAW;IACb,SAAS,CAAY;IACrB,OAAO,CAAW;IAClB,GAAG,CAAyB;IAErC,YAAY,OAA2B;QACrC,IAAI,OAAO,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACjC,MAAM,IAAI,eAAe,CACvB,qBAAqB,EACrB,uCAAuC,CACxC,CAAC;QACJ,CAAC;QACD,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC;QACnC,IAAI,CAAC,OAAO,GAAG,CAAC,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;QACpC,IAAI,CAAC,GAAG,GAAG;YACT,GAAG,kBAAkB,CAAC,UAAU,CAAC;YACjC,GAAG,CAAC,OAAO,CAAC,GAAG,IAAI,EAAE,CAAC;SACvB,CAAC;IACJ,CAAC;IAED,UAAU,CAAC,GAAW,EAAE,KAAa;QACnC,OAAO,IAAI,WAAW,CAAC;YACrB,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,GAAG,EAAE,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,EAAE,KAAK,EAAE;SACnC,CAAC,CAAC;IACL,CAAC;IAED,WAAW,CAAC,GAA2B;QACrC,OAAO,IAAI,WAAW,CAAC;YACrB,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,GAAG,EAAE,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,GAAG,EAAE;SAC7B,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,CAAC,GAAG,CAAC,IAAU;QACnB,MAAM,GAAG,GAAG;YACV,GAAG,IAAI,CAAC,GAAG;YACX,CAAC,8BAA8B,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,aAAa;YAC9D,CAAC,mBAAmB,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,GAAG;YACzC,CAAC,2BAA2B,CAAC,EAAE,IAAI,CAAC,MAAM;SAC3C,CAAC;QAEF,IAAI,MAAM,GAAG,EAAE,CAAC;QAChB,IAAI,QAAQ,GAAG,KAAK,CAAC;QACrB,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC;YACpD,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,GAAG,EAAE,IAAI,CAAC,SAAS,CAAC,GAAG;YACvB,GAAG;YACH,QAAQ,EAAE,CAAC,KAAK,EAAE,EAAE;gBAClB,MAAM,IAAI,KAAK,CAAC;YAClB,CAAC;SACF,CAAC,EAAE,CAAC;YACH,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;YAC5B,IAAI,CAAC,OAAO;gBAAE,SAAS;YACvB,MAAM,OAAO,GAAG,iBAAiB,CAAC,OAAO,CAAC,CAAC;YAC3C,QAAQ,GAAG,iBAAiB,CAAC,OAAO,CAAC,CAAC;YACtC,MAAM,OAAO,CAAC;YACd,IAAI,QAAQ;gBAAE,MAAM;QACtB,CAAC;QAED,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,MAAM,IAAI,eAAe,CACvB,+BAA+B,EAC/B,4EAA4E,MAAM,EAAE,CACrF,CAAC;QACJ,CAAC;IACH,CAAC;CACF;AAED,MAAM,UAAU,kBAAkB,CAChC,UAA8C,UAAU;IAExD,MAAM,GAAG,GAA2B,EAAE,CAAC;IACvC,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;QACnD,IAAI,KAAK,KAAK,SAAS;YAAE,SAAS;QAClC,IAAI,GAAG,KAAK,aAAa;YAAE,SAAS;QACpC,IACE,iBAAiB,CAAC,GAAG,CAAC,GAAG,CAAC;YAC1B,GAAG,KAAK,aAAa;YACrB,GAAG,CAAC,UAAU,CAAC,SAAS,CAAC,EACzB,CAAC;YACD,GAAG,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;QACnB,CAAC;IACH,CAAC;IACD,GAAG,CAAC,WAAW,GAAG,qBAAqB,CAAC;IACxC,OAAO,GAAG,CAAC;AACb,CAAC;AAED,SAAS,iBAAiB,CAAC,IAAY;IACrC,IAAI,CAAC;QACH,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAiB,CAAC;IAC1C,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,IAAI,eAAe,CACvB,yBAAyB,EACzB,oDAAoD,IAAI,EAAE,EAC1D,GAAG,CACJ,CAAC;IACJ,CAAC;AACH,CAAC"}

package/dist/FakeAgent.d.ts

@@ -0,0 +1,21 @@
+import type { AgentMessage } from "@aether-agent/sdk";
+import type { Agent } from "./Agent.js";
+import type { Task } from "./task.js";
+import type { Workspace } from "./workspace.js";
+/**
+ * In-memory {@link Agent} for testing eval harnesses without Docker. Streams a scripted transcript
+ * and optionally writes files into the workspace, mirroring how a real agent is observed.
+ */
+export declare class FakeAgent implements Agent {
+    readonly messages: AgentMessage[];
+    readonly fileWrites: ReadonlyArray<readonly [string, string]>;
+    readonly workspace: Workspace | undefined;
+    constructor(messages: AgentMessage[], fileWrites?: ReadonlyArray<readonly [string, string]>, workspace?: Workspace | undefined);
+    static success(): FakeAgent;
+    static withToolCall(toolName: string, result: string): FakeAgent;
+    static writesFile(path: string, contents: string): FakeAgent;
+    withFileWrite(path: string, contents: string): FakeAgent;
+    withWorkspace(workspace: Workspace): FakeAgent;
+    run(_task: Task): AsyncIterable<AgentMessage>;
+}
+//# sourceMappingURL=FakeAgent.d.ts.map

package/dist/FakeAgent.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"FakeAgent.d.ts","sourceRoot":"","sources":["../src/FakeAgent.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAEtD,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AACxC,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACtC,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAEhD;;;GAGG;AACH,qBAAa,SAAU,YAAW,KAAK;IAEnC,QAAQ,CAAC,QAAQ,EAAE,YAAY,EAAE;IACjC,QAAQ,CAAC,UAAU,EAAE,aAAa,CAAC,SAAS,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC7D,QAAQ,CAAC,SAAS,EAAE,SAAS,GAAG,SAAS;gBAFhC,QAAQ,EAAE,YAAY,EAAE,EACxB,UAAU,GAAE,aAAa,CAAC,SAAS,CAAC,MAAM,EAAE,MAAM,CAAC,CAAM,EACzD,SAAS,GAAE,SAAS,GAAG,SAAqB;IAGvD,MAAM,CAAC,OAAO,IAAI,SAAS;IAa3B,MAAM,CAAC,YAAY,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,SAAS;IAkBhE,MAAM,CAAC,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,SAAS;IAI5D,aAAa,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,SAAS;IAQxD,aAAa,CAAC,SAAS,EAAE,SAAS,GAAG,SAAS;IAIvC,GAAG,CAAC,KAAK,EAAE,IAAI,GAAG,aAAa,CAAC,YAAY,CAAC;CAkBrD"}

package/dist/FakeAgent.js

@@ -0,0 +1,69 @@
+import { mkdir, writeFile } from "node:fs/promises";
+import { dirname, join } from "node:path";
+import { AetherEvalError } from "./errors.js";
+/**
+ * In-memory {@link Agent} for testing eval harnesses without Docker. Streams a scripted transcript
+ * and optionally writes files into the workspace, mirroring how a real agent is observed.
+ */
+export class FakeAgent {
+    messages;
+    fileWrites;
+    workspace;
+    constructor(messages, fileWrites = [], workspace = undefined) {
+        this.messages = messages;
+        this.fileWrites = fileWrites;
+        this.workspace = workspace;
+    }
+    static success() {
+        return new FakeAgent([
+            {
+                type: "text",
+                message_id: "fake_1",
+                chunk: "Task completed successfully",
+                is_complete: true,
+                model_name: "fake",
+            },
+            { type: "done" },
+        ]);
+    }
+    static withToolCall(toolName, result) {
+        return new FakeAgent([
+            {
+                type: "tool_result",
+                model_name: "fake",
+                result: { id: "fake_call_1", name: toolName, arguments: "{}", result },
+            },
+            {
+                type: "text",
+                message_id: "fake_2",
+                chunk: "Task completed using tools",
+                is_complete: true,
+                model_name: "fake",
+            },
+            { type: "done" },
+        ]);
+    }
+    static writesFile(path, contents) {
+        return FakeAgent.success().withFileWrite(path, contents);
+    }
+    withFileWrite(path, contents) {
+        return new FakeAgent(this.messages, [...this.fileWrites, [path, contents]], this.workspace);
+    }
+    withWorkspace(workspace) {
+        return new FakeAgent(this.messages, this.fileWrites, workspace);
+    }
+    async *run(_task) {
+        if (this.fileWrites.length > 0 && !this.workspace) {
+            throw new AetherEvalError("configuration_error", "FakeAgent file writes require a bound workspace");
+        }
+        for (const [relativePath, contents] of this.fileWrites) {
+            const target = join(this.workspace.path, relativePath);
+            await mkdir(dirname(target), { recursive: true });
+            await writeFile(target, contents);
+        }
+        for (const message of this.messages) {
+            yield message;
+        }
+    }
+}
+//# sourceMappingURL=FakeAgent.js.map

package/dist/FakeAgent.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"FakeAgent.js","sourceRoot":"","sources":["../src/FakeAgent.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AACpD,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAG1C,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAK9C;;;GAGG;AACH,MAAM,OAAO,SAAS;IAET;IACA;IACA;IAHX,YACW,QAAwB,EACxB,aAAuD,EAAE,EACzD,YAAmC,SAAS;QAF5C,aAAQ,GAAR,QAAQ,CAAgB;QACxB,eAAU,GAAV,UAAU,CAA+C;QACzD,cAAS,GAAT,SAAS,CAAmC;IACpD,CAAC;IAEJ,MAAM,CAAC,OAAO;QACZ,OAAO,IAAI,SAAS,CAAC;YACnB;gBACE,IAAI,EAAE,MAAM;gBACZ,UAAU,EAAE,QAAQ;gBACpB,KAAK,EAAE,6BAA6B;gBACpC,WAAW,EAAE,IAAI;gBACjB,UAAU,EAAE,MAAM;aACnB;YACD,EAAE,IAAI,EAAE,MAAM,EAAE;SACjB,CAAC,CAAC;IACL,CAAC;IAED,MAAM,CAAC,YAAY,CAAC,QAAgB,EAAE,MAAc;QAClD,OAAO,IAAI,SAAS,CAAC;YACnB;gBACE,IAAI,EAAE,aAAa;gBACnB,UAAU,EAAE,MAAM;gBAClB,MAAM,EAAE,EAAE,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,QAAQ,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,EAAE;aACvE;YACD;gBACE,IAAI,EAAE,MAAM;gBACZ,UAAU,EAAE,QAAQ;gBACpB,KAAK,EAAE,4BAA4B;gBACnC,WAAW,EAAE,IAAI;gBACjB,UAAU,EAAE,MAAM;aACnB;YACD,EAAE,IAAI,EAAE,MAAM,EAAE;SACjB,CAAC,CAAC;IACL,CAAC;IAED,MAAM,CAAC,UAAU,CAAC,IAAY,EAAE,QAAgB;QAC9C,OAAO,SAAS,CAAC,OAAO,EAAE,CAAC,aAAa,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;IAC3D,CAAC;IAED,aAAa,CAAC,IAAY,EAAE,QAAgB;QAC1C,OAAO,IAAI,SAAS,CAClB,IAAI,CAAC,QAAQ,EACb,CAAC,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC,EACtC,IAAI,CAAC,SAAS,CACf,CAAC;IACJ,CAAC;IAED,aAAa,CAAC,SAAoB;QAChC,OAAO,IAAI,SAAS,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,UAAU,EAAE,SAAS,CAAC,CAAC;IAClE,CAAC;IAED,KAAK,CAAC,CAAC,GAAG,CAAC,KAAW;QACpB,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;YAClD,MAAM,IAAI,eAAe,CACvB,qBAAqB,EACrB,iDAAiD,CAClD,CAAC;QACJ,CAAC;QAED,KAAK,MAAM,CAAC,YAAY,EAAE,QAAQ,CAAC,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACvD,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,SAAU,CAAC,IAAI,EAAE,YAAY,CAAC,CAAC;YACxD,MAAM,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YAClD,MAAM,SAAS,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QACpC,CAAC;QAED,KAAK,MAAM,OAAO,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YACpC,MAAM,OAAO,CAAC;QAChB,CAAC;IACH,CAAC;CACF"}

package/dist/containers/container.d.ts

@@ -0,0 +1,51 @@
+import { type StartedTestContainer } from "testcontainers";
+import { Image } from "./image.js";
+import type { Workspace } from "../workspace.js";
+export interface BindMount {
+    source: string;
+    target: string;
+    mode?: "rw" | "ro" | "z" | "Z";
+}
+export interface ContainerExecOptions {
+    command: string[];
+    cwd?: string;
+    env?: Record<string, string>;
+    signal?: AbortSignal;
+}
+export interface ExecOutput {
+    exitCode: number;
+    stdout: string;
+    stderr: string;
+}
+export interface ContainerStreamingOptions {
+    command: string[];
+    cwd?: string;
+    env?: Record<string, string>;
+    signal?: AbortSignal;
+    onStderr?: (chunk: string) => void;
+}
+export declare class Container {
+    private readonly container;
+    private readonly ephemeralTempdirs;
+    readonly workspaceRoot = "/workspace";
+    readonly cwd: string;
+    constructor(container: StartedTestContainer, ephemeralTempdirs: string[], cwd: string);
+    static builder(image: Image): ContainerBuilder;
+    exec(options: ContainerExecOptions): Promise<ExecOutput>;
+    execShell(script: string): Promise<ExecOutput>;
+    execStreaming(options: ContainerStreamingOptions): AsyncIterable<string>;
+    [Symbol.asyncDispose](): Promise<void>;
+}
+export declare class ContainerBuilder {
+    private readonly image;
+    private envVars;
+    private mounts;
+    private ephemeralMounts;
+    constructor(image: Image);
+    withEnvVar(key: string, value: string): this;
+    withEnvVars(envVars: Record<string, string>): this;
+    withMount(mount: BindMount): this;
+    withEphemeralMount(target: string): this;
+    start(workspace: Workspace): Promise<Container>;
+}
+//# sourceMappingURL=container.d.ts.map

package/dist/containers/container.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"container.d.ts","sourceRoot":"","sources":["../../src/containers/container.ts"],"names":[],"mappings":"AAMA,OAAO,EAGL,KAAK,oBAAoB,EAE1B,MAAM,gBAAgB,CAAC;AAExB,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AACnC,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAIjD,MAAM,WAAW,SAAS;IACxB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,IAAI,GAAG,IAAI,GAAG,GAAG,GAAG,GAAG,CAAC;CAChC;AAED,MAAM,WAAW,oBAAoB;IACnC,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC7B,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB;AAED,MAAM,WAAW,UAAU;IACzB,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,yBAAyB;IACxC,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC7B,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,QAAQ,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;CACpC;AAED,qBAAa,SAAS;IAKlB,OAAO,CAAC,QAAQ,CAAC,SAAS;IAC1B,OAAO,CAAC,QAAQ,CAAC,iBAAiB;IALpC,QAAQ,CAAC,aAAa,gBAAgB;IACtC,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;gBAGF,SAAS,EAAE,oBAAoB,EAC/B,iBAAiB,EAAE,MAAM,EAAE,EAC5C,GAAG,EAAE,MAAM;IAKb,MAAM,CAAC,OAAO,CAAC,KAAK,EAAE,KAAK,GAAG,gBAAgB;IAIxC,IAAI,CAAC,OAAO,EAAE,oBAAoB,GAAG,OAAO,CAAC,UAAU,CAAC;IAYxD,SAAS,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,CAAC;IAM7C,aAAa,CAClB,OAAO,EAAE,yBAAyB,GACjC,aAAa,CAAC,MAAM,CAAC;IA6ClB,CAAC,MAAM,CAAC,YAAY,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC;CAI7C;AAED,qBAAa,gBAAgB;IAKf,OAAO,CAAC,QAAQ,CAAC,KAAK;IAJlC,OAAO,CAAC,OAAO,CAA8B;IAC7C,OAAO,CAAC,MAAM,CAAmB;IACjC,OAAO,CAAC,eAAe,CAAgB;gBAEV,KAAK,EAAE,KAAK;IAEzC,UAAU,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI;IAK5C,WAAW,CAAC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,IAAI;IAKlD,SAAS,CAAC,KAAK,EAAE,SAAS,GAAG,IAAI;IAKjC,kBAAkB,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI;IAKlC,KAAK,CAAC,SAAS,EAAE,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC;CAuCtD"}