@hona/openeval 0.2.0

package/src/infra/containers/candidate.ts

@@ -0,0 +1,218 @@
+import { mkdir, mkdtemp, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { resolve } from "node:path";
+import type {
+  BenchmarkDefinition,
+  ModelRef,
+  PreparationStep,
+  SessionArchive,
+  EvidenceRef,
+  OpenCodeStreamEvent,
+} from "../../types";
+import { CandidateContainer } from "./oci";
+import { EvidenceCapture } from "../evidence";
+import { createSessionDatabase, OPENCODE_VERSION } from "../opencode/host";
+import { credentialsFor, removeCredentials } from "../opencode/auth";
+import { runSession, clientFor, type SessionResult } from "../opencode/session";
+import { readArchivedSession } from "../opencode/archive";
+import { hash, errorMessage, writeJson } from "../files";
+import type { EvidenceFeed } from "../../evidence";
+import { readDurableCheckpoint } from "../opencode/checkpoint";
+
+type CandidateInput = {
+  model: ModelRef;
+  prompt: string;
+  workspace: string;
+  prepare: readonly PreparationStep[];
+  timeoutMs: number;
+  websearch: "exa" | false;
+  container: BenchmarkDefinition["container"];
+  imageId: string;
+  providers?: BenchmarkDefinition["candidate"]["providers"];
+};
+type CandidateResult = {
+  state: "completed" | "stopped" | "failed" | "timed_out";
+  session?: SessionArchive;
+  evidence?: EvidenceRef;
+  error?: string;
+};
+
+/** Owns the container and transport. Judge data cannot enter this interface. */
+export async function executeCandidate(
+  input: CandidateInput,
+  directory: string,
+  onEvent: (event: OpenCodeStreamEvent) => void,
+  controls: {
+    signal?: AbortSignal;
+    onEvidence?: (feed: EvidenceFeed) => void;
+  } = {},
+): Promise<CandidateResult> {
+  await mkdir(directory, { recursive: true });
+  const staging = await mkdtemp(
+    resolve(process.env.TMP ?? tmpdir(), "session-"),
+  );
+  let container: CandidateContainer | undefined,
+    captured = false,
+    rootId: string | undefined;
+  let outcome: SessionResult | undefined;
+  let capture: EvidenceCapture | undefined;
+  try {
+    const seedDatabase = resolve(staging, "opencode.db");
+    await createSessionDatabase(
+      seedDatabase,
+      credentialsFor(input.model, input.websearch),
+    );
+    container = await CandidateContainer.create(input.container, input.imageId);
+    await container.prepare(
+      input.workspace,
+      seedDatabase,
+      input.websearch,
+      input.prepare,
+      staging,
+      input.providers,
+    );
+    const initial = resolve(staging, "initial");
+    await container.snapshot(initial, staging);
+    capture = await EvidenceCapture.create(
+      resolve(directory, "evidence"),
+      initial,
+      { excludeDirectories: ["node_modules"] },
+    );
+    const recorded = new Set<string>();
+    let recordingError: unknown;
+    const record = (event: OpenCodeStreamEvent) => {
+      if (!("id" in event) || recorded.has(event.id)) return;
+      if (event.type === "session.created" && !event.data.parentID)
+        rootId ??= event.data.sessionID;
+      try {
+        capture!.event(event);
+        onEvent(event);
+        recorded.add(event.id);
+      } catch (error) {
+        recordingError = error;
+        throw error;
+      }
+    };
+    const url = await container.start(input.timeoutMs);
+    const client = clientFor(url, container.password);
+    let transportError: string | undefined;
+    try {
+      outcome = await runSession(
+        client,
+        {
+          model: input.model,
+          agent: "build",
+          prompt: input.prompt,
+          directory: "/workspace",
+          timeoutMs: input.timeoutMs,
+          signal: controls.signal,
+          onCreated: (id) => {
+            rootId = id;
+            controls.onEvidence?.(
+              capture!.feed(input.prompt, (cutoff, signal) =>
+                readDurableCheckpoint(client, id, cutoff, signal, record),
+              ),
+            );
+          },
+        },
+        record,
+      );
+    } catch (error) {
+      if (recordingError || !rootId) throw error;
+      transportError = errorMessage(error);
+      // The independent server deadline can close HTTP before final reads.
+      // Read the stopped, scrubbed database before deciding the outcome.
+    }
+    const deadlineExpired =
+      container.deadlineAt !== undefined && Date.now() >= container.deadlineAt;
+    capture.close();
+    await container.stop();
+    const database = resolve(directory, "opencode.db");
+    await container.saveDatabase(database);
+    removeCredentials(database);
+    if (!outcome || controls.onEvidence) {
+      const archived = await readArchivedSession(database, rootId!, record);
+      const previous = outcome;
+      outcome = previous
+        ? { ...archived.result, state: previous.state, error: previous.error }
+        : archived.result;
+      if (!archived.outcome) outcome.error = transportError ?? outcome.error;
+      if (
+        !previous &&
+        (archived.outcome === "interrupted" || !archived.outcome) &&
+        deadlineExpired
+      ) {
+        outcome.state = "timed_out";
+        outcome.error = "Candidate session exceeded its time limit";
+      } else if (
+        !previous &&
+        (archived.outcome === "interrupted" || !archived.outcome) &&
+        controls.signal?.aborted
+      ) {
+        outcome.state = "stopped";
+        outcome.error = undefined;
+      }
+    }
+    const final = resolve(directory, "workspace");
+    await container.snapshot(final, staging);
+    await writeJson(resolve(directory, "session.json"), outcome.sessions);
+    const manifest = await capture.finish({
+      prompt: input.prompt,
+      response: { text: outcome.text },
+      tools: outcome.tools,
+      workspace: final,
+    });
+    const session: SessionArchive = {
+      sessionId: outcome.sessions[0]!.info.id,
+      database: "opencode.db",
+      databaseHash: hash(await Bun.file(database).bytes()),
+      opencodeVersion: OPENCODE_VERSION,
+      accounting: outcome.accounting,
+    };
+    captured = true;
+    return {
+      state: outcome.state,
+      session,
+      evidence: { directory: "evidence", hash: manifest.sha256 },
+      error: outcome.error,
+    };
+  } catch (error) {
+    await container?.stop().catch(() => {});
+    let session: SessionArchive | undefined;
+    if (container) {
+      const database = resolve(directory, "opencode.db");
+      try {
+        await container.saveDatabase(database);
+        removeCredentials(database);
+        if (rootId)
+          session = {
+            sessionId: rootId,
+            database: "opencode.db",
+            databaseHash: hash(await Bun.file(database).bytes()),
+            opencodeVersion: OPENCODE_VERSION,
+            accounting: outcome?.accounting,
+          };
+      } catch (archiveError) {
+        await writeJson(resolve(directory, "archive-error.json"), {
+          error: errorMessage(archiveError),
+        });
+      }
+    }
+    if (outcome)
+      await writeJson(resolve(directory, "candidate-checkpoint.json"), outcome);
+    return {
+      state: "failed",
+      session,
+      error: `${errorMessage(error)}${container ? ` (workspace retained in ${container.name})` : ""}`,
+    };
+  } finally {
+    capture?.close();
+    if (captured) await container?.close();
+    await rm(staging, {
+      recursive: true,
+      force: true,
+      maxRetries: 5,
+      retryDelay: 100,
+    });
+  }
+}

package/src/infra/containers/docker.ts

@@ -0,0 +1,31 @@
+import type { Engine } from "../../types";
+
+/** The engine adapter owns argv, process cancellation, and diagnostics. */
+export async function engineCommand(
+  engine: Engine,
+  args: string[],
+  options: { output?: string; input?: string; timeoutMs?: number } = {},
+) {
+  const child = Bun.spawn([engine, ...args], {
+    stdout: "pipe",
+    stderr: "pipe",
+    stdin: options.input ? Bun.file(options.input) : "ignore",
+  });
+  const timer = setTimeout(() => child.kill(), options.timeoutMs ?? 120_000);
+  try {
+    const [code, stdout, stderr] = await Promise.all([
+      child.exited,
+      options.output
+        ? Bun.write(options.output, new Response(child.stdout)).then(() => "")
+        : new Response(child.stdout).text(),
+      new Response(child.stderr).text(),
+    ]);
+    if (code !== 0)
+      throw new Error(
+        `${engine} ${args[0]} failed: ${stderr.trim() || stdout.trim() || code}`,
+      );
+    return stdout.trim();
+  } finally {
+    clearTimeout(timer);
+  }
+}

package/src/infra/containers/oci.ts

@@ -0,0 +1,335 @@
+import { mkdir, rm, stat } from "node:fs/promises";
+import { resolve, posix, basename, dirname } from "node:path";
+import { randomUUID } from "node:crypto";
+import { fileURLToPath } from "node:url";
+import {
+  CANDIDATE_TIMEOUT_MS,
+  type BenchmarkDefinition,
+  type PreparationStep,
+} from "../../types";
+import { engineCommand } from "./docker";
+import { extractWorkspaceArchive } from "./transfer";
+import { treeHash, fingerprint, writeJson } from "../files";
+import { OPENCODE_VERSION } from "../opencode/host";
+import { judgingFingerprint } from "../judging";
+
+const runtimeDirectory = fileURLToPath(new URL("./runtime/", import.meta.url));
+export async function buildImage(config: BenchmarkDefinition["container"]) {
+  const source = await treeHash(runtimeDirectory);
+  await engineCommand(
+    config.engine,
+    [
+      "build",
+      "--label",
+      `openeval.runtime=${source}`,
+      "-t",
+      config.image,
+      runtimeDirectory,
+    ],
+    { timeoutMs: 600_000 },
+  );
+}
+export async function inspectImage(config: BenchmarkDefinition["container"]) {
+  const inspected = await engineCommand(config.engine, [
+    "image",
+    "inspect",
+    config.image,
+    "--format",
+    '{{.Id}}|{{index .Config.Labels "openeval.runtime"}}',
+  ]);
+  const [id, source] = inspected.split("|");
+  if (source !== (await treeHash(runtimeDirectory)))
+    throw new Error(
+      "Candidate runtime changed or the image is not initialized. Run the image command first.",
+    );
+  return id;
+}
+export async function runtimeFingerprint(
+  config: BenchmarkDefinition["container"],
+) {
+  const imageId = await inspectImage(config);
+  return {
+    imageId,
+    candidateHash: fingerprint({
+      imageId,
+      sdk: OPENCODE_VERSION,
+      engine: config.engine,
+      cpus: config.cpus,
+      memoryMiB: config.memoryMiB,
+      implementation: await treeHash(resolve(runtimeDirectory, ".."), {
+        ignore: /\.test\.[jt]sx?$/,
+      }),
+      sessions: await treeHash(
+        fileURLToPath(new URL("../opencode/", import.meta.url)),
+        { ignore: /\.test\.[jt]sx?$/ },
+      ),
+      evidence: await treeHash(
+        fileURLToPath(new URL("../evidence/", import.meta.url)),
+      ),
+      useCases: await Promise.all(
+        [
+          "run-eval.ts",
+          "run-eval-pipeline.ts",
+          "watch-evidence.ts",
+          "execution-budget.ts",
+        ].map((file) =>
+          Bun.file(
+            fileURLToPath(new URL(`../../app/${file}`, import.meta.url)),
+          ).text(),
+        ),
+      ),
+    }),
+    judgeHash: await judgingFingerprint(),
+  };
+}
+
+/** A private workspace and a normal OpenCode installation; no evaluator inputs or code. */
+export class CandidateContainer {
+  readonly name = `workspace-${randomUUID()}`;
+  readonly password = randomUUID();
+  deadlineAt?: number;
+  private workspaceVolume = `${this.name}-files`;
+  private homeVolume = `${this.name}-home`;
+  private stopped = false;
+  private removed = false;
+  private constructor(
+    private config: BenchmarkDefinition["container"],
+    readonly imageId: string,
+  ) {}
+  private command(
+    args: string[],
+    options?: Parameters<typeof engineCommand>[2],
+  ) {
+    return engineCommand(this.config.engine, args, options);
+  }
+
+  static async create(
+    config: BenchmarkDefinition["container"],
+    imageId: string,
+  ) {
+    const container = new CandidateContainer(config, imageId);
+    try {
+      for (const volume of [container.workspaceVolume, container.homeVolume])
+        await container.command(["volume", "create", volume]);
+      await container.command([
+        "run",
+        "--detach",
+        "--init",
+        "--name",
+        container.name,
+        "--hostname",
+        "workstation",
+        "--cap-drop=ALL",
+        "--cap-add=NET_ADMIN",
+        "--cap-add=SETUID",
+        "--cap-add=SETGID",
+        "--cap-add=SETPCAP",
+        "--security-opt=no-new-privileges",
+        "--pids-limit",
+        "256",
+        "--cpus",
+        String(config.cpus),
+        "--memory",
+        `${config.memoryMiB}m`,
+        "--publish",
+        "127.0.0.1::4096",
+        "--mount",
+        `type=volume,source=${container.workspaceVolume},target=/workspace`,
+        "--mount",
+        `type=volume,source=${container.homeVolume},target=/home/dev`,
+        imageId,
+      ]);
+      return container;
+    } catch (error) {
+      await container.close();
+      throw error;
+    }
+  }
+  async prepare(
+    workspace: string,
+    database: string,
+    websearch: "exa" | false,
+    steps: readonly PreparationStep[],
+    staging: string,
+    providers?: BenchmarkDefinition["candidate"]["providers"],
+  ) {
+    await this.upload(workspace, "/workspace", staging);
+    await this.upload(database, "/home/dev/.local/share/opencode", staging);
+    const config = resolve(staging, "opencode.json");
+    await writeJson(config, {
+      plugins: ["/opt/opencode/noninteractive"],
+      permissions: [{ action: "*", resource: "*", effect: "allow" }],
+      websearch: websearch ? { provider: websearch } : false,
+      ...(providers ? { providers } : {}),
+    });
+    await this.upload(config, "/home/dev/.config/opencode", staging);
+    for (const step of steps) {
+      const workdir = posix.resolve(
+        "/workspace",
+        step.cwd.replaceAll("\\", "/"),
+      );
+      if (workdir !== "/workspace" && !workdir.startsWith("/workspace/"))
+        throw new Error("Preparation directory escapes the workspace");
+      await this.command(
+        [
+          "exec",
+          "--user",
+          "dev",
+          "--workdir",
+          workdir,
+          this.name,
+          ...step.argv,
+        ],
+        { timeoutMs: 180_000 },
+      );
+    }
+  }
+  private async upload(source: string, destination: string, staging: string) {
+    const directory = (await stat(source)).isDirectory();
+    const archive = resolve(staging, `transfer-${randomUUID()}.tar`);
+    try {
+      const child = Bun.spawn(
+        [
+          "tar",
+          "-cf",
+          archive,
+          "-C",
+          directory ? source : dirname(source),
+          directory ? "." : basename(source),
+        ],
+        { stdout: "ignore", stderr: "pipe" },
+      );
+      const [code, error] = await Promise.all([
+        child.exited,
+        new Response(child.stderr).text(),
+      ]);
+      if (code !== 0) throw new Error(`Workspace transfer failed: ${error}`);
+      await this.command(
+        [
+          "exec",
+          "--user",
+          "dev",
+          "--interactive",
+          this.name,
+          "tar",
+          "--no-same-owner",
+          "-xf",
+          "-",
+          "-C",
+          destination,
+        ],
+        { input: archive },
+      );
+    } finally {
+      await rm(archive, { force: true });
+    }
+  }
+  async start(timeoutMs = CANDIDATE_TIMEOUT_MS) {
+    // The server deadline survives a host runner crash.
+    this.deadlineAt = Date.now() + Math.ceil(timeoutMs / 1000) * 1000;
+    await this.command([
+      "exec",
+      "--user",
+      "dev",
+      "--detach",
+      "--env",
+      `OPENCODE_SERVER_PASSWORD=${this.password}`,
+      this.name,
+      "sh",
+      "-c",
+      `exec timeout --signal=TERM --kill-after=5s ${Math.ceil(timeoutMs / 1000)}s bun /opt/opencode/server.mjs > /home/dev/.local/share/opencode/server.log 2>&1`,
+    ]);
+    const port = await this.command(["port", this.name, "4096/tcp"]);
+    const url = `http://${port.split(/\r?\n/)[0]}`;
+    const deadline = Date.now() + 30_000;
+    while (Date.now() < deadline) {
+      try {
+        if (
+          (
+            await fetch(`${url}/api/health`, {
+              headers: {
+                authorization: `Basic ${Buffer.from(`opencode:${this.password}`).toString("base64")}`,
+              },
+              signal: AbortSignal.timeout(1000),
+            })
+          ).ok
+        )
+          return url;
+      } catch {}
+      await Bun.sleep(100);
+    }
+    const log = await this.inspect([
+      "tail",
+      "-n",
+      "40",
+      "/home/dev/.local/share/opencode/server.log",
+    ]);
+    throw new Error(`OpenCode server did not become ready: ${log}`);
+  }
+  async snapshot(destination: string, staging: string) {
+    await mkdir(destination, { recursive: true });
+    const archive = resolve(staging, `files-${randomUUID()}.tar`);
+    try {
+      await this.command(
+        [
+          "run",
+          "--rm",
+          "--user",
+          "dev",
+          "--entrypoint",
+          "tar",
+          "--mount",
+          `type=volume,source=${this.workspaceVolume},target=/source,readonly`,
+          this.imageId,
+          "--exclude=node_modules",
+          "-cf",
+          "-",
+          "-C",
+          "/source",
+          ".",
+        ],
+        { output: archive },
+      );
+      await extractWorkspaceArchive(archive, destination, {
+        sourceRoot: "/workspace",
+      });
+    } finally {
+      await rm(archive, { force: true });
+    }
+  }
+  async stop() {
+    if (this.stopped) return;
+    await this.command(["stop", "--time", "15", this.name], {
+      timeoutMs: 30_000,
+    });
+    this.stopped = true;
+  }
+  async saveDatabase(destination: string) {
+    if (!this.stopped)
+      throw new Error("Stop the candidate before archiving its database");
+    for (const suffix of ["", "-wal", "-shm"]) {
+      try {
+        await this.command([
+          "cp",
+          `${this.name}:/home/dev/.local/share/opencode/opencode.db${suffix}`,
+          destination + suffix,
+        ]);
+      } catch (error) {
+        if (!suffix) throw error;
+      }
+    }
+  }
+  async inspect(command: string[]) {
+    return this.command(["exec", "--user", "dev", this.name, ...command]);
+  }
+  async close() {
+    if (this.removed) return;
+    this.removed = true;
+    await this.command(["rm", "--force", this.name]).catch(() => {});
+    for (const volume of [this.workspaceVolume, this.homeVolume])
+      await this.command(["volume", "rm", volume]).catch(() => {});
+  }
+  async [Symbol.asyncDispose]() {
+    await this.close();
+  }
+}

package/src/infra/containers/runtime/.dockerignore

@@ -0,0 +1,7 @@
+*
+!Dockerfile
+!package.json
+!server.mjs
+!noninteractive/
+!noninteractive/**
+!development-container

package/src/infra/containers/runtime/Dockerfile

@@ -0,0 +1,19 @@
+FROM oven/bun:1.4.2
+RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates git ripgrep curl unzip g++ make python3 nodejs npm tar procps \
+    && rm -rf /var/lib/apt/lists/*
+WORKDIR /opt/opencode
+RUN apt-get update && apt-get install -y --no-install-recommends iptables util-linux && rm -rf /var/lib/apt/lists/*
+COPY package.json ./
+RUN bun install --production
+COPY server.mjs ./
+COPY noninteractive ./noninteractive
+COPY development-container /usr/local/bin/development-container
+RUN useradd --create-home --uid 10001 dev \
+    && mkdir -p /workspace /home/dev/.config/opencode /home/dev/.local/share/opencode \
+    && chown -R dev:dev /workspace /home/dev \
+    && git config --system --add safe.directory '*' \
+    && chmod 755 /usr/local/bin/development-container
+ENV HOME=/home/dev
+WORKDIR /workspace
+ENTRYPOINT ["/usr/local/bin/development-container"]
+CMD ["sleep", "infinity"]

package/src/infra/containers/runtime/development-container

@@ -0,0 +1,19 @@
+#!/bin/sh
+set -eu
+
+# Allow replies to the local control connection and normal DNS/public egress.
+# A development session has no access to host/LAN services or metadata endpoints.
+iptables -A OUTPUT -m conntrack --ctstate ESTABLISHED,RELATED -j ACCEPT
+iptables -A OUTPUT -p udp --dport 53 -j ACCEPT
+iptables -A OUTPUT -p tcp --dport 53 -j ACCEPT
+for network in 10.0.0.0/8 172.16.0.0/12 192.168.0.0/16 169.254.0.0/16 100.64.0.0/10; do
+  iptables -A OUTPUT -d "$network" -j REJECT
+done
+ip6tables -A OUTPUT -m conntrack --ctstate ESTABLISHED,RELATED -j ACCEPT
+ip6tables -A OUTPUT -p udp --dport 53 -j ACCEPT
+ip6tables -A OUTPUT -p tcp --dport 53 -j ACCEPT
+for network in fc00::/7 fe80::/10; do
+  ip6tables -A OUTPUT -d "$network" -j REJECT
+done
+
+exec setpriv --reuid=dev --regid=dev --init-groups --bounding-set=-all --inh-caps=-all --ambient-caps=-all "$@"

package/src/infra/containers/runtime/noninteractive/index.mjs

@@ -0,0 +1,7 @@
+import { Plugin } from "@opencode/plugin/effect";
+
+export default Plugin.define({
+  id: "noninteractive",
+  effect: (context) =>
+    context.tool.transform((tools) => tools.remove("question")),
+});

package/src/infra/containers/runtime/noninteractive/package.json

@@ -0,0 +1,6 @@
+{
+  "name": "opencode-noninteractive",
+  "private": true,
+  "type": "module",
+  "exports": "./index.mjs"
+}

package/src/infra/containers/runtime/package.json

@@ -0,0 +1,10 @@
+{
+  "name": "opencode-runtime",
+  "private": true,
+  "type": "module",
+  "dependencies": {
+    "@opencode/server": "0.0.0-beta-19296",
+    "@opencode/plugin": "0.0.0-beta-19296",
+    "effect": "4.0.0-rc.112"
+  }
+}

package/src/infra/containers/runtime/server.mjs

@@ -0,0 +1,32 @@
+import { ServerProcess } from "@opencode/server/process";
+import { Effect, Fiber } from "effect";
+
+process.title = "opencode";
+const runtime = Effect.runFork(
+  Effect.scoped(
+    Effect.gen(function* () {
+      yield* ServerProcess.start({
+        hostname: "0.0.0.0",
+        port: 4096,
+        password: process.env.OPENCODE_SERVER_PASSWORD,
+        app: { name: "opencode", version: "0.0.0-beta-19296" },
+        database: { path: "/home/dev/.local/share/opencode/opencode.db" },
+        events: { persist: true },
+      });
+      yield* Effect.never;
+    }),
+  ),
+);
+let closing = false;
+for (const signal of ["SIGTERM", "SIGINT"])
+  process.on(signal, () => {
+    if (closing) return;
+    closing = true;
+    void Effect.runPromise(Fiber.interrupt(runtime)).finally(() =>
+      process.exit(0),
+    );
+  });
+void Effect.runPromise(Fiber.join(runtime)).catch((error) => {
+  console.error(error);
+  process.exit(1);
+});