@hona/openeval 0.2.0

package/src/app/serve-results.ts

@@ -0,0 +1,252 @@
+import { resolve, dirname, sep } from "node:path";
+import { watch } from "node:fs";
+import { mkdir } from "node:fs/promises";
+import { ResultReader } from "./read-results";
+import { readRecording } from "../infra/opencode/read-recording";
+import type { SessionSnapshot } from "../view";
+import type { RunEvent } from "../types";
+import type { EvidenceQuery } from "../infra/evidence";
+
+export async function serveResults(options: {
+  resultsPath: string;
+  port?: number;
+  assetsPath?: string;
+}) {
+  const root = resolve(options.resultsPath);
+  await mkdir(root, { recursive: true });
+  const reader = new ResultReader(root),
+    encoder = new TextEncoder();
+  const clients = new Set<ReadableStreamDefaultController<Uint8Array>>(),
+    streams = new Set<() => void>();
+  let changed: ReturnType<typeof setTimeout> | undefined;
+  const watcher = watch(root, { recursive: true }, (_event, file) => {
+    if (file && !/runner\.db(?:-wal|-shm)?$/.test(String(file))) return;
+    if (changed) return;
+    changed = setTimeout(() => {
+      changed = undefined;
+      for (const client of clients)
+        try {
+          client.enqueue(encoder.encode("event: change\ndata: {}\n\n"));
+        } catch {
+          clients.delete(client);
+        }
+    }, 250);
+  });
+  const server = Bun.serve({
+    hostname: "127.0.0.1",
+    port: options.port ?? 4173,
+    idleTimeout: 0,
+    async fetch(request) {
+      const url = new URL(request.url);
+      if (request.method !== "GET")
+        return new Response("Method not allowed", { status: 405 });
+      try {
+        if (url.pathname === "/api/results")
+          return Response.json(await reader.index());
+        if (url.pathname === "/api/activity")
+          return Response.json(await reader.activity());
+        if (url.pathname === "/api/judge-checks")
+          return Response.json(
+            await reader.judgeAudit(
+              url.searchParams.get("benchmark") ?? "",
+              url.searchParams.get("judge") ?? "",
+            ),
+          );
+        if (url.pathname === "/api/check-evidence") {
+          const query: EvidenceQuery = {
+            action: (url.searchParams.get("action") ??
+              "summary") as EvidenceQuery["action"],
+          };
+          for (const key of ["id", "path", "sessionID", "type"] as const) {
+            const value = url.searchParams.get(key);
+            if (value !== null) query[key] = value;
+          }
+          for (const key of ["offset", "limit"] as const) {
+            const value = url.searchParams.get(key);
+            if (value !== null) query[key] = Number(value);
+          }
+          const revision = url.searchParams.get("revision");
+          if (revision === "initial" || revision === "final")
+            query.revision = revision;
+          return Response.json(
+            await reader.checkEvidence(
+              url.searchParams.get("benchmark") ?? "",
+              url.searchParams.get("judge") ?? "",
+              url.searchParams.get("check") ?? "",
+              query,
+            ),
+          );
+        }
+        if (url.pathname === "/api/result")
+          return Response.json(
+            url.searchParams.get("format") === "runs"
+              ? await reader.evalRuns(url.searchParams.get("id") ?? "")
+              : await reader.summary(url.searchParams.get("id") ?? ""),
+          );
+        if (url.pathname === "/api/events") {
+          let owner: ReadableStreamDefaultController<Uint8Array>;
+          return new Response(
+            new ReadableStream<Uint8Array>({
+              start(controller) {
+                owner = controller;
+                clients.add(controller);
+                controller.enqueue(encoder.encode(": connected\n\n"));
+              },
+              cancel() {
+                clients.delete(owner);
+              },
+            }),
+            {
+              headers: {
+                "Content-Type": "text/event-stream",
+                "Cache-Control": "no-cache",
+              },
+            },
+          );
+        }
+        if (url.pathname === "/api/session") {
+          const benchmarkId = url.searchParams.get("benchmark") ?? "",
+            executionId = url.searchParams.get("execution") ?? "";
+          const recording = await reader.recording(benchmarkId, executionId);
+          const execution = recording.read;
+          const initial = execution();
+          if (!initial) {
+            recording.close();
+            return new Response("Unknown execution", { status: 404 });
+          }
+          const stage = "evalRunId" in initial.input ? "judge" : "candidate";
+          let stopped = false,
+            pumping = false,
+            cursor = 0,
+            snapshotSent = false;
+          const history: RunEvent[] = [];
+          let timer: ReturnType<typeof setInterval>;
+          let stop = () => {};
+          const body = new ReadableStream<Uint8Array>({
+            start(controller) {
+              const send = (event: string, data: unknown) => {
+                if (!stopped)
+                  controller.enqueue(
+                    encoder.encode(
+                      `event: ${event}\ndata: ${JSON.stringify(data)}\n\n`,
+                    ),
+                  );
+              };
+              stop = () => {
+                if (stopped) return;
+                stopped = true;
+                clearInterval(timer);
+                streams.delete(stop);
+                recording.close();
+                request.signal.removeEventListener("abort", stop);
+                try {
+                  controller.close();
+                } catch {}
+              };
+              streams.add(stop);
+              request.signal.addEventListener("abort", stop, { once: true });
+              const pump = async () => {
+                if (stopped || pumping) return;
+                pumping = true;
+                try {
+                  const current = execution()!;
+                  const records: RunEvent[] = [];
+                  for (;;) {
+                    const page = recording.events(cursor);
+                    if (!page.length) break;
+                    records.push(...page.map((item) => item.record));
+                    cursor = page.at(-1)!.sequence;
+                    if (page.length < 1000) break;
+                  }
+                  history.push(...records);
+                  const final = current.state !== "running";
+                  if (!snapshotSent || final) {
+                    const document = await readRecording(
+                      recording.directory,
+                      current,
+                      history,
+                    );
+                    send("snapshot", {
+                      name: stage,
+                      document,
+                      executionId,
+                    } satisfies SessionSnapshot);
+                    snapshotSent = true;
+                  } else if (records.length) send("records", records);
+                  send("state", {
+                    status:
+                      current.state === "running"
+                        ? "in_progress"
+                        : current.state === "completed" ||
+                            current.state === "stopped"
+                          ? "completed"
+                          : "failed",
+                    stage,
+                  });
+                  if (final) {
+                    send("done", {});
+                    stop();
+                  }
+                } catch (error) {
+                  send("session-error", {
+                    message:
+                      error instanceof Error ? error.message : String(error),
+                  });
+                  stop();
+                } finally {
+                  pumping = false;
+                }
+              };
+              controller.enqueue(encoder.encode("retry: 1000\n\n"));
+              timer = setInterval(() => void pump(), 250);
+              void pump();
+              if (request.signal.aborted) stop();
+            },
+            cancel() {
+              stop();
+            },
+          });
+          return new Response(body, {
+            headers: {
+              "Content-Type": "text/event-stream",
+              "Cache-Control": "no-cache, no-transform",
+              "X-Accel-Buffering": "no",
+            },
+          });
+        }
+        if (options.assetsPath && !url.pathname.startsWith("/api/")) {
+          const assets = resolve(options.assetsPath),
+            path = resolve(assets, "." + decodeURIComponent(url.pathname));
+          if (path !== assets && !path.startsWith(assets + sep))
+            return new Response("Not found", { status: 404 });
+          const file = Bun.file(path);
+          return new Response(
+            url.pathname !== "/" && (await file.exists())
+              ? file
+              : Bun.file(resolve(assets, "index.html")),
+          );
+        }
+        return new Response("Not found", { status: 404 });
+      } catch (error) {
+        return Response.json(
+          { error: error instanceof Error ? error.message : String(error) },
+          { status: 400 },
+        );
+      }
+    },
+  });
+  return {
+    url: `http://127.0.0.1:${server.port}`,
+    server,
+    async close() {
+      watcher.close();
+      if (changed) clearTimeout(changed);
+      for (const stop of streams) stop();
+      for (const client of clients)
+        try {
+          client.close();
+        } catch {}
+      await server.stop(true);
+    },
+  };
+}

package/src/app/snapshot-run.ts

@@ -0,0 +1,42 @@
+import { mkdir, writeFile } from "node:fs/promises";
+import { resolve, dirname } from "node:path";
+import { readBenchmarkRun } from "./read-run";
+import { fingerprint } from "../infra/files";
+import { benchmarkScores } from "./scores";
+
+/** Named selection snapshots, never copies of candidate execution.
+ * Reproducible audit inputs: https://arxiv.org/html/2607.07946#S7
+ */
+export async function snapshotBenchmarkRun(directory: string, name: string) {
+  if (!/^[a-zA-Z0-9][\w-]{0,63}$/.test(name))
+    throw new Error("Use a short alphanumeric snapshot name");
+  const root = resolve(directory),
+    records = readBenchmarkRun(root);
+  const slots = records.slots.filter((slot) => slot.active);
+  const payload = {
+    version: 1,
+    name,
+    capturedAt: new Date().toISOString(),
+    benchmark: records.benchmark,
+    slots,
+    evaluations: records.evals.filter((run) =>
+      slots.some((slot) => slot.evalRunId === run.id),
+    ),
+    judgments: records.judges.filter((run) =>
+      slots.some((slot) => slot.judgeRunId === run.id),
+    ),
+    scores: benchmarkScores(
+      records.benchmark.definition,
+      slots,
+      records.judges,
+    ),
+  };
+  const path = resolve(root, "maintenance/snapshots", `${name}.json`);
+  await mkdir(dirname(path), { recursive: true });
+  await writeFile(
+    path,
+    JSON.stringify({ hash: fingerprint(payload), payload }, null, 2),
+    { flag: "wx" },
+  );
+  return { path, hash: fingerprint(payload), slots: slots.length };
+}

package/src/app/watch-evidence.ts

@@ -0,0 +1,61 @@
+import type { EvidenceFeed, EvidenceView, JudgeSession } from "../evidence";
+import type { EarlyDecision } from "../types";
+
+/** Serial checks against fixed prefixes. Execution and persistence belong to the caller. */
+export async function watchEvidence(
+  feed: EvidenceFeed,
+  check: JudgeSession["check"],
+  signal: AbortSignal,
+  options: {
+    minIntervalMs?: number;
+    maxChecks?: number;
+    canCheck?: () => boolean;
+    onLimit?: (reason: string) => void;
+  } = {},
+): Promise<
+  | {
+      view: EvidenceView;
+      decision: Extract<EarlyDecision, { kind: "decided" }>;
+    }
+  | undefined
+> {
+  let revision = -1,
+    checks = 0,
+    lastStarted = 0;
+  for (;;) {
+    signal.throwIfAborted();
+    if (
+      checks >= (options.maxChecks ?? Infinity) ||
+      options.canCheck?.() === false
+    ) {
+      options.onLimit?.(
+        checks >= (options.maxChecks ?? Infinity)
+          ? "Monitor check limit reached"
+          : "Monitor cost limit reached",
+      );
+      return;
+    }
+    const wait = lastStarted + (options.minIntervalMs ?? 0) - Date.now();
+    if (wait > 0)
+      await new Promise<void>((resolve, reject) => {
+        const abort = () => {
+          clearTimeout(timer);
+          signal.removeEventListener("abort", abort);
+          reject(signal.reason);
+        };
+        const timer = setTimeout(() => {
+          signal.removeEventListener("abort", abort);
+          resolve();
+        }, wait);
+        signal.addEventListener("abort", abort, { once: true });
+        if (signal.aborted) abort();
+      });
+    const view = await feed.next(revision, signal);
+    if (!view) return;
+    lastStarted = Date.now();
+    checks++;
+    const decision = await check(view, signal);
+    revision = view.checkpoint.revision;
+    if (decision.kind === "decided") return { view, decision };
+  }
+}

package/src/cli.ts

@@ -0,0 +1,145 @@
+#!/usr/bin/env bun
+import { fileURLToPath } from "node:url";
+import { resolve } from "node:path";
+import {
+  runBenchmark,
+  loadBenchmark,
+  buildImage,
+  addModels,
+  retryEvalRun,
+  judgeRun,
+  serveResults,
+  mergeBenchmarkRuns,
+  snapshotBenchmarkRun,
+} from "./index";
+import type { ModelRef } from "./index";
+
+const args = process.argv.slice(2),
+  command = args[0] ?? "run";
+const option = (flag: string) => {
+  const index = args.indexOf(flag);
+  return index >= 0 ? args[index + 1] : undefined;
+};
+const models = (flag = "--model") =>
+  args.flatMap((arg, index) =>
+    arg === flag ? [args[index + 1] as ModelRef] : [],
+  );
+const benchmark = resolve(
+  option("--benchmark") ?? process.cwd(),
+);
+try {
+  if (command === "--help" || command === "help" || args.includes("--help")) {
+    console.log(`Usage: openeval <command> [options]
+
+Commands:
+  image                 Build the candidate container image
+  plan                  Show missing or changed work without executing it
+  run                   Execute missing or changed work in the current result
+  view                  Serve the results viewer
+  snapshot <run> <name>  Export a score snapshot
+  merge-runs <to> <from> Merge results into one aggregate
+  add-models <run>       Add models with repeated --model flags
+  retry <run> <eval-run> Retry a candidate execution
+  rejudge <run> <eval-run> Judge the saved evidence again
+
+Options:
+  --benchmark <dir>      Benchmark directory (default: current directory)
+  --run <dir>            Resume a specific result directory
+  --new                 Start a new result
+  --model <provider/id>  Add a model to the benchmark
+  --only-model <ref>     Execute only this model (repeatable)
+  --only-eval <id>       Execute only this eval (repeatable)
+  --only-repetition <n>  Execute only this repetition (repeatable)
+  --max-cost <usd>       Scheduling budget for this invocation
+  --final-only          Judge only after candidates finish
+  --port <port>         Viewer port (default: 4173)
+
+Requires Bun 1.4.2+, Docker, and an authenticated OpenCode installation.`);
+  } else if (command === "--version") {
+    console.log((await Bun.file(new URL("../package.json", import.meta.url)).json()).version);
+  } else if (command === "image") {
+    await buildImage((await loadBenchmark(benchmark)).container);
+    console.log("Candidate image ready");
+  } else if (command === "view") {
+    const viewer = await serveResults({
+      resultsPath: resolve(benchmark, "results"),
+      port: Number(option("--port") ?? 4173),
+      assetsPath: fileURLToPath(new URL("../viewer/", import.meta.url)),
+    });
+    console.log(viewer.url);
+  } else if (command === "run" || command === "plan") {
+    const result = await runBenchmark(benchmark, {
+      fresh: args.includes("--new"),
+      directory: option("--run"),
+      models: models().length ? models() : undefined,
+      onlyModels: models("--only-model").length
+        ? models("--only-model")
+        : undefined,
+      onlyEvals: args.flatMap((arg, index) =>
+        arg === "--only-eval" ? [args[index + 1]] : [],
+      ).length
+        ? args.flatMap((arg, index) =>
+            arg === "--only-eval" ? [args[index + 1]] : [],
+          )
+        : undefined,
+      maxCostUSD: args.includes("--max-cost")
+        ? Number(option("--max-cost"))
+        : undefined,
+      finalOnly: args.includes("--final-only"),
+      onlyRepetitions: args.includes("--only-repetition")
+        ? args.flatMap((arg, index) =>
+            arg === "--only-repetition" ? [Number(args[index + 1])] : [],
+          )
+        : undefined,
+      dryRun: command === "plan",
+    });
+    console.log(
+      JSON.stringify(
+        {
+          directory: result.directory,
+          estimate: result.estimate,
+          work: result.plan.map((item) => ({
+            eval: item.slot.evalId,
+            model: item.slot.model,
+            repetition: item.slot.repetition,
+            action: item.action,
+            reason: item.reason,
+          })),
+          status: result.benchmark?.state,
+        },
+        null,
+        2,
+      ),
+    );
+  } else if (command === "snapshot") {
+    if (!args[1] || !args[2])
+      throw new Error("snapshot requires a run directory and snapshot name");
+    console.log(await snapshotBenchmarkRun(resolve(args[1]), args[2]));
+  } else if (command === "merge-runs") {
+    if (!args[1] || !args[2])
+      throw new Error(
+        "merge-runs requires the aggregate and source run directories",
+      );
+    console.log(await mergeBenchmarkRuns(resolve(args[1]), resolve(args[2])));
+  } else if (command === "add-models") {
+    if (!args[1])
+      throw new Error("add-models requires a benchmark run directory");
+    console.log((await addModels(resolve(args[1]), models())).directory);
+  } else if (command === "retry" || command === "rejudge") {
+    if (!args[1] || !args[2])
+      throw new Error(
+        `${command} requires a benchmark run directory and eval run ID`,
+      );
+    const result =
+      command === "retry"
+        ? await retryEvalRun(resolve(args[1]), args[2])
+        : await judgeRun(resolve(args[1]), args[2]);
+    console.log(JSON.stringify(result, null, 2));
+  } else
+    throw new Error(
+      "Commands: image, plan, run, view, add-models, merge-runs, retry, rejudge, snapshot",
+    );
+} catch (error) {
+  console.error(error instanceof Error ? error.message : String(error));
+  process.exitCode = 1;
+}

package/src/evidence.ts

@@ -0,0 +1,40 @@
+import type { EvidenceCheckpoint, Judgment, EarlyDecision } from "./types";
+export type EvidenceQuery = {
+  action:
+    | "summary"
+    | "response"
+    | "events"
+    | "event"
+    | "messages"
+    | "message"
+    | "tools"
+    | "tool"
+    | "artifacts"
+    | "artifact"
+    | "diff";
+  sessionID?: string;
+  type?: string;
+  id?: string;
+  path?: string;
+  revision?: "initial" | "final";
+  offset?: number;
+  limit?: number;
+  encoding?: "utf8" | "base64";
+};
+
+/** Read-only capabilities shared by recorded archives and fixed live prefixes. */
+export interface EvidenceReader {
+  summary(): Record<string, unknown>;
+  query(input: EvidenceQuery): Promise<Record<string, unknown>>;
+}
+export interface EvidenceView extends EvidenceReader {
+  readonly checkpoint: EvidenceCheckpoint;
+}
+export interface EvidenceFeed {
+  /** Wait for meaningful evidence; coalesce pending changes to the latest prefix. */
+  next(after: number, signal: AbortSignal): Promise<EvidenceView | undefined>;
+}
+export interface JudgeSession {
+  check(evidence: EvidenceView, signal: AbortSignal): Promise<EarlyDecision>;
+  grade(evidence: EvidenceView, signal: AbortSignal): Promise<Judgment>;
+}

package/src/index.ts

@@ -0,0 +1,53 @@
+export type {
+  Benchmark,
+  Eval,
+  ModelRef,
+  ProviderDefinitions,
+  Judge,
+  BenchmarkRun,
+  EvalRun,
+  JudgeRun,
+  Judgment,
+  MetricDefinition,
+  MetricJudgment,
+  EvidenceCitation,
+  ToolCall,
+  MonitorPolicy,
+  SessionArchive,
+  EvidenceRef,
+  EvidenceCheckpoint,
+  EarlyDecision,
+  JudgeCheck,
+  EvalStop,
+  PlanItem,
+  Slot,
+} from "./types";
+export type {
+  EvidenceReader,
+  EvidenceView,
+  EvidenceFeed,
+  EvidenceQuery,
+  JudgeSession,
+} from "./evidence";
+export { CANDIDATE_TIMEOUT_MS } from "./types";
+export { rubricMetrics, metricMean, isScored } from "./judgment";
+export { loadBenchmark } from "./app/load-benchmark";
+export {
+  runBenchmark,
+  currentBenchmarkRun,
+  type RunBenchmarkOptions,
+} from "./app/run-benchmark";
+export { addModels } from "./app/add-models";
+export type { CostEstimate } from "./app/cost-plan";
+export { mergeBenchmarkRuns } from "./app/merge-runs";
+export { retryEvalRun } from "./app/retry-run";
+export { judgeRun, judgeRuns } from "./app/rejudge";
+export { judgeEvidence, recordEvidence } from "./app/judge-evidence";
+export {
+  readBenchmarkRun,
+  readEvidence,
+  verifyJudgmentEvidence,
+} from "./app/read-run";
+export { snapshotBenchmarkRun } from "./app/snapshot-run";
+export { buildImage } from "./infra/containers/oci";
+export { serveResults } from "./app/serve-results";