@hona/openeval 0.2.1 → 0.3.0

package/src/cli.ts

@@ -6,6 +6,7 @@ import {
   loadBenchmark,
   buildImage,
   addModels,
+  removeModels,
   retryEvalRun,
   judgeRun,
   serveResults,
@@ -39,6 +40,7 @@ Commands:
   snapshot <run> <name>  Export a score snapshot
   merge-runs <to> <from> Merge results into one aggregate
   add-models <run>       Add models with repeated --model flags
+  remove-models <run>    Remove active models while retaining their evidence
   retry <run> <eval-run> Retry a candidate execution
   rejudge <run> <eval-run> Judge the saved evidence again
 
@@ -46,7 +48,7 @@ Options:
   --benchmark <dir>      Benchmark directory (default: current directory)
   --run <dir>            Resume a specific result directory
   --new                 Start a new result
-  --model <provider/id>  Add a model to the benchmark
+  --model <provider/id>  Model to add or remove (repeatable)
   --only-model <ref>     Execute only this model (repeatable)
   --only-eval <id>       Execute only this eval (repeatable)
   --only-repetition <n>  Execute only this repetition (repeatable)
@@ -125,6 +127,16 @@ Requires Bun 1.4.2+, Docker, and an authenticated OpenCode installation.`);
     if (!args[1])
       throw new Error("add-models requires a benchmark run directory");
     console.log((await addModels(resolve(args[1]), models())).directory);
+  } else if (command === "remove-models") {
+    if (!args[1])
+      throw new Error("remove-models requires a benchmark run directory");
+    const result = await removeModels(resolve(args[1]), models());
+    console.log(JSON.stringify({
+      directory: result.directory,
+      removed: result.removed,
+      retiredSlots: result.retiredSlots,
+      status: result.benchmark.state,
+    }, null, 2));
   } else if (command === "retry" || command === "rejudge") {
     if (!args[1] || !args[2])
       throw new Error(
@@ -137,7 +149,7 @@ Requires Bun 1.4.2+, Docker, and an authenticated OpenCode installation.`);
     console.log(JSON.stringify(result, null, 2));
   } else
     throw new Error(
-      "Commands: image, plan, run, view, add-models, merge-runs, retry, rejudge, snapshot",
+      "Commands: image, plan, run, view, add-models, remove-models, merge-runs, retry, rejudge, snapshot",
     );
 } catch (error) {
   console.error(error instanceof Error ? error.message : String(error));

package/src/evidence.ts

@@ -11,7 +11,10 @@ export type EvidenceQuery = {
     | "tool"
     | "artifacts"
     | "artifact"
-    | "diff";
+    | "diff"
+    | "metrics";
+  /** A dot-separated measurement path for the metrics query. */
+  metric?: string;
   sessionID?: string;
   type?: string;
   id?: string;

package/src/index.ts

@@ -8,8 +8,8 @@ export type {
   EvalRun,
   JudgeRun,
   Judgment,
-  MetricDefinition,
-  MetricJudgment,
+  CriterionDefinition,
+  CriterionScore,
   EvidenceCitation,
   ToolCall,
   MonitorPolicy,
@@ -30,7 +30,18 @@ export type {
   JudgeSession,
 } from "./evidence";
 export { CANDIDATE_TIMEOUT_MS } from "./types";
-export { rubricMetrics, metricMean, isScored } from "./judgment";
+export { rubricCriteria, criterionMean, isScored } from "./judgment";
+export type {
+  JudgeContext,
+  JudgeFunction,
+  JsonValue,
+  ScoreValue,
+  RunMetrics,
+  ToolMetrics,
+  RecordedEvent,
+  RecordedMessage,
+  RecordedFile,
+} from "./judge-context";
 export { loadBenchmark } from "./app/load-benchmark";
 export {
   runBenchmark,
@@ -38,6 +49,7 @@ export {
   type RunBenchmarkOptions,
 } from "./app/run-benchmark";
 export { addModels } from "./app/add-models";
+export { removeModels } from "./app/remove-models";
 export type { CostEstimate } from "./app/cost-plan";
 export { mergeBenchmarkRuns } from "./app/merge-runs";
 export { retryEvalRun } from "./app/retry-run";
@@ -46,6 +58,7 @@ export { judgeEvidence, recordEvidence } from "./app/judge-evidence";
 export {
   readBenchmarkRun,
   readEvidence,
+  readRecording,
   verifyJudgmentEvidence,
 } from "./app/read-run";
 export { snapshotBenchmarkRun } from "./app/snapshot-run";

package/src/infra/evidence/index.ts

@@ -11,6 +11,7 @@ import type { EvidenceFeed, EvidenceView, EvidenceQuery } from "../../evidence";
 export type { EvidenceQuery } from "../../evidence";
 import { trackTool } from "./tool-calls";
 import { hash as sha256, writeJson } from "../files";
+import { measureRecording } from "../recording/metrics";
 type InferenceResponse = { text: string; textBlocks?: string[] };
 type ArtifactPolicy = { excludeDirectories: string[] };
 
@@ -25,7 +26,7 @@ export type EvidenceQueryAudit = {
   checkId?: string;
   checkpoint?: EvidenceCheckpoint;
 };
-type FileEntry = {
+export type FileEntry = {
   path: string;
   bytes: number;
   sha256?: string;
@@ -37,7 +38,7 @@ export type EvidenceEvent = {
   time: string;
   event: OpenCodeStreamEvent;
 };
-type EvidenceManifest = {
+export type EvidenceManifest = {
   version: 1;
   handle: "candidate";
   coverage: "recorded-session" | "recorded-prefix";
@@ -446,6 +447,28 @@ export class CandidateEvidence {
     readonly hash: string,
   ) {}
 
+  /** Native records for trusted code judges. LLM queries use the separate blinded view. */
+  rawEvents(): EvidenceEvent[] {
+    return structuredClone(this.records);
+  }
+
+  toolCalls(): ToolCall[] {
+    return structuredClone(this.manifest.tools);
+  }
+
+  files(revision: "initial" | "final" = "final"): FileEntry[] {
+    const snapshot = this.manifest[revision];
+    if (!snapshot)
+      throw new Error(`${revision} workspace snapshot is unavailable`);
+    return structuredClone(snapshot.files);
+  }
+
+  async readFile(path: string, revision: "initial" | "final" = "final") {
+    const entry = this.entry(path, revision);
+    if (!entry) throw new Error(`Artifact not found: ${path}`);
+    return this.bytes(entry);
+  }
+
   view(reference: EvidenceRef): EvidenceView {
     if (reference.hash !== this.hash)
       throw new Error("Evidence view reference does not match its archive");
@@ -629,6 +652,20 @@ export class CandidateEvidence {
 
   async query(input: EvidenceQuery): Promise<Record<string, unknown>> {
     if (input.action === "summary") return this.summary();
+    if (input.action === "metrics") {
+      const metrics = measureRecording(this.records, this.manifest.tools, {
+        directory: this.directory,
+        hash: this.hash,
+      });
+      if (!input.metric) return { ...metrics };
+      let value: unknown = metrics;
+      for (const key of input.metric.split(".")) {
+        if (!value || typeof value !== "object" || !Object.hasOwn(value, key))
+          throw new Error(`Unknown recorded metric: ${input.metric}`);
+        value = (value as Record<string, unknown>)[key];
+      }
+      return { metric: input.metric, value, evidence: metrics.evidence };
+    }
     if (input.action === "response")
       return textPage(this.manifest.response.text, input);
     if (input.action === "events")

package/src/infra/evidence/tool-calls.ts

@@ -8,6 +8,7 @@ export function trackTool(
   if (!("data" in event)) return;
   const data = event.data as {
     id?: string;
+    sessionID?: string;
     name?: string;
     assistantMessageID?: string;
     input?: Record<string, unknown>;
@@ -17,8 +18,10 @@ export function trackTool(
     executed?: boolean;
   };
   if (!data.id) return;
-  const value = calls.get(data.id) ?? {
+  const key = `${data.sessionID ?? ""}:${data.assistantMessageID ?? ""}:${data.id}`;
+  const value = calls.get(key) ?? {
     id: data.id,
+    sessionID: data.sessionID,
     name: data.name ?? "unknown",
     assistantMessageId: data.assistantMessageID ?? "",
     status: "preparing" as const,
@@ -45,5 +48,5 @@ export function trackTool(
     if (value.startedAt !== undefined)
       value.durationMs = event.created - value.startedAt;
   }
-  calls.set(value.id, value);
+  calls.set(key, value);
 }

package/src/infra/judging/agent.ts

@@ -18,7 +18,7 @@ export const JUDGE_TOOLS = [
  */
 export const JUDGE_AGENT = {
   id: "openeval-judge",
-  description: "Grades immutable recorded work against eval metrics",
+  description: "Grades immutable recorded work against eval criteria",
   mode: "primary",
   system: readFileSync(new URL("./judge-agent.md", import.meta.url), "utf8"),
   permissions: [
@@ -35,6 +35,7 @@ export async function judgeConfiguration(
   input: Pick<JudgeRunInput, "agent" | "rubric" | "websearch">,
   directory: string,
 ) {
+  if (!input.agent) throw new Error("An LLM judge requires an agent profile");
   const { id, ...agent } = input.agent;
   const configuration = resolve(directory, "configuration");
   const config = {
@@ -45,7 +46,7 @@ export async function judgeConfiguration(
         ...agent,
         // Keep the rubric in the native system prompt so compaction cannot lose
         // it. Request context and result data move through registered tools.
-        system: `${agent.system.trim()}\n\n# Eval metrics\n\n${input.rubric.trim()}`,
+        system: `${agent.system.trim()}\n\n# Eval rubric\n\n${input.rubric.trim()}`,
       },
     },
     websearch: input.websearch ? { provider: input.websearch } : false,

package/src/infra/judging/code-result.ts

@@ -0,0 +1,102 @@
+import type { JsonValue } from "../../judge-context";
+import type { CriterionScore, Judgment } from "../../types";
+import { criterionMean, isScored } from "../../judgment";
+
+/** Preserve author JSON exactly; reject values JSON would silently discard or coerce. */
+export function jsonOutput(
+  value: unknown,
+  path = "output",
+  seen = new Set<object>(),
+): JsonValue {
+  if (value === null || typeof value === "boolean" || typeof value === "string")
+    return value;
+  if (typeof value === "number" && Number.isFinite(value)) return value;
+  if (typeof value !== "object" || !value || seen.has(value))
+    throw new Error(
+      `${path} must be JSON-compatible (no undefined, non-finite numbers, or cycles)`,
+    );
+  if (
+    !Array.isArray(value) &&
+    ![Object.prototype, null].includes(Object.getPrototypeOf(value))
+  )
+    throw new Error(`${path} must be a plain JSON object`);
+  if (Object.getOwnPropertySymbols(value).length)
+    throw new Error(`${path} contains a symbol key`);
+  seen.add(value);
+  try {
+    if (Array.isArray(value))
+      return Array.from(value, (item, index) =>
+        jsonOutput(item, `${path}[${index}]`, seen),
+      );
+    return Object.fromEntries(
+      Object.entries(value).map(([key, item]) => [
+        key,
+        jsonOutput(item, `${path}.${key}`, seen),
+      ]),
+    );
+  } finally {
+    seen.delete(value);
+  }
+}
+
+export function codeJudgment(output: JsonValue): Judgment {
+  const values =
+    output &&
+    typeof output === "object" &&
+    !Array.isArray(output) &&
+    "scores" in output
+      ? output.scores
+      : {};
+  if (!values || typeof values !== "object" || Array.isArray(values))
+    throw new Error("output.scores must be an object keyed by criterion ID");
+  const scores = Object.fromEntries(
+    Object.entries(values).map(([id, supplied]): [string, CriterionScore] => {
+      if (!/^[a-z][a-z0-9_]*$/.test(id))
+        throw new Error(`Invalid criterion ID: ${id}`);
+      const value = typeof supplied === "boolean" ? Number(supplied) : supplied;
+      if (value !== null && !isScored(value))
+        throw new Error(
+          `scores.${id} must be a boolean, a finite number from 0 to 1, or null`,
+        );
+      return [
+        id,
+        {
+          value: value as number | null,
+          reason:
+            value === null
+              ? "judge.ts returned an unresolved score."
+              : `judge.ts returned ${String(supplied)}.`,
+          evidence: value === null ? [] : [{ kind: "recording" }],
+          source: "judge.ts",
+        },
+      ];
+    }),
+  );
+  return {
+    scores,
+    value: criterionMean(scores),
+    reason: Object.keys(scores).length
+      ? "Criterion scores computed by judge.ts."
+      : "The code judge returned data without criterion scores.",
+  };
+}
+
+export function combineJudgments(...judgments: Judgment[]): Judgment {
+  const scores: Record<string, CriterionScore> = {};
+  for (const judgment of judgments)
+    for (const [id, score] of Object.entries(judgment.scores)) {
+      if (Object.hasOwn(scores, id))
+        throw new Error(
+          `Duplicate criterion ID from judge.md and judge.ts: ${id}`,
+        );
+      scores[id] = score;
+    }
+  return {
+    scores,
+    value: criterionMean(scores),
+    reason: judgments
+      .map((judgment) => judgment.reason)
+      .filter(Boolean)
+      .join("\n"),
+  };
+}

package/src/infra/judging/code-source.ts

@@ -0,0 +1,96 @@
+import { dirname, isAbsolute, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+import type { CodeJudgeDefinition } from "../../judge-context";
+import { fingerprint } from "../files";
+
+/** Bundle local imports without executing author code during planning. */
+export async function compileCodeJudge(
+  file: string,
+): Promise<CodeJudgeDefinition> {
+  file = resolve(file);
+  const dependencies: Record<string, string> = {};
+  const sdkManifest = fileURLToPath(
+    new URL("../../../package.json", import.meta.url),
+  );
+  dependencies[sdkManifest] = await Bun.file(sdkManifest).text();
+  const external = new Set<string>();
+  const built = await Bun.build({
+    entrypoints: [file],
+    root: dirname(file),
+    target: "bun",
+    format: "esm",
+    packages: "external",
+    sourcemap: "external",
+    plugins: [
+      {
+        name: "judge-dependencies",
+        setup(build) {
+          build.onResolve({ filter: /^[^./]/ }, (args) => {
+            if (
+              args.path.startsWith("node:") ||
+              args.path.startsWith("bun:") ||
+              isAbsolute(args.path)
+            )
+              return;
+            const path = Bun.resolveSync(
+              args.path,
+              args.resolveDir || dirname(file),
+            );
+            external.add(path);
+            return { path, external: true };
+          });
+        },
+      },
+    ],
+  });
+  if (!built.success)
+    throw new Error(`Cannot compile judge.ts: ${built.logs.join("\n")}`);
+  const source = await built.outputs
+    .find((output) => output.kind === "entry-point")!
+    .text();
+  if (
+    !new Bun.Transpiler({ loader: "js" })
+      .scan(source)
+      .exports.includes("default")
+  )
+    throw new Error("judge.ts must have a default function export");
+  const sourceMap =
+    (await built.outputs
+      .find((output) => output.kind === "sourcemap")
+      ?.text()) ?? "";
+  const collect = async (directory: string, stopAtPackage: boolean) => {
+    for (;;) {
+      let found = false;
+      for (const name of [
+        "package.json",
+        "bun.lock",
+        "bun.lockb",
+        "package-lock.json",
+        "pnpm-lock.yaml",
+        "yarn.lock",
+      ]) {
+        const path = resolve(directory, name);
+        if (await Bun.file(path).exists()) {
+          dependencies[path] =
+            name === "bun.lockb"
+              ? Buffer.from(await Bun.file(path).bytes()).toString("base64")
+              : await Bun.file(path).text();
+          found ||= name === "package.json";
+        }
+      }
+      if (found && stopAtPackage) return;
+      const parent = dirname(directory);
+      if (parent === directory) return;
+      directory = parent;
+    }
+  };
+  await collect(dirname(file), false);
+  for (const path of external) await collect(dirname(path), true);
+  return {
+    file,
+    source,
+    sourceMap,
+    dependencies,
+    hash: fingerprint({ source, sourceMap, dependencies }),
+  };
+}

package/src/infra/judging/code-worker.ts

@@ -0,0 +1,26 @@
+import { pathToFileURL } from "node:url";
+import { openRecording, type RecordingInput } from "../recording";
+import { jsonOutput } from "./code-result";
+import { errorMessage, writeJson } from "../files";
+import type { JudgeFunction, RunMetrics } from "../../judge-context";
+
+type Job = RecordingInput & { module: string; output: string; scratch: string };
+const job: Job = JSON.parse(await Bun.stdin.text());
+let metrics: RunMetrics | undefined;
+async function grade() {
+  await using context = await openRecording(job, job.scratch);
+  metrics = context.metrics;
+  const module = await import(pathToFileURL(job.module).href);
+  if (typeof module.default !== "function")
+    throw new Error("judge.ts must default-export a function");
+  const output = jsonOutput(await (module.default as JudgeFunction)(context));
+  return { output, metrics: context.metrics };
+}
+try {
+  await writeJson(job.output, await grade());
+  process.exit(0);
+} catch (error) {
+  await writeJson(job.output, { error: errorMessage(error), metrics });
+  console.error(error);
+  process.exit(1);
+}

package/src/infra/judging/code.ts

@@ -0,0 +1,97 @@
+import { mkdir, rm } from "node:fs/promises";
+import { resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+import type {
+  CodeJudgeDefinition,
+  CodeJudgeExecution,
+  JsonValue,
+  RunMetrics,
+} from "../../judge-context";
+import type { RecordingInput } from "../recording";
+import { codeJudgment, jsonOutput } from "./code-result";
+import { errorMessage, writeJson } from "../files";
+
+/** A separate process bounds synchronous loops as well as asynchronous judges. */
+export async function executeCodeJudge(
+  code: CodeJudgeDefinition,
+  recording: RecordingInput,
+  directory: string,
+  timeoutMs: number,
+): Promise<CodeJudgeExecution> {
+  directory = resolve(directory);
+  await mkdir(directory, { recursive: true });
+  const module = resolve(directory, "judge.js"),
+    output = resolve(directory, "output.json");
+  const scratch = resolve(directory, "scratch");
+  const stdout = resolve(directory, "stdout.log"),
+    stderr = resolve(directory, "stderr.log");
+  await Bun.write(module, code.source);
+  await Bun.write(resolve(directory, "judge.js.map"), code.sourceMap);
+  await writeJson(resolve(directory, "dependencies.json"), code.dependencies);
+  const startedAt = new Date().toISOString();
+  let timer: ReturnType<typeof setTimeout> | undefined;
+  let timedOut = false;
+  let raw: JsonValue | undefined, metrics: RunMetrics | undefined;
+  const base = { startedAt, sourceHash: code.hash, stdout, stderr };
+  try {
+    const child = Bun.spawn(
+      [
+        process.execPath,
+        fileURLToPath(new URL("./code-worker.ts", import.meta.url)),
+      ],
+      {
+        cwd: directory,
+        stdin: "pipe",
+        stdout: Bun.file(stdout),
+        stderr: Bun.file(stderr),
+        windowsHide: true,
+      },
+    );
+    timer = setTimeout(() => {
+      timedOut = true;
+      child.kill();
+    }, timeoutMs);
+    await child.stdin.write(
+      JSON.stringify({ ...recording, module, output, scratch }),
+    );
+    child.stdin.end();
+    const exit = await child.exited;
+    if (timedOut)
+      throw new Error(`judge.ts exceeded its ${timeoutMs}ms time limit`);
+    const result = await Bun.file(output).json();
+    metrics = result.metrics;
+    if (exit !== 0 || result.error)
+      throw new Error(result.error ?? `judge.ts exited with code ${exit}`);
+    raw = jsonOutput(result.output);
+    const judgment = codeJudgment(raw);
+    return {
+      ...base,
+      state: "completed",
+      completedAt: new Date().toISOString(),
+      elapsedMs: Date.now() - Date.parse(startedAt),
+      output: raw,
+      metrics,
+      scores: Object.fromEntries(
+        Object.entries(judgment.scores).map(([id, score]) => [id, score.value]),
+      ),
+    };
+  } catch (error) {
+    return {
+      ...base,
+      state: timedOut ? "timed_out" : "failed",
+      completedAt: new Date().toISOString(),
+      elapsedMs: Date.now() - Date.parse(startedAt),
+      error: errorMessage(error),
+      output: raw,
+      metrics,
+    };
+  } finally {
+    clearTimeout(timer);
+    await rm(scratch, {
+      recursive: true,
+      force: true,
+      maxRetries: 5,
+      retryDelay: 100,
+    });
+  }
+}