@hona/openeval 0.2.2 → 0.3.0

package/src/app/judge-run.ts

@@ -1,22 +1,20 @@
 import { resolve, relative } from "node:path";
 import type { EvalRun, JudgeRun } from "../types";
 import type { ExecutionContext } from "./context";
-import { executeJudge } from "../infra/judging";
+import { gradeRecording } from "./grade-recording";
 import { errorMessage } from "../infra/files";
 import { canJudgeEval } from "./eval-state";
 import { JUDGE_PROTOCOL } from "../judgment";
 import { JUDGE_AGENT } from "../infra/judging/agent";
 
-/** A new judge session references existing evidence and becomes active only after success. */
+/** A new grading execution becomes active only after all present judges succeed. */
 export async function judgeEvalRun(
   context: ExecutionContext,
   candidate: EvalRun,
   monitoring: Pick<JudgeRun, "monitorError" | "monitorErrorAt"> = {},
 ): Promise<JudgeRun> {
   if (!canJudgeEval(candidate))
-    throw new Error(
-      "Judging requires finalized evidence from a completed, stopped, or timed-out eval run",
-    );
+    throw new Error("Judging requires finalized candidate evidence");
   const slot = context.results.slot(candidate.slotId)!;
   const definition = context.definition.evals.find(
     (item) => item.id === slot.evalId,
@@ -25,20 +23,22 @@ export async function judgeEvalRun(
     evalRunId: candidate.id,
     evidence: candidate.evidence,
     rubric: definition.judge,
-    agent: JUDGE_AGENT,
-    model: context.definition.judge.model,
+    agent: definition.judge ? JUDGE_AGENT : undefined,
+    model: definition.judge ? context.definition.judge.model : undefined,
+    kind: definition.code ? (definition.judge ? "hybrid" : "code") : "llm",
+    code: definition.code,
     judgeHash: slot.judgeHash,
     timeoutMs: context.definition.judge.timeoutMs,
     websearch: context.definition.judge.websearch,
     mode: "final",
     runtimeHash: context.runtime.judgeHash,
     protocol: JUDGE_PROTOCOL,
-    metrics: definition.metrics,
+    criteria: definition.criteria,
   });
   const directory = resolve(context.directory, "judge-runs", run.id);
   let completed: JudgeRun;
   try {
-    const { result, session, judgment } = await executeJudge(
+    const graded = await gradeRecording(
       {
         ...run.input,
         evidence: {
@@ -46,6 +46,14 @@ export async function judgeEvalRun(
           directory: resolve(context.directory, run.input.evidence.directory),
         },
       },
+      {
+        evidence: {
+          ...candidate.evidence,
+          directory: resolve(context.directory, candidate.evidence.directory),
+        },
+        run: candidate,
+        runDirectory: context.directory,
+      },
       directory,
       (event) => {
         const record = {
@@ -61,16 +69,25 @@ export async function judgeEvalRun(
     completed = {
       ...run,
       ...monitoring,
-      state: result.state,
-      judgment,
-      session: {
-        ...session,
-        database: relative(
-          context.directory,
-          resolve(directory, session.database),
-        ),
-      },
-      error: result.error,
+      state: graded.state,
+      judgment: graded.judgment,
+      code: graded.code
+        ? {
+            ...graded.code,
+            stdout: relative(context.directory, graded.code.stdout),
+            stderr: relative(context.directory, graded.code.stderr),
+          }
+        : undefined,
+      session: graded.session
+        ? {
+            ...graded.session,
+            database: relative(
+              context.directory,
+              resolve(directory, graded.session.database),
+            ),
+          }
+        : undefined,
+      error: graded.error,
       completedAt: new Date().toISOString(),
       elapsedMs: Date.now() - Date.parse(run.startedAt),
     };

package/src/app/load-benchmark.ts

@@ -9,7 +9,8 @@ import type {
   ModelRef,
 } from "../types";
 import { CANDIDATE_TIMEOUT_MS } from "../types";
-import { rubricMetrics } from "../judgment";
+import { rubricCriteria } from "../judgment";
+import { compileCodeJudge } from "../infra/judging/code-source";
 import { monitorPolicy } from "./monitor-policy";
 import {
   fingerprint,
@@ -41,9 +42,14 @@ async function declaration<T>(path: string): Promise<T> {
 async function loadEval(directory: string): Promise<EvalDefinition> {
   const id = basename(directory),
     prompt = Bun.file(resolve(directory, "prompt.md")),
-    judge = Bun.file(resolve(directory, "judge.md"));
-  if (!(await prompt.exists()) || !(await judge.exists()))
-    throw new Error(`${id} requires prompt.md and judge.md`);
+    judge = Bun.file(resolve(directory, "judge.md")),
+    codeFile = resolve(directory, "judge.ts");
+  const hasMarkdown = await judge.exists(),
+    hasCode = await Bun.file(codeFile).exists();
+  if (!(await prompt.exists()) || (!hasMarkdown && !hasCode))
+    throw new Error(
+      `${id} requires prompt.md and at least one of judge.md or judge.ts`,
+    );
   const settings = (await Bun.file(resolve(directory, "eval.ts")).exists())
     ? await declaration<Eval>(resolve(directory, "eval.ts"))
     : {};
@@ -56,6 +62,10 @@ async function loadEval(directory: string): Promise<EvalDefinition> {
   )
     throw new Error(`${id}/eval.ts has unsupported settings`);
   monitorPolicy(settings.earlyStop);
+  if (hasCode && settings.earlyStop)
+    throw new Error(
+      `${id}: judge.ts grades finalized recordings; earlyStop requires a Markdown-only judge`,
+    );
   const source: unknown[] = [];
   if (settings.workspace) {
     const workspace = settings.workspace;
@@ -157,9 +167,10 @@ async function loadEval(directory: string): Promise<EvalDefinition> {
       throw new Error(`${id}: preparation requires argv`);
   }
   const promptText = await prompt.text(),
-    judgeText = await judge.text();
-  if (!promptText.trim() || !judgeText.trim())
+    judgeText = hasMarkdown ? await judge.text() : "";
+  if (!promptText.trim() || (hasMarkdown && !judgeText.trim()))
     throw new Error(`${id}: prompt and judge must not be empty`);
+  const code = hasCode ? await compileCodeJudge(codeFile) : undefined;
   return {
     id,
     directory,
@@ -170,8 +181,9 @@ async function loadEval(directory: string): Promise<EvalDefinition> {
       settings: { workspace: settings.workspace, prepare: settings.prepare },
       source,
     }),
-    judgeHash: hash(judgeText),
-    metrics: rubricMetrics(judgeText),
+    judgeHash: fingerprint({ rubric: judgeText, code: code?.hash }),
+    criteria: hasMarkdown ? rubricCriteria(judgeText) : [],
+    ...(code ? { code } : {}),
     name: /^# (.+)$/m.exec(judgeText)?.[1] ?? id,
   };
 }
@@ -187,10 +199,9 @@ export async function loadBenchmark(
   if (
     !definition ||
     !Array.isArray(definition.models) ||
-    !definition.models.length ||
-    !definition.judge
+    !definition.models.length
   )
-    throw new Error("benchmark.ts requires models and a judge");
+    throw new Error("benchmark.ts requires models");
   if (
     Object.keys(definition).some(
       (key) =>
@@ -207,6 +218,18 @@ export async function loadBenchmark(
   )
     throw new Error("benchmark.ts contains unsupported settings");
   const models = definition.models.map(modelRef);
+  if (
+    definition.judge !== undefined &&
+    (!definition.judge ||
+      typeof definition.judge !== "object" ||
+      Array.isArray(definition.judge) ||
+      Object.keys(definition.judge).some(
+        (key) => !["model", "timeoutMs", "websearch"].includes(key),
+      ))
+  )
+    throw new Error(
+      "judge must be an object with model, timeoutMs, or websearch settings",
+    );
   if (new Set(models).size !== models.length)
     throw new Error("Benchmark contains duplicate models");
   const timeoutMs = positive(
@@ -224,6 +247,8 @@ export async function loadBenchmark(
   const evals = await Promise.all(
     directories.map((entry) => loadEval(resolve(evalRoot, entry.name))),
   );
+  if (evals.some((item) => item.judge) && !definition.judge?.model)
+    throw new Error("A benchmark containing judge.md requires judge.model");
   const concurrency = positive(definition.concurrency, 10, "Concurrency");
   if (concurrency < 2 && evals.some((item) => item.settings.earlyStop))
     throw new Error(
@@ -234,7 +259,7 @@ export async function loadBenchmark(
     throw new Error("Container engine must be docker or podman");
   for (const search of [
     definition.candidate?.websearch,
-    definition.judge.websearch,
+    definition.judge?.websearch,
   ])
     if (search !== undefined && search !== false && search !== "exa")
       throw new Error("Websearch must be exa or false");
@@ -253,9 +278,15 @@ export async function loadBenchmark(
         : {}),
     },
     judge: {
-      model: modelRef(definition.judge.model),
-      timeoutMs: positive(definition.judge.timeoutMs, 600_000, "Judge timeout"),
-      websearch: definition.judge.websearch ?? "exa",
+      ...(definition.judge?.model
+        ? { model: modelRef(definition.judge.model) }
+        : {}),
+      timeoutMs: positive(
+        definition.judge?.timeoutMs,
+        600_000,
+        "Judge timeout",
+      ),
+      websearch: definition.judge?.websearch ?? "exa",
     },
     container: {
       engine,

package/src/app/read-results.ts

@@ -26,6 +26,7 @@ import {
 } from "../runtime";
 import { canJudgeEval } from "./eval-state";
 import { CandidateEvidence, type EvidenceQuery } from "../infra/evidence";
+import { contained } from "../infra/files";
 
 const scheduled = (slot: Slot, benchmark: BenchmarkRun, now: number) =>
   benchmark.state === "running" &&
@@ -33,6 +34,8 @@ const scheduled = (slot: Slot, benchmark: BenchmarkRun, now: number) =>
   benchmark.scheduledSlotIds.includes(slot.id);
 
 const costOf = (run: EvalRun | JudgeRun, results?: Results): Cost => {
+  if ("evalRunId" in run.input && run.input.kind === "code")
+    return { usd: 0, reportedUSD: 0, complete: true };
   const usd = run.session?.accounting?.costUSD;
   return {
     usd: usd ?? null,
@@ -162,6 +165,21 @@ export class ResultReader {
     using results = await this.database(benchmarkId);
     const judge = results.judgeRun(judgeRunId);
     if (!judge) throw new Error("Unknown judge run");
+    const candidate = results.evalRun(judge.input.evalRunId);
+    const criteria = new Map(
+      judge.input.criteria.map((criterion) => [criterion.id, criterion]),
+    );
+    for (const id of Object.keys(judge.judgment?.scores ?? {}))
+      if (!criteria.has(id))
+        criteria.set(id, { id, name: id.replaceAll("_", " ") });
+    const log = async (path?: string) =>
+      path
+        ? (
+            await Bun.file(contained(dirname(results.path), path))
+              .text()
+              .catch(() => "")
+          ).slice(0, 40_000)
+        : undefined;
     return {
       judge: {
         id: judge.id,
@@ -172,8 +190,15 @@ export class ResultReader {
         monitorErrorAt: judge.monitorErrorAt,
         monitorLimit: judge.monitorLimit,
         judgment: judge.judgment,
+        code: judge.code,
+        error: judge.error,
       },
-      metrics: judge.input.metrics,
+      kind: judge.input.kind,
+      criteria: [...criteria.values()],
+      metrics: candidate?.metrics ?? judge.code?.metrics,
+      codeSource: judge.input.code?.source,
+      codeStdout: await log(judge.code?.stdout),
+      codeStderr: await log(judge.code?.stderr),
       stop: results.evalRun(judge.input.evalRunId)?.stop,
       checks: results.judgeChecks(judgeRunId).map((check) => {
         const start = timestampMs(check.executionStartedAt);
@@ -421,6 +446,7 @@ function stageState(
     runtime,
     cost: costOf(run, results),
     message: run.error,
+    ...("kind" in run.input ? { kind: run.input.kind } : {}),
   };
 }
 
@@ -470,6 +496,13 @@ function liveRun(
       evalStage.status,
     );
   const judgeStage: StageState = {
+    kind: benchmark.definition.evals.find((item) => item.id === slot.evalId)
+      ?.code
+      ? benchmark.definition.evals.find((item) => item.id === slot.evalId)
+          ?.judge
+        ? "hybrid"
+        : "code"
+      : "llm",
     ...stageState(
       judge,
       stale,
@@ -486,7 +519,7 @@ function liveRun(
     ...(judge?.state === "completed"
       ? {
           score: judge.judgment?.value ?? null,
-          metrics: judge.judgment?.metrics,
+          scores: judge.judgment?.scores,
         }
       : {}),
   };

package/src/app/read-run.ts

@@ -4,6 +4,8 @@ import { CandidateEvidence } from "../infra/evidence";
 import type { EvidenceRef, Judgment } from "../types";
 import type { EvidenceView } from "../evidence";
 import { validateCitations } from "../infra/judging/contract";
+import { openRecording } from "../infra/recording";
+import { tmpdir } from "node:os";
 
 /** Read-only records for maintenance clients. Finalized evidence remains authoritative. */
 export function readBenchmarkRun(directory: string) {
@@ -29,7 +31,26 @@ export async function readEvidence(
   return (await CandidateEvidence.open(ref.directory, ref.hash)).view(ref);
 }
 
-/** Reference integrity only; interpretation of task facts stays in judge.md. */
+/** Open all recorded data with lazy native readers; dispose after inspection. */
+export async function readRecording(directory: string, evalRunId: string) {
+  using results = new Results(resolve(directory, "runner.db"), true);
+  const run = results.evalRun(evalRunId);
+  if (!run?.evidence)
+    throw new Error("Select an EvalRun with finalized evidence");
+  return openRecording(
+    {
+      run,
+      runDirectory: resolve(directory),
+      evidence: {
+        ...run.evidence,
+        directory: resolve(directory, run.evidence.directory),
+      },
+    },
+    process.platform === "win32" ? "C:/tmp/opencode" : tmpdir(),
+  );
+}
+
+/** Reference integrity only; interpretation of task facts stays in the judges. */
 export async function verifyJudgmentEvidence(
   judgment: Judgment,
   reference: EvidenceRef,
@@ -38,6 +59,6 @@ export async function verifyJudgmentEvidence(
   await validateCitations(judgment, evidence);
   return {
     hash: evidence.checkpoint.hash,
-    metrics: judgment.metrics.length,
+    criteria: Object.keys(judgment.scores).length,
   };
 }

package/src/app/rejudge.ts

@@ -57,7 +57,8 @@ export async function judgeRuns(
             ...evalDefinition,
             judge: updated.judge,
             judgeHash: updated.judgeHash,
-            metrics: updated.metrics,
+            criteria: updated.criteria,
+            code: updated.code,
             name: updated.name,
           }
         : evalDefinition;

package/src/app/run-benchmark.ts

@@ -20,6 +20,7 @@ import { judgeEvalRun } from "./judge-run";
 import type { ExecutionContext } from "./context";
 import { canJudgeEval } from "./eval-state";
 import { isScored } from "../judgment";
+import { benchmarkScores } from "./scores";
 import { CostBudget, estimateWork } from "./cost-plan";
 
 export type RunBenchmarkOptions = {
@@ -79,6 +80,9 @@ export function finishBenchmark(context: ExecutionContext) {
       judges.some(
         (run) => run.id === slot.judgeRunId && isScored(run.judgment?.value),
       ),
+    ) &&
+    benchmarkScores(context.definition, slots, judges).every(
+      (score) => score.percentage !== null,
     );
   const currentInputs = planBenchmark(
     context.definition,

package/src/app/run-eval-pipeline.ts

@@ -67,6 +67,7 @@ export async function runEvalPipeline(
       evalRunId: candidate!.id,
       evidence: saved,
       rubric: definition.judge,
+      kind: "llm",
       agent: JUDGE_AGENT,
       model: context.definition.judge.model,
       judgeHash: slot.judgeHash,
@@ -75,7 +76,7 @@ export async function runEvalPipeline(
       mode: "monitor",
       runtimeHash: context.runtime.judgeHash,
       protocol: JUDGE_PROTOCOL,
-      metrics: definition.metrics,
+      criteria: definition.criteria,
       monitor: policy,
     });
     return saved;

package/src/app/run-eval.ts

@@ -3,6 +3,8 @@ import type { Slot, EvalRun } from "../types";
 import type { ExecutionContext } from "./context";
 import { executeCandidate } from "../infra/containers/candidate";
 import type { EvidenceFeed } from "../evidence";
+import { CandidateEvidence } from "../infra/evidence";
+import { measureRecording } from "../infra/recording/metrics";
 
 /** The normal candidate use case: record its inputs, execute, finalize its evidence. */
 export async function runEval(
@@ -88,6 +90,17 @@ export async function runEval(
         }
       : undefined,
   };
+  if (completed.evidence) {
+    const evidence = await CandidateEvidence.open(
+      resolve(context.directory, completed.evidence.directory),
+      completed.evidence.hash,
+    );
+    completed.metrics = measureRecording(
+      evidence.rawEvents(),
+      evidence.toolCalls(),
+      completed.evidence,
+    );
+  }
   context.results.finishEval(completed);
   return completed;
 }

package/src/app/scores.ts

@@ -1,5 +1,6 @@
 import type { BenchmarkDefinition, JudgeRun, Slot } from "../types";
 import { modelScore } from "../view";
+import { isScored } from "../judgment";
 
 /** Scores are derived from a single selection snapshot; every eval has equal weight. */
 export function benchmarkScores(
@@ -9,46 +10,87 @@ export function benchmarkScores(
   evalId?: string,
 ) {
   const index = new Map(judges.map((run) => [run.id, run]));
+  const evals = definition.evals.filter(
+    (item) => !evalId || item.id === evalId,
+  );
+  const definitions = new Map(
+    evals.map((item) => {
+      const criteria = new Map(
+        item.criteria.map((criterion) => [criterion.id, criterion]),
+      );
+      for (const slot of slots.filter(
+        (slot) => slot.active && slot.evalId === item.id,
+      )) {
+        const judgment = slot.judgeRunId
+          ? index.get(slot.judgeRunId)?.judgment
+          : undefined;
+        for (const id of Object.keys(judgment?.scores ?? {}))
+          if (!criteria.has(id))
+            criteria.set(id, { id, name: id.replaceAll("_", " ") });
+      }
+      return [item.id, [...criteria.values()]];
+    }),
+  );
   return definition.models.map((model) =>
     modelScore(
       model,
-      definition.evals
-        .filter((item) => !evalId || item.id === evalId)
-        .flatMap((item) => {
-          const metrics = item.metrics;
-          const selected = slots.filter(
-            (slot) =>
-              slot.active && slot.evalId === item.id && slot.model === model,
+      evals.flatMap((item) => {
+        const criteria = definitions.get(item.id)!;
+        const selected = slots.filter(
+          (slot) =>
+            slot.active && slot.evalId === item.id && slot.model === model,
+        );
+        return criteria.map((criterion) => {
+          const values = selected
+            .map(
+              (slot) =>
+                (slot.judgeRunId
+                  ? index.get(slot.judgeRunId)?.judgment?.scores
+                  : undefined)?.[criterion.id]?.value,
+            )
+            .filter(isScored);
+          const scoredSum = values.reduce<number>(
+            (sum, value) => sum + value,
+            0,
           );
-          return metrics.map((metric) => {
-            const values = selected
-              .map(
+          return {
+            eval: item.id,
+            criterion: criterion.id,
+            name: criterion.name,
+            value:
+              values.length === definition.repetitions
+                ? scoredSum / definition.repetitions
+                : null,
+            scored: values.length,
+            expected: definition.repetitions,
+            passed: values.filter((value) => value === 1).length,
+            scoredSum,
+          };
+        });
+      }),
+      evals.map((item) => item.id),
+      evals
+        .filter(
+          (item) =>
+            item.code &&
+            slots
+              .filter(
                 (slot) =>
-                  (slot.judgeRunId
-                    ? index.get(slot.judgeRunId)?.judgment?.metrics
-                    : undefined
-                  )?.find((value) => value.id === metric.id)?.value,
+                  slot.active &&
+                  slot.evalId === item.id &&
+                  slot.model === model,
               )
-              .filter((value): value is 0 | 1 => value === 0 || value === 1);
-            const scoredSum = values.reduce<number>(
-              (sum, value) => sum + value,
-              0,
-            );
-            return {
-              eval: item.id,
-              metric: metric.id,
-              name: metric.name,
-              value:
-                values.length === definition.repetitions
-                  ? scoredSum / definition.repetitions
-                  : null,
-              scored: values.length,
-              expected: definition.repetitions,
-              passed: scoredSum,
-              scoredSum,
-            };
-          });
-        }),
+              .some((slot) => {
+                const judge = slot.judgeRunId
+                  ? index.get(slot.judgeRunId)
+                  : undefined;
+                return (
+                  judge?.code?.state !== "completed" ||
+                  judge.input.code?.hash !== item.code!.hash
+                );
+              }),
+        )
+        .map((item) => item.id),
     ),
   );
 }

package/src/app/serve-results.ts

@@ -66,6 +66,8 @@ export async function serveResults(options: {
             if (value !== null) query[key] = Number(value);
           }
           const revision = url.searchParams.get("revision");
+          if (url.searchParams.has("metric"))
+            query.metric = url.searchParams.get("metric")!;
           if (revision === "initial" || revision === "final")
             query.revision = revision;
           return Response.json(

package/src/evidence.ts

@@ -11,7 +11,10 @@ export type EvidenceQuery = {
     | "tool"
     | "artifacts"
     | "artifact"
-    | "diff";
+    | "diff"
+    | "metrics";
+  /** A dot-separated measurement path for the metrics query. */
+  metric?: string;
   sessionID?: string;
   type?: string;
   id?: string;

package/src/index.ts

@@ -8,8 +8,8 @@ export type {
   EvalRun,
   JudgeRun,
   Judgment,
-  MetricDefinition,
-  MetricJudgment,
+  CriterionDefinition,
+  CriterionScore,
   EvidenceCitation,
   ToolCall,
   MonitorPolicy,
@@ -30,7 +30,18 @@ export type {
   JudgeSession,
 } from "./evidence";
 export { CANDIDATE_TIMEOUT_MS } from "./types";
-export { rubricMetrics, metricMean, isScored } from "./judgment";
+export { rubricCriteria, criterionMean, isScored } from "./judgment";
+export type {
+  JudgeContext,
+  JudgeFunction,
+  JsonValue,
+  ScoreValue,
+  RunMetrics,
+  ToolMetrics,
+  RecordedEvent,
+  RecordedMessage,
+  RecordedFile,
+} from "./judge-context";
 export { loadBenchmark } from "./app/load-benchmark";
 export {
   runBenchmark,
@@ -47,6 +58,7 @@ export { judgeEvidence, recordEvidence } from "./app/judge-evidence";
 export {
   readBenchmarkRun,
   readEvidence,
+  readRecording,
   verifyJudgmentEvidence,
 } from "./app/read-run";
 export { snapshotBenchmarkRun } from "./app/snapshot-run";