@hona/openeval 0.2.2 → 0.3.0

package/src/infra/evidence/index.ts

@@ -11,6 +11,7 @@ import type { EvidenceFeed, EvidenceView, EvidenceQuery } from "../../evidence";
 export type { EvidenceQuery } from "../../evidence";
 import { trackTool } from "./tool-calls";
 import { hash as sha256, writeJson } from "../files";
+import { measureRecording } from "../recording/metrics";
 type InferenceResponse = { text: string; textBlocks?: string[] };
 type ArtifactPolicy = { excludeDirectories: string[] };
 
@@ -25,7 +26,7 @@ export type EvidenceQueryAudit = {
   checkId?: string;
   checkpoint?: EvidenceCheckpoint;
 };
-type FileEntry = {
+export type FileEntry = {
   path: string;
   bytes: number;
   sha256?: string;
@@ -37,7 +38,7 @@ export type EvidenceEvent = {
   time: string;
   event: OpenCodeStreamEvent;
 };
-type EvidenceManifest = {
+export type EvidenceManifest = {
   version: 1;
   handle: "candidate";
   coverage: "recorded-session" | "recorded-prefix";
@@ -446,6 +447,28 @@ export class CandidateEvidence {
     readonly hash: string,
   ) {}
 
+  /** Native records for trusted code judges. LLM queries use the separate blinded view. */
+  rawEvents(): EvidenceEvent[] {
+    return structuredClone(this.records);
+  }
+
+  toolCalls(): ToolCall[] {
+    return structuredClone(this.manifest.tools);
+  }
+
+  files(revision: "initial" | "final" = "final"): FileEntry[] {
+    const snapshot = this.manifest[revision];
+    if (!snapshot)
+      throw new Error(`${revision} workspace snapshot is unavailable`);
+    return structuredClone(snapshot.files);
+  }
+
+  async readFile(path: string, revision: "initial" | "final" = "final") {
+    const entry = this.entry(path, revision);
+    if (!entry) throw new Error(`Artifact not found: ${path}`);
+    return this.bytes(entry);
+  }
+
   view(reference: EvidenceRef): EvidenceView {
     if (reference.hash !== this.hash)
       throw new Error("Evidence view reference does not match its archive");
@@ -629,6 +652,20 @@ export class CandidateEvidence {
 
   async query(input: EvidenceQuery): Promise<Record<string, unknown>> {
     if (input.action === "summary") return this.summary();
+    if (input.action === "metrics") {
+      const metrics = measureRecording(this.records, this.manifest.tools, {
+        directory: this.directory,
+        hash: this.hash,
+      });
+      if (!input.metric) return { ...metrics };
+      let value: unknown = metrics;
+      for (const key of input.metric.split(".")) {
+        if (!value || typeof value !== "object" || !Object.hasOwn(value, key))
+          throw new Error(`Unknown recorded metric: ${input.metric}`);
+        value = (value as Record<string, unknown>)[key];
+      }
+      return { metric: input.metric, value, evidence: metrics.evidence };
+    }
     if (input.action === "response")
       return textPage(this.manifest.response.text, input);
     if (input.action === "events")

package/src/infra/evidence/tool-calls.ts

@@ -8,6 +8,7 @@ export function trackTool(
   if (!("data" in event)) return;
   const data = event.data as {
     id?: string;
+    sessionID?: string;
     name?: string;
     assistantMessageID?: string;
     input?: Record<string, unknown>;
@@ -17,8 +18,10 @@ export function trackTool(
     executed?: boolean;
   };
   if (!data.id) return;
-  const value = calls.get(data.id) ?? {
+  const key = `${data.sessionID ?? ""}:${data.assistantMessageID ?? ""}:${data.id}`;
+  const value = calls.get(key) ?? {
     id: data.id,
+    sessionID: data.sessionID,
     name: data.name ?? "unknown",
     assistantMessageId: data.assistantMessageID ?? "",
     status: "preparing" as const,
@@ -45,5 +48,5 @@ export function trackTool(
     if (value.startedAt !== undefined)
       value.durationMs = event.created - value.startedAt;
   }
-  calls.set(value.id, value);
+  calls.set(key, value);
 }

package/src/infra/judging/agent.ts

@@ -18,7 +18,7 @@ export const JUDGE_TOOLS = [
  */
 export const JUDGE_AGENT = {
   id: "openeval-judge",
-  description: "Grades immutable recorded work against eval metrics",
+  description: "Grades immutable recorded work against eval criteria",
   mode: "primary",
   system: readFileSync(new URL("./judge-agent.md", import.meta.url), "utf8"),
   permissions: [
@@ -35,6 +35,7 @@ export async function judgeConfiguration(
   input: Pick<JudgeRunInput, "agent" | "rubric" | "websearch">,
   directory: string,
 ) {
+  if (!input.agent) throw new Error("An LLM judge requires an agent profile");
   const { id, ...agent } = input.agent;
   const configuration = resolve(directory, "configuration");
   const config = {
@@ -45,7 +46,7 @@ export async function judgeConfiguration(
         ...agent,
         // Keep the rubric in the native system prompt so compaction cannot lose
         // it. Request context and result data move through registered tools.
-        system: `${agent.system.trim()}\n\n# Eval metrics\n\n${input.rubric.trim()}`,
+        system: `${agent.system.trim()}\n\n# Eval rubric\n\n${input.rubric.trim()}`,
       },
     },
     websearch: input.websearch ? { provider: input.websearch } : false,

package/src/infra/judging/code-result.ts

@@ -0,0 +1,102 @@
+import type { JsonValue } from "../../judge-context";
+import type { CriterionScore, Judgment } from "../../types";
+import { criterionMean, isScored } from "../../judgment";
+
+/** Preserve author JSON exactly; reject values JSON would silently discard or coerce. */
+export function jsonOutput(
+  value: unknown,
+  path = "output",
+  seen = new Set<object>(),
+): JsonValue {
+  if (value === null || typeof value === "boolean" || typeof value === "string")
+    return value;
+  if (typeof value === "number" && Number.isFinite(value)) return value;
+  if (typeof value !== "object" || !value || seen.has(value))
+    throw new Error(
+      `${path} must be JSON-compatible (no undefined, non-finite numbers, or cycles)`,
+    );
+  if (
+    !Array.isArray(value) &&
+    ![Object.prototype, null].includes(Object.getPrototypeOf(value))
+  )
+    throw new Error(`${path} must be a plain JSON object`);
+  if (Object.getOwnPropertySymbols(value).length)
+    throw new Error(`${path} contains a symbol key`);
+  seen.add(value);
+  try {
+    if (Array.isArray(value))
+      return Array.from(value, (item, index) =>
+        jsonOutput(item, `${path}[${index}]`, seen),
+      );
+    return Object.fromEntries(
+      Object.entries(value).map(([key, item]) => [
+        key,
+        jsonOutput(item, `${path}.${key}`, seen),
+      ]),
+    );
+  } finally {
+    seen.delete(value);
+  }
+}
+
+export function codeJudgment(output: JsonValue): Judgment {
+  const values =
+    output &&
+    typeof output === "object" &&
+    !Array.isArray(output) &&
+    "scores" in output
+      ? output.scores
+      : {};
+  if (!values || typeof values !== "object" || Array.isArray(values))
+    throw new Error("output.scores must be an object keyed by criterion ID");
+  const scores = Object.fromEntries(
+    Object.entries(values).map(([id, supplied]): [string, CriterionScore] => {
+      if (!/^[a-z][a-z0-9_]*$/.test(id))
+        throw new Error(`Invalid criterion ID: ${id}`);
+      const value = typeof supplied === "boolean" ? Number(supplied) : supplied;
+      if (value !== null && !isScored(value))
+        throw new Error(
+          `scores.${id} must be a boolean, a finite number from 0 to 1, or null`,
+        );
+      return [
+        id,
+        {
+          value: value as number | null,
+          reason:
+            value === null
+              ? "judge.ts returned an unresolved score."
+              : `judge.ts returned ${String(supplied)}.`,
+          evidence: value === null ? [] : [{ kind: "recording" }],
+          source: "judge.ts",
+        },
+      ];
+    }),
+  );
+  return {
+    scores,
+    value: criterionMean(scores),
+    reason: Object.keys(scores).length
+      ? "Criterion scores computed by judge.ts."
+      : "The code judge returned data without criterion scores.",
+  };
+}
+
+export function combineJudgments(...judgments: Judgment[]): Judgment {
+  const scores: Record<string, CriterionScore> = {};
+  for (const judgment of judgments)
+    for (const [id, score] of Object.entries(judgment.scores)) {
+      if (Object.hasOwn(scores, id))
+        throw new Error(
+          `Duplicate criterion ID from judge.md and judge.ts: ${id}`,
+        );
+      scores[id] = score;
+    }
+  return {
+    scores,
+    value: criterionMean(scores),
+    reason: judgments
+      .map((judgment) => judgment.reason)
+      .filter(Boolean)
+      .join("\n"),
+  };
+}

package/src/infra/judging/code-source.ts

@@ -0,0 +1,96 @@
+import { dirname, isAbsolute, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+import type { CodeJudgeDefinition } from "../../judge-context";
+import { fingerprint } from "../files";
+
+/** Bundle local imports without executing author code during planning. */
+export async function compileCodeJudge(
+  file: string,
+): Promise<CodeJudgeDefinition> {
+  file = resolve(file);
+  const dependencies: Record<string, string> = {};
+  const sdkManifest = fileURLToPath(
+    new URL("../../../package.json", import.meta.url),
+  );
+  dependencies[sdkManifest] = await Bun.file(sdkManifest).text();
+  const external = new Set<string>();
+  const built = await Bun.build({
+    entrypoints: [file],
+    root: dirname(file),
+    target: "bun",
+    format: "esm",
+    packages: "external",
+    sourcemap: "external",
+    plugins: [
+      {
+        name: "judge-dependencies",
+        setup(build) {
+          build.onResolve({ filter: /^[^./]/ }, (args) => {
+            if (
+              args.path.startsWith("node:") ||
+              args.path.startsWith("bun:") ||
+              isAbsolute(args.path)
+            )
+              return;
+            const path = Bun.resolveSync(
+              args.path,
+              args.resolveDir || dirname(file),
+            );
+            external.add(path);
+            return { path, external: true };
+          });
+        },
+      },
+    ],
+  });
+  if (!built.success)
+    throw new Error(`Cannot compile judge.ts: ${built.logs.join("\n")}`);
+  const source = await built.outputs
+    .find((output) => output.kind === "entry-point")!
+    .text();
+  if (
+    !new Bun.Transpiler({ loader: "js" })
+      .scan(source)
+      .exports.includes("default")
+  )
+    throw new Error("judge.ts must have a default function export");
+  const sourceMap =
+    (await built.outputs
+      .find((output) => output.kind === "sourcemap")
+      ?.text()) ?? "";
+  const collect = async (directory: string, stopAtPackage: boolean) => {
+    for (;;) {
+      let found = false;
+      for (const name of [
+        "package.json",
+        "bun.lock",
+        "bun.lockb",
+        "package-lock.json",
+        "pnpm-lock.yaml",
+        "yarn.lock",
+      ]) {
+        const path = resolve(directory, name);
+        if (await Bun.file(path).exists()) {
+          dependencies[path] =
+            name === "bun.lockb"
+              ? Buffer.from(await Bun.file(path).bytes()).toString("base64")
+              : await Bun.file(path).text();
+          found ||= name === "package.json";
+        }
+      }
+      if (found && stopAtPackage) return;
+      const parent = dirname(directory);
+      if (parent === directory) return;
+      directory = parent;
+    }
+  };
+  await collect(dirname(file), false);
+  for (const path of external) await collect(dirname(path), true);
+  return {
+    file,
+    source,
+    sourceMap,
+    dependencies,
+    hash: fingerprint({ source, sourceMap, dependencies }),
+  };
+}

package/src/infra/judging/code-worker.ts

@@ -0,0 +1,26 @@
+import { pathToFileURL } from "node:url";
+import { openRecording, type RecordingInput } from "../recording";
+import { jsonOutput } from "./code-result";
+import { errorMessage, writeJson } from "../files";
+import type { JudgeFunction, RunMetrics } from "../../judge-context";
+
+type Job = RecordingInput & { module: string; output: string; scratch: string };
+const job: Job = JSON.parse(await Bun.stdin.text());
+let metrics: RunMetrics | undefined;
+async function grade() {
+  await using context = await openRecording(job, job.scratch);
+  metrics = context.metrics;
+  const module = await import(pathToFileURL(job.module).href);
+  if (typeof module.default !== "function")
+    throw new Error("judge.ts must default-export a function");
+  const output = jsonOutput(await (module.default as JudgeFunction)(context));
+  return { output, metrics: context.metrics };
+}
+try {
+  await writeJson(job.output, await grade());
+  process.exit(0);
+} catch (error) {
+  await writeJson(job.output, { error: errorMessage(error), metrics });
+  console.error(error);
+  process.exit(1);
+}

package/src/infra/judging/code.ts

@@ -0,0 +1,97 @@
+import { mkdir, rm } from "node:fs/promises";
+import { resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+import type {
+  CodeJudgeDefinition,
+  CodeJudgeExecution,
+  JsonValue,
+  RunMetrics,
+} from "../../judge-context";
+import type { RecordingInput } from "../recording";
+import { codeJudgment, jsonOutput } from "./code-result";
+import { errorMessage, writeJson } from "../files";
+
+/** A separate process bounds synchronous loops as well as asynchronous judges. */
+export async function executeCodeJudge(
+  code: CodeJudgeDefinition,
+  recording: RecordingInput,
+  directory: string,
+  timeoutMs: number,
+): Promise<CodeJudgeExecution> {
+  directory = resolve(directory);
+  await mkdir(directory, { recursive: true });
+  const module = resolve(directory, "judge.js"),
+    output = resolve(directory, "output.json");
+  const scratch = resolve(directory, "scratch");
+  const stdout = resolve(directory, "stdout.log"),
+    stderr = resolve(directory, "stderr.log");
+  await Bun.write(module, code.source);
+  await Bun.write(resolve(directory, "judge.js.map"), code.sourceMap);
+  await writeJson(resolve(directory, "dependencies.json"), code.dependencies);
+  const startedAt = new Date().toISOString();
+  let timer: ReturnType<typeof setTimeout> | undefined;
+  let timedOut = false;
+  let raw: JsonValue | undefined, metrics: RunMetrics | undefined;
+  const base = { startedAt, sourceHash: code.hash, stdout, stderr };
+  try {
+    const child = Bun.spawn(
+      [
+        process.execPath,
+        fileURLToPath(new URL("./code-worker.ts", import.meta.url)),
+      ],
+      {
+        cwd: directory,
+        stdin: "pipe",
+        stdout: Bun.file(stdout),
+        stderr: Bun.file(stderr),
+        windowsHide: true,
+      },
+    );
+    timer = setTimeout(() => {
+      timedOut = true;
+      child.kill();
+    }, timeoutMs);
+    await child.stdin.write(
+      JSON.stringify({ ...recording, module, output, scratch }),
+    );
+    child.stdin.end();
+    const exit = await child.exited;
+    if (timedOut)
+      throw new Error(`judge.ts exceeded its ${timeoutMs}ms time limit`);
+    const result = await Bun.file(output).json();
+    metrics = result.metrics;
+    if (exit !== 0 || result.error)
+      throw new Error(result.error ?? `judge.ts exited with code ${exit}`);
+    raw = jsonOutput(result.output);
+    const judgment = codeJudgment(raw);
+    return {
+      ...base,
+      state: "completed",
+      completedAt: new Date().toISOString(),
+      elapsedMs: Date.now() - Date.parse(startedAt),
+      output: raw,
+      metrics,
+      scores: Object.fromEntries(
+        Object.entries(judgment.scores).map(([id, score]) => [id, score.value]),
+      ),
+    };
+  } catch (error) {
+    return {
+      ...base,
+      state: timedOut ? "timed_out" : "failed",
+      completedAt: new Date().toISOString(),
+      elapsedMs: Date.now() - Date.parse(startedAt),
+      error: errorMessage(error),
+      output: raw,
+      metrics,
+    };
+  } finally {
+    clearTimeout(timer);
+    await rm(scratch, {
+      recursive: true,
+      force: true,
+      maxRetries: 5,
+      retryDelay: 100,
+    });
+  }
+}

package/src/infra/judging/contract.ts

@@ -1,11 +1,11 @@
 import type {
   EvidenceCitation,
   Judgment,
-  MetricDefinition,
-  MetricJudgment,
+  CriterionDefinition,
+  CriterionScore,
 } from "../../types";
 import type { EvidenceReader, EvidenceQuery } from "../../evidence";
-import { metricMean } from "../../judgment";
+import { criterionMean, isScored } from "../../judgment";
 import { Schema } from "effect";
 
 export const STRICT_OBJECT = {
@@ -14,68 +14,77 @@ export const STRICT_OBJECT = {
 
 const object = (value: unknown): value is Record<string, unknown> =>
   !!value && typeof value === "object" && !Array.isArray(value);
-const binary = (value: unknown) => value === 0 || value === 1 || value === null;
+const validScore = (value: unknown) =>
+  value === null || typeof value === "boolean" || isScored(value);
 const reason = (value: unknown): value is string =>
   typeof value === "string" && !!value.trim();
 
-/** Validate structured tool arguments; metric meaning remains the judge's job. */
-export function validateMetrics(
+/** Validate structured tool arguments; criterion meaning remains the judge's job. */
+export function validateCriteria(
   value: unknown,
-  expected: readonly MetricDefinition[],
+  expected: readonly CriterionDefinition[],
 ): Judgment {
   if (!object(value))
-    throw new Error("metrics must be an object keyed by metric ID");
+    throw new Error("scores must be an object keyed by criterion ID");
   const ids = Object.keys(value);
   if (
     ids.length !== expected.length ||
-    ids.some((id) => !expected.some((metric) => metric.id === id))
+    ids.some((id) => !expected.some((criterion) => criterion.id === id))
   )
     throw new Error(
-      `Submit exactly these metric IDs: ${expected.map((metric) => metric.id).join(", ")}`,
+      `Submit exactly these criterion IDs: ${expected.map((criterion) => criterion.id).join(", ")}`,
     );
-  const metrics = expected.map(({ id }): MetricJudgment => {
-    const item = value[id];
-    if (
-      !object(item) ||
-      !binary(item.value) ||
-      !reason(item.reason) ||
-      !Array.isArray(item.evidence) ||
-      item.evidence.length > 32 ||
-      (item.value !== null && !item.evidence.length)
-    )
-      throw new Error(
-        `Metric ${id} requires value 0, 1, or null, a non-empty reason, and 1–32 citations when decided`,
-      );
-    const evidence = item.evidence.map(parseCitation);
-    if (
-      item.sources !== undefined &&
-      (!Array.isArray(item.sources) ||
-        !item.sources.every(
-          (url) => typeof url === "string" && /^https?:\/\//.test(url),
-        ))
-    )
-      throw new Error("Metric sources must be HTTP(S) URLs");
-    return {
-      id,
-      value: item.value as 0 | 1 | null,
-      reason: item.reason,
-      evidence,
-      ...(item.sources ? { sources: [...(item.sources as string[])] } : {}),
-    };
-  });
+  const scores = Object.fromEntries(
+    expected.map(({ id }): [string, CriterionScore] => {
+      const item = value[id];
+      if (
+        !object(item) ||
+        !validScore(item.value) ||
+        !reason(item.reason) ||
+        !Array.isArray(item.evidence) ||
+        item.evidence.length > 32 ||
+        (item.value !== null && !item.evidence.length)
+      )
+        throw new Error(
+          `Criterion ${id} requires a boolean, a value from 0 to 1, or null, a non-empty reason, and 1–32 citations when decided`,
+        );
+      const evidence = item.evidence.map(parseCitation);
+      if (
+        item.sources !== undefined &&
+        (!Array.isArray(item.sources) ||
+          !item.sources.every(
+            (url) => typeof url === "string" && /^https?:\/\//.test(url),
+          ))
+      )
+        throw new Error("Criterion sources must be HTTP(S) URLs");
+      return [
+        id,
+        {
+          value:
+            typeof item.value === "boolean"
+              ? Number(item.value)
+              : (item.value as number | null),
+          reason: item.reason,
+          evidence,
+          source: "judge.md",
+          ...(item.sources ? { sources: [...(item.sources as string[])] } : {}),
+        },
+      ];
+    }),
+  );
   return {
-    value: metricMean(metrics),
-    reason: metrics
-      .map((metric) => `${metric.id}: ${metric.reason}`)
+    value: criterionMean(scores),
+    reason: Object.entries(scores)
+      .map(([id, score]) => `${id}: ${score.reason}`)
       .join("\n"),
-    metrics,
+    scores,
   };
 }
 
 function parseCitation(value: unknown): EvidenceCitation {
   if (
     !object(value) ||
-    !["response", "tool", "message", "event", "artifact"].includes(
+    !["response", "tool", "message", "event", "artifact", "metric"].includes(
       String(value.kind),
     )
   )
@@ -110,7 +119,7 @@ function parseCitation(value: unknown): EvidenceCitation {
   }
   if (!reason(value.id)) throw new Error("Citation requires a recorded ID");
   return {
-    kind: value.kind as "tool" | "message" | "event",
+    kind: value.kind as "tool" | "message" | "event" | "metric",
     id: value.id,
     ...extra,
   };
@@ -123,18 +132,22 @@ export async function validateCitations(
   judgment: Judgment,
   reader: EvidenceReader,
 ) {
-  for (const metric of judgment.metrics)
-    for (const citation of metric.evidence) {
+  for (const [id, score] of Object.entries(judgment.scores))
+    for (const citation of score.evidence) {
       const query: EvidenceQuery =
-        citation.kind === "artifact"
-          ? {
-              action: "artifact",
-              path: citation.path,
-              revision: citation.revision,
-            }
-          : citation.kind === "response"
-            ? { action: "response" }
-            : { action: citation.kind, id: citation.id };
+        citation.kind === "recording"
+          ? { action: "summary" }
+          : citation.kind === "metric"
+            ? { action: "metrics", metric: citation.id }
+            : citation.kind === "artifact"
+              ? {
+                  action: "artifact",
+                  path: citation.path,
+                  revision: citation.revision,
+                }
+              : citation.kind === "response"
+                ? { action: "response" }
+                : { action: citation.kind, id: citation.id };
       let offset = citation.offset ?? 0,
         found = false;
       for (let page = 0; page < 100; page++) {
@@ -154,12 +167,12 @@ export async function validateCitations(
         );
       }
       if (!found)
-        throw new Error(`Citation quote was not found for metric ${metric.id}`);
+        throw new Error(`Citation quote was not found for criterion ${id}`);
     }
 }
 
 /** The native tool validator exposes these shapes directly in Code Mode. */
-export function metricsSchema(metrics: readonly MetricDefinition[]) {
+export function criteriaSchema(criteria: readonly CriterionDefinition[]) {
   const citationFields = {
     quote: Schema.optional(Schema.String.check(Schema.isMaxLength(2000))),
     offset: Schema.optional(
@@ -175,7 +188,7 @@ export function metricsSchema(metrics: readonly MetricDefinition[]) {
     }).annotate(STRICT_OBJECT),
     Schema.Struct({
       ...citationFields,
-      kind: Schema.Literals(["tool", "message", "event"]),
+      kind: Schema.Literals(["tool", "message", "event", "metric"]),
       id: Schema.NonEmptyString,
     }).annotate(STRICT_OBJECT),
     Schema.Struct({
@@ -186,7 +199,12 @@ export function metricsSchema(metrics: readonly MetricDefinition[]) {
     }).annotate(STRICT_OBJECT),
   ]);
   const decision = Schema.Struct({
-    value: Schema.NullOr(Schema.Literals([0, 1])),
+    value: Schema.NullOr(
+      Schema.Union([
+        Schema.Boolean,
+        Schema.Number.check(Schema.isBetween({ minimum: 0, maximum: 1 })),
+      ]),
+    ),
     reason: Schema.NonEmptyString,
     evidence: Schema.Array(citation).check(
       Schema.isMinLength(0),
@@ -198,9 +216,9 @@ export function metricsSchema(metrics: readonly MetricDefinition[]) {
   }).annotate(STRICT_OBJECT);
   return Schema.Struct(
     Object.fromEntries(
-      metrics.map((metric) => [
-        metric.id,
-        decision.annotate({ description: metric.name }),
+      criteria.map((criterion) => [
+        criterion.id,
+        decision.annotate({ description: criterion.name }),
       ]),
     ),
   ).annotate(STRICT_OBJECT);