@hona/openeval 0.2.1 → 0.3.0

package/src/infra/recording/index.ts

@@ -0,0 +1,244 @@
+import { Database } from "bun:sqlite";
+import { mkdir, mkdtemp, rm, symlink } from "node:fs/promises";
+import { dirname, isAbsolute, relative, resolve, sep } from "node:path";
+import type { OpenCode } from "@opencode/sdk";
+import type {
+  JudgeContext,
+  RecordedMessage,
+  Revision,
+} from "../../judge-context";
+import type { EvalRun, EvidenceRef } from "../../types";
+import { CandidateEvidence } from "../evidence";
+import { contained, hash } from "../files";
+import { OPENCODE_VERSION } from "../opencode/version";
+import { measureRecording } from "./metrics";
+
+export type RecordingInput = {
+  evidence: EvidenceRef;
+  run?: EvalRun;
+  runDirectory?: string;
+};
+
+/** Own disposable native readers; the authoritative archive is only read. */
+export async function openRecording(
+  input: RecordingInput,
+  workDirectory: string,
+): Promise<JudgeContext & AsyncDisposable> {
+  const reference = {
+    ...input.evidence,
+    directory: resolve(input.evidence.directory),
+  };
+  const evidence = await CandidateEvidence.open(
+    reference.directory,
+    reference.hash,
+  );
+  const events = evidence.rawEvents(),
+    tools = evidence.toolCalls(),
+    run = input.run ?? null;
+  const metrics = measureRecording(events, tools, reference);
+  const archive = run?.session;
+  const rootID = archive?.sessionId ?? null;
+  let temporary: Promise<string> | undefined,
+    database: Promise<Database> | undefined;
+  let sdk: Promise<OpenCode.Interface> | undefined;
+  const temp = () =>
+    (temporary ??= (async () => {
+      await mkdir(workDirectory, { recursive: true });
+      return mkdtemp(resolve(workDirectory, "recording-"));
+    })());
+  const copyDatabase = async (name: string) => {
+    if (!archive || !input.runDirectory)
+      throw new Error("This recording has no native OpenCode database");
+    if (archive.opencodeVersion !== OPENCODE_VERSION)
+      throw new Error(
+        `Native reader ${OPENCODE_VERSION} cannot open archive version ${archive.opencodeVersion}`,
+      );
+    const bytes = await Bun.file(
+      resolve(input.runDirectory, archive.database),
+    ).bytes();
+    if (hash(bytes) !== archive.databaseHash)
+      throw new Error("Native OpenCode database hash mismatch");
+    const path = resolve(await temp(), name);
+    await Bun.write(path, bytes);
+    return path;
+  };
+  const nativeSDK = () =>
+    (sdk ??= (async () => {
+      const path = await copyDatabase("sdk.db");
+      const { OpenCode } = await import("@opencode/sdk");
+      return OpenCode.create({
+        database: { path },
+        config: {
+          directory: resolve(dirname(path), "configuration"),
+          project: false,
+          content: JSON.stringify({ websearch: false }),
+        },
+      });
+    })());
+  const sessions = async () => {
+    const host = await nativeSDK();
+    const records = [];
+    let cursor: string | undefined;
+    do {
+      const page = await host.session.list({ limit: 100, cursor });
+      records.push(...page.data);
+      cursor = page.cursor.next ?? undefined;
+    } while (cursor);
+    return records;
+  };
+  const readText = async (path: string, revision: Revision = "final") =>
+    new TextDecoder("utf-8", { fatal: true }).decode(
+      await evidence.readFile(path, revision),
+    );
+  return {
+    response: { text: evidence.manifest.response?.text ?? "" },
+    prompt: evidence.manifest.prompt,
+    run: run ? structuredClone(run) : null,
+    metrics: structuredClone(metrics),
+    recording: {
+      evidence: reference,
+      sessionID: rootID,
+      coverage: evidence.manifest.coverage,
+      events: (filter = {}) =>
+        structuredClone(
+          events.filter(
+            ({ event }) =>
+              (!filter.type || event.type.startsWith(filter.type)) &&
+              (!filter.sessionID ||
+                ("data" in event &&
+                  (event.data as { sessionID?: string }).sessionID ===
+                    filter.sessionID)),
+          ),
+        ),
+      tools: (filter = {}) =>
+        structuredClone(
+          tools.filter(
+            (call) =>
+              (!filter.name || call.name === filter.name) &&
+              (!filter.sessionID || call.sessionID === filter.sessionID),
+          ),
+        ),
+      sessions,
+      async messages(sessionID) {
+        const host = await nativeSDK();
+        const ids = sessionID
+          ? [sessionID]
+          : (await sessions()).map((session) => session.id);
+        const records: RecordedMessage[] = [];
+        for (const id of ids) {
+          let cursor: string | undefined;
+          do {
+            const page = await host.message.list({
+              sessionID: id,
+              limit: 100,
+              ...(cursor ? { cursor } : { order: "asc" }),
+            });
+            records.push(
+              ...page.data.map((message) => ({ sessionID: id, message })),
+            );
+            cursor = page.cursor.next ?? undefined;
+          } while (cursor);
+        }
+        return records;
+      },
+      async export(sessionID = rootID ?? undefined) {
+        if (!sessionID) throw new Error("This recording has no native session");
+        return (await nativeSDK()).session.export({
+          sessionID,
+          sanitize: false,
+        });
+      },
+    },
+    workspace: {
+      files: (revision = "final") => evidence.files(revision),
+      read: (path, revision = "final") => evidence.readFile(path, revision),
+      text: readText,
+      async diff(path) {
+        const initial = evidence
+          .files("initial")
+          .some((file) => file.path === path);
+        const final = evidence
+          .files("final")
+          .some((file) => file.path === path);
+        return {
+          initial: initial ? await readText(path, "initial") : null,
+          final: final ? await readText(path, "final") : null,
+        };
+      },
+      async materialize(revision = "final") {
+        const directory = await mkdtemp(resolve(await temp(), "workspace-"));
+        const files = evidence.files(revision);
+        for (const file of files.filter((file) => file.symlink === undefined)) {
+          const path = contained(directory, file.path);
+          await mkdir(dirname(path), { recursive: true });
+          await Bun.write(path, await evidence.readFile(file.path, revision));
+        }
+        for (const file of files.filter((file) => file.symlink !== undefined)) {
+          const path = contained(directory, file.path),
+            link = file.symlink!;
+          const target = link.startsWith("/workspace/")
+            ? contained(directory, link.slice("/workspace/".length))
+            : isAbsolute(link)
+              ? ""
+              : resolve(dirname(path), link);
+          if (
+            !target ||
+            (target !== directory && !target.startsWith(directory + sep))
+          )
+            throw new Error(
+              `Recorded symlink leaves its workspace: ${file.path}`,
+            );
+          await mkdir(dirname(path), { recursive: true });
+          const isDirectory = files.some((item) =>
+            item.path.startsWith(
+              relative(directory, target).replaceAll("\\", "/") + "/",
+            ),
+          );
+          await symlink(
+            process.platform === "win32" && isDirectory
+              ? target
+              : relative(dirname(path), target),
+            path,
+            process.platform === "win32" && isDirectory
+              ? "junction"
+              : isDirectory
+                ? "dir"
+                : "file",
+          );
+        }
+        return directory;
+      },
+    },
+    native: {
+      version: archive?.opencodeVersion ?? null,
+      readerVersion: OPENCODE_VERSION,
+      database: () =>
+        (database ??= (async () => {
+          const db = new Database(await copyDatabase("readonly.db"), {
+            readonly: true,
+          });
+          db.exec("PRAGMA query_only=ON");
+          return db;
+        })()),
+      sdk: nativeSDK,
+      schema: () => import("@opencode/schema"),
+    },
+    async [Symbol.asyncDispose]() {
+      try {
+        if (sdk) await (await sdk).close();
+      } finally {
+        try {
+          if (database) (await database).close();
+        } finally {
+          if (temporary)
+            await rm(await temporary, {
+              recursive: true,
+              force: true,
+              maxRetries: 5,
+              retryDelay: 100,
+            });
+        }
+      }
+    },
+  };
+}

package/src/infra/recording/metrics.ts

@@ -0,0 +1,193 @@
+import type { EvidenceRef, ToolCall } from "../../types";
+import type {
+  RecordedEvent,
+  RunMetrics,
+  ToolMetrics,
+} from "../../judge-context";
+import { mergeRuntime, runtimeMs, type RuntimeInterval } from "../../runtime";
+
+const toolsSummary = (tools: ToolCall[]): ToolMetrics => {
+  const succeeded = tools.filter((call) => call.status === "succeeded").length;
+  const failed = tools.filter((call) => call.status === "failed").length;
+  return {
+    calls: tools.length,
+    succeeded,
+    failed,
+    unfinished: tools.length - succeeded - failed,
+    errorRate: succeeded + failed ? failed / (succeeded + failed) : null,
+  };
+};
+
+/** Measurements use deduplicated native records, never scored criterion values. */
+export function measureRecording(
+  supplied: RecordedEvent[],
+  tools: ToolCall[],
+  evidence: EvidenceRef,
+): RunMetrics {
+  const events = [
+    ...new Map(
+      supplied.map((item) => [
+        "id" in item.event ? item.event.id : String(item.sequence),
+        item.event,
+      ]),
+    ).values(),
+  ].sort(
+    (a, b) =>
+      ("created" in a ? a.created : 0) - ("created" in b ? b.created : 0),
+  );
+  const count = (type: string) =>
+    events.filter((event) => event.type === type).length;
+  const tokens = {
+    input: 0,
+    output: 0,
+    reasoning: 0,
+    cache: { read: 0, write: 0 },
+  };
+  let reportedUSD = 0,
+    usageRecords = 0,
+    usageComplete = true,
+    costComplete = true;
+  const intervals: RuntimeInterval[] = [],
+    starts = new Map<string, number>();
+  const executions: RuntimeInterval[] = [],
+    executionStarts = new Map<string, number>();
+  const created = new Set<string>();
+  const sessions = new Set<string>(),
+    sequences = new Map<string, number[]>();
+  for (const event of events) {
+    if (!("data" in event)) continue;
+    const data = event.data as Record<string, unknown>;
+    if (typeof data.sessionID === "string") sessions.add(data.sessionID);
+    if (event.type === "session.created") created.add(event.data.sessionID);
+    if (event.type === "session.execution.started")
+      executionStarts.set(event.data.sessionID, event.created);
+    if (
+      [
+        "session.execution.succeeded",
+        "session.execution.failed",
+        "session.execution.interrupted",
+      ].includes(event.type) &&
+      typeof data.sessionID === "string" &&
+      "created" in event
+    ) {
+      const start = executionStarts.get(data.sessionID);
+      if (start !== undefined) executions.push({ start, end: event.created });
+      executionStarts.delete(data.sessionID);
+    }
+    if ("durable" in event) {
+      const values = sequences.get(event.durable.aggregateID) ?? [];
+      values.push(event.durable.seq);
+      sequences.set(event.durable.aggregateID, values);
+    }
+    const key = `${data.sessionID}:${data.assistantMessageID ?? "compaction"}`;
+    if (
+      event.type === "session.step.started" ||
+      event.type === "session.compaction.started"
+    )
+      starts.set(key, event.created);
+    if (
+      [
+        "session.step.ended",
+        "session.step.failed",
+        "session.compaction.ended",
+        "session.compaction.failed",
+      ].includes(event.type)
+    ) {
+      const start = starts.get(key);
+      if (start !== undefined && "created" in event)
+        intervals.push({ start, end: event.created });
+      starts.delete(key);
+    }
+    if (
+      ![
+        "session.step.ended",
+        "session.step.failed",
+        "session.usage.recorded",
+      ].includes(event.type)
+    )
+      continue;
+    usageRecords++;
+    if (typeof data.cost === "number" && Number.isFinite(data.cost))
+      reportedUSD += data.cost;
+    else costComplete = false;
+    const usage = data.tokens as typeof tokens | undefined;
+    if (
+      !usage ||
+      ![
+        usage.input,
+        usage.output,
+        usage.reasoning,
+        usage.cache?.read,
+        usage.cache?.write,
+      ].every(Number.isFinite)
+    ) {
+      usageComplete = false;
+      continue;
+    }
+    tokens.input += usage.input;
+    tokens.output += usage.output;
+    tokens.reasoning += usage.reasoning;
+    tokens.cache.read += usage.cache.read;
+    tokens.cache.write += usage.cache.write;
+  }
+  const measuredTokens = usageRecords && usageComplete ? tokens : null;
+  const cost = usageRecords && costComplete ? reportedUSD : null;
+  const clock = events.reduce(
+    (latest, event) =>
+      "created" in event ? Math.max(latest, event.created) : latest,
+    0,
+  );
+  const modelActiveMs = runtimeMs(mergeRuntime(intervals), clock);
+  const complete =
+    sequences.size > 0 &&
+    starts.size === 0 &&
+    executionStarts.size === 0 &&
+    [...sessions].every((id) => created.has(id)) &&
+    [...sequences.values()].every((values) => {
+      const sorted = [...new Set(values)].sort((a, b) => a - b);
+      return sorted.every(
+        (value, index) => index === 0 || value === sorted[index - 1] + 1,
+      );
+    });
+  return {
+    version: 1,
+    scope: "candidate",
+    complete,
+    sessions: sessions.size,
+    requests: {
+      started: count("session.step.started"),
+      succeeded: count("session.step.ended"),
+      failed: count("session.step.failed"),
+      retries: count("session.retry.scheduled"),
+    },
+    tokens: measuredTokens,
+    cost: { usd: cost, reportedUSD, complete: cost !== null },
+    tools: {
+      ...toolsSummary(tools),
+      byName: Object.fromEntries(
+        [...new Set(tools.map((call) => call.name))]
+          .sort()
+          .map((name) => [
+            name,
+            toolsSummary(tools.filter((call) => call.name === name)),
+          ]),
+      ),
+    },
+    compactions: {
+      started: count("session.compaction.started"),
+      completed: count("session.compaction.ended"),
+      failed: count("session.compaction.failed"),
+    },
+    timing: {
+      elapsedMs: executions.length
+        ? runtimeMs(mergeRuntime(executions), clock)
+        : null,
+      modelActiveMs,
+      outputTokensPerSecond:
+        measuredTokens && modelActiveMs > 0
+          ? (measuredTokens.output * 1000) / modelActiveMs
+          : null,
+    },
+    evidence,
+  };
+}

package/src/infra/sqlite/index.ts

@@ -5,7 +5,7 @@ import { mkdirSync, existsSync } from "node:fs";
 import { dirname, resolve } from "node:path";
 import { randomUUID } from "node:crypto";
 import * as tables from "./schema";
-import { isScored, JUDGE_PROTOCOL, metricMean } from "../../judgment";
+import { isScored, JUDGE_PROTOCOL, criterionMean } from "../../judgment";
 import type {
   BenchmarkRun,
   EvalRun,
@@ -19,7 +19,7 @@ import type {
 } from "../../types";
 
 const APPLICATION_ID = 0x4f455632;
-const SCHEMA_VERSION = 4;
+const SCHEMA_VERSION = 5;
 const timestamp = () => new Date().toISOString();
 
 /** One owner for run selection and append-only finalized execution records. */
@@ -257,13 +257,16 @@ export class Results {
   }
   startJudge(input: JudgeRunInput) {
     if (
-      !input.agent ||
-      !input.metrics.length ||
+      (!input.code &&
+        (!input.agent || !input.model || !input.criteria.length)) ||
+      (!!input.rubric &&
+        (!input.agent || !input.model || !input.criteria.length)) ||
+      !["llm", "code", "hybrid"].includes(input.kind) ||
       !input.runtimeHash ||
       input.protocol !== JUDGE_PROTOCOL
     )
       throw new Error(
-        "Judge inputs must use the current agent, metrics, runtime, and protocol",
+        "Judge inputs must include their code or LLM configuration, criteria, runtime, and protocol",
       );
     const value: JudgeRun = {
       id: `judge_${randomUUID()}`,
@@ -301,21 +304,28 @@ export class Results {
     const saved = this.judgeRun(value.id);
     if (!saved || saved.state !== "running")
       throw new Error("Judge run is not running");
-    if (
-      value.state === "completed" &&
-      (!value.judgment ||
-        !Array.isArray(value.judgment.metrics) ||
-        value.judgment.metrics.length !== value.input.metrics.length ||
-        new Set(value.judgment.metrics.map((metric) => metric.id)).size !==
-          value.input.metrics.length ||
-        value.judgment.metrics.some(
-          (metric) =>
-            !value.input.metrics.some((declared) => declared.id === metric.id),
+    if (value.state === "completed") {
+      const expected = [
+        ...value.input.criteria.map((criterion) => criterion.id),
+        ...Object.keys(value.code?.scores ?? {}),
+      ];
+      const scores = value.judgment?.scores;
+      if (
+        !scores ||
+        Array.isArray(scores) ||
+        new Set(expected).size !== expected.length ||
+        Object.keys(scores).length !== expected.length ||
+        expected.some((id) => !Object.hasOwn(scores, id)) ||
+        Object.values(scores).some(
+          (score) => score.value !== null && !isScored(score.value),
         ) ||
-        value.judgment.value !== metricMean(value.judgment.metrics) ||
-        (value.judgment.value !== null && !isScored(value.judgment.value)))
-    )
-      throw new Error("A completed judge run requires a valid judgment");
+        value.judgment!.value !== criterionMean(scores) ||
+        (value.input.code && value.code?.state !== "completed")
+      )
+        throw new Error(
+          "A completed judge run requires a valid, complete judgment",
+        );
+    }
     if (
       value.state === "completed" &&
       this.evalRun(value.input.evalRunId)?.evidence?.hash !==

package/src/judge-context.ts

@@ -0,0 +1,141 @@
+import type { Database } from "bun:sqlite";
+import type { OpenCode } from "@opencode/sdk";
+import type {
+  SessionInfo,
+  SessionMessageInfo,
+  SessionTransferData,
+} from "@opencode/client";
+import type {
+  EvalRun,
+  EvidenceRef,
+  OpenCodeStreamEvent,
+  ToolCall,
+  Cost,
+  Accounting,
+} from "./types";
+
+export type JsonValue =
+  | null
+  | boolean
+  | number
+  | string
+  | JsonValue[]
+  | { [key: string]: JsonValue };
+export type ScoreValue = boolean | number | null;
+
+/** Code judges are ordinary functions. Only an optional scores map has grading semantics. */
+export type JudgeFunction = (
+  context: JudgeContext,
+) => JsonValue | Promise<JsonValue>;
+
+export type ToolMetrics = {
+  calls: number;
+  succeeded: number;
+  failed: number;
+  unfinished: number;
+  /** Failed / (succeeded + failed); null when there are no terminal calls. */
+  errorRate: number | null;
+};
+export type RunMetrics = {
+  version: 1;
+  scope: "candidate";
+  /** False for constructed controls or incomplete durable execution history. */
+  complete: boolean;
+  sessions: number;
+  requests: {
+    started: number;
+    succeeded: number;
+    failed: number;
+    retries: number;
+  };
+  tokens: Accounting["tokens"] | null;
+  cost: Cost;
+  tools: ToolMetrics & { byName: Record<string, ToolMetrics> };
+  compactions: { started: number; completed: number; failed: number };
+  timing: {
+    elapsedMs: number | null;
+    modelActiveMs: number;
+    /** Output tokens / model-active seconds; overlapping requests count once. */
+    outputTokensPerSecond: number | null;
+  };
+  evidence: EvidenceRef;
+};
+
+export type RecordedEvent = {
+  sequence: number;
+  time: string;
+  event: OpenCodeStreamEvent;
+};
+export type RecordedMessage = {
+  sessionID: string;
+  message: SessionMessageInfo;
+};
+export type RecordedFile = {
+  path: string;
+  bytes: number;
+  sha256?: string;
+  symlink?: string;
+};
+export type Revision = "initial" | "final";
+
+export interface JudgeContext {
+  /** Missing response text is normalized to an empty string. */
+  readonly response: { readonly text: string };
+  readonly prompt: string;
+  /** Null for constructed evidence that did not execute a candidate. */
+  readonly run: Readonly<EvalRun> | null;
+  readonly metrics: Readonly<RunMetrics>;
+  readonly recording: {
+    readonly evidence: EvidenceRef;
+    readonly sessionID: string | null;
+    readonly coverage: "recorded-session" | "recorded-prefix";
+    events(filter?: { sessionID?: string; type?: string }): RecordedEvent[];
+    tools(filter?: { sessionID?: string; name?: string }): ToolCall[];
+    sessions(): Promise<SessionInfo[]>;
+    /** Full projected history, including messages before compaction. */
+    messages(sessionID?: string): Promise<RecordedMessage[]>;
+    export(sessionID?: string): Promise<SessionTransferData>;
+  };
+  readonly workspace: {
+    files(revision?: Revision): RecordedFile[];
+    read(path: string, revision?: Revision): Promise<Uint8Array>;
+    text(path: string, revision?: Revision): Promise<string>;
+    diff(
+      path: string,
+    ): Promise<{ initial: string | null; final: string | null }>;
+    /** Restore a disposable snapshot for author-owned verification commands. */
+    materialize(revision?: Revision): Promise<string>;
+  };
+  readonly native: {
+    readonly version: string | null;
+    readonly readerVersion: string;
+    /** Read-only SQL access to a verified, disposable native database copy. */
+    database(): Promise<Database>;
+    /** The pinned OpenCode API/SDK over a separate disposable database copy. */
+    sdk(): Promise<OpenCode.Interface>;
+    schema(): Promise<typeof import("@opencode/schema")>;
+  };
+}
+
+/** Frozen judging source and dependency identity, kept in host-side run records. */
+export type CodeJudgeDefinition = {
+  file: string;
+  hash: string;
+  source: string;
+  sourceMap: string;
+  dependencies: Record<string, string>;
+};
+export type CodeJudgeExecution = {
+  state: "completed" | "failed" | "timed_out";
+  startedAt: string;
+  completedAt: string;
+  elapsedMs: number;
+  /** Original JSON, before boolean score normalization. */
+  output?: JsonValue;
+  scores?: Record<string, number | null>;
+  metrics?: RunMetrics;
+  error?: string;
+  sourceHash: string;
+  stdout: string;
+  stderr: string;
+};