@hona/openeval 0.2.2 → 0.3.1

package/src/infra/recording/metrics.ts

@@ -0,0 +1,193 @@
+import type { EvidenceRef, ToolCall } from "../../types";
+import type {
+  RecordedEvent,
+  RunMetrics,
+  ToolMetrics,
+} from "../../judge-context";
+import { mergeRuntime, runtimeMs, type RuntimeInterval } from "../../runtime";
+
+const toolsSummary = (tools: ToolCall[]): ToolMetrics => {
+  const succeeded = tools.filter((call) => call.status === "succeeded").length;
+  const failed = tools.filter((call) => call.status === "failed").length;
+  return {
+    calls: tools.length,
+    succeeded,
+    failed,
+    unfinished: tools.length - succeeded - failed,
+    errorRate: succeeded + failed ? failed / (succeeded + failed) : null,
+  };
+};
+
+/** Measurements use deduplicated native records, never scored criterion values. */
+export function measureRecording(
+  supplied: RecordedEvent[],
+  tools: ToolCall[],
+  evidence: EvidenceRef,
+): RunMetrics {
+  const events = [
+    ...new Map(
+      supplied.map((item) => [
+        "id" in item.event ? item.event.id : String(item.sequence),
+        item.event,
+      ]),
+    ).values(),
+  ].sort(
+    (a, b) =>
+      ("created" in a ? a.created : 0) - ("created" in b ? b.created : 0),
+  );
+  const count = (type: string) =>
+    events.filter((event) => event.type === type).length;
+  const tokens = {
+    input: 0,
+    output: 0,
+    reasoning: 0,
+    cache: { read: 0, write: 0 },
+  };
+  let reportedUSD = 0,
+    usageRecords = 0,
+    usageComplete = true,
+    costComplete = true;
+  const intervals: RuntimeInterval[] = [],
+    starts = new Map<string, number>();
+  const executions: RuntimeInterval[] = [],
+    executionStarts = new Map<string, number>();
+  const created = new Set<string>();
+  const sessions = new Set<string>(),
+    sequences = new Map<string, number[]>();
+  for (const event of events) {
+    if (!("data" in event)) continue;
+    const data = event.data as Record<string, unknown>;
+    if (typeof data.sessionID === "string") sessions.add(data.sessionID);
+    if (event.type === "session.created") created.add(event.data.sessionID);
+    if (event.type === "session.execution.started")
+      executionStarts.set(event.data.sessionID, event.created);
+    if (
+      [
+        "session.execution.succeeded",
+        "session.execution.failed",
+        "session.execution.interrupted",
+      ].includes(event.type) &&
+      typeof data.sessionID === "string" &&
+      "created" in event
+    ) {
+      const start = executionStarts.get(data.sessionID);
+      if (start !== undefined) executions.push({ start, end: event.created });
+      executionStarts.delete(data.sessionID);
+    }
+    if ("durable" in event) {
+      const values = sequences.get(event.durable.aggregateID) ?? [];
+      values.push(event.durable.seq);
+      sequences.set(event.durable.aggregateID, values);
+    }
+    const key = `${data.sessionID}:${data.assistantMessageID ?? "compaction"}`;
+    if (
+      event.type === "session.step.started" ||
+      event.type === "session.compaction.started"
+    )
+      starts.set(key, event.created);
+    if (
+      [
+        "session.step.ended",
+        "session.step.failed",
+        "session.compaction.ended",
+        "session.compaction.failed",
+      ].includes(event.type)
+    ) {
+      const start = starts.get(key);
+      if (start !== undefined && "created" in event)
+        intervals.push({ start, end: event.created });
+      starts.delete(key);
+    }
+    if (
+      ![
+        "session.step.ended",
+        "session.step.failed",
+        "session.usage.recorded",
+      ].includes(event.type)
+    )
+      continue;
+    usageRecords++;
+    if (typeof data.cost === "number" && Number.isFinite(data.cost))
+      reportedUSD += data.cost;
+    else costComplete = false;
+    const usage = data.tokens as typeof tokens | undefined;
+    if (
+      !usage ||
+      ![
+        usage.input,
+        usage.output,
+        usage.reasoning,
+        usage.cache?.read,
+        usage.cache?.write,
+      ].every(Number.isFinite)
+    ) {
+      usageComplete = false;
+      continue;
+    }
+    tokens.input += usage.input;
+    tokens.output += usage.output;
+    tokens.reasoning += usage.reasoning;
+    tokens.cache.read += usage.cache.read;
+    tokens.cache.write += usage.cache.write;
+  }
+  const measuredTokens = usageRecords && usageComplete ? tokens : null;
+  const cost = usageRecords && costComplete ? reportedUSD : null;
+  const clock = events.reduce(
+    (latest, event) =>
+      "created" in event ? Math.max(latest, event.created) : latest,
+    0,
+  );
+  const modelActiveMs = runtimeMs(mergeRuntime(intervals), clock);
+  const complete =
+    sequences.size > 0 &&
+    starts.size === 0 &&
+    executionStarts.size === 0 &&
+    [...sessions].every((id) => created.has(id)) &&
+    [...sequences.values()].every((values) => {
+      const sorted = [...new Set(values)].sort((a, b) => a - b);
+      return sorted.every(
+        (value, index) => index === 0 || value === sorted[index - 1] + 1,
+      );
+    });
+  return {
+    version: 1,
+    scope: "candidate",
+    complete,
+    sessions: sessions.size,
+    requests: {
+      started: count("session.step.started"),
+      succeeded: count("session.step.ended"),
+      failed: count("session.step.failed"),
+      retries: count("session.retry.scheduled"),
+    },
+    tokens: measuredTokens,
+    cost: { usd: cost, reportedUSD, complete: cost !== null },
+    tools: {
+      ...toolsSummary(tools),
+      byName: Object.fromEntries(
+        [...new Set(tools.map((call) => call.name))]
+          .sort()
+          .map((name) => [
+            name,
+            toolsSummary(tools.filter((call) => call.name === name)),
+          ]),
+      ),
+    },
+    compactions: {
+      started: count("session.compaction.started"),
+      completed: count("session.compaction.ended"),
+      failed: count("session.compaction.failed"),
+    },
+    timing: {
+      elapsedMs: executions.length
+        ? runtimeMs(mergeRuntime(executions), clock)
+        : null,
+      modelActiveMs,
+      outputTokensPerSecond:
+        measuredTokens && modelActiveMs > 0
+          ? (measuredTokens.output * 1000) / modelActiveMs
+          : null,
+    },
+    evidence,
+  };
+}

package/src/infra/sqlite/index.ts

@@ -5,7 +5,7 @@ import { mkdirSync, existsSync } from "node:fs";
 import { dirname, resolve } from "node:path";
 import { randomUUID } from "node:crypto";
 import * as tables from "./schema";
-import { isScored, JUDGE_PROTOCOL, metricMean } from "../../judgment";
+import { isScored, JUDGE_PROTOCOL, criterionMean } from "../../judgment";
 import type {
   BenchmarkRun,
   EvalRun,
@@ -19,7 +19,7 @@ import type {
 } from "../../types";
 
 const APPLICATION_ID = 0x4f455632;
-const SCHEMA_VERSION = 4;
+const SCHEMA_VERSION = 5;
 const timestamp = () => new Date().toISOString();
 
 /** One owner for run selection and append-only finalized execution records. */
@@ -257,13 +257,16 @@ export class Results {
   }
   startJudge(input: JudgeRunInput) {
     if (
-      !input.agent ||
-      !input.metrics.length ||
+      (!input.code &&
+        (!input.agent || !input.model || !input.criteria.length)) ||
+      (!!input.rubric &&
+        (!input.agent || !input.model || !input.criteria.length)) ||
+      !["llm", "code", "hybrid"].includes(input.kind) ||
       !input.runtimeHash ||
       input.protocol !== JUDGE_PROTOCOL
     )
       throw new Error(
-        "Judge inputs must use the current agent, metrics, runtime, and protocol",
+        "Judge inputs must include their code or LLM configuration, criteria, runtime, and protocol",
       );
     const value: JudgeRun = {
       id: `judge_${randomUUID()}`,
@@ -301,21 +304,28 @@ export class Results {
     const saved = this.judgeRun(value.id);
     if (!saved || saved.state !== "running")
       throw new Error("Judge run is not running");
-    if (
-      value.state === "completed" &&
-      (!value.judgment ||
-        !Array.isArray(value.judgment.metrics) ||
-        value.judgment.metrics.length !== value.input.metrics.length ||
-        new Set(value.judgment.metrics.map((metric) => metric.id)).size !==
-          value.input.metrics.length ||
-        value.judgment.metrics.some(
-          (metric) =>
-            !value.input.metrics.some((declared) => declared.id === metric.id),
+    if (value.state === "completed") {
+      const expected = [
+        ...value.input.criteria.map((criterion) => criterion.id),
+        ...Object.keys(value.code?.scores ?? {}),
+      ];
+      const scores = value.judgment?.scores;
+      if (
+        !scores ||
+        Array.isArray(scores) ||
+        new Set(expected).size !== expected.length ||
+        Object.keys(scores).length !== expected.length ||
+        expected.some((id) => !Object.hasOwn(scores, id)) ||
+        Object.values(scores).some(
+          (score) => score.value !== null && !isScored(score.value),
         ) ||
-        value.judgment.value !== metricMean(value.judgment.metrics) ||
-        (value.judgment.value !== null && !isScored(value.judgment.value)))
-    )
-      throw new Error("A completed judge run requires a valid judgment");
+        value.judgment!.value !== criterionMean(scores) ||
+        (value.input.code && value.code?.state !== "completed")
+      )
+        throw new Error(
+          "A completed judge run requires a valid, complete judgment",
+        );
+    }
     if (
       value.state === "completed" &&
       this.evalRun(value.input.evalRunId)?.evidence?.hash !==

package/src/judge-context.ts

@@ -0,0 +1,141 @@
+import type { Database } from "bun:sqlite";
+import type { OpenCode } from "@opencode/sdk";
+import type {
+  SessionInfo,
+  SessionMessageInfo,
+  SessionTransferData,
+} from "@opencode/client";
+import type {
+  EvalRun,
+  EvidenceRef,
+  OpenCodeStreamEvent,
+  ToolCall,
+  Cost,
+  Accounting,
+} from "./types";
+
+export type JsonValue =
+  | null
+  | boolean
+  | number
+  | string
+  | JsonValue[]
+  | { [key: string]: JsonValue };
+export type ScoreValue = boolean | number | null;
+
+/** Code judges are ordinary functions. Only an optional scores map has grading semantics. */
+export type JudgeFunction = (
+  context: JudgeContext,
+) => JsonValue | Promise<JsonValue>;
+
+export type ToolMetrics = {
+  calls: number;
+  succeeded: number;
+  failed: number;
+  unfinished: number;
+  /** Failed / (succeeded + failed); null when there are no terminal calls. */
+  errorRate: number | null;
+};
+export type RunMetrics = {
+  version: 1;
+  scope: "candidate";
+  /** False for constructed controls or incomplete durable execution history. */
+  complete: boolean;
+  sessions: number;
+  requests: {
+    started: number;
+    succeeded: number;
+    failed: number;
+    retries: number;
+  };
+  tokens: Accounting["tokens"] | null;
+  cost: Cost;
+  tools: ToolMetrics & { byName: Record<string, ToolMetrics> };
+  compactions: { started: number; completed: number; failed: number };
+  timing: {
+    elapsedMs: number | null;
+    modelActiveMs: number;
+    /** Output tokens / model-active seconds; overlapping requests count once. */
+    outputTokensPerSecond: number | null;
+  };
+  evidence: EvidenceRef;
+};
+
+export type RecordedEvent = {
+  sequence: number;
+  time: string;
+  event: OpenCodeStreamEvent;
+};
+export type RecordedMessage = {
+  sessionID: string;
+  message: SessionMessageInfo;
+};
+export type RecordedFile = {
+  path: string;
+  bytes: number;
+  sha256?: string;
+  symlink?: string;
+};
+export type Revision = "initial" | "final";
+
+export interface JudgeContext {
+  /** Missing response text is normalized to an empty string. */
+  readonly response: { readonly text: string };
+  readonly prompt: string;
+  /** Null for constructed evidence that did not execute a candidate. */
+  readonly run: Readonly<EvalRun> | null;
+  readonly metrics: Readonly<RunMetrics>;
+  readonly recording: {
+    readonly evidence: EvidenceRef;
+    readonly sessionID: string | null;
+    readonly coverage: "recorded-session" | "recorded-prefix";
+    events(filter?: { sessionID?: string; type?: string }): RecordedEvent[];
+    tools(filter?: { sessionID?: string; name?: string }): ToolCall[];
+    sessions(): Promise<SessionInfo[]>;
+    /** Full projected history, including messages before compaction. */
+    messages(sessionID?: string): Promise<RecordedMessage[]>;
+    export(sessionID?: string): Promise<SessionTransferData>;
+  };
+  readonly workspace: {
+    files(revision?: Revision): RecordedFile[];
+    read(path: string, revision?: Revision): Promise<Uint8Array>;
+    text(path: string, revision?: Revision): Promise<string>;
+    diff(
+      path: string,
+    ): Promise<{ initial: string | null; final: string | null }>;
+    /** Restore a disposable snapshot for author-owned verification commands. */
+    materialize(revision?: Revision): Promise<string>;
+  };
+  readonly native: {
+    readonly version: string | null;
+    readonly readerVersion: string;
+    /** Read-only SQL access to a verified, disposable native database copy. */
+    database(): Promise<Database>;
+    /** The pinned OpenCode API/SDK over a separate disposable database copy. */
+    sdk(): Promise<OpenCode.Interface>;
+    schema(): Promise<typeof import("@opencode/schema")>;
+  };
+}
+
+/** Frozen judging source and dependency identity, kept in host-side run records. */
+export type CodeJudgeDefinition = {
+  file: string;
+  hash: string;
+  source: string;
+  sourceMap: string;
+  dependencies: Record<string, string>;
+};
+export type CodeJudgeExecution = {
+  state: "completed" | "failed" | "timed_out";
+  startedAt: string;
+  completedAt: string;
+  elapsedMs: number;
+  /** Original JSON, before boolean score normalization. */
+  output?: JsonValue;
+  scores?: Record<string, number | null>;
+  metrics?: RunMetrics;
+  error?: string;
+  sourceHash: string;
+  stdout: string;
+  stderr: string;
+};

package/src/judgment.ts

@@ -1,40 +1,49 @@
-import type { MetricDefinition, MetricJudgment } from "./types";
+import type { CriterionDefinition, CriterionScore } from "./types";
 
-export const JUDGE_PROTOCOL = 3;
+export const JUDGE_PROTOCOL = 4;
 export const isScored = (value: unknown): value is number =>
   typeof value === "number" &&
   Number.isFinite(value) &&
   value >= 0 &&
   value <= 1;
 
-/** Every rubric explicitly declares its metrics. */
-export function rubricMetrics(rubric: string): MetricDefinition[] {
-  const metrics = [
-    ...rubric.matchAll(/^## Metric: ([a-z][a-z0-9_]*)[ \t]*[—–-][ \t]*(.+)$/gm),
+/** The author declares Markdown criteria; code functions return their own scores. */
+export function rubricCriteria(rubric: string): CriterionDefinition[] {
+  if (/^## Metric:/m.test(rubric))
+    throw new Error("Use ## Criterion: id — Label for rubric declarations");
+  const criteria = [
+    ...rubric.matchAll(
+      /^## Criterion: ([a-z][a-z0-9_]*)[ \t]*[—–-][ \t]*(.+)$/gm,
+    ),
   ].map((match) => ({
     id: match[1],
     name: match[2].trim(),
   }));
-  if (new Set(metrics.map((metric) => metric.id)).size !== metrics.length)
-    throw new Error("Metric IDs must be unique within judge.md");
   if (
-    !metrics.length ||
-    metrics.some((metric) => !metric.name) ||
-    metrics.length !== (rubric.match(/^## Metric:/gm)?.length ?? 0)
+    new Set(criteria.map((criterion) => criterion.id)).size !== criteria.length
+  )
+    throw new Error("Criterion IDs must be unique within judge.md");
+  if (
+    !criteria.length ||
+    criteria.some((criterion) => !criterion.name) ||
+    criteria.length !== (rubric.match(/^## Criterion:/gm)?.length ?? 0)
   )
     throw new Error(
-      "Declare every metric as ## Metric: id — Label in judge.md",
+      "Declare every criterion as ## Criterion: id — Label in judge.md",
     );
-  return metrics;
+  return criteria;
 }
 
 /** Equal weighting within an eval; no task-specific policy is implemented here.
  * https://arxiv.org/html/2607.07946#S5.SS3 (equal task weighting)
  */
-export function metricMean(
-  metrics: readonly Pick<MetricJudgment, "value">[],
+export function criterionMean(
+  scores: Readonly<Record<string, Pick<CriterionScore, "value">>>,
 ): number | null {
-  return metrics.length && metrics.every((metric) => isScored(metric.value))
-    ? metrics.reduce((sum, metric) => sum + metric.value!, 0) / metrics.length
+  const criteria = Object.values(scores);
+  return criteria.length &&
+    criteria.every((criterion) => isScored(criterion.value))
+    ? criteria.reduce((sum, criterion) => sum + criterion.value!, 0) /
+        criteria.length
     : null;
 }

package/src/types.ts

@@ -4,6 +4,11 @@ import type {
   ConfigEntry,
   AgentInfo,
 } from "@opencode/client";
+import type {
+  CodeJudgeDefinition,
+  CodeJudgeExecution,
+  RunMetrics,
+} from "./judge-context";
 export type OpenCodeStreamEvent = OpenCodeEvent | SessionLogItem;
 
 export const CANDIDATE_TIMEOUT_MS = 45 * 60 * 1000;
@@ -13,18 +18,22 @@ export type ProviderDefinitions = NonNullable<
   Extract<ConfigEntry, { type: "document" }>["info"]["providers"]
 >;
 export type SessionState =
-  "running" | "completed" | "stopped" | "failed" | "timed_out";
+  | "running"
+  | "completed"
+  | "stopped"
+  | "failed"
+  | "timed_out";
 export type Stage = "candidate" | "judge";
 
 export type Judge = {
-  model: ModelRef;
+  model?: ModelRef;
   timeoutMs?: number;
   websearch?: "exa" | false;
 };
 export type Benchmark = {
   name?: string;
   models: readonly ModelRef[];
-  judge: Judge;
+  judge?: Judge;
   repetitions?: number;
   concurrency?: number;
   candidate?: {
@@ -48,7 +57,8 @@ export type MonitorPolicy = {
 export type Eval = {
   /** Let the host judge stop execution once the criterion is conclusively decided. */
   earlyStop?:
-    boolean | (Partial<MonitorPolicy> & { onlyModels?: readonly ModelRef[] });
+    | boolean
+    | (Partial<MonitorPolicy> & { onlyModels?: readonly ModelRef[] });
   workspace?: {
     repository?: string;
     /** Readable file overlays committed in order to the named refs, on the host. */
@@ -69,15 +79,17 @@ export type EvalDefinition = {
   settings: Eval;
   sourceHash: string;
   judgeHash: string;
-  /** Declared by `## Metric: id — Label` headings in judge.md. */
-  metrics: MetricDefinition[];
+  /** Criteria declared in judge.md; code-defined IDs are discovered from results. */
+  criteria: CriterionDefinition[];
+  /** A frozen, bundled code judge; never sent to the candidate. */
+  code?: CodeJudgeDefinition;
   name: string;
 };
 export type BenchmarkDefinition = {
   name: string;
   directory: string;
   models: ModelRef[];
-  judge: Required<Judge>;
+  judge: { model?: ModelRef; timeoutMs: number; websearch: "exa" | false };
   repetitions: number;
   concurrency: number;
   candidate: {
@@ -113,6 +125,7 @@ export type SessionArchive = {
 };
 export type ToolCall = {
   id: string;
+  sessionID?: string;
   name: string;
   assistantMessageId: string;
   status: "preparing" | "called" | "succeeded" | "failed";
@@ -186,44 +199,49 @@ export type EvalRun = {
   elapsedMs?: number;
   evidence?: EvidenceRef;
   session?: SessionArchive;
+  /** Candidate-only observations, independent of rubric scores. */
+  metrics?: RunMetrics;
   error?: string;
   replaces?: string;
   stop?: EvalStop;
 };
-export type MetricDefinition = { id: string; name: string };
+export type CriterionDefinition = { id: string; name: string };
 export type EvidenceCitation = (
   | { kind: "response" }
-  | { kind: "tool" | "message" | "event"; id: string }
+  | { kind: "recording" }
+  | { kind: "tool" | "message" | "event" | "metric"; id: string }
   | { kind: "artifact"; path: string; revision: "initial" | "final" }
 ) & { quote?: string; offset?: number };
-export type MetricJudgment = {
-  id: string;
-  value: 0 | 1 | null;
+export type CriterionScore = {
+  value: number | null;
   reason: string;
   evidence: EvidenceCitation[];
   sources?: string[];
+  source: "judge.md" | "judge.ts";
 };
-/** value is the equal-weight metric mean, or null if any metric is unknown. */
+/** value is the equal-weight criterion mean, or null if any score is unknown. */
 export type Judgment = {
   value: number | null;
   reason: string;
-  metrics: MetricJudgment[];
+  scores: Record<string, CriterionScore>;
 };
 export type JudgeRunInput = {
   evalRunId: string;
   evidence: EvidenceRef;
   rubric: string;
   /** Shared native agent profile used for this judgment. */
-  agent: Pick<AgentInfo, "id" | "mode" | "description" | "permissions"> & {
+  agent?: Pick<AgentInfo, "id" | "mode" | "description" | "permissions"> & {
     system: string;
   };
-  model: ModelRef;
+  model?: ModelRef;
+  kind: "llm" | "code" | "hybrid";
+  code?: CodeJudgeDefinition;
   judgeHash: string;
   timeoutMs: number;
   websearch: "exa" | false;
   mode: "monitor" | "final";
   runtimeHash: string;
-  metrics: MetricDefinition[];
+  criteria: CriterionDefinition[];
   protocol: number;
   monitor?: MonitorPolicy;
 };
@@ -237,6 +255,7 @@ export type JudgeRun = {
   archiveStartedAt?: string;
   judgment?: Judgment;
   session?: SessionArchive;
+  code?: CodeJudgeExecution;
   error?: string;
   activity?: "watching" | "queued" | "checking" | "finalizing";
   decisionCheckId?: string;