@hona/openeval 0.2.0

package/JUDGING.md

@@ -0,0 +1,123 @@
+# Judging recorded work
+
+## Shared judge agent
+
+OpenEval uses the native OpenCode V2 primary agent `openeval-judge` for final
+grading, live checks, rejudging, calibration, and independent audits. Its
+[base system prompt](src/infra/judging/judge-agent.md) owns the common evidence,
+citation, uncertainty, early-decision, and output rules. See
+[OpenCode V2 custom agents](https://opencode.ai/v2/docs/agents/).
+
+The native agent's system prompt contains the base instructions and one copy of
+the eval rubric. It remains present across compaction. User turns identify the
+phase; structured context and decisions move through registered Code Mode tools.
+The tools use native Effect schemas for argument validation and catalog types.
+
+Every JudgeRun requires its shared profile in `input.agent`, explicit metrics,
+mode, runtime hash, and protocol. The effective
+native configuration is archived at `configuration/opencode.json` in its judge
+directory. The profile contributes to the judge input fingerprint, so a shared
+prompt change schedules rejudging using saved evidence. Candidate input
+fingerprints do not include the judge profile.
+
+## Eval metric rubrics
+
+Declare one or more metrics in `judge.md`:
+
+```md
+# Advice quality
+
+## Metric: current_advice — Advice for the current situation
+Pass when the requested advice is present and its recommendations are available now.
+Fail when a recommendation is unavailable now, or the requested advice is omitted.
+
+## Metric: later_advice — Advice for later stages
+Judge later recommendations at their explicitly stated stage.
+```
+
+Keep metric-specific rules, accepted alternatives, and domain facts in that file.
+The LLM applies these rules. The SDK validates the declared IDs, binary metric
+values, recorded evidence references, and exact optional quotes. It calculates
+the equal-weight metric mean. A missing metric or invalid citation is a judge
+protocol error; missing source evidence can produce a null metric. Every rubric
+must declare at least one `## Metric: id — Label`, and every judgment contains
+the complete metric array.
+
+## Structured tool submissions
+
+| Tool | Data |
+| --- | --- |
+| `judge_context` | Current request ID, phase, metric declarations, and evidence index |
+| `candidate_evidence` | Recorded responses, tools, events, messages, and artifacts |
+| `submit_judgment` | Scores keyed by metric ID, with reasons and citations |
+| `continue_judging` | An early check's reason for needing more evidence |
+
+Native argument validation and citation validation return errors directly to the
+judge. It can correct arguments or retrieve better evidence and retry within the
+same native turn and existing timeout. The runner does not parse assistant text
+for JSON or start synthetic format-repair turns. Ending without an accepted tool
+submission fails the JudgeRun rather than assigning a candidate zero.
+
+Tool schemas stay stable across checks. Each request ID is bound to one fixed
+evidence view. Only the first valid submission is accepted; stale, concurrent, or
+cancelled submissions cannot overwrite it or affect a later check. Early
+submissions require every metric to be non-null and irreversible. Final grading
+permits null metrics and rejects `continue_judging`.
+
+Accepted submissions and semantic rejections are recorded in
+`judgment-submissions.json`, with their request/check IDs and checkpoints. Native
+schema failures are retained in the native tool transcript.
+
+Every decided metric cites recorded evidence. Citations can name a response,
+message ID, tool-call ID, event sequence, or initial/final artifact path. The
+evidence reference/checkpoint binds those citations to the exact recording.
+Source URLs are supplementary domain references, not substitutes for citations
+to the candidate's work.
+
+For example, the judge submits this from Code Mode after inspecting the recording:
+
+```js
+const context = await tools.judge_context({});
+await tools.submit_judgment({
+  requestId: context.requestId,
+  metrics: {
+    current_advice: {
+      value: 1,
+      reason: "The current-stage recommendation satisfies the rubric.",
+      evidence: [{ kind: "response" }],
+    },
+    later_advice: {
+      value: 0,
+      reason: "The later-stage recommendation violates the rubric.",
+      evidence: [{ kind: "response" }],
+    },
+  },
+});
+```
+
+The host stores the metric array and aggregate value `0.5`. A citation's optional `quote` must occur
+literally in the referenced response/tool/message/event/artifact. An artifact
+citation must specify `path` and `revision` (`initial` or `final`); other record
+citations specify `id`. Semantic interpretation remains the judge's job.
+
+`recordEvidence` creates a calibration recording from supplied text/tool records
+without executing a model or tool. `judgeEvidence` grades retained evidence into
+a new standalone audit directory without changing a benchmark's active scores.
+Use `judgeRuns` when an explicit rejudge should update active selections.
+
+Runtime uses recorded work intervals. Judge checks record `executionStartedAt`
+when they obtain a worker, separately from their queue-admission `startedAt`.
+`archiveStartedAt` records active finalization. A persistent judge's waiting time
+is excluded. A check cancelled before worker admission has no runtime. Calendar
+creation/completion dates remain provenance, not elapsed-runtime measurements.
+
+## Design references
+
+- [DeepSWE quality assurance](https://deepswe.datacurve.ai/blog/deepswe#qa):
+  acceptance breadth and alignment between the specification and grading.
+- [DeepSWE metrics](https://arxiv.org/html/2607.07946#S5.SS3): equal task weighting.
+- [JudgeBench](https://arxiv.org/abs/2410.12784): validate factual and logical
+  judging ability with known examples, rather than assuming model strength.
+
+Keep task-specific decisions in `judge.md`; executable SDK code handles the
+recording protocol and generic arithmetic, not task success rules.

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Hona
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,133 @@
+# OpenEval
+
+A typed prompt-plus-LLM-judge SDK for Bun. Run agents in isolated containers,
+record their work, grade the evidence, and explore the results in a local viewer.
+
+## Install
+
+Requires Bun 1.4.2 or later, Docker, and an authenticated OpenCode installation.
+The agent runtime currently uses OpenCode `0.0.0-beta-19296`.
+
+```sh
+bun add --exact @hona/openeval
+```
+
+## Declare a benchmark
+
+```text
+my-benchmark/
+  benchmark.ts
+  evals/
+    ask-dialect/
+      prompt.md
+      judge.md
+```
+
+`benchmark.ts`:
+
+```ts
+import type { Benchmark } from "@hona/openeval";
+
+export default {
+  models: ["provider/candidate-model"],
+  judge: { model: "provider/judge-model" },
+  repetitions: 3,
+} satisfies Benchmark;
+```
+
+Replace the model references with models connected in OpenCode.
+
+`evals/ask-dialect/prompt.md`:
+
+```md
+Write a SQL query for the ten most recent orders for a customer.
+```
+
+`evals/ask-dialect/judge.md`:
+
+```md
+# Requests the SQL dialect
+
+## Metric: asked_dialect — Asks which SQL dialect to use
+
+Pass when the agent asks which database or SQL dialect is in use.
+Fail when it assumes a dialect without asking. Asking alongside a draft counts.
+```
+
+Prompts are sent verbatim. Each metric is `0`, `1`, or `null` when evidence is
+insufficient. The judge supplies citations to the recording. A shared native
+judge agent supplies the evidence and submission protocol; rubrics contain the
+task-specific criteria. See [JUDGING.md](JUDGING.md).
+
+## Run and inspect
+
+From your benchmark directory:
+
+```sh
+bunx --bun @hona/openeval image
+bunx --bun @hona/openeval plan
+bunx --bun @hona/openeval run
+bunx --bun @hona/openeval view
+```
+
+The viewer opens at `http://127.0.0.1:4173`. Use `--benchmark <directory>` to point
+any command at another benchmark, or `--port <port>` for another viewer port.
+
+`run` resumes the current aggregate and reuses unchanged work. `--new` starts a
+separate result. Repeat `--only-eval`, `--only-model`, or `--only-repetition` to
+execute a small scope while retaining the full aggregate. `--max-cost <usd>`
+sets a scheduling budget for the invocation. Run `openeval --help` for commands.
+
+Scores average repetitions per metric, metrics per eval, then evals equally.
+Unresolved checks produce completion bounds until the final percentage is known.
+The viewer provides metric drilldowns and recorded candidate and judge sessions.
+Elapsed time measures active execution intervals, counting overlapping work once.
+
+## Prepare a workspace
+
+An eval's optional `workspace/` directory supplies files. An optional `eval.ts`
+can declare a pinned Git repository or readable revision overlays and preparation
+commands:
+
+```ts
+import type { Eval } from "@hona/openeval";
+
+export default {
+  prepare: [{ cwd: ".", argv: ["bun", "install", "--frozen-lockfile"] }],
+} satisfies Eval;
+```
+
+Preparation runs before recording the initial candidate workspace. Candidate
+containers receive project inputs, never judge rubrics or evaluator storage.
+Candidates finish naturally, meet an opted-in irreversible judge decision, or
+time out after at most 45 minutes. Finalized recordings remain immutable.
+
+## SDK
+
+```ts
+import { runBenchmark, serveResults } from "@hona/openeval";
+
+await runBenchmark("./my-benchmark");
+```
+
+The package also exports types, result readers, snapshot and rejudge functions,
+and reusable view/session data through `@hona/openeval/types`, `/results`,
+`/view`, and `/session`.
+
+## Develop and release
+
+```sh
+bun install --frozen-lockfile
+bun run typecheck
+bun test
+bun run release:pack
+bun run release:verify
+```
+
+The private benchmark client is maintained in a separate repository. It installs
+an exact published SDK version. This repository contains only the reusable SDK,
+CLI, viewer, and development inputs. See [RELEASING.md](https://github.com/Hona/openeval/blob/main/RELEASING.md).
+
+OpenEval is MIT licensed. The vendored session UI retains its upstream MIT
+license and revision in `vendor/session-ui`. Built viewer distributions include
+third-party license notices.

package/package.json

@@ -0,0 +1,39 @@
+{
+  "name": "@hona/openeval",
+  "version": "0.2.0",
+  "description": "Typed prompt-and-judge evaluations with isolated agents, recorded evidence, and a results viewer",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/Hona/openeval.git",
+    "directory": "packages/openeval"
+  },
+  "engines": {
+    "bun": ">=1.4.2"
+  },
+  "bin": {
+    "openeval": "src/cli.ts"
+  },
+  "files": ["src", "viewer", "LICENSE", "README.md", "JUDGING.md", "!src/**/*.test.ts"],
+  "publishConfig": {
+    "access": "public"
+  },
+  "type": "module",
+  "exports": {
+    ".": "./src/index.ts",
+    "./types": "./src/types.ts",
+    "./results": "./src/app/read-results.ts",
+    "./view": "./src/view.ts",
+    "./session": "./src/session-replay.ts"
+  },
+  "dependencies": {
+    "@types/tar-stream": "3.1.4",
+    "@opencode/client": "0.0.0-beta-19296",
+    "@opencode/core": "0.0.0-beta-19296",
+    "@opencode/sdk": "0.0.0-beta-19296",
+    "@opencode/util": "0.0.0-beta-19296",
+    "drizzle-orm": "0.45.2",
+    "effect": "4.0.0-rc.112",
+    "tar-stream": "3.1.7"
+  }
+}

package/src/app/add-models.ts

@@ -0,0 +1,18 @@
+import type { ModelRef, ExecutionObserver } from "../types";
+import { Results } from "../infra/sqlite";
+import { resolve } from "node:path";
+import { runBenchmark } from "./run-benchmark";
+
+export async function addModels(
+  directory: string,
+  models: readonly ModelRef[],
+  onEvent?: ExecutionObserver,
+) {
+  if (!models.length) throw new Error("Select models to add");
+  let source: string;
+  {
+    using results = new Results(resolve(directory, "runner.db"), true);
+    source = results.benchmark!.source;
+  }
+  return runBenchmark(source, { directory, models, onEvent });
+}

package/src/app/context.ts

@@ -0,0 +1,15 @@
+import type {
+  BenchmarkDefinition,
+  BenchmarkRun,
+  ExecutionObserver,
+} from "../types";
+import type { Results } from "../infra/sqlite";
+
+export type ExecutionContext = {
+  directory: string;
+  definition: BenchmarkDefinition;
+  runtime: BenchmarkRun["runtime"];
+  results: Results;
+  onEvent?: ExecutionObserver;
+  finalOnly?: boolean;
+};

package/src/app/cost-plan.ts

@@ -0,0 +1,118 @@
+import type {
+  BenchmarkDefinition,
+  EvalRun,
+  JudgeRun,
+  PlanItem,
+} from "../types";
+import type { Results } from "../infra/sqlite";
+
+export type CostEstimate = {
+  candidates: number;
+  judges: number;
+  estimatedUSD: number | null;
+  unknownItems: number;
+  items: Array<{ slotId: string; estimatedUSD: number | null }>;
+};
+const median = (values: number[]) => {
+  if (!values.length) return undefined;
+  values.sort((a, b) => a - b);
+  const middle = Math.floor(values.length / 2);
+  return values.length % 2
+    ? values[middle]
+    : (values[middle - 1] + values[middle]) / 2;
+};
+
+/** Empirical planning estimates, separate from reported spend.
+ * https://deepswe.datacurve.ai/blog/deepswe#beyond-pass-rate
+ */
+export function estimateWork(
+  definition: BenchmarkDefinition,
+  results: Results,
+  plan: readonly PlanItem[],
+): CostEstimate {
+  const evals = results.evalRuns(),
+    judges = results.judgeRuns();
+  const cost = (runs: Array<EvalRun | JudgeRun>) =>
+    median(
+      runs.flatMap((run) => {
+        const value = run.session?.accounting?.costUSD;
+        return value === undefined ? [] : [value];
+      }),
+    );
+  const ready = plan.filter(
+    (item) => item.action === "candidate" || item.action === "judge",
+  );
+  const items = ready.map((item) => {
+    const candidate =
+      cost(
+        evals.filter(
+          (run) =>
+            run.input.evalId === item.slot.evalId &&
+            run.input.model === item.slot.model,
+        ),
+      ) ??
+      cost(evals.filter((run) => run.input.model === item.slot.model)) ??
+      cost(evals);
+    const judge =
+      cost(
+        judges.filter(
+          (run) =>
+            run.input.model === definition.judge.model &&
+            evals.some(
+              (e) =>
+                e.id === run.input.evalRunId &&
+                e.input.evalId === item.slot.evalId,
+            ),
+        ),
+      ) ??
+      cost(
+        judges.filter((run) => run.input.model === definition.judge.model),
+      ) ??
+      cost(judges);
+    return {
+      slotId: item.slot.id,
+      estimatedUSD:
+        judge === undefined ||
+        (item.action === "candidate" && candidate === undefined)
+          ? null
+          : judge + (item.action === "candidate" ? candidate! : 0),
+    };
+  });
+  return {
+    candidates: ready.filter((item) => item.action === "candidate").length,
+    judges: ready.length,
+    estimatedUSD: items.every((item) => item.estimatedUSD !== null)
+      ? items.reduce((sum, item) => sum + item.estimatedUSD!, 0)
+      : null,
+    unknownItems: items.filter((item) => item.estimatedUSD === null).length,
+    items,
+  };
+}
+
+/** Scheduling guard: active work finishes naturally, so this is not a billing ceiling. */
+export class CostBudget {
+  private reserved = 0;
+  constructor(
+    readonly limit: number | undefined,
+    private spent: () => number,
+  ) {
+    if (limit !== undefined && (!Number.isFinite(limit) || limit < 0))
+      throw new Error("Cost budget must be a non-negative USD amount");
+  }
+  reserve(estimate: number | null): (() => void) | undefined {
+    if (this.limit === undefined) return () => {};
+    if (
+      estimate === null ||
+      this.spent() + this.reserved + estimate > this.limit
+    )
+      return;
+    this.reserved += estimate;
+    let released = false;
+    return () => {
+      if (!released) {
+        this.reserved -= estimate;
+        released = true;
+      }
+    };
+  }
+}

package/src/app/eval-state.ts

@@ -0,0 +1,10 @@
+import type { EvalRun, EvidenceRef } from "../types";
+
+/** A time limit ends execution, but recorded work can still satisfy the rubric. */
+export function canJudgeEval(
+  run: EvalRun | undefined,
+): run is EvalRun & { evidence: EvidenceRef } {
+  return (
+    !!run?.evidence && ["completed", "stopped", "timed_out"].includes(run.state)
+  );
+}

package/src/app/execution-budget.ts

@@ -0,0 +1,65 @@
+type Kind = "eval" | "judge";
+type Waiting = { kind: Kind; start(): void; cancel(): void };
+
+/** One stage budget. Live monitors reserve one lane so long evals cannot starve checks. */
+export class ExecutionBudget {
+  private active = 0;
+  private evals = 0;
+  private monitors = 0;
+  private waiting: Waiting[] = [];
+  constructor(readonly limit: number) {
+    if (!Number.isInteger(limit) || limit < 1)
+      throw new Error("Invalid concurrency limit");
+  }
+  monitor() {
+    if (this.limit < 2)
+      throw new Error("Early stopping requires concurrency of at least 2");
+    this.monitors++;
+    let released = false;
+    return () => {
+      if (released) return;
+      released = true;
+      this.monitors--;
+      this.pump();
+    };
+  }
+  run<T>(kind: Kind, work: () => Promise<T>, signal?: AbortSignal): Promise<T> {
+    if (signal?.aborted) return Promise.reject(signal.reason);
+    return new Promise<T>((resolve, reject) => {
+      const request: Waiting = {
+        kind,
+        cancel: () => {
+          this.waiting = this.waiting.filter((item) => item !== request);
+          signal?.removeEventListener("abort", request.cancel);
+          reject(signal?.reason);
+          this.pump();
+        },
+        start: () => {
+          signal?.removeEventListener("abort", request.cancel);
+          this.active++;
+          if (kind === "eval") this.evals++;
+          Promise.resolve()
+            .then(work)
+            .then(resolve, reject)
+            .finally(() => {
+              this.active--;
+              if (kind === "eval") this.evals--;
+              this.pump();
+            });
+        },
+      };
+      this.waiting.push(request);
+      signal?.addEventListener("abort", request.cancel, { once: true });
+      this.pump();
+    });
+  }
+  private pump() {
+    while (this.active < this.limit) {
+      let index = this.waiting.findIndex((item) => item.kind === "judge");
+      if (index < 0 && this.evals < this.limit - (this.monitors ? 1 : 0))
+        index = this.waiting.findIndex((item) => item.kind === "eval");
+      if (index < 0) return;
+      this.waiting.splice(index, 1)[0].start();
+    }
+  }
+}

package/src/app/execution-runtime.ts

@@ -0,0 +1,63 @@
+import type { BenchmarkRun, EvalRun, JudgeRun } from "../types";
+import type { Results } from "../infra/sqlite";
+import { mergeRuntime, timestampMs, type RuntimeInterval } from "../runtime";
+
+/** Read-only projection; finalized runs and their original timestamps are untouched. */
+export function executionRuntime(
+  run: EvalRun | JudgeRun,
+  results: Results,
+  benchmark: BenchmarkRun,
+  now: number,
+  current = true,
+): RuntimeInterval[] {
+  const start = timestampMs(run.startedAt),
+    completed = timestampMs(run.completedAt);
+  if (start === undefined) return [];
+  const heartbeat = timestampMs(benchmark.updatedAt) ?? start;
+  const live =
+    current &&
+    run.state === "running" &&
+    benchmark.state === "running" &&
+    now - heartbeat <= 30_000;
+  const lastObserved =
+    !current && completed === undefined
+      ? (results.eventBounds(run.id)?.last ?? start)
+      : undefined;
+  const closing = completed ?? Math.min(now, lastObserved ?? heartbeat);
+  const interval = (begin: number, end?: number): RuntimeInterval[] =>
+    Number.isFinite(begin) &&
+    (end === undefined || (Number.isFinite(end) && end >= begin))
+      ? [{ start: begin, ...(end === undefined ? {} : { end }) }]
+      : [];
+
+  if (!("evalRunId" in run.input) || run.input.mode !== "monitor") {
+    const end =
+      completed !== undefined
+        ? completed
+        : live
+          ? undefined
+          : Math.min(closing, start + run.input.timeoutMs);
+    return interval(start, end);
+  }
+
+  // A persistent judge is idle between checks. Its session lifetime is not work.
+  const judge = run as JudgeRun;
+  const checks = results.judgeChecks(run.id);
+  const intervals = checks.flatMap((check) => {
+    const begin = timestampMs(check.executionStartedAt);
+    if (begin === undefined) return []; // Cancelled before obtaining a worker.
+    const end =
+      timestampMs(check.completedAt) ??
+      (live && judge.activity === "checking" ? undefined : closing);
+    return interval(begin, end);
+  });
+  const archiveStart = timestampMs(judge.archiveStartedAt);
+  if (archiveStart !== undefined)
+    intervals.push(
+      ...interval(
+        archiveStart,
+        live && judge.activity === "finalizing" ? undefined : closing,
+      ),
+    );
+  return mergeRuntime(intervals);
+}

package/src/app/input-fingerprints.ts

@@ -0,0 +1,67 @@
+import type {
+  BenchmarkDefinition,
+  BenchmarkRun,
+  ModelRef,
+  JudgeRunInput,
+} from "../types";
+import { fingerprint } from "../infra/files";
+import { JUDGE_PROTOCOL } from "../judgment";
+import { JUDGE_AGENT } from "../infra/judging/agent";
+
+/** Declared, candidate-visible inputs. Harness implementation hashes are provenance. */
+export function candidateFingerprint(
+  definition: BenchmarkDefinition,
+  evalId: string,
+  model: ModelRef,
+  runtime: BenchmarkRun["runtime"],
+) {
+  const item = definition.evals.find((item) => item.id === evalId)!;
+  const [name] = model.split("#"),
+    slash = name.indexOf("/");
+  const provider = definition.candidate.providers?.[name.slice(0, slash)];
+  const { models, ...settings } = provider ?? {};
+  return fingerprint({
+    prompt: item.prompt,
+    source: item.sourceHash,
+    model,
+    image: runtime.imageId,
+    candidate: {
+      timeoutMs: definition.candidate.timeoutMs,
+      websearch: definition.candidate.websearch,
+      provider: provider
+        ? { ...settings, model: models?.[name.slice(slash + 1)] }
+        : undefined,
+    },
+    container: {
+      engine: definition.container.engine,
+      cpus: definition.container.cpus,
+      memoryMiB: definition.container.memoryMiB,
+    },
+  });
+}
+export const judgeFingerprint = (
+  definition: BenchmarkDefinition,
+  evalId: string,
+) =>
+  fingerprint({
+    rubric: definition.evals.find((item) => item.id === evalId)!.judge,
+    agent: JUDGE_AGENT,
+    judge: definition.judge,
+    protocol: JUDGE_PROTOCOL,
+  });
+export const savedJudgeFingerprint = (
+  input: Pick<
+    JudgeRunInput,
+    "rubric" | "agent" | "protocol" | "model" | "timeoutMs" | "websearch"
+  >,
+) =>
+  fingerprint({
+    rubric: input.rubric,
+    agent: input.agent,
+    protocol: input.protocol,
+    judge: {
+      model: input.model,
+      timeoutMs: input.timeoutMs,
+      websearch: input.websearch,
+    },
+  });