@hona/openeval 0.3.3 → 0.4.0

package/JUDGING.md

@@ -152,6 +152,63 @@ must declare at least one `## Criterion: id — Label`. A normalized Judgment ha
 an aggregate value and a scores map of CriterionScore objects containing value,
 reason, evidence, and source. The original code output is retained separately.
 
+## Criterion categories
+
+A category is any non-empty string on a criterion. Categories group results in
+the viewer and can compose a benchmark. They never change judge input,
+fingerprints, or the headline weighting.
+
+In `judge.md`, put one optional line directly below a criterion heading:
+
+```md
+## Criterion: asked_dialect — Asks for the SQL dialect
+Categories: misalignment, general
+```
+
+OpenEval removes that line before hashing the rubric and before the judge reads
+it, so adding or changing categories does not rejudge recorded evidence. A
+`Categories:` line anywhere else is an error.
+
+In `judge.ts`, export labels and categories for the scores it returns:
+
+```ts
+import type { CodeCriteria, JudgeContext } from "@hona/openeval";
+
+export const criteria = {
+  correct_answer: { name: "Correct answer", categories: ["general"] },
+} satisfies CodeCriteria;
+
+export default ({ response }: JudgeContext) => ({
+  scores: { correct_answer: response.text.trim() === "42" },
+});
+```
+
+Declare each criterion ID in one judge file only. Names are trimmed and matched
+case-insensitively; the first spelling is displayed. A criterion may have
+several categories, but one is usually clearer. A category score uses the same
+rule as the overall score: criteria are averaged within each eval, then evals
+are weighted equally. Unscored checks keep that category's score a range.
+
+Compose a benchmark from categories in `benchmark.ts`. Evals without a matching
+criterion are not run, and scores use only matching criteria:
+
+```ts
+export default {
+  models: ["opencode/gpt-6-astra#high"],
+  categories: ["coding", "verification"],
+} satisfies Benchmark;
+```
+
+`openeval run --only-category <name>` limits one invocation to evals with a
+matching criterion, like `--only-eval`, without changing the composition.
+
+Prefer published category sets so results are comparable:
+
+| Kind | Source | Categories |
+| --- | --- | --- |
+| Capability | [Artificial Analysis Intelligence Index](https://artificialanalysis.ai/methodology/intelligence-benchmarking) | `agents`, `coding`, `general`, `scientific-reasoning`; also `multilingual`, `vision` |
+| Agent failure | [MAST](https://arxiv.org/abs/2503.13657) (Cemri et al., NeurIPS 2025) | `specification`, `misalignment`, `verification` |
+
 ## Structured tool submissions
 
 | Tool | Data |

package/README.md

@@ -86,6 +86,10 @@ or does not provide a parameterized query.
 | Only asks which database | **1** | **0** |
 | Required recording is unavailable | **null** | **null** |
 
+Add an optional `Categories: misalignment` line under a heading to compare models
+by category in the viewer's radar chart and heatmap. Categories never trigger
+rejudging.
+
 → [Write good rubrics](https://openev.al/docs/rubrics/) · [Download the SQL starter](https://openev.al/starter.zip)
 
 ## One vocabulary

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hona/openeval",
-  "version": "0.3.3",
+  "version": "0.4.0",
   "description": "Code and LLM evaluations with isolated agents, recorded OpenCode data, and a results viewer",
   "license": "MIT",
   "repository": {

package/src/app/load-benchmark.ts

@@ -10,12 +10,17 @@ import type {
 } from "../types";
 import { CANDIDATE_TIMEOUT_MS } from "../types";
 import { rubricCriteria } from "../judgment";
+import {
+  categoryKey,
+  categoryList,
+  rubricCategories,
+} from "../criterion-categories";
 import { compileCodeJudge } from "../infra/judging/code-source";
+import { readCodeCriteria } from "../infra/judging/code-criteria";
 import { RUNTIME_IMAGE } from "../infra/opencode/version";
 import { monitorPolicy } from "./monitor-policy";
 import {
   fingerprint,
-  hash,
   relativePath,
   treeHash,
   contained,
@@ -35,10 +40,13 @@ export const modelRef = (value: unknown): ModelRef => {
     throw new Error(`Invalid model reference: ${String(value)}`);
   return value as ModelRef;
 };
+/** Bun caches modules by path, ignoring URL queries; clear the entry so edits load. */
+async function freshModule(path: string): Promise<Record<string, unknown>> {
+  delete require.cache[path];
+  return import(pathToFileURL(path).href);
+}
 async function declaration<T>(path: string): Promise<T> {
-  const url = pathToFileURL(path);
-  url.searchParams.set("version", hash(await Bun.file(path).bytes()));
-  return (await import(url.href)).default;
+  return (await freshModule(path)).default as T;
 }
 async function loadEval(directory: string): Promise<EvalDefinition> {
   const id = basename(directory),
@@ -168,10 +176,23 @@ async function loadEval(directory: string): Promise<EvalDefinition> {
       throw new Error(`${id}: preparation requires argv`);
   }
   const promptText = await prompt.text(),
-    judgeText = hasMarkdown ? await judge.text() : "";
+    rubric = rubricCategories(hasMarkdown ? await judge.text() : ""),
+    judgeText = rubric.rubric;
   if (!promptText.trim() || (hasMarkdown && !judgeText.trim()))
     throw new Error(`${id}: prompt and judge must not be empty`);
   const code = hasCode ? await compileCodeJudge(codeFile) : undefined;
+  const criteria = hasMarkdown ? rubricCriteria(judgeText) : [];
+  const declared = hasCode ? await codeCriteria(id, codeFile) : [];
+  if (declared.some((item) => criteria.some(({ id }) => id === item.id)))
+    throw new Error(
+      `${id}: declare each criterion in either judge.md or judge.ts, not both`,
+    );
+  const categories = Object.fromEntries(
+    [
+      ...Object.entries(rubric.categories),
+      ...declared.map((item) => [item.id, item.categories] as const),
+    ].filter(([, names]) => names.length),
+  );
   return {
     id,
     directory,
@@ -183,12 +204,49 @@ async function loadEval(directory: string): Promise<EvalDefinition> {
       source,
     }),
     judgeHash: fingerprint({ rubric: judgeText, code: code?.hash }),
-    criteria: hasMarkdown ? rubricCriteria(judgeText) : [],
+    criteria,
+    ...(declared.length
+      ? { codeCriteria: declared.map(({ id, name }) => ({ id, name })) }
+      : {}),
+    ...(Object.keys(categories).length ? { categories } : {}),
     ...(code ? { code } : {}),
     name: /^# (.+)$/m.exec(judgeText)?.[1] ?? id,
   };
 }
 
+/** Reads judge.ts's optional `criteria` export as data; planning never executes judge code. */
+async function codeCriteria(id: string, path: string) {
+  const declared = await readCodeCriteria(path);
+  if (declared === undefined) return [];
+  if (!declared || typeof declared !== "object" || Array.isArray(declared))
+    throw new Error(`${id}/judge.ts criteria must be an object`);
+  return Object.entries(declared).map(([criterion, value]) => {
+    const where = `${id}/judge.ts criterion ${criterion}`;
+    if (!/^[a-z][a-z0-9_]*$/.test(criterion))
+      throw new Error(`${where}: use a lowercase snake_case ID`);
+    if (
+      !value ||
+      typeof value !== "object" ||
+      Array.isArray(value) ||
+      Object.keys(value).some((key) => !["name", "categories"].includes(key))
+    )
+      throw new Error(`${where}: declare only name and categories`);
+    const { name, categories = [] } = value as {
+      name?: unknown;
+      categories?: unknown;
+    };
+    if (name !== undefined && (typeof name !== "string" || !name.trim()))
+      throw new Error(`${where}: name must be a non-empty string`);
+    if (!Array.isArray(categories))
+      throw new Error(`${where}: categories must be an array`);
+    return {
+      id: criterion,
+      name: (name as string | undefined)?.trim() ?? criterion.replaceAll("_", " "),
+      categories: categoryList(categories, where),
+    };
+  });
+}
+
 export async function loadBenchmark(
   path: string,
 ): Promise<BenchmarkDefinition> {
@@ -214,10 +272,19 @@ export async function loadBenchmark(
           "concurrency",
           "candidate",
           "container",
+          "categories",
         ].includes(key),
     )
   )
     throw new Error("benchmark.ts contains unsupported settings");
+  if (
+    definition.categories !== undefined &&
+    (!Array.isArray(definition.categories) || !definition.categories.length)
+  )
+    throw new Error("benchmark.ts categories must be a non-empty array");
+  const categories = definition.categories
+    ? categoryList(definition.categories, "benchmark.ts")
+    : undefined;
   const models = definition.models.map(modelRef);
   if (
     definition.judge !== undefined &&
@@ -245,9 +312,21 @@ export async function loadBenchmark(
     .filter((entry) => entry.isDirectory() && !entry.name.startsWith("."))
     .sort((a, b) => a.name.localeCompare(b.name));
   if (!directories.length) throw new Error("Benchmark has no eval folders");
-  const evals = await Promise.all(
+  const declared = await Promise.all(
     directories.map((entry) => loadEval(resolve(evalRoot, entry.name))),
   );
+  const keys = categories?.map(categoryKey);
+  const evals = keys
+    ? declared.filter((item) =>
+        Object.values(item.categories ?? {}).some((names) =>
+          names.some((name) => keys.includes(categoryKey(name))),
+        ),
+      )
+    : declared;
+  if (!evals.length)
+    throw new Error(
+      `No criteria match the benchmark categories: ${categories!.join(", ")}`,
+    );
   if (evals.some((item) => item.judge) && !definition.judge?.model)
     throw new Error("A benchmark containing judge.md requires judge.model");
   const concurrency = positive(definition.concurrency, 10, "Concurrency");
@@ -295,5 +374,6 @@ export async function loadBenchmark(
       cpus: positive(definition.container?.cpus, 2, "CPU count"),
       memoryMiB: positive(definition.container?.memoryMiB, 4096, "Memory"),
     },
+    ...(categories ? { categories } : {}),
   };
 }

package/src/app/read-results.ts

@@ -14,7 +14,7 @@ import type {
   StageStatus,
   JudgeAudit,
 } from "../view";
-import { sumCosts } from "../view";
+import { categoryCatalog, sumCosts } from "../view";
 import { benchmarkScores } from "./scores";
 import { executionRuntime } from "./execution-runtime";
 import {
@@ -316,6 +316,7 @@ export class ResultReader {
           evals.map((item) => [item.id, item.name]),
         ),
         modelNames: run.modelNames ?? {},
+        categories: categoryCatalog(scores[0]?.components ?? []),
         cost: evalId ? evalCosts[evalId] : sumCosts(Object.values(evalCosts)),
         evalCosts,
         runtime,

package/src/app/run-benchmark.ts

@@ -23,6 +23,11 @@ import { canJudgeEval } from "./eval-state";
 import { isScored } from "../judgment";
 import { benchmarkScores } from "./scores";
 import { CostBudget, estimateWork } from "./cost-plan";
+import {
+  categoryKey,
+  categoryList,
+  inCategories,
+} from "../criterion-categories";
 
 export type RunBenchmarkOptions = {
   directory?: string;
@@ -32,6 +37,8 @@ export type RunBenchmarkOptions = {
   /** Execute work only for these models while retaining the full aggregate. */
   onlyModels?: readonly ModelRef[];
   onlyEvals?: readonly string[];
+  /** Execute only evals with a criterion in any of these categories. */
+  onlyCategories?: readonly string[];
   onlyRepetitions?: readonly number[];
   /** Stop admitting work when reported spend plus reservations reaches this amount. */
   maxCostUSD?: number;
@@ -39,6 +46,29 @@ export type RunBenchmarkOptions = {
   onEvent?: ExecutionObserver;
 };
 
+/** Intersect --only-eval with evals that have a criterion in any --only-category. */
+function scopedEvals(
+  definition: BenchmarkDefinition,
+  options: Pick<RunBenchmarkOptions, "onlyEvals" | "onlyCategories">,
+) {
+  if (!options.onlyCategories) return options.onlyEvals;
+  const keys = categoryList(options.onlyCategories, "--only-category").map(
+    categoryKey,
+  );
+  if (!keys.length) throw new Error("Select at least one category");
+  const matching = definition.evals
+    .filter((item) =>
+      Object.values(item.categories ?? {}).some((names) =>
+        inCategories({ categories: names }, keys),
+      ),
+    )
+    .map((item) => item.id)
+    .filter((id) => !options.onlyEvals || options.onlyEvals.includes(id));
+  if (!matching.length)
+    throw new Error("No selected evals have criteria in these categories");
+  return matching;
+}
+
 export async function currentBenchmarkRun(directory: string) {
   const root = resolve(directory, "results");
   const dirs = (await readdir(root, { withFileTypes: true }).catch(() => []))
@@ -141,6 +171,7 @@ export async function runBenchmark(
       ))
   )
     throw new Error("Select configured eval IDs with --only-eval");
+  const onlyEvals = scopedEvals(definition, options);
   new CostBudget(options.maxCostUSD, () => 0);
   const selected = options.onlyModels?.map(modelRef);
   if (selected && !selected.length)
@@ -192,11 +223,11 @@ export async function runBenchmark(
     results.slots().map((slot) => [slot.id, slot]),
   );
   let plan = planBenchmark(definition, runtime, results);
-  if (selected || options.onlyEvals || options.onlyRepetitions) {
+  if (selected || onlyEvals || options.onlyRepetitions) {
     const retained = new Map(results.slots().map((slot) => [slot.id, slot]));
     plan = plan.map((item) =>
       ((!selected || selected.includes(item.slot.model)) &&
-        (!options.onlyEvals || options.onlyEvals.includes(item.slot.evalId)) &&
+        (!onlyEvals || onlyEvals.includes(item.slot.evalId)) &&
         (!options.onlyRepetitions ||
           options.onlyRepetitions.includes(item.slot.repetition))) ||
       item.action === "reuse"
@@ -244,7 +275,7 @@ export async function runBenchmark(
     execution: {
       startedAt: now,
       onlyModels: selected,
-      onlyEvals: options.onlyEvals,
+      onlyEvals,
       onlyRepetitions: options.onlyRepetitions,
       budgetUSD: options.maxCostUSD,
       estimatedUSD: estimate.estimatedUSD,

package/src/app/scores.ts

@@ -1,6 +1,7 @@
 import type { BenchmarkDefinition, JudgeRun, Slot } from "../types";
 import { modelScore } from "../view";
 import { isScored } from "../judgment";
+import { categoryKey, inCategories } from "../criterion-categories";
 
 /** Scores are derived from a single selection snapshot; every eval has equal weight. */
 export function benchmarkScores(
@@ -13,10 +14,14 @@ export function benchmarkScores(
   const evals = definition.evals.filter(
     (item) => !evalId || item.id === evalId,
   );
+  const keys = definition.categories?.map(categoryKey);
   const definitions = new Map(
     evals.map((item) => {
       const criteria = new Map(
-        item.criteria.map((criterion) => [criterion.id, criterion]),
+        [...item.criteria, ...(item.codeCriteria ?? [])].map((criterion) => [
+          criterion.id,
+          criterion,
+        ]),
       );
       for (const slot of slots.filter(
         (slot) => slot.active && slot.evalId === item.id,
@@ -28,7 +33,16 @@ export function benchmarkScores(
           if (!criteria.has(id))
             criteria.set(id, { id, name: id.replaceAll("_", " ") });
       }
-      return [item.id, [...criteria.values()]];
+      const categorized = [...criteria.values()].map((criterion) => ({
+        ...criterion,
+        categories: item.categories?.[criterion.id] ?? [],
+      }));
+      return [
+        item.id,
+        keys
+          ? categorized.filter((criterion) => inCategories(criterion, keys))
+          : categorized,
+      ];
     }),
   );
   return definition.models.map((model) =>
@@ -57,6 +71,9 @@ export function benchmarkScores(
             eval: item.id,
             criterion: criterion.id,
             name: criterion.name,
+            ...(criterion.categories.length
+              ? { categories: criterion.categories }
+              : {}),
             value:
               values.length === definition.repetitions
                 ? scoredSum / definition.repetitions

package/src/cli.ts

@@ -51,6 +51,7 @@ Options:
   --model <provider/id>  Model to add or remove (repeatable)
   --only-model <ref>     Execute only this model (repeatable)
   --only-eval <id>       Execute only this eval (repeatable)
+  --only-category <name> Execute only evals with criteria in this category (repeatable)
   --only-repetition <n>  Execute only this repetition (repeatable)
   --max-cost <usd>       Scheduling budget for this invocation
   --final-only          Judge only after candidates finish
@@ -84,6 +85,11 @@ Requires Bun 1.4.2+, Docker, and an authenticated OpenCode installation.`);
             arg === "--only-eval" ? [args[index + 1]] : [],
           )
         : undefined,
+      onlyCategories: args.includes("--only-category")
+        ? args.flatMap((arg, index) =>
+            arg === "--only-category" ? [args[index + 1]] : [],
+          )
+        : undefined,
       maxCostUSD: args.includes("--max-cost")
         ? Number(option("--max-cost"))
         : undefined,

package/src/criterion-categories.ts

@@ -0,0 +1,53 @@
+/** Categories are reporting metadata: any non-empty string, matched case-insensitively. */
+export const categoryKey = (name: string) => name.trim().toLowerCase();
+
+/** Trim names and drop case-insensitive duplicates, keeping the first spelling. */
+export function categoryList(values: readonly unknown[], where: string) {
+  const names: string[] = [],
+    keys = new Set<string>();
+  for (const value of values) {
+    if (typeof value !== "string" || !value.trim())
+      throw new Error(`${where}: categories must be non-empty strings`);
+    const name = value.trim();
+    if (keys.has(categoryKey(name))) continue;
+    keys.add(categoryKey(name));
+    names.push(name);
+  }
+  return names;
+}
+
+const CATEGORY_LINE = /^Categor(?:y|ies):[ \t]*(.*?)\r?$/i;
+const HEADING = /^## Criterion: ([a-z][a-z0-9_]*)[ \t]*[—–-]/;
+
+/** Reads the optional `Categories:` line directly below each `## Criterion:` heading.
+ * The returned rubric omits those lines, so adding or changing categories never
+ * changes judge input or judge fingerprints.
+ */
+export function rubricCategories(rubric: string) {
+  const lines = rubric.split("\n"),
+    kept: string[] = [],
+    categories: Record<string, string[]> = {};
+  for (let index = 0; index < lines.length; index++) {
+    const line = lines[index]!;
+    if (CATEGORY_LINE.test(line))
+      throw new Error(
+        "Put a Categories: line directly below its ## Criterion: heading",
+      );
+    kept.push(line);
+    const heading = HEADING.exec(line),
+      next = heading ? CATEGORY_LINE.exec(lines[index + 1] ?? "") : null;
+    if (!heading || !next) continue;
+    categories[heading[1]!] = categoryList(
+      next[1]!.split(","),
+      `Criterion ${heading[1]}`,
+    );
+    index++;
+  }
+  return { rubric: kept.join("\n"), categories };
+}
+
+/** Whether a scored criterion belongs to any of the selected category keys. */
+export const inCategories = (
+  part: { categories?: readonly string[] },
+  keys: readonly string[],
+) => !!part.categories?.some((name) => keys.includes(categoryKey(name)));

package/src/index.ts

@@ -9,6 +9,7 @@ export type {
   JudgeRun,
   Judgment,
   CriterionDefinition,
+  CodeCriteria,
   CriterionScore,
   EvidenceCitation,
   ToolCall,
@@ -31,6 +32,10 @@ export type {
 } from "./evidence";
 export { CANDIDATE_TIMEOUT_MS } from "./types";
 export { rubricCriteria, criterionMean, isScored } from "./judgment";
+export {
+  categoryKey,
+  rubricCategories,
+} from "./criterion-categories";
 export type {
   JudgeContext,
   JudgeFunction,

package/src/infra/containers/catalog.ts

@@ -1,18 +1,64 @@
 import { mkdir, mkdtemp, rm } from "node:fs/promises";
 import { tmpdir } from "node:os";
 import { resolve } from "node:path";
-import type { BenchmarkDefinition, ModelNames } from "../../types";
+import type {
+  BenchmarkDefinition,
+  ModelNames,
+  ModelRef,
+  ProviderDefinitions,
+} from "../../types";
 import { CandidateContainer } from "./oci";
 import { createSessionDatabase } from "../opencode/host";
 import { createCredentialSnapshot, integrationFor } from "../opencode/auth";
 import { clientFor, parseModel } from "../opencode/session";
+import { candidateProviders } from "../../app/input-fingerprints";
 
 const CATALOG_TIMEOUT_MS = 60_000;
 
-/** Display names from the candidates' OpenCode catalog, including declared provider overrides. */
+type CatalogDefinition = Pick<
+  BenchmarkDefinition,
+  "models" | "candidate" | "container"
+>;
+
+/** Display names from the candidates' OpenCode catalog, including declared provider overrides.
+ * Like candidates, each container receives only one integration's credentials and its models' provider configuration.
+ */
 export async function readModelNames(
-  definition: Pick<BenchmarkDefinition, "models" | "candidate" | "container">,
+  definition: CatalogDefinition,
+  imageId: string,
+): Promise<ModelNames> {
+  const names = await Promise.all(
+    catalogRequests(definition).map((request) =>
+      readIntegrationNames(definition.container, imageId, request).catch(
+        () => ({}),
+      ),
+    ),
+  );
+  return Object.assign({}, ...names);
+}
+
+export type CatalogRequest = {
+  integration: string;
+  models: readonly ModelRef[];
+  providers?: ProviderDefinitions;
+};
+
+/** One request per credential integration, scoped like the candidates it names. */
+export const catalogRequests = (
+  definition: Pick<BenchmarkDefinition, "models" | "candidate">,
+): CatalogRequest[] =>
+  [...Map.groupBy(definition.models, integrationFor)].map(
+    ([integration, models]) => ({
+      integration,
+      models,
+      providers: scopedProviders(definition.candidate.providers, models),
+    }),
+  );
+
+async function readIntegrationNames(
+  container: BenchmarkDefinition["container"],
   imageId: string,
+  { integration, models, providers }: CatalogRequest,
 ): Promise<ModelNames> {
   const staging = await mkdtemp(
     resolve(process.env.TMP ?? tmpdir(), "catalog-"),
@@ -23,31 +69,19 @@ export async function readModelNames(
     await mkdir(workspace);
     await createSessionDatabase(
       database,
-      createCredentialSnapshot(undefined, [
-        ...new Set(definition.models.map(integrationFor)),
-      ]),
-    );
-    await using container = await CandidateContainer.create(
-      definition.container,
-      imageId,
-    );
-    await container.prepare(
-      workspace,
-      database,
-      false,
-      [],
-      staging,
-      definition.candidate.providers,
+      createCredentialSnapshot(undefined, [integration]),
     );
+    await using catalog = await CandidateContainer.create(container, imageId);
+    await catalog.prepare(workspace, database, false, [], staging, providers);
     const client = clientFor(
-      await container.start(CATALOG_TIMEOUT_MS),
-      container.password,
+      await catalog.start(CATALOG_TIMEOUT_MS),
+      catalog.password,
     );
     const location = { directory: "/workspace" };
     await client.plugin.awaitActivation({ location });
     const listed = (await client.model.list({ location })).data;
     return Object.fromEntries(
-      definition.models.flatMap((model) => {
+      models.flatMap((model) => {
         const selected = parseModel(model),
           found = listed.find(
             (item) =>
@@ -68,3 +102,25 @@ export async function readModelNames(
     });
   }
 }
+
+/** Union of what each listed candidate would receive, without other providers' entries. */
+function scopedProviders(
+  providers: ProviderDefinitions | undefined,
+  models: readonly ModelRef[],
+): ProviderDefinitions | undefined {
+  let scoped: NonNullable<ProviderDefinitions> | undefined;
+  for (const model of models)
+    for (const [id, entry] of Object.entries(
+      candidateProviders(providers, model) ?? {},
+    )) {
+      scoped ??= {};
+      scoped[id] = {
+        ...scoped[id],
+        ...entry,
+        ...(scoped[id]?.models || entry.models
+          ? { models: { ...scoped[id]?.models, ...entry.models } }
+          : {}),
+      };
+    }
+  return scoped;
+}