@hona/openeval 0.3.2 → 0.4.0

package/JUDGING.md

@@ -152,6 +152,63 @@ must declare at least one `## Criterion: id — Label`. A normalized Judgment ha
 an aggregate value and a scores map of CriterionScore objects containing value,
 reason, evidence, and source. The original code output is retained separately.
 
+## Criterion categories
+
+A category is any non-empty string on a criterion. Categories group results in
+the viewer and can compose a benchmark. They never change judge input,
+fingerprints, or the headline weighting.
+
+In `judge.md`, put one optional line directly below a criterion heading:
+
+```md
+## Criterion: asked_dialect — Asks for the SQL dialect
+Categories: misalignment, general
+```
+
+OpenEval removes that line before hashing the rubric and before the judge reads
+it, so adding or changing categories does not rejudge recorded evidence. A
+`Categories:` line anywhere else is an error.
+
+In `judge.ts`, export labels and categories for the scores it returns:
+
+```ts
+import type { CodeCriteria, JudgeContext } from "@hona/openeval";
+
+export const criteria = {
+  correct_answer: { name: "Correct answer", categories: ["general"] },
+} satisfies CodeCriteria;
+
+export default ({ response }: JudgeContext) => ({
+  scores: { correct_answer: response.text.trim() === "42" },
+});
+```
+
+Declare each criterion ID in one judge file only. Names are trimmed and matched
+case-insensitively; the first spelling is displayed. A criterion may have
+several categories, but one is usually clearer. A category score uses the same
+rule as the overall score: criteria are averaged within each eval, then evals
+are weighted equally. Unscored checks keep that category's score a range.
+
+Compose a benchmark from categories in `benchmark.ts`. Evals without a matching
+criterion are not run, and scores use only matching criteria:
+
+```ts
+export default {
+  models: ["opencode/gpt-6-astra#high"],
+  categories: ["coding", "verification"],
+} satisfies Benchmark;
+```
+
+`openeval run --only-category <name>` limits one invocation to evals with a
+matching criterion, like `--only-eval`, without changing the composition.
+
+Prefer published category sets so results are comparable:
+
+| Kind | Source | Categories |
+| --- | --- | --- |
+| Capability | [Artificial Analysis Intelligence Index](https://artificialanalysis.ai/methodology/intelligence-benchmarking) | `agents`, `coding`, `general`, `scientific-reasoning`; also `multilingual`, `vision` |
+| Agent failure | [MAST](https://arxiv.org/abs/2503.13657) (Cemri et al., NeurIPS 2025) | `specification`, `misalignment`, `verification` |
+
 ## Structured tool submissions
 
 | Tool | Data |

package/README.md

@@ -86,6 +86,10 @@ or does not provide a parameterized query.
 | Only asks which database | **1** | **0** |
 | Required recording is unavailable | **null** | **null** |
 
+Add an optional `Categories: misalignment` line under a heading to compare models
+by category in the viewer's radar chart and heatmap. Categories never trigger
+rejudging.
+
 → [Write good rubrics](https://openev.al/docs/rubrics/) · [Download the SQL starter](https://openev.al/starter.zip)
 
 ## One vocabulary
@@ -211,13 +215,21 @@ await runBenchmark("./my-benchmark", {
 
 ## Write evals with an agent
 
+Copy the **Agent prompt** at [openev.al](https://openev.al), or use
+[agent-start.md](https://github.com/Hona/openeval/blob/main/agent-start.md). It walks through project selection,
+prerequisites, the writing skill, one eval, models, and an optional first run.
+The prompt starts from [llms.txt](https://openev.al/llms.txt); documentation pages
+also support Markdown fetches and direct `index.md` URLs.
+
 Use the public [Eval Writing skill](https://github.com/Hona/openeval/tree/main/.opencode/skills/eval-writing)
 to turn a real failure into an eval, review a rubric, or investigate misleading
 scores. It guides an agent through concrete false-pass/false-failure examples,
 accepted alternatives, evidence requirements, and human-reviewed calibration.
 
 Copy the whole `.opencode/skills/eval-writing/` directory, including `references/`,
-into the same path in your project. For global use, copy it to
+into the same path in your project, or extract the
+[project skill ZIP](https://openev.al/eval-writing.zip) into your project root.
+For global use, copy it to
 `~/.config/opencode/skills/eval-writing/`. Then run **`/eval-writing`** in OpenCode.
 
 > Use eval-writing to review this task and rubric. Show me the strongest false

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hona/openeval",
-  "version": "0.3.2",
+  "version": "0.4.0",
   "description": "Code and LLM evaluations with isolated agents, recorded OpenCode data, and a results viewer",
   "license": "MIT",
   "repository": {

package/src/app/input-fingerprints.ts

@@ -3,11 +3,46 @@ import type {
   BenchmarkRun,
   ModelRef,
   JudgeRunInput,
+  ProviderDefinitions,
 } from "../types";
 import { fingerprint } from "../infra/files";
 import { JUDGE_PROTOCOL } from "../judgment";
 import { JUDGE_AGENT } from "../infra/judging/agent";
 
+/** The candidate's own provider settings and model override; other entries do not apply to it. */
+function providerScope(
+  providers: ProviderDefinitions | undefined,
+  model: ModelRef,
+) {
+  const [name] = model.split("#"),
+    slash = name.indexOf("/"),
+    id = name.slice(0, slash),
+    modelId = name.slice(slash + 1);
+  const { models, ...settings } = providers?.[id] ?? {};
+  const override = models?.[modelId];
+  return override || Object.keys(settings).length
+    ? { id, modelId, settings, override }
+    : undefined;
+}
+
+/** Provider configuration installed in one candidate container. */
+export function candidateProviders(
+  providers: ProviderDefinitions | undefined,
+  model: ModelRef,
+): ProviderDefinitions | undefined {
+  const scope = providerScope(providers, model);
+  return scope
+    ? {
+        [scope.id]: {
+          ...scope.settings,
+          ...(scope.override
+            ? { models: { [scope.modelId]: scope.override } }
+            : {}),
+        },
+      }
+    : undefined;
+}
+
 /** Declared, candidate-visible inputs. Harness implementation hashes are provenance. */
 export function candidateFingerprint(
   definition: BenchmarkDefinition,
@@ -16,10 +51,7 @@ export function candidateFingerprint(
   runtime: BenchmarkRun["runtime"],
 ) {
   const item = definition.evals.find((item) => item.id === evalId)!;
-  const [name] = model.split("#"),
-    slash = name.indexOf("/");
-  const provider = definition.candidate.providers?.[name.slice(0, slash)];
-  const { models, ...settings } = provider ?? {};
+  const scope = providerScope(definition.candidate.providers, model);
   return fingerprint({
     prompt: item.prompt,
     source: item.sourceHash,
@@ -28,9 +60,7 @@ export function candidateFingerprint(
     candidate: {
       timeoutMs: definition.candidate.timeoutMs,
       websearch: definition.candidate.websearch,
-      provider: provider
-        ? { ...settings, model: models?.[name.slice(slash + 1)] }
-        : undefined,
+      provider: scope && { ...scope.settings, model: scope.override },
     },
     container: {
       engine: definition.container.engine,

package/src/app/load-benchmark.ts

@@ -10,12 +10,17 @@ import type {
 } from "../types";
 import { CANDIDATE_TIMEOUT_MS } from "../types";
 import { rubricCriteria } from "../judgment";
+import {
+  categoryKey,
+  categoryList,
+  rubricCategories,
+} from "../criterion-categories";
 import { compileCodeJudge } from "../infra/judging/code-source";
+import { readCodeCriteria } from "../infra/judging/code-criteria";
 import { RUNTIME_IMAGE } from "../infra/opencode/version";
 import { monitorPolicy } from "./monitor-policy";
 import {
   fingerprint,
-  hash,
   relativePath,
   treeHash,
   contained,
@@ -35,10 +40,13 @@ export const modelRef = (value: unknown): ModelRef => {
     throw new Error(`Invalid model reference: ${String(value)}`);
   return value as ModelRef;
 };
+/** Bun caches modules by path, ignoring URL queries; clear the entry so edits load. */
+async function freshModule(path: string): Promise<Record<string, unknown>> {
+  delete require.cache[path];
+  return import(pathToFileURL(path).href);
+}
 async function declaration<T>(path: string): Promise<T> {
-  const url = pathToFileURL(path);
-  url.searchParams.set("version", hash(await Bun.file(path).bytes()));
-  return (await import(url.href)).default;
+  return (await freshModule(path)).default as T;
 }
 async function loadEval(directory: string): Promise<EvalDefinition> {
   const id = basename(directory),
@@ -168,10 +176,23 @@ async function loadEval(directory: string): Promise<EvalDefinition> {
       throw new Error(`${id}: preparation requires argv`);
   }
   const promptText = await prompt.text(),
-    judgeText = hasMarkdown ? await judge.text() : "";
+    rubric = rubricCategories(hasMarkdown ? await judge.text() : ""),
+    judgeText = rubric.rubric;
   if (!promptText.trim() || (hasMarkdown && !judgeText.trim()))
     throw new Error(`${id}: prompt and judge must not be empty`);
   const code = hasCode ? await compileCodeJudge(codeFile) : undefined;
+  const criteria = hasMarkdown ? rubricCriteria(judgeText) : [];
+  const declared = hasCode ? await codeCriteria(id, codeFile) : [];
+  if (declared.some((item) => criteria.some(({ id }) => id === item.id)))
+    throw new Error(
+      `${id}: declare each criterion in either judge.md or judge.ts, not both`,
+    );
+  const categories = Object.fromEntries(
+    [
+      ...Object.entries(rubric.categories),
+      ...declared.map((item) => [item.id, item.categories] as const),
+    ].filter(([, names]) => names.length),
+  );
   return {
     id,
     directory,
@@ -183,12 +204,49 @@ async function loadEval(directory: string): Promise<EvalDefinition> {
       source,
     }),
     judgeHash: fingerprint({ rubric: judgeText, code: code?.hash }),
-    criteria: hasMarkdown ? rubricCriteria(judgeText) : [],
+    criteria,
+    ...(declared.length
+      ? { codeCriteria: declared.map(({ id, name }) => ({ id, name })) }
+      : {}),
+    ...(Object.keys(categories).length ? { categories } : {}),
     ...(code ? { code } : {}),
     name: /^# (.+)$/m.exec(judgeText)?.[1] ?? id,
   };
 }
 
+/** Reads judge.ts's optional `criteria` export as data; planning never executes judge code. */
+async function codeCriteria(id: string, path: string) {
+  const declared = await readCodeCriteria(path);
+  if (declared === undefined) return [];
+  if (!declared || typeof declared !== "object" || Array.isArray(declared))
+    throw new Error(`${id}/judge.ts criteria must be an object`);
+  return Object.entries(declared).map(([criterion, value]) => {
+    const where = `${id}/judge.ts criterion ${criterion}`;
+    if (!/^[a-z][a-z0-9_]*$/.test(criterion))
+      throw new Error(`${where}: use a lowercase snake_case ID`);
+    if (
+      !value ||
+      typeof value !== "object" ||
+      Array.isArray(value) ||
+      Object.keys(value).some((key) => !["name", "categories"].includes(key))
+    )
+      throw new Error(`${where}: declare only name and categories`);
+    const { name, categories = [] } = value as {
+      name?: unknown;
+      categories?: unknown;
+    };
+    if (name !== undefined && (typeof name !== "string" || !name.trim()))
+      throw new Error(`${where}: name must be a non-empty string`);
+    if (!Array.isArray(categories))
+      throw new Error(`${where}: categories must be an array`);
+    return {
+      id: criterion,
+      name: (name as string | undefined)?.trim() ?? criterion.replaceAll("_", " "),
+      categories: categoryList(categories, where),
+    };
+  });
+}
+
 export async function loadBenchmark(
   path: string,
 ): Promise<BenchmarkDefinition> {
@@ -214,10 +272,19 @@ export async function loadBenchmark(
           "concurrency",
           "candidate",
           "container",
+          "categories",
         ].includes(key),
     )
   )
     throw new Error("benchmark.ts contains unsupported settings");
+  if (
+    definition.categories !== undefined &&
+    (!Array.isArray(definition.categories) || !definition.categories.length)
+  )
+    throw new Error("benchmark.ts categories must be a non-empty array");
+  const categories = definition.categories
+    ? categoryList(definition.categories, "benchmark.ts")
+    : undefined;
   const models = definition.models.map(modelRef);
   if (
     definition.judge !== undefined &&
@@ -245,9 +312,21 @@ export async function loadBenchmark(
     .filter((entry) => entry.isDirectory() && !entry.name.startsWith("."))
     .sort((a, b) => a.name.localeCompare(b.name));
   if (!directories.length) throw new Error("Benchmark has no eval folders");
-  const evals = await Promise.all(
+  const declared = await Promise.all(
     directories.map((entry) => loadEval(resolve(evalRoot, entry.name))),
   );
+  const keys = categories?.map(categoryKey);
+  const evals = keys
+    ? declared.filter((item) =>
+        Object.values(item.categories ?? {}).some((names) =>
+          names.some((name) => keys.includes(categoryKey(name))),
+        ),
+      )
+    : declared;
+  if (!evals.length)
+    throw new Error(
+      `No criteria match the benchmark categories: ${categories!.join(", ")}`,
+    );
   if (evals.some((item) => item.judge) && !definition.judge?.model)
     throw new Error("A benchmark containing judge.md requires judge.model");
   const concurrency = positive(definition.concurrency, 10, "Concurrency");
@@ -295,5 +374,6 @@ export async function loadBenchmark(
       cpus: positive(definition.container?.cpus, 2, "CPU count"),
       memoryMiB: positive(definition.container?.memoryMiB, 4096, "Memory"),
     },
+    ...(categories ? { categories } : {}),
   };
 }

package/src/app/merge-runs.ts

@@ -88,6 +88,7 @@ export async function mergeBenchmarkRuns(
     {
       ...previous,
       definition,
+      modelNames: { ...source.benchmark!.modelNames, ...previous.modelNames },
       state: "incomplete",
       updatedAt: new Date().toISOString(),
       sources: [

package/src/app/read-results.ts

@@ -14,7 +14,7 @@ import type {
   StageStatus,
   JudgeAudit,
 } from "../view";
-import { sumCosts } from "../view";
+import { categoryCatalog, sumCosts } from "../view";
 import { benchmarkScores } from "./scores";
 import { executionRuntime } from "./execution-runtime";
 import {
@@ -315,6 +315,8 @@ export class ResultReader {
         evalNames: Object.fromEntries(
           evals.map((item) => [item.id, item.name]),
         ),
+        modelNames: run.modelNames ?? {},
+        categories: categoryCatalog(scores[0]?.components ?? []),
         cost: evalId ? evalCosts[evalId] : sumCosts(Object.values(evalCosts)),
         evalCosts,
         runtime,

package/src/app/run-benchmark.ts

@@ -10,6 +10,7 @@ import type {
 } from "../types";
 import { Results } from "../infra/sqlite";
 import { runtimeFingerprint } from "../infra/containers/oci";
+import { readModelNames } from "../infra/containers/catalog";
 import { prepareWorkspace } from "../infra/containers/workspace";
 import { fingerprint, errorMessage } from "../infra/files";
 import { loadBenchmark, modelRef } from "./load-benchmark";
@@ -22,6 +23,11 @@ import { canJudgeEval } from "./eval-state";
 import { isScored } from "../judgment";
 import { benchmarkScores } from "./scores";
 import { CostBudget, estimateWork } from "./cost-plan";
+import {
+  categoryKey,
+  categoryList,
+  inCategories,
+} from "../criterion-categories";
 
 export type RunBenchmarkOptions = {
   directory?: string;
@@ -31,6 +37,8 @@ export type RunBenchmarkOptions = {
   /** Execute work only for these models while retaining the full aggregate. */
   onlyModels?: readonly ModelRef[];
   onlyEvals?: readonly string[];
+  /** Execute only evals with a criterion in any of these categories. */
+  onlyCategories?: readonly string[];
   onlyRepetitions?: readonly number[];
   /** Stop admitting work when reported spend plus reservations reaches this amount. */
   maxCostUSD?: number;
@@ -38,6 +46,29 @@ export type RunBenchmarkOptions = {
   onEvent?: ExecutionObserver;
 };
 
+/** Intersect --only-eval with evals that have a criterion in any --only-category. */
+function scopedEvals(
+  definition: BenchmarkDefinition,
+  options: Pick<RunBenchmarkOptions, "onlyEvals" | "onlyCategories">,
+) {
+  if (!options.onlyCategories) return options.onlyEvals;
+  const keys = categoryList(options.onlyCategories, "--only-category").map(
+    categoryKey,
+  );
+  if (!keys.length) throw new Error("Select at least one category");
+  const matching = definition.evals
+    .filter((item) =>
+      Object.values(item.categories ?? {}).some((names) =>
+        inCategories({ categories: names }, keys),
+      ),
+    )
+    .map((item) => item.id)
+    .filter((id) => !options.onlyEvals || options.onlyEvals.includes(id));
+  if (!matching.length)
+    throw new Error("No selected evals have criteria in these categories");
+  return matching;
+}
+
 export async function currentBenchmarkRun(directory: string) {
   const root = resolve(directory, "results");
   const dirs = (await readdir(root, { withFileTypes: true }).catch(() => []))
@@ -140,6 +171,7 @@ export async function runBenchmark(
       ))
   )
     throw new Error("Select configured eval IDs with --only-eval");
+  const onlyEvals = scopedEvals(definition, options);
   new CostBudget(options.maxCostUSD, () => 0);
   const selected = options.onlyModels?.map(modelRef);
   if (selected && !selected.length)
@@ -191,11 +223,11 @@ export async function runBenchmark(
     results.slots().map((slot) => [slot.id, slot]),
   );
   let plan = planBenchmark(definition, runtime, results);
-  if (selected || options.onlyEvals || options.onlyRepetitions) {
+  if (selected || onlyEvals || options.onlyRepetitions) {
     const retained = new Map(results.slots().map((slot) => [slot.id, slot]));
     plan = plan.map((item) =>
       ((!selected || selected.includes(item.slot.model)) &&
-        (!options.onlyEvals || options.onlyEvals.includes(item.slot.evalId)) &&
+        (!onlyEvals || onlyEvals.includes(item.slot.evalId)) &&
         (!options.onlyRepetitions ||
           options.onlyRepetitions.includes(item.slot.repetition))) ||
       item.action === "reuse"
@@ -215,11 +247,17 @@ export async function runBenchmark(
     (item) => item.action === "candidate" || item.action === "judge",
   );
   const previous = results.benchmark;
+  const modelNames = {
+    ...previous?.modelNames,
+    // Names are display metadata; an unavailable catalog keeps the recorded names.
+    ...(await readModelNames(definition, runtime.imageId).catch(() => ({}))),
+  };
   if (
     !work.length &&
     previous &&
     fingerprint(previous.definition) === fingerprint(definition) &&
-    fingerprint(previous.runtime) === fingerprint(runtime)
+    fingerprint(previous.runtime) === fingerprint(runtime) &&
+    fingerprint(previous.modelNames ?? {}) === fingerprint(modelNames)
   )
     return { directory, plan, estimate, benchmark: previous };
   const benchmark: BenchmarkRun = {
@@ -232,11 +270,12 @@ export async function runBenchmark(
     scheduledSlotIds: work.map((item) => item.slot.id),
     definition,
     runtime,
+    modelNames,
     sources: previous?.sources,
     execution: {
       startedAt: now,
       onlyModels: selected,
-      onlyEvals: options.onlyEvals,
+      onlyEvals,
       onlyRepetitions: options.onlyRepetitions,
       budgetUSD: options.maxCostUSD,
       estimatedUSD: estimate.estimatedUSD,

package/src/app/run-eval.ts

@@ -5,6 +5,7 @@ import { executeCandidate } from "../infra/containers/candidate";
 import type { EvidenceFeed } from "../evidence";
 import { CandidateEvidence } from "../infra/evidence";
 import { measureRecording } from "../infra/recording/metrics";
+import { candidateProviders } from "./input-fingerprints";
 
 /** The normal candidate use case: record its inputs, execute, finalize its evidence. */
 export async function runEval(
@@ -42,7 +43,10 @@ export async function runEval(
       websearch: context.definition.candidate.websearch,
       container: context.definition.container,
       imageId: context.runtime.imageId,
-      providers: context.definition.candidate.providers,
+      providers: candidateProviders(
+        context.definition.candidate.providers,
+        slot.model,
+      ),
     },
     directory,
     (event) => {

package/src/app/scores.ts

@@ -1,6 +1,7 @@
 import type { BenchmarkDefinition, JudgeRun, Slot } from "../types";
 import { modelScore } from "../view";
 import { isScored } from "../judgment";
+import { categoryKey, inCategories } from "../criterion-categories";
 
 /** Scores are derived from a single selection snapshot; every eval has equal weight. */
 export function benchmarkScores(
@@ -13,10 +14,14 @@ export function benchmarkScores(
   const evals = definition.evals.filter(
     (item) => !evalId || item.id === evalId,
   );
+  const keys = definition.categories?.map(categoryKey);
   const definitions = new Map(
     evals.map((item) => {
       const criteria = new Map(
-        item.criteria.map((criterion) => [criterion.id, criterion]),
+        [...item.criteria, ...(item.codeCriteria ?? [])].map((criterion) => [
+          criterion.id,
+          criterion,
+        ]),
       );
       for (const slot of slots.filter(
         (slot) => slot.active && slot.evalId === item.id,
@@ -28,7 +33,16 @@ export function benchmarkScores(
           if (!criteria.has(id))
             criteria.set(id, { id, name: id.replaceAll("_", " ") });
       }
-      return [item.id, [...criteria.values()]];
+      const categorized = [...criteria.values()].map((criterion) => ({
+        ...criterion,
+        categories: item.categories?.[criterion.id] ?? [],
+      }));
+      return [
+        item.id,
+        keys
+          ? categorized.filter((criterion) => inCategories(criterion, keys))
+          : categorized,
+      ];
     }),
   );
   return definition.models.map((model) =>
@@ -57,6 +71,9 @@ export function benchmarkScores(
             eval: item.id,
             criterion: criterion.id,
             name: criterion.name,
+            ...(criterion.categories.length
+              ? { categories: criterion.categories }
+              : {}),
             value:
               values.length === definition.repetitions
                 ? scoredSum / definition.repetitions

package/src/cli.ts

@@ -51,6 +51,7 @@ Options:
   --model <provider/id>  Model to add or remove (repeatable)
   --only-model <ref>     Execute only this model (repeatable)
   --only-eval <id>       Execute only this eval (repeatable)
+  --only-category <name> Execute only evals with criteria in this category (repeatable)
   --only-repetition <n>  Execute only this repetition (repeatable)
   --max-cost <usd>       Scheduling budget for this invocation
   --final-only          Judge only after candidates finish
@@ -84,6 +85,11 @@ Requires Bun 1.4.2+, Docker, and an authenticated OpenCode installation.`);
             arg === "--only-eval" ? [args[index + 1]] : [],
           )
         : undefined,
+      onlyCategories: args.includes("--only-category")
+        ? args.flatMap((arg, index) =>
+            arg === "--only-category" ? [args[index + 1]] : [],
+          )
+        : undefined,
       maxCostUSD: args.includes("--max-cost")
         ? Number(option("--max-cost"))
         : undefined,

package/src/criterion-categories.ts

@@ -0,0 +1,53 @@
+/** Categories are reporting metadata: any non-empty string, matched case-insensitively. */
+export const categoryKey = (name: string) => name.trim().toLowerCase();
+
+/** Trim names and drop case-insensitive duplicates, keeping the first spelling. */
+export function categoryList(values: readonly unknown[], where: string) {
+  const names: string[] = [],
+    keys = new Set<string>();
+  for (const value of values) {
+    if (typeof value !== "string" || !value.trim())
+      throw new Error(`${where}: categories must be non-empty strings`);
+    const name = value.trim();
+    if (keys.has(categoryKey(name))) continue;
+    keys.add(categoryKey(name));
+    names.push(name);
+  }
+  return names;
+}
+
+const CATEGORY_LINE = /^Categor(?:y|ies):[ \t]*(.*?)\r?$/i;
+const HEADING = /^## Criterion: ([a-z][a-z0-9_]*)[ \t]*[—–-]/;
+
+/** Reads the optional `Categories:` line directly below each `## Criterion:` heading.
+ * The returned rubric omits those lines, so adding or changing categories never
+ * changes judge input or judge fingerprints.
+ */
+export function rubricCategories(rubric: string) {
+  const lines = rubric.split("\n"),
+    kept: string[] = [],
+    categories: Record<string, string[]> = {};
+  for (let index = 0; index < lines.length; index++) {
+    const line = lines[index]!;
+    if (CATEGORY_LINE.test(line))
+      throw new Error(
+        "Put a Categories: line directly below its ## Criterion: heading",
+      );
+    kept.push(line);
+    const heading = HEADING.exec(line),
+      next = heading ? CATEGORY_LINE.exec(lines[index + 1] ?? "") : null;
+    if (!heading || !next) continue;
+    categories[heading[1]!] = categoryList(
+      next[1]!.split(","),
+      `Criterion ${heading[1]}`,
+    );
+    index++;
+  }
+  return { rubric: kept.join("\n"), categories };
+}
+
+/** Whether a scored criterion belongs to any of the selected category keys. */
+export const inCategories = (
+  part: { categories?: readonly string[] },
+  keys: readonly string[],
+) => !!part.categories?.some((name) => keys.includes(categoryKey(name)));

package/src/index.ts

@@ -9,6 +9,7 @@ export type {
   JudgeRun,
   Judgment,
   CriterionDefinition,
+  CodeCriteria,
   CriterionScore,
   EvidenceCitation,
   ToolCall,
@@ -31,6 +32,10 @@ export type {
 } from "./evidence";
 export { CANDIDATE_TIMEOUT_MS } from "./types";
 export { rubricCriteria, criterionMean, isScored } from "./judgment";
+export {
+  categoryKey,
+  rubricCategories,
+} from "./criterion-categories";
 export type {
   JudgeContext,
   JudgeFunction,