chaffjs 0.18.0 → 0.20.0

package/src/grade/variants-input.ts

@@ -0,0 +1,54 @@
+import type { GradeResult } from "./result.ts";
+import { isGradeResult } from "./results-read.ts";
+
+// The results `compareVariants()` is given, checked and grouped by variant. Pure. A harness builds this input by hand, so
+// a result without an id, without a variant or under an id its variant already has is said, not silently dropped.
+
+/** Each variant's results, by variant label then id, in order of first appearance. */
+export type VariantGroups = ReadonlyMap<string, ReadonlyMap<string, GradeResult>>;
+
+/** Labelled results, or each variant's results under its label. */
+export type VariantInput = readonly GradeResult[] | Readonly<Record<string, readonly GradeResult[]>>;
+
+type Labelled = { readonly variant: unknown; readonly result: unknown; readonly place: string };
+
+const isRecord = (value: unknown): value is Record<string, unknown> => typeof value === "object" && value !== null && !Array.isArray(value);
+
+const labelledOf = (input: unknown): Labelled[] | undefined => {
+  if (Array.isArray(input))
+    return input.map((result: unknown, index) => ({ variant: isRecord(result) ? result["variant"] : undefined, result, place: `results[${String(index)}]` }));
+  if (!isRecord(input)) return undefined;
+  return Object.entries(input).flatMap(([variant, results]) =>
+    Array.isArray(results)
+      ? results.map((result: unknown, index) => ({ variant, result, place: `${variant}[${String(index)}]` }))
+      : [{ variant, result: results, place: variant }],
+  );
+};
+
+/** A whole result line, as `--out` writes it, with an id to match by: a partial object would break the table, not be compared. */
+const isResult = (value: unknown): value is GradeResult => isGradeResult(value) && value.id !== "";
+
+/** Why one entry cannot be compared, or undefined when it can. */
+const problemOf = (entry: Labelled, groups: ReadonlyMap<string, ReadonlyMap<string, GradeResult>>): string | undefined => {
+  if (!isResult(entry.result)) return `${entry.place}: not a grade result with an id`;
+  if (typeof entry.variant !== "string" || entry.variant === "") return `${entry.place}: id "${entry.result.id}" has no variant`;
+  return groups.get(entry.variant)?.has(entry.result.id) === true
+    ? `${entry.place}: id "${entry.result.id}" is already in variant "${entry.variant}"`
+    : undefined;
+};
+
+/** The results grouped by variant, or every reason some cannot be. No result at all is a problem, not an empty comparison. */
+export const variantGroupsOf = (input: unknown): { readonly groups: VariantGroups } | { readonly problems: readonly string[] } => {
+  const labelled = labelledOf(input);
+  if (labelled === undefined) return { problems: ["results must be an array of grade results, or { variant: results[] }"] };
+  const groups = new Map<string, Map<string, GradeResult>>();
+  const problems = labelled.flatMap((entry) => {
+    const problem = problemOf(entry, groups);
+    if (problem !== undefined || !isResult(entry.result) || typeof entry.variant !== "string") return problem === undefined ? [] : [problem];
+    const byId = groups.get(entry.variant) ?? new Map<string, GradeResult>();
+    groups.set(entry.variant, byId.set(entry.result.id, { ...entry.result, variant: entry.variant }));
+    return [];
+  });
+  if (problems.length > 0) return { problems };
+  return groups.size === 0 ? { problems: ["no results to compare"] } : { groups };
+};

package/src/grade/variants-text.ts

@@ -0,0 +1,88 @@
+import type { Texts } from "../ui.ts";
+
+export type VariantText = {
+  readonly heading: (compared: number, variants: number) => string;
+  readonly passed: string;
+  readonly passedCell: (passed: number, outputs: number, rate: string) => string;
+  readonly factsDropped: string;
+  readonly factsAdded: string;
+  readonly citationsFailed: string;
+  readonly citationsCell: (failed: number, checked: number) => string;
+  readonly unsupportedFacts: string;
+  readonly penalty: string;
+  readonly ratesHeading: (unit: string) => string;
+  readonly unit: Readonly<Record<"char" | "word", string>>;
+  readonly disagreeHeading: (count: number) => string;
+  readonly disagreement: (id: string, passedIn: string, failedIn: string) => string;
+  readonly failedIn: (variant: string, reasons: string) => string;
+  readonly noDisagreement: string;
+  readonly missing: (id: string, variants: string) => string;
+  readonly readOtherwise: (ids: string) => string;
+  readonly notComparedHeading: string;
+  /** Markdown only: the run's heading and its totals. */
+  readonly runHeading: (path: string) => string;
+  readonly totals: (total: number, passed: number) => string;
+  readonly failedHeading: string;
+  readonly regressionsHeading: (count: number) => string;
+  readonly noRegression: string;
+  readonly stamp: string;
+  readonly separator: string;
+};
+
+export const VARIANT_TEXT: Texts<VariantText> = {
+  ja: {
+    heading: (compared, variants) => `${String(variants)} つの variant を並べた: どの variant にもある id の出力 ${String(compared)} 件`,
+    passed: "通った",
+    passedCell: (passed, outputs, rate) => `${String(passed)}/${String(outputs)}（${rate}）`,
+    factsDropped: "落ちた事実",
+    factsAdded: "足された事実",
+    citationsFailed: "外れた引用",
+    citationsCell: (failed, checked) => `${String(failed)}/${String(checked)}`,
+    unsupportedFacts: "一節に無い事実",
+    penalty: "減点の和",
+    ratesHeading: (unit) => `ルールごとの率（1,000 ${unit}あたり）`,
+    unit: { char: "字", word: "語" },
+    disagreeHeading: (count) => `合否が分かれた出力 ${String(count)} 件`,
+    disagreement: (id, passedIn, failedIn) => `${id}: 通った ${passedIn}、落ちた ${failedIn}`,
+    failedIn: (variant, reasons) => `${variant}（${reasons}）`,
+    noDisagreement: "合否が分かれた出力はありません",
+    missing: (id, variants) => `${id}: ${variants} に無い`,
+    readOtherwise: (ids) => `言語かジャンルが違う: ${ids}`,
+    notComparedHeading: "比べなかった id",
+    runHeading: (path) => `chaff grade: ${path}`,
+    totals: (total, passed) => `${String(total)} 件の出力、${String(passed)} 件が通り、${String(total - passed)} 件が落ちた`,
+    failedHeading: "落ちた出力",
+    regressionsHeading: (count) => `前の回と比べた回帰 ${String(count)} 件`,
+    noRegression: "前の回と比べた回帰はありません",
+    stamp: "再現の印",
+    separator: "、",
+  },
+  en: {
+    heading: (compared, variants) =>
+      `${String(variants)} variants side by side: ${String(compared)} ${compared === 1 ? "output" : "outputs"} with an id every variant has`,
+    passed: "Passed",
+    passedCell: (passed, outputs, rate) => `${String(passed)}/${String(outputs)} (${rate})`,
+    factsDropped: "Facts dropped",
+    factsAdded: "Facts added",
+    citationsFailed: "Quotations failed",
+    citationsCell: (failed, checked) => `${String(failed)}/${String(checked)}`,
+    unsupportedFacts: "Unsupported facts",
+    penalty: "Penalty points",
+    ratesHeading: (unit) => `Rule rates (per 1,000 ${unit})`,
+    unit: { char: "characters", word: "words" },
+    disagreeHeading: (count) => `${String(count)} ${count === 1 ? "output" : "outputs"} where pass or fail differs`,
+    disagreement: (id, passedIn, failedIn) => `${id}: passed in ${passedIn}; failed in ${failedIn}`,
+    failedIn: (variant, reasons) => `${variant} (${reasons})`,
+    noDisagreement: "Every variant passed or failed the same outputs",
+    missing: (id, variants) => `${id}: missing from ${variants}`,
+    readOtherwise: (ids) => `another language or genre: ${ids}`,
+    notComparedHeading: "Ids not compared",
+    runHeading: (path) => `chaff grade: ${path}`,
+    totals: (total, passed) => `${String(total)} outputs, ${String(passed)} passed, ${String(total - passed)} failed`,
+    failedHeading: "Failed outputs",
+    regressionsHeading: (count) => `${String(count)} ${count === 1 ? "regression" : "regressions"} against the baseline`,
+    noRegression: "No regression against the baseline",
+    stamp: "Stamp",
+    separator: "; ",
+  },
+};

package/src/grade/variants.ts

@@ -0,0 +1,118 @@
+import type { LengthUnit } from "../plugin.ts";
+import { rateRows, sameReading } from "./pairing.ts";
+import type { GradeResult } from "./result.ts";
+import { summaryOf } from "./summary.ts";
+import { variantGroupsOf, type VariantGroups } from "./variants-input.ts";
+
+// Several variants (prompts, models, settings) graded on the same inputs, side by side (spec §29.5). Pure. Outputs are
+// matched by id: only an id every variant answered, read the same way, is compared, so each column covers the same tasks.
+
+/** One variant over the compared ids. */
+export type VariantColumn = {
+  readonly variant: string;
+  readonly outputs: number;
+  readonly passed: number;
+  /** The share of outputs that passed, in percent to one decimal. Undefined when no id was compared. */
+  readonly passRate?: number | undefined;
+  /** Facts dropped and added against the reference, not counting the kinds the rubric allows. */
+  readonly facts: { readonly dropped: number; readonly added: number };
+  readonly citations: { readonly checked: number; readonly failed: number };
+  /** Facts checked against the retrieved passages, and those in no passage. Only when some output had contexts. */
+  readonly contexts?: { readonly checked: number; readonly unsupported: number } | undefined;
+  /** Only with a `grade:` rubric. */
+  readonly penalty?: number | undefined;
+};
+
+/** One rule's rate per 1,000 units in each variant, by variant label; 0 where the variant had no finding of it. */
+export type VariantRates = { readonly rule: string; readonly unit: LengthUnit; readonly rates: Readonly<Record<string, number>> };
+
+/** An id some variants passed and others failed, with why each failed. */
+export type Disagreement = {
+  readonly id: string;
+  readonly passedIn: readonly string[];
+  readonly failedIn: readonly { readonly variant: string; readonly failedBecause: readonly string[] }[];
+};
+
+export type VariantComparison = {
+  /** In the order they first appear. */
+  readonly variants: readonly string[];
+  /** The ids every variant answered, read the same way. */
+  readonly compared: readonly string[];
+  /** Ids some variant did not answer, and which. */
+  readonly missing: readonly { readonly id: string; readonly missingFrom: readonly string[] }[];
+  /** Ids every variant answered, but in another language or genre: not the same task graded the same way. */
+  readonly readOtherwise: readonly string[];
+  readonly columns: readonly VariantColumn[];
+  readonly rules: readonly VariantRates[];
+  readonly disagreements: readonly Disagreement[];
+};
+
+const PERCENT = 100;
+const PERCENT_SCALE = 10;
+
+const percentOf = (passed: number, outputs: number): number | undefined =>
+  outputs === 0 ? undefined : Math.round((passed * PERCENT * PERCENT_SCALE) / outputs) / PERCENT_SCALE;
+
+const sum = (counts: Readonly<Record<string, number>>): number => Object.values(counts).reduce((total, count) => total + count, 0);
+
+const columnOf = (variant: string, results: readonly GradeResult[]): VariantColumn => {
+  const summary = summaryOf(results);
+  return {
+    variant,
+    outputs: summary.total,
+    passed: summary.passed,
+    passRate: percentOf(summary.passed, summary.total),
+    facts: { dropped: sum(summary.facts.dropped), added: sum(summary.facts.added) },
+    citations: summary.citations,
+    ...(summary.contexts === undefined ? {} : { contexts: { checked: summary.contexts.checked, unsupported: sum(summary.contexts.unsupported) } }),
+    ...(summary.penalty === undefined ? {} : { penalty: summary.penalty }),
+  };
+};
+
+const disagreementOf = (id: string, row: readonly GradeResult[], variants: readonly string[]): Disagreement[] => {
+  const passedIn = variants.filter((_, index) => row[index]?.pass === true);
+  if (passedIn.length === 0 || passedIn.length === variants.length) return [];
+  const failedIn = variants.flatMap((variant, index) => {
+    const result = row[index];
+    return result === undefined || result.pass ? [] : [{ variant, failedBecause: result.failedBecause }];
+  });
+  return [{ id, passedIn, failedIn }];
+};
+
+/** Each id in order of first appearance, with its result in each variant, or undefined where the variant has none. */
+const rowsOf = (grouped: VariantGroups, variants: readonly string[]): { readonly id: string; readonly row: readonly (GradeResult | undefined)[] }[] => {
+  const ids = [...new Set([...grouped.values()].flatMap((byId) => [...byId.keys()]))];
+  return ids.map((id) => ({ id, row: variants.map((variant) => grouped.get(variant)?.get(id)) }));
+};
+
+const isComplete = (row: readonly (GradeResult | undefined)[]): row is readonly GradeResult[] => row.every((result) => result !== undefined);
+
+/** The variants side by side over the ids they all answered. `grouped`: each variant's results by id, the variants in order. */
+export const compareVariantGroups = (grouped: VariantGroups): VariantComparison => {
+  const variants = [...grouped.keys()];
+  const rows = rowsOf(grouped, variants);
+  const complete = rows.flatMap(({ id, row }) => (isComplete(row) ? [{ id, row }] : []));
+  const compared = complete.filter(({ row }) => sameReading(row));
+  const byVariant = variants.map((_, index) => compared.flatMap(({ row }) => row.slice(index, index + 1)));
+  return {
+    variants,
+    compared: compared.map(({ id }) => id),
+    missing: rows.flatMap(({ id, row }) => {
+      const missingFrom = variants.filter((_, index) => row[index] === undefined);
+      return missingFrom.length === 0 ? [] : [{ id, missingFrom }];
+    }),
+    readOtherwise: complete.filter(({ row }) => !sameReading(row)).map(({ id }) => id),
+    columns: variants.map((variant, index) => columnOf(variant, byVariant[index] ?? [])),
+    rules: rateRows(byVariant)
+      .filter((row) => row.rates.some((rate) => rate > 0))
+      .map(({ rule, unit, rates }) => ({ rule, unit, rates: Object.fromEntries(variants.map((variant, index) => [variant, rates[index] ?? 0])) })),
+    disagreements: compared.flatMap(({ id, row }) => disagreementOf(id, row, variants)),
+  };
+};
+
+/** A run's results compared variant by variant, or undefined when none has a variant label (the command line's input already checked). */
+export const variantsOfRun = (results: readonly GradeResult[]): VariantComparison | undefined => {
+  if (!results.some((result) => result.variant !== undefined)) return undefined;
+  const read = variantGroupsOf(results);
+  return "groups" in read ? compareVariantGroups(read.groups) : undefined;
+};

package/src/grade/verdict.ts

@@ -0,0 +1,27 @@
+import type { GradeFact, GradeResult } from "./result.ts";
+
+/** What pass or fail is decided from. */
+export type Graded = Pick<GradeResult, "findings" | "facts" | "citations" | "contexts">;
+
+export type Verdict = { readonly pass: boolean; readonly failedBecause: readonly string[] };
+
+/** A condition over a count, written as the reason when the count goes past its limit: `facts.dropped 3 > 0`. */
+export const overLimit = (name: string, count: number, limit: number): string[] => (count > limit ? [`${name} ${String(count)} > ${String(limit)}`] : []);
+
+/** Facts that count against the output: those whose kind was not allowed to change. */
+export const counted = (facts: readonly GradeFact[]): number => facts.filter((fact) => !fact.allowed).length;
+
+export const verdictOf = (failedBecause: readonly string[]): Verdict => ({ pass: failedBecause.length === 0, failedBecause });
+
+/**
+ * The pass or fail with no `grade:` in chaff.yaml (spec §29.3): an error finding, a fact dropped or added against the
+ * reference, a quotation not found, or a fact no retrieved passage states fails the output. Warnings and info are rates to compare, not a verdict on one output.
+ */
+export const defaultVerdict = (graded: Graded): Verdict =>
+  verdictOf([
+    ...overLimit("findings.error", graded.findings.filter((finding) => finding.level === "error").length, 0),
+    ...overLimit("facts.dropped", counted(graded.facts?.dropped ?? []), 0),
+    ...overLimit("facts.added", counted(graded.facts?.added ?? []), 0),
+    ...overLimit("citations.failed", graded.citations?.failed.length ?? 0, 0),
+    ...overLimit("contexts.unsupported", counted(graded.contexts?.unsupported ?? []), 0),
+  ]);

package/src/grade-api.ts

@@ -0,0 +1,184 @@
+// `chaffjs/grade`: chaff's grader for an in-process eval harness (spec §29.6). One output at a time, with the same
+// result as one line of `chaff grade --out`: both go through gradeItem with the same settings and stamp. A separate entry
+// from `chaffjs/api`, so the plugin API's version and the result's shape are not bound to one number.
+import { statSync } from "node:fs";
+import { resolve } from "node:path";
+import { packageFor } from "./adapter-load.ts";
+import { CLI_TEXT } from "./cli-text.ts";
+import { EMPTY, loadConfig, type Config } from "./config/load.ts";
+import { withStyle } from "./config/style.ts";
+import { withExtensions } from "./extension/load.ts";
+import { GENRES } from "./genre.ts";
+import { gradeItem, type GradeSetup } from "./grade/grade-item.ts";
+import { readItem } from "./grade/item.ts";
+import type { GradeResult } from "./grade/result.ts";
+import { parseRubric, type Rubric } from "./grade/rubric.ts";
+import { gradeSetup } from "./grade/setup.ts";
+import { GRADE_TEXT } from "./grade/text.ts";
+import { compareVariantGroups, type VariantComparison } from "./grade/variants.ts";
+import { variantGroupsOf, type VariantInput } from "./grade/variants-input.ts";
+import { settingProblems } from "./setting-problems.ts";
+import { loadStyles } from "./style-load.ts";
+
+export type {
+  FailedCitation,
+  GradeCitations,
+  GradeContexts,
+  GradeFact,
+  GradeFacts,
+  GradeFinding,
+  GradeResult,
+  GradeScore,
+  NotRunEntry,
+  ScoreItem,
+  Stamp,
+  SupportedFact,
+} from "./grade/result.ts";
+export type { OutputSize } from "./grade/rates.ts";
+export { toScorer, type ChaffScore } from "./grade/scorer.ts";
+export type { Disagreement, VariantColumn, VariantComparison, VariantRates } from "./grade/variants.ts";
+export type { VariantInput } from "./grade/variants-input.ts";
+
+export type GradeOptions = {
+  /** The result's id. "output" when left out. */
+  readonly id?: string;
+  /** The document the output was made from: facts are checked against it, as `chaff compare` does. */
+  readonly reference?: string;
+  /** Each source's name and text. */
+  readonly sources?: Readonly<Record<string, string>>;
+  /** What the output quoted. `source` may be left out when there is one source. */
+  readonly citations?: readonly { readonly source?: string; readonly address: string; readonly quote: string }[];
+  /** The retrieved passages the output was meant to rest on: each fact of the output is looked for in them. */
+  readonly contexts?: readonly string[];
+  readonly language?: string;
+  readonly genre?: string;
+  /** The prompt, model or setting that produced the output, kept on the result for compareVariants(). */
+  readonly variant?: string;
+  /** Run the experimental rules too, as --experimental does. chaff.yaml's `experimental` when left out. */
+  readonly experimental?: boolean;
+  /** chaff.yaml's path, or settings already read. Without it chaff's defaults apply and no file is read. */
+  readonly config?: string | Config;
+};
+
+/** An input or a chaff.yaml grade() cannot use: the same cases end `chaff grade` with exit 2. */
+export class GradeInputError extends Error {
+  readonly problems: readonly string[];
+
+  constructor(problems: readonly string[]) {
+    super(problems.join("\n"));
+    this.name = "GradeInputError";
+    this.problems = problems;
+  }
+}
+
+const TEXT = GRADE_TEXT.en;
+
+/** Each chaff.yaml read, by its path and when it was last changed: one file is read and its code loaded once. */
+const readConfigs = new Map<string, Config>();
+
+const KEPT = 16;
+
+/** Adds to a cache, dropping the oldest entry past KEPT. */
+const remember = <T>(cache: Map<string, T>, key: string, value: T): T => {
+  cache.set(key, value);
+  const [oldest] = cache.keys();
+  if (cache.size > KEPT && oldest !== undefined) cache.delete(oldest);
+  return value;
+};
+
+/** chaff.yaml read as the command line reads it: its style applied and the code it names loaded. */
+const readConfig = async (path: string): Promise<Config> => {
+  const file = resolve(path);
+  const key = `${file}\n${String(statSync(file).mtimeMs)}`;
+  return readConfigs.get(key) ?? remember(readConfigs, key, await withExtensions(withStyle(loadConfig(file), loadStyles())));
+};
+
+/** A chaff.yaml that is missing or not YAML is an input grade() cannot use, as it is for the command line. */
+const readNamedConfig = async (path: string): Promise<Config> => {
+  try {
+    return await readConfig(path);
+  } catch (error) {
+    throw new GradeInputError([TEXT.unreadable(path, error instanceof Error ? error.message : String(error))]);
+  }
+};
+
+/** The settings to grade with, checked as the command line checks them, whether read here or given already read. */
+const configOf = async (given: string | Config | undefined): Promise<Config> => {
+  if (given === undefined) return EMPTY;
+  const config = typeof given === "string" ? await readNamedConfig(given) : given;
+  const problems = settingProblems("grade", undefined, config, CLI_TEXT.en);
+  if (problems.length > 0) throw new GradeInputError(problems);
+  return config;
+};
+
+const rubricOf = (config: Config): Rubric | undefined => {
+  const parsed = parseRubric(config.grade);
+  if ("problems" in parsed) throw new GradeInputError(parsed.problems.map((problem) => TEXT.rubricProblem(problem)));
+  return parsed.rubric;
+};
+
+/**
+ * Setups for each settings object, by what else decides them, so a harness grading many outputs loads the language
+ * packages and hashes the rules once. Keyed by the object itself: two chaff.yaml files that read alike may load other code.
+ */
+const setups = new WeakMap<Config, Map<string, Promise<GradeSetup>>>();
+
+const setupFor = (config: Config, experimental: boolean, languages: readonly string[]): Promise<GradeSetup> => {
+  const run = { experimental, genre: undefined };
+  const own = setups.get(config) ?? new Map<string, Promise<GradeSetup>>();
+  setups.set(config, own);
+  const key = `${String(experimental)}\n${languages.join(",")}`;
+  const known = own.get(key);
+  if (known !== undefined) return known;
+  const made = remember(own, key, gradeSetup(config, run, languages, rubricOf(config)));
+  // A language package that failed to load may be installed before the next call: do not keep the failure.
+  made.catch(() => own.delete(key));
+  return made;
+};
+
+/** The setup, or GradeInputError when a language package cannot be loaded, as `chaff grade` ends with 2 then. */
+const loadedSetup = async (config: Config, experimental: boolean, languages: readonly string[]): Promise<GradeSetup> => {
+  try {
+    return await setupFor(config, experimental, languages);
+  } catch (error) {
+    if (error instanceof GradeInputError) throw error;
+    throw new GradeInputError([TEXT.cannotLoad(error instanceof Error ? error.message : String(error))]);
+  }
+};
+
+/**
+ * Grades one output (spec §29.6): its findings and rates, its facts against `reference`, its `citations` against
+ * `sources`, and pass or fail by chaff.yaml's `grade:` or the default. Reads a file only when `config` is a path, and
+ * never writes one. Throws GradeInputError for an input or a chaff.yaml `chaff grade` would refuse with exit 2.
+ */
+export const grade = async (output: string, options: GradeOptions = {}): Promise<GradeResult> => {
+  const config = await configOf(options.config);
+  const raw = {
+    id: options.id ?? "output",
+    output,
+    reference: options.reference,
+    sources: options.sources,
+    citations: options.citations,
+    contexts: options.contexts,
+    language: options.language,
+    genre: options.genre,
+    variant: options.variant,
+  };
+  const read = readItem(raw, { isLanguage: (language) => packageFor(language) !== undefined, genres: GENRES });
+  if ("problem" in read) throw new GradeInputError([TEXT.problem(read.problem)]);
+  const languages = read.item.language === undefined ? [] : [read.item.language];
+  return gradeItem(read.item, await loadedSetup(config, options.experimental ?? config.experimental, languages));
+};
+
+/**
+ * Several variants' results side by side on the same inputs (spec §29.5), as `chaff grade` prints them for a file whose
+ * lines carry a variant: per variant the pass rate, each rule's rate, facts dropped and added, and failed quotations,
+ * over the ids every variant answered; and the ids where pass or fail differs. Takes results labelled by `variant`
+ * (grade() with `variant`, or `chaff grade --out`), or `{ [variant]: results }`. Throws GradeInputError for a result
+ * without an id or a variant, or an id given twice in one variant.
+ */
+export const compareVariants = (results: VariantInput): VariantComparison => {
+  const read = variantGroupsOf(results);
+  if ("problems" in read) throw new GradeInputError(read.problems);
+  return compareVariantGroups(read.groups);
+};

package/src/heading-numbering.ts

@@ -0,0 +1,106 @@
+// 見出しの番号の付け方。純粋な関数だけを置く。どの語が番号を持つか（第1章、Chapter 1）と、番号を付けない前後の見出し
+// （目次、参考文献）は言語パッケージの語彙表が言う。
+
+/** 番号の無い見出しの書き方。ほかの書き方は "dot"（1.）や "label:章"（第1章）のように、番号を持つ語ごとに分ける。 */
+export const UNNUMBERED = "none";
+
+/** 番号を持つ語。before は語が番号の前（Chapter 1、ステップ1）、after は番号の後ろ（第1章、1章）。 */
+export type NumberLabel = { readonly word: string; readonly position: "before" | "after" };
+
+const ARABIC = "[0-9０-９]+";
+const KANJI = "[一二三四五六七八九十百]+";
+const ROMAN = "[IVXⅠ-Ⅻ]+";
+
+/** 番号の後ろに続く見出しの言葉の頭。字で始まる。「8.2.3 (2026-08-06)」の版や「3 reasons」の数量は番号と読まない。 */
+const WORDS_FOLLOW = "(?=\\s*[\\p{L}])";
+
+/**
+ * 字だけで決まる書き方。上から順に試す。ラテン文字一字とローマ数字（A. と II.）は同じ書き方とする。どちらも番号の後ろに言葉が続くときだけ。
+ * 区切りの記号の無い数（「1 Introduction」）は数量（「10 reasons」）と見分けられないので読まない。
+ */
+const SHAPES: ReadonlyArray<readonly [string, RegExp]> = [
+  ["paren", /^[(（][0-9０-９]+[)）]/u],
+  ["close-paren", /^[0-9０-９]+[)）]/u],
+  ["dot", new RegExp(`^${ARABIC}[a-z]?(?:[.．]${ARABIC})*[.．](?![0-9０-９])${WORDS_FOLLOW}`, "u")],
+  ["dotted", new RegExp(`^${ARABIC}(?:[.．]${ARABIC})+\\s${WORDS_FOLLOW}`, "u")],
+  ["circled", /^[①-⑳]/u],
+  ["kanji", /^[一二三四五六七八九十]+[、.．]/u],
+  ["letter", /^(?:[IVX]+|[A-Z])[.)]\s+(?=\p{L})/u],
+];
+
+const escaped = (word: string): string => word.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&");
+
+/** 番号を持つ語は書いたとおりか、全部大文字で照らす（Chapter と CHAPTER。小文字の section は本文の語）。 */
+const casings = (word: string): string => [...new Set([word, word.toUpperCase()])].map(escaped).join("|");
+
+const labelPattern = (label: NumberLabel): RegExp =>
+  label.position === "before"
+    ? new RegExp(`^(?:${casings(label.word)})(?:\\s*(?:${ARABIC}|${KANJI}|${ROMAN})|\\s+[A-Z])(?![A-Za-z])`, "u")
+    : new RegExp(`^(?:第(?:${ARABIC}|${KANJI}|${ROMAN})|${ARABIC})${escaped(label.word)}`, "u");
+
+/** 見出しの頭の強調の印と空白。「**1. 概要**」も「1. 概要」と読む。 */
+const LEAD = /^[\s*_]+/u;
+
+/** 見出しの言葉から、番号の書き方を読む。番号を持つ語（第1章）を先に、字だけの形（1.）を後に見る。 */
+export const numberingStyleOf = (heading: string, labels: readonly NumberLabel[]): string => {
+  const text = heading.replace(LEAD, "");
+  const label = labels.find((candidate) => labelPattern(candidate).test(text));
+  if (label !== undefined) return `label:${label.word.toLowerCase()}`;
+  return SHAPES.find(([, shape]) => shape.test(text))?.[0] ?? UNNUMBERED;
+};
+
+/** 見出し 1 つ。depth は深さ（#の数）、style は番号の書き方、skipped は番号を付けない前後の見出し（目次・参考文献）。 */
+export type NumberedHeading = { readonly depth: number; readonly style: string; readonly skipped: boolean };
+
+/** 親（直前の浅い見出し）と深さが同じ見出しの組。番号は兄弟の見出しのあいだで揃える。位置は渡した並びの添え字。 */
+export const siblingGroups = (headings: readonly NumberedHeading[]): number[][] => {
+  const parents = headings.map((heading, at) => headings.slice(0, at).findLastIndex((before) => before.depth < heading.depth));
+  const groups = new Map<string, number[]>();
+  headings.forEach((heading, at) => {
+    const key = `${String(parents[at])}:${String(heading.depth)}`;
+    groups.set(key, [...(groups.get(key) ?? []), at]);
+  });
+  return [...groups.values()];
+};
+
+/** 指す見出しと、何に対して少ないか。kind は numbered（番号の有無）か style（番号の書き方）。majority は多いほうの書き方で、count はその数。 */
+export type NumberingMinority = {
+  readonly at: number;
+  readonly kind: "numbered" | "style";
+  readonly majority: string;
+  readonly count: number;
+  /** 多いほうの書き方の、最初の兄弟の位置。指摘が例に引く。 */
+  readonly example: number;
+};
+
+type Split = { readonly minority: readonly number[]; readonly majority: string; readonly count: number; readonly example: number };
+
+const minorityBy = (members: readonly number[], keyOf: (at: number) => string): Split | undefined => {
+  const counts = new Map<string, number>();
+  members.forEach((at) => counts.set(keyOf(at), (counts.get(keyOf(at)) ?? 0) + 1));
+  const [top, second] = [...counts].toSorted((left, right) => right[1] - left[1]);
+  if (top === undefined || second === undefined || top[1] === second[1]) return undefined;
+  const example = members.find((at) => keyOf(at) === top[0]) ?? -1;
+  return { minority: members.filter((at) => keyOf(at) !== top[0]), majority: top[0], count: top[1], example };
+};
+
+const minoritiesOf = (split: Split | undefined, kind: NumberingMinority["kind"]): NumberingMinority[] =>
+  split === undefined ? [] : split.minority.map((at) => ({ at, kind, majority: split.majority, count: split.count, example: split.example }));
+
+/**
+ * 兄弟の見出しのうち、少ないほうの書き方のもの。先に番号の有無を比べ（「1. 概要」「2. 手順」「まとめ」の「まとめ」）、
+ * 番号の付いた見出しのあいだで書き方を比べる（「1.」「2.」「3）」の「3）」）。同じ数なら言わない（二つだけの兄弟は必ず同じ数か一つの書き方）。前後の見出し（目次・参考文献）は数えない。
+ */
+export const numberingMinorities = (headings: readonly NumberedHeading[]): NumberingMinority[] =>
+  siblingGroups(headings).flatMap((group) => {
+    const members = group.filter((at) => headings[at]?.skipped !== true);
+    const styleOf = (at: number): string => headings[at]?.style ?? UNNUMBERED;
+    const numbered = members.filter((at) => styleOf(at) !== UNNUMBERED);
+    return [
+      ...minoritiesOf(
+        minorityBy(members, (at) => (styleOf(at) === UNNUMBERED ? UNNUMBERED : "numbered")),
+        "numbered",
+      ),
+      ...minoritiesOf(minorityBy(numbered, styleOf), "style"),
+    ];
+  });

package/src/kana-spelling.ts

@@ -0,0 +1,49 @@
+// Keys that make two Japanese spellings of one word meet: the reading of the dictionary form (出来る and できる are both デキル),
+// the kanji a spelling keeps (引っ越し and 引越し both keep 引越), and a katakana word with its small kana and ー evened out
+// (ウィンドウ and ウインドウ). Pure: the tokens come in from the language adapter.
+
+const HIRAGANA_TO_KATAKANA = 0x60;
+const HIRAGANA = /[ぁ-ゖ]/gu;
+const KANA_ONLY = /^[ぁ-ゖァ-ヺー]+$/u;
+const KANJI = /[\p{Script=Han}々]/gu;
+
+/** Hiragana as katakana, the script the adapter reads in. */
+export const toKatakana = (text: string): string => text.replace(HIRAGANA, (char) => String.fromCodePoint((char.codePointAt(0) ?? 0) + HIRAGANA_TO_KATAKANA));
+
+export const isKanaOnly = (text: string): boolean => KANA_ONLY.test(text);
+
+/** The kanji of a spelling, in order: 引越 for both 引っ越し and 引越し, nothing for できる. */
+export const kanjiSkeleton = (text: string): string => [...text.matchAll(KANJI)].map((match) => match[0]).join("");
+
+/** The length, in UTF-16 units, of the start both strings share, compared character by character (𠮟 is one character, two units). */
+const sharedPrefixLength = (left: string, right: string): number => {
+  const [leftChars, rightChars] = [[...left], [...right]];
+  const differs = leftChars.findIndex((char, at) => rightChars[at] !== char);
+  const shared = differs === -1 ? leftChars.slice(0, rightChars.length) : leftChars.slice(0, differs);
+  return shared.join("").length;
+};
+
+/**
+ * The reading of the dictionary form, from the reading of the written form: 下さい (クダサイ) with lemma 下さる reads クダサル.
+ * The written form and the lemma differ only in their kana ending, so that ending is swapped in the reading. When the ending is
+ * not kana, or the reading does not end with it (来 read キ for 来る), there is no answer.
+ */
+export const lemmaReading = (surface: string, lemma: string, reading: string): string | undefined => {
+  const shared = sharedPrefixLength(surface, lemma);
+  const [writtenEnd, lemmaEnd] = [surface.slice(shared), lemma.slice(shared)];
+  if ((writtenEnd !== "" && !isKanaOnly(writtenEnd)) || (lemmaEnd !== "" && !isKanaOnly(lemmaEnd))) return undefined;
+  const spoken = toKatakana(writtenEnd);
+  if (!reading.endsWith(spoken)) return undefined;
+  return reading.slice(0, reading.length - spoken.length) + toKatakana(lemmaEnd);
+};
+
+const SMALL_TO_LARGE: Readonly<Record<string, string>> = { ァ: "ア", ィ: "イ", ゥ: "ウ", ェ: "エ", ォ: "オ", ヵ: "カ", ヶ: "ケ", ヮ: "ワ" };
+const VU: Readonly<Record<string, string>> = { ヴァ: "バ", ヴィ: "ビ", ヴェ: "ベ", ヴォ: "ボ", ヴ: "ブ" };
+
+/** A katakana word with ー dropped, small vowels made large, ヴ read as バ行 and エイ as エ: インターフェース and インタフェイス meet. */
+export const katakanaKey = (surface: string): string =>
+  surface
+    .replace(/ヴ[ァィェォ]?/gu, (match) => VU[match] ?? match)
+    .replace(/[ァィゥェォヵヶヮ]/gu, (char) => SMALL_TO_LARGE[char] ?? char)
+    .replace(/ー/gu, "")
+    .replace(/エイ/gu, "エ");