chaffjs 0.18.0 → 0.20.0

package/src/grade/rubric.ts

@@ -0,0 +1,192 @@
+import { isAtomKind, type AtomKind } from "../compare/atom.ts";
+
+// The `grade:` section of chaff.yaml (spec §29.4), read and checked. Pure. A section that cannot be read stops the run
+// (exit 2): a rubric that silently checks less than was written would pass outputs it was meant to fail.
+
+export type RuleLimit = {
+  /** Findings allowed per output. */
+  readonly max?: number | undefined;
+  /** Findings allowed per 1,000 characters or words. */
+  readonly maxRate?: number | undefined;
+  /** Penalty points per finding. */
+  readonly weight?: number | undefined;
+};
+
+type FactLimits = {
+  readonly dropped?: number | undefined;
+  readonly added?: number | undefined;
+  readonly allowDropped: readonly AtomKind[];
+  readonly allowAdded: readonly AtomKind[];
+};
+
+type CitationLimits = { readonly failed?: number | undefined; readonly required: boolean };
+
+type ContextLimits = { readonly unsupported?: number | undefined; readonly allowUnsupported: readonly AtomKind[]; readonly required: boolean };
+
+export type Rubric = {
+  readonly rules: Readonly<Record<string, RuleLimit>>;
+  /** Undefined when not written: chaff.yaml's top-level required_sections apply. */
+  readonly requiredSections?: readonly string[] | undefined;
+  readonly facts?: FactLimits | undefined;
+  readonly citations?: CitationLimits | undefined;
+  readonly contexts?: ContextLimits | undefined;
+  readonly penalty?: number | undefined;
+};
+
+/** What a value under `grade:` should have been. */
+export type Expected = "map" | "count" | "number" | "boolean" | "words" | "kinds" | "known-key";
+
+/** Where under `grade:` (a dotted path), what it should be, and what was written. */
+export type RubricProblem = { readonly path: string; readonly expected: Expected; readonly written: string };
+
+type Read<T> = { readonly value: T; readonly problems: readonly RubricProblem[] };
+
+const isRecord = (value: unknown): value is Record<string, unknown> => typeof value === "object" && value !== null && !Array.isArray(value);
+
+const printed = (value: unknown): string => JSON.stringify(value) ?? String(value);
+
+const problem = (path: string, expected: Expected, written: unknown): RubricProblem => ({ path, expected, written: printed(written) });
+
+const ok = <T>(value: T): Read<T> => ({ value, problems: [] });
+
+const isCount = (value: unknown): value is number => Number.isInteger(value) && typeof value === "number" && value >= 0;
+
+const isAmount = (value: unknown): value is number => typeof value === "number" && Number.isFinite(value) && value >= 0;
+
+/** An optional number under `path`: a count (a whole number) or an amount (any number from 0). */
+const optional = (raw: Record<string, unknown>, key: string, path: string, expected: "count" | "number"): Read<number | undefined> => {
+  const value = raw[key];
+  if (value === undefined) return ok(undefined);
+  const valid = expected === "count" ? isCount : isAmount;
+  return valid(value) ? ok(value) : { value: undefined, problems: [problem(`${path}.${key}`, expected, value)] };
+};
+
+/** Keys that are not among `known`. A misspelt `max_rte` would otherwise be a limit nobody checks. */
+const unknownKeys = (raw: Record<string, unknown>, known: readonly string[], path: string): RubricProblem[] =>
+  Object.keys(raw)
+    .filter((key) => !known.includes(key))
+    .map((key) => problem(`${path}.${key}`, "known-key", raw[key]));
+
+const ruleLimit = (raw: unknown, path: string): Read<RuleLimit> => {
+  if (!isRecord(raw)) return { value: {}, problems: [problem(path, "map", raw)] };
+  const max = optional(raw, "max", path, "count");
+  const maxRate = optional(raw, "max_rate", path, "number");
+  const weight = optional(raw, "weight", path, "number");
+  return {
+    value: { max: max.value, maxRate: maxRate.value, weight: weight.value },
+    problems: [...max.problems, ...maxRate.problems, ...weight.problems, ...unknownKeys(raw, ["max", "max_rate", "weight"], path)],
+  };
+};
+
+const rulesOf = (raw: unknown): Read<Record<string, RuleLimit>> => {
+  if (raw === undefined) return ok({});
+  if (!isRecord(raw)) return { value: {}, problems: [problem("grade.rules", "map", raw)] };
+  const read = Object.entries(raw).map(([id, limit]) => ({ id, read: ruleLimit(limit, `grade.rules.${id}`) }));
+  return { value: Object.fromEntries(read.map((entry) => [entry.id, entry.read.value])), problems: read.flatMap((entry) => entry.read.problems) };
+};
+
+const wordsOf = (raw: unknown, path: string): Read<string[] | undefined> => {
+  if (raw === undefined) return ok(undefined);
+  const valid = Array.isArray(raw) && raw.every((entry) => typeof entry === "string" && entry.trim() !== "");
+  return valid ? ok(raw.map((entry: string) => entry.trim())) : { value: undefined, problems: [problem(path, "words", raw)] };
+};
+
+const kindsOf = (raw: unknown, path: string): Read<AtomKind[]> => {
+  if (raw === undefined) return ok([]);
+  const entries: readonly unknown[] = Array.isArray(raw) ? raw : [raw];
+  const kinds = entries.filter((entry): entry is AtomKind => typeof entry === "string" && isAtomKind(entry));
+  return kinds.length === entries.length ? ok(kinds) : { value: kinds, problems: [problem(path, "kinds", raw)] };
+};
+
+const factsOf = (raw: unknown): Read<FactLimits | undefined> => {
+  if (raw === undefined) return ok(undefined);
+  if (!isRecord(raw)) return { value: undefined, problems: [problem("grade.facts", "map", raw)] };
+  const dropped = optional(raw, "dropped", "grade.facts", "count");
+  const added = optional(raw, "added", "grade.facts", "count");
+  const allowDropped = kindsOf(raw["allow_dropped"], "grade.facts.allow_dropped");
+  const allowAdded = kindsOf(raw["allow_added"], "grade.facts.allow_added");
+  return {
+    value: { dropped: dropped.value, added: added.value, allowDropped: allowDropped.value, allowAdded: allowAdded.value },
+    problems: [
+      ...dropped.problems,
+      ...added.problems,
+      ...allowDropped.problems,
+      ...allowAdded.problems,
+      ...unknownKeys(raw, ["dropped", "added", "allow_dropped", "allow_added"], "grade.facts"),
+    ],
+  };
+};
+
+/** `required: true|false` under `path`: false when not written. */
+const requiredOf = (raw: Record<string, unknown>, path: string): Read<boolean> => {
+  const required = raw["required"];
+  return required === undefined || typeof required === "boolean"
+    ? ok(required === true)
+    : { value: false, problems: [problem(`${path}.required`, "boolean", required)] };
+};
+
+const citationsOf = (raw: unknown): Read<CitationLimits | undefined> => {
+  if (raw === undefined) return ok(undefined);
+  if (!isRecord(raw)) return { value: undefined, problems: [problem("grade.citations", "map", raw)] };
+  const failed = optional(raw, "failed", "grade.citations", "count");
+  const required = requiredOf(raw, "grade.citations");
+  return {
+    value: { failed: failed.value, required: required.value },
+    problems: [...failed.problems, ...required.problems, ...unknownKeys(raw, ["failed", "required"], "grade.citations")],
+  };
+};
+
+const contextsOf = (raw: unknown): Read<ContextLimits | undefined> => {
+  if (raw === undefined) return ok(undefined);
+  if (!isRecord(raw)) return { value: undefined, problems: [problem("grade.contexts", "map", raw)] };
+  const unsupported = optional(raw, "unsupported", "grade.contexts", "count");
+  const allowUnsupported = kindsOf(raw["allow_unsupported"], "grade.contexts.allow_unsupported");
+  const required = requiredOf(raw, "grade.contexts");
+  return {
+    value: { unsupported: unsupported.value, allowUnsupported: allowUnsupported.value, required: required.value },
+    problems: [
+      ...unsupported.problems,
+      ...allowUnsupported.problems,
+      ...required.problems,
+      ...unknownKeys(raw, ["unsupported", "allow_unsupported", "required"], "grade.contexts"),
+    ],
+  };
+};
+
+const RUBRIC_KEYS: readonly string[] = ["rules", "required_sections", "facts", "citations", "contexts", "penalty"];
+
+/**
+ * The rubric, or every problem in it. No `grade:` key is no rubric: the default pass or fail applies (spec §29.3). A bare
+ * `grade:` (null) is an empty rubric, so writing the key alone still turns the default off, as the spec says it does.
+ */
+export const parseRubric = (raw: unknown): { readonly rubric: Rubric | undefined } | { readonly problems: readonly RubricProblem[] } => {
+  if (raw === undefined) return { rubric: undefined };
+  if (raw === null) return { rubric: { rules: {} } };
+  if (!isRecord(raw)) return { problems: [problem("grade", "map", raw)] };
+  const rules = rulesOf(raw["rules"]);
+  const sections = wordsOf(raw["required_sections"], "grade.required_sections");
+  const facts = factsOf(raw["facts"]);
+  const citations = citationsOf(raw["citations"]);
+  const contexts = contextsOf(raw["contexts"]);
+  const penalty = optional(raw, "penalty", "grade", "number");
+  const problems = [
+    ...rules.problems,
+    ...sections.problems,
+    ...facts.problems,
+    ...citations.problems,
+    ...contexts.problems,
+    ...penalty.problems,
+    ...unknownKeys(raw, RUBRIC_KEYS, "grade"),
+  ];
+  if (problems.length > 0) return { problems };
+  return {
+    rubric: {
+      rules: rules.value,
+      requiredSections: sections.value,
+      facts: facts.value,
+      citations: citations.value,
+      contexts: contexts.value,
+      penalty: penalty.value,
+    },
+  };
+};

package/src/grade/scorer.ts

@@ -0,0 +1,33 @@
+import { tally } from "./order.ts";
+import type { GradeResult } from "./result.ts";
+
+// The common scorer shape of eval frameworks (promptfoo, autoevals, evalite, DeepEval, Inspect AI…) as a projection of
+// one result: a score, pass, a reason a person can read, and the whole breakdown as metadata. Spec §29.6.
+
+export type ChaffScore = {
+  readonly name: "chaff";
+  /** 1 when the output passed, 0 when it failed. Never the penalty mapped onto 0–1: chaff sets no full marks (§29.4). */
+  readonly score: 0 | 1;
+  readonly pass: boolean;
+  readonly reason: string;
+  readonly metadata: Omit<GradeResult, "pass">;
+};
+
+const findingsWord = (count: number): string => (count === 1 ? "finding" : "findings");
+
+/** The findings in one phrase: "3 findings: closing-cliche ×2, ai-tell ×1". */
+const findingsPhrase = (result: GradeResult): string => {
+  const counts = Object.entries(tally(result.findings.map((finding) => finding.rule))).map(([rule, count]) => `${rule} ×${String(count)}`);
+  return counts.length === 0 ? "no findings" : `${String(result.findings.length)} ${findingsWord(result.findings.length)}: ${counts.join(", ")}`;
+};
+
+const penaltyPhrase = (result: GradeResult): string[] => (result.score === undefined ? [] : [`penalty ${String(result.score.penalty)}`]);
+
+/** Why it passed or failed, then what was found: the failed conditions first, since they decide the score. */
+export const reasonOf = (result: GradeResult): string =>
+  [result.pass ? "passed" : `failed: ${result.failedBecause.join("; ")}`, findingsPhrase(result), ...penaltyPhrase(result)].join(" — ");
+
+export const toScorer = (result: GradeResult): ChaffScore => {
+  const { pass, ...rest } = result;
+  return { name: "chaff", score: pass ? 1 : 0, pass, reason: reasonOf(result), metadata: rest };
+};

package/src/grade/setup.ts

@@ -0,0 +1,43 @@
+import { loadAdapter } from "../adapter-load.ts";
+import type { Config } from "../config/load.ts";
+import { rulesOf } from "../custom/load.ts";
+import { loadGenres } from "../genre-load.ts";
+import { loadProfiles } from "../profile/load.ts";
+import { VERSION_LINES } from "../version.ts";
+import type { GradeSetup } from "./grade-item.ts";
+import { compareText } from "./order.ts";
+import type { Rubric } from "./rubric.ts";
+import { stampOf, type RuleSet } from "./stamp.ts";
+
+/** The languages every stamp covers, whatever a run's outputs are written in, so two runs over different outputs stay comparable. */
+const BUNDLED_LANGUAGES: readonly string[] = ["en", "ja"];
+
+/** The rules each language runs and the word lists they read, with the genres' presets. Loads each language package. */
+const ruleSetOf = async (config: Config, languages: readonly string[]): Promise<RuleSet> => {
+  const read = await Promise.all(
+    languages.map(async (language) => {
+      const adapter = await loadAdapter(language);
+      return { language, rules: rulesOf(language, config), lexicons: { ...adapter.lexicons, ...config.extensions?.lexicons[language] } };
+    }),
+  );
+  return {
+    rules: Object.fromEntries(read.map((entry) => [entry.language, entry.rules])),
+    lexicons: Object.fromEntries(read.map((entry) => [entry.language, entry.lexicons])),
+    genres: loadGenres(),
+    profiles: loadProfiles(),
+  };
+};
+
+/** A run's settings, rubric and stamp. `languages`: those the items name, beyond the bundled ones. Throws when a language package cannot be loaded. */
+export const gradeSetup = async (
+  config: Config,
+  run: { readonly experimental: boolean; readonly genre: string | undefined },
+  languages: readonly string[],
+  rubric: Rubric | undefined,
+): Promise<GradeSetup> => {
+  // chaff.yaml's language and its by_path languages are loaded here too, so a missing package stops the run before grading.
+  const named = [config.language, ...config.byPath.map((rule) => rule.language)].filter((language) => language !== undefined);
+  const all = [...new Set([...BUNDLED_LANGUAGES, ...named, ...languages])].toSorted(compareText);
+  const settings = { config, experimental: run.experimental, genre: run.genre };
+  return { ...settings, rubric, stamp: stampOf(VERSION_LINES, await ruleSetOf(config, all), settings) };
+};

package/src/grade/stamp.ts

@@ -0,0 +1,60 @@
+import { createHash } from "node:crypto";
+import type { Config } from "../config/load.ts";
+import { sortedByKey } from "./order.ts";
+import type { Stamp } from "./result.ts";
+
+// The reproducibility stamp (spec §29.7). Two results are compared only when their rules and settings hashes match;
+// the chaff version is recorded but not compared, so a release that only rewords messages keeps old results usable.
+
+const isRecord = (value: unknown): value is Record<string, unknown> => typeof value === "object" && value !== null && !Array.isArray(value);
+
+/** JSON with every object's keys sorted, so the order a file or a map was written in does not change the hash. */
+export const canonicalJson = (value: unknown): string =>
+  JSON.stringify(value, (_key, inner: unknown) => (isRecord(inner) ? sortedByKey(Object.entries(inner)) : inner)) ?? "null";
+
+export const digestOf = (value: unknown): string => `sha256:${createHash("sha256").update(canonicalJson(value)).digest("hex")}`;
+
+/** What decides which findings a run gives, other than the rules: chaff.yaml as it applies, the genre and --experimental. */
+export type GradeSettings = {
+  readonly config: Config;
+  readonly experimental: boolean;
+  /** --genre, which wins over chaff.yaml for every item that does not name its own. */
+  readonly genre: string | undefined;
+};
+
+/** The parts of chaff.yaml that change a finding, a fact or a verdict. Paths and the AI judge do not. */
+export const settingsOf = (settings: GradeSettings): Record<string, unknown> => {
+  const { config } = settings;
+  return {
+    experimental: settings.experimental,
+    genre: settings.genre ?? config.genre ?? null,
+    language: config.language ?? null,
+    profile: config.profile ?? null,
+    rules: config.rules,
+    limits: config.limits,
+    options: config.options ?? {},
+    style: config.style ?? null,
+    styleApplied: config.applied ?? null,
+    jargon: config.jargon,
+    prefer: config.prefer,
+    requiredSections: config.requiredSections,
+    names: config.names,
+    byPath: config.byPath,
+    // A bare `grade:` reads as null and is an empty rubric, which grades otherwise than no `grade:` at all.
+    grade: config.grade === undefined ? "none" : config.grade,
+  };
+};
+
+/** What the rules are: each language's rule definitions and word lists, the genres' presets and the document profiles. */
+export type RuleSet = {
+  readonly rules: Readonly<Record<string, unknown>>;
+  readonly lexicons: Readonly<Record<string, unknown>>;
+  readonly genres: unknown;
+  readonly profiles: unknown;
+};
+
+export const stampOf = (versionLines: readonly string[], ruleSet: RuleSet, settings: GradeSettings): Stamp => ({
+  chaff: versionLines.join(", "),
+  rules: digestOf(ruleSet),
+  settings: digestOf(settingsOf(settings)),
+});

package/src/grade/summary.ts

@@ -0,0 +1,112 @@
+import type { LengthUnit } from "../plugin.ts";
+import { compareText, sortedByKey, tally } from "./order.ts";
+import { rateOf } from "./rates.ts";
+import type { GradeFact, GradeResult, NotRunEntry, Stamp } from "./result.ts";
+
+// The summary of a run (spec §29.3), from the per-output results alone, so the same results always sum the same way.
+
+/** Per length unit: Japanese outputs are measured in characters and English ones in words, and the two never add up. */
+type PerUnit = Partial<Record<LengthUnit, number>>;
+
+type RuleSummary = { readonly findings: number; readonly outputs: number; readonly rate: PerUnit };
+
+export type GradeSummary = {
+  readonly total: number;
+  readonly passed: number;
+  readonly failed: readonly { readonly id: string; readonly variant?: string | undefined; readonly failedBecause: readonly string[] }[];
+  readonly size: PerUnit;
+  readonly rules: Readonly<Record<string, RuleSummary>>;
+  readonly facts: { readonly dropped: Readonly<Record<string, number>>; readonly added: Readonly<Record<string, number>> };
+  readonly citations: { readonly checked: number; readonly failed: number };
+  /** Only when some output had contexts: how many, their facts checked, and the facts no passage states by kind. */
+  readonly contexts?: { readonly outputs: number; readonly checked: number; readonly unsupported: Readonly<Record<string, number>> } | undefined;
+  readonly notRun: readonly (NotRunEntry & { readonly outputs: number })[];
+  /** The penalty points of every output, added up. Only with a `grade:` rubric. */
+  readonly penalty?: number | undefined;
+  readonly stamp: Stamp | undefined;
+};
+
+const sizesOf = (results: readonly GradeResult[]): PerUnit =>
+  results.reduce<PerUnit>((sizes, result) => ({ ...sizes, [result.size.unit]: (sizes[result.size.unit] ?? 0) + result.size.value }), {});
+
+const countIn = (result: GradeResult, rule: string): number => result.findings.filter((finding) => finding.rule === rule).length;
+
+/** A rule's findings over every output of one unit, per 1,000 of that unit. */
+const ratePerUnit = (results: readonly GradeResult[], rule: string, sizes: PerUnit): PerUnit =>
+  Object.fromEntries(
+    Object.entries(sizes).flatMap(([unit, size]) => {
+      const count = results.filter((result) => result.size.unit === unit).reduce((sum, result) => sum + countIn(result, rule), 0);
+      const rate = count === 0 ? undefined : rateOf(count, size);
+      return rate === undefined ? [] : [[unit, rate]];
+    }),
+  );
+
+const rulesOf = (results: readonly GradeResult[], sizes: PerUnit): Record<string, RuleSummary> => {
+  const ruleIds = [...new Set(results.flatMap((result) => result.findings.map((finding) => finding.rule)))];
+  return sortedByKey(
+    ruleIds.map((rule): [string, RuleSummary] => [
+      rule,
+      {
+        findings: results.reduce((sum, result) => sum + countIn(result, rule), 0),
+        outputs: results.filter((result) => countIn(result, rule) > 0).length,
+        rate: ratePerUnit(results, rule, sizes),
+      },
+    ]),
+  );
+};
+
+const kindsOf = (facts: readonly GradeFact[]): Record<string, number> => tally(facts.filter((fact) => !fact.allowed).map((fact) => fact.kind));
+
+/** Each rule not run, with the reason, and how many outputs it did not run on. */
+const notRunOf = (results: readonly GradeResult[]): GradeSummary["notRun"] => {
+  const counts = new Map<string, { entry: NotRunEntry; outputs: number }>();
+  results.forEach((result) =>
+    result.notRun.forEach((entry) => {
+      const key = `${entry.rule}\n${entry.reason}`;
+      counts.set(key, { entry, outputs: (counts.get(key)?.outputs ?? 0) + 1 });
+    }),
+  );
+  return [...counts.values()].map(({ entry, outputs }) => ({ ...entry, outputs })).toSorted((left, right) => compareText(left.rule, right.rule));
+};
+
+const penaltyOf = (results: readonly GradeResult[]): { readonly penalty?: number } => {
+  const scored = results.flatMap((result) => (result.score === undefined ? [] : [result.score.penalty]));
+  return scored.length === 0 ? {} : { penalty: scored.reduce((sum, points) => sum + points, 0) };
+};
+
+const contextsOf = (results: readonly GradeResult[]): Pick<GradeSummary, "contexts"> => {
+  const read = results.flatMap((result) => (result.contexts === undefined ? [] : [result.contexts]));
+  if (read.length === 0) return {};
+  return {
+    contexts: {
+      outputs: read.length,
+      checked: read.reduce((sum, contexts) => sum + contexts.checked, 0),
+      unsupported: kindsOf(read.flatMap((contexts) => contexts.unsupported)),
+    },
+  };
+};
+
+export const summaryOf = (results: readonly GradeResult[]): GradeSummary => {
+  const sizes = sizesOf(results);
+  return {
+    total: results.length,
+    passed: results.filter((result) => result.pass).length,
+    failed: results
+      .filter((result) => !result.pass)
+      .map((result) => ({ id: result.id, ...(result.variant === undefined ? {} : { variant: result.variant }), failedBecause: result.failedBecause })),
+    size: sizes,
+    rules: rulesOf(results, sizes),
+    facts: {
+      dropped: kindsOf(results.flatMap((result) => result.facts?.dropped ?? [])),
+      added: kindsOf(results.flatMap((result) => result.facts?.added ?? [])),
+    },
+    citations: {
+      checked: results.reduce((sum, result) => sum + (result.citations?.checked ?? 0), 0),
+      failed: results.reduce((sum, result) => sum + (result.citations?.failed.length ?? 0), 0),
+    },
+    ...contextsOf(results),
+    notRun: notRunOf(results),
+    ...penaltyOf(results),
+    stamp: results[0]?.stamp,
+  };
+};

package/src/grade/text.ts

@@ -0,0 +1,164 @@
+import type { Texts } from "../ui.ts";
+import { ATOM_KINDS } from "../compare/atom.ts";
+import type { ItemProblem, ItemProblemKind } from "./item.ts";
+import type { Expected, RubricProblem } from "./rubric.ts";
+
+export type GradeText = {
+  readonly usage: string;
+  readonly noReference: string;
+  readonly noCitations: string;
+  readonly noContexts: string;
+  readonly contextsUnread: (kind: string, reason: string) => string;
+  readonly uncheckedSentences: (count: number) => string;
+  readonly unknownRule: string;
+  readonly rubricProblem: (problem: RubricProblem) => string;
+  readonly penalty: (total: number) => string;
+  readonly noStructure: (source: string, language: string) => string;
+  readonly unreadable: (path: string, why: string) => string;
+  readonly problem: (problem: ItemProblem) => string;
+  readonly cannotLoad: (why: string) => string;
+  readonly wrote: (path: string, count: number) => string;
+  readonly totals: (path: string, total: number, passed: number) => string;
+  readonly failedHeading: string;
+  readonly ratesHeading: (unit: string) => string;
+  readonly unit: Readonly<Record<"char" | "word", string>>;
+  readonly outputsWith: (count: number) => string;
+  readonly facts: (dropped: number, added: number, droppedKinds: string, addedKinds: string) => string;
+  readonly citations: (checked: number, failed: number) => string;
+  readonly contexts: (outputs: number, checked: number, unsupported: number, kinds: string) => string;
+  readonly notRunHeading: (count: number) => string;
+  readonly inOutputs: (count: number) => string;
+  readonly stamp: string;
+};
+
+/** Where the problem is, when it is on a line of a file. An item given to grade() has none. */
+const lineJa = (problem: ItemProblem): string => (problem.line > 0 ? `${String(problem.line)} 行目: ` : "");
+
+const lineEn = (problem: ItemProblem): string => (problem.line > 0 ? `line ${String(problem.line)}: ` : "");
+
+const PROBLEM_JA: Readonly<Record<ItemProblemKind, (problem: ItemProblem) => string>> = {
+  "not-json": (problem) => `${lineJa(problem)}JSON として読めません（${problem.detail ?? ""}）`,
+  "not-object": (problem) => `${lineJa(problem)}1 行に 1 つのオブジェクト {…} を書いてください`,
+  "no-id": (problem) => `${lineJa(problem)}id（空でない文字列）がありません`,
+  "no-output": (problem) => `${lineJa(problem)}output（文字列）がありません`,
+  "not-string": (problem) => `${lineJa(problem)}${problem.detail ?? ""} は文字列で書いてください`,
+  "not-text-map": (problem) => `${lineJa(problem)}${problem.detail ?? ""} は { "名前": "本文" } の形で書いてください`,
+  "not-citations": (problem) => `${lineJa(problem)}citations は [{ "source", "address", "quote" }] の配列で書いてください`,
+  "citations-without-sources": (problem) => `${lineJa(problem)}citations があるのに sources がありません。引用を照らす原文を sources に入れてください`,
+  "not-contexts": (problem) => `${lineJa(problem)}contexts は文字列の配列（["検索で取った一節", …]）で書いてください`,
+  "duplicate-id": (problem) => `${lineJa(problem)}id "${problem.detail ?? ""}" は ${String(problem.first ?? 0)} 行目と同じです`,
+  "no-variant": (problem) =>
+    `${lineJa(problem)}${problem.detail ?? ""}（どの prompt や model の出力かを示す名前）がありません。並べて比べるファイルでは、どの行にも書いてください`,
+  "unknown-source": (problem) => `${lineJa(problem)}引用の source "${problem.detail ?? ""}" が sources にありません`,
+  "which-source": (problem) => `${lineJa(problem)}番地 ${problem.detail ?? ""} の引用に source がありません。原文が二つ以上あるときは名前を書いてください`,
+  "unknown-language": (problem) => `${lineJa(problem)}language "${problem.detail ?? ""}" は読めません`,
+  "unknown-genre": (problem) => `${lineJa(problem)}genre "${problem.detail ?? ""}" は chaff の知らないジャンルです（chaff genres で一覧）`,
+  empty: () => "採点する出力が 1 行もありません",
+};
+
+const PROBLEM_EN: typeof PROBLEM_JA = {
+  "not-json": (problem) => `${lineEn(problem)}not valid JSON (${problem.detail ?? ""})`,
+  "not-object": (problem) => `${lineEn(problem)}write one object {…} per line`,
+  "no-id": (problem) => `${lineEn(problem)}no id (a non-empty string)`,
+  "no-output": (problem) => `${lineEn(problem)}no output (a string)`,
+  "not-string": (problem) => `${lineEn(problem)}${problem.detail ?? ""} must be a string`,
+  "not-text-map": (problem) => `${lineEn(problem)}${problem.detail ?? ""} must be { "name": "text" }`,
+  "not-citations": (problem) => `${lineEn(problem)}citations must be an array of { "source", "address", "quote" }`,
+  "citations-without-sources": (problem) => `${lineEn(problem)}citations without sources; put the text they quote in sources`,
+  "not-contexts": (problem) => `${lineEn(problem)}contexts must be an array of strings (["a retrieved passage", …])`,
+  "duplicate-id": (problem) => `${lineEn(problem)}id "${problem.detail ?? ""}" is already on line ${String(problem.first ?? 0)}`,
+  "no-variant": (problem) => `${lineEn(problem)}no ${problem.detail ?? ""} (the variant's name); when a file compares variants, every line needs one`,
+  "unknown-source": (problem) => `${lineEn(problem)}the citation's source "${problem.detail ?? ""}" is not in sources`,
+  "which-source": (problem) => `${lineEn(problem)}the citation of ${problem.detail ?? ""} names no source; with two or more sources, name one`,
+  "unknown-language": (problem) => `${lineEn(problem)}language "${problem.detail ?? ""}" cannot be read`,
+  "unknown-genre": (problem) => `${lineEn(problem)}genre "${problem.detail ?? ""}" is not one chaff knows (chaff genres lists them)`,
+  empty: () => "no output to grade: the file has no item",
+};
+
+const EXPECTED_JA: Readonly<Record<Expected, string>> = {
+  map: "キーと値の組（{ … }）",
+  count: " 0 以上の整数",
+  number: " 0 以上の数",
+  boolean: " true か false",
+  words: "空でない文字列の並び",
+  kinds: `事実の種類（${ATOM_KINDS.join("・")}）の並び`,
+  "known-key": "知っているキーだけ",
+};
+
+const EXPECTED_EN: Readonly<Record<Expected, string>> = {
+  map: "a map ({ … })",
+  count: "a whole number from 0",
+  number: "a number from 0",
+  boolean: "true or false",
+  words: "a list of non-empty strings",
+  kinds: `a list of fact kinds (${ATOM_KINDS.join(", ")})`,
+  "known-key": "a key chaff knows",
+};
+
+/** A bracketed aside, or nothing when there is nothing to say. */
+const aside = (open: string, inner: string, close: string): string => (inner === "" ? "" : [open, inner, close].join(""));
+
+export const GRADE_TEXT: Texts<GradeText> = {
+  ja: {
+    usage:
+      "使い方: chaff grade <items.jsonl> [--out <results.jsonl>] [--baseline <前の results.jsonl>] [--variant-key <欄の名前>] [--format text|json|markdown] [--json] [--compact] [--experimental] [--genre <ジャンル>]",
+    noReference: "reference が無い（事実は reference と照らす）",
+    noCitations: "citations が無い（chaff は出力から引用を推測しない）",
+    noContexts: "contexts が無い（回答の事実は、検索で取った一節と照らす）",
+    contextsUnread: (kind, reason) => `contexts と照らすとき、${kind} を読み切れなかった（${reason}）。その種類の事実は、一節にあっても見つからないことがある`,
+    uncheckedSentences: (count) => `数・日付・名前・引用の無い文 ${String(count)} 件は、contexts に支えられているかを照らしていない（意味を読む必要がある）`,
+    unknownRule: "grade: に書かれているが、chaff の知らないルール",
+    rubricProblem: (problem) => `chaff.yaml の ${problem.path} は${EXPECTED_JA[problem.expected]}で書いてください（書かれていたのは ${problem.written}）`,
+    penalty: (total) => `減点の和: ${String(total)}`,
+    noStructure: (source, language) => `原文 ${source} の言語 ${language} のパッケージは文書の構造を読めない`,
+    unreadable: (path, why) => `${path} を読めませんでした: ${why}`,
+    problem: (problem) => PROBLEM_JA[problem.kind](problem),
+    cannotLoad: (why) => `言語パッケージを読み込めません: ${why}`,
+    wrote: (path, count) => `出力ごとの結果を書きました: ${path}（${String(count)} 行）`,
+    totals: (path, total, passed) => `${path}: ${String(total)} 件の出力、${String(passed)} 件が通り、${String(total - passed)} 件が落ちた`,
+    failedHeading: "落ちた出力",
+    ratesHeading: (unit) => `ルールごとの率（1,000 ${unit}あたり、指摘のあった出力の数）`,
+    unit: { char: "字", word: "語" },
+    outputsWith: (count) => `${String(count)} 件`,
+    facts: (dropped, added, droppedKinds, addedKinds) =>
+      `事実: 落ちた ${String(dropped)}${aside("（", droppedKinds, "）")}、足された ${String(added)}${aside("（", addedKinds, "）")}`,
+    citations: (checked, failed) => `引用: ${String(checked)} 件を照らし、${String(failed)} 件が外れた`,
+    contexts: (outputs, checked, unsupported, kinds) =>
+      `contexts: ${String(outputs)} 件の出力の事実 ${String(checked)} 件を照らし、${String(unsupported)} 件がどの一節にも無かった${aside("（", kinds, "）")}`,
+    notRunHeading: (count) => `動かなかったもの ${String(count)} 件`,
+    inOutputs: (count) => `${String(count)} 件の出力`,
+    stamp: "再現の印",
+  },
+  en: {
+    usage:
+      "usage: chaff grade <items.jsonl> [--out <results.jsonl>] [--baseline <earlier results.jsonl>] [--variant-key <field>] [--format text|json|markdown] [--json] [--compact] [--experimental] [--genre <genre>]",
+    noReference: "no reference given (facts are checked against a reference)",
+    noCitations: "no citations given (chaff does not guess quotations from the output)",
+    noContexts: "no contexts given (the output's facts are checked against retrieved passages)",
+    contextsUnread: (kind, reason) =>
+      `against contexts, ${kind} could not be fully read (${reason}); a fact of that kind may be in a passage and still not be found`,
+    uncheckedSentences: (count) =>
+      `${String(count)} ${count === 1 ? "sentence states" : "sentences state"} no number, date, name or quotation, so their support by contexts is not checked (it needs reading)`,
+    unknownRule: "named under grade: but not a rule chaff knows",
+    rubricProblem: (problem) => `chaff.yaml: ${problem.path} must be ${EXPECTED_EN[problem.expected]} (found ${problem.written})`,
+    penalty: (total) => `Penalty points: ${String(total)}`,
+    noStructure: (source, language) => `the ${language} package cannot read the structure of source ${source}`,
+    unreadable: (path, why) => `Could not read ${path}: ${why}`,
+    problem: (problem) => PROBLEM_EN[problem.kind](problem),
+    cannotLoad: (why) => `Cannot load a language package: ${why}`,
+    wrote: (path, count) => `Wrote one result per output: ${path} (${String(count)} lines)`,
+    totals: (path, total, passed) => `${path}: ${String(total)} outputs, ${String(passed)} passed, ${String(total - passed)} failed`,
+    failedHeading: "Failed outputs",
+    ratesHeading: (unit) => `Rule rates (per 1,000 ${unit}, outputs with a finding)`,
+    unit: { char: "characters", word: "words" },
+    outputsWith: (count) => `${String(count)} ${count === 1 ? "output" : "outputs"}`,
+    facts: (dropped, added, droppedKinds, addedKinds) =>
+      `Facts: ${String(dropped)} dropped${aside(" (", droppedKinds, ")")}, ${String(added)} added${aside(" (", addedKinds, ")")}`,
+    citations: (checked, failed) => `Quotations: ${String(checked)} checked, ${String(failed)} failed`,
+    contexts: (outputs, checked, unsupported, kinds) =>
+      `Contexts: ${String(checked)} facts checked in ${String(outputs)} ${outputs === 1 ? "output" : "outputs"}, ${String(unsupported)} in no passage${aside(" (", kinds, ")")}`,
+    notRunHeading: (count) => `${String(count)} not run`,
+    inOutputs: (count) => `${String(count)} ${count === 1 ? "output" : "outputs"}`,
+    stamp: "Stamp",
+  },
+};