chaffjs 0.19.0 → 0.20.0

package/src/grade/summary.ts

@@ -0,0 +1,112 @@
+import type { LengthUnit } from "../plugin.ts";
+import { compareText, sortedByKey, tally } from "./order.ts";
+import { rateOf } from "./rates.ts";
+import type { GradeFact, GradeResult, NotRunEntry, Stamp } from "./result.ts";
+
+// The summary of a run (spec §29.3), from the per-output results alone, so the same results always sum the same way.
+
+/** Per length unit: Japanese outputs are measured in characters and English ones in words, and the two never add up. */
+type PerUnit = Partial<Record<LengthUnit, number>>;
+
+type RuleSummary = { readonly findings: number; readonly outputs: number; readonly rate: PerUnit };
+
+export type GradeSummary = {
+  readonly total: number;
+  readonly passed: number;
+  readonly failed: readonly { readonly id: string; readonly variant?: string | undefined; readonly failedBecause: readonly string[] }[];
+  readonly size: PerUnit;
+  readonly rules: Readonly<Record<string, RuleSummary>>;
+  readonly facts: { readonly dropped: Readonly<Record<string, number>>; readonly added: Readonly<Record<string, number>> };
+  readonly citations: { readonly checked: number; readonly failed: number };
+  /** Only when some output had contexts: how many, their facts checked, and the facts no passage states by kind. */
+  readonly contexts?: { readonly outputs: number; readonly checked: number; readonly unsupported: Readonly<Record<string, number>> } | undefined;
+  readonly notRun: readonly (NotRunEntry & { readonly outputs: number })[];
+  /** The penalty points of every output, added up. Only with a `grade:` rubric. */
+  readonly penalty?: number | undefined;
+  readonly stamp: Stamp | undefined;
+};
+
+const sizesOf = (results: readonly GradeResult[]): PerUnit =>
+  results.reduce<PerUnit>((sizes, result) => ({ ...sizes, [result.size.unit]: (sizes[result.size.unit] ?? 0) + result.size.value }), {});
+
+const countIn = (result: GradeResult, rule: string): number => result.findings.filter((finding) => finding.rule === rule).length;
+
+/** A rule's findings over every output of one unit, per 1,000 of that unit. */
+const ratePerUnit = (results: readonly GradeResult[], rule: string, sizes: PerUnit): PerUnit =>
+  Object.fromEntries(
+    Object.entries(sizes).flatMap(([unit, size]) => {
+      const count = results.filter((result) => result.size.unit === unit).reduce((sum, result) => sum + countIn(result, rule), 0);
+      const rate = count === 0 ? undefined : rateOf(count, size);
+      return rate === undefined ? [] : [[unit, rate]];
+    }),
+  );
+
+const rulesOf = (results: readonly GradeResult[], sizes: PerUnit): Record<string, RuleSummary> => {
+  const ruleIds = [...new Set(results.flatMap((result) => result.findings.map((finding) => finding.rule)))];
+  return sortedByKey(
+    ruleIds.map((rule): [string, RuleSummary] => [
+      rule,
+      {
+        findings: results.reduce((sum, result) => sum + countIn(result, rule), 0),
+        outputs: results.filter((result) => countIn(result, rule) > 0).length,
+        rate: ratePerUnit(results, rule, sizes),
+      },
+    ]),
+  );
+};
+
+const kindsOf = (facts: readonly GradeFact[]): Record<string, number> => tally(facts.filter((fact) => !fact.allowed).map((fact) => fact.kind));
+
+/** Each rule not run, with the reason, and how many outputs it did not run on. */
+const notRunOf = (results: readonly GradeResult[]): GradeSummary["notRun"] => {
+  const counts = new Map<string, { entry: NotRunEntry; outputs: number }>();
+  results.forEach((result) =>
+    result.notRun.forEach((entry) => {
+      const key = `${entry.rule}\n${entry.reason}`;
+      counts.set(key, { entry, outputs: (counts.get(key)?.outputs ?? 0) + 1 });
+    }),
+  );
+  return [...counts.values()].map(({ entry, outputs }) => ({ ...entry, outputs })).toSorted((left, right) => compareText(left.rule, right.rule));
+};
+
+const penaltyOf = (results: readonly GradeResult[]): { readonly penalty?: number } => {
+  const scored = results.flatMap((result) => (result.score === undefined ? [] : [result.score.penalty]));
+  return scored.length === 0 ? {} : { penalty: scored.reduce((sum, points) => sum + points, 0) };
+};
+
+const contextsOf = (results: readonly GradeResult[]): Pick<GradeSummary, "contexts"> => {
+  const read = results.flatMap((result) => (result.contexts === undefined ? [] : [result.contexts]));
+  if (read.length === 0) return {};
+  return {
+    contexts: {
+      outputs: read.length,
+      checked: read.reduce((sum, contexts) => sum + contexts.checked, 0),
+      unsupported: kindsOf(read.flatMap((contexts) => contexts.unsupported)),
+    },
+  };
+};
+
+export const summaryOf = (results: readonly GradeResult[]): GradeSummary => {
+  const sizes = sizesOf(results);
+  return {
+    total: results.length,
+    passed: results.filter((result) => result.pass).length,
+    failed: results
+      .filter((result) => !result.pass)
+      .map((result) => ({ id: result.id, ...(result.variant === undefined ? {} : { variant: result.variant }), failedBecause: result.failedBecause })),
+    size: sizes,
+    rules: rulesOf(results, sizes),
+    facts: {
+      dropped: kindsOf(results.flatMap((result) => result.facts?.dropped ?? [])),
+      added: kindsOf(results.flatMap((result) => result.facts?.added ?? [])),
+    },
+    citations: {
+      checked: results.reduce((sum, result) => sum + (result.citations?.checked ?? 0), 0),
+      failed: results.reduce((sum, result) => sum + (result.citations?.failed.length ?? 0), 0),
+    },
+    ...contextsOf(results),
+    notRun: notRunOf(results),
+    ...penaltyOf(results),
+    stamp: results[0]?.stamp,
+  };
+};

package/src/grade/text.ts

@@ -0,0 +1,164 @@
+import type { Texts } from "../ui.ts";
+import { ATOM_KINDS } from "../compare/atom.ts";
+import type { ItemProblem, ItemProblemKind } from "./item.ts";
+import type { Expected, RubricProblem } from "./rubric.ts";
+
+export type GradeText = {
+  readonly usage: string;
+  readonly noReference: string;
+  readonly noCitations: string;
+  readonly noContexts: string;
+  readonly contextsUnread: (kind: string, reason: string) => string;
+  readonly uncheckedSentences: (count: number) => string;
+  readonly unknownRule: string;
+  readonly rubricProblem: (problem: RubricProblem) => string;
+  readonly penalty: (total: number) => string;
+  readonly noStructure: (source: string, language: string) => string;
+  readonly unreadable: (path: string, why: string) => string;
+  readonly problem: (problem: ItemProblem) => string;
+  readonly cannotLoad: (why: string) => string;
+  readonly wrote: (path: string, count: number) => string;
+  readonly totals: (path: string, total: number, passed: number) => string;
+  readonly failedHeading: string;
+  readonly ratesHeading: (unit: string) => string;
+  readonly unit: Readonly<Record<"char" | "word", string>>;
+  readonly outputsWith: (count: number) => string;
+  readonly facts: (dropped: number, added: number, droppedKinds: string, addedKinds: string) => string;
+  readonly citations: (checked: number, failed: number) => string;
+  readonly contexts: (outputs: number, checked: number, unsupported: number, kinds: string) => string;
+  readonly notRunHeading: (count: number) => string;
+  readonly inOutputs: (count: number) => string;
+  readonly stamp: string;
+};
+
+/** Where the problem is, when it is on a line of a file. An item given to grade() has none. */
+const lineJa = (problem: ItemProblem): string => (problem.line > 0 ? `${String(problem.line)} 行目: ` : "");
+
+const lineEn = (problem: ItemProblem): string => (problem.line > 0 ? `line ${String(problem.line)}: ` : "");
+
+const PROBLEM_JA: Readonly<Record<ItemProblemKind, (problem: ItemProblem) => string>> = {
+  "not-json": (problem) => `${lineJa(problem)}JSON として読めません（${problem.detail ?? ""}）`,
+  "not-object": (problem) => `${lineJa(problem)}1 行に 1 つのオブジェクト {…} を書いてください`,
+  "no-id": (problem) => `${lineJa(problem)}id（空でない文字列）がありません`,
+  "no-output": (problem) => `${lineJa(problem)}output（文字列）がありません`,
+  "not-string": (problem) => `${lineJa(problem)}${problem.detail ?? ""} は文字列で書いてください`,
+  "not-text-map": (problem) => `${lineJa(problem)}${problem.detail ?? ""} は { "名前": "本文" } の形で書いてください`,
+  "not-citations": (problem) => `${lineJa(problem)}citations は [{ "source", "address", "quote" }] の配列で書いてください`,
+  "citations-without-sources": (problem) => `${lineJa(problem)}citations があるのに sources がありません。引用を照らす原文を sources に入れてください`,
+  "not-contexts": (problem) => `${lineJa(problem)}contexts は文字列の配列（["検索で取った一節", …]）で書いてください`,
+  "duplicate-id": (problem) => `${lineJa(problem)}id "${problem.detail ?? ""}" は ${String(problem.first ?? 0)} 行目と同じです`,
+  "no-variant": (problem) =>
+    `${lineJa(problem)}${problem.detail ?? ""}（どの prompt や model の出力かを示す名前）がありません。並べて比べるファイルでは、どの行にも書いてください`,
+  "unknown-source": (problem) => `${lineJa(problem)}引用の source "${problem.detail ?? ""}" が sources にありません`,
+  "which-source": (problem) => `${lineJa(problem)}番地 ${problem.detail ?? ""} の引用に source がありません。原文が二つ以上あるときは名前を書いてください`,
+  "unknown-language": (problem) => `${lineJa(problem)}language "${problem.detail ?? ""}" は読めません`,
+  "unknown-genre": (problem) => `${lineJa(problem)}genre "${problem.detail ?? ""}" は chaff の知らないジャンルです（chaff genres で一覧）`,
+  empty: () => "採点する出力が 1 行もありません",
+};
+
+const PROBLEM_EN: typeof PROBLEM_JA = {
+  "not-json": (problem) => `${lineEn(problem)}not valid JSON (${problem.detail ?? ""})`,
+  "not-object": (problem) => `${lineEn(problem)}write one object {…} per line`,
+  "no-id": (problem) => `${lineEn(problem)}no id (a non-empty string)`,
+  "no-output": (problem) => `${lineEn(problem)}no output (a string)`,
+  "not-string": (problem) => `${lineEn(problem)}${problem.detail ?? ""} must be a string`,
+  "not-text-map": (problem) => `${lineEn(problem)}${problem.detail ?? ""} must be { "name": "text" }`,
+  "not-citations": (problem) => `${lineEn(problem)}citations must be an array of { "source", "address", "quote" }`,
+  "citations-without-sources": (problem) => `${lineEn(problem)}citations without sources; put the text they quote in sources`,
+  "not-contexts": (problem) => `${lineEn(problem)}contexts must be an array of strings (["a retrieved passage", …])`,
+  "duplicate-id": (problem) => `${lineEn(problem)}id "${problem.detail ?? ""}" is already on line ${String(problem.first ?? 0)}`,
+  "no-variant": (problem) => `${lineEn(problem)}no ${problem.detail ?? ""} (the variant's name); when a file compares variants, every line needs one`,
+  "unknown-source": (problem) => `${lineEn(problem)}the citation's source "${problem.detail ?? ""}" is not in sources`,
+  "which-source": (problem) => `${lineEn(problem)}the citation of ${problem.detail ?? ""} names no source; with two or more sources, name one`,
+  "unknown-language": (problem) => `${lineEn(problem)}language "${problem.detail ?? ""}" cannot be read`,
+  "unknown-genre": (problem) => `${lineEn(problem)}genre "${problem.detail ?? ""}" is not one chaff knows (chaff genres lists them)`,
+  empty: () => "no output to grade: the file has no item",
+};
+
+const EXPECTED_JA: Readonly<Record<Expected, string>> = {
+  map: "キーと値の組（{ … }）",
+  count: " 0 以上の整数",
+  number: " 0 以上の数",
+  boolean: " true か false",
+  words: "空でない文字列の並び",
+  kinds: `事実の種類（${ATOM_KINDS.join("・")}）の並び`,
+  "known-key": "知っているキーだけ",
+};
+
+const EXPECTED_EN: Readonly<Record<Expected, string>> = {
+  map: "a map ({ … })",
+  count: "a whole number from 0",
+  number: "a number from 0",
+  boolean: "true or false",
+  words: "a list of non-empty strings",
+  kinds: `a list of fact kinds (${ATOM_KINDS.join(", ")})`,
+  "known-key": "a key chaff knows",
+};
+
+/** A bracketed aside, or nothing when there is nothing to say. */
+const aside = (open: string, inner: string, close: string): string => (inner === "" ? "" : [open, inner, close].join(""));
+
+export const GRADE_TEXT: Texts<GradeText> = {
+  ja: {
+    usage:
+      "使い方: chaff grade <items.jsonl> [--out <results.jsonl>] [--baseline <前の results.jsonl>] [--variant-key <欄の名前>] [--format text|json|markdown] [--json] [--compact] [--experimental] [--genre <ジャンル>]",
+    noReference: "reference が無い（事実は reference と照らす）",
+    noCitations: "citations が無い（chaff は出力から引用を推測しない）",
+    noContexts: "contexts が無い（回答の事実は、検索で取った一節と照らす）",
+    contextsUnread: (kind, reason) => `contexts と照らすとき、${kind} を読み切れなかった（${reason}）。その種類の事実は、一節にあっても見つからないことがある`,
+    uncheckedSentences: (count) => `数・日付・名前・引用の無い文 ${String(count)} 件は、contexts に支えられているかを照らしていない（意味を読む必要がある）`,
+    unknownRule: "grade: に書かれているが、chaff の知らないルール",
+    rubricProblem: (problem) => `chaff.yaml の ${problem.path} は${EXPECTED_JA[problem.expected]}で書いてください（書かれていたのは ${problem.written}）`,
+    penalty: (total) => `減点の和: ${String(total)}`,
+    noStructure: (source, language) => `原文 ${source} の言語 ${language} のパッケージは文書の構造を読めない`,
+    unreadable: (path, why) => `${path} を読めませんでした: ${why}`,
+    problem: (problem) => PROBLEM_JA[problem.kind](problem),
+    cannotLoad: (why) => `言語パッケージを読み込めません: ${why}`,
+    wrote: (path, count) => `出力ごとの結果を書きました: ${path}（${String(count)} 行）`,
+    totals: (path, total, passed) => `${path}: ${String(total)} 件の出力、${String(passed)} 件が通り、${String(total - passed)} 件が落ちた`,
+    failedHeading: "落ちた出力",
+    ratesHeading: (unit) => `ルールごとの率（1,000 ${unit}あたり、指摘のあった出力の数）`,
+    unit: { char: "字", word: "語" },
+    outputsWith: (count) => `${String(count)} 件`,
+    facts: (dropped, added, droppedKinds, addedKinds) =>
+      `事実: 落ちた ${String(dropped)}${aside("（", droppedKinds, "）")}、足された ${String(added)}${aside("（", addedKinds, "）")}`,
+    citations: (checked, failed) => `引用: ${String(checked)} 件を照らし、${String(failed)} 件が外れた`,
+    contexts: (outputs, checked, unsupported, kinds) =>
+      `contexts: ${String(outputs)} 件の出力の事実 ${String(checked)} 件を照らし、${String(unsupported)} 件がどの一節にも無かった${aside("（", kinds, "）")}`,
+    notRunHeading: (count) => `動かなかったもの ${String(count)} 件`,
+    inOutputs: (count) => `${String(count)} 件の出力`,
+    stamp: "再現の印",
+  },
+  en: {
+    usage:
+      "usage: chaff grade <items.jsonl> [--out <results.jsonl>] [--baseline <earlier results.jsonl>] [--variant-key <field>] [--format text|json|markdown] [--json] [--compact] [--experimental] [--genre <genre>]",
+    noReference: "no reference given (facts are checked against a reference)",
+    noCitations: "no citations given (chaff does not guess quotations from the output)",
+    noContexts: "no contexts given (the output's facts are checked against retrieved passages)",
+    contextsUnread: (kind, reason) =>
+      `against contexts, ${kind} could not be fully read (${reason}); a fact of that kind may be in a passage and still not be found`,
+    uncheckedSentences: (count) =>
+      `${String(count)} ${count === 1 ? "sentence states" : "sentences state"} no number, date, name or quotation, so their support by contexts is not checked (it needs reading)`,
+    unknownRule: "named under grade: but not a rule chaff knows",
+    rubricProblem: (problem) => `chaff.yaml: ${problem.path} must be ${EXPECTED_EN[problem.expected]} (found ${problem.written})`,
+    penalty: (total) => `Penalty points: ${String(total)}`,
+    noStructure: (source, language) => `the ${language} package cannot read the structure of source ${source}`,
+    unreadable: (path, why) => `Could not read ${path}: ${why}`,
+    problem: (problem) => PROBLEM_EN[problem.kind](problem),
+    cannotLoad: (why) => `Cannot load a language package: ${why}`,
+    wrote: (path, count) => `Wrote one result per output: ${path} (${String(count)} lines)`,
+    totals: (path, total, passed) => `${path}: ${String(total)} outputs, ${String(passed)} passed, ${String(total - passed)} failed`,
+    failedHeading: "Failed outputs",
+    ratesHeading: (unit) => `Rule rates (per 1,000 ${unit}, outputs with a finding)`,
+    unit: { char: "characters", word: "words" },
+    outputsWith: (count) => `${String(count)} ${count === 1 ? "output" : "outputs"}`,
+    facts: (dropped, added, droppedKinds, addedKinds) =>
+      `Facts: ${String(dropped)} dropped${aside(" (", droppedKinds, ")")}, ${String(added)} added${aside(" (", addedKinds, ")")}`,
+    citations: (checked, failed) => `Quotations: ${String(checked)} checked, ${String(failed)} failed`,
+    contexts: (outputs, checked, unsupported, kinds) =>
+      `Contexts: ${String(checked)} facts checked in ${String(outputs)} ${outputs === 1 ? "output" : "outputs"}, ${String(unsupported)} in no passage${aside(" (", kinds, ")")}`,
+    notRunHeading: (count) => `${String(count)} not run`,
+    inOutputs: (count) => `${String(count)} ${count === 1 ? "output" : "outputs"}`,
+    stamp: "Stamp",
+  },
+};

package/src/grade/variants-input.ts

@@ -0,0 +1,54 @@
+import type { GradeResult } from "./result.ts";
+import { isGradeResult } from "./results-read.ts";
+
+// The results `compareVariants()` is given, checked and grouped by variant. Pure. A harness builds this input by hand, so
+// a result without an id, without a variant or under an id its variant already has is said, not silently dropped.
+
+/** Each variant's results, by variant label then id, in order of first appearance. */
+export type VariantGroups = ReadonlyMap<string, ReadonlyMap<string, GradeResult>>;
+
+/** Labelled results, or each variant's results under its label. */
+export type VariantInput = readonly GradeResult[] | Readonly<Record<string, readonly GradeResult[]>>;
+
+type Labelled = { readonly variant: unknown; readonly result: unknown; readonly place: string };
+
+const isRecord = (value: unknown): value is Record<string, unknown> => typeof value === "object" && value !== null && !Array.isArray(value);
+
+const labelledOf = (input: unknown): Labelled[] | undefined => {
+  if (Array.isArray(input))
+    return input.map((result: unknown, index) => ({ variant: isRecord(result) ? result["variant"] : undefined, result, place: `results[${String(index)}]` }));
+  if (!isRecord(input)) return undefined;
+  return Object.entries(input).flatMap(([variant, results]) =>
+    Array.isArray(results)
+      ? results.map((result: unknown, index) => ({ variant, result, place: `${variant}[${String(index)}]` }))
+      : [{ variant, result: results, place: variant }],
+  );
+};
+
+/** A whole result line, as `--out` writes it, with an id to match by: a partial object would break the table, not be compared. */
+const isResult = (value: unknown): value is GradeResult => isGradeResult(value) && value.id !== "";
+
+/** Why one entry cannot be compared, or undefined when it can. */
+const problemOf = (entry: Labelled, groups: ReadonlyMap<string, ReadonlyMap<string, GradeResult>>): string | undefined => {
+  if (!isResult(entry.result)) return `${entry.place}: not a grade result with an id`;
+  if (typeof entry.variant !== "string" || entry.variant === "") return `${entry.place}: id "${entry.result.id}" has no variant`;
+  return groups.get(entry.variant)?.has(entry.result.id) === true
+    ? `${entry.place}: id "${entry.result.id}" is already in variant "${entry.variant}"`
+    : undefined;
+};
+
+/** The results grouped by variant, or every reason some cannot be. No result at all is a problem, not an empty comparison. */
+export const variantGroupsOf = (input: unknown): { readonly groups: VariantGroups } | { readonly problems: readonly string[] } => {
+  const labelled = labelledOf(input);
+  if (labelled === undefined) return { problems: ["results must be an array of grade results, or { variant: results[] }"] };
+  const groups = new Map<string, Map<string, GradeResult>>();
+  const problems = labelled.flatMap((entry) => {
+    const problem = problemOf(entry, groups);
+    if (problem !== undefined || !isResult(entry.result) || typeof entry.variant !== "string") return problem === undefined ? [] : [problem];
+    const byId = groups.get(entry.variant) ?? new Map<string, GradeResult>();
+    groups.set(entry.variant, byId.set(entry.result.id, { ...entry.result, variant: entry.variant }));
+    return [];
+  });
+  if (problems.length > 0) return { problems };
+  return groups.size === 0 ? { problems: ["no results to compare"] } : { groups };
+};

package/src/grade/variants-text.ts

@@ -0,0 +1,88 @@
+import type { Texts } from "../ui.ts";
+
+export type VariantText = {
+  readonly heading: (compared: number, variants: number) => string;
+  readonly passed: string;
+  readonly passedCell: (passed: number, outputs: number, rate: string) => string;
+  readonly factsDropped: string;
+  readonly factsAdded: string;
+  readonly citationsFailed: string;
+  readonly citationsCell: (failed: number, checked: number) => string;
+  readonly unsupportedFacts: string;
+  readonly penalty: string;
+  readonly ratesHeading: (unit: string) => string;
+  readonly unit: Readonly<Record<"char" | "word", string>>;
+  readonly disagreeHeading: (count: number) => string;
+  readonly disagreement: (id: string, passedIn: string, failedIn: string) => string;
+  readonly failedIn: (variant: string, reasons: string) => string;
+  readonly noDisagreement: string;
+  readonly missing: (id: string, variants: string) => string;
+  readonly readOtherwise: (ids: string) => string;
+  readonly notComparedHeading: string;
+  /** Markdown only: the run's heading and its totals. */
+  readonly runHeading: (path: string) => string;
+  readonly totals: (total: number, passed: number) => string;
+  readonly failedHeading: string;
+  readonly regressionsHeading: (count: number) => string;
+  readonly noRegression: string;
+  readonly stamp: string;
+  readonly separator: string;
+};
+
+export const VARIANT_TEXT: Texts<VariantText> = {
+  ja: {
+    heading: (compared, variants) => `${String(variants)} つの variant を並べた: どの variant にもある id の出力 ${String(compared)} 件`,
+    passed: "通った",
+    passedCell: (passed, outputs, rate) => `${String(passed)}/${String(outputs)}（${rate}）`,
+    factsDropped: "落ちた事実",
+    factsAdded: "足された事実",
+    citationsFailed: "外れた引用",
+    citationsCell: (failed, checked) => `${String(failed)}/${String(checked)}`,
+    unsupportedFacts: "一節に無い事実",
+    penalty: "減点の和",
+    ratesHeading: (unit) => `ルールごとの率（1,000 ${unit}あたり）`,
+    unit: { char: "字", word: "語" },
+    disagreeHeading: (count) => `合否が分かれた出力 ${String(count)} 件`,
+    disagreement: (id, passedIn, failedIn) => `${id}: 通った ${passedIn}、落ちた ${failedIn}`,
+    failedIn: (variant, reasons) => `${variant}（${reasons}）`,
+    noDisagreement: "合否が分かれた出力はありません",
+    missing: (id, variants) => `${id}: ${variants} に無い`,
+    readOtherwise: (ids) => `言語かジャンルが違う: ${ids}`,
+    notComparedHeading: "比べなかった id",
+    runHeading: (path) => `chaff grade: ${path}`,
+    totals: (total, passed) => `${String(total)} 件の出力、${String(passed)} 件が通り、${String(total - passed)} 件が落ちた`,
+    failedHeading: "落ちた出力",
+    regressionsHeading: (count) => `前の回と比べた回帰 ${String(count)} 件`,
+    noRegression: "前の回と比べた回帰はありません",
+    stamp: "再現の印",
+    separator: "、",
+  },
+  en: {
+    heading: (compared, variants) =>
+      `${String(variants)} variants side by side: ${String(compared)} ${compared === 1 ? "output" : "outputs"} with an id every variant has`,
+    passed: "Passed",
+    passedCell: (passed, outputs, rate) => `${String(passed)}/${String(outputs)} (${rate})`,
+    factsDropped: "Facts dropped",
+    factsAdded: "Facts added",
+    citationsFailed: "Quotations failed",
+    citationsCell: (failed, checked) => `${String(failed)}/${String(checked)}`,
+    unsupportedFacts: "Unsupported facts",
+    penalty: "Penalty points",
+    ratesHeading: (unit) => `Rule rates (per 1,000 ${unit})`,
+    unit: { char: "characters", word: "words" },
+    disagreeHeading: (count) => `${String(count)} ${count === 1 ? "output" : "outputs"} where pass or fail differs`,
+    disagreement: (id, passedIn, failedIn) => `${id}: passed in ${passedIn}; failed in ${failedIn}`,
+    failedIn: (variant, reasons) => `${variant} (${reasons})`,
+    noDisagreement: "Every variant passed or failed the same outputs",
+    missing: (id, variants) => `${id}: missing from ${variants}`,
+    readOtherwise: (ids) => `another language or genre: ${ids}`,
+    notComparedHeading: "Ids not compared",
+    runHeading: (path) => `chaff grade: ${path}`,
+    totals: (total, passed) => `${String(total)} outputs, ${String(passed)} passed, ${String(total - passed)} failed`,
+    failedHeading: "Failed outputs",
+    regressionsHeading: (count) => `${String(count)} ${count === 1 ? "regression" : "regressions"} against the baseline`,
+    noRegression: "No regression against the baseline",
+    stamp: "Stamp",
+    separator: "; ",
+  },
+};

package/src/grade/variants.ts

@@ -0,0 +1,118 @@
+import type { LengthUnit } from "../plugin.ts";
+import { rateRows, sameReading } from "./pairing.ts";
+import type { GradeResult } from "./result.ts";
+import { summaryOf } from "./summary.ts";
+import { variantGroupsOf, type VariantGroups } from "./variants-input.ts";
+
+// Several variants (prompts, models, settings) graded on the same inputs, side by side (spec §29.5). Pure. Outputs are
+// matched by id: only an id every variant answered, read the same way, is compared, so each column covers the same tasks.
+
+/** One variant over the compared ids. */
+export type VariantColumn = {
+  readonly variant: string;
+  readonly outputs: number;
+  readonly passed: number;
+  /** The share of outputs that passed, in percent to one decimal. Undefined when no id was compared. */
+  readonly passRate?: number | undefined;
+  /** Facts dropped and added against the reference, not counting the kinds the rubric allows. */
+  readonly facts: { readonly dropped: number; readonly added: number };
+  readonly citations: { readonly checked: number; readonly failed: number };
+  /** Facts checked against the retrieved passages, and those in no passage. Only when some output had contexts. */
+  readonly contexts?: { readonly checked: number; readonly unsupported: number } | undefined;
+  /** Only with a `grade:` rubric. */
+  readonly penalty?: number | undefined;
+};
+
+/** One rule's rate per 1,000 units in each variant, by variant label; 0 where the variant had no finding of it. */
+export type VariantRates = { readonly rule: string; readonly unit: LengthUnit; readonly rates: Readonly<Record<string, number>> };
+
+/** An id some variants passed and others failed, with why each failed. */
+export type Disagreement = {
+  readonly id: string;
+  readonly passedIn: readonly string[];
+  readonly failedIn: readonly { readonly variant: string; readonly failedBecause: readonly string[] }[];
+};
+
+export type VariantComparison = {
+  /** In the order they first appear. */
+  readonly variants: readonly string[];
+  /** The ids every variant answered, read the same way. */
+  readonly compared: readonly string[];
+  /** Ids some variant did not answer, and which. */
+  readonly missing: readonly { readonly id: string; readonly missingFrom: readonly string[] }[];
+  /** Ids every variant answered, but in another language or genre: not the same task graded the same way. */
+  readonly readOtherwise: readonly string[];
+  readonly columns: readonly VariantColumn[];
+  readonly rules: readonly VariantRates[];
+  readonly disagreements: readonly Disagreement[];
+};
+
+const PERCENT = 100;
+const PERCENT_SCALE = 10;
+
+const percentOf = (passed: number, outputs: number): number | undefined =>
+  outputs === 0 ? undefined : Math.round((passed * PERCENT * PERCENT_SCALE) / outputs) / PERCENT_SCALE;
+
+const sum = (counts: Readonly<Record<string, number>>): number => Object.values(counts).reduce((total, count) => total + count, 0);
+
+const columnOf = (variant: string, results: readonly GradeResult[]): VariantColumn => {
+  const summary = summaryOf(results);
+  return {
+    variant,
+    outputs: summary.total,
+    passed: summary.passed,
+    passRate: percentOf(summary.passed, summary.total),
+    facts: { dropped: sum(summary.facts.dropped), added: sum(summary.facts.added) },
+    citations: summary.citations,
+    ...(summary.contexts === undefined ? {} : { contexts: { checked: summary.contexts.checked, unsupported: sum(summary.contexts.unsupported) } }),
+    ...(summary.penalty === undefined ? {} : { penalty: summary.penalty }),
+  };
+};
+
+const disagreementOf = (id: string, row: readonly GradeResult[], variants: readonly string[]): Disagreement[] => {
+  const passedIn = variants.filter((_, index) => row[index]?.pass === true);
+  if (passedIn.length === 0 || passedIn.length === variants.length) return [];
+  const failedIn = variants.flatMap((variant, index) => {
+    const result = row[index];
+    return result === undefined || result.pass ? [] : [{ variant, failedBecause: result.failedBecause }];
+  });
+  return [{ id, passedIn, failedIn }];
+};
+
+/** Each id in order of first appearance, with its result in each variant, or undefined where the variant has none. */
+const rowsOf = (grouped: VariantGroups, variants: readonly string[]): { readonly id: string; readonly row: readonly (GradeResult | undefined)[] }[] => {
+  const ids = [...new Set([...grouped.values()].flatMap((byId) => [...byId.keys()]))];
+  return ids.map((id) => ({ id, row: variants.map((variant) => grouped.get(variant)?.get(id)) }));
+};
+
+const isComplete = (row: readonly (GradeResult | undefined)[]): row is readonly GradeResult[] => row.every((result) => result !== undefined);
+
+/** The variants side by side over the ids they all answered. `grouped`: each variant's results by id, the variants in order. */
+export const compareVariantGroups = (grouped: VariantGroups): VariantComparison => {
+  const variants = [...grouped.keys()];
+  const rows = rowsOf(grouped, variants);
+  const complete = rows.flatMap(({ id, row }) => (isComplete(row) ? [{ id, row }] : []));
+  const compared = complete.filter(({ row }) => sameReading(row));
+  const byVariant = variants.map((_, index) => compared.flatMap(({ row }) => row.slice(index, index + 1)));
+  return {
+    variants,
+    compared: compared.map(({ id }) => id),
+    missing: rows.flatMap(({ id, row }) => {
+      const missingFrom = variants.filter((_, index) => row[index] === undefined);
+      return missingFrom.length === 0 ? [] : [{ id, missingFrom }];
+    }),
+    readOtherwise: complete.filter(({ row }) => !sameReading(row)).map(({ id }) => id),
+    columns: variants.map((variant, index) => columnOf(variant, byVariant[index] ?? [])),
+    rules: rateRows(byVariant)
+      .filter((row) => row.rates.some((rate) => rate > 0))
+      .map(({ rule, unit, rates }) => ({ rule, unit, rates: Object.fromEntries(variants.map((variant, index) => [variant, rates[index] ?? 0])) })),
+    disagreements: compared.flatMap(({ id, row }) => disagreementOf(id, row, variants)),
+  };
+};
+
+/** A run's results compared variant by variant, or undefined when none has a variant label (the command line's input already checked). */
+export const variantsOfRun = (results: readonly GradeResult[]): VariantComparison | undefined => {
+  if (!results.some((result) => result.variant !== undefined)) return undefined;
+  const read = variantGroupsOf(results);
+  return "groups" in read ? compareVariantGroups(read.groups) : undefined;
+};

package/src/grade/verdict.ts

@@ -0,0 +1,27 @@
+import type { GradeFact, GradeResult } from "./result.ts";
+
+/** What pass or fail is decided from. */
+export type Graded = Pick<GradeResult, "findings" | "facts" | "citations" | "contexts">;
+
+export type Verdict = { readonly pass: boolean; readonly failedBecause: readonly string[] };
+
+/** A condition over a count, written as the reason when the count goes past its limit: `facts.dropped 3 > 0`. */
+export const overLimit = (name: string, count: number, limit: number): string[] => (count > limit ? [`${name} ${String(count)} > ${String(limit)}`] : []);
+
+/** Facts that count against the output: those whose kind was not allowed to change. */
+export const counted = (facts: readonly GradeFact[]): number => facts.filter((fact) => !fact.allowed).length;
+
+export const verdictOf = (failedBecause: readonly string[]): Verdict => ({ pass: failedBecause.length === 0, failedBecause });
+
+/**
+ * The pass or fail with no `grade:` in chaff.yaml (spec §29.3): an error finding, a fact dropped or added against the
+ * reference, a quotation not found, or a fact no retrieved passage states fails the output. Warnings and info are rates to compare, not a verdict on one output.
+ */
+export const defaultVerdict = (graded: Graded): Verdict =>
+  verdictOf([
+    ...overLimit("findings.error", graded.findings.filter((finding) => finding.level === "error").length, 0),
+    ...overLimit("facts.dropped", counted(graded.facts?.dropped ?? []), 0),
+    ...overLimit("facts.added", counted(graded.facts?.added ?? []), 0),
+    ...overLimit("citations.failed", graded.citations?.failed.length ?? 0, 0),
+    ...overLimit("contexts.unsupported", counted(graded.contexts?.unsupported ?? []), 0),
+  ]);