chaffjs 0.19.0 → 0.21.0

package/src/grade/item.ts

@@ -0,0 +1,208 @@
+// One line of `chaff grade`'s input: a model's output and what to check it against. Pure: reads text, returns items
+// or what is wrong with them, so a harness that writes the file can be told which line to fix.
+import { resultKey } from "./result-name.ts";
+
+export type GradeCitation = { readonly source: string; readonly address: string; readonly quote: string };
+
+export type GradeItem = {
+  readonly id: string;
+  readonly output: string;
+  readonly reference?: string | undefined;
+  /** Each source's name and text. A citation names the one it quotes. */
+  readonly sources: Readonly<Record<string, string>>;
+  /** What the output quoted. Undefined when not given: chaff does not guess quotations from the output's text. */
+  readonly citations?: readonly GradeCitation[] | undefined;
+  readonly language?: string | undefined;
+  readonly genre?: string | undefined;
+  /** The retrieved passages the output was meant to rest on. Undefined when not given; empty when retrieval found none. */
+  readonly contexts?: readonly string[] | undefined;
+  /** The prompt, model or setting that produced this output. Outputs of several variants are compared on the same ids. */
+  readonly variant?: string | undefined;
+};
+
+/** The fields an item already has a meaning for: none of them can name a variant. */
+export const ITEM_FIELDS: readonly string[] = ["id", "output", "reference", "sources", "citations", "contexts", "language", "genre"];
+
+/** Which field names an output's variant, and whether every line must have it (the key was given on the command line). */
+export type VariantField = { readonly key: string; readonly required: boolean };
+
+export const DEFAULT_VARIANT_FIELD: VariantField = { key: "variant", required: false };
+
+export type ItemProblemKind =
+  | "not-json"
+  | "not-object"
+  | "no-id"
+  | "no-output"
+  | "not-string"
+  | "not-text-map"
+  | "not-citations"
+  | "citations-without-sources"
+  | "not-contexts"
+  | "duplicate-id"
+  | "no-variant"
+  | "unknown-source"
+  | "which-source"
+  | "unknown-language"
+  | "unknown-genre"
+  | "empty";
+
+/** Why a line cannot be graded. `detail`: the field, the value or the parser's message; `first`: the line an id was first on. */
+export type ItemProblem = { readonly kind: ItemProblemKind; readonly line: number; readonly detail?: string; readonly first?: number };
+
+/** What an item may name: the languages chaff can load and the genres it knows. */
+export type ItemVocabulary = { readonly isLanguage: (language: string) => boolean; readonly genres: readonly string[] };
+
+const isRecord = (value: unknown): value is Record<string, unknown> => typeof value === "object" && value !== null && !Array.isArray(value);
+
+const isTextMap = (value: unknown): value is Record<string, string> => isRecord(value) && Object.values(value).every((text) => typeof text === "string");
+
+type Parsed<T> = { readonly value: T } | { readonly problem: ItemProblem };
+
+const optionalString = (raw: Record<string, unknown>, field: string, line: number): Parsed<string | undefined> => {
+  const value = raw[field];
+  if (value === undefined || typeof value === "string") return { value };
+  return { problem: { kind: "not-string", line, detail: field } };
+};
+
+type RawCitation = { readonly source?: unknown; readonly address: string; readonly quote: string };
+
+const isRawCitation = (value: unknown): value is RawCitation =>
+  isRecord(value) && typeof value["address"] === "string" && typeof value["quote"] === "string" && ["string", "undefined"].includes(typeof value["source"]);
+
+/** A citation's source: the one named, or the only one given. */
+const citedSource = (citation: RawCitation, sources: Readonly<Record<string, string>>, line: number): Parsed<string> => {
+  const names = Object.keys(sources);
+  if (typeof citation.source === "string") {
+    return names.includes(citation.source) ? { value: citation.source } : { problem: { kind: "unknown-source", line, detail: citation.source } };
+  }
+  const [only] = names;
+  return names.length === 1 && only !== undefined ? { value: only } : { problem: { kind: "which-source", line, detail: citation.address } };
+};
+
+const citationsOf = (raw: unknown, sources: Readonly<Record<string, string>>, line: number): Parsed<GradeCitation[] | undefined> => {
+  if (raw === undefined) return { value: undefined };
+  if (!Array.isArray(raw) || !raw.every(isRawCitation)) return { problem: { kind: "not-citations", line } };
+  if (Object.keys(sources).length === 0) return { problem: { kind: "citations-without-sources", line } };
+  const entries: readonly RawCitation[] = raw;
+  const resolved = entries.map((citation) => ({ citation, source: citedSource(citation, sources, line) }));
+  const failed = resolved.find((entry) => "problem" in entry.source);
+  if (failed !== undefined && "problem" in failed.source) return failed.source;
+  return {
+    value: resolved.flatMap(({ citation, source }) => ("value" in source ? [{ source: source.value, address: citation.address, quote: citation.quote }] : [])),
+  };
+};
+
+/** The retrieved passages: an array of strings. An empty array is a retrieval that found nothing, not a missing field. */
+const contextsOf = (raw: unknown, line: number): Parsed<readonly string[] | undefined> => {
+  if (raw === undefined) return { value: undefined };
+  const valid = Array.isArray(raw) && raw.every((passage) => typeof passage === "string");
+  return valid ? { value: raw.map(String) } : { problem: { kind: "not-contexts", line } };
+};
+
+const sourcesOf = (raw: unknown, line: number): Parsed<Readonly<Record<string, string>>> => {
+  if (raw === undefined) return { value: {} };
+  return isTextMap(raw) ? { value: raw } : { problem: { kind: "not-text-map", line, detail: "sources" } };
+};
+
+/** The fields that are optional strings, read together so the first wrong one is reported. */
+const textFields = (raw: Record<string, unknown>, line: number): Parsed<{ reference?: string; language?: string; genre?: string }> => {
+  const read = (["reference", "language", "genre"] as const).map((field) => ({ field, parsed: optionalString(raw, field, line) }));
+  const failed = read.find((entry) => "problem" in entry.parsed);
+  if (failed !== undefined && "problem" in failed.parsed) return failed.parsed;
+  return {
+    value: Object.fromEntries(
+      read.flatMap((entry) => ("value" in entry.parsed && entry.parsed.value !== undefined ? [[entry.field, entry.parsed.value]] : [])),
+    ),
+  };
+};
+
+const vocabularyProblem = (fields: { language?: string; genre?: string }, vocabulary: ItemVocabulary, line: number): ItemProblem | undefined => {
+  if (fields.language !== undefined && !vocabulary.isLanguage(fields.language)) return { kind: "unknown-language", line, detail: fields.language };
+  if (fields.genre !== undefined && !vocabulary.genres.includes(fields.genre)) return { kind: "unknown-genre", line, detail: fields.genre };
+  return undefined;
+};
+
+/** The variant label: a non-empty string under the variant field, or none. */
+const variantOf = (raw: Record<string, unknown>, key: string, line: number): Parsed<string | undefined> => {
+  const value = raw[key];
+  if (value === undefined) return { value };
+  return typeof value === "string" && value !== "" ? { value } : { problem: { kind: "not-string", line, detail: key } };
+};
+
+const itemOf = (raw: Record<string, unknown>, line: number, vocabulary: ItemVocabulary, variantKey: string): Parsed<GradeItem> => {
+  const id = raw["id"];
+  if (typeof id !== "string" || id === "") return { problem: { kind: "no-id", line } };
+  const output = raw["output"];
+  if (typeof output !== "string") return { problem: { kind: "no-output", line } };
+  const fields = textFields(raw, line);
+  if ("problem" in fields) return fields;
+  const unknown = vocabularyProblem(fields.value, vocabulary, line);
+  if (unknown !== undefined) return { problem: unknown };
+  const variant = variantOf(raw, variantKey, line);
+  if ("problem" in variant) return variant;
+  const sources = sourcesOf(raw["sources"], line);
+  if ("problem" in sources) return sources;
+  const citations = citationsOf(raw["citations"], sources.value, line);
+  if ("problem" in citations) return citations;
+  const contexts = contextsOf(raw["contexts"], line);
+  if ("problem" in contexts) return contexts;
+  const passages = contexts.value === undefined ? {} : { contexts: contexts.value };
+  const labelled = variant.value === undefined ? {} : { variant: variant.value };
+  return { value: { id, output, ...fields.value, sources: sources.value, citations: citations.value, ...passages, ...labelled } };
+};
+
+/** One item from a value already parsed, as `grade()` receives it; `line` is 0 when it came from no file. */
+export const readItem = (raw: unknown, vocabulary: ItemVocabulary, line = 0): { readonly item: GradeItem } | { readonly problem: ItemProblem } => {
+  const read = isRecord(raw) ? itemOf(raw, line, vocabulary, DEFAULT_VARIANT_FIELD.key) : { problem: { kind: "not-object" as const, line } };
+  return "value" in read ? { item: read.value } : read;
+};
+
+const parseLine = (text: string, line: number, vocabulary: ItemVocabulary, variantKey: string): Parsed<GradeItem> => {
+  try {
+    const raw: unknown = JSON.parse(text);
+    return isRecord(raw) ? itemOf(raw, line, vocabulary, variantKey) : { problem: { kind: "not-object", line } };
+  } catch (error) {
+    return { problem: { kind: "not-json", line, detail: error instanceof Error ? error.message : String(error) } };
+  }
+};
+
+type ReadLine = { readonly item: GradeItem; readonly line: number };
+
+/** An id seen on an earlier line of the same variant: two outputs under one id could not be paired in an A/B comparison. */
+const duplicates = (items: readonly ReadLine[]): ItemProblem[] => {
+  const firstLine = new Map<string, number>();
+  return items.flatMap(({ item, line }) => {
+    const key = resultKey(item);
+    const first = firstLine.get(key);
+    if (first === undefined) firstLine.set(key, line);
+    const detail = item.variant === undefined ? item.id : `${item.id} (${item.variant})`;
+    return first === undefined ? [] : [{ kind: "duplicate-id", line, detail, first }];
+  });
+};
+
+/** Lines without a variant in a file that compares variants: their output would belong to no column. */
+const unlabelled = (items: readonly ReadLine[], field: VariantField): ItemProblem[] => {
+  const labelled = field.required || items.some(({ item }) => item.variant !== undefined);
+  if (!labelled) return [];
+  return items.filter(({ item }) => item.variant === undefined).map(({ line }) => ({ kind: "no-variant", line, detail: field.key }));
+};
+
+/** Every item of a JSONL text, or every problem found. A blank line is skipped; no item at all is a problem, not a clean run. */
+export const parseItems = (
+  text: string,
+  vocabulary: ItemVocabulary,
+  variantField: VariantField = DEFAULT_VARIANT_FIELD,
+): { readonly items: readonly GradeItem[] } | { readonly problems: readonly ItemProblem[] } => {
+  const lines = text.split(/\r?\n/u).map((body, index) => ({ body, line: index + 1 }));
+  const parsed = lines
+    .filter(({ body }) => body.trim() !== "")
+    .map(({ body, line }) => ({ line, parsed: parseLine(body, line, vocabulary, variantField.key) }));
+  const read = parsed.flatMap(({ line, parsed: entry }) => ("value" in entry ? [{ item: entry.value, line }] : []));
+  const problems = [
+    ...parsed.flatMap(({ parsed: entry }) => ("problem" in entry ? [entry.problem] : [])),
+    ...duplicates(read),
+    ...unlabelled(read, variantField),
+  ];
+  if (problems.length > 0) return { problems: problems.toSorted((left, right) => left.line - right.line) };
+  return read.length === 0 ? { problems: [{ kind: "empty", line: 0 }] } : { items: read.map((entry) => entry.item) };
+};

package/src/grade/order.ts

@@ -0,0 +1,10 @@
+/** Code-unit order, the same on every machine: a hash or a result must not depend on the locale it was made in. */
+export const compareText = (left: string, right: string): number => (left < right ? -1 : Number(left > right));
+
+/** An object whose keys are in compareText order. */
+export const sortedByKey = <T>(entries: readonly (readonly [string, T])[]): Record<string, T> =>
+  Object.fromEntries(entries.toSorted(([left], [right]) => compareText(left, right)));
+
+/** How many times each value occurs, by value in compareText order. */
+export const tally = (values: readonly string[]): Record<string, number> =>
+  sortedByKey(Object.entries(values.reduce<Record<string, number>>((counts, value) => ({ ...counts, [value]: (counts[value] ?? 0) + 1 }), {})));

package/src/grade/pairing.ts

@@ -0,0 +1,22 @@
+import type { LengthUnit } from "../plugin.ts";
+import { compareText } from "./order.ts";
+import type { GradeResult } from "./result.ts";
+import { summaryOf } from "./summary.ts";
+
+// What --baseline (two runs over time) and the variant comparison (several prompts or models on one run) share: outputs
+// matched by id within a variant, read the same way, and each rule's rate per group over the matched outputs. Pure.
+
+/** Read in one language and genre: otherwise not the same task graded the same way. */
+export const sameReading = (results: readonly GradeResult[]): boolean =>
+  results.every((result) => result.language === results[0]?.language && result.genre === results[0]?.genre);
+
+/** One rule's rate per 1,000 units of one unit, in each group, in the order the groups were given. 0 where it had none. */
+export type RateRow = { readonly rule: string; readonly unit: LengthUnit; readonly rates: readonly number[] };
+
+/** Each rule's rate in each group, per unit. `groups` hold the same tasks, so the rates compare like with like. */
+export const rateRows = (groups: readonly (readonly GradeResult[])[]): RateRow[] => {
+  const summaries = groups.map((group) => summaryOf(group).rules);
+  const rules = [...new Set(summaries.flatMap((rules) => Object.keys(rules)))].toSorted(compareText);
+  const units = [...new Set(groups.flatMap((group) => group.map((result) => result.size.unit)))].toSorted(compareText);
+  return rules.flatMap((rule) => units.map((unit) => ({ rule, unit, rates: summaries.map((rules) => rules[rule]?.rate[unit] ?? 0) })));
+};

package/src/grade/rates.ts

@@ -0,0 +1,39 @@
+import { lengthOf } from "../measure.ts";
+import { tally } from "./order.ts";
+import type { LengthUnit, ProseDocument } from "../plugin.ts";
+
+/** An output's length in the unit lint measures sentences in: characters for Japanese, words for English. */
+export type OutputSize = { readonly unit: LengthUnit; readonly value: number };
+
+/** Rates are findings per this many units, so a short and a long output are measured alike. */
+const RATE_BASE = 1000;
+
+const RATE_DECIMALS = 1;
+
+export const sizeOf = (doc: Pick<ProseDocument, "sentences" | "lengthUnit">): OutputSize => ({
+  unit: doc.lengthUnit,
+  value: doc.sentences.reduce((sum, sentence) => sum + lengthOf(sentence, doc.lengthUnit), 0),
+});
+
+const rounded = (value: number): number => {
+  const scale = 10 ** RATE_DECIMALS;
+  return Math.round(value * scale) / scale;
+};
+
+/** Findings per RATE_BASE units, unrounded: a limit is checked against this, so a rate just over it never rounds down to it. */
+export const exactRateOf = (count: number, size: number): number | undefined => (size > 0 ? (count * RATE_BASE) / size : undefined);
+
+/** Findings per RATE_BASE units, rounded to show. Undefined for an output with no length: a rate over nothing is not zero. */
+export const rateOf = (count: number, size: number): number | undefined => {
+  const exact = exactRateOf(count, size);
+  return exact === undefined ? undefined : rounded(exact);
+};
+
+/** Each rule's rate over one output. An output with no length gets none, its findings still listed. */
+export const ratesOf = (rules: readonly string[], size: OutputSize): Record<string, number> =>
+  Object.fromEntries(
+    Object.entries(tally(rules)).flatMap(([rule, count]) => {
+      const rate = rateOf(count, size.value);
+      return rate === undefined ? [] : [[rule, rate]];
+    }),
+  );

package/src/grade/render-baseline.ts

@@ -0,0 +1,74 @@
+import type { LengthUnit } from "../plugin.ts";
+import type { Comparison, ItemChange, RuleMovement } from "./baseline.ts";
+import type { BaselineText } from "./baseline-text.ts";
+import { block } from "./render.ts";
+
+const UNITS: readonly LengthUnit[] = ["word", "char"];
+
+const listed = (ids: readonly string[], text: BaselineText): string => (ids.length === 0 ? text.none : ids.join(", "));
+
+const signed = (value: number): string => `${value > 0 ? "+" : ""}${value.toFixed(1)}`;
+
+const movementLine = (movement: RuleMovement, width: number, text: BaselineText): string =>
+  [
+    `  ${movement.rule.padEnd(width)}  ${movement.before.toFixed(1)} → ${movement.after.toFixed(1)}  (${signed(movement.after - movement.before)})`,
+    ...(movement.increasedIn.length === 0 ? [] : [text.more(movement.increasedIn.join(", "))]),
+    ...(movement.decreasedIn.length === 0 ? [] : [text.fewer(movement.decreasedIn.join(", "))]),
+    ...(movement.inRubric ? [text.inRubric] : []),
+  ].join("  ");
+
+const movementBlocks = (comparison: Comparison, text: BaselineText): string[] =>
+  UNITS.flatMap((unit) => {
+    const movements = comparison.rules.filter((movement) => movement.unit === unit);
+    const width = movements.reduce((widest, movement) => Math.max(widest, movement.rule.length), 0);
+    return block(
+      text.ratesHeading(text.unit[unit]),
+      movements.map((movement) => movementLine(movement, width, text)),
+    );
+  });
+
+const factsOf = (facts: ItemChange["dropped"], text: BaselineText): string =>
+  facts.length === 0 ? text.none : facts.map((fact) => `${fact.kind} ${fact.text}`).join(", ");
+
+const itemLine = (change: ItemChange, text: BaselineText): string =>
+  [
+    `  ${change.id}: ${text.droppedAdded(factsOf(change.dropped, text), factsOf(change.added, text))}`,
+    ...(change.citations.length === 0 ? [] : [text.newCitations(change.citations.map((citation) => `${citation.source} ${citation.address}`).join(", "))]),
+  ].join(text.separator);
+
+const unpairedLines = (comparison: Comparison, text: BaselineText): string[] =>
+  comparison.onlyBefore.length + comparison.onlyAfter.length + comparison.readOtherwise.length === 0
+    ? []
+    : [text.unpaired(listed(comparison.onlyBefore, text), listed(comparison.onlyAfter, text), listed(comparison.readOtherwise, text))];
+
+const regressionLines = (comparison: Comparison, text: BaselineText): string[] =>
+  comparison.regressions.length === 0
+    ? ["", text.noRegression]
+    : block(
+        text.regressionsHeading(comparison.regressions.length),
+        comparison.regressions.map((reason) => `  ✗ ${reason}`),
+      );
+
+/** For a person: what was paired, how each rule moved and where, what newly failed or passed, new facts, and the verdict. */
+export const renderComparison = (path: string, comparison: Comparison, text: BaselineText): string =>
+  [
+    text.heading(path, comparison.paired),
+    ...unpairedLines(comparison, text),
+    ...movementBlocks(comparison, text),
+    "",
+    text.newlyFailed(listed(comparison.newlyFailed, text)),
+    text.newlyPassed(listed(comparison.newlyPassed, text)),
+    ...(comparison.penalty === undefined ? [] : [text.penalty(comparison.penalty.before, comparison.penalty.after)]),
+    ...block(
+      text.newFactsHeading,
+      comparison.items.map((change) => itemLine(change, text)),
+    ),
+    ...regressionLines(comparison, text),
+  ].join("\n");
+
+/** For a CI log: one line per regression, then the verdict. */
+export const renderComparisonCompact = (comparison: Comparison, text: BaselineText): string =>
+  [
+    ...comparison.regressions.map((reason) => `regression\t${reason}`),
+    comparison.regressions.length === 0 ? text.noRegression : text.regressionsHeading(comparison.regressions.length),
+  ].join("\n");

package/src/grade/render-markdown.ts

@@ -0,0 +1,62 @@
+import type { LengthUnit } from "../plugin.ts";
+import type { Comparison } from "./baseline.ts";
+import { factsLine } from "./render.ts";
+import { resultName } from "./result-name.ts";
+import { markdownTable, renderVariantsMarkdown } from "./render-variants.ts";
+import type { GradeSummary } from "./summary.ts";
+import type { GradeText } from "./text.ts";
+import type { VariantComparison } from "./variants.ts";
+import type { VariantText } from "./variants-text.ts";
+
+// `chaff grade --format markdown`: a run's summary as a pull request comment. The same numbers as the screen.
+
+const UNITS: readonly LengthUnit[] = ["word", "char"];
+
+export type MarkdownTexts = { readonly grade: GradeText; readonly variants: VariantText };
+
+/** One rate table per unit, as on the screen: a rate per 1,000 characters and one per 1,000 words are not on one scale. */
+const rateTables = (summary: GradeSummary, text: GradeText): string[] =>
+  UNITS.flatMap((unit) => {
+    const rows = Object.entries(summary.rules).flatMap(([rule, entry]) => {
+      const rate = entry.rate[unit];
+      return rate === undefined ? [] : [[rule, rate.toFixed(1), text.outputsWith(entry.outputs)]];
+    });
+    return rows.length === 0 ? [] : ["", `**${text.ratesHeading(text.unit[unit])}**`, "", ...markdownTable([["", "", ""], ...rows])];
+  });
+
+const failedLines = (summary: GradeSummary, text: VariantText): string[] =>
+  summary.failed.length === 0
+    ? []
+    : ["", `**${text.failedHeading}**`, "", ...summary.failed.map((entry) => `- \`${resultName(entry)}\`: ${entry.failedBecause.join(", ")}`)];
+
+const baselineLines = (comparison: Comparison | undefined, text: VariantText): string[] => {
+  if (comparison === undefined) return [];
+  if (comparison.regressions.length === 0) return ["", text.noRegression];
+  return ["", `**${text.regressionsHeading(comparison.regressions.length)}**`, "", ...comparison.regressions.map((reason) => `- ${reason}`)];
+};
+
+const stampLine = (summary: GradeSummary, text: VariantText): string[] =>
+  summary.stamp === undefined
+    ? []
+    : ["", `<sub>${text.stamp}: ${summary.stamp.chaff} · rules ${summary.stamp.rules} · settings ${summary.stamp.settings}</sub>`];
+
+/** The totals, the failed outputs, the variants side by side (or the rates when there are none), the regressions, and the stamp. */
+export const renderMarkdown = (
+  path: string,
+  summary: GradeSummary,
+  compared: { readonly variants?: VariantComparison | undefined; readonly baseline?: Comparison | undefined },
+  texts: MarkdownTexts,
+): string =>
+  [
+    `## ${texts.variants.runHeading(path)}`,
+    "",
+    texts.variants.totals(summary.total, summary.passed),
+    ...failedLines(summary, texts.variants),
+    ...(compared.variants === undefined ? rateTables(summary, texts.grade) : ["", renderVariantsMarkdown(compared.variants, texts.variants)]),
+    "",
+    factsLine(summary, texts.grade),
+    "",
+    texts.grade.citations(summary.citations.checked, summary.citations.failed),
+    ...baselineLines(compared.baseline, texts.variants),
+    ...stampLine(summary, texts.variants),
+  ].join("\n");

package/src/grade/render-variants.ts

@@ -0,0 +1,115 @@
+import type { LengthUnit } from "../plugin.ts";
+import { widthOf } from "../render/rules-table.ts";
+import { block, table } from "./render.ts";
+import type { Disagreement, VariantColumn, VariantComparison } from "./variants.ts";
+import type { VariantText } from "./variants-text.ts";
+
+// The variant comparison for a person (a screen table), for a CI log (one line per disagreement), and for a pull request
+// comment (Markdown tables). Every form is built from the same rows, so they never say different things.
+
+const UNITS: readonly LengthUnit[] = ["word", "char"];
+
+type Rows = readonly (readonly string[])[];
+
+const passedCell = (column: VariantColumn, text: VariantText): string =>
+  text.passedCell(column.passed, column.outputs, column.passRate === undefined ? "–" : `${String(column.passRate)}%`);
+
+/** The header and one row per measure, a column per variant. */
+const measureRows = (comparison: VariantComparison, text: VariantText): Rows => {
+  const { columns } = comparison;
+  const scored = columns.some((column) => column.penalty !== undefined);
+  const grounded = columns.some((column) => column.contexts !== undefined);
+  return [
+    ["", ...comparison.variants],
+    [text.passed, ...columns.map((column) => passedCell(column, text))],
+    [text.factsDropped, ...columns.map((column) => String(column.facts.dropped))],
+    [text.factsAdded, ...columns.map((column) => String(column.facts.added))],
+    [text.citationsFailed, ...columns.map((column) => text.citationsCell(column.citations.failed, column.citations.checked))],
+    ...(grounded
+      ? [[text.unsupportedFacts, ...columns.map((column) => text.citationsCell(column.contexts?.unsupported ?? 0, column.contexts?.checked ?? 0))]]
+      : []),
+    ...(scored ? [[text.penalty, ...columns.map((column) => String(column.penalty ?? 0))]] : []),
+  ];
+};
+
+/** Each rule's rate in each variant, for one unit; empty when no rule fired in that unit. */
+const rateRowsOf = (comparison: VariantComparison, unit: LengthUnit): Rows =>
+  comparison.rules.filter((row) => row.unit === unit).map((row) => [row.rule, ...comparison.variants.map((variant) => (row.rates[variant] ?? 0).toFixed(1))]);
+
+const disagreementLine = (entry: Disagreement, text: VariantText): string =>
+  text.disagreement(
+    entry.id,
+    entry.passedIn.join(", "),
+    entry.failedIn.map((failed) => text.failedIn(failed.variant, failed.failedBecause.join(", "))).join(text.separator),
+  );
+
+const notComparedLines = (comparison: VariantComparison, text: VariantText): string[] => [
+  ...comparison.missing.map((entry) => text.missing(entry.id, entry.missingFrom.join(", "))),
+  ...(comparison.readOtherwise.length === 0 ? [] : [text.readOtherwise(comparison.readOtherwise.join(", "))]),
+];
+
+const disagreementBlock = (comparison: VariantComparison, text: VariantText, bullet: string): string[] =>
+  comparison.disagreements.length === 0
+    ? ["", text.noDisagreement]
+    : block(
+        text.disagreeHeading(comparison.disagreements.length),
+        comparison.disagreements.map((entry) => `${bullet}${disagreementLine(entry, text)}`),
+      );
+
+/** For a person: the measures side by side, the rates, where pass or fail differs, and what could not be compared. */
+export const renderVariants = (comparison: VariantComparison, text: VariantText): string =>
+  [
+    text.heading(comparison.compared.length, comparison.variants.length),
+    ...table(measureRows(comparison, text), widthOf),
+    ...UNITS.flatMap((unit) => {
+      const rows = rateRowsOf(comparison, unit);
+      return rows.length === 0 ? [] : block(text.ratesHeading(text.unit[unit]), table([["", ...comparison.variants], ...rows], widthOf));
+    }),
+    ...disagreementBlock(comparison, text, "  ✗ "),
+    ...block(
+      text.notComparedHeading,
+      notComparedLines(comparison, text).map((line) => `  ${line}`),
+    ),
+  ].join("\n");
+
+/** For a CI log: one line per output where pass or fail differs, then the count. */
+export const renderVariantsCompact = (comparison: VariantComparison, text: VariantText): string =>
+  [
+    ...comparison.disagreements.map(
+      (entry) => `disagree\t${entry.id}\tpass ${entry.passedIn.join(",")}\tfail ${entry.failedIn.map((failed) => failed.variant).join(",")}`,
+    ),
+    comparison.disagreements.length === 0 ? text.noDisagreement : text.disagreeHeading(comparison.disagreements.length),
+  ].join("\n");
+
+/** A table cell: a pipe would end the cell, and a line break the row. */
+const cell = (value: string): string => value.replaceAll("|", "\\|").replaceAll(/\r?\n/gu, " ");
+
+export const markdownTable = (rows: Rows): string[] => {
+  const [header, ...body] = rows;
+  if (header === undefined) return [];
+  const line = (cells: readonly string[]): string => `| ${cells.map(cell).join(" | ")} |`;
+  return [line(header), line(header.map(() => "---")), ...body.map(line)];
+};
+
+/** For a pull request comment: the same measures and rates as Markdown tables, and where pass or fail differs. */
+export const renderVariantsMarkdown = (comparison: VariantComparison, text: VariantText): string =>
+  [
+    `### ${text.heading(comparison.compared.length, comparison.variants.length)}`,
+    "",
+    ...markdownTable(measureRows(comparison, text)),
+    ...UNITS.flatMap((unit) => {
+      const rows = rateRowsOf(comparison, unit);
+      return rows.length === 0 ? [] : ["", `**${text.ratesHeading(text.unit[unit])}**`, "", ...markdownTable([["", ...comparison.variants], ...rows])];
+    }),
+    ...(comparison.disagreements.length === 0
+      ? ["", text.noDisagreement]
+      : [
+          "",
+          `**${text.disagreeHeading(comparison.disagreements.length)}**`,
+          "",
+          ...comparison.disagreements.map((entry) => `- ${disagreementLine(entry, text)}`),
+        ]),
+    ...(notComparedLines(comparison, text).length === 0
+      ? []
+      : ["", `**${text.notComparedHeading}**`, "", ...notComparedLines(comparison, text).map((line) => `- ${line}`)]),
+  ].join("\n");

package/src/grade/render.ts

@@ -0,0 +1,84 @@
+import type { LengthUnit } from "../plugin.ts";
+import type { GradeResult } from "./result.ts";
+import { resultName } from "./result-name.ts";
+import type { GradeSummary } from "./summary.ts";
+import type { GradeText } from "./text.ts";
+
+const UNITS: readonly LengthUnit[] = ["word", "char"];
+
+export const block = (heading: string, lines: readonly string[]): string[] => (lines.length === 0 ? [] : ["", heading, ...lines]);
+
+type Measure = (cell: string) => number;
+
+const lengthOfCell: Measure = (cell) => cell.length;
+
+const padCell = (cell: string, width: number, measure: Measure): string => cell + " ".repeat(Math.max(0, width - measure(cell)));
+
+const padded = (cells: readonly string[], widths: readonly number[], measure: Measure): string =>
+  `  ${cells.map((cell, index) => padCell(cell, widths[index] ?? 0, measure)).join("  ")}`.trimEnd();
+
+/** Columns as wide as their widest cell, each cell measured by `measure`. */
+export const table = (rows: readonly (readonly string[])[], measure: Measure = lengthOfCell): string[] => {
+  const widths = rows.reduce<number[]>((max, row) => row.map((cell, index) => Math.max(max[index] ?? 0, measure(cell))), []);
+  return rows.map((row) => padded(row, widths, measure));
+};
+
+/** One table per unit: a rate per 1,000 characters and one per 1,000 words are not on one scale. */
+const rateBlocks = (summary: GradeSummary, text: GradeText): string[] =>
+  UNITS.flatMap((unit) => {
+    const rows = Object.entries(summary.rules).flatMap(([rule, entry]) => {
+      const rate = entry.rate[unit];
+      return rate === undefined ? [] : [[rule, rate.toFixed(1), text.outputsWith(entry.outputs)]];
+    });
+    return block(text.ratesHeading(text.unit[unit]), table(rows));
+  });
+
+const kindsLine = (counts: Readonly<Record<string, number>>): string =>
+  Object.entries(counts)
+    .map(([kind, count]) => `${kind} ${String(count)}`)
+    .join(", ");
+
+const sum = (counts: Readonly<Record<string, number>>): number => Object.values(counts).reduce((total, count) => total + count, 0);
+
+export const factsLine = (summary: GradeSummary, text: GradeText): string =>
+  text.facts(sum(summary.facts.dropped), sum(summary.facts.added), kindsLine(summary.facts.dropped), kindsLine(summary.facts.added));
+
+const contextsLine = (summary: GradeSummary, text: GradeText): string[] =>
+  summary.contexts === undefined
+    ? []
+    : [text.contexts(summary.contexts.outputs, summary.contexts.checked, sum(summary.contexts.unsupported), kindsLine(summary.contexts.unsupported))];
+
+const stampLines = (summary: GradeSummary, text: GradeText): string[] =>
+  summary.stamp === undefined ? [] : ["", `${text.stamp}: ${summary.stamp.chaff}`, `  rules ${summary.stamp.rules}`, `  settings ${summary.stamp.settings}`];
+
+/** For a person: the totals, each failed output with why, the rates, facts and quotations, what did not run, and the stamp. */
+export const renderSummary = (path: string, summary: GradeSummary, text: GradeText): string =>
+  [
+    text.totals(path, summary.total, summary.passed),
+    ...block(
+      text.failedHeading,
+      summary.failed.map((entry) => `  ✗ ${resultName(entry)}: ${entry.failedBecause.join(", ")}`),
+    ),
+    ...rateBlocks(summary, text),
+    "",
+    factsLine(summary, text),
+    text.citations(summary.citations.checked, summary.citations.failed),
+    ...contextsLine(summary, text),
+    ...(summary.penalty === undefined ? [] : [text.penalty(summary.penalty)]),
+    ...block(text.notRunHeading(summary.notRun.length), table(summary.notRun.map((entry) => [entry.rule, text.inOutputs(entry.outputs), entry.reason]))),
+    ...stampLines(summary, text),
+  ].join("\n");
+
+/** For grep and a CI log: one line per output, its id, pass or fail and why, then the totals. */
+export const renderCompact = (path: string, results: readonly GradeResult[], summary: GradeSummary, text: GradeText): string =>
+  [
+    ...results.map((result) =>
+      [
+        resultName(result),
+        result.pass ? "pass" : "fail",
+        ...(result.score === undefined ? [] : [`penalty ${String(result.score.penalty)}`]),
+        ...(result.pass ? [] : [result.failedBecause.join(", ")]),
+      ].join("\t"),
+    ),
+    text.totals(path, summary.total, summary.passed),
+  ].join("\n");

package/src/grade/result-name.ts

@@ -0,0 +1,9 @@
+import type { GradeResult } from "./result.ts";
+
+/** The same task: an id within its variant. Two variants' outputs for one id are different outputs of one task. */
+/** A JSON pair, so no character a label or id may hold can make two pairs one key. */
+export const resultKey = (result: Pick<GradeResult, "id" | "variant">): string => JSON.stringify([result.variant ?? null, result.id]);
+
+/** An output as a person reads its name: the id, and its variant when it has one. */
+export const resultName = (result: Pick<GradeResult, "id" | "variant">): string =>
+  result.variant === undefined ? result.id : `${result.id} (${result.variant})`;