chaffjs 0.18.0 → 0.20.0

package/src/grade/render-baseline.ts

@@ -0,0 +1,74 @@
+import type { LengthUnit } from "../plugin.ts";
+import type { Comparison, ItemChange, RuleMovement } from "./baseline.ts";
+import type { BaselineText } from "./baseline-text.ts";
+import { block } from "./render.ts";
+
+const UNITS: readonly LengthUnit[] = ["word", "char"];
+
+const listed = (ids: readonly string[], text: BaselineText): string => (ids.length === 0 ? text.none : ids.join(", "));
+
+const signed = (value: number): string => `${value > 0 ? "+" : ""}${value.toFixed(1)}`;
+
+const movementLine = (movement: RuleMovement, width: number, text: BaselineText): string =>
+  [
+    `  ${movement.rule.padEnd(width)}  ${movement.before.toFixed(1)} → ${movement.after.toFixed(1)}  (${signed(movement.after - movement.before)})`,
+    ...(movement.increasedIn.length === 0 ? [] : [text.more(movement.increasedIn.join(", "))]),
+    ...(movement.decreasedIn.length === 0 ? [] : [text.fewer(movement.decreasedIn.join(", "))]),
+    ...(movement.inRubric ? [text.inRubric] : []),
+  ].join("  ");
+
+const movementBlocks = (comparison: Comparison, text: BaselineText): string[] =>
+  UNITS.flatMap((unit) => {
+    const movements = comparison.rules.filter((movement) => movement.unit === unit);
+    const width = movements.reduce((widest, movement) => Math.max(widest, movement.rule.length), 0);
+    return block(
+      text.ratesHeading(text.unit[unit]),
+      movements.map((movement) => movementLine(movement, width, text)),
+    );
+  });
+
+const factsOf = (facts: ItemChange["dropped"], text: BaselineText): string =>
+  facts.length === 0 ? text.none : facts.map((fact) => `${fact.kind} ${fact.text}`).join(", ");
+
+const itemLine = (change: ItemChange, text: BaselineText): string =>
+  [
+    `  ${change.id}: ${text.droppedAdded(factsOf(change.dropped, text), factsOf(change.added, text))}`,
+    ...(change.citations.length === 0 ? [] : [text.newCitations(change.citations.map((citation) => `${citation.source} ${citation.address}`).join(", "))]),
+  ].join(text.separator);
+
+const unpairedLines = (comparison: Comparison, text: BaselineText): string[] =>
+  comparison.onlyBefore.length + comparison.onlyAfter.length + comparison.readOtherwise.length === 0
+    ? []
+    : [text.unpaired(listed(comparison.onlyBefore, text), listed(comparison.onlyAfter, text), listed(comparison.readOtherwise, text))];
+
+const regressionLines = (comparison: Comparison, text: BaselineText): string[] =>
+  comparison.regressions.length === 0
+    ? ["", text.noRegression]
+    : block(
+        text.regressionsHeading(comparison.regressions.length),
+        comparison.regressions.map((reason) => `  ✗ ${reason}`),
+      );
+
+/** For a person: what was paired, how each rule moved and where, what newly failed or passed, new facts, and the verdict. */
+export const renderComparison = (path: string, comparison: Comparison, text: BaselineText): string =>
+  [
+    text.heading(path, comparison.paired),
+    ...unpairedLines(comparison, text),
+    ...movementBlocks(comparison, text),
+    "",
+    text.newlyFailed(listed(comparison.newlyFailed, text)),
+    text.newlyPassed(listed(comparison.newlyPassed, text)),
+    ...(comparison.penalty === undefined ? [] : [text.penalty(comparison.penalty.before, comparison.penalty.after)]),
+    ...block(
+      text.newFactsHeading,
+      comparison.items.map((change) => itemLine(change, text)),
+    ),
+    ...regressionLines(comparison, text),
+  ].join("\n");
+
+/** For a CI log: one line per regression, then the verdict. */
+export const renderComparisonCompact = (comparison: Comparison, text: BaselineText): string =>
+  [
+    ...comparison.regressions.map((reason) => `regression\t${reason}`),
+    comparison.regressions.length === 0 ? text.noRegression : text.regressionsHeading(comparison.regressions.length),
+  ].join("\n");

package/src/grade/render-markdown.ts

@@ -0,0 +1,62 @@
+import type { LengthUnit } from "../plugin.ts";
+import type { Comparison } from "./baseline.ts";
+import { factsLine } from "./render.ts";
+import { resultName } from "./result-name.ts";
+import { markdownTable, renderVariantsMarkdown } from "./render-variants.ts";
+import type { GradeSummary } from "./summary.ts";
+import type { GradeText } from "./text.ts";
+import type { VariantComparison } from "./variants.ts";
+import type { VariantText } from "./variants-text.ts";
+
+// `chaff grade --format markdown`: a run's summary as a pull request comment. The same numbers as the screen.
+
+const UNITS: readonly LengthUnit[] = ["word", "char"];
+
+export type MarkdownTexts = { readonly grade: GradeText; readonly variants: VariantText };
+
+/** One rate table per unit, as on the screen: a rate per 1,000 characters and one per 1,000 words are not on one scale. */
+const rateTables = (summary: GradeSummary, text: GradeText): string[] =>
+  UNITS.flatMap((unit) => {
+    const rows = Object.entries(summary.rules).flatMap(([rule, entry]) => {
+      const rate = entry.rate[unit];
+      return rate === undefined ? [] : [[rule, rate.toFixed(1), text.outputsWith(entry.outputs)]];
+    });
+    return rows.length === 0 ? [] : ["", `**${text.ratesHeading(text.unit[unit])}**`, "", ...markdownTable([["", "", ""], ...rows])];
+  });
+
+const failedLines = (summary: GradeSummary, text: VariantText): string[] =>
+  summary.failed.length === 0
+    ? []
+    : ["", `**${text.failedHeading}**`, "", ...summary.failed.map((entry) => `- \`${resultName(entry)}\`: ${entry.failedBecause.join(", ")}`)];
+
+const baselineLines = (comparison: Comparison | undefined, text: VariantText): string[] => {
+  if (comparison === undefined) return [];
+  if (comparison.regressions.length === 0) return ["", text.noRegression];
+  return ["", `**${text.regressionsHeading(comparison.regressions.length)}**`, "", ...comparison.regressions.map((reason) => `- ${reason}`)];
+};
+
+const stampLine = (summary: GradeSummary, text: VariantText): string[] =>
+  summary.stamp === undefined
+    ? []
+    : ["", `<sub>${text.stamp}: ${summary.stamp.chaff} · rules ${summary.stamp.rules} · settings ${summary.stamp.settings}</sub>`];
+
+/** The totals, the failed outputs, the variants side by side (or the rates when there are none), the regressions, and the stamp. */
+export const renderMarkdown = (
+  path: string,
+  summary: GradeSummary,
+  compared: { readonly variants?: VariantComparison | undefined; readonly baseline?: Comparison | undefined },
+  texts: MarkdownTexts,
+): string =>
+  [
+    `## ${texts.variants.runHeading(path)}`,
+    "",
+    texts.variants.totals(summary.total, summary.passed),
+    ...failedLines(summary, texts.variants),
+    ...(compared.variants === undefined ? rateTables(summary, texts.grade) : ["", renderVariantsMarkdown(compared.variants, texts.variants)]),
+    "",
+    factsLine(summary, texts.grade),
+    "",
+    texts.grade.citations(summary.citations.checked, summary.citations.failed),
+    ...baselineLines(compared.baseline, texts.variants),
+    ...stampLine(summary, texts.variants),
+  ].join("\n");

package/src/grade/render-variants.ts

@@ -0,0 +1,115 @@
+import type { LengthUnit } from "../plugin.ts";
+import { widthOf } from "../render/rules-table.ts";
+import { block, table } from "./render.ts";
+import type { Disagreement, VariantColumn, VariantComparison } from "./variants.ts";
+import type { VariantText } from "./variants-text.ts";
+
+// The variant comparison for a person (a screen table), for a CI log (one line per disagreement), and for a pull request
+// comment (Markdown tables). Every form is built from the same rows, so they never say different things.
+
+const UNITS: readonly LengthUnit[] = ["word", "char"];
+
+type Rows = readonly (readonly string[])[];
+
+const passedCell = (column: VariantColumn, text: VariantText): string =>
+  text.passedCell(column.passed, column.outputs, column.passRate === undefined ? "–" : `${String(column.passRate)}%`);
+
+/** The header and one row per measure, a column per variant. */
+const measureRows = (comparison: VariantComparison, text: VariantText): Rows => {
+  const { columns } = comparison;
+  const scored = columns.some((column) => column.penalty !== undefined);
+  const grounded = columns.some((column) => column.contexts !== undefined);
+  return [
+    ["", ...comparison.variants],
+    [text.passed, ...columns.map((column) => passedCell(column, text))],
+    [text.factsDropped, ...columns.map((column) => String(column.facts.dropped))],
+    [text.factsAdded, ...columns.map((column) => String(column.facts.added))],
+    [text.citationsFailed, ...columns.map((column) => text.citationsCell(column.citations.failed, column.citations.checked))],
+    ...(grounded
+      ? [[text.unsupportedFacts, ...columns.map((column) => text.citationsCell(column.contexts?.unsupported ?? 0, column.contexts?.checked ?? 0))]]
+      : []),
+    ...(scored ? [[text.penalty, ...columns.map((column) => String(column.penalty ?? 0))]] : []),
+  ];
+};
+
+/** Each rule's rate in each variant, for one unit; empty when no rule fired in that unit. */
+const rateRowsOf = (comparison: VariantComparison, unit: LengthUnit): Rows =>
+  comparison.rules.filter((row) => row.unit === unit).map((row) => [row.rule, ...comparison.variants.map((variant) => (row.rates[variant] ?? 0).toFixed(1))]);
+
+const disagreementLine = (entry: Disagreement, text: VariantText): string =>
+  text.disagreement(
+    entry.id,
+    entry.passedIn.join(", "),
+    entry.failedIn.map((failed) => text.failedIn(failed.variant, failed.failedBecause.join(", "))).join(text.separator),
+  );
+
+const notComparedLines = (comparison: VariantComparison, text: VariantText): string[] => [
+  ...comparison.missing.map((entry) => text.missing(entry.id, entry.missingFrom.join(", "))),
+  ...(comparison.readOtherwise.length === 0 ? [] : [text.readOtherwise(comparison.readOtherwise.join(", "))]),
+];
+
+const disagreementBlock = (comparison: VariantComparison, text: VariantText, bullet: string): string[] =>
+  comparison.disagreements.length === 0
+    ? ["", text.noDisagreement]
+    : block(
+        text.disagreeHeading(comparison.disagreements.length),
+        comparison.disagreements.map((entry) => `${bullet}${disagreementLine(entry, text)}`),
+      );
+
+/** For a person: the measures side by side, the rates, where pass or fail differs, and what could not be compared. */
+export const renderVariants = (comparison: VariantComparison, text: VariantText): string =>
+  [
+    text.heading(comparison.compared.length, comparison.variants.length),
+    ...table(measureRows(comparison, text), widthOf),
+    ...UNITS.flatMap((unit) => {
+      const rows = rateRowsOf(comparison, unit);
+      return rows.length === 0 ? [] : block(text.ratesHeading(text.unit[unit]), table([["", ...comparison.variants], ...rows], widthOf));
+    }),
+    ...disagreementBlock(comparison, text, "  ✗ "),
+    ...block(
+      text.notComparedHeading,
+      notComparedLines(comparison, text).map((line) => `  ${line}`),
+    ),
+  ].join("\n");
+
+/** For a CI log: one line per output where pass or fail differs, then the count. */
+export const renderVariantsCompact = (comparison: VariantComparison, text: VariantText): string =>
+  [
+    ...comparison.disagreements.map(
+      (entry) => `disagree\t${entry.id}\tpass ${entry.passedIn.join(",")}\tfail ${entry.failedIn.map((failed) => failed.variant).join(",")}`,
+    ),
+    comparison.disagreements.length === 0 ? text.noDisagreement : text.disagreeHeading(comparison.disagreements.length),
+  ].join("\n");
+
+/** A table cell: a pipe would end the cell, and a line break the row. */
+const cell = (value: string): string => value.replaceAll("|", "\\|").replaceAll(/\r?\n/gu, " ");
+
+export const markdownTable = (rows: Rows): string[] => {
+  const [header, ...body] = rows;
+  if (header === undefined) return [];
+  const line = (cells: readonly string[]): string => `| ${cells.map(cell).join(" | ")} |`;
+  return [line(header), line(header.map(() => "---")), ...body.map(line)];
+};
+
+/** For a pull request comment: the same measures and rates as Markdown tables, and where pass or fail differs. */
+export const renderVariantsMarkdown = (comparison: VariantComparison, text: VariantText): string =>
+  [
+    `### ${text.heading(comparison.compared.length, comparison.variants.length)}`,
+    "",
+    ...markdownTable(measureRows(comparison, text)),
+    ...UNITS.flatMap((unit) => {
+      const rows = rateRowsOf(comparison, unit);
+      return rows.length === 0 ? [] : ["", `**${text.ratesHeading(text.unit[unit])}**`, "", ...markdownTable([["", ...comparison.variants], ...rows])];
+    }),
+    ...(comparison.disagreements.length === 0
+      ? ["", text.noDisagreement]
+      : [
+          "",
+          `**${text.disagreeHeading(comparison.disagreements.length)}**`,
+          "",
+          ...comparison.disagreements.map((entry) => `- ${disagreementLine(entry, text)}`),
+        ]),
+    ...(notComparedLines(comparison, text).length === 0
+      ? []
+      : ["", `**${text.notComparedHeading}**`, "", ...notComparedLines(comparison, text).map((line) => `- ${line}`)]),
+  ].join("\n");

package/src/grade/render.ts

@@ -0,0 +1,84 @@
+import type { LengthUnit } from "../plugin.ts";
+import type { GradeResult } from "./result.ts";
+import { resultName } from "./result-name.ts";
+import type { GradeSummary } from "./summary.ts";
+import type { GradeText } from "./text.ts";
+
+const UNITS: readonly LengthUnit[] = ["word", "char"];
+
+export const block = (heading: string, lines: readonly string[]): string[] => (lines.length === 0 ? [] : ["", heading, ...lines]);
+
+type Measure = (cell: string) => number;
+
+const lengthOfCell: Measure = (cell) => cell.length;
+
+const padCell = (cell: string, width: number, measure: Measure): string => cell + " ".repeat(Math.max(0, width - measure(cell)));
+
+const padded = (cells: readonly string[], widths: readonly number[], measure: Measure): string =>
+  `  ${cells.map((cell, index) => padCell(cell, widths[index] ?? 0, measure)).join("  ")}`.trimEnd();
+
+/** Columns as wide as their widest cell, each cell measured by `measure`. */
+export const table = (rows: readonly (readonly string[])[], measure: Measure = lengthOfCell): string[] => {
+  const widths = rows.reduce<number[]>((max, row) => row.map((cell, index) => Math.max(max[index] ?? 0, measure(cell))), []);
+  return rows.map((row) => padded(row, widths, measure));
+};
+
+/** One table per unit: a rate per 1,000 characters and one per 1,000 words are not on one scale. */
+const rateBlocks = (summary: GradeSummary, text: GradeText): string[] =>
+  UNITS.flatMap((unit) => {
+    const rows = Object.entries(summary.rules).flatMap(([rule, entry]) => {
+      const rate = entry.rate[unit];
+      return rate === undefined ? [] : [[rule, rate.toFixed(1), text.outputsWith(entry.outputs)]];
+    });
+    return block(text.ratesHeading(text.unit[unit]), table(rows));
+  });
+
+const kindsLine = (counts: Readonly<Record<string, number>>): string =>
+  Object.entries(counts)
+    .map(([kind, count]) => `${kind} ${String(count)}`)
+    .join(", ");
+
+const sum = (counts: Readonly<Record<string, number>>): number => Object.values(counts).reduce((total, count) => total + count, 0);
+
+export const factsLine = (summary: GradeSummary, text: GradeText): string =>
+  text.facts(sum(summary.facts.dropped), sum(summary.facts.added), kindsLine(summary.facts.dropped), kindsLine(summary.facts.added));
+
+const contextsLine = (summary: GradeSummary, text: GradeText): string[] =>
+  summary.contexts === undefined
+    ? []
+    : [text.contexts(summary.contexts.outputs, summary.contexts.checked, sum(summary.contexts.unsupported), kindsLine(summary.contexts.unsupported))];
+
+const stampLines = (summary: GradeSummary, text: GradeText): string[] =>
+  summary.stamp === undefined ? [] : ["", `${text.stamp}: ${summary.stamp.chaff}`, `  rules ${summary.stamp.rules}`, `  settings ${summary.stamp.settings}`];
+
+/** For a person: the totals, each failed output with why, the rates, facts and quotations, what did not run, and the stamp. */
+export const renderSummary = (path: string, summary: GradeSummary, text: GradeText): string =>
+  [
+    text.totals(path, summary.total, summary.passed),
+    ...block(
+      text.failedHeading,
+      summary.failed.map((entry) => `  ✗ ${resultName(entry)}: ${entry.failedBecause.join(", ")}`),
+    ),
+    ...rateBlocks(summary, text),
+    "",
+    factsLine(summary, text),
+    text.citations(summary.citations.checked, summary.citations.failed),
+    ...contextsLine(summary, text),
+    ...(summary.penalty === undefined ? [] : [text.penalty(summary.penalty)]),
+    ...block(text.notRunHeading(summary.notRun.length), table(summary.notRun.map((entry) => [entry.rule, text.inOutputs(entry.outputs), entry.reason]))),
+    ...stampLines(summary, text),
+  ].join("\n");
+
+/** For grep and a CI log: one line per output, its id, pass or fail and why, then the totals. */
+export const renderCompact = (path: string, results: readonly GradeResult[], summary: GradeSummary, text: GradeText): string =>
+  [
+    ...results.map((result) =>
+      [
+        resultName(result),
+        result.pass ? "pass" : "fail",
+        ...(result.score === undefined ? [] : [`penalty ${String(result.score.penalty)}`]),
+        ...(result.pass ? [] : [result.failedBecause.join(", ")]),
+      ].join("\t"),
+    ),
+    text.totals(path, summary.total, summary.passed),
+  ].join("\n");

package/src/grade/result-name.ts

@@ -0,0 +1,9 @@
+import type { GradeResult } from "./result.ts";
+
+/** The same task: an id within its variant. Two variants' outputs for one id are different outputs of one task. */
+/** A JSON pair, so no character a label or id may hold can make two pairs one key. */
+export const resultKey = (result: Pick<GradeResult, "id" | "variant">): string => JSON.stringify([result.variant ?? null, result.id]);
+
+/** An output as a person reads its name: the id, and its variant when it has one. */
+export const resultName = (result: Pick<GradeResult, "id" | "variant">): string =>
+  result.variant === undefined ? result.id : `${result.id} (${result.variant})`;

package/src/grade/result.ts

@@ -0,0 +1,82 @@
+import type { AtomKind } from "../compare/atom.ts";
+import type { CitationStatus } from "../structure/cite.ts";
+import type { Severity } from "../plugin.ts";
+import type { OutputSize } from "./rates.ts";
+
+// The shape of one line of `chaff grade --out`, and of what `grade()` returns. Spec §29.3.
+
+export type GradeFinding = {
+  readonly rule: string;
+  readonly level: Severity;
+  readonly line: number;
+  readonly column: number;
+  readonly message: string;
+};
+
+/** A dropped or added fact, as `compare --json` writes it. */
+export type GradeFact = { readonly kind: AtomKind; readonly key: string; readonly text: string; readonly line: number; readonly allowed: boolean };
+
+/** `compare`'s outcome against the item's reference. */
+export type GradeFacts = { readonly dropped: readonly GradeFact[]; readonly added: readonly GradeFact[]; readonly reformed: number };
+
+export type FailedCitation = {
+  readonly source: string;
+  readonly address: string;
+  readonly quote: string;
+  readonly status: Exclude<CitationStatus, "ok">;
+  readonly foundAt?: string | undefined;
+  readonly line?: number | undefined;
+};
+
+export type GradeCitations = { readonly checked: number; readonly failed: readonly FailedCitation[] };
+
+/** A fact of the output that a retrieved passage states, and the first passage (0-based) that does. */
+export type SupportedFact = { readonly kind: AtomKind; readonly text: string; readonly line: number; readonly passage: number };
+
+/** The output's checkable facts against the item's retrieved passages (`contexts`). */
+export type GradeContexts = {
+  readonly passages: number;
+  /** Facts of the kinds checked against the passages: numbers, dates, times, URLs, code, names and quotations. */
+  readonly checked: number;
+  readonly supported: readonly SupportedFact[];
+  /** Facts no passage states; a quotation not found word for word in any passage. */
+  readonly unsupported: readonly GradeFact[];
+  /** Sentences that state no checkable fact: whether a passage supports them needs reading, which chaff does not do. */
+  readonly uncheckedSentences: number;
+};
+
+/** A rule, or a check (`compare`, `cite`), that did not run on this output, and why. */
+export type NotRunEntry = { readonly rule: string; readonly reason: string };
+
+/** What makes two results comparable (spec §29.7): the same rules and settings. */
+export type Stamp = { readonly chaff: string; readonly rules: string; readonly settings: string };
+
+/** One finding's penalty points, so every point names the finding it came from. */
+export type ScoreItem = { readonly points: number; readonly rule: string; readonly line: number };
+
+/** The penalty sum under a `grade:` rubric (spec §29.4). Never a score out of a maximum. */
+export type GradeScore = { readonly penalty: number; readonly items: readonly ScoreItem[] };
+
+export type GradeResult = {
+  readonly id: string;
+  /** The item's variant label, when it had one. */
+  readonly variant?: string | undefined;
+  readonly language: string;
+  readonly genre: string;
+  readonly size: OutputSize;
+  readonly findings: readonly GradeFinding[];
+  readonly rates: Readonly<Record<string, number>>;
+  readonly notRun: readonly NotRunEntry[];
+  /** Null when the item had no reference. */
+  readonly facts: GradeFacts | null;
+  /** Null when the item gave no citations. */
+  readonly citations: GradeCitations | null;
+  /** Only when the item gave contexts, so results of items without them keep their shape. */
+  readonly contexts?: GradeContexts | undefined;
+  /** Only with a `grade:` rubric. */
+  readonly score?: GradeScore | undefined;
+  readonly pass: boolean;
+  /** Each condition the output failed, with how far: `facts.dropped 3 > 0`. */
+  readonly failedBecause: readonly string[];
+  readonly stamp: Stamp;
+};

package/src/grade/results-read.ts

@@ -0,0 +1,111 @@
+import { resultKey } from "./result-name.ts";
+import type { GradeResult } from "./result.ts";
+
+// An earlier run's results (`--out`), read back for `--baseline`. Pure. Only lines chaff grade wrote are accepted: a
+// baseline that half-reads would compare against outputs that were never graded.
+
+const isRecord = (value: unknown): value is Record<string, unknown> => typeof value === "object" && value !== null && !Array.isArray(value);
+
+const isString = (value: unknown): value is string => typeof value === "string";
+
+const isNumber = (value: unknown): value is number => typeof value === "number" && Number.isFinite(value);
+
+const isArrayOf = (value: unknown, each: (entry: unknown) => boolean): boolean => Array.isArray(value) && value.every(each);
+
+const isFinding = (value: unknown): boolean =>
+  isRecord(value) && isString(value["rule"]) && isString(value["level"]) && isNumber(value["line"]) && isNumber(value["column"]) && isString(value["message"]);
+
+const isFact = (value: unknown): boolean =>
+  isRecord(value) &&
+  isString(value["kind"]) &&
+  isString(value["key"]) &&
+  isString(value["text"]) &&
+  isNumber(value["line"]) &&
+  typeof value["allowed"] === "boolean";
+
+const isFailedCitation = (value: unknown): boolean =>
+  isRecord(value) && isString(value["source"]) && isString(value["address"]) && isString(value["quote"]) && isString(value["status"]);
+
+const isFacts = (value: unknown): boolean => value === null || (isRecord(value) && isArrayOf(value["dropped"], isFact) && isArrayOf(value["added"], isFact));
+
+const isCitations = (value: unknown): boolean =>
+  value === null || (isRecord(value) && isNumber(value["checked"]) && isArrayOf(value["failed"], isFailedCitation));
+
+const isSupported = (value: unknown): boolean =>
+  isRecord(value) && isString(value["kind"]) && isString(value["text"]) && isNumber(value["line"]) && isNumber(value["passage"]);
+
+/** Absent when the item gave no contexts. */
+const isContexts = (value: unknown): boolean =>
+  value === undefined ||
+  (isRecord(value) &&
+    isNumber(value["passages"]) &&
+    isNumber(value["checked"]) &&
+    isArrayOf(value["supported"], isSupported) &&
+    isArrayOf(value["unsupported"], isFact) &&
+    isNumber(value["uncheckedSentences"]));
+
+const isStamp = (value: unknown): boolean => isRecord(value) && isString(value["chaff"]) && isString(value["rules"]) && isString(value["settings"]);
+
+const isSize = (value: unknown): boolean => isRecord(value) && (value["unit"] === "char" || value["unit"] === "word") && isNumber(value["value"]);
+
+const isScoreItem = (value: unknown): boolean => isRecord(value) && isNumber(value["points"]) && isString(value["rule"]) && isNumber(value["line"]);
+
+const isScore = (value: unknown): boolean => value === undefined || (isRecord(value) && isNumber(value["penalty"]) && isArrayOf(value["items"], isScoreItem));
+
+const isNotRun = (value: unknown): boolean => isRecord(value) && isString(value["rule"]) && isString(value["reason"]);
+
+const isRates = (value: unknown): boolean => isRecord(value) && Object.values(value).every(isNumber);
+
+/** The id, and the variant when the output had one. */
+const isLabel = (value: Record<string, unknown>): boolean => isString(value["id"]) && (value["variant"] === undefined || isString(value["variant"]));
+
+/** A line `--out` writes, every field checked: a line that only looks like one would be compared as if it had been graded. */
+export const isGradeResult = (value: unknown): value is GradeResult =>
+  isRecord(value) &&
+  isLabel(value) &&
+  isString(value["language"]) &&
+  isString(value["genre"]) &&
+  isSize(value["size"]) &&
+  isArrayOf(value["findings"], isFinding) &&
+  isRates(value["rates"]) &&
+  isArrayOf(value["notRun"], isNotRun) &&
+  isFacts(value["facts"]) &&
+  isCitations(value["citations"]) &&
+  isContexts(value["contexts"]) &&
+  isScore(value["score"]) &&
+  typeof value["pass"] === "boolean" &&
+  isArrayOf(value["failedBecause"], isString) &&
+  isStamp(value["stamp"]);
+
+const parsedLine = (body: string): unknown => {
+  try {
+    return JSON.parse(body);
+  } catch {
+    return undefined;
+  }
+};
+
+/** The lines whose id an earlier line of the same variant already has: one run never grades two outputs under one id. */
+const repeatedIds = (read: readonly { readonly line: number; readonly value: unknown }[]): number[] => {
+  const seen = new Set<string>();
+  return read.flatMap(({ line, value }) => {
+    if (!isGradeResult(value)) return [];
+    const repeated = seen.has(resultKey(value));
+    seen.add(resultKey(value));
+    return repeated ? [line] : [];
+  });
+};
+
+/** Every result line, or the line numbers that are not one (or repeat an id). No result at all is not a baseline. */
+export const parseResults = (text: string): { readonly results: readonly GradeResult[] } | { readonly badLines: readonly number[] } => {
+  const lines = text
+    .split(/\r?\n/u)
+    .map((body, index) => ({ body, line: index + 1 }))
+    .filter(({ body }) => body.trim() !== "");
+  const read = lines.map(({ body, line }) => ({ line, value: parsedLine(body) }));
+  const badLines = [...read.filter((entry) => !isGradeResult(entry.value)).map((entry) => entry.line), ...repeatedIds(read)].toSorted(
+    (left, right) => left - right,
+  );
+  if (badLines.length > 0 || read.length === 0) return { badLines: read.length === 0 ? [0] : badLines };
+  return { results: read.map((entry) => entry.value).filter(isGradeResult) };
+};

package/src/grade/rubric-verdict.ts

@@ -0,0 +1,86 @@
+import { exactRateOf, type OutputSize } from "./rates.ts";
+import type { GradeScore } from "./result.ts";
+import type { Rubric, RuleLimit } from "./rubric.ts";
+import { counted, overLimit, verdictOf, type Graded, type Verdict } from "./verdict.ts";
+
+// Pass or fail and the penalty score under a `grade:` rubric (spec §29.4). Only what the rubric writes decides: an error
+// finding of a rule the rubric does not name is a rate, not a failure, so the reason for a failure is always in `grade:`.
+
+/** What the rubric reads of one output beyond its findings, facts and citations. */
+export type RubricInput = Graded & {
+  readonly size: OutputSize;
+  /** The required sections the output has no heading for. */
+  readonly missingSections: readonly string[];
+  /** The item gave sources but no citations. */
+  readonly uncited: boolean;
+};
+
+const countOf = (input: RubricInput, rule: string): number => input.findings.filter((finding) => finding.rule === rule).length;
+
+/** A rate as a failure reason shows it: to three decimals, enough to tell it from a limit written to one. */
+const shownRate = (rate: number): string => String(Math.round(rate * RATE_SHOWN_SCALE) / RATE_SHOWN_SCALE);
+
+const RATE_SHOWN_SCALE = 1000;
+
+const ruleFailures = (input: RubricInput, rule: string, limit: RuleLimit): string[] => {
+  const count = countOf(input, rule);
+  const rate = exactRateOf(count, input.size.value);
+  return [
+    ...(limit.max === undefined ? [] : overLimit(`rules.${rule}`, count, limit.max)),
+    ...(limit.maxRate === undefined || rate === undefined || rate <= limit.maxRate ? [] : [`rules.${rule}.rate ${shownRate(rate)} > ${String(limit.maxRate)}`]),
+  ];
+};
+
+const sectionFailures = (input: RubricInput): string[] =>
+  overLimit("required_sections.missing", input.missingSections.length, 0).map((reason) => `${reason}: ${input.missingSections.join(", ")}`);
+
+const factFailures = (input: RubricInput, rubric: Rubric): string[] => {
+  const limits = rubric.facts;
+  if (limits === undefined || input.facts === null) return [];
+  return [
+    ...(limits.dropped === undefined ? [] : overLimit("facts.dropped", counted(input.facts.dropped), limits.dropped)),
+    ...(limits.added === undefined ? [] : overLimit("facts.added", counted(input.facts.added), limits.added)),
+  ];
+};
+
+const citationFailures = (input: RubricInput, rubric: Rubric): string[] => {
+  const limits = rubric.citations;
+  if (limits === undefined) return [];
+  return [
+    ...(limits.failed === undefined || input.citations === null ? [] : overLimit("citations.failed", input.citations.failed.length, limits.failed)),
+    ...(limits.required && input.uncited ? ["citations.required: sources given, no citations"] : []),
+  ];
+};
+
+const contextFailures = (input: RubricInput, rubric: Rubric): string[] => {
+  const limits = rubric.contexts;
+  if (limits === undefined) return [];
+  return [
+    ...(limits.unsupported === undefined || input.contexts === undefined
+      ? []
+      : overLimit("contexts.unsupported", counted(input.contexts.unsupported), limits.unsupported)),
+    ...(limits.required && input.contexts === undefined ? ["contexts.required: no contexts given"] : []),
+  ];
+};
+
+/** One item per finding of a weighted rule, so every point names the finding it came from; the penalty is their sum. */
+export const scoreOf = (input: Pick<RubricInput, "findings">, rubric: Rubric): GradeScore => {
+  const items = input.findings.flatMap((finding) => {
+    const weight = rubric.rules[finding.rule]?.weight;
+    return weight === undefined || weight === 0 ? [] : [{ points: weight, rule: finding.rule, line: finding.line }];
+  });
+  return { penalty: items.reduce((sum, item) => sum + item.points, 0), items };
+};
+
+export const rubricVerdict = (input: RubricInput, rubric: Rubric): { readonly verdict: Verdict; readonly score: GradeScore } => {
+  const score = scoreOf(input, rubric);
+  const failures = [
+    ...Object.entries(rubric.rules).flatMap(([rule, limit]) => ruleFailures(input, rule, limit)),
+    ...sectionFailures(input),
+    ...factFailures(input, rubric),
+    ...citationFailures(input, rubric),
+    ...contextFailures(input, rubric),
+    ...(rubric.penalty === undefined ? [] : overLimit("score.penalty", score.penalty, rubric.penalty)),
+  ];
+  return { verdict: verdictOf(failures), score };
+};