chaffjs 0.19.0 → 0.20.0

package/src/grade/baseline.ts

@@ -0,0 +1,140 @@
+import type { LengthUnit } from "../plugin.ts";
+import { tally } from "./order.ts";
+import { rateRows, sameReading } from "./pairing.ts";
+import { resultKey, resultName } from "./result-name.ts";
+import type { FailedCitation, GradeFact, GradeResult, Stamp } from "./result.ts";
+
+// A run compared with an earlier one (spec §29.5). Pure. Outputs are paired by id; a pair is compared only when both
+// were read the same way (language and genre), and the runs only when their stamps say the same rules and settings.
+
+/** How a rule's rate moved over the paired outputs of one unit, and which outputs gained or lost findings of it. */
+export type RuleMovement = {
+  readonly rule: string;
+  readonly unit: LengthUnit;
+  readonly before: number;
+  readonly after: number;
+  readonly increasedIn: readonly string[];
+  readonly decreasedIn: readonly string[];
+  /** Named under the rubric's rules: only these count as a regression when they increase. */
+  readonly inRubric: boolean;
+};
+
+/** What an output newly dropped or added, or newly failed to quote, against the same reference and sources. */
+export type ItemChange = {
+  readonly id: string;
+  readonly dropped: readonly GradeFact[];
+  readonly added: readonly GradeFact[];
+  readonly citations: readonly FailedCitation[];
+};
+
+export type Comparison = {
+  readonly paired: number;
+  readonly onlyBefore: readonly string[];
+  readonly onlyAfter: readonly string[];
+  /** Paired by id but read in another language or genre: not the same task graded the same way. */
+  readonly readOtherwise: readonly string[];
+  readonly rules: readonly RuleMovement[];
+  readonly newlyFailed: readonly string[];
+  readonly newlyPassed: readonly string[];
+  readonly items: readonly ItemChange[];
+  readonly penalty?: { readonly before: number; readonly after: number } | undefined;
+  /** Each reason this is a regression; empty when it is none. */
+  readonly regressions: readonly string[];
+};
+
+/** Why two runs cannot be compared: their rules or settings differ, or the baseline mixes several stamps. */
+export type StampCheck = { readonly comparable: true } | { readonly comparable: false; readonly differ: readonly ("rules" | "settings" | "mixed")[] };
+
+const STAMP_PARTS: readonly ("rules" | "settings")[] = ["rules", "settings"];
+
+export const stampCheck = (before: readonly GradeResult[], current: Stamp): StampCheck => {
+  // One earlier run has one stamp, its chaff version included; results joined from several runs are not one baseline.
+  const stamps = new Set(before.map((result) => `${result.stamp.chaff}\n${result.stamp.rules}\n${result.stamp.settings}`));
+  const [first] = before;
+  if (stamps.size > 1 || first === undefined) return { comparable: false, differ: ["mixed"] };
+  const differ = STAMP_PARTS.filter((part) => first.stamp[part] !== current[part]);
+  return differ.length === 0 ? { comparable: true } : { comparable: false, differ };
+};
+
+type Pair = { readonly before: GradeResult; readonly after: GradeResult };
+
+const countIn = (result: GradeResult, rule: string): number => result.findings.filter((finding) => finding.rule === rule).length;
+
+const movementsOf = (pairs: readonly Pair[], rubricRules: ReadonlySet<string>): RuleMovement[] =>
+  rateRows([pairs.map((pair) => pair.before), pairs.map((pair) => pair.after)]).flatMap(({ rule, unit, rates: [before = 0, after = 0] }) => {
+    const inUnit = pairs.filter((pair) => pair.after.size.unit === unit);
+    const increasedIn = inUnit.filter((pair) => countIn(pair.after, rule) > countIn(pair.before, rule)).map((pair) => resultName(pair.after));
+    const decreasedIn = inUnit.filter((pair) => countIn(pair.after, rule) < countIn(pair.before, rule)).map((pair) => resultName(pair.after));
+    if (increasedIn.length === 0 && decreasedIn.length === 0 && before === after) return [];
+    return [{ rule, unit, before, after, increasedIn, decreasedIn, inRubric: rubricRules.has(rule) }];
+  });
+
+const factKey = (fact: GradeFact): string => `${fact.kind}\n${fact.key}`;
+
+/** The facts of `after` beyond as many as `before` had of each: a fact dropped in both runs is not new. */
+const newFacts = (before: readonly GradeFact[], after: readonly GradeFact[]): GradeFact[] => {
+  const earlier = tally(before.filter((fact) => !fact.allowed).map(factKey));
+  const seen = new Map<string, number>();
+  return after
+    .filter((fact) => !fact.allowed)
+    .filter((fact) => {
+      const key = factKey(fact);
+      seen.set(key, (seen.get(key) ?? 0) + 1);
+      return (seen.get(key) ?? 0) > (earlier[key] ?? 0);
+    });
+};
+
+const citationKey = (citation: FailedCitation): string => `${citation.source}\n${citation.address}\n${citation.quote}`;
+
+const itemChange = ({ before, after }: Pair): ItemChange => {
+  const failedBefore = new Set((before.citations?.failed ?? []).map(citationKey));
+  return {
+    id: resultName(after),
+    dropped: newFacts(before.facts?.dropped ?? [], after.facts?.dropped ?? []),
+    added: newFacts(before.facts?.added ?? [], after.facts?.added ?? []),
+    citations: (after.citations?.failed ?? []).filter((citation) => !failedBefore.has(citationKey(citation))),
+  };
+};
+
+const penaltyOf = (results: readonly GradeResult[]): number => results.reduce((sum, result) => sum + (result.score?.penalty ?? 0), 0);
+
+const scored = (pairs: readonly Pair[]): boolean => pairs.some((pair) => pair.after.score !== undefined);
+
+const regressionsOf = (newlyFailed: readonly string[], movements: readonly RuleMovement[], penalty: Comparison["penalty"]): string[] => [
+  ...newlyFailed.map((id) => `${id}: passed, now fails`),
+  ...movements
+    .filter((movement) => movement.inRubric && movement.increasedIn.length > 0)
+    .map((movement) => `rules.${movement.rule}: more findings in ${movement.increasedIn.join(", ")}`),
+  ...(penalty !== undefined && penalty.after > penalty.before ? [`score.penalty ${String(penalty.before)} → ${String(penalty.after)}`] : []),
+];
+
+const pairRead = (pair: Pair): boolean => sameReading([pair.before, pair.after]);
+
+/**
+ * `after` compared with `before`, output by output: an output is paired with the earlier one of its id and variant.
+ * `rubricRules`: the rules the rubric names, whose increase is a regression.
+ */
+export const compareRuns = (before: readonly GradeResult[], after: readonly GradeResult[], rubricRules: ReadonlySet<string>): Comparison => {
+  const earlier = new Map(before.map((result) => [resultKey(result), result]));
+  const keys = new Set(after.map(resultKey));
+  const matched = after.flatMap((result) => {
+    const match = earlier.get(resultKey(result));
+    return match === undefined ? [] : [{ before: match, after: result }];
+  });
+  const pairs = matched.filter(pairRead);
+  const rules = movementsOf(pairs, rubricRules);
+  const newlyFailed = pairs.filter((pair) => pair.before.pass && !pair.after.pass).map((pair) => resultName(pair.after));
+  const penalty = scored(pairs) ? { before: penaltyOf(pairs.map((pair) => pair.before)), after: penaltyOf(pairs.map((pair) => pair.after)) } : undefined;
+  return {
+    paired: pairs.length,
+    onlyBefore: before.filter((result) => !keys.has(resultKey(result))).map(resultName),
+    onlyAfter: after.filter((result) => !earlier.has(resultKey(result))).map(resultName),
+    readOtherwise: matched.filter((pair) => !pairRead(pair)).map((pair) => resultName(pair.after)),
+    rules,
+    newlyFailed,
+    newlyPassed: pairs.filter((pair) => !pair.before.pass && pair.after.pass).map((pair) => resultName(pair.after)),
+    items: pairs.map(itemChange).filter((change) => change.dropped.length + change.added.length + change.citations.length > 0),
+    ...(penalty === undefined ? {} : { penalty }),
+    regressions: regressionsOf(newlyFailed, rules, penalty),
+  };
+};

package/src/grade/checks.ts

@@ -0,0 +1,73 @@
+import { documentLanguage } from "../check-source.ts";
+import type { Config } from "../config/load.ts";
+import { outcomeOf, type Allowed } from "../compare/outcome.ts";
+import { factsOf } from "../commands/compare.ts";
+import { documentFromSource } from "../commands/read-document.ts";
+import { treeFromSource } from "../commands/tree.ts";
+import { checkCitations, type CitationResult } from "../structure/cite.ts";
+import type { StructureNode } from "../plugin.ts";
+import type { UiLanguage } from "../ui.ts";
+import type { GradeCitation, GradeItem } from "./item.ts";
+import type { FailedCitation, GradeCitations, GradeFact, GradeFacts, NotRunEntry } from "./result.ts";
+import { GRADE_TEXT } from "./text.ts";
+
+// What an output is checked against besides the rules: its reference (compare) and its sources (cite). Each is read from
+// memory through the same code as `chaff compare` and `chaff cite` read files.
+
+/** The output is read under this name, so it is Markdown as a model's answer usually is. */
+export const OUTPUT_PATH = "output.md";
+const REFERENCE_PATH = "reference.md";
+const READ_AS_NAMED = /\.(?:md|markdown|mdx|txt)$/iu;
+
+/** A source is Markdown unless its name says it is a plain text file (a contract without headings, read by its numbering). */
+export const sourcePathOf = (name: string): string => (READ_AS_NAMED.test(name) ? name : `${name}.md`);
+
+/** The settings, and the output's genre, which every text of one item is read with, so a source's profile follows the task. */
+export type ItemReading = { readonly genre: string; readonly config: Config };
+
+const factOf = (change: GradeFact): GradeFact => ({ kind: change.kind, key: change.key, text: change.text, line: change.line, allowed: change.allowed });
+
+/** `compare`'s outcome, the reference before and the output after, read as `chaff compare` reads two files. */
+export const factsAgainst = async (
+  texts: { readonly reference: string; readonly output: string; readonly outputLanguage: string },
+  reading: ItemReading,
+  allowed: Allowed,
+): Promise<GradeFacts> => {
+  const { config, genre } = reading;
+  const referenceLanguage = documentLanguage(REFERENCE_PATH, texts.reference, config);
+  const before = await documentFromSource(REFERENCE_PATH, texts.reference, { language: referenceLanguage, genre }, config, true);
+  const after = await documentFromSource(OUTPUT_PATH, texts.output, { language: texts.outputLanguage, genre }, config, true);
+  const outcome = outcomeOf(factsOf(REFERENCE_PATH, before, config.names), factsOf(OUTPUT_PATH, after, config.names), allowed);
+  return { dropped: outcome.dropped.map(factOf), added: outcome.added.map(factOf), reformed: outcome.reformed.length };
+};
+
+type SourceTreeRead = { readonly tree: StructureNode } | { readonly language: string };
+
+const sourceTree = async (name: string, text: string, reading: ItemReading): Promise<SourceTreeRead> => {
+  const path = sourcePathOf(name);
+  const language = documentLanguage(path, text, reading.config);
+  const tree = await treeFromSource(path, text, language, reading.genre, reading.config);
+  return tree === undefined ? { language } : { tree };
+};
+
+const failedOf = (source: string, result: CitationResult): FailedCitation[] =>
+  result.status === "ok"
+    ? []
+    : [{ source, address: result.citation.address, quote: result.citation.quote, status: result.status, foundAt: result.foundAt, line: result.line }];
+
+export type CitationsRead = { readonly citations: GradeCitations } | { readonly notRun: NotRunEntry };
+
+/** Each citation checked against the source it names, as `chaff cite` checks it. A source chaff cannot read as a tree is not a failed quotation. */
+export const citationsAgainst = async (item: GradeItem, citations: readonly GradeCitation[], reading: ItemReading, ui: UiLanguage): Promise<CitationsRead> => {
+  const names = [...new Set(citations.map((citation) => citation.source))];
+  const trees = await Promise.all(names.map(async (name) => ({ name, read: await sourceTree(name, item.sources[name] ?? "", reading) })));
+  const unread = trees.find((entry) => !("tree" in entry.read));
+  if (unread !== undefined && "language" in unread.read)
+    return { notRun: { rule: "cite", reason: GRADE_TEXT[ui].noStructure(unread.name, unread.read.language) } };
+  const failed = trees.flatMap(({ name, read }) => {
+    if (!("tree" in read)) return [];
+    const own = citations.filter((citation) => citation.source === name);
+    return checkCitations(item.sources[name] ?? "", read.tree, own).flatMap((result) => failedOf(name, result));
+  });
+  return { citations: { checked: citations.length, failed } };
+};

package/src/grade/context-support.ts

@@ -0,0 +1,60 @@
+import type { Atom, AtomKind, Extraction, UnreadReason } from "../compare/atom.ts";
+import { outcomeOf } from "../compare/outcome.ts";
+import type { GradeFact, SupportedFact } from "./result.ts";
+
+// Whether each checkable fact of an answer is stated in some retrieved passage (spec §29.3, contexts). Pure. Facts are
+// read and matched as `chaff compare --distinct` matches them; a quotation must be in a passage word for word. Meaning
+// is never judged: a sentence with no checkable fact is counted as not checked, not as supported.
+
+/** The kinds an answer is checked for. Headings, clause references and footnotes describe the answer, not the world. */
+export const CONTEXT_KINDS: readonly AtomKind[] = ["number", "date", "time", "url", "code", "name", "quote"];
+
+/** A kind not fully read in the answer or a passage, and why: no fact of it found there is not "none". */
+export type UnreadKind = { readonly kind: AtomKind; readonly reason: UnreadReason };
+
+export type ContextSupport = {
+  readonly checked: number;
+  readonly supported: readonly SupportedFact[];
+  readonly unsupported: readonly GradeFact[];
+  readonly unread: readonly UnreadKind[];
+};
+
+/** One passage as read: its facts, and its text to find a quotation in. */
+export type Passage = { readonly extraction: Extraction; readonly text: string };
+
+/** Text as a quotation key spells it: width and runs of white space are how it is written, not what it says. */
+const spelled = (text: string): string => text.normalize("NFKC").replace(/\s+/gu, " ");
+
+const factId = (atom: Atom): string => [atom.kind, atom.key, String(atom.line), atom.text].join("\u0000");
+
+/** The facts of the answer this passage states: a quotation word for word, any other fact as compare --distinct keeps it. */
+const statedIn = (answer: Extraction, passage: Passage): ReadonlySet<string> => {
+  const unstated = new Set(
+    outcomeOf({ path: "context", extraction: passage.extraction }, { path: "answer", extraction: answer }, undefined, "distinct").added.map(factId),
+  );
+  const text = spelled(passage.text);
+  return new Set(answer.atoms.filter((atom) => (atom.kind === "quote" ? text.includes(atom.key) : !unstated.has(factId(atom)))).map(factId));
+};
+
+const unreadOf = (extractions: readonly Extraction[]): UnreadKind[] => {
+  const seen = new Map<string, UnreadKind>();
+  extractions.forEach((extraction) =>
+    extraction.unread.filter((entry) => CONTEXT_KINDS.includes(entry.kind)).forEach((entry) => seen.set(`${entry.kind}\n${entry.reason}`, entry)),
+  );
+  return [...seen.values()];
+};
+
+/** Each checkable fact of the answer, supported by the first passage that states it, or unsupported. `allowed`: kinds that may be. */
+export const contextSupport = (answer: Extraction, passages: readonly Passage[], allowed: ReadonlySet<AtomKind>): ContextSupport => {
+  const stated = passages.map((passage) => statedIn(answer, passage));
+  const checked = answer.atoms.filter((atom) => CONTEXT_KINDS.includes(atom.kind));
+  const placed = checked.map((atom) => ({ atom, passage: stated.findIndex((ids) => ids.has(factId(atom))) }));
+  return {
+    checked: checked.length,
+    supported: placed.flatMap(({ atom, passage }) => (passage === -1 ? [] : [{ kind: atom.kind, text: atom.text, line: atom.line, passage }])),
+    unsupported: placed.flatMap(({ atom, passage }) =>
+      passage === -1 ? [{ kind: atom.kind, key: atom.key, text: atom.text, line: atom.line, allowed: allowed.has(atom.kind) }] : [],
+    ),
+    unread: unreadOf([answer, ...passages.map((passage) => passage.extraction)]),
+  };
+};

package/src/grade/contexts.ts

@@ -0,0 +1,53 @@
+import { documentLanguage } from "../check-source.ts";
+import type { AtomKind } from "../compare/atom.ts";
+import { factsOf } from "../commands/compare.ts";
+import { documentFromSource } from "../commands/read-document.ts";
+import { lineStarts, placeOf } from "../position.ts";
+import type { ProseDocument } from "../plugin.ts";
+import type { ItemReading } from "./checks.ts";
+import { contextSupport, type UnreadKind } from "./context-support.ts";
+import type { GradeContexts } from "./result.ts";
+
+// An output's facts against the passages it was meant to rest on (spec §29.3, contexts): each passage read as `chaff
+// compare` reads a document, so a number, date or name is the same fact here as there.
+
+const contextPath = (index: number): string => `context-${String(index + 1)}.md`;
+
+type Placed = { readonly line: number; readonly text: string };
+
+/** Each sentence as written, and the lines it covers, first to last. */
+const sentencesOf = (doc: ProseDocument): { readonly first: number; readonly last: number; readonly written: string }[] => {
+  const starts = lineStarts(doc.source);
+  return doc.sentences.map((sentence) => ({
+    first: placeOf(starts, sentence.span.start).line,
+    last: placeOf(starts, Math.max(sentence.span.start, sentence.span.end - 1)).line,
+    written: doc.source.slice(sentence.span.start, sentence.span.end),
+  }));
+};
+
+/** Sentences that state no checked fact: chaff cannot say whether a passage supports them. A fact is placed by its line and how it is written. */
+const uncheckedOf = (doc: ProseDocument, facts: readonly Placed[]): number =>
+  sentencesOf(doc).filter(({ first, last, written }) => !facts.some((fact) => fact.line >= first && fact.line <= last && written.includes(fact.text))).length;
+
+export type ContextsRead = { readonly contexts: GradeContexts; readonly unread: readonly UnreadKind[] };
+
+/** The output's checkable facts, each found in some passage or not. `allowed`: kinds the rubric lets go unsupported. */
+export const contextsAgainst = async (
+  texts: { readonly output: string; readonly outputPath: string; readonly outputLanguage: string; readonly contexts: readonly string[] },
+  reading: ItemReading,
+  allowed: ReadonlySet<AtomKind>,
+): Promise<ContextsRead> => {
+  const { config, genre } = reading;
+  const answer = await documentFromSource(texts.outputPath, texts.output, { language: texts.outputLanguage, genre }, config, true);
+  const passages = await Promise.all(
+    texts.contexts.map(async (text, index) => {
+      const path = contextPath(index);
+      const read = await documentFromSource(path, text, { language: documentLanguage(path, text, config), genre }, config, true);
+      return { extraction: factsOf(path, read, config.names).extraction, text };
+    }),
+  );
+  const support = contextSupport(factsOf(texts.outputPath, answer, config.names).extraction, passages, allowed);
+  const { checked, supported, unsupported, unread } = support;
+  const uncheckedSentences = uncheckedOf(answer.doc, [...supported, ...unsupported]);
+  return { contexts: { passages: passages.length, checked, supported, unsupported, uncheckedSentences }, unread };
+};

package/src/grade/grade-item.ts

@@ -0,0 +1,133 @@
+import { checkSource, type SourceCheck } from "../check-source.ts";
+import type { Allowed } from "../compare/outcome.ts";
+import { missingSections } from "../detectors/team.ts";
+import { messageOf } from "../render/text.ts";
+import { uiLanguageOf, type UiLanguage } from "../ui.ts";
+import type { Finding } from "../plugin.ts";
+import { citationsAgainst, factsAgainst, OUTPUT_PATH, type CitationsRead, type ItemReading } from "./checks.ts";
+import { contextsAgainst, type ContextsRead } from "./contexts.ts";
+import type { GradeItem } from "./item.ts";
+import { ratesOf, sizeOf, type OutputSize } from "./rates.ts";
+import type { GradeCitations, GradeFacts, GradeFinding, GradeResult, GradeScore, NotRunEntry, Stamp } from "./result.ts";
+import type { Rubric } from "./rubric.ts";
+import { rubricVerdict } from "./rubric-verdict.ts";
+import type { GradeSettings } from "./stamp.ts";
+import { GRADE_TEXT } from "./text.ts";
+import { defaultVerdict, type Verdict } from "./verdict.ts";
+
+/** One run's settings, its `grade:` rubric when chaff.yaml has one, and its stamp, which every result carries. */
+export type GradeSetup = GradeSettings & { readonly stamp: Stamp; readonly rubric?: Rubric | undefined };
+
+const findingOf = (finding: Finding, check: SourceCheck): GradeFinding => {
+  const rule = check.rules.find((entry) => entry.id === finding.rule);
+  return {
+    rule: finding.rule,
+    level: finding.severity,
+    line: finding.line,
+    column: finding.column,
+    message: rule === undefined ? finding.rule : messageOf(rule, finding, check.language),
+  };
+};
+
+/** The fact kinds the rubric lets change (`allow_dropped`, `allow_added`), as compare's --allow-dropped and --allow-added. */
+const allowedBy = (rubric: Rubric | undefined): Allowed => ({
+  dropped: new Set(rubric?.facts?.allowDropped ?? []),
+  added: new Set(rubric?.facts?.allowAdded ?? []),
+});
+
+/** One output as read: linted, measured, and checked against its reference and sources. */
+type Checked = {
+  readonly check: SourceCheck;
+  readonly ui: UiLanguage;
+  readonly findings: readonly GradeFinding[];
+  readonly size: OutputSize;
+  readonly facts: GradeFacts | null;
+  readonly cited: CitationsRead | undefined;
+  readonly contexts: ContextsRead | undefined;
+};
+
+const checkItem = async (item: GradeItem, setup: GradeSetup): Promise<Checked> => {
+  const choice = { language: item.language, genre: item.genre ?? setup.genre, experimental: setup.experimental };
+  const check = await checkSource(OUTPUT_PATH, item.output, setup.config, choice);
+  const ui = uiLanguageOf(check.language);
+  const reading: ItemReading = { genre: check.genre.genre, config: setup.config };
+  const texts = { reference: item.reference ?? "", output: item.output, outputLanguage: check.language };
+  return {
+    check,
+    ui,
+    findings: check.applied.kept.map((finding) => findingOf(finding, check)),
+    size: sizeOf(check.doc),
+    facts: item.reference === undefined ? null : await factsAgainst(texts, reading, allowedBy(setup.rubric)),
+    cited: item.citations === undefined ? undefined : await citationsAgainst(item, item.citations, reading, ui),
+    contexts: await contextsOf(item, check.language, reading, setup.rubric),
+  };
+};
+
+const contextsOf = async (item: GradeItem, outputLanguage: string, reading: ItemReading, rubric: Rubric | undefined): Promise<ContextsRead | undefined> => {
+  if (item.contexts === undefined) return undefined;
+  const texts = { output: item.output, outputPath: OUTPUT_PATH, outputLanguage, contexts: item.contexts };
+  return contextsAgainst(texts, reading, new Set(rubric?.contexts?.allowUnsupported ?? []));
+};
+
+/** What the contexts check could not see: not given at all, kinds not fully read, and sentences with nothing to check. */
+const contextsNotRun = (checked: Checked): NotRunEntry[] => {
+  const text = GRADE_TEXT[checked.ui];
+  if (checked.contexts === undefined) return [{ rule: "contexts", reason: text.noContexts }];
+  const { contexts, unread } = checked.contexts;
+  return [
+    ...unread.map((entry) => ({ rule: "contexts", reason: text.contextsUnread(entry.kind, entry.reason) })),
+    ...(contexts.uncheckedSentences === 0 ? [] : [{ rule: "contexts", reason: text.uncheckedSentences(contexts.uncheckedSentences) }]),
+  ];
+};
+
+const citationsOf = (checked: Checked): GradeCitations | null => (checked.cited !== undefined && "citations" in checked.cited ? checked.cited.citations : null);
+
+/** The rubric's rules this output's rules do not include: a misspelt id would otherwise pass every output silently. */
+const unknownRubricRules = (checked: Checked, rubric: Rubric | undefined): NotRunEntry[] =>
+  Object.keys(rubric?.rules ?? {})
+    .filter((id) => !checked.check.rules.some((rule) => rule.id === id))
+    .map((rule) => ({ rule, reason: GRADE_TEXT[checked.ui].unknownRule }));
+
+const notRunOf = (checked: Checked, rubric: Rubric | undefined): NotRunEntry[] => [
+  ...checked.check.raw.skipped.map((skipped) => ({ rule: skipped.rule, reason: skipped.why })),
+  ...unknownRubricRules(checked, rubric),
+  ...(checked.facts === null ? [{ rule: "compare", reason: GRADE_TEXT[checked.ui].noReference }] : []),
+  ...(checked.cited === undefined ? [{ rule: "cite", reason: GRADE_TEXT[checked.ui].noCitations }] : []),
+  ...(checked.cited !== undefined && "notRun" in checked.cited ? [checked.cited.notRun] : []),
+  ...contextsNotRun(checked),
+];
+
+/** Pass or fail: by the rubric when chaff.yaml has `grade:`, with its penalty score; else by the default of spec §29.3. */
+const judge = (item: GradeItem, checked: Checked, setup: GradeSetup): { readonly verdict: Verdict; readonly score?: GradeScore } => {
+  const graded = { findings: checked.findings, facts: checked.facts, citations: citationsOf(checked), contexts: checked.contexts?.contexts };
+  if (setup.rubric === undefined) return { verdict: defaultVerdict(graded) };
+  const wanted = setup.rubric.requiredSections ?? checked.check.doc.requiredSections;
+  const uncited = Object.keys(item.sources).length > 0 && item.citations === undefined;
+  return rubricVerdict({ ...graded, size: checked.size, missingSections: missingSections(checked.check.doc, wanted), uncited }, setup.rubric);
+};
+
+/** Grades one output (spec §29.3): its findings and their rates, the facts against its reference, its quotations, and pass or fail. */
+export const gradeItem = async (item: GradeItem, setup: GradeSetup): Promise<GradeResult> => {
+  const checked = await checkItem(item, setup);
+  const { verdict, score } = judge(item, checked, setup);
+  return {
+    id: item.id,
+    ...(item.variant === undefined ? {} : { variant: item.variant }),
+    language: checked.check.language,
+    genre: checked.check.genre.genre,
+    size: checked.size,
+    findings: checked.findings,
+    rates: ratesOf(
+      checked.findings.map((finding) => finding.rule),
+      checked.size,
+    ),
+    notRun: notRunOf(checked, setup.rubric),
+    facts: checked.facts,
+    citations: citationsOf(checked),
+    ...(checked.contexts === undefined ? {} : { contexts: checked.contexts.contexts }),
+    ...(score === undefined ? {} : { score }),
+    pass: verdict.pass,
+    failedBecause: verdict.failedBecause,
+    stamp: setup.stamp,
+  };
+};