chaffjs 0.19.0 → 0.20.0

package/src/grade/result.ts

@@ -0,0 +1,82 @@
+import type { AtomKind } from "../compare/atom.ts";
+import type { CitationStatus } from "../structure/cite.ts";
+import type { Severity } from "../plugin.ts";
+import type { OutputSize } from "./rates.ts";
+
+// The shape of one line of `chaff grade --out`, and of what `grade()` returns. Spec §29.3.
+
+export type GradeFinding = {
+  readonly rule: string;
+  readonly level: Severity;
+  readonly line: number;
+  readonly column: number;
+  readonly message: string;
+};
+
+/** A dropped or added fact, as `compare --json` writes it. */
+export type GradeFact = { readonly kind: AtomKind; readonly key: string; readonly text: string; readonly line: number; readonly allowed: boolean };
+
+/** `compare`'s outcome against the item's reference. */
+export type GradeFacts = { readonly dropped: readonly GradeFact[]; readonly added: readonly GradeFact[]; readonly reformed: number };
+
+export type FailedCitation = {
+  readonly source: string;
+  readonly address: string;
+  readonly quote: string;
+  readonly status: Exclude<CitationStatus, "ok">;
+  readonly foundAt?: string | undefined;
+  readonly line?: number | undefined;
+};
+
+export type GradeCitations = { readonly checked: number; readonly failed: readonly FailedCitation[] };
+
+/** A fact of the output that a retrieved passage states, and the first passage (0-based) that does. */
+export type SupportedFact = { readonly kind: AtomKind; readonly text: string; readonly line: number; readonly passage: number };
+
+/** The output's checkable facts against the item's retrieved passages (`contexts`). */
+export type GradeContexts = {
+  readonly passages: number;
+  /** Facts of the kinds checked against the passages: numbers, dates, times, URLs, code, names and quotations. */
+  readonly checked: number;
+  readonly supported: readonly SupportedFact[];
+  /** Facts no passage states; a quotation not found word for word in any passage. */
+  readonly unsupported: readonly GradeFact[];
+  /** Sentences that state no checkable fact: whether a passage supports them needs reading, which chaff does not do. */
+  readonly uncheckedSentences: number;
+};
+
+/** A rule, or a check (`compare`, `cite`), that did not run on this output, and why. */
+export type NotRunEntry = { readonly rule: string; readonly reason: string };
+
+/** What makes two results comparable (spec §29.7): the same rules and settings. */
+export type Stamp = { readonly chaff: string; readonly rules: string; readonly settings: string };
+
+/** One finding's penalty points, so every point names the finding it came from. */
+export type ScoreItem = { readonly points: number; readonly rule: string; readonly line: number };
+
+/** The penalty sum under a `grade:` rubric (spec §29.4). Never a score out of a maximum. */
+export type GradeScore = { readonly penalty: number; readonly items: readonly ScoreItem[] };
+
+export type GradeResult = {
+  readonly id: string;
+  /** The item's variant label, when it had one. */
+  readonly variant?: string | undefined;
+  readonly language: string;
+  readonly genre: string;
+  readonly size: OutputSize;
+  readonly findings: readonly GradeFinding[];
+  readonly rates: Readonly<Record<string, number>>;
+  readonly notRun: readonly NotRunEntry[];
+  /** Null when the item had no reference. */
+  readonly facts: GradeFacts | null;
+  /** Null when the item gave no citations. */
+  readonly citations: GradeCitations | null;
+  /** Only when the item gave contexts, so results of items without them keep their shape. */
+  readonly contexts?: GradeContexts | undefined;
+  /** Only with a `grade:` rubric. */
+  readonly score?: GradeScore | undefined;
+  readonly pass: boolean;
+  /** Each condition the output failed, with how far: `facts.dropped 3 > 0`. */
+  readonly failedBecause: readonly string[];
+  readonly stamp: Stamp;
+};

package/src/grade/results-read.ts

@@ -0,0 +1,111 @@
+import { resultKey } from "./result-name.ts";
+import type { GradeResult } from "./result.ts";
+
+// An earlier run's results (`--out`), read back for `--baseline`. Pure. Only lines chaff grade wrote are accepted: a
+// baseline that half-reads would compare against outputs that were never graded.
+
+const isRecord = (value: unknown): value is Record<string, unknown> => typeof value === "object" && value !== null && !Array.isArray(value);
+
+const isString = (value: unknown): value is string => typeof value === "string";
+
+const isNumber = (value: unknown): value is number => typeof value === "number" && Number.isFinite(value);
+
+const isArrayOf = (value: unknown, each: (entry: unknown) => boolean): boolean => Array.isArray(value) && value.every(each);
+
+const isFinding = (value: unknown): boolean =>
+  isRecord(value) && isString(value["rule"]) && isString(value["level"]) && isNumber(value["line"]) && isNumber(value["column"]) && isString(value["message"]);
+
+const isFact = (value: unknown): boolean =>
+  isRecord(value) &&
+  isString(value["kind"]) &&
+  isString(value["key"]) &&
+  isString(value["text"]) &&
+  isNumber(value["line"]) &&
+  typeof value["allowed"] === "boolean";
+
+const isFailedCitation = (value: unknown): boolean =>
+  isRecord(value) && isString(value["source"]) && isString(value["address"]) && isString(value["quote"]) && isString(value["status"]);
+
+const isFacts = (value: unknown): boolean => value === null || (isRecord(value) && isArrayOf(value["dropped"], isFact) && isArrayOf(value["added"], isFact));
+
+const isCitations = (value: unknown): boolean =>
+  value === null || (isRecord(value) && isNumber(value["checked"]) && isArrayOf(value["failed"], isFailedCitation));
+
+const isSupported = (value: unknown): boolean =>
+  isRecord(value) && isString(value["kind"]) && isString(value["text"]) && isNumber(value["line"]) && isNumber(value["passage"]);
+
+/** Absent when the item gave no contexts. */
+const isContexts = (value: unknown): boolean =>
+  value === undefined ||
+  (isRecord(value) &&
+    isNumber(value["passages"]) &&
+    isNumber(value["checked"]) &&
+    isArrayOf(value["supported"], isSupported) &&
+    isArrayOf(value["unsupported"], isFact) &&
+    isNumber(value["uncheckedSentences"]));
+
+const isStamp = (value: unknown): boolean => isRecord(value) && isString(value["chaff"]) && isString(value["rules"]) && isString(value["settings"]);
+
+const isSize = (value: unknown): boolean => isRecord(value) && (value["unit"] === "char" || value["unit"] === "word") && isNumber(value["value"]);
+
+const isScoreItem = (value: unknown): boolean => isRecord(value) && isNumber(value["points"]) && isString(value["rule"]) && isNumber(value["line"]);
+
+const isScore = (value: unknown): boolean => value === undefined || (isRecord(value) && isNumber(value["penalty"]) && isArrayOf(value["items"], isScoreItem));
+
+const isNotRun = (value: unknown): boolean => isRecord(value) && isString(value["rule"]) && isString(value["reason"]);
+
+const isRates = (value: unknown): boolean => isRecord(value) && Object.values(value).every(isNumber);
+
+/** The id, and the variant when the output had one. */
+const isLabel = (value: Record<string, unknown>): boolean => isString(value["id"]) && (value["variant"] === undefined || isString(value["variant"]));
+
+/** A line `--out` writes, every field checked: a line that only looks like one would be compared as if it had been graded. */
+export const isGradeResult = (value: unknown): value is GradeResult =>
+  isRecord(value) &&
+  isLabel(value) &&
+  isString(value["language"]) &&
+  isString(value["genre"]) &&
+  isSize(value["size"]) &&
+  isArrayOf(value["findings"], isFinding) &&
+  isRates(value["rates"]) &&
+  isArrayOf(value["notRun"], isNotRun) &&
+  isFacts(value["facts"]) &&
+  isCitations(value["citations"]) &&
+  isContexts(value["contexts"]) &&
+  isScore(value["score"]) &&
+  typeof value["pass"] === "boolean" &&
+  isArrayOf(value["failedBecause"], isString) &&
+  isStamp(value["stamp"]);
+
+const parsedLine = (body: string): unknown => {
+  try {
+    return JSON.parse(body);
+  } catch {
+    return undefined;
+  }
+};
+
+/** The lines whose id an earlier line of the same variant already has: one run never grades two outputs under one id. */
+const repeatedIds = (read: readonly { readonly line: number; readonly value: unknown }[]): number[] => {
+  const seen = new Set<string>();
+  return read.flatMap(({ line, value }) => {
+    if (!isGradeResult(value)) return [];
+    const repeated = seen.has(resultKey(value));
+    seen.add(resultKey(value));
+    return repeated ? [line] : [];
+  });
+};
+
+/** Every result line, or the line numbers that are not one (or repeat an id). No result at all is not a baseline. */
+export const parseResults = (text: string): { readonly results: readonly GradeResult[] } | { readonly badLines: readonly number[] } => {
+  const lines = text
+    .split(/\r?\n/u)
+    .map((body, index) => ({ body, line: index + 1 }))
+    .filter(({ body }) => body.trim() !== "");
+  const read = lines.map(({ body, line }) => ({ line, value: parsedLine(body) }));
+  const badLines = [...read.filter((entry) => !isGradeResult(entry.value)).map((entry) => entry.line), ...repeatedIds(read)].toSorted(
+    (left, right) => left - right,
+  );
+  if (badLines.length > 0 || read.length === 0) return { badLines: read.length === 0 ? [0] : badLines };
+  return { results: read.map((entry) => entry.value).filter(isGradeResult) };
+};

package/src/grade/rubric-verdict.ts

@@ -0,0 +1,86 @@
+import { exactRateOf, type OutputSize } from "./rates.ts";
+import type { GradeScore } from "./result.ts";
+import type { Rubric, RuleLimit } from "./rubric.ts";
+import { counted, overLimit, verdictOf, type Graded, type Verdict } from "./verdict.ts";
+
+// Pass or fail and the penalty score under a `grade:` rubric (spec §29.4). Only what the rubric writes decides: an error
+// finding of a rule the rubric does not name is a rate, not a failure, so the reason for a failure is always in `grade:`.
+
+/** What the rubric reads of one output beyond its findings, facts and citations. */
+export type RubricInput = Graded & {
+  readonly size: OutputSize;
+  /** The required sections the output has no heading for. */
+  readonly missingSections: readonly string[];
+  /** The item gave sources but no citations. */
+  readonly uncited: boolean;
+};
+
+const countOf = (input: RubricInput, rule: string): number => input.findings.filter((finding) => finding.rule === rule).length;
+
+/** A rate as a failure reason shows it: to three decimals, enough to tell it from a limit written to one. */
+const shownRate = (rate: number): string => String(Math.round(rate * RATE_SHOWN_SCALE) / RATE_SHOWN_SCALE);
+
+const RATE_SHOWN_SCALE = 1000;
+
+const ruleFailures = (input: RubricInput, rule: string, limit: RuleLimit): string[] => {
+  const count = countOf(input, rule);
+  const rate = exactRateOf(count, input.size.value);
+  return [
+    ...(limit.max === undefined ? [] : overLimit(`rules.${rule}`, count, limit.max)),
+    ...(limit.maxRate === undefined || rate === undefined || rate <= limit.maxRate ? [] : [`rules.${rule}.rate ${shownRate(rate)} > ${String(limit.maxRate)}`]),
+  ];
+};
+
+const sectionFailures = (input: RubricInput): string[] =>
+  overLimit("required_sections.missing", input.missingSections.length, 0).map((reason) => `${reason}: ${input.missingSections.join(", ")}`);
+
+const factFailures = (input: RubricInput, rubric: Rubric): string[] => {
+  const limits = rubric.facts;
+  if (limits === undefined || input.facts === null) return [];
+  return [
+    ...(limits.dropped === undefined ? [] : overLimit("facts.dropped", counted(input.facts.dropped), limits.dropped)),
+    ...(limits.added === undefined ? [] : overLimit("facts.added", counted(input.facts.added), limits.added)),
+  ];
+};
+
+const citationFailures = (input: RubricInput, rubric: Rubric): string[] => {
+  const limits = rubric.citations;
+  if (limits === undefined) return [];
+  return [
+    ...(limits.failed === undefined || input.citations === null ? [] : overLimit("citations.failed", input.citations.failed.length, limits.failed)),
+    ...(limits.required && input.uncited ? ["citations.required: sources given, no citations"] : []),
+  ];
+};
+
+const contextFailures = (input: RubricInput, rubric: Rubric): string[] => {
+  const limits = rubric.contexts;
+  if (limits === undefined) return [];
+  return [
+    ...(limits.unsupported === undefined || input.contexts === undefined
+      ? []
+      : overLimit("contexts.unsupported", counted(input.contexts.unsupported), limits.unsupported)),
+    ...(limits.required && input.contexts === undefined ? ["contexts.required: no contexts given"] : []),
+  ];
+};
+
+/** One item per finding of a weighted rule, so every point names the finding it came from; the penalty is their sum. */
+export const scoreOf = (input: Pick<RubricInput, "findings">, rubric: Rubric): GradeScore => {
+  const items = input.findings.flatMap((finding) => {
+    const weight = rubric.rules[finding.rule]?.weight;
+    return weight === undefined || weight === 0 ? [] : [{ points: weight, rule: finding.rule, line: finding.line }];
+  });
+  return { penalty: items.reduce((sum, item) => sum + item.points, 0), items };
+};
+
+export const rubricVerdict = (input: RubricInput, rubric: Rubric): { readonly verdict: Verdict; readonly score: GradeScore } => {
+  const score = scoreOf(input, rubric);
+  const failures = [
+    ...Object.entries(rubric.rules).flatMap(([rule, limit]) => ruleFailures(input, rule, limit)),
+    ...sectionFailures(input),
+    ...factFailures(input, rubric),
+    ...citationFailures(input, rubric),
+    ...contextFailures(input, rubric),
+    ...(rubric.penalty === undefined ? [] : overLimit("score.penalty", score.penalty, rubric.penalty)),
+  ];
+  return { verdict: verdictOf(failures), score };
+};

package/src/grade/rubric.ts

@@ -0,0 +1,192 @@
+import { isAtomKind, type AtomKind } from "../compare/atom.ts";
+
+// The `grade:` section of chaff.yaml (spec §29.4), read and checked. Pure. A section that cannot be read stops the run
+// (exit 2): a rubric that silently checks less than was written would pass outputs it was meant to fail.
+
+export type RuleLimit = {
+  /** Findings allowed per output. */
+  readonly max?: number | undefined;
+  /** Findings allowed per 1,000 characters or words. */
+  readonly maxRate?: number | undefined;
+  /** Penalty points per finding. */
+  readonly weight?: number | undefined;
+};
+
+type FactLimits = {
+  readonly dropped?: number | undefined;
+  readonly added?: number | undefined;
+  readonly allowDropped: readonly AtomKind[];
+  readonly allowAdded: readonly AtomKind[];
+};
+
+type CitationLimits = { readonly failed?: number | undefined; readonly required: boolean };
+
+type ContextLimits = { readonly unsupported?: number | undefined; readonly allowUnsupported: readonly AtomKind[]; readonly required: boolean };
+
+export type Rubric = {
+  readonly rules: Readonly<Record<string, RuleLimit>>;
+  /** Undefined when not written: chaff.yaml's top-level required_sections apply. */
+  readonly requiredSections?: readonly string[] | undefined;
+  readonly facts?: FactLimits | undefined;
+  readonly citations?: CitationLimits | undefined;
+  readonly contexts?: ContextLimits | undefined;
+  readonly penalty?: number | undefined;
+};
+
+/** What a value under `grade:` should have been. */
+export type Expected = "map" | "count" | "number" | "boolean" | "words" | "kinds" | "known-key";
+
+/** Where under `grade:` (a dotted path), what it should be, and what was written. */
+export type RubricProblem = { readonly path: string; readonly expected: Expected; readonly written: string };
+
+type Read<T> = { readonly value: T; readonly problems: readonly RubricProblem[] };
+
+const isRecord = (value: unknown): value is Record<string, unknown> => typeof value === "object" && value !== null && !Array.isArray(value);
+
+const printed = (value: unknown): string => JSON.stringify(value) ?? String(value);
+
+const problem = (path: string, expected: Expected, written: unknown): RubricProblem => ({ path, expected, written: printed(written) });
+
+const ok = <T>(value: T): Read<T> => ({ value, problems: [] });
+
+const isCount = (value: unknown): value is number => Number.isInteger(value) && typeof value === "number" && value >= 0;
+
+const isAmount = (value: unknown): value is number => typeof value === "number" && Number.isFinite(value) && value >= 0;
+
+/** An optional number under `path`: a count (a whole number) or an amount (any number from 0). */
+const optional = (raw: Record<string, unknown>, key: string, path: string, expected: "count" | "number"): Read<number | undefined> => {
+  const value = raw[key];
+  if (value === undefined) return ok(undefined);
+  const valid = expected === "count" ? isCount : isAmount;
+  return valid(value) ? ok(value) : { value: undefined, problems: [problem(`${path}.${key}`, expected, value)] };
+};
+
+/** Keys that are not among `known`. A misspelt `max_rte` would otherwise be a limit nobody checks. */
+const unknownKeys = (raw: Record<string, unknown>, known: readonly string[], path: string): RubricProblem[] =>
+  Object.keys(raw)
+    .filter((key) => !known.includes(key))
+    .map((key) => problem(`${path}.${key}`, "known-key", raw[key]));
+
+const ruleLimit = (raw: unknown, path: string): Read<RuleLimit> => {
+  if (!isRecord(raw)) return { value: {}, problems: [problem(path, "map", raw)] };
+  const max = optional(raw, "max", path, "count");
+  const maxRate = optional(raw, "max_rate", path, "number");
+  const weight = optional(raw, "weight", path, "number");
+  return {
+    value: { max: max.value, maxRate: maxRate.value, weight: weight.value },
+    problems: [...max.problems, ...maxRate.problems, ...weight.problems, ...unknownKeys(raw, ["max", "max_rate", "weight"], path)],
+  };
+};
+
+const rulesOf = (raw: unknown): Read<Record<string, RuleLimit>> => {
+  if (raw === undefined) return ok({});
+  if (!isRecord(raw)) return { value: {}, problems: [problem("grade.rules", "map", raw)] };
+  const read = Object.entries(raw).map(([id, limit]) => ({ id, read: ruleLimit(limit, `grade.rules.${id}`) }));
+  return { value: Object.fromEntries(read.map((entry) => [entry.id, entry.read.value])), problems: read.flatMap((entry) => entry.read.problems) };
+};
+
+const wordsOf = (raw: unknown, path: string): Read<string[] | undefined> => {
+  if (raw === undefined) return ok(undefined);
+  const valid = Array.isArray(raw) && raw.every((entry) => typeof entry === "string" && entry.trim() !== "");
+  return valid ? ok(raw.map((entry: string) => entry.trim())) : { value: undefined, problems: [problem(path, "words", raw)] };
+};
+
+const kindsOf = (raw: unknown, path: string): Read<AtomKind[]> => {
+  if (raw === undefined) return ok([]);
+  const entries: readonly unknown[] = Array.isArray(raw) ? raw : [raw];
+  const kinds = entries.filter((entry): entry is AtomKind => typeof entry === "string" && isAtomKind(entry));
+  return kinds.length === entries.length ? ok(kinds) : { value: kinds, problems: [problem(path, "kinds", raw)] };
+};
+
+const factsOf = (raw: unknown): Read<FactLimits | undefined> => {
+  if (raw === undefined) return ok(undefined);
+  if (!isRecord(raw)) return { value: undefined, problems: [problem("grade.facts", "map", raw)] };
+  const dropped = optional(raw, "dropped", "grade.facts", "count");
+  const added = optional(raw, "added", "grade.facts", "count");
+  const allowDropped = kindsOf(raw["allow_dropped"], "grade.facts.allow_dropped");
+  const allowAdded = kindsOf(raw["allow_added"], "grade.facts.allow_added");
+  return {
+    value: { dropped: dropped.value, added: added.value, allowDropped: allowDropped.value, allowAdded: allowAdded.value },
+    problems: [
+      ...dropped.problems,
+      ...added.problems,
+      ...allowDropped.problems,
+      ...allowAdded.problems,
+      ...unknownKeys(raw, ["dropped", "added", "allow_dropped", "allow_added"], "grade.facts"),
+    ],
+  };
+};
+
+/** `required: true|false` under `path`: false when not written. */
+const requiredOf = (raw: Record<string, unknown>, path: string): Read<boolean> => {
+  const required = raw["required"];
+  return required === undefined || typeof required === "boolean"
+    ? ok(required === true)
+    : { value: false, problems: [problem(`${path}.required`, "boolean", required)] };
+};
+
+const citationsOf = (raw: unknown): Read<CitationLimits | undefined> => {
+  if (raw === undefined) return ok(undefined);
+  if (!isRecord(raw)) return { value: undefined, problems: [problem("grade.citations", "map", raw)] };
+  const failed = optional(raw, "failed", "grade.citations", "count");
+  const required = requiredOf(raw, "grade.citations");
+  return {
+    value: { failed: failed.value, required: required.value },
+    problems: [...failed.problems, ...required.problems, ...unknownKeys(raw, ["failed", "required"], "grade.citations")],
+  };
+};
+
+const contextsOf = (raw: unknown): Read<ContextLimits | undefined> => {
+  if (raw === undefined) return ok(undefined);
+  if (!isRecord(raw)) return { value: undefined, problems: [problem("grade.contexts", "map", raw)] };
+  const unsupported = optional(raw, "unsupported", "grade.contexts", "count");
+  const allowUnsupported = kindsOf(raw["allow_unsupported"], "grade.contexts.allow_unsupported");
+  const required = requiredOf(raw, "grade.contexts");
+  return {
+    value: { unsupported: unsupported.value, allowUnsupported: allowUnsupported.value, required: required.value },
+    problems: [
+      ...unsupported.problems,
+      ...allowUnsupported.problems,
+      ...required.problems,
+      ...unknownKeys(raw, ["unsupported", "allow_unsupported", "required"], "grade.contexts"),
+    ],
+  };
+};
+
+const RUBRIC_KEYS: readonly string[] = ["rules", "required_sections", "facts", "citations", "contexts", "penalty"];
+
+/**
+ * The rubric, or every problem in it. No `grade:` key is no rubric: the default pass or fail applies (spec §29.3). A bare
+ * `grade:` (null) is an empty rubric, so writing the key alone still turns the default off, as the spec says it does.
+ */
+export const parseRubric = (raw: unknown): { readonly rubric: Rubric | undefined } | { readonly problems: readonly RubricProblem[] } => {
+  if (raw === undefined) return { rubric: undefined };
+  if (raw === null) return { rubric: { rules: {} } };
+  if (!isRecord(raw)) return { problems: [problem("grade", "map", raw)] };
+  const rules = rulesOf(raw["rules"]);
+  const sections = wordsOf(raw["required_sections"], "grade.required_sections");
+  const facts = factsOf(raw["facts"]);
+  const citations = citationsOf(raw["citations"]);
+  const contexts = contextsOf(raw["contexts"]);
+  const penalty = optional(raw, "penalty", "grade", "number");
+  const problems = [
+    ...rules.problems,
+    ...sections.problems,
+    ...facts.problems,
+    ...citations.problems,
+    ...contexts.problems,
+    ...penalty.problems,
+    ...unknownKeys(raw, RUBRIC_KEYS, "grade"),
+  ];
+  if (problems.length > 0) return { problems };
+  return {
+    rubric: {
+      rules: rules.value,
+      requiredSections: sections.value,
+      facts: facts.value,
+      citations: citations.value,
+      contexts: contexts.value,
+      penalty: penalty.value,
+    },
+  };
+};

package/src/grade/scorer.ts

@@ -0,0 +1,33 @@
+import { tally } from "./order.ts";
+import type { GradeResult } from "./result.ts";
+
+// The common scorer shape of eval frameworks (promptfoo, autoevals, evalite, DeepEval, Inspect AI…) as a projection of
+// one result: a score, pass, a reason a person can read, and the whole breakdown as metadata. Spec §29.6.
+
+export type ChaffScore = {
+  readonly name: "chaff";
+  /** 1 when the output passed, 0 when it failed. Never the penalty mapped onto 0–1: chaff sets no full marks (§29.4). */
+  readonly score: 0 | 1;
+  readonly pass: boolean;
+  readonly reason: string;
+  readonly metadata: Omit<GradeResult, "pass">;
+};
+
+const findingsWord = (count: number): string => (count === 1 ? "finding" : "findings");
+
+/** The findings in one phrase: "3 findings: closing-cliche ×2, ai-tell ×1". */
+const findingsPhrase = (result: GradeResult): string => {
+  const counts = Object.entries(tally(result.findings.map((finding) => finding.rule))).map(([rule, count]) => `${rule} ×${String(count)}`);
+  return counts.length === 0 ? "no findings" : `${String(result.findings.length)} ${findingsWord(result.findings.length)}: ${counts.join(", ")}`;
+};
+
+const penaltyPhrase = (result: GradeResult): string[] => (result.score === undefined ? [] : [`penalty ${String(result.score.penalty)}`]);
+
+/** Why it passed or failed, then what was found: the failed conditions first, since they decide the score. */
+export const reasonOf = (result: GradeResult): string =>
+  [result.pass ? "passed" : `failed: ${result.failedBecause.join("; ")}`, findingsPhrase(result), ...penaltyPhrase(result)].join(" — ");
+
+export const toScorer = (result: GradeResult): ChaffScore => {
+  const { pass, ...rest } = result;
+  return { name: "chaff", score: pass ? 1 : 0, pass, reason: reasonOf(result), metadata: rest };
+};

package/src/grade/setup.ts

@@ -0,0 +1,43 @@
+import { loadAdapter } from "../adapter-load.ts";
+import type { Config } from "../config/load.ts";
+import { rulesOf } from "../custom/load.ts";
+import { loadGenres } from "../genre-load.ts";
+import { loadProfiles } from "../profile/load.ts";
+import { VERSION_LINES } from "../version.ts";
+import type { GradeSetup } from "./grade-item.ts";
+import { compareText } from "./order.ts";
+import type { Rubric } from "./rubric.ts";
+import { stampOf, type RuleSet } from "./stamp.ts";
+
+/** The languages every stamp covers, whatever a run's outputs are written in, so two runs over different outputs stay comparable. */
+const BUNDLED_LANGUAGES: readonly string[] = ["en", "ja"];
+
+/** The rules each language runs and the word lists they read, with the genres' presets. Loads each language package. */
+const ruleSetOf = async (config: Config, languages: readonly string[]): Promise<RuleSet> => {
+  const read = await Promise.all(
+    languages.map(async (language) => {
+      const adapter = await loadAdapter(language);
+      return { language, rules: rulesOf(language, config), lexicons: { ...adapter.lexicons, ...config.extensions?.lexicons[language] } };
+    }),
+  );
+  return {
+    rules: Object.fromEntries(read.map((entry) => [entry.language, entry.rules])),
+    lexicons: Object.fromEntries(read.map((entry) => [entry.language, entry.lexicons])),
+    genres: loadGenres(),
+    profiles: loadProfiles(),
+  };
+};
+
+/** A run's settings, rubric and stamp. `languages`: those the items name, beyond the bundled ones. Throws when a language package cannot be loaded. */
+export const gradeSetup = async (
+  config: Config,
+  run: { readonly experimental: boolean; readonly genre: string | undefined },
+  languages: readonly string[],
+  rubric: Rubric | undefined,
+): Promise<GradeSetup> => {
+  // chaff.yaml's language and its by_path languages are loaded here too, so a missing package stops the run before grading.
+  const named = [config.language, ...config.byPath.map((rule) => rule.language)].filter((language) => language !== undefined);
+  const all = [...new Set([...BUNDLED_LANGUAGES, ...named, ...languages])].toSorted(compareText);
+  const settings = { config, experimental: run.experimental, genre: run.genre };
+  return { ...settings, rubric, stamp: stampOf(VERSION_LINES, await ruleSetOf(config, all), settings) };
+};

package/src/grade/stamp.ts

@@ -0,0 +1,60 @@
+import { createHash } from "node:crypto";
+import type { Config } from "../config/load.ts";
+import { sortedByKey } from "./order.ts";
+import type { Stamp } from "./result.ts";
+
+// The reproducibility stamp (spec §29.7). Two results are compared only when their rules and settings hashes match;
+// the chaff version is recorded but not compared, so a release that only rewords messages keeps old results usable.
+
+const isRecord = (value: unknown): value is Record<string, unknown> => typeof value === "object" && value !== null && !Array.isArray(value);
+
+/** JSON with every object's keys sorted, so the order a file or a map was written in does not change the hash. */
+export const canonicalJson = (value: unknown): string =>
+  JSON.stringify(value, (_key, inner: unknown) => (isRecord(inner) ? sortedByKey(Object.entries(inner)) : inner)) ?? "null";
+
+export const digestOf = (value: unknown): string => `sha256:${createHash("sha256").update(canonicalJson(value)).digest("hex")}`;
+
+/** What decides which findings a run gives, other than the rules: chaff.yaml as it applies, the genre and --experimental. */
+export type GradeSettings = {
+  readonly config: Config;
+  readonly experimental: boolean;
+  /** --genre, which wins over chaff.yaml for every item that does not name its own. */
+  readonly genre: string | undefined;
+};
+
+/** The parts of chaff.yaml that change a finding, a fact or a verdict. Paths and the AI judge do not. */
+export const settingsOf = (settings: GradeSettings): Record<string, unknown> => {
+  const { config } = settings;
+  return {
+    experimental: settings.experimental,
+    genre: settings.genre ?? config.genre ?? null,
+    language: config.language ?? null,
+    profile: config.profile ?? null,
+    rules: config.rules,
+    limits: config.limits,
+    options: config.options ?? {},
+    style: config.style ?? null,
+    styleApplied: config.applied ?? null,
+    jargon: config.jargon,
+    prefer: config.prefer,
+    requiredSections: config.requiredSections,
+    names: config.names,
+    byPath: config.byPath,
+    // A bare `grade:` reads as null and is an empty rubric, which grades otherwise than no `grade:` at all.
+    grade: config.grade === undefined ? "none" : config.grade,
+  };
+};
+
+/** What the rules are: each language's rule definitions and word lists, the genres' presets and the document profiles. */
+export type RuleSet = {
+  readonly rules: Readonly<Record<string, unknown>>;
+  readonly lexicons: Readonly<Record<string, unknown>>;
+  readonly genres: unknown;
+  readonly profiles: unknown;
+};
+
+export const stampOf = (versionLines: readonly string[], ruleSet: RuleSet, settings: GradeSettings): Stamp => ({
+  chaff: versionLines.join(", "),
+  rules: digestOf(ruleSet),
+  settings: digestOf(settingsOf(settings)),
+});