chaffjs 0.33.0 → 0.35.0

package/src/grade/expect.ts

@@ -0,0 +1,74 @@
+import type { Atom } from "../compare/atom.ts";
+import { outcomeOf, type Compared } from "../compare/outcome.ts";
+
+// An item's known answer (`expect`): facts the output must state and facts it must not. Pure: the shape of the field,
+// which one fact an entry states, and whether the output states it, decided as `chaff compare --distinct` decides it.
+
+export type GradeExpect = { readonly facts: readonly string[]; readonly absent: readonly string[] };
+
+export const EXPECT_LISTS = ["facts", "absent"] as const;
+
+export type ExpectList = (typeof EXPECT_LISTS)[number];
+
+const isExpectList = (key: string): key is ExpectList => EXPECT_LISTS.some((list) => list === key);
+
+const isRecord = (value: unknown): value is Record<string, unknown> => typeof value === "object" && value !== null && !Array.isArray(value);
+
+const isEntryList = (value: unknown): value is string[] => Array.isArray(value) && value.every((entry) => typeof entry === "string" && entry.trim() !== "");
+
+/**
+ * The field as written, or the path of its first wrong part. A key other than facts and absent is wrong, not ignored:
+ * a misspelt list would check nothing and pass every output. A missing list is an empty one.
+ */
+export const expectOf = (raw: unknown): { readonly expect: GradeExpect | undefined } | { readonly wrong: string } => {
+  if (raw === undefined) return { expect: undefined };
+  if (!isRecord(raw)) return { wrong: "expect" };
+  const unknown = Object.keys(raw).find((key) => !isExpectList(key));
+  if (unknown !== undefined) return { wrong: `expect.${unknown}` };
+  const wrongList = EXPECT_LISTS.find((list) => raw[list] !== undefined && !isEntryList(raw[list]));
+  if (wrongList !== undefined) return { wrong: `expect.${wrongList}` };
+  const listOf = (list: ExpectList): readonly string[] => {
+    const value = raw[list];
+    return isEntryList(value) ? value : [];
+  };
+  return { expect: { facts: listOf("facts"), absent: listOf("absent") } };
+};
+
+/** The one fact an entry states, or why it states none or several. */
+export type EntryFact = { readonly fact: Atom } | { readonly unread: "no-fact" | "several-facts" };
+
+/**
+ * The fact an entry states. Several facts are one when exactly one of them is the whole entry: a quotation 「12,000円」
+ * is read as the quotation and the number inside it, and the entry means the quotation.
+ */
+export const factOfEntry = (entry: string, atoms: readonly Atom[]): EntryFact => {
+  const [only] = atoms;
+  if (atoms.length === 1 && only !== undefined) return { fact: only };
+  if (atoms.length === 0) return { unread: "no-fact" };
+  const whole = atoms.filter((atom) => atom.text === entry.trim());
+  const [wholeOnly] = whole;
+  return whole.length === 1 && wholeOnly !== undefined ? { fact: wholeOnly } : { unread: "several-facts" };
+};
+
+/** Where an entry is in the item: `expect.facts[0]`. */
+export const entryPath = (list: ExpectList, index: number): string => `expect.${list}[${String(index)}]`;
+
+/** One entry as read: its list, its place, its text, and its document's facts with only the entry's fact kept. */
+export type ReadEntry = { readonly list: ExpectList; readonly index: number; readonly entry: string; readonly read: Compared };
+
+/** An entry that states one fact, compared as compare compares two documents: the output states it at least once. */
+export const statesFact = (entry: Compared, output: Compared): boolean => outcomeOf(entry, output, undefined, "distinct").dropped.length === 0;
+
+/** What the output was checked against: how many entries, the facts it does not state, and the absent ones it does. */
+export type GradeExpectResult = { readonly checked: number; readonly missing: readonly string[]; readonly present: readonly string[] };
+
+/** Each entry against the output, by the entry's text as written in the item. */
+export const expectAgainst = (entries: readonly ReadEntry[], output: Compared): GradeExpectResult => ({
+  checked: entries.length,
+  missing: entries.filter((entry) => entry.list === "facts" && !statesFact(entry.read, output)).map((entry) => entry.entry),
+  present: entries.filter((entry) => entry.list === "absent" && statesFact(entry.read, output)).map((entry) => entry.entry),
+});
+
+/** The failure reasons: `expect.missing 12,000円`, `expect.present 15,000円`. Absent when the item had no expect. */
+export const expectFailures = (result: GradeExpectResult | undefined): string[] =>
+  result === undefined ? [] : [...result.missing.map((entry) => `expect.missing ${entry}`), ...result.present.map((entry) => `expect.present ${entry}`)];

package/src/grade/grade-item.ts

@@ -10,11 +10,14 @@ import { ratesOf, sizeOf, type OutputSize } from "./rates.ts";
 import type { GradeCitations, GradeFacts, GradeFinding, GradeResult, GradeScore, NotRunEntry, Stamp } from "./result.ts";
 import type { Rubric } from "./rubric.ts";
 import { rubricVerdict } from "./rubric-verdict.ts";
+import { rubricForTask } from "./task-rubric.ts";
 import type { GradeSettings } from "./stamp.ts";
 import { GRADE_TEXT } from "./text.ts";
 import { aiScoreOfDocument } from "../ai-score/of-document.ts";
 import { gradeAiScoreOf } from "./ai-score.ts";
-import { defaultVerdict, type Verdict } from "./verdict.ts";
+import { defaultVerdict, verdictOf, type Verdict } from "./verdict.ts";
+import { expectFailures, type GradeExpectResult } from "./expect.ts";
+import { expectOfItem } from "./expect-read.ts";
 
 /** One run's settings, its `grade:` rubric when chaff.yaml has one, and its stamp, which every result carries. */
 export type GradeSetup = GradeSettings & { readonly stamp: Stamp; readonly rubric?: Rubric | undefined };
@@ -34,6 +37,7 @@ type Checked = {
   readonly facts: GradeFacts | null;
   readonly cited: CitationsRead | undefined;
   readonly contexts: ContextsRead | undefined;
+  readonly expect: GradeExpectResult | undefined;
 };
 
 const checkItem = async (item: GradeItem, setup: GradeSetup): Promise<Checked> => {
@@ -50,6 +54,7 @@ const checkItem = async (item: GradeItem, setup: GradeSetup): Promise<Checked> =
     facts: item.reference === undefined ? null : await factsAgainst(texts, reading, allowedBy(setup.rubric)),
     cited: item.citations === undefined ? undefined : await citationsAgainst(item, item.citations, reading, ui),
     contexts: await contextsOf(item, check.language, reading, setup.rubric),
+    expect: await expectOfItem(item, { ...reading, language: check.language }),
   };
 };
 
@@ -97,12 +102,17 @@ const judge = (item: GradeItem, checked: Checked, setup: GradeSetup): { readonly
 };
 
 /** Grades one output (spec §29.3): its findings and their rates, the facts against its reference, its quotations, and pass or fail. */
-export const gradeItem = async (item: GradeItem, setup: GradeSetup): Promise<GradeResult> => {
+export const gradeItem = async (item: GradeItem, runSetup: GradeSetup): Promise<GradeResult> => {
+  const setup = item.task === undefined ? runSetup : { ...runSetup, rubric: rubricForTask(runSetup.rubric, item.task) };
   const checked = await checkItem(item, setup);
-  const { verdict, score } = judge(item, checked, setup);
+  const judged = judge(item, checked, setup);
+  const { score } = judged;
+  // expect is the item's own answer, so it fails the output with or without a `grade:` rubric.
+  const verdict = verdictOf([...judged.verdict.failedBecause, ...expectFailures(checked.expect)]);
   return {
     id: item.id,
     ...(item.variant === undefined ? {} : { variant: item.variant }),
+    ...(item.task === undefined ? {} : { task: item.task }),
     language: checked.check.language,
     genre: checked.check.genre.genre,
     size: checked.size,
@@ -115,6 +125,7 @@ export const gradeItem = async (item: GradeItem, setup: GradeSetup): Promise<Gra
     facts: checked.facts,
     citations: citationsOf(checked),
     ...(checked.contexts === undefined ? {} : { contexts: checked.contexts.contexts }),
+    ...(checked.expect === undefined ? {} : { expect: checked.expect }),
     ...(score === undefined ? {} : { score }),
     aiScore: gradeAiScoreOf(aiScoreOfDocument(checked.check.doc, checked.check.rules, checked.check.genre.genre)),
     pass: verdict.pass,

package/src/grade/item.ts

@@ -1,5 +1,6 @@
 // One line of `chaff grade`'s input: a model's output and what to check it against. Pure: reads text, returns items
 // or what is wrong with them, so a harness that writes the file can be told which line to fix.
+import { expectOf, type GradeExpect } from "./expect.ts";
 import { resultKey } from "./result-name.ts";
 
 export type GradeCitation = { readonly source: string; readonly address: string; readonly quote: string };
@@ -18,10 +19,14 @@ export type GradeItem = {
   readonly contexts?: readonly string[] | undefined;
   /** The prompt, model or setting that produced this output. Outputs of several variants are compared on the same ids. */
   readonly variant?: string | undefined;
+  /** The rubric task this item is graded under (`grade.tasks.<name>`). Undefined: the top-level rubric. */
+  readonly task?: string | undefined;
+  /** Facts the output must state and facts it must not: a question's known answer. Undefined when not given. */
+  readonly expect?: GradeExpect | undefined;
 };
 
 /** The fields an item already has a meaning for: none of them can name a variant. */
-export const ITEM_FIELDS: readonly string[] = ["id", "output", "reference", "sources", "citations", "contexts", "language", "genre"];
+export const ITEM_FIELDS: readonly string[] = ["id", "output", "reference", "sources", "citations", "contexts", "language", "genre", "expect", "task"];
 
 /** Which field names an output's variant, and whether every line must have it (the key was given on the command line). */
 export type VariantField = { readonly key: string; readonly required: boolean };
@@ -44,13 +49,22 @@ export type ItemProblemKind =
   | "which-source"
   | "unknown-language"
   | "unknown-genre"
+  | "not-expect"
+  | "expect-no-fact"
+  | "expect-several-facts"
+  | "unknown-task"
   | "empty";
 
 /** Why a line cannot be graded. `detail`: the field, the value or the parser's message; `first`: the line an id was first on. */
 export type ItemProblem = { readonly kind: ItemProblemKind; readonly line: number; readonly detail?: string; readonly first?: number };
 
 /** What an item may name: the languages chaff can load and the genres it knows. */
-export type ItemVocabulary = { readonly isLanguage: (language: string) => boolean; readonly genres: readonly string[] };
+export type ItemVocabulary = {
+  readonly isLanguage: (language: string) => boolean;
+  readonly genres: readonly string[];
+  /** The tasks the rubric defines. None given: an item naming a task is refused, as there is no rubric for it. */
+  readonly tasks?: readonly string[] | undefined;
+};
 
 const isRecord = (value: unknown): value is Record<string, unknown> => typeof value === "object" && value !== null && !Array.isArray(value);
 
@@ -129,6 +143,27 @@ const variantOf = (raw: Record<string, unknown>, key: string, line: number): Par
   return typeof value === "string" && value !== "" ? { value } : { problem: { kind: "not-string", line, detail: key } };
 };
 
+/** The item's task: a non-empty string the rubric defines, or none. Never a silent fall back to the top-level rubric. */
+const taskOf = (raw: unknown, vocabulary: ItemVocabulary, line: number): Parsed<string | undefined> => {
+  if (raw === undefined) return { value: undefined };
+  if (typeof raw !== "string" || raw === "") return { problem: { kind: "not-string", line, detail: "task" } };
+  return (vocabulary.tasks ?? []).includes(raw) ? { value: raw } : { problem: { kind: "unknown-task", line, detail: raw } };
+};
+
+/** The item's variant and task, each written only when given, checked in that order. */
+const labelsOf = (
+  raw: Record<string, unknown>,
+  line: number,
+  vocabulary: ItemVocabulary,
+  variantKey: string,
+): Parsed<{ readonly variant?: string; readonly task?: string }> => {
+  const variant = variantOf(raw, variantKey, line);
+  if ("problem" in variant) return variant;
+  const task = taskOf(raw["task"], vocabulary, line);
+  if ("problem" in task) return task;
+  return { value: { ...(variant.value === undefined ? {} : { variant: variant.value }), ...(task.value === undefined ? {} : { task: task.value }) } };
+};
+
 const itemOf = (raw: Record<string, unknown>, line: number, vocabulary: ItemVocabulary, variantKey: string): Parsed<GradeItem> => {
   const id = raw["id"];
   if (typeof id !== "string" || id === "") return { problem: { kind: "no-id", line } };
@@ -138,17 +173,22 @@ const itemOf = (raw: Record<string, unknown>, line: number, vocabulary: ItemVoca
   if ("problem" in fields) return fields;
   const unknown = vocabularyProblem(fields.value, vocabulary, line);
   if (unknown !== undefined) return { problem: unknown };
-  const variant = variantOf(raw, variantKey, line);
-  if ("problem" in variant) return variant;
+  const labels = labelsOf(raw, line, vocabulary, variantKey);
+  if ("problem" in labels) return labels;
   const sources = sourcesOf(raw["sources"], line);
   if ("problem" in sources) return sources;
   const citations = citationsOf(raw["citations"], sources.value, line);
   if ("problem" in citations) return citations;
   const contexts = contextsOf(raw["contexts"], line);
   if ("problem" in contexts) return contexts;
+  const expect = expectOf(raw["expect"]);
+  if ("wrong" in expect) return { problem: { kind: "not-expect", line, detail: expect.wrong } };
   const passages = contexts.value === undefined ? {} : { contexts: contexts.value };
-  const labelled = variant.value === undefined ? {} : { variant: variant.value };
-  return { value: { id, output, ...fields.value, sources: sources.value, citations: citations.value, ...passages, ...labelled } };
+  const expected = expect.expect === undefined ? {} : { expect: expect.expect };
+  const { variant, task } = labels.value;
+  const labelled = variant === undefined ? {} : { variant };
+  const tasked = task === undefined ? {} : { task };
+  return { value: { id, output, ...fields.value, sources: sources.value, citations: citations.value, ...passages, ...labelled, ...expected, ...tasked } };
 };
 
 /** One item from a value already parsed, as `grade()` receives it; `line` is 0 when it came from no file. */
@@ -192,7 +232,7 @@ export const parseItems = (
   text: string,
   vocabulary: ItemVocabulary,
   variantField: VariantField = DEFAULT_VARIANT_FIELD,
-): { readonly items: readonly GradeItem[] } | { readonly problems: readonly ItemProblem[] } => {
+): { readonly items: readonly GradeItem[]; readonly lines: readonly number[] } | { readonly problems: readonly ItemProblem[] } => {
   const lines = text.split(/\r?\n/u).map((body, index) => ({ body, line: index + 1 }));
   const parsed = lines
     .filter(({ body }) => body.trim() !== "")
@@ -204,5 +244,5 @@ export const parseItems = (
     ...unlabelled(read, variantField),
   ];
   if (problems.length > 0) return { problems: problems.toSorted((left, right) => left.line - right.line) };
-  return read.length === 0 ? { problems: [{ kind: "empty", line: 0 }] } : { items: read.map((entry) => entry.item) };
+  return read.length === 0 ? { problems: [{ kind: "empty", line: 0 }] } : { items: read.map((entry) => entry.item), lines: read.map((entry) => entry.line) };
 };

package/src/grade/result.ts

@@ -3,6 +3,7 @@ import type { CitationStatus } from "../structure/cite.ts";
 import type { Severity } from "../plugin.ts";
 import type { OutputSize } from "./rates.ts";
 import type { GradeAiScore } from "./ai-score.ts";
+import type { GradeExpectResult } from "./expect.ts";
 
 // The shape of one line of `chaff grade --out`, and of what `grade()` returns. Spec §29.3.
 
@@ -62,6 +63,8 @@ export type GradeResult = {
   readonly id: string;
   /** The item's variant label, when it had one. */
   readonly variant?: string | undefined;
+  /** The rubric task the output was graded under, when the item named one. */
+  readonly task?: string | undefined;
   readonly language: string;
   readonly genre: string;
   readonly size: OutputSize;
@@ -74,6 +77,8 @@ export type GradeResult = {
   readonly citations: GradeCitations | null;
   /** Only when the item gave contexts, so results of items without them keep their shape. */
   readonly contexts?: GradeContexts | undefined;
+  /** Only when the item gave expect: the entries checked, the facts not stated and the absent ones stated. */
+  readonly expect?: GradeExpectResult | undefined;
   /** Only with a `grade:` rubric. */
   readonly score?: GradeScore | undefined;
   /** The AI-likeness quick score. Absent only in results written before it existed. Never part of pass or fail. */

package/src/grade/results-read.ts

@@ -72,8 +72,13 @@ const isRates = (value: unknown): boolean => isRecord(value) && Object.values(va
 /** The id, and the variant when the output had one. */
 const isLabel = (value: Record<string, unknown>): boolean => isString(value["id"]) && (value["variant"] === undefined || isString(value["variant"]));
 
-/** The parts a result carries only sometimes: contexts, a rubric's score, the quick score. Each is checked when present. */
-const hasOptionalParts = (value: Record<string, unknown>): boolean => isContexts(value["contexts"]) && isScore(value["score"]) && isAiScore(value["aiScore"]);
+/** Absent when the item gave no expect. */
+const isExpect = (value: unknown): boolean =>
+  value === undefined || (isRecord(value) && isNumber(value["checked"]) && isArrayOf(value["missing"], isString) && isArrayOf(value["present"], isString));
+
+/** The parts a result carries only sometimes: contexts, expect, a rubric's score, the quick score. Each is checked when present. */
+const hasOptionalParts = (value: Record<string, unknown>): boolean =>
+  isContexts(value["contexts"]) && isExpect(value["expect"]) && isScore(value["score"]) && isAiScore(value["aiScore"]);
 
 /** A line `--out` writes, every field checked: a line that only looks like one would be compared as if it had been graded. */
 export const isGradeResult = (value: unknown): value is GradeResult =>

package/src/grade/rubric-file.ts

@@ -0,0 +1,46 @@
+import { createHash } from "node:crypto";
+import { extname } from "node:path";
+import { parse } from "yaml";
+import { parseRubric, type Rubric, type RubricProblem } from "./rubric.ts";
+
+// A rubric file given with `chaff grade --rubric <file>`: the schema of chaff.yaml's `grade:`, at the file's top level.
+// Pure: the caller reads the file. Its paths drop the `grade.` prefix, because in the file they start at the top.
+
+type Format = "yaml" | "json";
+
+const FORMATS: Readonly<Record<string, Format>> = { ".yaml": "yaml", ".yml": "yaml", ".json": "json" };
+
+export type RubricFileProblem =
+  { readonly kind: "format" } | { readonly kind: "syntax"; readonly why: string } | { readonly kind: "rubric"; readonly problem: RubricProblem };
+
+export type RubricFile = { readonly rubric: Rubric; readonly digest: string } | { readonly problems: readonly RubricFileProblem[] };
+
+const formatOf = (path: string): Format | undefined => FORMATS[extname(path).toLowerCase()];
+
+const syntaxOf = (content: string, format: Format): { readonly raw: unknown } | { readonly why: string } => {
+  try {
+    // An empty file reads as null in YAML but throws in JSON; both are an empty rubric, as a bare `grade:` is.
+    if (content.trim() === "") return { raw: null };
+    return { raw: format === "json" ? JSON.parse(content) : parse(content) };
+  } catch (error) {
+    return { why: error instanceof Error ? error.message : String(error) };
+  }
+};
+
+/** `grade.rules.x` in chaff.yaml is `rules.x` in a rubric file; `grade` itself is the file's top level, the empty path. */
+export const filePathOf = (gradePath: string): string => (gradePath === "grade" ? "" : gradePath.replace(/^grade\./u, ""));
+
+/** The sha256 of the file's bytes as read, recorded in the stamp so runs under different rubric files are not compared silently. */
+export const rubricFileDigest = (content: string): string => `sha256:${createHash("sha256").update(content).digest("hex")}`;
+
+/** The rubric a file holds, or every problem with it. An empty file is an empty rubric, as a bare `grade:` is. */
+export const parseRubricFile = (path: string, content: string): RubricFile => {
+  const format = formatOf(path);
+  if (format === undefined) return { problems: [{ kind: "format" }] };
+  const read = syntaxOf(content, format);
+  if ("why" in read) return { problems: [{ kind: "syntax", why: read.why }] };
+  const parsed = parseRubric(read.raw ?? null);
+  if ("problems" in parsed)
+    return { problems: parsed.problems.map((problem) => ({ kind: "rubric", problem: { ...problem, path: filePathOf(problem.path) } })) };
+  return { rubric: parsed.rubric ?? { rules: {} }, digest: rubricFileDigest(content) };
+};

package/src/grade/rubric.ts

@@ -31,6 +31,8 @@ export type Rubric = {
   readonly citations?: CitationLimits | undefined;
   readonly contexts?: ContextLimits | undefined;
   readonly penalty?: number | undefined;
+  /** Each task's own rubric, as written: what it does not write comes from the top level (task-rubric.ts). */
+  readonly tasks?: Readonly<Record<string, Rubric>> | undefined;
 };
 
 /** What a value under `grade:` should have been. */
@@ -78,10 +80,10 @@ const ruleLimit = (raw: unknown, path: string): Read<RuleLimit> => {
   };
 };
 
-const rulesOf = (raw: unknown): Read<Record<string, RuleLimit>> => {
+const rulesOf = (raw: unknown, base: string): Read<Record<string, RuleLimit>> => {
   if (raw === undefined) return ok({});
-  if (!isRecord(raw)) return { value: {}, problems: [problem("grade.rules", "map", raw)] };
-  const read = Object.entries(raw).map(([id, limit]) => ({ id, read: ruleLimit(limit, `grade.rules.${id}`) }));
+  if (!isRecord(raw)) return { value: {}, problems: [problem(`${base}.rules`, "map", raw)] };
+  const read = Object.entries(raw).map(([id, limit]) => ({ id, read: ruleLimit(limit, `${base}.rules.${id}`) }));
   return { value: Object.fromEntries(read.map((entry) => [entry.id, entry.read.value])), problems: read.flatMap((entry) => entry.read.problems) };
 };
 
@@ -98,13 +100,14 @@ const kindsOf = (raw: unknown, path: string): Read<AtomKind[]> => {
   return kinds.length === entries.length ? ok(kinds) : { value: kinds, problems: [problem(path, "kinds", raw)] };
 };
 
-const factsOf = (raw: unknown): Read<FactLimits | undefined> => {
+const factsOf = (raw: unknown, base: string): Read<FactLimits | undefined> => {
   if (raw === undefined) return ok(undefined);
-  if (!isRecord(raw)) return { value: undefined, problems: [problem("grade.facts", "map", raw)] };
-  const dropped = optional(raw, "dropped", "grade.facts", "count");
-  const added = optional(raw, "added", "grade.facts", "count");
-  const allowDropped = kindsOf(raw["allow_dropped"], "grade.facts.allow_dropped");
-  const allowAdded = kindsOf(raw["allow_added"], "grade.facts.allow_added");
+  const path = `${base}.facts`;
+  if (!isRecord(raw)) return { value: undefined, problems: [problem(path, "map", raw)] };
+  const dropped = optional(raw, "dropped", path, "count");
+  const added = optional(raw, "added", path, "count");
+  const allowDropped = kindsOf(raw["allow_dropped"], `${path}.allow_dropped`);
+  const allowAdded = kindsOf(raw["allow_added"], `${path}.allow_added`);
   return {
     value: { dropped: dropped.value, added: added.value, allowDropped: allowDropped.value, allowAdded: allowAdded.value },
     problems: [
@@ -112,7 +115,7 @@ const factsOf = (raw: unknown): Read<FactLimits | undefined> => {
       ...added.problems,
       ...allowDropped.problems,
       ...allowAdded.problems,
-      ...unknownKeys(raw, ["dropped", "added", "allow_dropped", "allow_added"], "grade.facts"),
+      ...unknownKeys(raw, ["dropped", "added", "allow_dropped", "allow_added"], path),
     ],
   };
 };
@@ -125,62 +128,51 @@ const requiredOf = (raw: Record<string, unknown>, path: string): Read<boolean> =
     : { value: false, problems: [problem(`${path}.required`, "boolean", required)] };
 };
 
-const citationsOf = (raw: unknown): Read<CitationLimits | undefined> => {
+const citationsOf = (raw: unknown, base: string): Read<CitationLimits | undefined> => {
   if (raw === undefined) return ok(undefined);
-  if (!isRecord(raw)) return { value: undefined, problems: [problem("grade.citations", "map", raw)] };
-  const failed = optional(raw, "failed", "grade.citations", "count");
-  const required = requiredOf(raw, "grade.citations");
+  const path = `${base}.citations`;
+  if (!isRecord(raw)) return { value: undefined, problems: [problem(path, "map", raw)] };
+  const failed = optional(raw, "failed", path, "count");
+  const required = requiredOf(raw, path);
   return {
     value: { failed: failed.value, required: required.value },
-    problems: [...failed.problems, ...required.problems, ...unknownKeys(raw, ["failed", "required"], "grade.citations")],
+    problems: [...failed.problems, ...required.problems, ...unknownKeys(raw, ["failed", "required"], path)],
   };
 };
 
-const contextsOf = (raw: unknown): Read<ContextLimits | undefined> => {
+const contextsOf = (raw: unknown, base: string): Read<ContextLimits | undefined> => {
   if (raw === undefined) return ok(undefined);
-  if (!isRecord(raw)) return { value: undefined, problems: [problem("grade.contexts", "map", raw)] };
-  const unsupported = optional(raw, "unsupported", "grade.contexts", "count");
-  const allowUnsupported = kindsOf(raw["allow_unsupported"], "grade.contexts.allow_unsupported");
-  const required = requiredOf(raw, "grade.contexts");
+  const path = `${base}.contexts`;
+  if (!isRecord(raw)) return { value: undefined, problems: [problem(path, "map", raw)] };
+  const unsupported = optional(raw, "unsupported", path, "count");
+  const allowUnsupported = kindsOf(raw["allow_unsupported"], `${path}.allow_unsupported`);
+  const required = requiredOf(raw, path);
   return {
     value: { unsupported: unsupported.value, allowUnsupported: allowUnsupported.value, required: required.value },
     problems: [
       ...unsupported.problems,
       ...allowUnsupported.problems,
       ...required.problems,
-      ...unknownKeys(raw, ["unsupported", "allow_unsupported", "required"], "grade.contexts"),
+      ...unknownKeys(raw, ["unsupported", "allow_unsupported", "required"], path),
     ],
   };
 };
 
-const RUBRIC_KEYS: readonly string[] = ["rules", "required_sections", "facts", "citations", "contexts", "penalty"];
+/** The keys a task's rubric may write: every top-level key but `tasks`, so a task cannot hold tasks of its own. */
+const TASK_KEYS: readonly string[] = ["rules", "required_sections", "facts", "citations", "contexts", "penalty"];
 
-/**
- * The rubric, or every problem in it. No `grade:` key is no rubric: the default pass or fail applies (spec §29.3). A bare
- * `grade:` (null) is an empty rubric, so writing the key alone still turns the default off, as the spec says it does.
- */
-export const parseRubric = (raw: unknown): { readonly rubric: Rubric | undefined } | { readonly problems: readonly RubricProblem[] } => {
-  if (raw === undefined) return { rubric: undefined };
-  if (raw === null) return { rubric: { rules: {} } };
-  if (!isRecord(raw)) return { problems: [problem("grade", "map", raw)] };
-  const rules = rulesOf(raw["rules"]);
-  const sections = wordsOf(raw["required_sections"], "grade.required_sections");
-  const facts = factsOf(raw["facts"]);
-  const citations = citationsOf(raw["citations"]);
-  const contexts = contextsOf(raw["contexts"]);
-  const penalty = optional(raw, "penalty", "grade", "number");
-  const problems = [
-    ...rules.problems,
-    ...sections.problems,
-    ...facts.problems,
-    ...citations.problems,
-    ...contexts.problems,
-    ...penalty.problems,
-    ...unknownKeys(raw, RUBRIC_KEYS, "grade"),
-  ];
-  if (problems.length > 0) return { problems };
+const RUBRIC_KEYS: readonly string[] = [...TASK_KEYS, "tasks"];
+
+/** One rubric's keys under `base` (`grade`, or `grade.tasks.<name>`); a key not among `keys` is a problem. */
+const rubricAt = (raw: Record<string, unknown>, base: string, keys: readonly string[]): Read<Rubric> => {
+  const rules = rulesOf(raw["rules"], base);
+  const sections = wordsOf(raw["required_sections"], `${base}.required_sections`);
+  const facts = factsOf(raw["facts"], base);
+  const citations = citationsOf(raw["citations"], base);
+  const contexts = contextsOf(raw["contexts"], base);
+  const penalty = optional(raw, "penalty", base, "number");
   return {
-    rubric: {
+    value: {
       rules: rules.value,
       requiredSections: sections.value,
       facts: facts.value,
@@ -188,5 +180,43 @@ export const parseRubric = (raw: unknown): { readonly rubric: Rubric | undefined
       contexts: contexts.value,
       penalty: penalty.value,
     },
+    problems: [
+      ...rules.problems,
+      ...sections.problems,
+      ...facts.problems,
+      ...citations.problems,
+      ...contexts.problems,
+      ...penalty.problems,
+      ...unknownKeys(raw, keys, base),
+    ],
   };
 };
+
+/** One task's rubric. A bare `<name>:` (null) writes nothing, so the task inherits the whole top-level rubric. */
+const taskOf = (raw: unknown, path: string): Read<Rubric> => {
+  if (raw === null) return ok({ rules: {} });
+  if (!isRecord(raw)) return { value: { rules: {} }, problems: [problem(path, "map", raw)] };
+  return rubricAt(raw, path, TASK_KEYS);
+};
+
+const tasksOf = (raw: unknown): Read<Record<string, Rubric> | undefined> => {
+  if (raw === undefined) return ok(undefined);
+  if (!isRecord(raw)) return { value: undefined, problems: [problem("grade.tasks", "map", raw)] };
+  const read = Object.entries(raw).map(([name, task]) => ({ name, read: taskOf(task, `grade.tasks.${name}`) }));
+  return { value: Object.fromEntries(read.map((entry) => [entry.name, entry.read.value])), problems: read.flatMap((entry) => entry.read.problems) };
+};
+
+/**
+ * The rubric, or every problem in it. No `grade:` key is no rubric: the default pass or fail applies (spec §29.3). A bare
+ * `grade:` (null) is an empty rubric, so writing the key alone still turns the default off, as the spec says it does.
+ */
+export const parseRubric = (raw: unknown): { readonly rubric: Rubric | undefined } | { readonly problems: readonly RubricProblem[] } => {
+  if (raw === undefined) return { rubric: undefined };
+  if (raw === null) return { rubric: { rules: {} } };
+  if (!isRecord(raw)) return { problems: [problem("grade", "map", raw)] };
+  const top = rubricAt(raw, "grade", RUBRIC_KEYS);
+  const tasks = tasksOf(raw["tasks"]);
+  const problems = [...top.problems, ...tasks.problems];
+  if (problems.length > 0) return { problems };
+  return { rubric: tasks.value === undefined ? top.value : { ...top.value, tasks: tasks.value } };
+};

package/src/grade/setup.ts

@@ -28,16 +28,16 @@ const ruleSetOf = async (config: Config, languages: readonly string[]): Promise<
   };
 };
 
-/** A run's settings, rubric and stamp. `languages`: those the items name, beyond the bundled ones. Throws when a language package cannot be loaded. */
+/** A run's settings, rubric and stamp. `run.rubricFile`: a --rubric file's digest, when the rubric came from one. `languages`: those the items name, beyond the bundled ones. Throws when a language package cannot be loaded. */
 export const gradeSetup = async (
   config: Config,
-  run: { readonly experimental: boolean; readonly genre: string | undefined },
+  run: { readonly experimental: boolean; readonly genre: string | undefined; readonly rubricFile?: string | undefined },
   languages: readonly string[],
   rubric: Rubric | undefined,
 ): Promise<GradeSetup> => {
   // chaff.yaml's language and its by_path languages are loaded here too, so a missing package stops the run before grading.
   const named = [config.language, ...config.byPath.map((rule) => rule.language)].filter((language) => language !== undefined);
   const all = [...new Set([...BUNDLED_LANGUAGES, ...named, ...languages])].toSorted(compareText);
-  const settings = { config, experimental: run.experimental, genre: run.genre };
+  const settings = { config, experimental: run.experimental, genre: run.genre, rubricFile: run.rubricFile };
   return { ...settings, rubric, stamp: stampOf(VERSION_LINES, await ruleSetOf(config, all), settings) };
 };

package/src/grade/stamp.ts

@@ -20,6 +20,15 @@ export type GradeSettings = {
   readonly experimental: boolean;
   /** --genre, which wins over chaff.yaml for every item that does not name its own. */
   readonly genre: string | undefined;
+  /** The digest of a --rubric file, which replaces chaff.yaml's `grade:` for the run. */
+  readonly rubricFile?: string | undefined;
+};
+
+/** The rubric a run grades by: a --rubric file by its digest, else chaff.yaml's `grade:`. */
+const gradeOf = (settings: GradeSettings): unknown => {
+  if (settings.rubricFile !== undefined) return { file: settings.rubricFile };
+  // A bare `grade:` reads as null and is an empty rubric, which grades otherwise than no `grade:` at all.
+  return settings.config.grade === undefined ? "none" : settings.config.grade;
 };
 
 /** The parts of chaff.yaml that change a finding, a fact or a verdict. Paths and the AI judge do not. */
@@ -40,8 +49,7 @@ export const settingsOf = (settings: GradeSettings): Record<string, unknown> =>
     requiredSections: config.requiredSections,
     names: config.names,
     byPath: config.byPath,
-    // A bare `grade:` reads as null and is an empty rubric, which grades otherwise than no `grade:` at all.
-    grade: config.grade === undefined ? "none" : config.grade,
+    grade: gradeOf(settings),
   };
 };
 

package/src/grade/task-rubric.ts

@@ -0,0 +1,39 @@
+import type { Rubric } from "./rubric.ts";
+
+// The rubric one item is graded by: the top-level rubric with its task's rubric laid over it. Pure.
+// A key the task writes replaces the top-level key whole (`facts`, `citations`, `contexts` are not merged field by
+// field); a key it does not write is inherited. `rules` is merged per rule id: the task's entry for an id replaces the
+// top-level entry for that id, and every other top-level id still applies.
+
+/** The task names a rubric defines; none without a rubric or without `tasks`. */
+export const taskNamesOf = (rubric: Rubric | undefined): readonly string[] => Object.keys(rubric?.tasks ?? {});
+
+/** Each rule id the rubric names and where, from the rubric's top level: `rules`, or `tasks.<name>.rules`. */
+export const rubricRuleEntries = (rubric: Rubric | undefined): readonly { readonly path: string; readonly id: string }[] => [
+  ...Object.keys(rubric?.rules ?? {}).map((id) => ({ path: "rules", id })),
+  ...Object.entries(rubric?.tasks ?? {}).flatMap(([name, task]) => Object.keys(task.rules).map((id) => ({ path: `tasks.${name}.rules`, id }))),
+];
+
+const overlay = (top: Rubric, task: Rubric): Rubric => ({
+  rules: { ...top.rules, ...task.rules },
+  requiredSections: task.requiredSections ?? top.requiredSections,
+  facts: task.facts ?? top.facts,
+  citations: task.citations ?? top.citations,
+  contexts: task.contexts ?? top.contexts,
+  penalty: task.penalty ?? top.penalty,
+});
+
+/**
+ * The rubric for an item of `task`. Without a task, the rubric as it is. A task the rubric does not define is refused
+ * when the items are read, so here it cannot happen; it throws rather than fall back to the top level silently.
+ */
+export const rubricForTask = (rubric: Rubric | undefined, task: string | undefined): Rubric | undefined => {
+  if (task === undefined) return rubric;
+  const own = rubric?.tasks?.[task];
+  if (rubric === undefined || own === undefined) throw new Error(`chaff grade: no rubric for task "${task}"`);
+  return overlay(rubric, own);
+};
+
+/** The rule ids an output of `task` is graded by: whose rise in that output is a regression against a baseline. */
+export const taskRuleIds = (rubric: Rubric | undefined, task: string | undefined): ReadonlySet<string> =>
+  new Set(Object.keys(rubricForTask(rubric, task)?.rules ?? {}));