chaffjs 0.32.0 → 0.34.0

package/src/facts/term-values.ts

@@ -2,6 +2,7 @@ import { sameValue, type FactValue } from "./fact-values.ts";
 import type { Fact } from "./labelled-facts.ts";
 import type { TermWord } from "./document-terms.ts";
 import { withoutEdgeMarks } from "./trim-marks.ts";
+import { qualifiedKeyOf, type QualifierWords } from "./qualified-labels.ts";
 
 /**
  * 文書全体で一つの値を持つ項目（保証期間、warranty period）を主語にした文の期間。名前付きの値の読み（labelled-facts）は、区切りの
@@ -24,6 +25,14 @@ export type TermValueWords = {
   /** 起算の句の印（から は値の前の句の終わり、from は値の後ろの句の頭）。 */
   readonly starts: readonly PlacedWord[];
   readonly valueEnds: readonly string[];
+  /** 名前のすぐ後ろで、その値を何に使ったかを言う句の頭（used for the）。後ろの句ごと落としても同じ項目。 */
+  readonly restating?: readonly string[];
+  /** 名前と値を囲む型（を*として）。* が値。型の値は、期間でなくても読む。 */
+  readonly frames?: readonly string[];
+  /** 文の値に条件を付ける語（場合、if）。型の前の句にあれば読まない。 */
+  readonly conditions?: readonly string[];
+  /** 名前に条件を付ける型（*の、for *）。 */
+  readonly qualifiers?: QualifierWords;
 };
 
 const folded = (text: string): string => text.normalize("NFKC").toLowerCase().replace(/\s+/gu, " ").trim();
@@ -73,12 +82,35 @@ const withoutSelf = (key: string, words: TermValueWords): string => {
   return (self.position === "before" ? key.slice(length) : key.slice(0, key.length - length)).trim();
 };
 
-/** 名前が語彙表の項目なら、その語の形。 */
-export const termKeyOf = (label: string, words: TermValueWords): string | undefined => {
+/** 語彙表の項目の形。語彙表の語は条件の付いた名前の形（扶養親族の数 は 数 (扶養親族)）なので、名前もその形にして比べる。 */
+const termOf = (key: string, words: TermValueWords): string | undefined => {
+  const qualified = words.qualifiers === undefined ? key : qualifiedKeyOf(key, words.qualifiers);
+  return words.terms.map((word) => folded(word.pattern)).find((pattern) => pattern === key || pattern === qualified);
+};
+
+/** 何に使ったかを言う句から後ろを落とした名前（number of dependants used for the allowance の number of dependants）。 */
+const withoutRestating = (key: string, words: TermValueWords): string | undefined =>
+  (words.restating ?? [])
+    .flatMap((word) => {
+      const mark = ` ${folded(word)}`;
+      const at = key.indexOf(mark);
+      const next = key.charAt(at + mark.length);
+      return at > 0 && (next === "" || next === " ") ? [key.slice(0, at)] : [];
+    })
+    .find((head) => termOf(head, words) !== undefined);
+
+/** 名前が語彙表の項目なら、その語の形。restated: 何に使ったかを言う句を落として、項目になった。 */
+const termKeyWith = (label: string, words: TermValueWords): { readonly key: string; readonly restated: boolean } | undefined => {
   const key = withoutSelf(withoutDeterminer(folded(withoutEdgeMarks(label)), words), words);
-  return words.terms.map((word) => folded(word.pattern)).find((pattern) => pattern === key);
+  const term = termOf(key, words);
+  if (term !== undefined) return { key: term, restated: false };
+  const restated = withoutRestating(key, words);
+  return restated === undefined ? undefined : { key: restated, restated: true };
 };
 
+/** 名前が語彙表の項目なら、その語の形。 */
+export const termKeyOf = (label: string, words: TermValueWords): string | undefined => termKeyWith(label, words)?.key;
+
 type Span = { readonly start: number; readonly end: number };
 
 const valuesIn = (values: readonly FactValue[], span: Span): FactValue[] => values.filter((value) => span.start <= value.start && value.end <= span.end);
@@ -91,7 +123,7 @@ const separatorsIn = (text: string, separator: string): number[] =>
     .map((match) => match.index)
     .filter((index) => !isLatin(separator) || (text.charAt(index - 1) === " " && [" ", ""].includes(text.charAt(index + separator.length))));
 
-type Subject = { readonly key: string; readonly label: string; readonly lead: Span | undefined; readonly restStart: number };
+type Subject = { readonly key: string; readonly label: string; readonly lead: Span | undefined; readonly restStart: number; readonly restated: boolean };
 
 /** 文の頭から区切りまでの主語。読点の前は条件の句。 */
 const subjectOf = (source: string, from: number, to: number, words: TermValueWords): Subject | undefined => {
@@ -101,9 +133,10 @@ const subjectOf = (source: string, from: number, to: number, words: TermValueWor
       const head = clause.slice(0, index);
       const comma = [...head.matchAll(CLAUSE_COMMAS)].at(-1)?.index;
       const label = withoutEdgeMarks(comma === undefined ? head : head.slice(comma + 1));
-      const key = termKeyOf(label, words);
-      if (key === undefined) continue;
-      return { key, label, lead: comma === undefined ? undefined : { start: from, end: from + comma }, restStart: from + index + separator.length };
+      const term = termKeyWith(label, words);
+      if (term === undefined) continue;
+      const lead = comma === undefined ? undefined : { start: from, end: from + comma };
+      return { ...term, label, lead, restStart: from + index + separator.length };
     }
   }
   return undefined;
@@ -170,10 +203,10 @@ const isWorkedExample = (lead: Span, starts: readonly Span[], values: readonly F
   return starts.some((span) => valuesIn(values, span).some((value) => value.kind === "date" && sameValue(date, value)));
 };
 
-/** 主語と区切りの後ろの期間（保証期間は、お買い上げ日から24か月です）。 */
-const subjectFact = (source: string, value: FactValue, values: readonly FactValue[], words: TermValueWords): Fact | undefined => {
+/** 主語と区切りの後ろの期間（保証期間は、お買い上げ日から24か月です）。restatedOnly: 何に使ったかを言う句の付いた主語だけ。 */
+const subjectFact = (source: string, value: FactValue, values: readonly FactValue[], words: TermValueWords, restatedOnly = false): Fact | undefined => {
   const subject = subjectOf(source, clauseStartOf(source, value.start), value.start, words);
-  if (subject === undefined) return undefined;
+  if (subject === undefined || (restatedOnly && !subject.restated)) return undefined;
   const before = startBeforeValue(source, { start: subject.restStart, end: value.start }, values, words);
   if (before === undefined) return undefined;
   const after = startAfterValue(source, value.end, values, words);
@@ -226,11 +259,59 @@ const unitNamedFact = (source: string, value: FactValue, words: TermValueWords):
 
 const DURATION_UNITS = new Set(["day", "week", "month", "year"]);
 
-/** 語彙表の項目を主語にした文の、期間の値。values は文書の中の順の値（日付を含む）。 */
+/** 型の前と後ろの字（を*として の を と として）。 */
+const frameParts = (frame: string): readonly [string, string] | undefined => {
+  const at = frame.indexOf("*");
+  return at <= 0 || at === frame.length - 1 ? undefined : [frame.slice(0, at), frame.slice(at + 1)];
+};
+
+/** 型の前の句（扶養控除は、）が、値も条件の語も含まないか。条件の句（登録された場合、）の後ろの値は、その場合の値。 */
+const isPlainLead = (source: string, lead: Span, values: readonly FactValue[], words: TermValueWords): boolean => {
+  const written = folded(source.slice(lead.start, lead.end));
+  return valuesIn(values, lead).length === 0 && !(words.conditions ?? []).some((word) => written.includes(folded(word)));
+};
+
+/** 型の名前（扶養親族の数を2人として の 扶養親族の数）。名前は読点の後ろから。 */
+const framedFact = (
+  source: string,
+  value: FactValue,
+  values: readonly FactValue[],
+  words: TermValueWords,
+  [open, close]: readonly [string, string],
+): Fact | undefined => {
+  const from = clauseStartOf(source, value.start);
+  const labelEnd = value.start - open.length;
+  if (labelEnd <= from || source.slice(labelEnd, value.start) !== open || !source.startsWith(close, value.end)) return undefined;
+  const head = source.slice(from, labelEnd);
+  const comma = [...head.matchAll(CLAUSE_COMMAS)].at(-1)?.index ?? -1;
+  const label = withoutEdgeMarks(head.slice(comma + 1));
+  const key = termKeyOf(label, words);
+  if (key === undefined || !isPlainLead(source, { start: from, end: from + Math.max(comma, 0) }, values, words)) return undefined;
+  return { label, key, value };
+};
+
+/** 名前と値を囲む型の値（扶養控除は、扶養親族の数を2人として計算しています）。 */
+const frameFact = (source: string, value: FactValue, values: readonly FactValue[], words: TermValueWords): Fact | undefined =>
+  (words.frames ?? [])
+    .flatMap((frame) => {
+      const parts = frameParts(frame);
+      const fact = parts === undefined ? undefined : framedFact(source, value, values, words, parts);
+      return fact === undefined ? [] : [fact];
+    })
+    .at(0);
+
+const durationFact = (source: string, value: FactValue, values: readonly FactValue[], words: TermValueWords): Fact | undefined =>
+  bracketFact(source, value, words) ?? subjectFact(source, value, values, words) ?? unitNamedFact(source, value, words);
+
+/**
+ * 語彙表の項目を主語にした文の、期間の値。期間でない数は、何に使ったかを言う句の付いた主語（the number of dependants used for the
+ * allowance is 2）か、名前と値を囲む型（扶養親族の数を2人として）のときだけ。values は文書の中の順の値（日付を含む）。
+ */
 export const termValues = (source: string, values: readonly FactValue[], words: TermValueWords): Fact[] =>
   values
-    .filter((value) => value.kind === "quantity" && DURATION_UNITS.has(value.unit))
+    .filter((value) => value.kind === "quantity")
     .flatMap((value) => {
-      const fact = bracketFact(source, value, words) ?? subjectFact(source, value, values, words) ?? unitNamedFact(source, value, words);
+      const read = DURATION_UNITS.has(value.unit) ? durationFact(source, value, values, words) : subjectFact(source, value, values, words, true);
+      const fact = read ?? frameFact(source, value, values, words);
       return fact === undefined ? [] : [fact];
     });

package/src/fix-plan/text.ts

@@ -109,7 +109,7 @@ export const FIX_PLAN_TEXT: Texts<FixPlanText> = {
     spots: "見つけた箇所",
     at: (line) => `${String(line)} 行目`,
     notRunHeading: "動かなかったルール",
-    notExperimental: "試験中のルールは動かしていません。AIっぽさのルールの多くは試験中です。--experimental を付けると見ます。",
+    notExperimental: "試験中のルールは動かしていません。--experimental を付けると見ます。",
     checksHeading: "直したあとの確かめ",
     saveAs: (path) => `書き直したものを ${path} に保存して、次を実行します。`,
     checksNote:
@@ -174,7 +174,7 @@ export const FIX_PLAN_TEXT: Texts<FixPlanText> = {
     spots: "Spots",
     at: (line) => `line ${String(line)}`,
     notRunHeading: "Rules that did not run",
-    notExperimental: "The experimental rules did not run, and most rules for generated text are experimental. Add --experimental to run them.",
+    notExperimental: "The experimental rules did not run. Add --experimental to run them.",
     checksHeading: "Check after rewriting",
     saveAs: (path) => `Save the rewrite as ${path} and run:`,
     checksNote:

package/src/grade/baseline.ts

@@ -60,13 +60,22 @@ type Pair = { readonly before: GradeResult; readonly after: GradeResult };
 
 const countIn = (result: GradeResult, rule: string): number => result.findings.filter((finding) => finding.rule === rule).length;
 
-const movementsOf = (pairs: readonly Pair[], rubricRules: ReadonlySet<string>): RuleMovement[] =>
+/** The rules the rubric names for one output: with rubric tasks, its task's rules over the top level's. */
+export type RubricRulesOf = (result: GradeResult) => ReadonlySet<string>;
+
+/** A rule's movement, and the outputs it rose in whose own rubric names it: only those are a regression. */
+type Moved = { readonly movement: RuleMovement; readonly regressedIn: readonly string[] };
+
+const movementsOf = (pairs: readonly Pair[], rubricRulesOf: RubricRulesOf): Moved[] =>
   rateRows([pairs.map((pair) => pair.before), pairs.map((pair) => pair.after)]).flatMap(({ rule, unit, rates: [before = 0, after = 0] }) => {
     const inUnit = pairs.filter((pair) => pair.after.size.unit === unit);
-    const increasedIn = inUnit.filter((pair) => countIn(pair.after, rule) > countIn(pair.before, rule)).map((pair) => resultName(pair.after));
+    const increased = inUnit.filter((pair) => countIn(pair.after, rule) > countIn(pair.before, rule));
+    const increasedIn = increased.map((pair) => resultName(pair.after));
     const decreasedIn = inUnit.filter((pair) => countIn(pair.after, rule) < countIn(pair.before, rule)).map((pair) => resultName(pair.after));
     if (increasedIn.length === 0 && decreasedIn.length === 0 && before === after) return [];
-    return [{ rule, unit, before, after, increasedIn, decreasedIn, inRubric: rubricRules.has(rule) }];
+    const graded = (pair: Pair): boolean => rubricRulesOf(pair.after).has(rule);
+    const regressedIn = increased.filter(graded).map((pair) => resultName(pair.after));
+    return [{ movement: { rule, unit, before, after, increasedIn, decreasedIn, inRubric: inUnit.some(graded) }, regressedIn }];
   });
 
 const factKey = (fact: GradeFact): string => `${fact.kind}\n${fact.key}`;
@@ -100,11 +109,9 @@ const penaltyOf = (results: readonly GradeResult[]): number => results.reduce((s
 
 const scored = (pairs: readonly Pair[]): boolean => pairs.some((pair) => pair.after.score !== undefined);
 
-const regressionsOf = (newlyFailed: readonly string[], movements: readonly RuleMovement[], penalty: Comparison["penalty"]): string[] => [
+const regressionsOf = (newlyFailed: readonly string[], moved: readonly Moved[], penalty: Comparison["penalty"]): string[] => [
   ...newlyFailed.map((id) => `${id}: passed, now fails`),
-  ...movements
-    .filter((movement) => movement.inRubric && movement.increasedIn.length > 0)
-    .map((movement) => `rules.${movement.rule}: more findings in ${movement.increasedIn.join(", ")}`),
+  ...moved.filter((entry) => entry.regressedIn.length > 0).map((entry) => `rules.${entry.movement.rule}: more findings in ${entry.regressedIn.join(", ")}`),
   ...(penalty !== undefined && penalty.after > penalty.before ? [`score.penalty ${String(penalty.before)} → ${String(penalty.after)}`] : []),
 ];
 
@@ -112,9 +119,10 @@ const pairRead = (pair: Pair): boolean => sameReading([pair.before, pair.after])
 
 /**
  * `after` compared with `before`, output by output: an output is paired with the earlier one of its id and variant.
- * `rubricRules`: the rules the rubric names, whose increase is a regression.
+ * `rubricRules`: the rules the rubric names, whose increase is a regression; per output when the rubric has tasks.
  */
-export const compareRuns = (before: readonly GradeResult[], after: readonly GradeResult[], rubricRules: ReadonlySet<string>): Comparison => {
+export const compareRuns = (before: readonly GradeResult[], after: readonly GradeResult[], rubricRules: ReadonlySet<string> | RubricRulesOf): Comparison => {
+  const rubricRulesOf: RubricRulesOf = typeof rubricRules === "function" ? rubricRules : () => rubricRules;
   const earlier = new Map(before.map((result) => [resultKey(result), result]));
   const keys = new Set(after.map(resultKey));
   const matched = after.flatMap((result) => {
@@ -122,7 +130,7 @@ export const compareRuns = (before: readonly GradeResult[], after: readonly Grad
     return match === undefined ? [] : [{ before: match, after: result }];
   });
   const pairs = matched.filter(pairRead);
-  const rules = movementsOf(pairs, rubricRules);
+  const moved = movementsOf(pairs, rubricRulesOf);
   const newlyFailed = pairs.filter((pair) => pair.before.pass && !pair.after.pass).map((pair) => resultName(pair.after));
   const penalty = scored(pairs) ? { before: penaltyOf(pairs.map((pair) => pair.before)), after: penaltyOf(pairs.map((pair) => pair.after)) } : undefined;
   return {
@@ -130,11 +138,11 @@ export const compareRuns = (before: readonly GradeResult[], after: readonly Grad
     onlyBefore: before.filter((result) => !keys.has(resultKey(result))).map(resultName),
     onlyAfter: after.filter((result) => !earlier.has(resultKey(result))).map(resultName),
     readOtherwise: matched.filter((pair) => !pairRead(pair)).map((pair) => resultName(pair.after)),
-    rules,
+    rules: moved.map((entry) => entry.movement),
     newlyFailed,
     newlyPassed: pairs.filter((pair) => !pair.before.pass && pair.after.pass).map((pair) => resultName(pair.after)),
     items: pairs.map(itemChange).filter((change) => change.dropped.length + change.added.length + change.citations.length > 0),
     ...(penalty === undefined ? {} : { penalty }),
-    regressions: regressionsOf(newlyFailed, rules, penalty),
+    regressions: regressionsOf(newlyFailed, moved, penalty),
   };
 };

package/src/grade/expect-read.ts

@@ -0,0 +1,65 @@
+import { documentLanguage } from "../check-source.ts";
+import { factsOf } from "../commands/compare.ts";
+import { documentFromSource } from "../commands/read-document.ts";
+import type { Compared } from "../compare/outcome.ts";
+import { resolveGenre } from "../resolve-genre.ts";
+import { OUTPUT_PATH, type ItemReading } from "./checks.ts";
+import { EXPECT_LISTS, entryPath, expectAgainst, factOfEntry, type GradeExpect, type GradeExpectResult, type ReadEntry } from "./expect.ts";
+import type { GradeItem, ItemProblem } from "./item.ts";
+
+// An item's `expect` entries read as compare reads a document, in the output's language and genre, so an entry and
+// the output spell one fact alike.
+
+const ENTRY_PATH = "expect.md";
+
+type EntryRead = { readonly entry: ReadEntry } | { readonly problem: Omit<ItemProblem, "line"> };
+
+const readEntry = async (list: ReadEntry["list"], index: number, entry: string, reading: ItemReading & { readonly language: string }): Promise<EntryRead> => {
+  const prose = await documentFromSource(ENTRY_PATH, entry, { language: reading.language, genre: reading.genre }, reading.config, true);
+  const facts = factsOf(ENTRY_PATH, prose, reading.config.names ?? []);
+  const found = factOfEntry(entry, facts.extraction.atoms);
+  if ("unread" in found)
+    return { problem: { kind: found.unread === "no-fact" ? "expect-no-fact" : "expect-several-facts", detail: `${entryPath(list, index)} ${entry}` } };
+  return { entry: { list, index, entry, read: { path: ENTRY_PATH, extraction: { ...facts.extraction, atoms: [found.fact] } } } };
+};
+
+const readEntries = (expect: GradeExpect, reading: ItemReading & { readonly language: string }): Promise<EntryRead[]> =>
+  Promise.all(EXPECT_LISTS.flatMap((list) => expect[list].map((entry, index) => readEntry(list, index, entry, reading))));
+
+/** The output's own facts, read as `chaff compare` reads the document after. */
+const outputFacts = async (output: string, reading: ItemReading & { readonly language: string }): Promise<Compared> => {
+  const prose = await documentFromSource(OUTPUT_PATH, output, { language: reading.language, genre: reading.genre }, reading.config, true);
+  return factsOf(OUTPUT_PATH, prose, reading.config.names ?? []);
+};
+
+/** The language and genre the output is graded in, as checkSource chooses them. */
+const itemReading = (item: GradeItem, setup: { readonly config: ItemReading["config"]; readonly genre?: string | undefined }) => ({
+  config: setup.config,
+  language: item.language ?? documentLanguage(OUTPUT_PATH, item.output, setup.config),
+  genre: resolveGenre(OUTPUT_PATH, item.output, setup.config, item.genre ?? setup.genre).genre,
+});
+
+/** Each entry chaff cannot read as exactly one fact, on the item's line: the run stops before grading (exit 2). */
+export const expectProblems = async (
+  item: GradeItem,
+  line: number,
+  setup: { readonly config: ItemReading["config"]; readonly genre?: string | undefined },
+): Promise<ItemProblem[]> => {
+  if (item.expect === undefined) return [];
+  const read = await readEntries(item.expect, itemReading(item, setup));
+  return read.flatMap((entry) => ("problem" in entry ? [{ ...entry.problem, line }] : []));
+};
+
+/**
+ * The output against the item's expect, read in the language and genre it was graded in. Throws for an entry that is
+ * not one fact: `chaff grade` refuses such an item before grading (expectProblems), so none reaches here.
+ */
+export const expectOfItem = async (item: GradeItem, reading: ItemReading & { readonly language: string }): Promise<GradeExpectResult | undefined> => {
+  if (item.expect === undefined) return undefined;
+  const read = await readEntries(item.expect, reading);
+  const entries = read.flatMap((entry) => {
+    if ("problem" in entry) throw new Error(`${item.id}: ${entry.problem.detail ?? ""} is not one fact`);
+    return [entry.entry];
+  });
+  return expectAgainst(entries, await outputFacts(item.output, reading));
+};

package/src/grade/expect.ts

@@ -0,0 +1,74 @@
+import type { Atom } from "../compare/atom.ts";
+import { outcomeOf, type Compared } from "../compare/outcome.ts";
+
+// An item's known answer (`expect`): facts the output must state and facts it must not. Pure: the shape of the field,
+// which one fact an entry states, and whether the output states it, decided as `chaff compare --distinct` decides it.
+
+export type GradeExpect = { readonly facts: readonly string[]; readonly absent: readonly string[] };
+
+export const EXPECT_LISTS = ["facts", "absent"] as const;
+
+export type ExpectList = (typeof EXPECT_LISTS)[number];
+
+const isExpectList = (key: string): key is ExpectList => EXPECT_LISTS.some((list) => list === key);
+
+const isRecord = (value: unknown): value is Record<string, unknown> => typeof value === "object" && value !== null && !Array.isArray(value);
+
+const isEntryList = (value: unknown): value is string[] => Array.isArray(value) && value.every((entry) => typeof entry === "string" && entry.trim() !== "");
+
+/**
+ * The field as written, or the path of its first wrong part. A key other than facts and absent is wrong, not ignored:
+ * a misspelt list would check nothing and pass every output. A missing list is an empty one.
+ */
+export const expectOf = (raw: unknown): { readonly expect: GradeExpect | undefined } | { readonly wrong: string } => {
+  if (raw === undefined) return { expect: undefined };
+  if (!isRecord(raw)) return { wrong: "expect" };
+  const unknown = Object.keys(raw).find((key) => !isExpectList(key));
+  if (unknown !== undefined) return { wrong: `expect.${unknown}` };
+  const wrongList = EXPECT_LISTS.find((list) => raw[list] !== undefined && !isEntryList(raw[list]));
+  if (wrongList !== undefined) return { wrong: `expect.${wrongList}` };
+  const listOf = (list: ExpectList): readonly string[] => {
+    const value = raw[list];
+    return isEntryList(value) ? value : [];
+  };
+  return { expect: { facts: listOf("facts"), absent: listOf("absent") } };
+};
+
+/** The one fact an entry states, or why it states none or several. */
+export type EntryFact = { readonly fact: Atom } | { readonly unread: "no-fact" | "several-facts" };
+
+/**
+ * The fact an entry states. Several facts are one when exactly one of them is the whole entry: a quotation 「12,000円」
+ * is read as the quotation and the number inside it, and the entry means the quotation.
+ */
+export const factOfEntry = (entry: string, atoms: readonly Atom[]): EntryFact => {
+  const [only] = atoms;
+  if (atoms.length === 1 && only !== undefined) return { fact: only };
+  if (atoms.length === 0) return { unread: "no-fact" };
+  const whole = atoms.filter((atom) => atom.text === entry.trim());
+  const [wholeOnly] = whole;
+  return whole.length === 1 && wholeOnly !== undefined ? { fact: wholeOnly } : { unread: "several-facts" };
+};
+
+/** Where an entry is in the item: `expect.facts[0]`. */
+export const entryPath = (list: ExpectList, index: number): string => `expect.${list}[${String(index)}]`;
+
+/** One entry as read: its list, its place, its text, and its document's facts with only the entry's fact kept. */
+export type ReadEntry = { readonly list: ExpectList; readonly index: number; readonly entry: string; readonly read: Compared };
+
+/** An entry that states one fact, compared as compare compares two documents: the output states it at least once. */
+export const statesFact = (entry: Compared, output: Compared): boolean => outcomeOf(entry, output, undefined, "distinct").dropped.length === 0;
+
+/** What the output was checked against: how many entries, the facts it does not state, and the absent ones it does. */
+export type GradeExpectResult = { readonly checked: number; readonly missing: readonly string[]; readonly present: readonly string[] };
+
+/** Each entry against the output, by the entry's text as written in the item. */
+export const expectAgainst = (entries: readonly ReadEntry[], output: Compared): GradeExpectResult => ({
+  checked: entries.length,
+  missing: entries.filter((entry) => entry.list === "facts" && !statesFact(entry.read, output)).map((entry) => entry.entry),
+  present: entries.filter((entry) => entry.list === "absent" && statesFact(entry.read, output)).map((entry) => entry.entry),
+});
+
+/** The failure reasons: `expect.missing 12,000円`, `expect.present 15,000円`. Absent when the item had no expect. */
+export const expectFailures = (result: GradeExpectResult | undefined): string[] =>
+  result === undefined ? [] : [...result.missing.map((entry) => `expect.missing ${entry}`), ...result.present.map((entry) => `expect.present ${entry}`)];

package/src/grade/grade-item.ts

@@ -10,11 +10,14 @@ import { ratesOf, sizeOf, type OutputSize } from "./rates.ts";
 import type { GradeCitations, GradeFacts, GradeFinding, GradeResult, GradeScore, NotRunEntry, Stamp } from "./result.ts";
 import type { Rubric } from "./rubric.ts";
 import { rubricVerdict } from "./rubric-verdict.ts";
+import { rubricForTask } from "./task-rubric.ts";
 import type { GradeSettings } from "./stamp.ts";
 import { GRADE_TEXT } from "./text.ts";
 import { aiScoreOfDocument } from "../ai-score/of-document.ts";
 import { gradeAiScoreOf } from "./ai-score.ts";
-import { defaultVerdict, type Verdict } from "./verdict.ts";
+import { defaultVerdict, verdictOf, type Verdict } from "./verdict.ts";
+import { expectFailures, type GradeExpectResult } from "./expect.ts";
+import { expectOfItem } from "./expect-read.ts";
 
 /** One run's settings, its `grade:` rubric when chaff.yaml has one, and its stamp, which every result carries. */
 export type GradeSetup = GradeSettings & { readonly stamp: Stamp; readonly rubric?: Rubric | undefined };
@@ -34,6 +37,7 @@ type Checked = {
   readonly facts: GradeFacts | null;
   readonly cited: CitationsRead | undefined;
   readonly contexts: ContextsRead | undefined;
+  readonly expect: GradeExpectResult | undefined;
 };
 
 const checkItem = async (item: GradeItem, setup: GradeSetup): Promise<Checked> => {
@@ -50,6 +54,7 @@ const checkItem = async (item: GradeItem, setup: GradeSetup): Promise<Checked> =
     facts: item.reference === undefined ? null : await factsAgainst(texts, reading, allowedBy(setup.rubric)),
     cited: item.citations === undefined ? undefined : await citationsAgainst(item, item.citations, reading, ui),
     contexts: await contextsOf(item, check.language, reading, setup.rubric),
+    expect: await expectOfItem(item, { ...reading, language: check.language }),
   };
 };
 
@@ -97,12 +102,17 @@ const judge = (item: GradeItem, checked: Checked, setup: GradeSetup): { readonly
 };
 
 /** Grades one output (spec §29.3): its findings and their rates, the facts against its reference, its quotations, and pass or fail. */
-export const gradeItem = async (item: GradeItem, setup: GradeSetup): Promise<GradeResult> => {
+export const gradeItem = async (item: GradeItem, runSetup: GradeSetup): Promise<GradeResult> => {
+  const setup = item.task === undefined ? runSetup : { ...runSetup, rubric: rubricForTask(runSetup.rubric, item.task) };
   const checked = await checkItem(item, setup);
-  const { verdict, score } = judge(item, checked, setup);
+  const judged = judge(item, checked, setup);
+  const { score } = judged;
+  // expect is the item's own answer, so it fails the output with or without a `grade:` rubric.
+  const verdict = verdictOf([...judged.verdict.failedBecause, ...expectFailures(checked.expect)]);
   return {
     id: item.id,
     ...(item.variant === undefined ? {} : { variant: item.variant }),
+    ...(item.task === undefined ? {} : { task: item.task }),
     language: checked.check.language,
     genre: checked.check.genre.genre,
     size: checked.size,
@@ -115,6 +125,7 @@ export const gradeItem = async (item: GradeItem, setup: GradeSetup): Promise<Gra
     facts: checked.facts,
     citations: citationsOf(checked),
     ...(checked.contexts === undefined ? {} : { contexts: checked.contexts.contexts }),
+    ...(checked.expect === undefined ? {} : { expect: checked.expect }),
     ...(score === undefined ? {} : { score }),
     aiScore: gradeAiScoreOf(aiScoreOfDocument(checked.check.doc, checked.check.rules, checked.check.genre.genre)),
     pass: verdict.pass,

package/src/grade/item.ts

@@ -1,5 +1,6 @@
 // One line of `chaff grade`'s input: a model's output and what to check it against. Pure: reads text, returns items
 // or what is wrong with them, so a harness that writes the file can be told which line to fix.
+import { expectOf, type GradeExpect } from "./expect.ts";
 import { resultKey } from "./result-name.ts";
 
 export type GradeCitation = { readonly source: string; readonly address: string; readonly quote: string };
@@ -18,10 +19,14 @@ export type GradeItem = {
   readonly contexts?: readonly string[] | undefined;
   /** The prompt, model or setting that produced this output. Outputs of several variants are compared on the same ids. */
   readonly variant?: string | undefined;
+  /** The rubric task this item is graded under (`grade.tasks.<name>`). Undefined: the top-level rubric. */
+  readonly task?: string | undefined;
+  /** Facts the output must state and facts it must not: a question's known answer. Undefined when not given. */
+  readonly expect?: GradeExpect | undefined;
 };
 
 /** The fields an item already has a meaning for: none of them can name a variant. */
-export const ITEM_FIELDS: readonly string[] = ["id", "output", "reference", "sources", "citations", "contexts", "language", "genre"];
+export const ITEM_FIELDS: readonly string[] = ["id", "output", "reference", "sources", "citations", "contexts", "language", "genre", "expect", "task"];
 
 /** Which field names an output's variant, and whether every line must have it (the key was given on the command line). */
 export type VariantField = { readonly key: string; readonly required: boolean };
@@ -44,13 +49,22 @@ export type ItemProblemKind =
   | "which-source"
   | "unknown-language"
   | "unknown-genre"
+  | "not-expect"
+  | "expect-no-fact"
+  | "expect-several-facts"
+  | "unknown-task"
   | "empty";
 
 /** Why a line cannot be graded. `detail`: the field, the value or the parser's message; `first`: the line an id was first on. */
 export type ItemProblem = { readonly kind: ItemProblemKind; readonly line: number; readonly detail?: string; readonly first?: number };
 
 /** What an item may name: the languages chaff can load and the genres it knows. */
-export type ItemVocabulary = { readonly isLanguage: (language: string) => boolean; readonly genres: readonly string[] };
+export type ItemVocabulary = {
+  readonly isLanguage: (language: string) => boolean;
+  readonly genres: readonly string[];
+  /** The tasks the rubric defines. None given: an item naming a task is refused, as there is no rubric for it. */
+  readonly tasks?: readonly string[] | undefined;
+};
 
 const isRecord = (value: unknown): value is Record<string, unknown> => typeof value === "object" && value !== null && !Array.isArray(value);
 
@@ -129,6 +143,27 @@ const variantOf = (raw: Record<string, unknown>, key: string, line: number): Par
   return typeof value === "string" && value !== "" ? { value } : { problem: { kind: "not-string", line, detail: key } };
 };
 
+/** The item's task: a non-empty string the rubric defines, or none. Never a silent fall back to the top-level rubric. */
+const taskOf = (raw: unknown, vocabulary: ItemVocabulary, line: number): Parsed<string | undefined> => {
+  if (raw === undefined) return { value: undefined };
+  if (typeof raw !== "string" || raw === "") return { problem: { kind: "not-string", line, detail: "task" } };
+  return (vocabulary.tasks ?? []).includes(raw) ? { value: raw } : { problem: { kind: "unknown-task", line, detail: raw } };
+};
+
+/** The item's variant and task, each written only when given, checked in that order. */
+const labelsOf = (
+  raw: Record<string, unknown>,
+  line: number,
+  vocabulary: ItemVocabulary,
+  variantKey: string,
+): Parsed<{ readonly variant?: string; readonly task?: string }> => {
+  const variant = variantOf(raw, variantKey, line);
+  if ("problem" in variant) return variant;
+  const task = taskOf(raw["task"], vocabulary, line);
+  if ("problem" in task) return task;
+  return { value: { ...(variant.value === undefined ? {} : { variant: variant.value }), ...(task.value === undefined ? {} : { task: task.value }) } };
+};
+
 const itemOf = (raw: Record<string, unknown>, line: number, vocabulary: ItemVocabulary, variantKey: string): Parsed<GradeItem> => {
   const id = raw["id"];
   if (typeof id !== "string" || id === "") return { problem: { kind: "no-id", line } };
@@ -138,17 +173,22 @@ const itemOf = (raw: Record<string, unknown>, line: number, vocabulary: ItemVoca
   if ("problem" in fields) return fields;
   const unknown = vocabularyProblem(fields.value, vocabulary, line);
   if (unknown !== undefined) return { problem: unknown };
-  const variant = variantOf(raw, variantKey, line);
-  if ("problem" in variant) return variant;
+  const labels = labelsOf(raw, line, vocabulary, variantKey);
+  if ("problem" in labels) return labels;
   const sources = sourcesOf(raw["sources"], line);
   if ("problem" in sources) return sources;
   const citations = citationsOf(raw["citations"], sources.value, line);
   if ("problem" in citations) return citations;
   const contexts = contextsOf(raw["contexts"], line);
   if ("problem" in contexts) return contexts;
+  const expect = expectOf(raw["expect"]);
+  if ("wrong" in expect) return { problem: { kind: "not-expect", line, detail: expect.wrong } };
   const passages = contexts.value === undefined ? {} : { contexts: contexts.value };
-  const labelled = variant.value === undefined ? {} : { variant: variant.value };
-  return { value: { id, output, ...fields.value, sources: sources.value, citations: citations.value, ...passages, ...labelled } };
+  const expected = expect.expect === undefined ? {} : { expect: expect.expect };
+  const { variant, task } = labels.value;
+  const labelled = variant === undefined ? {} : { variant };
+  const tasked = task === undefined ? {} : { task };
+  return { value: { id, output, ...fields.value, sources: sources.value, citations: citations.value, ...passages, ...labelled, ...expected, ...tasked } };
 };
 
 /** One item from a value already parsed, as `grade()` receives it; `line` is 0 when it came from no file. */
@@ -192,7 +232,7 @@ export const parseItems = (
   text: string,
   vocabulary: ItemVocabulary,
   variantField: VariantField = DEFAULT_VARIANT_FIELD,
-): { readonly items: readonly GradeItem[] } | { readonly problems: readonly ItemProblem[] } => {
+): { readonly items: readonly GradeItem[]; readonly lines: readonly number[] } | { readonly problems: readonly ItemProblem[] } => {
   const lines = text.split(/\r?\n/u).map((body, index) => ({ body, line: index + 1 }));
   const parsed = lines
     .filter(({ body }) => body.trim() !== "")
@@ -204,5 +244,5 @@ export const parseItems = (
     ...unlabelled(read, variantField),
   ];
   if (problems.length > 0) return { problems: problems.toSorted((left, right) => left.line - right.line) };
-  return read.length === 0 ? { problems: [{ kind: "empty", line: 0 }] } : { items: read.map((entry) => entry.item) };
+  return read.length === 0 ? { problems: [{ kind: "empty", line: 0 }] } : { items: read.map((entry) => entry.item), lines: read.map((entry) => entry.line) };
 };

package/src/grade/result.ts

@@ -3,6 +3,7 @@ import type { CitationStatus } from "../structure/cite.ts";
 import type { Severity } from "../plugin.ts";
 import type { OutputSize } from "./rates.ts";
 import type { GradeAiScore } from "./ai-score.ts";
+import type { GradeExpectResult } from "./expect.ts";
 
 // The shape of one line of `chaff grade --out`, and of what `grade()` returns. Spec §29.3.
 
@@ -62,6 +63,8 @@ export type GradeResult = {
   readonly id: string;
   /** The item's variant label, when it had one. */
   readonly variant?: string | undefined;
+  /** The rubric task the output was graded under, when the item named one. */
+  readonly task?: string | undefined;
   readonly language: string;
   readonly genre: string;
   readonly size: OutputSize;
@@ -74,6 +77,8 @@ export type GradeResult = {
   readonly citations: GradeCitations | null;
   /** Only when the item gave contexts, so results of items without them keep their shape. */
   readonly contexts?: GradeContexts | undefined;
+  /** Only when the item gave expect: the entries checked, the facts not stated and the absent ones stated. */
+  readonly expect?: GradeExpectResult | undefined;
   /** Only with a `grade:` rubric. */
   readonly score?: GradeScore | undefined;
   /** The AI-likeness quick score. Absent only in results written before it existed. Never part of pass or fail. */

package/src/grade/results-read.ts

@@ -72,8 +72,13 @@ const isRates = (value: unknown): boolean => isRecord(value) && Object.values(va
 /** The id, and the variant when the output had one. */
 const isLabel = (value: Record<string, unknown>): boolean => isString(value["id"]) && (value["variant"] === undefined || isString(value["variant"]));
 
-/** The parts a result carries only sometimes: contexts, a rubric's score, the quick score. Each is checked when present. */
-const hasOptionalParts = (value: Record<string, unknown>): boolean => isContexts(value["contexts"]) && isScore(value["score"]) && isAiScore(value["aiScore"]);
+/** Absent when the item gave no expect. */
+const isExpect = (value: unknown): boolean =>
+  value === undefined || (isRecord(value) && isNumber(value["checked"]) && isArrayOf(value["missing"], isString) && isArrayOf(value["present"], isString));
+
+/** The parts a result carries only sometimes: contexts, expect, a rubric's score, the quick score. Each is checked when present. */
+const hasOptionalParts = (value: Record<string, unknown>): boolean =>
+  isContexts(value["contexts"]) && isExpect(value["expect"]) && isScore(value["score"]) && isAiScore(value["aiScore"]);
 
 /** A line `--out` writes, every field checked: a line that only looks like one would be compared as if it had been graded. */
 export const isGradeResult = (value: unknown): value is GradeResult =>