chaffjs 0.33.0 → 0.34.0

package/src/grade/rubric-file.ts

@@ -0,0 +1,46 @@
+import { createHash } from "node:crypto";
+import { extname } from "node:path";
+import { parse } from "yaml";
+import { parseRubric, type Rubric, type RubricProblem } from "./rubric.ts";
+
+// A rubric file given with `chaff grade --rubric <file>`: the schema of chaff.yaml's `grade:`, at the file's top level.
+// Pure: the caller reads the file. Its paths drop the `grade.` prefix, because in the file they start at the top.
+
+type Format = "yaml" | "json";
+
+const FORMATS: Readonly<Record<string, Format>> = { ".yaml": "yaml", ".yml": "yaml", ".json": "json" };
+
+export type RubricFileProblem =
+  { readonly kind: "format" } | { readonly kind: "syntax"; readonly why: string } | { readonly kind: "rubric"; readonly problem: RubricProblem };
+
+export type RubricFile = { readonly rubric: Rubric; readonly digest: string } | { readonly problems: readonly RubricFileProblem[] };
+
+const formatOf = (path: string): Format | undefined => FORMATS[extname(path).toLowerCase()];
+
+const syntaxOf = (content: string, format: Format): { readonly raw: unknown } | { readonly why: string } => {
+  try {
+    // An empty file reads as null in YAML but throws in JSON; both are an empty rubric, as a bare `grade:` is.
+    if (content.trim() === "") return { raw: null };
+    return { raw: format === "json" ? JSON.parse(content) : parse(content) };
+  } catch (error) {
+    return { why: error instanceof Error ? error.message : String(error) };
+  }
+};
+
+/** `grade.rules.x` in chaff.yaml is `rules.x` in a rubric file; `grade` itself is the file's top level, the empty path. */
+export const filePathOf = (gradePath: string): string => (gradePath === "grade" ? "" : gradePath.replace(/^grade\./u, ""));
+
+/** The sha256 of the file's bytes as read, recorded in the stamp so runs under different rubric files are not compared silently. */
+export const rubricFileDigest = (content: string): string => `sha256:${createHash("sha256").update(content).digest("hex")}`;
+
+/** The rubric a file holds, or every problem with it. An empty file is an empty rubric, as a bare `grade:` is. */
+export const parseRubricFile = (path: string, content: string): RubricFile => {
+  const format = formatOf(path);
+  if (format === undefined) return { problems: [{ kind: "format" }] };
+  const read = syntaxOf(content, format);
+  if ("why" in read) return { problems: [{ kind: "syntax", why: read.why }] };
+  const parsed = parseRubric(read.raw ?? null);
+  if ("problems" in parsed)
+    return { problems: parsed.problems.map((problem) => ({ kind: "rubric", problem: { ...problem, path: filePathOf(problem.path) } })) };
+  return { rubric: parsed.rubric ?? { rules: {} }, digest: rubricFileDigest(content) };
+};

package/src/grade/rubric.ts

@@ -31,6 +31,8 @@ export type Rubric = {
   readonly citations?: CitationLimits | undefined;
   readonly contexts?: ContextLimits | undefined;
   readonly penalty?: number | undefined;
+  /** Each task's own rubric, as written: what it does not write comes from the top level (task-rubric.ts). */
+  readonly tasks?: Readonly<Record<string, Rubric>> | undefined;
 };
 
 /** What a value under `grade:` should have been. */
@@ -78,10 +80,10 @@ const ruleLimit = (raw: unknown, path: string): Read<RuleLimit> => {
   };
 };
 
-const rulesOf = (raw: unknown): Read<Record<string, RuleLimit>> => {
+const rulesOf = (raw: unknown, base: string): Read<Record<string, RuleLimit>> => {
   if (raw === undefined) return ok({});
-  if (!isRecord(raw)) return { value: {}, problems: [problem("grade.rules", "map", raw)] };
-  const read = Object.entries(raw).map(([id, limit]) => ({ id, read: ruleLimit(limit, `grade.rules.${id}`) }));
+  if (!isRecord(raw)) return { value: {}, problems: [problem(`${base}.rules`, "map", raw)] };
+  const read = Object.entries(raw).map(([id, limit]) => ({ id, read: ruleLimit(limit, `${base}.rules.${id}`) }));
   return { value: Object.fromEntries(read.map((entry) => [entry.id, entry.read.value])), problems: read.flatMap((entry) => entry.read.problems) };
 };
 
@@ -98,13 +100,14 @@ const kindsOf = (raw: unknown, path: string): Read<AtomKind[]> => {
   return kinds.length === entries.length ? ok(kinds) : { value: kinds, problems: [problem(path, "kinds", raw)] };
 };
 
-const factsOf = (raw: unknown): Read<FactLimits | undefined> => {
+const factsOf = (raw: unknown, base: string): Read<FactLimits | undefined> => {
   if (raw === undefined) return ok(undefined);
-  if (!isRecord(raw)) return { value: undefined, problems: [problem("grade.facts", "map", raw)] };
-  const dropped = optional(raw, "dropped", "grade.facts", "count");
-  const added = optional(raw, "added", "grade.facts", "count");
-  const allowDropped = kindsOf(raw["allow_dropped"], "grade.facts.allow_dropped");
-  const allowAdded = kindsOf(raw["allow_added"], "grade.facts.allow_added");
+  const path = `${base}.facts`;
+  if (!isRecord(raw)) return { value: undefined, problems: [problem(path, "map", raw)] };
+  const dropped = optional(raw, "dropped", path, "count");
+  const added = optional(raw, "added", path, "count");
+  const allowDropped = kindsOf(raw["allow_dropped"], `${path}.allow_dropped`);
+  const allowAdded = kindsOf(raw["allow_added"], `${path}.allow_added`);
   return {
     value: { dropped: dropped.value, added: added.value, allowDropped: allowDropped.value, allowAdded: allowAdded.value },
     problems: [
@@ -112,7 +115,7 @@ const factsOf = (raw: unknown): Read<FactLimits | undefined> => {
       ...added.problems,
       ...allowDropped.problems,
       ...allowAdded.problems,
-      ...unknownKeys(raw, ["dropped", "added", "allow_dropped", "allow_added"], "grade.facts"),
+      ...unknownKeys(raw, ["dropped", "added", "allow_dropped", "allow_added"], path),
     ],
   };
 };
@@ -125,62 +128,51 @@ const requiredOf = (raw: Record<string, unknown>, path: string): Read<boolean> =
     : { value: false, problems: [problem(`${path}.required`, "boolean", required)] };
 };
 
-const citationsOf = (raw: unknown): Read<CitationLimits | undefined> => {
+const citationsOf = (raw: unknown, base: string): Read<CitationLimits | undefined> => {
   if (raw === undefined) return ok(undefined);
-  if (!isRecord(raw)) return { value: undefined, problems: [problem("grade.citations", "map", raw)] };
-  const failed = optional(raw, "failed", "grade.citations", "count");
-  const required = requiredOf(raw, "grade.citations");
+  const path = `${base}.citations`;
+  if (!isRecord(raw)) return { value: undefined, problems: [problem(path, "map", raw)] };
+  const failed = optional(raw, "failed", path, "count");
+  const required = requiredOf(raw, path);
   return {
     value: { failed: failed.value, required: required.value },
-    problems: [...failed.problems, ...required.problems, ...unknownKeys(raw, ["failed", "required"], "grade.citations")],
+    problems: [...failed.problems, ...required.problems, ...unknownKeys(raw, ["failed", "required"], path)],
   };
 };
 
-const contextsOf = (raw: unknown): Read<ContextLimits | undefined> => {
+const contextsOf = (raw: unknown, base: string): Read<ContextLimits | undefined> => {
   if (raw === undefined) return ok(undefined);
-  if (!isRecord(raw)) return { value: undefined, problems: [problem("grade.contexts", "map", raw)] };
-  const unsupported = optional(raw, "unsupported", "grade.contexts", "count");
-  const allowUnsupported = kindsOf(raw["allow_unsupported"], "grade.contexts.allow_unsupported");
-  const required = requiredOf(raw, "grade.contexts");
+  const path = `${base}.contexts`;
+  if (!isRecord(raw)) return { value: undefined, problems: [problem(path, "map", raw)] };
+  const unsupported = optional(raw, "unsupported", path, "count");
+  const allowUnsupported = kindsOf(raw["allow_unsupported"], `${path}.allow_unsupported`);
+  const required = requiredOf(raw, path);
   return {
     value: { unsupported: unsupported.value, allowUnsupported: allowUnsupported.value, required: required.value },
     problems: [
       ...unsupported.problems,
       ...allowUnsupported.problems,
       ...required.problems,
-      ...unknownKeys(raw, ["unsupported", "allow_unsupported", "required"], "grade.contexts"),
+      ...unknownKeys(raw, ["unsupported", "allow_unsupported", "required"], path),
     ],
   };
 };
 
-const RUBRIC_KEYS: readonly string[] = ["rules", "required_sections", "facts", "citations", "contexts", "penalty"];
+/** The keys a task's rubric may write: every top-level key but `tasks`, so a task cannot hold tasks of its own. */
+const TASK_KEYS: readonly string[] = ["rules", "required_sections", "facts", "citations", "contexts", "penalty"];
 
-/**
- * The rubric, or every problem in it. No `grade:` key is no rubric: the default pass or fail applies (spec §29.3). A bare
- * `grade:` (null) is an empty rubric, so writing the key alone still turns the default off, as the spec says it does.
- */
-export const parseRubric = (raw: unknown): { readonly rubric: Rubric | undefined } | { readonly problems: readonly RubricProblem[] } => {
-  if (raw === undefined) return { rubric: undefined };
-  if (raw === null) return { rubric: { rules: {} } };
-  if (!isRecord(raw)) return { problems: [problem("grade", "map", raw)] };
-  const rules = rulesOf(raw["rules"]);
-  const sections = wordsOf(raw["required_sections"], "grade.required_sections");
-  const facts = factsOf(raw["facts"]);
-  const citations = citationsOf(raw["citations"]);
-  const contexts = contextsOf(raw["contexts"]);
-  const penalty = optional(raw, "penalty", "grade", "number");
-  const problems = [
-    ...rules.problems,
-    ...sections.problems,
-    ...facts.problems,
-    ...citations.problems,
-    ...contexts.problems,
-    ...penalty.problems,
-    ...unknownKeys(raw, RUBRIC_KEYS, "grade"),
-  ];
-  if (problems.length > 0) return { problems };
+const RUBRIC_KEYS: readonly string[] = [...TASK_KEYS, "tasks"];
+
+/** One rubric's keys under `base` (`grade`, or `grade.tasks.<name>`); a key not among `keys` is a problem. */
+const rubricAt = (raw: Record<string, unknown>, base: string, keys: readonly string[]): Read<Rubric> => {
+  const rules = rulesOf(raw["rules"], base);
+  const sections = wordsOf(raw["required_sections"], `${base}.required_sections`);
+  const facts = factsOf(raw["facts"], base);
+  const citations = citationsOf(raw["citations"], base);
+  const contexts = contextsOf(raw["contexts"], base);
+  const penalty = optional(raw, "penalty", base, "number");
   return {
-    rubric: {
+    value: {
       rules: rules.value,
       requiredSections: sections.value,
       facts: facts.value,
@@ -188,5 +180,43 @@ export const parseRubric = (raw: unknown): { readonly rubric: Rubric | undefined
       contexts: contexts.value,
       penalty: penalty.value,
     },
+    problems: [
+      ...rules.problems,
+      ...sections.problems,
+      ...facts.problems,
+      ...citations.problems,
+      ...contexts.problems,
+      ...penalty.problems,
+      ...unknownKeys(raw, keys, base),
+    ],
   };
 };
+
+/** One task's rubric. A bare `<name>:` (null) writes nothing, so the task inherits the whole top-level rubric. */
+const taskOf = (raw: unknown, path: string): Read<Rubric> => {
+  if (raw === null) return ok({ rules: {} });
+  if (!isRecord(raw)) return { value: { rules: {} }, problems: [problem(path, "map", raw)] };
+  return rubricAt(raw, path, TASK_KEYS);
+};
+
+const tasksOf = (raw: unknown): Read<Record<string, Rubric> | undefined> => {
+  if (raw === undefined) return ok(undefined);
+  if (!isRecord(raw)) return { value: undefined, problems: [problem("grade.tasks", "map", raw)] };
+  const read = Object.entries(raw).map(([name, task]) => ({ name, read: taskOf(task, `grade.tasks.${name}`) }));
+  return { value: Object.fromEntries(read.map((entry) => [entry.name, entry.read.value])), problems: read.flatMap((entry) => entry.read.problems) };
+};
+
+/**
+ * The rubric, or every problem in it. No `grade:` key is no rubric: the default pass or fail applies (spec §29.3). A bare
+ * `grade:` (null) is an empty rubric, so writing the key alone still turns the default off, as the spec says it does.
+ */
+export const parseRubric = (raw: unknown): { readonly rubric: Rubric | undefined } | { readonly problems: readonly RubricProblem[] } => {
+  if (raw === undefined) return { rubric: undefined };
+  if (raw === null) return { rubric: { rules: {} } };
+  if (!isRecord(raw)) return { problems: [problem("grade", "map", raw)] };
+  const top = rubricAt(raw, "grade", RUBRIC_KEYS);
+  const tasks = tasksOf(raw["tasks"]);
+  const problems = [...top.problems, ...tasks.problems];
+  if (problems.length > 0) return { problems };
+  return { rubric: tasks.value === undefined ? top.value : { ...top.value, tasks: tasks.value } };
+};

package/src/grade/setup.ts

@@ -28,16 +28,16 @@ const ruleSetOf = async (config: Config, languages: readonly string[]): Promise<
   };
 };
 
-/** A run's settings, rubric and stamp. `languages`: those the items name, beyond the bundled ones. Throws when a language package cannot be loaded. */
+/** A run's settings, rubric and stamp. `run.rubricFile`: a --rubric file's digest, when the rubric came from one. `languages`: those the items name, beyond the bundled ones. Throws when a language package cannot be loaded. */
 export const gradeSetup = async (
   config: Config,
-  run: { readonly experimental: boolean; readonly genre: string | undefined },
+  run: { readonly experimental: boolean; readonly genre: string | undefined; readonly rubricFile?: string | undefined },
   languages: readonly string[],
   rubric: Rubric | undefined,
 ): Promise<GradeSetup> => {
   // chaff.yaml's language and its by_path languages are loaded here too, so a missing package stops the run before grading.
   const named = [config.language, ...config.byPath.map((rule) => rule.language)].filter((language) => language !== undefined);
   const all = [...new Set([...BUNDLED_LANGUAGES, ...named, ...languages])].toSorted(compareText);
-  const settings = { config, experimental: run.experimental, genre: run.genre };
+  const settings = { config, experimental: run.experimental, genre: run.genre, rubricFile: run.rubricFile };
   return { ...settings, rubric, stamp: stampOf(VERSION_LINES, await ruleSetOf(config, all), settings) };
 };

package/src/grade/stamp.ts

@@ -20,6 +20,15 @@ export type GradeSettings = {
   readonly experimental: boolean;
   /** --genre, which wins over chaff.yaml for every item that does not name its own. */
   readonly genre: string | undefined;
+  /** The digest of a --rubric file, which replaces chaff.yaml's `grade:` for the run. */
+  readonly rubricFile?: string | undefined;
+};
+
+/** The rubric a run grades by: a --rubric file by its digest, else chaff.yaml's `grade:`. */
+const gradeOf = (settings: GradeSettings): unknown => {
+  if (settings.rubricFile !== undefined) return { file: settings.rubricFile };
+  // A bare `grade:` reads as null and is an empty rubric, which grades otherwise than no `grade:` at all.
+  return settings.config.grade === undefined ? "none" : settings.config.grade;
 };
 
 /** The parts of chaff.yaml that change a finding, a fact or a verdict. Paths and the AI judge do not. */
@@ -40,8 +49,7 @@ export const settingsOf = (settings: GradeSettings): Record<string, unknown> =>
     requiredSections: config.requiredSections,
     names: config.names,
     byPath: config.byPath,
-    // A bare `grade:` reads as null and is an empty rubric, which grades otherwise than no `grade:` at all.
-    grade: config.grade === undefined ? "none" : config.grade,
+    grade: gradeOf(settings),
   };
 };
 

package/src/grade/task-rubric.ts

@@ -0,0 +1,39 @@
+import type { Rubric } from "./rubric.ts";
+
+// The rubric one item is graded by: the top-level rubric with its task's rubric laid over it. Pure.
+// A key the task writes replaces the top-level key whole (`facts`, `citations`, `contexts` are not merged field by
+// field); a key it does not write is inherited. `rules` is merged per rule id: the task's entry for an id replaces the
+// top-level entry for that id, and every other top-level id still applies.
+
+/** The task names a rubric defines; none without a rubric or without `tasks`. */
+export const taskNamesOf = (rubric: Rubric | undefined): readonly string[] => Object.keys(rubric?.tasks ?? {});
+
+/** Each rule id the rubric names and where, from the rubric's top level: `rules`, or `tasks.<name>.rules`. */
+export const rubricRuleEntries = (rubric: Rubric | undefined): readonly { readonly path: string; readonly id: string }[] => [
+  ...Object.keys(rubric?.rules ?? {}).map((id) => ({ path: "rules", id })),
+  ...Object.entries(rubric?.tasks ?? {}).flatMap(([name, task]) => Object.keys(task.rules).map((id) => ({ path: `tasks.${name}.rules`, id }))),
+];
+
+const overlay = (top: Rubric, task: Rubric): Rubric => ({
+  rules: { ...top.rules, ...task.rules },
+  requiredSections: task.requiredSections ?? top.requiredSections,
+  facts: task.facts ?? top.facts,
+  citations: task.citations ?? top.citations,
+  contexts: task.contexts ?? top.contexts,
+  penalty: task.penalty ?? top.penalty,
+});
+
+/**
+ * The rubric for an item of `task`. Without a task, the rubric as it is. A task the rubric does not define is refused
+ * when the items are read, so here it cannot happen; it throws rather than fall back to the top level silently.
+ */
+export const rubricForTask = (rubric: Rubric | undefined, task: string | undefined): Rubric | undefined => {
+  if (task === undefined) return rubric;
+  const own = rubric?.tasks?.[task];
+  if (rubric === undefined || own === undefined) throw new Error(`chaff grade: no rubric for task "${task}"`);
+  return overlay(rubric, own);
+};
+
+/** The rule ids an output of `task` is graded by: whose rise in that output is a regression against a baseline. */
+export const taskRuleIds = (rubric: Rubric | undefined, task: string | undefined): ReadonlySet<string> =>
+  new Set(Object.keys(rubricForTask(rubric, task)?.rules ?? {}));

package/src/grade/text.ts

@@ -11,7 +11,11 @@ export type GradeText = {
   readonly contextsUnread: (kind: string, reason: string) => string;
   readonly uncheckedSentences: (count: number) => string;
   readonly unknownRule: string;
-  readonly rubricProblem: (problem: RubricProblem) => string;
+  /** A problem under chaff.yaml's `grade:`, or, given `file`, under a --rubric file's top level. */
+  readonly rubricProblem: (problem: RubricProblem, file?: string) => string;
+  /** A --rubric file whose extension is none of .yaml, .yml, .json. */
+  readonly rubricFormat: (file: string) => string;
+  readonly rubricSyntax: (file: string, why: string) => string;
   readonly penalty: (total: number) => string;
   /** The outputs at each level of the AI-likeness quick score. */
   readonly aiScore: (low: number, medium: number, high: number, notScored: number) => string;
@@ -55,6 +59,12 @@ const PROBLEM_JA: Readonly<Record<ItemProblemKind, (problem: ItemProblem) => str
   "which-source": (problem) => `${lineJa(problem)}番地 ${problem.detail ?? ""} の引用に source がありません。原文が二つ以上あるときは名前を書いてください`,
   "unknown-language": (problem) => `${lineJa(problem)}language "${problem.detail ?? ""}" は読めません`,
   "unknown-genre": (problem) => `${lineJa(problem)}genre "${problem.detail ?? ""}" は chaff の知らないジャンルです（chaff genres で一覧）`,
+  "not-expect": (problem) => `${lineJa(problem)}${problem.detail ?? ""} は { "facts": ["…"], "absent": ["…"] } の形で、空でない文字列の並びを書いてください`,
+  "expect-no-fact": (problem) =>
+    `${lineJa(problem)}${problem.detail ?? ""} から事実を読めません（数・日付・時刻・URL・コード・名前・引用のどれか 1 つを書いてください）`,
+  "expect-several-facts": (problem) =>
+    `${lineJa(problem)}${problem.detail ?? ""} は事実を 2 つ以上含みます（compare が 1 つの事実と読む書き方で、1 項目に 1 つ書いてください）`,
+  "unknown-task": (problem) => `${lineJa(problem)}task "${problem.detail ?? ""}" は採点基準の tasks にありません`,
   empty: () => "採点する出力が 1 行もありません",
 };
 
@@ -74,6 +84,12 @@ const PROBLEM_EN: typeof PROBLEM_JA = {
   "which-source": (problem) => `${lineEn(problem)}the citation of ${problem.detail ?? ""} names no source; with two or more sources, name one`,
   "unknown-language": (problem) => `${lineEn(problem)}language "${problem.detail ?? ""}" cannot be read`,
   "unknown-genre": (problem) => `${lineEn(problem)}genre "${problem.detail ?? ""}" is not one chaff knows (chaff genres lists them)`,
+  "not-expect": (problem) => `${lineEn(problem)}${problem.detail ?? ""} must be { "facts": ["…"], "absent": ["…"] } with non-empty strings`,
+  "expect-no-fact": (problem) =>
+    `${lineEn(problem)}${problem.detail ?? ""} states no fact chaff can read (write one number, date, time, URL, code, name or quotation)`,
+  "expect-several-facts": (problem) =>
+    `${lineEn(problem)}${problem.detail ?? ""} states more than one fact (write one per entry, in a form compare reads as one fact)`,
+  "unknown-task": (problem) => `${lineEn(problem)}task "${problem.detail ?? ""}" is not among the rubric's tasks`,
   empty: () => "no output to grade: the file has no item",
 };
 
@@ -103,14 +119,17 @@ const aside = (open: string, inner: string, close: string): string => (inner ===
 export const GRADE_TEXT: Texts<GradeText> = {
   ja: {
     usage:
-      "使い方: chaff grade <items.jsonl> [--out <results.jsonl>] [--baseline <前の results.jsonl>] [--allow-stamp-mismatch] [--variant-key <欄の名前>] [--format text|json|markdown] [--json] [--compact] [--experimental] [--genre <ジャンル>]",
+      "使い方: chaff grade <items.jsonl> [--rubric <rubric.yaml|.json>] [--out <results.jsonl>] [--baseline <前の results.jsonl>] [--allow-stamp-mismatch] [--variant-key <欄の名前>] [--format text|json|markdown] [--json] [--compact] [--experimental] [--genre <ジャンル>]",
     noReference: "reference が無い（事実は reference と照らす）",
     noCitations: "citations が無い（chaff は出力から引用を推測しない）",
     noContexts: "contexts が無い（回答の事実は、検索で取った一節と照らす）",
     contextsUnread: (kind, reason) => `contexts と照らすとき、${kind} を読み切れなかった（${reason}）。その種類の事実は、一節にあっても見つからないことがある`,
     uncheckedSentences: (count) => `数・日付・名前・引用の無い文 ${String(count)} 件は、contexts に支えられているかを照らしていない（意味を読む必要がある）`,
     unknownRule: "grade: に書かれているが、chaff の知らないルール",
-    rubricProblem: (problem) => `chaff.yaml の ${problem.path} は${EXPECTED_JA[problem.expected]}で書いてください（書かれていたのは ${problem.written}）`,
+    rubricProblem: (problem, file) =>
+      `${file ?? "chaff.yaml"} の ${problem.path === "" ? "最上位" : problem.path} は${EXPECTED_JA[problem.expected]}で書いてください（書かれていたのは ${problem.written}）`,
+    rubricFormat: (file) => `${file}: --rubric のファイルは .yaml・.yml・.json のどれかにしてください`,
+    rubricSyntax: (file, why) => `${file} を採点の基準として読めませんでした: ${why}`,
     penalty: (total) => `減点の和: ${String(total)}`,
     aiScore: (low, medium, high, notScored) =>
       `AI らしさ: 低 ${String(low)}、中 ${String(medium)}、高 ${String(high)}、測っていない ${String(notScored)}（書いたのが AI かどうかの判定ではありません。合否には入りません）`,
@@ -135,7 +154,7 @@ export const GRADE_TEXT: Texts<GradeText> = {
   },
   en: {
     usage:
-      "usage: chaff grade <items.jsonl> [--out <results.jsonl>] [--baseline <earlier results.jsonl>] [--allow-stamp-mismatch] [--variant-key <field>] [--format text|json|markdown] [--json] [--compact] [--experimental] [--genre <genre>]",
+      "usage: chaff grade <items.jsonl> [--rubric <rubric.yaml|.json>] [--out <results.jsonl>] [--baseline <earlier results.jsonl>] [--allow-stamp-mismatch] [--variant-key <field>] [--format text|json|markdown] [--json] [--compact] [--experimental] [--genre <genre>]",
     noReference: "no reference given (facts are checked against a reference)",
     noCitations: "no citations given (chaff does not guess quotations from the output)",
     noContexts: "no contexts given (the output's facts are checked against retrieved passages)",
@@ -144,7 +163,10 @@ export const GRADE_TEXT: Texts<GradeText> = {
     uncheckedSentences: (count) =>
       `${String(count)} ${count === 1 ? "sentence states" : "sentences state"} no number, date, name or quotation, so their support by contexts is not checked (it needs reading)`,
     unknownRule: "named under grade: but not a rule chaff knows",
-    rubricProblem: (problem) => `chaff.yaml: ${problem.path} must be ${EXPECTED_EN[problem.expected]} (found ${problem.written})`,
+    rubricProblem: (problem, file) =>
+      `${file ?? "chaff.yaml"}: ${problem.path === "" ? "the top level" : problem.path} must be ${EXPECTED_EN[problem.expected]} (found ${problem.written})`,
+    rubricFormat: (file) => `${file}: a --rubric file must end in .yaml, .yml or .json`,
+    rubricSyntax: (file, why) => `Could not read ${file} as a rubric: ${why}`,
     penalty: (total) => `Penalty points: ${String(total)}`,
     aiScore: (low, medium, high, notScored) =>
       `AI-likeness: low ${String(low)}, medium ${String(medium)}, high ${String(high)}, not scored ${String(notScored)} (not a verdict on whether AI wrote them; not part of pass or fail)`,

package/src/name-char-reading.ts

@@ -22,3 +22,19 @@ export const readingOfWords = (words: readonly ReadToken[], chars: CharReadings)
   const readings = words.map((token) => tokenReadingOf(token, chars));
   return words.length > 0 && readings.every(isRead) ? readings.join("") : undefined;
 };
+
+/** 名の字の読みで読み直す、名の字数の下限。一字の名（太 を フトシ）は、二字の名の中の読み（健太 の タ）で読まない。 */
+const MIN_GIVEN_LETTERS = 2;
+
+/**
+ * 名をもう一通りに読んだもの。解析器は名を別の読みに切ることがある（建太 を タテ と フトシ）。字がどれも givenChars（名の中での
+ * 字の読み）にある語はその字の読みをつなぎ、ほかの語は tokenReadingOf で読む。一字の名と、読めない語のある名は読まない。
+ */
+export const givenNameReadingOf = (words: readonly ReadToken[], givenChars: CharReadings, chars: CharReadings): string | undefined => {
+  if (words.reduce((letters, token) => letters + [...token.surface].length, 0) < MIN_GIVEN_LETTERS) return undefined;
+  const readings = words.map((token) => {
+    const letters = [...token.surface].map((letter) => givenChars.get(letter));
+    return letters.length > 0 && letters.every(isRead) ? letters.join("") : tokenReadingOf(token, chars);
+  });
+  return readings.every(isRead) ? readings.join("") : undefined;
+};

package/src/name-end-letter.ts

@@ -0,0 +1,52 @@
+// 名前の語の終わりの一字の抜けか足し（Whitmore と Whitmor、Hollybrook と Hollybrooke）。ふつうの語なら語の形の違い
+// （Service と Services、America と American）なので、どちらも名前にしか使わない語に限る。
+
+const LATIN_WORD = /^[a-z]+$/u;
+const VOWEL = /[aeiouy]$/u;
+
+/** 足すと別の語の形になる字。s は複数と所有（Rental と Rentals）。 */
+const PLURAL_LETTER = "s";
+/** 母音で終わる語に足すと、その土地の人や言葉の語になる字（Korea と Korean、Rica と Rican）。 */
+const DEMONYM_LETTER = "n";
+
+/** 比べる語の長さ。三字の語（Ann と Anne）は別の名前のことが多い。 */
+const MIN_END_LETTER_LENGTH = 4;
+/** 終わりの一字の抜けか足しを、名前の最後でない語で見る長さ（Hollybrook と Hollybrooke）。 */
+const MIN_END_SLIP_ANYWHERE = 8;
+
+const isFormLetter = (shorter: string, added: string): boolean => added === PLURAL_LETTER || (added === DEMONYM_LETTER && VOWEL.test(shorter));
+
+/** 二つの英字の語が、終わりの一字の抜けか足しだけ違うか。足した字が語の形を作る字（s、母音の後ろの n）なら違うと見ない。 */
+export const isEndLetterApart = (left: string, right: string): boolean => {
+  const [shorter, longer] = left.length < right.length ? [left, right] : [right, left];
+  if (!LATIN_WORD.test(shorter) || !LATIN_WORD.test(longer) || shorter.length < MIN_END_LETTER_LENGTH) return false;
+  return longer.length === shorter.length + 1 && longer.startsWith(shorter) && !isFormLetter(shorter, longer.slice(shorter.length));
+};
+
+/**
+ * 文書での語の使い方（小文字で）。ordinary は名前と読んだ語のうち小文字で書けばふつうの語（Rental、Green）、common は文書が
+ * 小文字で書くか、品詞解析が普通名詞と読む語（hurricane、lake）。
+ */
+export type WordUse = { readonly ordinary: ReadonlySet<string>; readonly common: ReadonlySet<string> };
+
+export const NO_WORD_USE: WordUse = { ordinary: new Set(), common: new Set() };
+
+/**
+ * 名前の語（words、小文字）の at の語を usual と書いた名前があるとき、その違いを書き損じと見るか。違う語が最後の語（姓）なら、
+ * 終わりの一字の抜けか足しか、四字以上の一字違い（oneLetterApart）で、前の語がどれも文書の中で普通名詞として書かない語（Rachel、Mr）
+ * のとき。Hurricane Hanna と Hurricane Hannah は名前の違う二つの嵐。最後でない語なら、八字以上の終わりの一字の抜けか足しで、名前の
+ * 最後の語がふつうの語（Residences、Products）のとき。人の名の終わりの一字だけ違う名前（Christian と Christiane）は別の人のことが多い。
+ */
+export const isEndWordSlip = (
+  words: readonly string[],
+  at: number,
+  usual: string,
+  use: WordUse,
+  oneLetterApart: (left: string, right: string) => boolean,
+): boolean => {
+  const word = words[at] ?? "";
+  const shorter = Math.min(word.length, usual.length);
+  if (at < words.length - 1) return shorter >= MIN_END_SLIP_ANYWHERE && use.ordinary.has(words.at(-1) ?? "") && isEndLetterApart(word, usual);
+  if (words.slice(0, at).some((before) => use.common.has(before))) return false;
+  return isEndLetterApart(word, usual) || (shorter >= MIN_END_LETTER_LENGTH && oneLetterApart(word, usual));
+};

package/src/name-neighbours.ts

@@ -47,11 +47,12 @@ export const withKnownNeighbours = (mentions: readonly NameMention[], source: st
         surface: `${before} ${mention.surface}`,
         offset: mention.offset - 1 - before.length,
         reading: undefined,
+        givenNameReading: undefined,
         words: [before, ...mention.words],
       };
     }
     if (after !== undefined && known.has(`${mention.surface} ${after}`) && !taken(end + 1, end + 1 + after.length)) {
-      return { ...mention, surface: `${mention.surface} ${after}`, reading: undefined, words: [...mention.words, after] };
+      return { ...mention, surface: `${mention.surface} ${after}`, reading: undefined, givenNameReading: undefined, words: [...mention.words, after] };
     }
     return mention;
   });

package/src/name-separator.ts

@@ -34,6 +34,38 @@ const isNameLine = (lineTokens: readonly Token[]): boolean => lineTokens.length
 const isOrganizationName = (word: string, wordTokens: readonly Token[], words: OrganizationWords): boolean =>
   words.units.has(wordTokens.at(-1)?.surface ?? "") || words.forms.some((form) => word.length > form.length && (word.startsWith(form) || word.endsWith(form)));
 
+/** 行の頭の項目名とコロン（所属：、振込先：）。項目名は空白も数字も含まない短い語。 */
+const FIELD_LABEL = /^[ \t]*(?:[-*+][ \t]+)?[^\s：:。\d]{1,12}[：:][ \t\u3000]*/u;
+
+/** 文の終わり。値を文で書いた行（会場：本社 5階で行います。）の空白は、文の中の空け方として数える。 */
+const SENTENCE_END = /[。！？]/u;
+
+/** 値を句で書いた行（目標：参加者 100人を想定）に出る品詞。名前と番号を並べた値には出ない。 */
+const CLAUSE_POS = new Set(["VERB", "AUX", "ADP"]);
+
+/** 数の前に書いて量の一部になる語（約 3 か月、およそ 3 日、全 5 店舗）や、文の中の語（本社は 5階）の品詞。 */
+const QUANTITY_LEAD_POS = new Set(["ADJ", "ADV", "ADP", "AUX", "VERB", "SCONJ", "PART"]);
+
+/** 値の組の最も短い長さ。1 字の語（計 5件）は量の前置き。 */
+const MIN_PART_LENGTH = 2;
+
+/**
+ * 「所属：経済学部 経済学科 1年」「物件：メゾンかえで 203号室」「振込先：みなと銀行 普通 1234567」のように、項目名とコロンの後ろに
+ * 値の組を空白で並べた行で、組と、数で始まる次の組のあいだの空白。値の区切りで、日本語と数字のあいだの空け方ではない。
+ * boundary は空けた数字の前の境目（text[offset] が空白）。tokens は文書全体の座標で、base は text の先頭の位置。
+ */
+export const isFieldSeparator = (text: string, boundary: Boundary, tokens: readonly Token[] | undefined, base: number): boolean => {
+  if (tokens === undefined || boundary.kind !== "before-digit" || !boundary.spaced) return false;
+  const line = lineAround(text, boundary.offset);
+  const label = FIELD_LABEL.exec(text.slice(line.start, line.end));
+  if (label === null || SENTENCE_END.test(text.slice(line.start, line.end))) return false;
+  const valueStart = line.start + label[0].length;
+  if (tokensIn(tokens, { start: valueStart, end: line.end }, base).some((token) => CLAUSE_POS.has(token.pos))) return false;
+  const partStart = Math.max(wordStart(text, boundary.offset), valueStart);
+  const partEnd = tokens.find((token) => token.span.end === base + boundary.offset);
+  return boundary.offset - partStart >= MIN_PART_LENGTH && partEnd !== undefined && !QUANTITY_LEAD_POS.has(partEnd.pos);
+};
+
 /**
  * 「株式会社あおば電子 経営企画部 IR担当」「東京本社 PR室」のように、名前だけを空白で並べた行で、空白の両側がどちらも組織の名前のときの空白。
  * 名前と名前の区切りで、英字の前後の空け方ではない。文の中の空白（この IT システムは）や、組織の名前でない語のあいだ（生成AI 活用事例、Patch リリース）は区切りと読まない。