chaffjs 0.18.0 → 0.20.0

package/src/fix-plan/render.ts

@@ -0,0 +1,146 @@
+import type { RewritePair } from "../rule-guide.ts";
+import { shapeMeasures } from "../outline/render.ts";
+import { OUTLINE_TEXT } from "../outline/text.ts";
+import { uiLanguageOf } from "../ui.ts";
+import type { FixPlan, PhraseHint, PlanSpot, RulePlan } from "./plan.ts";
+import { FIX_PLAN_TEXT, type FixPlanText } from "./text.ts";
+import { STRUCTURE_TARGET_TEXT } from "./structure-target-text.ts";
+
+/** A phrase quoted the way the document's language quotes: 「…」 in Japanese, "…" otherwise. */
+const quoted = (text: string, language: string): string => (language === "ja" ? `「${text}」` : `"${text}"`);
+
+const bullets = (items: readonly string[]): string[] => items.map((item) => `- ${item}`);
+
+/** Each line of a passage as a Markdown quotation, so a list inside it stays a list and does not join the plan's own. */
+const blockquote = (passage: string): string[] =>
+  passage
+    .trimEnd()
+    .split("\n")
+    .map((line) => (line === "" ? ">" : `> ${line}`));
+
+const labelled = (label: string, lines: readonly string[]): string[] => (lines.length === 0 ? [] : ["", `**${label}**`, "", ...lines]);
+
+const pairLines = (pair: RewritePair | undefined, text: FixPlanText): string[] =>
+  pair === undefined ? [] : [`${text.before}:`, "", ...blockquote(pair.before), "", `${text.after}:`, "", ...blockquote(pair.after)];
+
+const hintLine = (hint: PhraseHint, language: string): string => `- ${quoted(hint.phrase, language)} → ${hint.rewrite}`;
+
+const spotLines = (spot: PlanSpot, text: FixPlanText, language: string): string[] => [
+  `- ${text.at(spot.line)}: ${quoted(spot.quote, language)}`,
+  `  ${spot.message}`,
+];
+
+const rulePlanLines = (rule: RulePlan, text: FixPlanText, language: string): string[] => [
+  "",
+  `### \`${rule.rule}\` ${rule.name}`,
+  "",
+  `**${text.direction}**: ${rule.direction}`,
+  ...labelled(text.keep, bullets(rule.keep)),
+  ...labelled(text.avoid, bullets(rule.avoid)),
+  ...labelled(text.example, pairLines(rule.pair, text)),
+  ...labelled(
+    text.phrases,
+    rule.hints.map((hint) => hintLine(hint, language)),
+  ),
+  ...labelled(
+    text.spots,
+    rule.spots.flatMap((spot) => spotLines(spot, text, language)),
+  ),
+];
+
+const modeLines = (plan: FixPlan, text: FixPlanText): string[] => [
+  "",
+  `## ${text.modeHeading}: ${text.modeName[plan.mode.mode]}`,
+  "",
+  text.modeReason[plan.mode.reason],
+  "",
+  ...bullets(text.modeWays),
+];
+
+const signalLines = (plan: FixPlan, text: FixPlanText): string[] => {
+  const outline = text.outlineLine(shapeMeasures(plan.outline.shape, plan.outline.unit, OUTLINE_TEXT[uiLanguageOf(plan.language)]));
+  const said = plan.signals.length === 0 ? [text.noSignals, ""] : [];
+  return ["", `## ${text.signalsHeading}`, "", ...said, ...bullets([...plan.signals.map((signal) => `\`${signal.rule}\`: ${signal.message}`), outline])];
+};
+
+/** The structure score and, for each measure past the human limit, the target from the baseline; then when to stop. */
+const structureLines = (plan: FixPlan): string[] => {
+  const text = STRUCTURE_TARGET_TEXT[uiLanguageOf(plan.language)];
+  const targets = plan.targets.map((target) => text.target[target.id](target));
+  const stop = targets.length === 0 ? [] : ["", text.stop(plan.structure.limit)];
+  return [
+    "",
+    `## ${text.heading}`,
+    "",
+    text.score(plan.structure.score, plan.structure.limit, plan.structure.compared),
+    "",
+    ...(targets.length === 0 ? [text.noTargets] : bullets(targets)),
+    ...stop,
+  ];
+};
+
+const notRunLines = (plan: FixPlan, text: FixPlanText): string[] => {
+  const note = plan.experimental ? [] : [text.notExperimental];
+  const listed = plan.notRun;
+  if (note.length === 0 && listed.length === 0) return [];
+  return [
+    "",
+    `## ${text.notRunHeading}`,
+    "",
+    ...note,
+    ...(note.length > 0 && listed.length > 0 ? [""] : []),
+    ...bullets(listed.map((skipped) => `\`${skipped.rule}\`: ${skipped.why}`)),
+  ];
+};
+
+const checkLines = (plan: FixPlan, text: FixPlanText): string[] => [
+  "",
+  `## ${text.checksHeading}`,
+  "",
+  text.saveAs(plan.rewrittenPath),
+  "",
+  "```bash",
+  ...plan.checks,
+  "```",
+  "",
+  text.checksNote,
+];
+
+const textOf = (plan: FixPlan): FixPlanText => FIX_PLAN_TEXT[uiLanguageOf(plan.language)];
+
+/** The plan as Markdown, in the document's language: what an agent reads before rewriting. */
+export const renderFixPlanMarkdown = (plan: FixPlan): string => {
+  const text = textOf(plan);
+  return [
+    `# ${text.title(plan.path)}`,
+    "",
+    text.about(plan.language, plan.genre),
+    "",
+    text.intro,
+    "",
+    `## ${text.constraintsHeading}`,
+    "",
+    ...text.constraints.map((constraint, index) => `${String(index + 1)}. ${constraint}`),
+    ...modeLines(plan, text),
+    ...signalLines(plan, text),
+    ...structureLines(plan),
+    ...(plan.rules.length === 0 ? [] : ["", `## ${text.rulesHeading}`]),
+    ...plan.rules.flatMap((rule) => rulePlanLines(rule, text, plan.language)),
+    ...notRunLines(plan, text),
+    ...checkLines(plan, text),
+  ].join("\n");
+};
+
+/** The same plan as JSON, with the constraints and the mode's reason in the document's language. */
+export const renderFixPlanJson = (plan: FixPlan): string => {
+  const text = textOf(plan);
+  return JSON.stringify(
+    {
+      ...plan,
+      constraints: text.constraints,
+      mode: { ...plan.mode, name: text.modeName[plan.mode.mode], why: text.modeReason[plan.mode.reason] },
+    },
+    null,
+    2,
+  );
+};

package/src/fix-plan/structure-target-text.ts

@@ -0,0 +1,90 @@
+import type { Texts } from "../ui.ts";
+import type { FeatureId } from "../structure-shape/features.ts";
+import type { StructureTarget } from "./structure-targets.ts";
+
+export type StructureTargetText = {
+  readonly heading: string;
+  /** The score, what it counts, and the count at which the plan rewrites from the outline. */
+  readonly score: (score: number, limit: number, compared: number) => string;
+  readonly noTargets: string;
+  /** One target per measure past the human limit, from the baseline's numbers. */
+  readonly target: Readonly<Record<FeatureId, (target: StructureTarget) => string>>;
+  /** When the rewrite of the outline is done. */
+  readonly stop: (limit: number) => string;
+};
+
+const v = (value: number): string => String(value);
+
+const withDetail = (detail: string | undefined, wrap: (detail: string) => string): string => (detail === undefined ? "" : wrap(detail));
+
+const formJa = (form: string): string => `（多いのは「${form}」）`;
+const quotedJa = (heading: string): string => `「${heading}」`;
+const pairsJa = (pairs: string): string => `（${pairs}）`;
+const formEn = (form: string): string => ` (most: "${form}")`;
+const quotedEn = (heading: string): string => ` "${heading}"`;
+const pairsEn = (pairs: string): string => ` (${pairs})`;
+
+export const STRUCTURE_TARGET_TEXT: Texts<StructureTargetText> = {
+  ja: {
+    heading: "構成の目標",
+    score: (score, limit, compared) =>
+      `構成の AI らしさ: ${v(score)}（人の記事の 9 割を超えた項目の数。比べた ${v(compared)} 項目のうち）。${v(limit)} 以上なら構成から書き直します。`,
+    noTargets: "構成の項目は、どれも人の記事の範囲にあります。",
+    target: {
+      "heading-density": (target) => {
+        const budget = target.headings;
+        const counts =
+          budget === undefined ? "" : `いま ${v(budget.now)} 個。この長さの人の記事なら多くて ${v(budget.most)} 個、ふつうは ${v(budget.usual)} 個です。`;
+        return `見出しを減らす: ${counts}一段落ごとに見出しを立てず、中身のまとまりごとに立てます。`;
+      },
+      "short-sections": (target) => `1〜2 段落しかない節を、前後の節と合わせる: いま ${v(target.value)}%（人の記事の 9 割は ${v(target.limit)}% まで）。`,
+      "section-uniformity": (target) =>
+        `節の長さを揃えない: ばらつきがいま ${v(target.value)}%（人の記事の 9 割は ${v(target.limit)}% 以上、ふつうは ${v(target.median)}%）。書くことの少ない節は短く、多い節は長くします。`,
+      "heading-forms": (target) =>
+        `見出しの形を変える: いま ${v(target.value)}% が決まった形${withDetail(target.detail, formJa)}です（人の記事の 9 割は ${v(target.limit)}% まで）。「〜とは」「〜のメリット」をやめ、その節で言うことを見出しにします。`,
+      "three-subsections": (target) =>
+        `3 つの小見出しに分けた見出しを、中身の数に合わせる: いま ${v(target.value)} か所（人の記事の 9 割は ${v(target.limit)} か所まで）。`,
+      bookends: () => "「はじめに」と「まとめ」の見出しを外す: 要点か具体的な場面から書き始め、最後の節は本文の続きにします。",
+      "closing-restatement": (target) =>
+        `本文を言い直す${withDetail(target.detail, quotedJa)}を消す: 言い直しがいま ${v(target.value)}%（人の記事の 9 割は ${v(target.limit)}% まで）。言いたいことは前の節の最後に入れます。`,
+      "three-item-lists": (target) => `3 項目に揃えた箇条書きを、中身の数に合わせる: いま ${v(target.value)}%（人の記事の 9 割は ${v(target.limit)}% まで）。`,
+      "bold-labels": (target) => `太字の札で始まる項目 ${v(target.value)} 個を文に戻す（人の記事の 9 割は ${v(target.limit)} 個まで）。`,
+      "emoji-headings": (target) => `見出しの絵文字 ${v(target.value)} 個を外す（人の記事の 9 割は ${v(target.limit)} 個まで）。`,
+      "pro-con": (target) => `メリットとデメリットを対にした節${withDetail(target.detail, pairsJa)}を、代わりに払うものも書いた一つの話にまとめる。`,
+    },
+    stop: (limit) =>
+      `\`chaff outline <元の文書> <書き直した文書>\` で、上の項目に ✗ が付かなくなり、構成の AI らしさが ${v(limit)} 未満になったら構成の書き直しを止めます。`,
+  },
+  en: {
+    heading: "Structure targets",
+    score: (score, limit, compared) =>
+      `Structure score: ${v(score)} (measures past 90% of human articles, of ${v(compared)} compared). At ${v(limit)} or more, rewrite from the outline.`,
+    noTargets: "Every structure measure is within the range of human articles.",
+    target: {
+      "heading-density": (target) => {
+        const budget = target.headings;
+        const counts =
+          budget === undefined ? "" : `${v(budget.now)} now; a human article this long has ${v(budget.most)} at most, ${v(budget.usual)} usually. `;
+        return `Fewer headings: ${counts}Give a heading to a unit of content, not to every paragraph.`;
+      },
+      "short-sections": (target) =>
+        `Merge sections of one or two paragraphs into their neighbours: ${v(target.value)}% now (90% of human articles: ${v(target.limit)}% or less).`,
+      "section-uniformity": (target) =>
+        `Let section lengths vary: variation ${v(target.value)}% now (90% of human articles: ${v(target.limit)}% or more, median ${v(target.median)}%). Thin sections stay thin, full ones run long.`,
+      "heading-forms": (target) =>
+        `Vary the heading forms: ${v(target.value)}% are in a stock form now${withDetail(target.detail, formEn)} (90% of human articles: ${v(target.limit)}% or less). Say in the heading what the section says, not "What is X" or "Benefits of X".`,
+      "three-subsections": (target) =>
+        `Split a section as many ways as its content has: ${v(target.value)} headings split into three now (90% of human articles: ${v(target.limit)} or fewer).`,
+      bookends: () => 'Drop the "Introduction" and "Conclusion" headings: open with the point or a concrete scene, and let the last section continue the body.',
+      "closing-restatement": (target) =>
+        `Cut the closing${withDetail(target.detail, quotedEn)} that restates the body: ${v(target.value)}% restated now (90% of human articles: ${v(target.limit)}% or less). Put the point at the end of the section before.`,
+      "three-item-lists": (target) =>
+        `Give each list as many items as its content has: ${v(target.value)}% of lists have three now (90% of human articles: ${v(target.limit)}% or less).`,
+      "bold-labels": (target) => `Turn the ${v(target.value)} bold-label list items back into sentences (90% of human articles: ${v(target.limit)} or fewer).`,
+      "emoji-headings": (target) => `Take the emoji out of ${v(target.value)} headings (90% of human articles: ${v(target.limit)} or fewer).`,
+      "pro-con": (target) => `Fold the paired pros and cons sections${withDetail(target.detail, pairsEn)} into one account that also says what it costs.`,
+    },
+    stop: (limit) =>
+      `Stop rewriting the outline when \`chaff outline <original> <rewrite>\` marks none of these with ✗ and the structure score is under ${v(limit)}.`,
+  },
+};

package/src/fix-plan/structure-targets.ts

@@ -0,0 +1,46 @@
+import type { FeatureId } from "../structure-shape/features.ts";
+import type { Placement, StructureScore } from "../structure-shape/score.ts";
+
+/** How many section headings the document has, and how many a human article of its length has at most and usually. */
+export type HeadingBudget = { readonly now: number; readonly most: number; readonly usual: number };
+
+/** One structure measure to bring back into the human range: where it is, where the human limit is, and the median. */
+export type StructureTarget = {
+  readonly id: FeatureId;
+  readonly value: number;
+  readonly limit: number;
+  readonly median: number;
+  readonly detail?: string;
+  /** For heading density only: the measure turned into a count of headings for this document's length. */
+  readonly headings?: HeadingBudget;
+};
+
+const PER = 1000;
+
+const budgetOf = (placement: Placement, value: number, length: number): HeadingBudget => ({
+  now: placement.feature.count ?? Math.round((value * length) / PER),
+  most: Math.floor(((placement.limit ?? 0) * length) / PER),
+  usual: Math.round(((placement.median ?? 0) * length) / PER),
+});
+
+const targetOf = (placement: Placement, length: number): StructureTarget[] => {
+  const { value, detail, id } = placement.feature;
+  if (!placement.beyond || value === undefined || placement.limit === undefined || placement.median === undefined) return [];
+  return [
+    {
+      id,
+      value,
+      limit: placement.limit,
+      median: placement.median,
+      ...(detail === undefined ? {} : { detail }),
+      ...(id === "heading-density" ? { headings: budgetOf(placement, value, length) } : {}),
+    },
+  ];
+};
+
+/**
+ * The measures past the human limit, each as a target from the baseline's numbers. `length` is the document's length in
+ * its unit (characters or words), which turns the heading density into a count of headings.
+ */
+export const structureTargetsOf = (score: StructureScore, length: number): StructureTarget[] =>
+  score.placements.flatMap((placement) => targetOf(placement, length));

package/src/fix-plan/text.ts

@@ -0,0 +1,136 @@
+import type { Texts } from "../ui.ts";
+import type { FixMode, ModeReason } from "./mode.ts";
+
+export type FixPlanText = {
+  readonly usage: string;
+  readonly notFound: (path: string) => string;
+  readonly title: (path: string) => string;
+  readonly about: (language: string, genre: string) => string;
+  readonly intro: string;
+  readonly constraintsHeading: string;
+  readonly constraints: readonly string[];
+  readonly modeHeading: string;
+  readonly modeName: Readonly<Record<FixMode, string>>;
+  readonly modeReason: Readonly<Record<ModeReason, string>>;
+  readonly modeWays: readonly string[];
+  readonly signalsHeading: string;
+  readonly noSignals: string;
+  readonly outlineLine: (measures: string) => string;
+  readonly rulesHeading: string;
+  readonly direction: string;
+  readonly keep: string;
+  readonly avoid: string;
+  readonly example: string;
+  readonly before: string;
+  readonly after: string;
+  readonly phrases: string;
+  readonly spots: string;
+  readonly at: (line: number) => string;
+  readonly notRunHeading: string;
+  readonly notExperimental: string;
+  readonly checksHeading: string;
+  readonly saveAs: (path: string) => string;
+  readonly checksNote: string;
+};
+
+export const FIX_PLAN_TEXT: Texts<FixPlanText> = {
+  ja: {
+    usage: "使い方: chaff fix-plan <file> [--experimental] [--genre <ジャンル>] [--json]",
+    notFound: (path) => `${path} がありません。`,
+    title: (path) => `直す計画: ${path}`,
+    about: (language, genre) => `言語 ${language}、ジャンル ${genre}`,
+    intro:
+      "chaff が機械で見つけた箇所と、それぞれの直す方向です。chaff は書き直しません。書き直すのは、この計画を読む人か AI です。直したら、最後の確かめのコマンドを実行してください。",
+    constraintsHeading: "守ること",
+    constraints: [
+      "事実、数、日付、条件、名前を変えない。",
+      "元の文書に無い事実、人、数、原因、例を足さない。",
+      "直すのに文書に無い具体（誰が、いつ、どれだけ）が要るときは、作らずに［ ］で空けて書き手に聞く。",
+      "書き直しは 2 回まで。指摘を消すためだけに書き直しを繰り返さない。",
+    ],
+    modeHeading: "勧める直し方",
+    modeName: { none: "直すところなし", light: "軽く直す（Light）", bold: "大胆に直す（Bold）", full: "全面書き直し（Full）" },
+    modeReason: {
+      "nothing-found": "chaff は何も見つけませんでした。",
+      composite: "ai-generated-composite が出ています。文の言い回しを直しても、生成文の骨組みが残ります。",
+      structure: "構成の項目のいくつもが、人の記事の 9 割を超えています（下の「構成の目標」）。文を直しても、見出しの立て方と節の形が残ります。",
+      genre: "ブログやエッセイは、構成から書き直すほうが書き手の望みに合います。",
+      signals: "文書全体を測るルールが 2 つ以上出ています。見出しの構成は残し、節ごとに文を書き直します。",
+      spots: "指摘は箇所ごとです。指摘された所だけを直します。",
+    },
+    modeWays: [
+      "軽く直す: 指摘された箇所だけ。",
+      "大胆に直す: 見出しの構成は残し、節ごとに文を書き直す。",
+      "全面書き直し: 事実と主張を控えてから、構成から書き直す。頼まれた内容が「全面的に」「一から」なら、これを選ぶ。",
+    ],
+    signalsHeading: "文書全体の目印",
+    noSignals: "文書全体を測るルールは、何も言っていません。",
+    outlineLine: (measures) => `構成: ${measures}`,
+    rulesHeading: "ルールごとの直し方",
+    direction: "直す方向",
+    keep: "変えないもの",
+    avoid: "やりがちな間違い",
+    example: "例",
+    before: "前",
+    after: "後",
+    phrases: "言い回しごとの手がかり",
+    spots: "見つけた箇所",
+    at: (line) => `${String(line)} 行目`,
+    notRunHeading: "動かなかったルール",
+    notExperimental: "試験中のルールは動かしていません。AIっぽさのルールの多くは試験中です。--experimental を付けると見ます。",
+    checksHeading: "直したあとの確かめ",
+    saveAs: (path) => `書き直したものを ${path} に保存して、次を実行します。`,
+    checksNote:
+      "1 つめは書き直した文書の指摘、2 つめは事実が落ちていないか・足されていないか、3 つめは構成の変化です。数は chaff の出力から取り、形容詞で言わないでください。",
+  },
+  en: {
+    usage: "usage: chaff fix-plan <file> [--experimental] [--genre <genre>] [--json]",
+    notFound: (path) => `${path} does not exist.`,
+    title: (path) => `Fix plan: ${path}`,
+    about: (language, genre) => `language ${language}, genre ${genre}`,
+    intro:
+      "What chaff found by machine, and how to rewrite each kind of spot. chaff does not rewrite; whoever reads this plan does, a person or an AI. When done, run the checks at the end.",
+    constraintsHeading: "Constraints",
+    constraints: [
+      "Keep every fact, number, date, condition and name.",
+      "Add no fact, person, number, cause or example the original does not have.",
+      "Where a fix needs a specific the text does not give (who, when, how much), do not invent it: leave [ ] and ask the writer.",
+      "At most two passes. Do not rewrite again just to silence a finding.",
+    ],
+    modeHeading: "Recommended way",
+    modeName: { none: "Nothing to fix", light: "Light", bold: "Bold", full: "Full rewrite" },
+    modeReason: {
+      "nothing-found": "chaff found nothing.",
+      composite: "ai-generated-composite fires. Fixing the wording would leave the skeleton of generated text.",
+      structure:
+        'Several structure measures lie past 90% of human articles (see "Structure targets" below). Fixing the sentences would leave the headings and the shape of the sections.',
+      genre: "For a blog post or an essay, what the writer wants changed is usually the structure.",
+      signals: "Two or more rules that measure the whole document fire. Keep the outline and rewrite the prose section by section.",
+      spots: "The findings are single spots. Rewrite only those.",
+    },
+    modeWays: [
+      "Light: only the flagged spots.",
+      "Bold: keep the outline, rewrite the prose of each section.",
+      'Full: take an inventory of the facts and claims, then rewrite from the structure up. If the request says "from scratch", choose this.',
+    ],
+    signalsHeading: "Document-level signals",
+    noSignals: "None of the rules that measure the whole document said anything.",
+    outlineLine: (measures) => `Outline: ${measures}`,
+    rulesHeading: "How to rewrite, rule by rule",
+    direction: "Direction",
+    keep: "Keep",
+    avoid: "Avoid",
+    example: "Example",
+    before: "before",
+    after: "after",
+    phrases: "Hints for the phrases found",
+    spots: "Spots",
+    at: (line) => `line ${String(line)}`,
+    notRunHeading: "Rules that did not run",
+    notExperimental: "The experimental rules did not run, and most rules for generated text are experimental. Add --experimental to run them.",
+    checksHeading: "Check after rewriting",
+    saveAs: (path) => `Save the rewrite as ${path} and run:`,
+    checksNote:
+      "The first lists the rewrite's findings, the second checks that no fact was dropped or added, the third shows how the outline moved. Take the numbers from chaff's output, not adjectives.",
+  },
+};

package/src/grade/baseline-text.ts

@@ -0,0 +1,80 @@
+import type { Texts } from "../ui.ts";
+
+export type BaselineText = {
+  readonly unreadable: (path: string, lines: string) => string;
+  readonly notComparable: (path: string, differ: string) => string;
+  readonly mismatchAllowed: (path: string, differ: string) => string;
+  readonly differ: Readonly<Record<"rules" | "settings" | "mixed", string>>;
+  readonly heading: (path: string, paired: number) => string;
+  readonly unpaired: (onlyBefore: string, onlyAfter: string, readOtherwise: string) => string;
+  readonly ratesHeading: (unit: string) => string;
+  readonly unit: Readonly<Record<"char" | "word", string>>;
+  readonly more: (ids: string) => string;
+  readonly fewer: (ids: string) => string;
+  readonly inRubric: string;
+  readonly newlyFailed: (ids: string) => string;
+  readonly newlyPassed: (ids: string) => string;
+  readonly newFactsHeading: string;
+  readonly droppedAdded: (dropped: string, added: string) => string;
+  readonly newCitations: (addresses: string) => string;
+  readonly penalty: (before: number, after: number) => string;
+  readonly regressionsHeading: (count: number) => string;
+  readonly noRegression: string;
+  readonly none: string;
+  /** Between the parts of one output's line. */
+  readonly separator: string;
+};
+
+export const BASELINE_TEXT: Texts<BaselineText> = {
+  ja: {
+    unreadable: (path, lines) => `${path}: chaff grade --out の 1 回分の結果として読めない行があります（${lines} 行目。形が違うか、id が重なっている）`,
+    notComparable: (path, differ) =>
+      `${path} とは比べません: ${differ}が違います。ルールか設定が変わった差を、prompt や model の差として読まないため（--allow-stamp-mismatch で比べる）`,
+    mismatchAllowed: (path, differ) =>
+      `注意: ${path} とは${differ}が違います。--allow-stamp-mismatch で比べています。差には prompt や model の差でないものが混ざります`,
+    differ: { rules: "ルール", settings: "設定", mixed: "結果ごとの再現の印" },
+    heading: (path, paired) => `${path} と比べた: ${String(paired)} 件の出力が組になった`,
+    unpaired: (onlyBefore, onlyAfter, readOtherwise) => `比べられない: 前の回だけ ${onlyBefore}、今回だけ ${onlyAfter}、言語かジャンルが違う ${readOtherwise}`,
+    ratesHeading: (unit) => `ルールごとの率（1,000 ${unit}あたり、前 → 後）`,
+    unit: { char: "字", word: "語" },
+    more: (ids) => `増えた: ${ids}`,
+    fewer: (ids) => `減った: ${ids}`,
+    inRubric: "（grade: のルール）",
+    newlyFailed: (ids) => `新しく落ちた: ${ids}`,
+    newlyPassed: (ids) => `新しく通った: ${ids}`,
+    newFactsHeading: "新しく落ちた・足された事実と、新しく外れた引用",
+    droppedAdded: (dropped, added) => `落ちた ${dropped}、足された ${added}`,
+    newCitations: (addresses) => `外れた引用 ${addresses}`,
+    penalty: (before, after) => `減点の和: ${String(before)} → ${String(after)}`,
+    regressionsHeading: (count) => `回帰 ${String(count)} 件`,
+    noRegression: "回帰はありません",
+    none: "なし",
+    separator: "、",
+  },
+  en: {
+    unreadable: (path, lines) => `${path}: not the results of one chaff grade --out run (line ${lines}: another shape, or an id already used)`,
+    notComparable: (path, differ) =>
+      `Not compared with ${path}: the ${differ} differ. A change of rules or settings would read as a change of prompt or model (--allow-stamp-mismatch compares anyway)`,
+    mismatchAllowed: (path, differ) =>
+      `Note: the ${differ} differ from ${path}; compared because of --allow-stamp-mismatch. Not every change below comes from the prompt or model`,
+    differ: { rules: "rules", settings: "settings", mixed: "stamps of its results" },
+    heading: (path, paired) => `Compared with ${path}: ${String(paired)} paired ${paired === 1 ? "output" : "outputs"}`,
+    unpaired: (onlyBefore, onlyAfter, readOtherwise) =>
+      `Not compared: only before ${onlyBefore}; only now ${onlyAfter}; another language or genre ${readOtherwise}`,
+    ratesHeading: (unit) => `Rule rates (per 1,000 ${unit}, before → after)`,
+    unit: { char: "characters", word: "words" },
+    more: (ids) => `more in ${ids}`,
+    fewer: (ids) => `fewer in ${ids}`,
+    inRubric: "(a grade: rule)",
+    newlyFailed: (ids) => `Newly failed: ${ids}`,
+    newlyPassed: (ids) => `Newly passed: ${ids}`,
+    newFactsHeading: "New dropped or added facts, and new failed quotations",
+    droppedAdded: (dropped, added) => `dropped ${dropped}; added ${added}`,
+    newCitations: (addresses) => `failed quotations ${addresses}`,
+    penalty: (before, after) => `Penalty points: ${String(before)} → ${String(after)}`,
+    regressionsHeading: (count) => `${String(count)} ${count === 1 ? "regression" : "regressions"}`,
+    noRegression: "No regression",
+    none: "none",
+    separator: "; ",
+  },
+};

package/src/grade/baseline.ts

@@ -0,0 +1,140 @@
+import type { LengthUnit } from "../plugin.ts";
+import { tally } from "./order.ts";
+import { rateRows, sameReading } from "./pairing.ts";
+import { resultKey, resultName } from "./result-name.ts";
+import type { FailedCitation, GradeFact, GradeResult, Stamp } from "./result.ts";
+
+// A run compared with an earlier one (spec §29.5). Pure. Outputs are paired by id; a pair is compared only when both
+// were read the same way (language and genre), and the runs only when their stamps say the same rules and settings.
+
+/** How a rule's rate moved over the paired outputs of one unit, and which outputs gained or lost findings of it. */
+export type RuleMovement = {
+  readonly rule: string;
+  readonly unit: LengthUnit;
+  readonly before: number;
+  readonly after: number;
+  readonly increasedIn: readonly string[];
+  readonly decreasedIn: readonly string[];
+  /** Named under the rubric's rules: only these count as a regression when they increase. */
+  readonly inRubric: boolean;
+};
+
+/** What an output newly dropped or added, or newly failed to quote, against the same reference and sources. */
+export type ItemChange = {
+  readonly id: string;
+  readonly dropped: readonly GradeFact[];
+  readonly added: readonly GradeFact[];
+  readonly citations: readonly FailedCitation[];
+};
+
+export type Comparison = {
+  readonly paired: number;
+  readonly onlyBefore: readonly string[];
+  readonly onlyAfter: readonly string[];
+  /** Paired by id but read in another language or genre: not the same task graded the same way. */
+  readonly readOtherwise: readonly string[];
+  readonly rules: readonly RuleMovement[];
+  readonly newlyFailed: readonly string[];
+  readonly newlyPassed: readonly string[];
+  readonly items: readonly ItemChange[];
+  readonly penalty?: { readonly before: number; readonly after: number } | undefined;
+  /** Each reason this is a regression; empty when it is none. */
+  readonly regressions: readonly string[];
+};
+
+/** Why two runs cannot be compared: their rules or settings differ, or the baseline mixes several stamps. */
+export type StampCheck = { readonly comparable: true } | { readonly comparable: false; readonly differ: readonly ("rules" | "settings" | "mixed")[] };
+
+const STAMP_PARTS: readonly ("rules" | "settings")[] = ["rules", "settings"];
+
+export const stampCheck = (before: readonly GradeResult[], current: Stamp): StampCheck => {
+  // One earlier run has one stamp, its chaff version included; results joined from several runs are not one baseline.
+  const stamps = new Set(before.map((result) => `${result.stamp.chaff}\n${result.stamp.rules}\n${result.stamp.settings}`));
+  const [first] = before;
+  if (stamps.size > 1 || first === undefined) return { comparable: false, differ: ["mixed"] };
+  const differ = STAMP_PARTS.filter((part) => first.stamp[part] !== current[part]);
+  return differ.length === 0 ? { comparable: true } : { comparable: false, differ };
+};
+
+type Pair = { readonly before: GradeResult; readonly after: GradeResult };
+
+const countIn = (result: GradeResult, rule: string): number => result.findings.filter((finding) => finding.rule === rule).length;
+
+const movementsOf = (pairs: readonly Pair[], rubricRules: ReadonlySet<string>): RuleMovement[] =>
+  rateRows([pairs.map((pair) => pair.before), pairs.map((pair) => pair.after)]).flatMap(({ rule, unit, rates: [before = 0, after = 0] }) => {
+    const inUnit = pairs.filter((pair) => pair.after.size.unit === unit);
+    const increasedIn = inUnit.filter((pair) => countIn(pair.after, rule) > countIn(pair.before, rule)).map((pair) => resultName(pair.after));
+    const decreasedIn = inUnit.filter((pair) => countIn(pair.after, rule) < countIn(pair.before, rule)).map((pair) => resultName(pair.after));
+    if (increasedIn.length === 0 && decreasedIn.length === 0 && before === after) return [];
+    return [{ rule, unit, before, after, increasedIn, decreasedIn, inRubric: rubricRules.has(rule) }];
+  });
+
+const factKey = (fact: GradeFact): string => `${fact.kind}\n${fact.key}`;
+
+/** The facts of `after` beyond as many as `before` had of each: a fact dropped in both runs is not new. */
+const newFacts = (before: readonly GradeFact[], after: readonly GradeFact[]): GradeFact[] => {
+  const earlier = tally(before.filter((fact) => !fact.allowed).map(factKey));
+  const seen = new Map<string, number>();
+  return after
+    .filter((fact) => !fact.allowed)
+    .filter((fact) => {
+      const key = factKey(fact);
+      seen.set(key, (seen.get(key) ?? 0) + 1);
+      return (seen.get(key) ?? 0) > (earlier[key] ?? 0);
+    });
+};
+
+const citationKey = (citation: FailedCitation): string => `${citation.source}\n${citation.address}\n${citation.quote}`;
+
+const itemChange = ({ before, after }: Pair): ItemChange => {
+  const failedBefore = new Set((before.citations?.failed ?? []).map(citationKey));
+  return {
+    id: resultName(after),
+    dropped: newFacts(before.facts?.dropped ?? [], after.facts?.dropped ?? []),
+    added: newFacts(before.facts?.added ?? [], after.facts?.added ?? []),
+    citations: (after.citations?.failed ?? []).filter((citation) => !failedBefore.has(citationKey(citation))),
+  };
+};
+
+const penaltyOf = (results: readonly GradeResult[]): number => results.reduce((sum, result) => sum + (result.score?.penalty ?? 0), 0);
+
+const scored = (pairs: readonly Pair[]): boolean => pairs.some((pair) => pair.after.score !== undefined);
+
+const regressionsOf = (newlyFailed: readonly string[], movements: readonly RuleMovement[], penalty: Comparison["penalty"]): string[] => [
+  ...newlyFailed.map((id) => `${id}: passed, now fails`),
+  ...movements
+    .filter((movement) => movement.inRubric && movement.increasedIn.length > 0)
+    .map((movement) => `rules.${movement.rule}: more findings in ${movement.increasedIn.join(", ")}`),
+  ...(penalty !== undefined && penalty.after > penalty.before ? [`score.penalty ${String(penalty.before)} → ${String(penalty.after)}`] : []),
+];
+
+const pairRead = (pair: Pair): boolean => sameReading([pair.before, pair.after]);
+
+/**
+ * `after` compared with `before`, output by output: an output is paired with the earlier one of its id and variant.
+ * `rubricRules`: the rules the rubric names, whose increase is a regression.
+ */
+export const compareRuns = (before: readonly GradeResult[], after: readonly GradeResult[], rubricRules: ReadonlySet<string>): Comparison => {
+  const earlier = new Map(before.map((result) => [resultKey(result), result]));
+  const keys = new Set(after.map(resultKey));
+  const matched = after.flatMap((result) => {
+    const match = earlier.get(resultKey(result));
+    return match === undefined ? [] : [{ before: match, after: result }];
+  });
+  const pairs = matched.filter(pairRead);
+  const rules = movementsOf(pairs, rubricRules);
+  const newlyFailed = pairs.filter((pair) => pair.before.pass && !pair.after.pass).map((pair) => resultName(pair.after));
+  const penalty = scored(pairs) ? { before: penaltyOf(pairs.map((pair) => pair.before)), after: penaltyOf(pairs.map((pair) => pair.after)) } : undefined;
+  return {
+    paired: pairs.length,
+    onlyBefore: before.filter((result) => !keys.has(resultKey(result))).map(resultName),
+    onlyAfter: after.filter((result) => !earlier.has(resultKey(result))).map(resultName),
+    readOtherwise: matched.filter((pair) => !pairRead(pair)).map((pair) => resultName(pair.after)),
+    rules,
+    newlyFailed,
+    newlyPassed: pairs.filter((pair) => !pair.before.pass && pair.after.pass).map((pair) => resultName(pair.after)),
+    items: pairs.map(itemChange).filter((change) => change.dropped.length + change.added.length + change.citations.length > 0),
+    ...(penalty === undefined ? {} : { penalty }),
+    regressions: regressionsOf(newlyFailed, rules, penalty),
+  };
+};