chaffjs 0.16.0 → 0.18.0

package/src/commands/tree.ts

@@ -1,5 +1,4 @@
-import { plainSource } from "../plain-source.ts";
-import { readFile } from "node:fs/promises";
+import { readDocumentFile } from "../files.ts";
 import { loadAdapter } from "../adapter-load.ts";
 import { applyByPath } from "../config/by-path.ts";
 import { profileFor } from "../profile/for-file.ts";
@@ -8,6 +7,7 @@ import { guessLanguage } from "../detect.ts";
 import { isMarkdownPath } from "../structure/markdown-path.ts";
 import { buildStructure } from "../structure/of.ts";
 import { toSexp } from "../structure/sexp.ts";
+import { foldPostOrder } from "../tree-walk.ts";
 import type { Config } from "../config/load.ts";
 import type { StructureNode } from "../plugin.ts";
 import type { Texts, UiLanguage } from "../ui.ts";
@@ -23,16 +23,19 @@ const TEXT: Texts<{
   readonly usage: string;
   readonly unreadable: (path: string, why: string) => string;
   readonly noStructure: (path: string, language: string) => string;
+  readonly tooDeepForJson: (path: string, depth: number) => string;
 }> = {
   ja: {
     usage: "使い方: chaff tree <file>... [--format sexp|json] [--language ja|en|…]",
     unreadable: (path, why) => `${path} を読めませんでした: ${why}`,
     noStructure: (path, language) => `${path}: 言語 ${language} のパッケージは文書の構造を読めません（structure がありません）`,
+    tooDeepForJson: (path, depth) => `${path}: 木が ${String(depth)} 段と深すぎて、字下げした JSON に書き出せません（--format sexp なら書けることがあります）`,
   },
   en: {
     usage: "usage: chaff tree <file>... [--format sexp|json] [--language ja|en|…]",
     unreadable: (path, why) => `Could not read ${path}: ${why}`,
     noStructure: (path, language) => `${path}: the ${language} package cannot read a document's structure (it has no structure)`,
+    tooDeepForJson: (path, depth) => `${path}: the tree is ${String(depth)} levels deep, too deep to write as indented JSON (--format sexp may still write it)`,
   },
 };
 
@@ -48,7 +51,7 @@ export const treeTargets = (argv: readonly string[]): string[] =>
 
 export const readSource = async (path: string, context: TreeContext): Promise<string | undefined> => {
   try {
-    return plainSource(await readFile(path, "utf8"));
+    return await readDocumentFile(path);
   } catch (err) {
     console.error(treeText(context).unreadable(path, err instanceof Error ? err.message : String(err)));
     return undefined;
@@ -83,13 +86,46 @@ export const readTree = async (path: string, argv: readonly string[], context: T
   // 日本語は形態素で数量と日付を読む。解析器が無ければ単位の表で読むので、木は作れる。
   await adapter.prepare?.({ pos: true });
   const profile = profileFor(context.config, path, source, language, resolveGenre(path, source, context.config, context.flag(argv, "--genre")).genre);
-  return { source, tree: buildStructure({ path, source, language, markdown: isMarkdownPath(path), profile }, adapter.structure) };
+  return { source, tree: buildStructure({ path, source, language, markdown: isMarkdownPath(path), profile, lexicons: adapter.lexicons }, adapter.structure) };
+};
+
+/**
+ * 字下げした JSON。書けないほど深い木（言語パッケージの番号が何千段も入れ子になる）なら undefined。
+ * JSON.stringify は 1 段ごとに呼び直すのでスタックが尽き、字下げの合計は深さの二乗で増えて文字列の上限を超える。どちらも RangeError になる。
+ */
+export const treeJson = (tree: StructureNode): string | undefined => {
+  try {
+    return JSON.stringify(tree, null, 2);
+  } catch (err) {
+    if (err instanceof RangeError) return undefined;
+    throw err;
+  }
+};
+
+/** 根を 1 段目と数えた、いちばん深い葉までの段の数。 */
+export const depthOf = (tree: StructureNode): number =>
+  foldPostOrder(
+    tree,
+    (node) => node.children,
+    (_node, depths: readonly number[]) => depths.reduce((deepest, depth) => Math.max(deepest, depth), 0) + 1,
+  );
+
+/** 出力する文字列。JSON に書けないほど深い木なら、その深さ。 */
+export const renderTree = (tree: StructureNode, format: string | undefined): { readonly text: string } | { readonly tooDeep: number } => {
+  if (format !== "json") return { text: toSexp(tree) };
+  const json = treeJson(tree);
+  return json === undefined ? { tooDeep: depthOf(tree) } : { text: json };
 };
 
 const printTree = async (path: string, argv: readonly string[], context: TreeContext): Promise<boolean> => {
   const read = await readTree(path, argv, context);
   if (read === undefined) return false;
-  console.log(context.flag(argv, "--format") === "json" ? JSON.stringify(read.tree, null, 2) : toSexp(read.tree));
+  const rendered = renderTree(read.tree, context.flag(argv, "--format"));
+  if ("tooDeep" in rendered) {
+    console.error(treeText(context).tooDeepForJson(path, rendered.tooDeep));
+    return false;
+  }
+  console.log(rendered.text);
   return true;
 };
 

package/src/compare/atom.ts

@@ -0,0 +1,33 @@
+/** The kinds of fact `chaff compare` reads, in the order it reports them. */
+export const ATOM_KINDS = ["number", "date", "time", "url", "code", "name", "quote", "heading", "reference", "footnote"] as const;
+
+export type AtomKind = (typeof ATOM_KINDS)[number];
+
+export const isAtomKind = (value: string): value is AtomKind => ATOM_KINDS.some((kind) => kind === value);
+
+/**
+ * One fact as a document states it. Two atoms are the same fact when kind and key are equal; `text` is how it was
+ * written (1,000円 and 1000円 share a key). Position is only for showing where it is: comparing ignores it.
+ */
+export type Atom = {
+  readonly kind: AtomKind;
+  readonly key: string;
+  readonly text: string;
+  readonly line: number;
+  /** A number written with no unit the reader knows. It may stand for the same number written with a unit. */
+  readonly unitless?: boolean;
+  /** For a number: its value alone, without the unit. */
+  readonly value?: string;
+};
+
+/** A kind this document could not be fully read for, and why. Zero facts of it then does not mean none were there. */
+export type Unread = { readonly kind: AtomKind; readonly reason: UnreadReason };
+
+export type UnreadReason = "no-structure" | "no-dates" | "no-pos" | "plain-text";
+
+export type Extraction = {
+  readonly atoms: readonly Atom[];
+  readonly unread: readonly Unread[];
+  /** The prose spelled as name keys are, so the other document can ask whether a name is written here at all. */
+  readonly nameText: string;
+};

package/src/compare/clock-time.ts

@@ -0,0 +1,86 @@
+import type { Span } from "../plugin.ts";
+import { overlapsAny, spanIndex } from "./spans.ts";
+
+/** A time of day and where it is written. key is "HH:MM" on a 24-hour clock, with ":SS" when seconds are given. */
+export type ClockTime = Span & { readonly key: string };
+
+const DIGIT = "[0-9０-９]";
+const COLON = "[:：]";
+
+/** 10:30, 15:30:05, 3:30 p.m. */
+const COLON_TIME = new RegExp(
+  `(?<![0-9０-９])(${DIGIT}{1,2})${COLON}(${DIGIT}{2})(?:${COLON}(${DIGIT}{2}))?(?:\\s?([ap])\\.?m\\.?(?![a-z]))?(?![0-9０-９])`,
+  "giu",
+);
+
+/** 3 p.m., 11am. */
+const MERIDIEM_TIME = new RegExp(`(?<![0-9０-９:：])(${DIGIT}{1,2})\\s?([ap])\\.?m\\.?(?![a-z])`, "giu");
+
+/** 午後3時30分, 10時半, 9時. */
+const JAPANESE_TIME = new RegExp(`(午前|午後)?(${DIGIT}{1,2})時(?:(${DIGIT}{1,2})分|(半))?`, "gu");
+
+const LAST_HOUR = 24;
+const MINUTES_PER_HOUR = 60;
+const HALF_HOUR = 30;
+const NOON = 12;
+const PADDED = 2;
+
+type Reading = { readonly hour: number; readonly minute: number; readonly second?: number | undefined; readonly afternoon?: boolean | undefined };
+
+const hourOf = (hour: number, afternoon: boolean | undefined): number => {
+  if (afternoon === undefined) return hour;
+  if (afternoon) return hour < NOON ? hour + NOON : hour;
+  return hour === NOON ? 0 : hour;
+};
+
+const padded = (value: number): string => String(value).padStart(PADDED, "0");
+
+/** Whether the hour is one on its clock: up to 12 with a.m. or p.m., else up to 23, and 24 only as 24:00. */
+const isHour = ({ hour, minute, second, afternoon }: Reading): boolean => {
+  if (afternoon !== undefined) return hour <= NOON;
+  return hour < LAST_HOUR || (hour === LAST_HOUR && minute === 0 && (second ?? 0) === 0);
+};
+
+/** The key, or undefined when the numbers are not a time of day (24:30, 13 p.m., 3:75). */
+const keyOf = (reading: Reading): string | undefined => {
+  const { hour, minute, second } = reading;
+  if (!isHour(reading) || minute >= MINUTES_PER_HOUR || (second ?? 0) >= MINUTES_PER_HOUR) return undefined;
+  const parts = [hourOf(hour, reading.afternoon), minute, ...(second === undefined ? [] : [second])];
+  return parts.map(padded).join(":");
+};
+
+const numberOf = (digits: string | undefined): number | undefined => (digits === undefined ? undefined : Number(digits.normalize("NFKC")));
+
+const meridiem = (letter: string | undefined): boolean | undefined => (letter === undefined ? undefined : letter.toLowerCase() === "p");
+
+const japaneseMeridiem = (word: string | undefined): boolean | undefined => (word === undefined ? undefined : word === "午後");
+
+type Reader = { readonly pattern: RegExp; readonly read: (match: RegExpMatchArray) => Reading };
+
+const READERS: readonly Reader[] = [
+  {
+    pattern: JAPANESE_TIME,
+    read: (match) => ({
+      hour: numberOf(match[2]) ?? 0,
+      minute: match[4] === undefined ? (numberOf(match[3]) ?? 0) : HALF_HOUR,
+      afternoon: japaneseMeridiem(match[1]),
+    }),
+  },
+  {
+    pattern: COLON_TIME,
+    read: (match) => ({ hour: numberOf(match[1]) ?? 0, minute: numberOf(match[2]) ?? 0, second: numberOf(match[3]), afternoon: meridiem(match[4]) }),
+  },
+  { pattern: MERIDIEM_TIME, read: (match) => ({ hour: numberOf(match[1]) ?? 0, minute: 0, afternoon: meridiem(match[2]) }) },
+];
+
+/** Times of day in text, in the order written. Where two readings overlap, the earlier reader in READERS wins. */
+export const clockTimes = (text: string): ClockTime[] =>
+  READERS.reduce<ClockTime[]>((found, reader) => {
+    const taken = spanIndex(found);
+    const fresh = [...text.matchAll(reader.pattern)].flatMap((match) => {
+      const key = keyOf(reader.read(match));
+      const span = { start: match.index, end: match.index + match[0].length };
+      return key === undefined || overlapsAny(taken, span) ? [] : [{ ...span, key }];
+    });
+    return [...found, ...fresh];
+  }, []).toSorted((left, right) => left.start - right.start);

package/src/compare/extract.ts

@@ -0,0 +1,78 @@
+import { lineStarts, placeOf } from "../position.ts";
+import type { MarkdownNode } from "../markdown-node.ts";
+import type { ProseDocument, Span, StructurePatterns } from "../plugin.ts";
+import type { Atom, Extraction, Unread } from "./atom.ts";
+import { clockTimes } from "./clock-time.ts";
+import { factTextOf, type FactText } from "./fact-text.ts";
+import { codeAtoms, headingAtoms, urlAtoms } from "./markup-atoms.ts";
+import { nameKey, properNouns, readsProperNouns } from "./proper-nouns.ts";
+import { spanIndex } from "./spans.ts";
+import { atomsOf, bareNumbers, footnotes, numericDates, quotations, teamNames, type TextInput } from "./text-atoms.ts";
+import { treeFacts, type TreeFacts } from "./tree-atoms.ts";
+
+export type ExtractInput = {
+  readonly doc: ProseDocument;
+  /** The Markdown tree, or undefined for a plain-text document. */
+  readonly root: MarkdownNode | undefined;
+  /** The language package's structure reader: numbers with units, dates and references come from it. */
+  readonly structure: StructurePatterns | undefined;
+  /** chaff.yaml's `names:`. */
+  readonly names: readonly string[];
+};
+
+const NO_TREE_FACTS: TreeFacts = { atoms: [], taken: [] };
+
+const STRUCTURE_KINDS = ["number", "date", "reference"] as const;
+
+/** The kinds this document could not be fully read for. Each says why, so "nothing dropped" is never "nothing read". */
+const unreadOf = (input: ExtractInput): Unread[] => [
+  ...(input.structure === undefined ? STRUCTURE_KINDS.map((kind): Unread => ({ kind, reason: "no-structure" })) : []),
+  ...(input.structure !== undefined && input.structure.dates === undefined ? [{ kind: "date", reason: "no-dates" } as const] : []),
+  ...(readsProperNouns(input.doc) ? [] : [{ kind: "name", reason: "no-pos" } as const]),
+  ...(input.root === undefined ? [{ kind: "code", reason: "plain-text" } as const] : []),
+];
+
+type Readers = { readonly facts: FactText; readonly text: TextInput; readonly lineOf: (offset: number) => number };
+
+/** Facts read from the prose itself. Each reader skips what an earlier one took, so one figure is one fact. */
+const proseFacts = (input: ExtractInput, readers: Readers, tree: TreeFacts, times: readonly Span[]): Atom[] => {
+  const { facts, text, lineOf } = readers;
+  const marks = footnotes(facts.codeless);
+  const dates = numericDates(facts.text, spanIndex([...tree.taken, ...times]));
+  const names = teamNames(facts.text, input.names);
+  const taken = [...tree.taken, ...times, ...marks, ...dates];
+  return [
+    ...atomsOf(dates, "date", text),
+    ...atomsOf(marks, "footnote", text),
+    ...atomsOf(names, "name", text),
+    ...bareNumbers(text, spanIndex([...taken, ...facts.listMarkers])),
+    ...quotations(text),
+    // "April" in a date and "Section" in a reference are read with them, not again as names.
+    ...properNouns({ doc: input.doc, taken: spanIndex([...taken, ...names, ...facts.blanked]), lineOf }),
+  ];
+};
+
+const structureFacts = (input: ExtractInput, facts: FactText, times: readonly Span[], lineOf: (offset: number) => number): TreeFacts => {
+  const tree = input.doc.structure;
+  return tree === undefined ? NO_TREE_FACTS : treeFacts({ tree, source: input.doc.source, blanked: facts.blanked, times: spanIndex(times), lineOf });
+};
+
+/** Every fact atom of one document, and the kinds it could not be fully read for. */
+export const extractFacts = (input: ExtractInput): Extraction => {
+  const source = input.doc.source;
+  const starts = lineStarts(source);
+  const lineOf = (offset: number): number => placeOf(starts, offset).line;
+  const facts = factTextOf(source, input.root);
+  const text = { text: facts.text, source, lineOf };
+  const times = clockTimes(facts.text);
+  const tree = structureFacts(input, facts, times, lineOf);
+  const atoms = [
+    ...tree.atoms,
+    ...atomsOf(times, "time", text),
+    ...proseFacts(input, { facts, text, lineOf }, tree, times),
+    ...urlAtoms(input.doc.markup, facts.bareUrls, lineOf),
+    ...(input.root === undefined ? [] : codeAtoms(input.root, lineOf)),
+    ...headingAtoms(input.doc.markup, lineOf),
+  ];
+  return { atoms, unread: unreadOf(input), nameText: nameKey(facts.text) };
+};

package/src/compare/fact-text.ts

@@ -0,0 +1,68 @@
+import { bareUrls, type BareUrl } from "../bare-url.ts";
+import { linkChrome } from "../document.ts";
+import { opaqueSpans } from "../markdown-read.ts";
+import { spanOf, type MarkdownNode } from "../markdown-node.ts";
+import { maskSpans } from "../mask.ts";
+import { eachPreOrder } from "../tree-walk.ts";
+import type { Span } from "../plugin.ts";
+import { overlapsAny, spanIndex, type SpanIndex } from "./spans.ts";
+
+/**
+ * The document as the facts in prose are read from it: the same length as the source, with what is not prose
+ * blanked. Code, HTML and front matter are compared as code or not at all; a link's target and a bare URL are
+ * compared as URLs. Blanking them keeps the digits of a URL from being read again as numbers.
+ */
+export type FactText = {
+  /** The source with code and URLs blanked. Offsets are the source's. */
+  readonly text: string;
+  /** The source with only code blanked: footnote marks are link syntax, so they are read before links are blanked. */
+  readonly codeless: string;
+  /** Where the blanked parts are. A fact the structure reader found there is not a fact of the prose. */
+  readonly blanked: SpanIndex;
+  readonly bareUrls: readonly BareUrl[];
+  /** Ordered-list markers (`1.`): they number the list, they do not state a number. */
+  readonly listMarkers: readonly Span[];
+};
+
+/** Nodes whose whole span is not read as prose: a link reference definition's URL is read as a URL, an image is not text. */
+const WHOLE = new Set(["definition", "image", "imageReference"]);
+const LINKS = new Set(["link", "linkReference"]);
+
+/** `1.` or `1)` at the start of an ordered list item. */
+const ORDERED_MARKER = /^\s*\d{1,9}[.)]/u;
+
+const listMarker = (node: MarkdownNode, source: string): Span[] => {
+  const span = spanOf(node);
+  if (node.type !== "listItem" || span === undefined) return [];
+  const marker = ORDERED_MARKER.exec(source.slice(span.start, span.end));
+  return marker === null ? [] : [{ start: span.start, end: span.start + marker[0].length }];
+};
+
+type MarkdownParts = { readonly chrome: Span[]; readonly linkSpans: Span[]; readonly listMarkers: Span[] };
+
+const markdownParts = (root: MarkdownNode, source: string): MarkdownParts => {
+  const parts: MarkdownParts = { chrome: [], linkSpans: [], listMarkers: [] };
+  eachPreOrder(root, (node) => {
+    const span = spanOf(node);
+    if (span === undefined) return;
+    if (WHOLE.has(node.type)) parts.chrome.push(span);
+    if (LINKS.has(node.type)) {
+      parts.chrome.push(...linkChrome(node));
+      parts.linkSpans.push(span);
+    }
+    parts.listMarkers.push(...listMarker(node, source));
+  });
+  return parts;
+};
+
+/** root is the Markdown tree, or undefined for a plain-text document, where nothing is code and URLs are bare. */
+export const factTextOf = (source: string, root: MarkdownNode | undefined): FactText => {
+  const code = root === undefined ? [] : opaqueSpans(root);
+  const codeless = maskSpans(source, code);
+  const parts = root === undefined ? { chrome: [], linkSpans: [], listMarkers: [] } : markdownParts(root, source);
+  const insideLinks = spanIndex(parts.linkSpans);
+  // An autolink's text is its URL: it is read once, as the link.
+  const bare = bareUrls(codeless).filter((url) => !overlapsAny(insideLinks, url));
+  const blanked = [...code, ...parts.chrome, ...bare];
+  return { text: maskSpans(source, blanked), codeless, blanked: spanIndex(blanked), bareUrls: bare, listMarkers: parts.listMarkers };
+};

package/src/compare/markup-atoms.ts

@@ -0,0 +1,29 @@
+import { spanOf, type MarkdownNode } from "../markdown-node.ts";
+import { eachPreOrder } from "../tree-walk.ts";
+import type { Markup } from "../plugin.ts";
+import type { BareUrl } from "../bare-url.ts";
+import type { Atom } from "./atom.ts";
+
+const CODE = new Set(["code", "inlineCode"]);
+
+/** Inline code and code blocks, compared as written: the key is the code itself, without its backticks or fences. */
+export const codeAtoms = (root: MarkdownNode, lineOf: (offset: number) => number): Atom[] => {
+  const found: Atom[] = [];
+  eachPreOrder(root, (node) => {
+    const span = spanOf(node);
+    if (!CODE.has(node.type) || span === undefined) return;
+    const code = node.value ?? "";
+    found.push({ kind: "code", key: code, text: code, line: lineOf(span.start) });
+  });
+  return found;
+};
+
+/** Where links point (a Markdown link, an autolink, a used reference definition) and URLs written bare. */
+export const urlAtoms = (markup: Markup | undefined, bare: readonly BareUrl[], lineOf: (offset: number) => number): Atom[] => [
+  ...(markup?.links ?? []).map((link) => ({ kind: "url" as const, key: link.destination, text: link.destination, line: lineOf(link.start) })),
+  ...bare.map((url) => ({ kind: "url" as const, key: url.url, text: url.url, line: lineOf(url.start) })),
+];
+
+/** Markdown headings as structure: the key is the level, so a reworded heading is the same heading in another form. */
+export const headingAtoms = (markup: Markup | undefined, lineOf: (offset: number) => number): Atom[] =>
+  (markup?.headings ?? []).map((heading) => ({ kind: "heading", key: "#".repeat(heading.depth), text: heading.text, line: lineOf(heading.start) }));

package/src/compare/match.ts

@@ -0,0 +1,105 @@
+import type { Atom } from "./atom.ts";
+
+/** The same fact written another way: 1,000円 and 1000円, 2026年4月1日 and 2026/4/1, a reworded heading. */
+export type Reformed = { readonly before: Atom; readonly after: Atom };
+
+export type Comparison = {
+  /** In the first document and not in the second. */
+  readonly dropped: readonly Atom[];
+  /** In the second document and not in the first. */
+  readonly added: readonly Atom[];
+  readonly reformed: readonly Reformed[];
+};
+
+const identity = (atom: Atom): string => `${atom.kind}\u0000${atom.key}`;
+
+const groupBy = <T>(items: readonly T[], keyOf: (item: T) => string): Map<string, T[]> => {
+  const groups = new Map<string, T[]>();
+  items.forEach((item) => {
+    const key = keyOf(item);
+    const group = groups.get(key);
+    if (group === undefined) groups.set(key, [item]);
+    else group.push(item);
+  });
+  return groups;
+};
+
+/** Takes and removes the first item the queue for key holds. */
+type Queues<T> = { readonly groups: Map<string, T[]>; readonly taken: Map<string, number> };
+
+const queuesOf = <T>(items: readonly T[], keyOf: (item: T) => string): Queues<T> => ({ groups: groupBy(items, keyOf), taken: new Map() });
+
+/** The next item not yet taken under key. A cursor, not shift(): shift moves every item left, which is quadratic over a long queue. */
+const takeFrom = <T>(queues: Queues<T>, key: string): T | undefined => {
+  const next = queues.taken.get(key) ?? 0;
+  const item = queues.groups.get(key)?.[next];
+  if (item !== undefined) queues.taken.set(key, next + 1);
+  return item;
+};
+
+/** Atoms of one fact: written the same way they need no report; written differently they pair as reformed; the rest are surplus. */
+const matchGroup = (before: readonly Atom[], after: readonly Atom[]): Comparison => {
+  const afterByText = queuesOf(after, (atom) => atom.text);
+  const sameText = new Set<Atom>();
+  const unmatchedBefore = before.filter((atom) => {
+    const twin = takeFrom(afterByText, atom.text);
+    if (twin !== undefined) sameText.add(twin);
+    return twin === undefined;
+  });
+  const unmatchedAfter = after.filter((atom) => !sameText.has(atom));
+  const pairs = Math.min(unmatchedBefore.length, unmatchedAfter.length);
+  return {
+    reformed: unmatchedBefore.slice(0, pairs).map((atom, index) => ({ before: atom, after: unmatchedAfter[index] ?? atom })),
+    dropped: unmatchedBefore.slice(pairs),
+    added: unmatchedAfter.slice(pairs),
+  };
+};
+
+const unitQueue = (value: string | undefined, unitless: boolean): string => `${value ?? ""}\u0000${unitless ? "bare" : "unit"}`;
+
+/**
+ * A number one side wrote with a unit the language package read (30 ＧＢ) and the other wrote so the unit was not read
+ * (30GB) is one number in another form. Two different units (5 件, 5 人) stay a dropped and an added fact.
+ */
+const pairUnits = (comparison: Comparison): Comparison => {
+  const addedNumbers = queuesOf(
+    comparison.added.filter((atom) => atom.kind === "number"),
+    (atom) => unitQueue(atom.value, atom.unitless === true),
+  );
+  const paired = new Set<Atom>();
+  const reformed = comparison.dropped.flatMap((atom): Reformed[] => {
+    // Two unitless numbers of one value share a key and were already paired; only the other side's kind can match.
+    const found = atom.kind === "number" ? takeFrom(addedNumbers, unitQueue(atom.value, atom.unitless !== true)) : undefined;
+    if (found === undefined) return [];
+    paired.add(atom).add(found);
+    return [{ before: atom, after: found }];
+  });
+  return {
+    dropped: comparison.dropped.filter((atom) => !paired.has(atom)),
+    added: comparison.added.filter((atom) => !paired.has(atom)),
+    reformed: [...comparison.reformed, ...reformed],
+  };
+};
+
+const byLine = (left: Atom, right: Atom): number => left.line - right.line;
+
+/**
+ * The facts of two documents compared as multisets: where a fact is written does not matter, how many times it is
+ * stated does. Pure: the same atoms always give the same comparison.
+ */
+export const compareAtoms = (before: readonly Atom[], after: readonly Atom[]): Comparison => {
+  const beforeGroups = groupBy(before, identity);
+  const afterGroups = groupBy(after, identity);
+  const keys = [...new Set([...beforeGroups.keys(), ...afterGroups.keys()])];
+  const groups = keys.map((key) => matchGroup(beforeGroups.get(key) ?? [], afterGroups.get(key) ?? []));
+  const paired = pairUnits({
+    dropped: groups.flatMap((group) => group.dropped),
+    added: groups.flatMap((group) => group.added),
+    reformed: groups.flatMap((group) => group.reformed),
+  });
+  return {
+    dropped: paired.dropped.toSorted(byLine),
+    added: paired.added.toSorted(byLine),
+    reformed: paired.reformed.toSorted((left, right) => byLine(left.before, right.before)),
+  };
+};

package/src/compare/outcome.ts

@@ -0,0 +1,70 @@
+import { type Atom, type AtomKind, type Extraction, type Unread } from "./atom.ts";
+import { compareAtoms, type Reformed } from "./match.ts";
+import { unwrittenNames } from "./written-names.ts";
+
+/** One document as compared: where it is, how many facts of each kind it states, and what could not be read. */
+export type Side = {
+  readonly path: string;
+  readonly counts: Readonly<Record<AtomKind, number>>;
+  readonly unread: readonly Unread[];
+};
+
+/** A dropped or added fact, and whether the command line allowed its kind to change. */
+export type Change = Atom & { readonly allowed: boolean };
+
+export type Outcome = {
+  readonly before: Side;
+  readonly after: Side;
+  readonly dropped: readonly Change[];
+  readonly added: readonly Change[];
+  readonly reformed: readonly Reformed[];
+  /** No fact dropped or added, other than of a kind allowed to be. */
+  readonly ok: boolean;
+};
+
+/** Kinds whose facts may be dropped or added on purpose (--allow-dropped, --allow-added). */
+export type Allowed = { readonly dropped: ReadonlySet<AtomKind>; readonly added: ReadonlySet<AtomKind> };
+
+const NOTHING_ALLOWED: Allowed = { dropped: new Set(), added: new Set() };
+
+const NO_FACTS: Readonly<Record<AtomKind, number>> = {
+  number: 0,
+  date: 0,
+  time: 0,
+  url: 0,
+  code: 0,
+  name: 0,
+  quote: 0,
+  heading: 0,
+  reference: 0,
+  footnote: 0,
+};
+
+const countsOf = (atoms: readonly Atom[]): Record<AtomKind, number> => {
+  const counts = { ...NO_FACTS };
+  atoms.forEach((atom) => {
+    counts[atom.kind] += 1;
+  });
+  return counts;
+};
+
+const sideOf = (path: string, extraction: Extraction): Side => ({ path, counts: countsOf(extraction.atoms), unread: extraction.unread });
+
+const marked = (atoms: readonly Atom[], allowed: ReadonlySet<AtomKind>): Change[] => atoms.map((atom) => ({ ...atom, allowed: allowed.has(atom.kind) }));
+
+export type Compared = { readonly path: string; readonly extraction: Extraction };
+
+/** What `chaff compare` reports: the comparison of two documents' facts, with the allowed kinds set aside. */
+export const outcomeOf = (before: Compared, after: Compared, allowed: Allowed = NOTHING_ALLOWED): Outcome => {
+  const comparison = compareAtoms(before.extraction.atoms, after.extraction.atoms);
+  const dropped = marked(unwrittenNames(comparison.dropped, before.extraction.atoms, after.extraction.nameText), allowed.dropped);
+  const added = marked(unwrittenNames(comparison.added, after.extraction.atoms, before.extraction.nameText), allowed.added);
+  return {
+    before: sideOf(before.path, before.extraction),
+    after: sideOf(after.path, after.extraction),
+    dropped,
+    added,
+    reformed: comparison.reformed,
+    ok: [...dropped, ...added].every((change) => change.allowed),
+  };
+};

package/src/compare/proper-nouns.ts

@@ -0,0 +1,65 @@
+import type { ProseDocument, Span, Token } from "../plugin.ts";
+import type { Atom } from "./atom.ts";
+import { overlapsAny, type SpanIndex } from "./spans.ts";
+
+const PROPER_NOUN = "PROPN";
+
+/** Two proper nouns with at most one space between them are one name: New York, 日本 銀行. */
+const MAX_GAP = 1;
+
+type Run = Span & { readonly words: number };
+
+const joins = (run: Run | undefined, token: Token, text: string): run is Run =>
+  run !== undefined && token.span.start - run.end <= MAX_GAP && text.slice(run.end, token.span.start).trim() === "";
+
+/** Adjacent proper nouns, each run one name. */
+const runsOf = (tokens: readonly Token[], text: string): Run[] =>
+  tokens.reduce<Run[]>((runs, token) => {
+    if (token.pos !== PROPER_NOUN) return runs;
+    const last = runs.at(-1);
+    if (joins(last, token, text)) runs[runs.length - 1] = { start: last.start, end: token.span.end, words: last.words + 1 };
+    else runs.push({ start: token.span.start, end: token.span.end, words: 1 });
+    return runs;
+  }, []);
+
+/** Words written in lower case. A capitalised word the same document also writes this way is a common word that began a sentence. */
+const LOWER_CASE_WORD = /(?<![\p{L}\p{M}])\p{Ll}[\p{L}\p{M}]*/gu;
+
+const lowerCaseWords = (source: string): ReadonlySet<string> => new Set(source.match(LOWER_CASE_WORD) ?? []);
+
+const withLowerFirst = (word: string): string => `${word.charAt(0).toLowerCase()}${word.slice(1)}`;
+
+/** A one-word name the document also writes in lower case ("Scammers" opening a sentence, "scammers" elsewhere). */
+const isCommonWord = (run: Run, written: string, lowerCase: ReadonlySet<string>): boolean => {
+  const lowered = withLowerFirst(written);
+  return run.words === 1 && lowered !== written && lowerCase.has(lowered);
+};
+
+export type NounInput = {
+  readonly doc: ProseDocument;
+  /** Facts other readers took and what is not prose: a proper noun inside one is read as that, not again here. */
+  readonly taken: SpanIndex;
+  readonly lineOf: (offset: number) => number;
+};
+
+/** A name as one spelling: full-width letters and line breaks inside a name do not make another name. */
+export const nameKey = (written: string): string => written.normalize("NFKC").replace(/\s+/gu, " ");
+
+/** Whether the language package tagged parts of speech: without them, a document with sentences has no proper nouns to read. */
+export const readsProperNouns = (doc: ProseDocument): boolean => doc.sentences.length === 0 || doc.sentences.some((sentence) => sentence.tokens !== undefined);
+
+/**
+ * Proper nouns as the part-of-speech tagger reads them in sentences. Headings are left out: they are compared as
+ * headings, may be reworded, and a title-case heading makes every word look like a name.
+ */
+export const properNouns = (input: NounInput): Atom[] => {
+  const { source } = input.doc;
+  const lowerCase = lowerCaseWords(source);
+  return input.doc.sentences.flatMap((sentence) =>
+    runsOf(sentence.tokens ?? [], source).flatMap((run): Atom[] => {
+      const text = source.slice(run.start, run.end);
+      if (overlapsAny(input.taken, run) || isCommonWord(run, text, lowerCase)) return [];
+      return [{ kind: "name", key: nameKey(text), text, line: input.lineOf(run.start) }];
+    }),
+  );
+};