chaffjs 0.17.0 → 0.19.0

package/src/commands/feedback.ts

@@ -1,4 +1,4 @@
-import { plainSource } from "../plain-source.ts";
+import { readDocumentFile } from "../files.ts";
 import { existsSync, readFileSync, writeFileSync } from "node:fs";
 import { basename, join } from "node:path";
 import type { Finding, RuleDefinition } from "../plugin.ts";
@@ -7,6 +7,8 @@ import { messageOf } from "../render/text.ts";
 import type { Texts, UiLanguage } from "../ui.ts";
 import { excerptsAround } from "../feedback/excerpt.ts";
 import { feedbackDraft, type FeedbackDraft, type FeedbackKind, type ReportedFinding } from "../feedback/draft.ts";
+import { notRunAmong } from "../not-run.ts";
+import type { Skipped } from "../run.ts";
 
 export const FEEDBACK_FILE = ".chaff-feedback.md";
 const NEW_ISSUE = "https://github.com/isamu/lab/issues/new";
@@ -18,6 +20,10 @@ export type Checked = {
   readonly rules: readonly RuleDefinition[];
   readonly language: string;
   readonly genre: string;
+  /** The rules this check did not run, and why. */
+  readonly skipped: readonly Skipped[];
+  /** How the check was run, as the reader of a report would repeat it (runConditions). */
+  readonly conditions: readonly string[];
 };
 
 export type FeedbackContext = {
@@ -36,6 +42,8 @@ const TEXT: Texts<{
   readonly notFound: (path: string) => string;
   readonly badLine: (lines: number) => string;
   readonly noMatch: (list: string) => string;
+  readonly notRun: (rule: string, why: string) => string;
+  readonly runExperimental: string;
   readonly several: (list: string) => string;
   readonly none: string;
   readonly written: (file: string, lines: string) => string;
@@ -51,6 +59,8 @@ const TEXT: Texts<{
     notFound: (path) => `${path} がありません。`,
     badLine: (lines) => `--line には 1〜${String(lines)} の行番号を書いてください。`,
     noMatch: (list) => `その指摘が見つかりません。この文書の指摘:\n${list}`,
+    notRun: (rule, why) => `${rule} は今回動いていません（${why}）。`,
+    runExperimental: "試験中の rule です。--experimental を付けてかけ直してください。",
     several: (list) => `その rule の指摘がいくつもあります。--line で一つ選んでください:\n${list}`,
     none: "  （指摘はありません）",
     written: (file, lines) => `報告の下書きを ${file} に書きました。文書から載せたのは ${lines} だけです。送る前に読んで、要らない所は消してください。`,
@@ -66,6 +76,8 @@ const TEXT: Texts<{
     notFound: (path) => `${path} does not exist.`,
     badLine: (lines) => `--line must be a line number from 1 to ${String(lines)}.`,
     noMatch: (list) => `No such finding. The findings in this document:\n${list}`,
+    notRun: (rule, why) => `${rule} did not run in this check (${why}).`,
+    runExperimental: "It is experimental: run again with --experimental.",
     several: (list) => `That rule has several findings here. Pick one with --line:\n${list}`,
     none: "  (no findings)",
     written: (file, lines) =>
@@ -86,15 +98,21 @@ const chosen = (findings: readonly Finding[], rule: string | undefined, line: nu
 const describe = (findings: readonly Finding[], none: string): string =>
   findings.length === 0 ? none : findings.map((finding) => `  ${String(finding.line)}  ${finding.rule}`).join("\n");
 
-/** Pure: the reported rules' own lines of chaff.yaml's rules (a numeric limit wins over a level), or undefined when none is set. */
-export const settingsOf = (config: Pick<Config, "rules" | "limits">, ruleIds: readonly string[]): string | undefined => {
+/**
+ * Pure: the reported rules' own lines of chaff.yaml's rules (a numeric limit wins over a level), under the style chaff.yaml
+ * names, or undefined when none is set. A level the style decided is the style's, so only the style line reproduces it.
+ */
+export const settingsOf = (config: Pick<Config, "rules" | "limits" | "applied">, ruleIds: readonly string[]): string | undefined => {
+  const fromStyle = new Set(config.applied?.levelsFrom ?? []);
   const lines = [...new Set(ruleIds)].flatMap((id) => {
     const limit = config.limits[id];
     const level = config.rules[id];
     if (limit !== undefined) return [`  ${id}: ${String(limit)}`];
-    return level === undefined ? [] : [`  ${id}: ${level}`];
+    return level === undefined || fromStyle.has(id) ? [] : [`  ${id}: ${level}`];
   });
-  return lines.length === 0 ? undefined : ["rules:", ...lines].join("\n");
+  const style = config.applied === undefined ? [] : [`style: ${config.applied.style}`];
+  const rules = lines.length === 0 ? [] : ["rules:", ...lines];
+  return style.length + rules.length === 0 ? undefined : [...style, ...rules].join("\n");
 };
 
 /** Pure: a title on one line, no longer than GitHub shows. */
@@ -151,13 +169,32 @@ const sendInstructions = (title: string, lines: readonly number[], ui: UiLanguag
   ].join("\n");
 };
 
+/**
+ * Pure: how a check was run, in the words a reader would type to run it again: the flags that change what is found.
+ * Experimental rules switched on in chaff.yaml count too, said as chaff.yaml says it.
+ */
+export const runConditions = (argv: readonly string[], genreFlag: string | undefined, experimentalInConfig: boolean): string[] => {
+  const experimental = argv.includes("--experimental") ? ["--experimental"] : [];
+  const fromConfig = experimental.length === 0 && experimentalInConfig ? ["experimental: true (chaff.yaml)"] : [];
+  return [...experimental, ...fromConfig, ...(genreFlag === undefined ? [] : [`--genre ${genreFlag}`])];
+};
+
+/** No finding matched: say so with the findings there are, and first why the asked-for rule did not run, if it did not. */
+const noMatch = (request: Request, checked: Checked, ui: UiLanguage): string => {
+  const text = TEXT[ui];
+  const list = text.noMatch(describe(checked.findings, text.none));
+  const [notRun] = request.rule === undefined ? [] : notRunAmong([request.rule], checked.skipped);
+  if (notRun === undefined) return list;
+  return [text.notRun(notRun.rule, notRun.why), ...(notRun.needsExperimental ? [text.runExperimental] : []), list].join("\n");
+};
+
 /** Why the request cannot be drafted, or the findings to report. */
 const pick = (request: Request, checked: Checked, lineCount: number, ui: UiLanguage): { readonly error: string } | { readonly findings: Finding[] } => {
   const text = TEXT[ui];
   if (request.line !== undefined && (request.line < 1 || request.line > lineCount)) return { error: text.badLine(lineCount) };
   const picked = chosen(checked.findings, request.rule, request.line);
   if (request.kind === "missed") return { findings: picked };
-  if (picked.length === 0) return { error: text.noMatch(describe(checked.findings, text.none)) };
+  if (picked.length === 0) return { error: noMatch(request, checked, ui) };
   // One finding per report: a rule picked alone could otherwise pull excerpts from all over the document.
   if (picked.length > 1) return { error: text.several(describe(picked, text.none)) };
   return { findings: picked };
@@ -179,7 +216,7 @@ export const runFeedback = async (targets: readonly string[], argv: readonly str
     console.error(text.notFound(request.path));
     return 1;
   }
-  const source = plainSource(readFileSync(request.path, "utf8"));
+  const source = await readDocumentFile(request.path);
   const checked = await context.check(request.path);
   const picked = pick(request, checked, source.split("\n").length, context.ui);
   if ("error" in picked) {
@@ -198,6 +235,7 @@ export const runFeedback = async (targets: readonly string[], argv: readonly str
       fileName: basename(request.path),
       language: checked.language,
       genre: checked.genre,
+      conditions: checked.conditions,
       findings,
       line: lines[0] ?? 0,
       excerpts: excerptsAround(source, lines),

package/src/commands/fix-plan.ts

@@ -0,0 +1,53 @@
+import { existsSync } from "node:fs";
+import { outlineOf } from "../outline/shape.ts";
+import { buildFixPlan } from "../fix-plan/plan.ts";
+import { renderFixPlanJson, renderFixPlanMarkdown } from "../fix-plan/render.ts";
+import { FIX_PLAN_TEXT } from "../fix-plan/text.ts";
+import type { Checked } from "./feedback.ts";
+import { readDocument } from "./read-document.ts";
+import type { TreeContext } from "./tree.ts";
+
+/** Options whose value is the next argument. That value is not a file. */
+const VALUED: ReadonlySet<string> = new Set(["--language", "--genre"]);
+
+export const fixPlanTargets = (argv: readonly string[]): string[] =>
+  argv.slice(1).filter((arg, index, all) => !arg.startsWith("--") && !VALUED.has(all[index - 1] ?? ""));
+
+export type FixPlanContext = TreeContext & {
+  /** The file's findings as lint finds them, shelved ones included. */
+  readonly check: (path: string) => Promise<Omit<Checked, "conditions">>;
+};
+
+/**
+ * An instruction document for whoever rewrites the file: the findings grouped by rule, each rule's rewrite direction,
+ * the document-level signals, a recommended mode and the checks to run after. Deterministic; sends nothing anywhere.
+ */
+export const runFixPlan = async (targets: readonly string[], argv: readonly string[], context: FixPlanContext): Promise<number> => {
+  const [path] = targets;
+  if (path === undefined || targets.length > 1) {
+    console.error(FIX_PLAN_TEXT[context.ui ?? "ja"].usage);
+    return 1;
+  }
+  if (!existsSync(path)) {
+    console.error(FIX_PLAN_TEXT[context.ui ?? "ja"].notFound(path));
+    return 1;
+  }
+  const checked = await context.check(path);
+  // The outline is read in the language lint chose, so the plan's findings and its outline describe one reading.
+  const prose = await readDocument(path, ["fix-plan", "--language", checked.language, ...argv.slice(1)], context, false);
+  if (prose === undefined) return 1;
+  const plan = buildFixPlan({
+    path,
+    language: checked.language,
+    genre: checked.genre,
+    experimental: context.config.experimental || argv.includes("--experimental"),
+    genreFlag: context.flag(argv, "--genre"),
+    findings: checked.findings,
+    rules: checked.rules,
+    skipped: checked.skipped,
+    outline: outlineOf(prose.doc),
+    phrases: prose.doc.lexicons["ai-tell"] ?? [],
+  });
+  console.log(argv.includes("--json") ? renderFixPlanJson(plan) : renderFixPlanMarkdown(plan));
+  return 0;
+};

package/src/commands/outline.ts

@@ -0,0 +1,49 @@
+import { uiLanguageOf } from "../ui.ts";
+import { outlineOf } from "../outline/shape.ts";
+import { renderOutlineCompact, renderOutlineFriendly, renderOutlineJson, type DocumentOutline } from "../outline/render.ts";
+import { OUTLINE_TEXT } from "../outline/text.ts";
+import { readDocument } from "./read-document.ts";
+import type { TreeContext } from "./tree.ts";
+
+/** Options whose value is the next argument. That value is not a file. */
+const VALUED: ReadonlySet<string> = new Set(["--language", "--genre"]);
+
+export const outlineTargets = (argv: readonly string[]): string[] =>
+  argv.slice(1).filter((arg, index, all) => !arg.startsWith("--") && !VALUED.has(all[index - 1] ?? ""));
+
+const MOST_FILES = 2;
+
+const readOutline = async (path: string, argv: readonly string[], context: TreeContext): Promise<DocumentOutline | undefined> => {
+  const prose = await readDocument(path, argv, context, false);
+  return prose === undefined ? undefined : { path, language: prose.language, outline: outlineOf(prose.doc) };
+};
+
+/** The files in the order given, or undefined once one cannot be read (its error is already printed). */
+const readAll = async (paths: readonly string[], argv: readonly string[], context: TreeContext): Promise<DocumentOutline[] | undefined> =>
+  paths.reduce<Promise<DocumentOutline[] | undefined>>(async (previous, path) => {
+    const earlier = await previous;
+    if (earlier === undefined) return undefined;
+    const next = await readOutline(path, argv, context);
+    return next === undefined ? undefined : [...earlier, next];
+  }, Promise.resolve([]));
+
+const renderFor = (argv: readonly string[], documents: readonly DocumentOutline[]): string => {
+  const text = OUTLINE_TEXT[uiLanguageOf(documents[0]?.language)];
+  if (argv.includes("--json")) return renderOutlineJson(documents);
+  return argv.includes("--compact") ? renderOutlineCompact(documents, text) : renderOutlineFriendly(documents, text);
+};
+
+/**
+ * A document's outline and shape (headings, average section length, text in lists, bold), or two documents' side by
+ * side: a restructure measured by machine. Measures; never judges, so it always ends with 0 once the files are read.
+ */
+export const runOutline = async (targets: readonly string[], argv: readonly string[], context: TreeContext): Promise<number> => {
+  if (targets.length === 0 || targets.length > MOST_FILES) {
+    console.error(OUTLINE_TEXT[context.ui ?? "ja"].usage);
+    return 1;
+  }
+  const documents = await readAll(targets, argv, context);
+  if (documents === undefined) return 1;
+  console.log(renderFor(argv, documents));
+  return 0;
+};

package/src/commands/read-document.ts

@@ -0,0 +1,24 @@
+import { loadAdapter } from "../adapter-load.ts";
+import { buildDocument, teamRules } from "../document.ts";
+import { profileFor } from "../profile/for-file.ts";
+import { resolveGenre } from "../resolve-genre.ts";
+import type { ProseDocument, StructurePatterns } from "../plugin.ts";
+import { readSource, treeLanguage, type TreeContext } from "./tree.ts";
+
+/** One document as lint reads it, and its language package's structure reader. */
+export type ReadDocument = { readonly doc: ProseDocument; readonly language: string; readonly structure: StructurePatterns | undefined };
+
+/**
+ * Reads one file as lint does: its language, genre, document profile and the team's rules. `pos` loads the
+ * part-of-speech tagger, which costs time, so only a command that reads names asks for it.
+ */
+export const readDocument = async (path: string, argv: readonly string[], context: TreeContext, pos: boolean): Promise<ReadDocument | undefined> => {
+  const source = await readSource(path, context);
+  if (source === undefined) return undefined;
+  const language = treeLanguage(path, source, argv, context);
+  const adapter = await loadAdapter(language);
+  await adapter.prepare?.({ pos });
+  const genre = resolveGenre(path, source, context.config, context.flag(argv, "--genre")).genre;
+  const doc = buildDocument(path, source, adapter, teamRules(context.config), profileFor(context.config, path, source, language, genre));
+  return { doc, language, structure: adapter.structure };
+};

package/src/commands/suppressions.ts

@@ -0,0 +1,19 @@
+import { collectTargets } from "../files.ts";
+import { renderSuppressions, type PerFile } from "../render/suppressions.ts";
+import type { UiLanguage } from "../ui.ts";
+
+/** `chaff suppressions`: what the stets in these files (this folder by default) silence, and the stets whose rule did not run. */
+export const runSuppressions = async (
+  targets: readonly string[],
+  inspect: (path: string) => Promise<{ readonly perFile: PerFile }>,
+  ui: UiLanguage,
+): Promise<number> => {
+  const inspected = await Promise.all(collectTargets(targets.length > 0 ? targets : ["."]).map(inspect));
+  console.log(
+    renderSuppressions(
+      inspected.map((file) => file.perFile),
+      ui,
+    ),
+  );
+  return 0;
+};

package/src/commands/test.ts

@@ -1,11 +1,9 @@
-import { plainSource } from "../plain-source.ts";
-import { readFile } from "node:fs/promises";
 import { join } from "node:path";
 import { loadAdapter } from "../adapter-load.ts";
 import { applyByPath } from "../config/by-path.ts";
 import { buildDocument, teamRules } from "../document.ts";
 import { guessLanguage } from "../detect.ts";
-import { collectTargets } from "../files.ts";
+import { collectTargets, readDocumentFile } from "../files.ts";
 import { loadRules } from "../rule-load.ts";
 import { counted } from "../render/plural.ts";
 import { CHECKS_FILE, loadChecks, type UserCheck } from "../checks.ts";
@@ -87,13 +85,13 @@ type SemanticDocument = { readonly doc: ProseDocument; readonly rules: RuleDefin
 
 /** 意味を読む検査に渡す文書。絞り込みが品詞を使うなら、解析器を読み込んでから作る。dry-run も本番も同じ文書を見る。 */
 const semanticDocument = async (path: string, config: Config, resolveGenre: TestContext["resolveGenre"]): Promise<SemanticDocument> => {
-  const source = plainSource(await readFile(path, "utf8"));
+  const source = await readDocumentFile(path);
   const language = applyByPath(config.byPath, config.baseDir, path).language ?? config.language ?? guessLanguage(source).language;
   const adapter = await loadAdapter(language);
   const { genre } = resolveGenre(path, source, config);
   const rules = loadRules(language);
   await adapter.prepare?.(semanticNeeds(rules, config.rules, genre));
-  const doc = buildDocument(path, source, adapter, teamRules(config), profileFor(config, path, source, language, genre));
+  const doc = buildDocument(path, source, adapter, teamRules(config, language), profileFor(config, path, source, language, genre));
   return { doc, rules, language, genre };
 };
 

package/src/commands/tree.ts

@@ -1,5 +1,4 @@
-import { plainSource } from "../plain-source.ts";
-import { readFile } from "node:fs/promises";
+import { readDocumentFile } from "../files.ts";
 import { loadAdapter } from "../adapter-load.ts";
 import { applyByPath } from "../config/by-path.ts";
 import { profileFor } from "../profile/for-file.ts";
@@ -8,6 +7,7 @@ import { guessLanguage } from "../detect.ts";
 import { isMarkdownPath } from "../structure/markdown-path.ts";
 import { buildStructure } from "../structure/of.ts";
 import { toSexp } from "../structure/sexp.ts";
+import { foldPostOrder } from "../tree-walk.ts";
 import type { Config } from "../config/load.ts";
 import type { StructureNode } from "../plugin.ts";
 import type { Texts, UiLanguage } from "../ui.ts";
@@ -23,16 +23,19 @@ const TEXT: Texts<{
   readonly usage: string;
   readonly unreadable: (path: string, why: string) => string;
   readonly noStructure: (path: string, language: string) => string;
+  readonly tooDeepForJson: (path: string, depth: number) => string;
 }> = {
   ja: {
     usage: "使い方: chaff tree <file>... [--format sexp|json] [--language ja|en|…]",
     unreadable: (path, why) => `${path} を読めませんでした: ${why}`,
     noStructure: (path, language) => `${path}: 言語 ${language} のパッケージは文書の構造を読めません（structure がありません）`,
+    tooDeepForJson: (path, depth) => `${path}: 木が ${String(depth)} 段と深すぎて、字下げした JSON に書き出せません（--format sexp なら書けることがあります）`,
   },
   en: {
     usage: "usage: chaff tree <file>... [--format sexp|json] [--language ja|en|…]",
     unreadable: (path, why) => `Could not read ${path}: ${why}`,
     noStructure: (path, language) => `${path}: the ${language} package cannot read a document's structure (it has no structure)`,
+    tooDeepForJson: (path, depth) => `${path}: the tree is ${String(depth)} levels deep, too deep to write as indented JSON (--format sexp may still write it)`,
   },
 };
 
@@ -48,7 +51,7 @@ export const treeTargets = (argv: readonly string[]): string[] =>
 
 export const readSource = async (path: string, context: TreeContext): Promise<string | undefined> => {
   try {
-    return plainSource(await readFile(path, "utf8"));
+    return await readDocumentFile(path);
   } catch (err) {
     console.error(treeText(context).unreadable(path, err instanceof Error ? err.message : String(err)));
     return undefined;
@@ -83,13 +86,46 @@ export const readTree = async (path: string, argv: readonly string[], context: T
   // 日本語は形態素で数量と日付を読む。解析器が無ければ単位の表で読むので、木は作れる。
   await adapter.prepare?.({ pos: true });
   const profile = profileFor(context.config, path, source, language, resolveGenre(path, source, context.config, context.flag(argv, "--genre")).genre);
-  return { source, tree: buildStructure({ path, source, language, markdown: isMarkdownPath(path), profile }, adapter.structure) };
+  return { source, tree: buildStructure({ path, source, language, markdown: isMarkdownPath(path), profile, lexicons: adapter.lexicons }, adapter.structure) };
+};
+
+/**
+ * 字下げした JSON。書けないほど深い木（言語パッケージの番号が何千段も入れ子になる）なら undefined。
+ * JSON.stringify は 1 段ごとに呼び直すのでスタックが尽き、字下げの合計は深さの二乗で増えて文字列の上限を超える。どちらも RangeError になる。
+ */
+export const treeJson = (tree: StructureNode): string | undefined => {
+  try {
+    return JSON.stringify(tree, null, 2);
+  } catch (err) {
+    if (err instanceof RangeError) return undefined;
+    throw err;
+  }
+};
+
+/** 根を 1 段目と数えた、いちばん深い葉までの段の数。 */
+export const depthOf = (tree: StructureNode): number =>
+  foldPostOrder(
+    tree,
+    (node) => node.children,
+    (_node, depths: readonly number[]) => depths.reduce((deepest, depth) => Math.max(deepest, depth), 0) + 1,
+  );
+
+/** 出力する文字列。JSON に書けないほど深い木なら、その深さ。 */
+export const renderTree = (tree: StructureNode, format: string | undefined): { readonly text: string } | { readonly tooDeep: number } => {
+  if (format !== "json") return { text: toSexp(tree) };
+  const json = treeJson(tree);
+  return json === undefined ? { tooDeep: depthOf(tree) } : { text: json };
 };
 
 const printTree = async (path: string, argv: readonly string[], context: TreeContext): Promise<boolean> => {
   const read = await readTree(path, argv, context);
   if (read === undefined) return false;
-  console.log(context.flag(argv, "--format") === "json" ? JSON.stringify(read.tree, null, 2) : toSexp(read.tree));
+  const rendered = renderTree(read.tree, context.flag(argv, "--format"));
+  if ("tooDeep" in rendered) {
+    console.error(treeText(context).tooDeepForJson(path, rendered.tooDeep));
+    return false;
+  }
+  console.log(rendered.text);
   return true;
 };
 

package/src/compare/atom.ts

@@ -0,0 +1,33 @@
+/** The kinds of fact `chaff compare` reads, in the order it reports them. */
+export const ATOM_KINDS = ["number", "date", "time", "url", "code", "name", "quote", "heading", "reference", "footnote"] as const;
+
+export type AtomKind = (typeof ATOM_KINDS)[number];
+
+export const isAtomKind = (value: string): value is AtomKind => ATOM_KINDS.some((kind) => kind === value);
+
+/**
+ * One fact as a document states it. Two atoms are the same fact when kind and key are equal; `text` is how it was
+ * written (1,000円 and 1000円 share a key). Position is only for showing where it is: comparing ignores it.
+ */
+export type Atom = {
+  readonly kind: AtomKind;
+  readonly key: string;
+  readonly text: string;
+  readonly line: number;
+  /** A number written with no unit the reader knows. It may stand for the same number written with a unit. */
+  readonly unitless?: boolean;
+  /** For a number: its value alone, without the unit. */
+  readonly value?: string;
+};
+
+/** A kind this document could not be fully read for, and why. Zero facts of it then does not mean none were there. */
+export type Unread = { readonly kind: AtomKind; readonly reason: UnreadReason };
+
+export type UnreadReason = "no-structure" | "no-dates" | "no-pos" | "plain-text";
+
+export type Extraction = {
+  readonly atoms: readonly Atom[];
+  readonly unread: readonly Unread[];
+  /** The prose spelled as name keys are, so the other document can ask whether a name is written here at all. */
+  readonly nameText: string;
+};

package/src/compare/clock-time.ts

@@ -0,0 +1,89 @@
+import type { Span } from "../plugin.ts";
+import { overlapsAny, spanIndex } from "./spans.ts";
+
+/** A time of day and where it is written. key is "HH:MM" on a 24-hour clock, with ":SS" when seconds are given. */
+export type ClockTime = Span & { readonly key: string };
+
+const DIGIT = "[0-9０-９]";
+const COLON = "[:：]";
+
+/** 10:30, 15:30:05, 3:30 p.m. */
+const COLON_TIME = new RegExp(
+  `(?<![0-9０-９])(${DIGIT}{1,2})${COLON}(${DIGIT}{2})(?:${COLON}(${DIGIT}{2}))?(?:\\s?([ap])\\.?m\\.?(?![a-z]))?(?![0-9０-９])`,
+  "giu",
+);
+
+/** 3 p.m., 11am. */
+const MERIDIEM_TIME = new RegExp(`(?<![0-9０-９:：])(${DIGIT}{1,2})\\s?([ap])\\.?m\\.?(?![a-z])`, "giu");
+
+/**
+ * 午後3時30分, 10時半, 9時. 時間 is a length of time (8時間), not a time of day, and the hour is not the end of a longer
+ * number (1.2時間, 123時).
+ */
+const JAPANESE_TIME = new RegExp(`(午前|午後)?(?<![0-9０-９.．])(${DIGIT}{1,2})時(?!間)(?:(${DIGIT}{1,2})分|(半))?`, "gu");
+
+const LAST_HOUR = 24;
+const MINUTES_PER_HOUR = 60;
+const HALF_HOUR = 30;
+const NOON = 12;
+const PADDED = 2;
+
+type Reading = { readonly hour: number; readonly minute: number; readonly second?: number | undefined; readonly afternoon?: boolean | undefined };
+
+const hourOf = (hour: number, afternoon: boolean | undefined): number => {
+  if (afternoon === undefined) return hour;
+  if (afternoon) return hour < NOON ? hour + NOON : hour;
+  return hour === NOON ? 0 : hour;
+};
+
+const padded = (value: number): string => String(value).padStart(PADDED, "0");
+
+/** Whether the hour is one on its clock: up to 12 with a.m. or p.m., else up to 23, and 24 only as 24:00. */
+const isHour = ({ hour, minute, second, afternoon }: Reading): boolean => {
+  if (afternoon !== undefined) return hour <= NOON;
+  return hour < LAST_HOUR || (hour === LAST_HOUR && minute === 0 && (second ?? 0) === 0);
+};
+
+/** The key, or undefined when the numbers are not a time of day (24:30, 13 p.m., 3:75). */
+const keyOf = (reading: Reading): string | undefined => {
+  const { hour, minute, second } = reading;
+  if (!isHour(reading) || minute >= MINUTES_PER_HOUR || (second ?? 0) >= MINUTES_PER_HOUR) return undefined;
+  const parts = [hourOf(hour, reading.afternoon), minute, ...(second === undefined ? [] : [second])];
+  return parts.map(padded).join(":");
+};
+
+const numberOf = (digits: string | undefined): number | undefined => (digits === undefined ? undefined : Number(digits.normalize("NFKC")));
+
+const meridiem = (letter: string | undefined): boolean | undefined => (letter === undefined ? undefined : letter.toLowerCase() === "p");
+
+const japaneseMeridiem = (word: string | undefined): boolean | undefined => (word === undefined ? undefined : word === "午後");
+
+type Reader = { readonly pattern: RegExp; readonly read: (match: RegExpMatchArray) => Reading };
+
+const READERS: readonly Reader[] = [
+  {
+    pattern: JAPANESE_TIME,
+    read: (match) => ({
+      hour: numberOf(match[2]) ?? 0,
+      minute: match[4] === undefined ? (numberOf(match[3]) ?? 0) : HALF_HOUR,
+      afternoon: japaneseMeridiem(match[1]),
+    }),
+  },
+  {
+    pattern: COLON_TIME,
+    read: (match) => ({ hour: numberOf(match[1]) ?? 0, minute: numberOf(match[2]) ?? 0, second: numberOf(match[3]), afternoon: meridiem(match[4]) }),
+  },
+  { pattern: MERIDIEM_TIME, read: (match) => ({ hour: numberOf(match[1]) ?? 0, minute: 0, afternoon: meridiem(match[2]) }) },
+];
+
+/** Times of day in text, in the order written. Where two readings overlap, the earlier reader in READERS wins. */
+export const clockTimes = (text: string): ClockTime[] =>
+  READERS.reduce<ClockTime[]>((found, reader) => {
+    const taken = spanIndex(found);
+    const fresh = [...text.matchAll(reader.pattern)].flatMap((match) => {
+      const key = keyOf(reader.read(match));
+      const span = { start: match.index, end: match.index + match[0].length };
+      return key === undefined || overlapsAny(taken, span) ? [] : [{ ...span, key }];
+    });
+    return [...found, ...fresh];
+  }, []).toSorted((left, right) => left.start - right.start);

package/src/compare/distinct.ts

@@ -0,0 +1,13 @@
+import type { Atom } from "./atom.ts";
+import { identity } from "./match.ts";
+
+/**
+ * The changes left when facts are compared as sets (--distinct): a fact the other document states at least once is
+ * kept, however many times either states it. A summary that repeated the body can be cut without its repeats reading
+ * as dropped. A name counts as stated when its spelling is in the other document's text (otherNameText), as when
+ * names are counted.
+ */
+export const unstatedChanges = (changes: readonly Atom[], other: readonly Atom[], otherNameText: string): Atom[] => {
+  const stated = new Set(other.map(identity));
+  return changes.filter((atom) => !stated.has(identity(atom)) && !(atom.kind === "name" && otherNameText.includes(atom.key)));
+};

package/src/compare/extract.ts

@@ -0,0 +1,80 @@
+import { lineStarts, placeOf } from "../position.ts";
+import type { MarkdownNode } from "../markdown-node.ts";
+import type { ProseDocument, Span, StructurePatterns } from "../plugin.ts";
+import type { Atom, Extraction, Unread } from "./atom.ts";
+import { clockTimes } from "./clock-time.ts";
+import { factTextOf, type FactText } from "./fact-text.ts";
+import { codeAtoms, headingAtoms, urlAtoms } from "./markup-atoms.ts";
+import { properNouns, readsProperNouns } from "./proper-nouns.ts";
+import { factKey, seenTextOf, wholeSpanOf } from "./fact-key.ts";
+import { spanIndex } from "./spans.ts";
+import { atomsOf, bareNumbers, footnotes, numericDates, quotations, teamNames, type TextInput } from "./text-atoms.ts";
+import { treeFacts, type TreeFacts } from "./tree-atoms.ts";
+
+export type ExtractInput = {
+  readonly doc: ProseDocument;
+  /** The Markdown tree, or undefined for a plain-text document. */
+  readonly root: MarkdownNode | undefined;
+  /** The language package's structure reader: numbers with units, dates and references come from it. */
+  readonly structure: StructurePatterns | undefined;
+  /** chaff.yaml's `names:`. */
+  readonly names: readonly string[];
+};
+
+const NO_TREE_FACTS: TreeFacts = { atoms: [], taken: [] };
+
+const STRUCTURE_KINDS = ["number", "date", "reference"] as const;
+
+/** The kinds this document could not be fully read for. Each says why, so "nothing dropped" is never "nothing read". */
+const unreadOf = (input: ExtractInput): Unread[] => [
+  ...(input.structure === undefined ? STRUCTURE_KINDS.map((kind): Unread => ({ kind, reason: "no-structure" })) : []),
+  ...(input.structure !== undefined && input.structure.dates === undefined ? [{ kind: "date", reason: "no-dates" } as const] : []),
+  ...(readsProperNouns(input.doc) ? [] : [{ kind: "name", reason: "no-pos" } as const]),
+  ...(input.root === undefined ? [{ kind: "code", reason: "plain-text" } as const] : []),
+];
+
+type Readers = { readonly facts: FactText; readonly text: TextInput; readonly lineOf: (offset: number) => number };
+
+/** Facts read from the prose itself. Each reader skips what an earlier one took, so one figure is one fact. */
+const proseFacts = (input: ExtractInput, readers: Readers, tree: TreeFacts, times: readonly Span[]): Atom[] => {
+  const { facts, text, lineOf } = readers;
+  const marks = footnotes(facts.codeless);
+  const dates = numericDates(facts.text, spanIndex([...tree.taken, ...times]));
+  const seen = seenTextOf(facts.text, facts.unseen);
+  const names = teamNames(seen.text, input.names).map((found) => wholeSpanOf(seen, found));
+  const taken = [...tree.taken, ...times, ...marks, ...dates];
+  return [
+    ...atomsOf(dates, "date", text),
+    ...atomsOf(marks, "footnote", text),
+    ...atomsOf(names, "name", text),
+    ...bareNumbers(text, spanIndex([...taken, ...facts.listMarkers])),
+    ...quotations(text, facts.unseen),
+    // "April" in a date and "Section" in a reference are read with them, not again as names.
+    ...properNouns({ doc: input.doc, taken: spanIndex([...taken, ...names, ...facts.blanked]), lineOf, unseen: facts.unseen }),
+  ];
+};
+
+const structureFacts = (input: ExtractInput, facts: FactText, times: readonly Span[], lineOf: (offset: number) => number): TreeFacts => {
+  const tree = input.doc.structure;
+  return tree === undefined ? NO_TREE_FACTS : treeFacts({ tree, source: input.doc.source, blanked: facts.blanked, times: spanIndex(times), lineOf });
+};
+
+/** Every fact atom of one document, and the kinds it could not be fully read for. */
+export const extractFacts = (input: ExtractInput): Extraction => {
+  const source = input.doc.source;
+  const starts = lineStarts(source);
+  const lineOf = (offset: number): number => placeOf(starts, offset).line;
+  const facts = factTextOf(source, input.root);
+  const text = { text: facts.text, source, lineOf };
+  const times = clockTimes(facts.text);
+  const tree = structureFacts(input, facts, times, lineOf);
+  const atoms = [
+    ...tree.atoms,
+    ...atomsOf(times, "time", text),
+    ...proseFacts(input, { facts, text, lineOf }, tree, times),
+    ...urlAtoms(input.doc.markup, facts.bareUrls, lineOf),
+    ...(input.root === undefined ? [] : codeAtoms(input.root, lineOf)),
+    ...headingAtoms(input.doc.markup, lineOf),
+  ];
+  return { atoms, unread: unreadOf(input), nameText: factKey(facts.text, { start: 0, end: source.length }, facts.unseen) };
+};

package/src/compare/fact-key.ts

@@ -0,0 +1,31 @@
+import { spansWithin, withoutSpans } from "../soft-break.ts";
+import type { Span } from "../plugin.ts";
+
+/**
+ * The text of a span as one spelling: what a reader never sees removed (unseen is sorted and does not overlap: a line
+ * break that vanishes between two wide characters, the marks of bold), any other run of white space one space,
+ * full-width and half-width forms alike.
+ */
+export const factKey = (text: string, span: Span, unseen: readonly Span[]): string => {
+  const inside = spansWithin(unseen, span).map((part) => ({ start: part.start - span.start, end: part.end - span.start }));
+  return withoutSpans(text.slice(span.start, span.end), inside).normalize("NFKC").replace(/\s+/gu, " ");
+};
+
+/** A text without its unseen parts, and where each of its characters stands in the whole text. */
+export type SeenText = { readonly text: string; readonly offsets: readonly number[] };
+
+/** The text as a reader sees it, for finding words that a vanishing line break or bold marks split (日本\n銀行). */
+export const seenTextOf = (text: string, unseen: readonly Span[]): SeenText => {
+  const kept = [...unseen, { start: text.length, end: text.length }].map((part, index) => ({ start: unseen[index - 1]?.end ?? 0, end: part.start }));
+  return {
+    text: kept.map((part) => text.slice(part.start, part.end)).join(""),
+    offsets: kept.flatMap((part) => Array.from({ length: part.end - part.start }, (_, index) => part.start + index)),
+  };
+};
+
+/** A span found in the seen text, as a span of the whole text: from its first character to just after its last. */
+export const wholeSpanOf = <T extends Span>(seen: SeenText, found: T): T => ({
+  ...found,
+  start: seen.offsets[found.start] ?? found.start,
+  end: (seen.offsets[found.end - 1] ?? found.end - 1) + 1,
+});