chaffjs 0.17.0 → 0.19.0

package/src/custom/regex-safety.ts

@@ -0,0 +1,118 @@
+// Whether a team's regular expression can run away. JavaScript's engine backtracks, so a pattern such as (a+)+ takes time
+// exponential in the length of a line that almost matches. Pure: the pattern is read as text, never run on a document.
+
+/** Longer patterns are refused: a rule that needs one is better written as several. */
+export const MAX_PATTERN_LENGTH = 500;
+
+/**
+ * More repeats of varying count (*, +, {1,9}) than this are refused. Repeats side by side (a*a*a*b, a{0,99}a{0,99}…) do
+ * not nest, but their work multiplies: a line that almost matches costs its length to the power of their number.
+ */
+export const MAX_REPEATS = 3;
+
+export type RegexRefusal = "too-long" | "nested-quantifier" | "too-many-repeats" | "backreference" | "empty-match" | "invalid";
+
+/** An open group: where its body starts, and whether it holds a repeat of varying count (a+, a?, a{1,3}) or an alternation. */
+type Group = { readonly start: number; varies: boolean; alternation: boolean };
+
+/** A quantifier: its length, whether it has no upper bound, whether its count varies, and whether it can repeat more than once. */
+type Quantifier = { readonly length: number; readonly unbounded: boolean; readonly varies: boolean; readonly many: boolean };
+
+type Scan = { at: number; repeats: number; readonly stack: Group[] };
+
+const QUANTIFIER_START: ReadonlySet<string> = new Set(["*", "+", "?", "{"]);
+const BRACE = /^\{(\d+)(,(\d*))?\}\??/u;
+const BACKREFERENCE = /^\\(?:[1-9]|k<)/u;
+const CLASS_BODY = /^\^?\]?(?:\\.|[^\]\\])*\]/u;
+
+/** The quantifier at pattern[at], or undefined when none is there. */
+const quantifierAt = (pattern: string, at: number): Quantifier | undefined => {
+  const char = pattern[at];
+  if (char === undefined || !QUANTIFIER_START.has(char)) return undefined;
+  if (char !== "{") return { length: pattern[at + 1] === "?" ? 2 : 1, unbounded: char !== "?", varies: true, many: char !== "?" };
+  const brace = BRACE.exec(pattern.slice(at));
+  if (brace === null) return undefined;
+  const [least, most] = [Number(brace[1]), brace[2] === undefined ? Number(brace[1]) : Number(brace[3] || Infinity)];
+  return { length: brace[0].length, unbounded: most === Infinity, varies: most !== least, many: most > 1 };
+};
+
+/** Just past the "]" that closes the character class opened at pattern[at]. */
+const classEnd = (pattern: string, at: number): number => {
+  const close = CLASS_BODY.exec(pattern.slice(at + 1));
+  return close === null ? pattern.length : at + 1 + close[0].length;
+};
+
+const innermost = (scan: Scan): Group => scan.stack[scan.stack.length - 1] ?? { start: 0, varies: false, alternation: false };
+
+/** A fixed string: plain characters, and punctuation escaped (Mr\.). \d, \w and the like are classes, not literals. */
+const LITERAL = /^(?:[^\\[\](){}*+?.|^$]|\\[^A-Za-z0-9])+$/u;
+
+/**
+ * Alternatives that are plain words starting with different characters ((cat|dog)+) never compete for the same text, so
+ * repeating them is linear. Any other alternation under a repeat ((a|aa)*) may try every split of the line.
+ */
+const distinctWords = (body: string): boolean => {
+  const alternatives = body.replace(/^\?:/u, "").split("|");
+  const firsts = alternatives.map((alternative) => (alternative.startsWith("\\") ? alternative.charAt(1) : alternative.charAt(0)));
+  return alternatives.every((alternative) => LITERAL.test(alternative)) && new Set(firsts).size === firsts.length;
+};
+
+/** Whether the group's text can be split between repeats in more than one way: it varies in length, or its alternatives compete. */
+const competes = (pattern: string, closed: Group, end: number): boolean =>
+  closed.varies || (closed.alternation && !distinctWords(pattern.slice(closed.start, end - 1)));
+
+/**
+ * After an atom that ends at end (closed is the group it closes, if any). A group repeated a varying number of times, more than
+ * once ((…)+, (…)*, (…){1,10}), that can itself match in more than one way ((a+), (a{1,10}), (a|aa)) is refused.
+ */
+const afterAtom = (pattern: string, scan: Scan, end: number, closed: Group | undefined): RegexRefusal | undefined => {
+  const quantifier = quantifierAt(pattern, end);
+  scan.at = end + (quantifier?.length ?? 0);
+  const repeatsVariably = quantifier !== undefined && quantifier.varies && quantifier.many;
+  if (repeatsVariably) scan.repeats += 1;
+  if (scan.repeats > MAX_REPEATS) return "too-many-repeats";
+  // A group that can match in more than one way makes the group around it so too, quantified or not: ((a|aa))+ is (a|aa)+.
+  const closedCompetes = closed !== undefined && competes(pattern, closed, end);
+  if (closedCompetes && repeatsVariably) return "nested-quantifier";
+  if (quantifier?.varies === true || closedCompetes) innermost(scan).varies = true;
+  return undefined;
+};
+
+/** Reads one escape, class, bracket or character at scan.at. */
+const step = (pattern: string, scan: Scan): RegexRefusal | undefined => {
+  const char = pattern[scan.at];
+  if (char === "\\") return BACKREFERENCE.test(pattern.slice(scan.at)) ? "backreference" : afterAtom(pattern, scan, scan.at + 2, undefined);
+  if (char === "[") return afterAtom(pattern, scan, classEnd(pattern, scan.at), undefined);
+  if (char === "(") {
+    scan.stack.push({ start: scan.at + 1, varies: false, alternation: false });
+    scan.at += 1;
+    return undefined;
+  }
+  if (char === ")") return afterAtom(pattern, scan, scan.at + 1, scan.stack.length > 1 ? scan.stack.pop() : undefined);
+  if (char === "|") innermost(scan).alternation = true;
+  return afterAtom(pattern, scan, scan.at + 1, undefined);
+};
+
+/**
+ * A group repeated without bound that holds a repeat without bound or an alternation ((a+)+, (a|aa)*, (\w+\s?)*) is the shape
+ * that backtracks exponentially. The pattern is read once, left to right, with a stack of open groups.
+ */
+const nestedRepeat = (pattern: string): RegexRefusal | undefined => {
+  const scan: Scan = { at: 0, repeats: 0, stack: [{ start: 0, varies: false, alternation: false }] };
+  while (scan.at < pattern.length) {
+    const refusal = step(pattern, scan);
+    if (refusal !== undefined) return refusal;
+  }
+  return undefined;
+};
+
+/** Why the pattern is refused, or undefined when it may run. flags are the ones the rule runs with ("u", "iu"). */
+export const regexRefusal = (pattern: string, flags: string): RegexRefusal | undefined => {
+  if (pattern.length > MAX_PATTERN_LENGTH) return "too-long";
+  try {
+    if (new RegExp(pattern, flags).test("")) return "empty-match";
+  } catch {
+    return "invalid";
+  }
+  return nestedRepeat(pattern);
+};

package/src/custom/token-pattern.ts

@@ -0,0 +1,84 @@
+import type { Token, TokenCondition } from "../plugin.ts";
+
+// A team's morphology pattern: a run of tokens, each matching a condition on its part of speech, base form or surface.
+// 「〜を行う」 is [{ pos: 名詞 }, { surface: を }, { base: 行う }]. Pure: tokens come from the language adapter.
+
+/**
+ * Parts of speech are Universal Dependencies tags (NOUN, VERB, ADP), the same for every language. A team may also write the
+ * everyday name in Japanese or English; each maps to the tags the adapters give for it.
+ */
+const POS_NAMES: Readonly<Record<string, readonly string[]>> = {
+  名詞: ["NOUN", "PROPN", "PRON", "NUM"],
+  固有名詞: ["PROPN"],
+  代名詞: ["PRON"],
+  動詞: ["VERB"],
+  形容詞: ["ADJ"],
+  副詞: ["ADV"],
+  助詞: ["ADP", "PART", "SCONJ", "CCONJ"],
+  助動詞: ["AUX"],
+  接続詞: ["CCONJ", "SCONJ"],
+  連体詞: ["DET"],
+  記号: ["PUNCT", "SYM"],
+  noun: ["NOUN", "PROPN", "PRON"],
+  verb: ["VERB"],
+  adjective: ["ADJ"],
+  adverb: ["ADV"],
+  preposition: ["ADP"],
+  determiner: ["DET"],
+  pronoun: ["PRON"],
+  conjunction: ["CCONJ", "SCONJ"],
+  auxiliary: ["AUX"],
+};
+
+const UPOS: ReadonlySet<string> = new Set([
+  "ADJ",
+  "ADP",
+  "ADV",
+  "AUX",
+  "CCONJ",
+  "DET",
+  "INTJ",
+  "NOUN",
+  "NUM",
+  "PART",
+  "PRON",
+  "PROPN",
+  "PUNCT",
+  "SCONJ",
+  "SYM",
+  "VERB",
+  "X",
+]);
+
+/** The tags a written part of speech stands for, or undefined when chaff does not know the name. */
+export const posTags = (written: string): readonly string[] | undefined => {
+  const upper = written.toUpperCase();
+  if (UPOS.has(upper)) return [upper];
+  return POS_NAMES[written] ?? POS_NAMES[written.toLowerCase()];
+};
+
+/** The names a team may write, for the message about an unknown one. */
+export const POS_WRITTEN_NAMES: readonly string[] = [...UPOS, ...Object.keys(POS_NAMES)];
+
+const sameWord = (left: string, right: string): boolean => left.toLowerCase() === right.toLowerCase();
+
+const holds = (token: Token, condition: TokenCondition): boolean =>
+  (condition.pos === undefined || condition.pos.includes(token.pos)) &&
+  (condition.surface === undefined || sameWord(token.surface, condition.surface)) &&
+  (condition.base === undefined || sameWord(token.lemma ?? token.surface, condition.base));
+
+/** Every run of consecutive tokens where each holds its condition, as [first, last] token indexes. Runs do not overlap. */
+export const tokenRuns = (tokens: readonly Token[], conditions: readonly TokenCondition[]): (readonly [number, number])[] => {
+  if (conditions.length === 0) return [];
+  return tokens.reduce<{ runs: (readonly [number, number])[]; next: number }>(
+    (acc, _token, start) => {
+      if (start < acc.next || start + conditions.length > tokens.length) return acc;
+      const matches = conditions.every((condition, offset) => {
+        const token = tokens[start + offset];
+        return token !== undefined && holds(token, condition);
+      });
+      return matches ? { runs: [...acc.runs, [start, start + conditions.length - 1]], next: start + conditions.length } : acc;
+    },
+    { runs: [], next: 0 },
+  ).runs;
+};

package/src/detect-language.ts

@@ -6,7 +6,7 @@
  * 権威ある点数で、こちらは何も読み込んでいない状態での見当付け。
  * 似た計算を両方が持つのは、この順序から来る避けられない重複。
  *
- * 呼ぶ側が、コードブロック・インラインコード・URL・front matter を先に落としておくこと。
+ * 呼ぶ側が、コードブロック・インラインコード・URL・front matter を先に落としておくこと（guessLanguage は language-sample.ts で落とす）。
  * 技術文書は英語のコードを大量に含むため、落とさないと日本語文書が英語と判定される。
  */
 const JAPANESE = /[぀-ゟ゠-ヿ一-鿿]/gu;

package/src/detect.ts

@@ -1,4 +1,11 @@
+import { readFileSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+import { parse } from "yaml";
 import { japaneseRatio, latinRatio } from "./detect-language.ts";
+import { referenceListSpans } from "./reference-lists.ts";
+import { withoutSpans } from "./soft-break.ts";
+import { languageSample } from "./language-sample.ts";
 
 export type LanguageGuess = { readonly language: string; readonly confidence: number; readonly from: string };
 
@@ -8,8 +15,34 @@ export type LanguageGuess = { readonly language: string; readonly confidence: nu
  */
 const JAPANESE_FLOOR = 0.15;
 
+const HEADINGS_FILE = join(dirname(fileURLToPath(import.meta.url)), "..", "reference-headings.yaml");
+
+const isStringList = (value: unknown): value is string[] => Array.isArray(value) && value.every((entry) => typeof entry === "string");
+
+const readHeadings = (): string[] => {
+  const raw: unknown = parse(readFileSync(HEADINGS_FILE, "utf8"));
+  const headings: unknown = typeof raw === "object" && raw !== null && "headings" in raw ? raw.headings : undefined;
+  if (!isStringList(headings)) throw new Error(`${HEADINGS_FILE}: headings は文字列の並びであること`);
+  return headings;
+};
+
+const REFERENCE_HEADINGS = readHeadings();
+
+/** 本文だけ。文献一覧は引いた文献の言語で書かれるので数えない。本文に字が無ければ（文献一覧だけの文書）全体。 */
+const bodyOf = (source: string): string => {
+  const body = withoutSpans(source, referenceListSpans(source, REFERENCE_HEADINGS));
+  return body.trim() === "" ? source : body;
+};
+
+/** The body without code, URLs and markup, or the whole body when nothing else is left (a document of only code). */
+const proseOf = (body: string): string => {
+  const sample = languageSample(body);
+  return sample.trim() === "" ? body : sample;
+};
+
 export const guessLanguage = (source: string): LanguageGuess => {
-  const japanese = japaneseRatio(source);
+  const prose = proseOf(bodyOf(source));
+  const japanese = japaneseRatio(prose);
   if (japanese >= JAPANESE_FLOOR) return { language: "ja", confidence: japanese, from: "content" };
-  return { language: "en", confidence: latinRatio(source), from: "content" };
+  return { language: "en", confidence: latinRatio(prose), from: "content" };
 };

package/src/detectors/acronym-context.ts

@@ -1,5 +1,5 @@
 import { escapeRegExp } from "../orthography.ts";
-import { ROMAN_NUMERAL } from "./roman-numeral.ts";
+import { ROMAN_NUMERAL, romanRangeOf } from "./roman-numeral.ts";
 import { citationKeySpans } from "./citation-key.ts";
 import { dottedNameSpans } from "./dotted-name.ts";
 import { letterJoinedNameSpans } from "./letter-joined-name.ts";
@@ -24,6 +24,8 @@ export type NotationWords = {
   readonly emphasis: readonly string[];
   /** 番号を後ろに書く、文書の区切りの名前（Part、Section、Title）。 */
   readonly divisions: readonly string[];
+  /** 番号を後ろに書く、点の付いた略した名前（Vol.、No.、FIG.）。 */
+  readonly abbreviatedLabels: readonly string[];
   /** 速記録が大文字で書く発言者の姓の前に置く敬称（Mr.、Mrs.、Madam）。 */
   readonly honorifics: readonly string[];
   /** 大文字で書く姓の前に置く肩書き（Prime Minister、Governor）。後ろに略語も来る（the President NASA memo）ので、名前の形を求める。 */
@@ -44,11 +46,21 @@ const ANY_MERIDIEM = String.raw`[AaPp]\.?[Mm]\.?`;
 const AMOUNT = String.raw`(?:\d{1,3}(?:,\d{3})+|\d+)(?:\.\d+)?`;
 
 /**
- * 区切りの名前のすぐ後ろのローマ数字は番号（Part II、Section VIII、TITLE IV）。名前は語彙表の書き方か、全部大文字。
- * CD や CI のようにローマ数字としても読める略語は、この位置でなければ略語のまま数える。
+ * 区切りの名前や略した名前のすぐ後ろの番号（Part II、Section VIII、TITLE IV、Vol. XLIII、FIG. 3、FIGS. 1A-1C）。名前は語彙表の
+ * 書き方か、全部大文字。ローマ数字は範囲（Part II-IV）も番号。CD や CI のようにローマ数字としても読める略語は、この位置で
+ * なければ略語のまま数える。
  */
 const divisionOf = (words: readonly string[]): string => oneOf(words.flatMap((word) => [word, word.toUpperCase()]));
 
+/** 図や表の番号は短い（FIG. 3、TABLE 12、FIG. 1A）。長い数（FIG. 12345）の前の大文字は略語のまま数える。 */
+const SHORT_NUMBER = String.raw`\d{1,3}[A-Za-z]?`;
+const NUMBER_END = String.raw`(?![\p{L}\p{N}_&])`;
+
+const numberedPatterns = (words: NotationWords): readonly RegExp[] => {
+  const label = String.raw`(?<![\p{L}\p{N}_])${divisionOf([...words.divisions, ...words.abbreviatedLabels])}\s+`;
+  return [new RegExp(`${label}${romanRangeOf(ROMAN_NUMERAL)}${NUMBER_END}`, "gu"), new RegExp(`${label}${SHORT_NUMBER}${NUMBER_END}`, "gu")];
+};
+
 /** 敬称のすぐ後ろの大文字の姓（Mr. HAWLEY、Dr. O'NEIL）。速記録は発言者をこう書き、略語ではない。 */
 const SURNAME_IN_CAPITALS = String.raw`[A-Z]+(?:['’-][A-Z]+)*(?![\p{L}\p{N}_])`;
 
@@ -71,7 +83,7 @@ const patternsOf = (words: NotationWords): readonly RegExp[] => [
   new RegExp(String.raw`(?<![\w.,])${AMOUNT}\s*${oneOf(words.currencies)}`, "gu"),
   new RegExp(String.raw`,\s+${oneOf(words.usStates)}\s+\d{5}(?:-\d{4})?(?!\d)`, "gu"),
   new RegExp(oneOf(words.emphasis), "gu"),
-  new RegExp(String.raw`(?<![\p{L}\p{N}_])${divisionOf(words.divisions)}\s+${ROMAN_NUMERAL}(?![\p{L}\p{N}_&])`, "gu"),
+  ...numberedPatterns(words),
   new RegExp(String.raw`(?<![\p{L}\p{N}_])${oneOf(words.honorifics)}\s+${SURNAME_IN_CAPITALS}`, "gu"),
   ...titledName(oneOf(words.titles)),
 ];

package/src/detectors/ai-phrasing.ts

@@ -0,0 +1,141 @@
+import type { Detector, Finding, Lexicon, LexiconEntry, ProseDocument, Sentence } from "../plugin.ts";
+import { wordsOf } from "./structure.ts";
+import { MIN_DOCUMENT_LENGTH } from "./signals.ts";
+import { entryIn, entryOpens } from "./lexicon-match.ts";
+import { contrastSentences, type ContrastWords } from "./contrast-frame.ts";
+import { placeholderSpans } from "./placeholder-text.ts";
+import { colonLeadIns } from "./list-lead-in.ts";
+
+// 生成文に多い形。語はどれも言語パッケージの語彙表が持ち、ここは形だけを知る。
+
+const PER = 1000;
+/** 密度は小数 1 桁まで。日本語の 1000 字あたりは 1 と 2 の間で人と生成文が分かれる。 */
+const TENTHS = 10;
+/** 重みの合計を 10 倍した点で比べる（ai-tell と同じ）。重み 0.5 の言い回し 2 つで 10。 */
+const SCORE_SCALE = 10;
+/** 1 つは重なりではない。短い文書では 1 つでも密度が上限を超えるので、数でも止める。 */
+const PILE = 2;
+
+/** 当たった文と語。offset は指摘する位置で、無ければ文の頭。 */
+type Hit = { readonly sentence: Sentence; readonly matched: string; readonly offset?: number };
+
+const findingOf = (hit: Hit, values: Readonly<Record<string, number>>): Finding => ({
+  rule: "",
+  severity: "info",
+  line: 0,
+  column: 0,
+  quote: hit.sentence.text.trim(),
+  values: { matched: hit.matched, ...values, offset: hit.offset ?? hit.sentence.span.start },
+});
+
+/** 1000 語（日本語は 1000 字）あたりの数が上限を超えたら、当たった文をすべて言う。短い文書は密度が暴れるので測らない。 */
+const densityFindings = (doc: ProseDocument, hits: readonly Hit[], limit: number): Finding[] => {
+  const length = wordsOf(doc);
+  if (hits.length < PILE || length < MIN_DOCUMENT_LENGTH[doc.lengthUnit]) return [];
+  const density = Math.round((hits.length / length) * PER * TENTHS) / TENTHS;
+  return density <= limit ? [] : hits.map((hit) => findingOf(hit, { count: hits.length, density, limit }));
+};
+
+const firstIn = (sentence: Sentence, lexicon: Lexicon): LexiconEntry | undefined => lexicon.find((entry) => entryIn(sentence, entry));
+
+const contrastWordsOf = (doc: ProseDocument, frames: Lexicon): ContrastWords => ({
+  frames,
+  leads: doc.lexicons["contrast-lead"] ?? [],
+  turns: doc.lexicons["contrast-turn"] ?? [],
+});
+
+/** 対比の枠（It's not X, it's Y、単なる X ではなく）。1 つなら論点の整理だが、重なると言い回しの癖になる。 */
+export const contrastFraming: Detector = (doc, options): Finding[] => {
+  const words = contrastWordsOf(doc, options.lexicon ?? []);
+  const hits = contrastSentences(doc.sentences, words).map((sentence) => ({
+    sentence,
+    matched: (firstIn(sentence, words.frames) ?? firstIn(sentence, words.leads))?.pattern ?? "",
+  }));
+  return densityFindings(doc, hits, options.limit);
+};
+
+/** 語彙表の語で始まる文。 */
+const openerHits = (doc: ProseDocument, lexicon: Lexicon): Hit[] =>
+  doc.sentences.flatMap((sentence) => {
+    const opener = lexicon.find((entry) => entryOpens(sentence, entry));
+    return opener === undefined ? [] : [{ sentence, matched: opener.pattern }];
+  });
+
+/** 文頭の決まった接ぎ（Moreover、さらに）の密度。話を運ぶ語（However、また）は語彙表に入れない。 */
+export const openerDensity: Detector = (doc, options): Finding[] => densityFindings(doc, openerHits(doc, options.lexicon ?? []), options.limit);
+
+/**
+ * 予告の文頭（重要なのは、Here's the thing）の数。limit は指摘に要る数。
+ * 密度ではなく数で見る。人の記事も長さによらず 1 つ 2 つは書き、生成文は短い記事にも重ねる。
+ */
+export const openerPile: Detector = (doc, options): Finding[] => {
+  const hits = openerHits(doc, options.lexicon ?? []);
+  return hits.length === 0 || hits.length < options.limit ? [] : hits.map((hit) => findingOf(hit, { count: hits.length, limit: options.limit }));
+};
+
+/**
+ * コロンで箇条書きへ渡す文（以下の通りです：）。1 つなら案内だが、節ごとに続くと説明が箇条書きの前置きだけになる。
+ * 数えるのは文書の言語の文だけ。和文の中の英文（Examples include:）は、英語ではふつうの書き方なので数えない。
+ */
+export const colonLeadIn: Detector = (doc, options): Finding[] => {
+  const inDocumentLanguage = doc.sentences.filter((sentence) => sentence.embeddedLanguage === undefined);
+  const hits = colonLeadIns(inDocumentLanguage, doc.lists, doc.listSpans, doc.source).map((sentence) => ({ sentence, matched: sentence.text.trim() }));
+  return densityFindings(doc, hits, options.limit);
+};
+
+const distinct = (lexicon: Lexicon): LexiconEntry[] => [...new Map(lexicon.map((entry) => [entry.pattern.toLowerCase(), entry])).values()];
+
+const weightOf = (entry: LexiconEntry): number => entry.weight ?? 1;
+
+const keyOf = (entry: LexiconEntry): string => entry.pattern.toLowerCase();
+
+type Tally = { readonly score: number; readonly counted: ReadonlySet<string>; readonly hits: readonly Hit[] };
+
+const NOTHING: Tally = { score: 0, counted: new Set(), hits: [] };
+
+/** 1 つの文を足す。重みは、まだ数えていない語のうち最も重いもの。文の語はすべて数えたことにする。 */
+const withSentence = (tally: Tally, sentence: Sentence, inSentence: readonly LexiconEntry[]): Tally => {
+  const fresh = inSentence.filter((entry) => !tally.counted.has(keyOf(entry)));
+  const added = fresh.length === 0 ? 0 : Math.max(...fresh.map(weightOf));
+  const matched = (fresh[0] ?? inSentence[0])?.pattern ?? "";
+  return {
+    score: tally.score + added,
+    counted: new Set([...tally.counted, ...inSentence.map(keyOf)]),
+    hits: [...tally.hits, { sentence, matched }],
+  };
+};
+
+/**
+ * 1 つの文は 1 つの言い回しとして数える（Let me know if you'd like a more detailed breakdown は礼 1 つ）。
+ * 前の文で数えた語は、別の文にあっても数えない（I hope this helps を 2 回書いても礼 1 つ）。
+ */
+const tallyOf = (sentences: readonly Sentence[], entries: readonly LexiconEntry[]): Tally =>
+  sentences.reduce<Tally>((tally, sentence) => {
+    const inSentence = entries.filter((entry) => entryIn(sentence, entry));
+    return inSentence.length === 0 ? tally : withSentence(tally, sentence, inSentence);
+  }, NOTHING);
+
+/**
+ * 会話の返事の名残（As of my last knowledge update、私の知識は）。重みを足した点が上限に届いたら、当たった文をすべて言う。
+ * 名残そのもの（知識の期限、AI としての断り）は 1 つで届き、人も書く礼（I hope this helps）は 2 つ重なって届く。
+ */
+export const assistantResidue: Detector = (doc, options): Finding[] => {
+  const tally = tallyOf(doc.sentences, distinct(options.lexicon ?? []));
+  const score = Math.round(tally.score * SCORE_SCALE);
+  if (tally.hits.length === 0 || score < options.limit) return [];
+  return tally.hits.map((hit) => findingOf(hit, { count: tally.counted.size, density: score, limit: options.limit }));
+};
+
+/** 埋め忘れた雛形の空欄（[Your Name]、【会社名】）。limit は指摘に要る数。 */
+export const unfilledPlaceholder: Detector = (doc, options): Finding[] => {
+  const words = options.lexicon ?? [];
+  const blanks = doc.sentences.flatMap((sentence) =>
+    placeholderSpans(sentence.text, words).map((span) => ({
+      sentence,
+      matched: sentence.text.slice(span.start, span.end),
+      offset: sentence.span.start + span.start,
+    })),
+  );
+  if (blanks.length === 0 || blanks.length < options.limit) return [];
+  return blanks.map((blank) => findingOf(blank, { count: blanks.length, limit: options.limit }));
+};

package/src/detectors/announced-count.ts

@@ -0,0 +1,25 @@
+import type { Detector, Finding, ProseDocument } from "../plugin.ts";
+import { announcedCountMismatches, type CountWords } from "../announced-count.ts";
+import { quoteAt } from "./structure-tree.ts";
+
+const patternsOf = (doc: ProseDocument, id: string, position?: "before" | "after"): string[] =>
+  (doc.lexicons[id] ?? []).filter((entry) => position === undefined || entry.position === position).map((entry) => entry.pattern);
+
+const countWordsOf = (doc: ProseDocument): CountWords => ({
+  anchors: patternsOf(doc, "count-anchor"),
+  numbers: patternsOf(doc, "count-number"),
+  counters: patternsOf(doc, "count-counter"),
+  hedgesBefore: patternsOf(doc, "count-hedge", "before"),
+  hedgesAfter: patternsOf(doc, "count-hedge", "after"),
+});
+
+/** 予告の数（以下の3点）と、すぐ下の箇条書きの項目の数が違う。 */
+export const announcedCount: Detector = (doc): Finding[] =>
+  announcedCountMismatches(doc.source, doc.lists, countWordsOf(doc)).map((mismatch) => ({
+    rule: "announced-count-mismatch",
+    severity: "warning",
+    line: 0,
+    column: 0,
+    quote: quoteAt(doc.source, mismatch.offset),
+    values: { phrase: mismatch.phrase, announced: mismatch.announced, listed: mismatch.listed, offset: mismatch.offset },
+  }));

package/src/detectors/bold-label.ts

@@ -0,0 +1,55 @@
+import type { Detector, Finding, ProseDocument, Span } from "../plugin.ts";
+
+/** A bold run longer than this is a bold sentence, not a label. */
+const MAX_LABEL_CHARS = 40;
+
+/** The bullet or number that opens a list item. */
+const ITEM_MARKER = /^[ \t]*(?:[-*+]|\d{1,9}[.)])[ \t]+/u;
+
+/** A task box, after the marker or at the start of an item whose marker was already cut. */
+const TASK_BOX = /^\[[ xX]\][ \t]+/u;
+
+/** `**Label**: text` or `**Label:** text`. The label holds no delimiter, so `**a** and **b**: text` is not one label. */
+const BOLD_LABEL = new RegExp(String.raw`^(\*\*|__)(?!\s)((?:(?!\1)[^\n]){1,${MAX_LABEL_CHARS}}?)(?:\1[ \t]*[:：]|[:：]\1)[ \t]*\S`, "u");
+
+/** A label that is only code (`--flag`, `timeout_ms`) names an option or a field: the way reference documentation lists them. */
+const CODE_SPAN = /^`[^`]+`$/u;
+
+/** A link around the whole label: inline, reference, collapsed or shortcut. Its text is what the reader sees. */
+const WHOLE_LINK = /^\[([^\]\n]+)\](?:\([^)\s]*\)|\[[^\]\n]*\])?$/u;
+
+const isCodeOnly = (label: string): boolean => CODE_SPAN.test(label.replace(WHOLE_LINK, "$1"));
+
+/** The label a list item opens with in bold before a colon and more text, or undefined. `item` is the item as written, marker and all. */
+export const boldLabelOf = (item: string): string | undefined => {
+  const label = BOLD_LABEL.exec(item.replace(ITEM_MARKER, "").replace(TASK_BOX, ""))?.[2]?.trim();
+  return label === undefined || isCodeOnly(label) ? undefined : label;
+};
+
+/** Whether the line holding `offset` is quoted (`> - **a**：…`): someone else's words, not the writer's to fix. */
+export const isQuotedAt = (source: string, offset: number): boolean => source.slice(source.lastIndexOf("\n", offset - 1) + 1, offset).includes(">");
+
+/** The list items, in document order, that open with a bold label and are not quoted. */
+export const boldLabelItems = (doc: ProseDocument): Span[] =>
+  doc.lists
+    .flatMap((list) => list.itemSpans)
+    .filter((span) => !isQuotedAt(doc.source, span.start) && boldLabelOf(doc.source.slice(span.start, span.end)) !== undefined)
+    .toSorted((left, right) => left.start - right.start);
+
+/** A count, not a density: a person writes one or two labels even in a long article. */
+export const boldLabelList: Detector = (doc, options): Finding[] => {
+  const items = boldLabelItems(doc);
+  const first = items[0];
+  if (first === undefined || items.length < options.limit) return [];
+  const written = doc.source.slice(first.start, first.end);
+  return [
+    {
+      rule: "bold-label-list",
+      severity: "info",
+      line: 0,
+      column: 0,
+      quote: (written.split("\n")[0] ?? "").trim(),
+      values: { label: boldLabelOf(written) ?? "", count: items.length, limit: options.limit, offset: first.start },
+    },
+  ];
+};

package/src/detectors/broken-link.ts

@@ -0,0 +1,90 @@
+import type { Detector, Finding, Markup, ProseDocument, Span } from "../plugin.ts";
+import { findingAt, markupOf, quoteOf } from "./markup-finding.ts";
+
+/** ページの先頭を指す書き方。見出しが無くても行き先がある。 */
+const PAGE_TOP = new Set(["", "top"]);
+
+/** 名前を比べる形。字と数字だけを小文字で残す。見出しから名前を作る決まりは表示するものごとに違う（空白を - にする・記号を消す）ので、その違いを畳む。 */
+export const anchorKey = (name: string): string => name.toLowerCase().replace(/[^\p{L}\p{N}]/gu, "");
+
+/** 同じ見出しが二つあると、表示するものは二つ目以降に -1、-2 を付ける。 */
+const DUPLICATE_SUFFIX = /-(\d+)$/u;
+
+const decoded = (fragment: string): string => {
+  try {
+    return decodeURIComponent(fragment);
+  } catch {
+    return fragment;
+  }
+};
+
+/** 「&」を and と読んで名前を作るもの（M&IE → m-and-ie）もある。 */
+const namesOf = (text: string): string[] => [...new Set([text, text.replaceAll("&", " and ")])];
+
+const keysOf = (name: string): string[] => [...new Set(namesOf(name).map(anchorKey))];
+
+/**
+ * 文書の中の名前を持つところ。headings は見出しから作る名前の形ごとの数（-1、-2 は見出しだけに付く）。
+ * ids は書き手が付けた名前そのもの。見出しと違って作り方の違いが無いので、書いたとおりに比べる。
+ */
+export type Anchors = { readonly ids: ReadonlySet<string>; readonly headings: ReadonlyMap<string, number> };
+
+export const anchorsOf = (markup: Markup): Anchors => ({
+  ids: markup.ids,
+  headings: markup.headings
+    .flatMap((heading) => keysOf(heading.text))
+    .reduce((counts, key) => counts.set(key, (counts.get(key) ?? 0) + 1), new Map<string, number>()),
+});
+
+/** `#setup-2` は、同じ形の見出しが 3 つ以上あるときだけ行き先がある（二つ目が -1）。 */
+const reachesDuplicate = (fragment: string, headings: ReadonlyMap<string, number>): boolean => {
+  const suffix = DUPLICATE_SUFFIX.exec(fragment);
+  if (suffix === null) return false;
+  return (headings.get(anchorKey(fragment.slice(0, suffix.index))) ?? 0) > Number(suffix[1]);
+};
+
+/** `#…` の行き先が、文書の中の見出しか id を指しているか。 */
+export const reachesAnchor = (destination: string, anchors: Anchors): boolean => {
+  const fragment = decoded(destination.slice(1));
+  if (PAGE_TOP.has(fragment.toLowerCase())) return true;
+  return anchors.ids.has(fragment) || anchors.headings.has(anchorKey(fragment)) || reachesDuplicate(fragment, anchors.headings);
+};
+
+/**
+ * 参照の形で書いたのに名前の定義（`[label]: url`）が無いもの（`[text][label]`、`[text][]`）。字のまま表示される。
+ * 定義があれば CommonMark はリンクにするので、字のまま残った参照の形は定義が無い。英数字のすぐ後ろの括弧（A[i][j]）は配列の添字。
+ */
+const REFERENCE_LIKE = /(?<![\\A-Za-z0-9_\]])\[([^[\]\n]+)\]\[([^[\]\n]*)\]/gu;
+
+/** 数だけの括弧（[1][2]、[3, 4]）は、参照の記法ではなく文献の番号を並べたもの。 */
+const CITATION_NUMBERS = /^[\d\s,，、–-]+$/u;
+
+/** 画像の参照（`![alt][label]`）の頭の印。指摘の範囲に含める。 */
+const IMAGE_MARK = "!";
+
+export const unresolvedReferences = (source: string, markup: Markup): Span[] =>
+  markup.texts.flatMap((text) =>
+    [...source.slice(text.start, text.end).matchAll(REFERENCE_LIKE)]
+      .filter((match) => !CITATION_NUMBERS.test(match[1] ?? ""))
+      .map((match) => ({ start: text.start + match.index, end: text.start + match.index + match[0].length }))
+      .map((span) => (source.charAt(span.start - 1) === IMAGE_MARK ? { start: span.start - 1, end: span.end } : span)),
+  );
+
+const emptyLinks = (doc: ProseDocument, markup: Markup): Finding[] =>
+  markup.links.filter((link) => link.destination.trim() === "").map((link) => findingAt(doc, link, { link: quoteOf(doc.source, link) }, "empty"));
+
+const missingAnchors = (doc: ProseDocument, markup: Markup): Finding[] => {
+  const anchors = anchorsOf(markup);
+  return markup.links
+    .filter((link) => link.destination.startsWith("#") && !reachesAnchor(link.destination, anchors))
+    .map((link) => findingAt(doc, link, { link: quoteOf(doc.source, link), target: link.destination }, "anchor"));
+};
+
+export const brokenLink: Detector = (doc): Finding[] => {
+  const markup = markupOf(doc);
+  if (markup === undefined) return [];
+  const references = unresolvedReferences(doc.source, markup).map((span) => findingAt(doc, span, { link: quoteOf(doc.source, span) }, "reference"));
+  return [...emptyLinks(doc, markup), ...missingAnchors(doc, markup), ...references].toSorted(
+    (left, right) => Number(left.values["offset"]) - Number(right.values["offset"]),
+  );
+};

package/src/detectors/cited-name.ts

@@ -1,13 +1,14 @@
 import type { StructureNode, Token } from "../plugin.ts";
+import { preOrder } from "../tree-walk.ts";
 
 /** 他の文書を指す参照（民法第709条の第709条）の始まりの位置から、その文書の名前（民法）へ。 */
 export type CitedNames = ReadonlyMap<number, string>;
 
-const namedReferences = (node: StructureNode): (readonly [number, string])[] => {
-  const document = node.attrs["document"];
-  const own: (readonly [number, string])[] = node.kind === "reference" && typeof document === "string" ? [[node.span.start, document]] : [];
-  return [...own, ...node.children.flatMap(namedReferences)];
-};
+const namedReferences = (root: StructureNode): (readonly [number, string])[] =>
+  preOrder(root).flatMap((node) => {
+    const document = node.attrs["document"];
+    return node.kind === "reference" && typeof document === "string" ? [[node.span.start, document] as const] : [];
+  });
 
 export const citedNamesOf = (structure: StructureNode | undefined): CitedNames => new Map(structure === undefined ? [] : namedReferences(structure));