chaffjs 0.34.0 → 0.36.0

package/src/detectors/subject-name-line.ts

@@ -0,0 +1,66 @@
+// A line under a heading that gives the name of what the heading is about, not a sentence restating it: the label of a
+// link or an attachment the heading introduces (### 実施要綱について → 実施要綱), or a list named after the heading
+// (## 皆さんの声 → 「皆さんの声」一覧). A converted page keeps such labels as plain lines. Pure.
+import { isContent } from "./content-morphemes.ts";
+import type { Token } from "../plugin.ts";
+
+/** A title is made of these: names, the words that build a name (annual, の, of, and), no predicate. */
+const TITLE_POS: ReadonlySet<string> = new Set(["NOUN", "PROPN", "NUM", "PUNCT", "SYM", "X", "ADJ", "ADP", "DET", "CCONJ"]);
+/** A line that ends like a sentence says something, whatever its words. */
+const SENTENCE_END = /[。．.!?！？]$/u;
+
+const isTitle = (tokens: readonly Token[]): boolean => tokens.some(isContent) && tokens.every((token) => TITLE_POS.has(token.pos));
+
+const baseForm = (token: Token): string => (token.lemma ?? token.surface).toLowerCase();
+
+/** Brackets a line quotes a title in. */
+const QUOTES: readonly (readonly [string, string])[] = [
+  ["「", "」"],
+  ["『", "』"],
+  ["“", "”"],
+  ['"', '"'],
+];
+
+type Span = { readonly start: number; readonly end: number };
+
+/** Where the line quotes the whole heading as a title, in the tokens' coordinates. */
+const quotedHeading = (heading: string, line: string, lineStart: number): Span | undefined => {
+  const quoted = QUOTES.map(([open, close]) => `${open}${heading}${close}`).find((form) => line.includes(form));
+  if (quoted === undefined) return undefined;
+  const start = lineStart + line.indexOf(quoted);
+  return { start, end: start + quoted.length };
+};
+
+/** 「見出し」一覧: the heading quoted as a title, and only a name after it that says what the line is (一覧, page). */
+const namesListAfterHeading = (outside: readonly Token[]): boolean => isTitle(outside);
+
+/**
+ * 実施要綱 under ### 実施要綱について: the line is a title, and what the heading has beyond it only frames the name (について,
+ * About, 紹介). A line that copies the heading whole (### 研究科 → 研究科) is a repeat, and one that drops a word of the
+ * heading's own (### 精算方法の概要 → 精算方法) does not name the heading's subject.
+ */
+const namesHeadingTopic = (heading: readonly Token[], line: readonly Token[], announcing: ReadonlySet<string>): boolean => {
+  if (!isTitle(line)) return false;
+  const named = new Set(line.map(baseForm));
+  const beyond = heading.filter((token) => token.pos !== "PUNCT" && !named.has(baseForm(token)));
+  return beyond.length > 0 && beyond.every((token) => !isContent(token) || announcing.has(baseForm(token)));
+};
+
+/**
+ * The first line under a heading names what the heading is about. `heading` is the heading's text without its number
+ * label, `headingTokens` its tokens; `lineTokens` are the line's, in the coordinates `lineStart` gives. `announcing`
+ * holds the words that only frame a subject (the heading-announcer lexicon's ungrouped entries).
+ */
+export const namesHeadingSubject = (
+  heading: string,
+  headingTokens: readonly Token[],
+  line: string,
+  lineStart: number,
+  lineTokens: readonly Token[],
+  announcing: ReadonlySet<string>,
+): boolean => {
+  if (line.trim().includes("\n") || SENTENCE_END.test(line.trim()) || heading.trim() === "") return false;
+  const quoted = quotedHeading(heading.trim(), line, lineStart);
+  if (quoted !== undefined) return namesListAfterHeading(lineTokens.filter((token) => token.span.start < quoted.start || token.span.start >= quoted.end));
+  return namesHeadingTopic(headingTokens, lineTokens, announcing);
+};

package/src/detectors/table-header.ts

@@ -53,7 +53,21 @@ const rawCells = (line: Line): { readonly raw: string; readonly from: number }[]
   return inner.at(-1)?.raw.trim() === "" ? inner.slice(0, -1) : inner;
 };
 
-const labelCell = (line: Line, { raw, from }: { readonly raw: string; readonly from: number }): HeaderCell[] => {
+/**
+ * An inline HTML tag in a cell (`<div style="…">`, `</div>`, `<br/>`): the reader sees none of it. A tag name ends at a space,
+ * a slash or `>`, so an autolink (`<https://…>`) is not one; an attribute in double quotes may hold a `>`.
+ */
+const HTML_TAG = /<\/?[A-Za-z][\w-]*(?:\s(?:"[^"]*"|[^<>"])*)?\/?>/gu;
+
+/**
+ * The cell with its HTML tags blanked out, same length, so the words keep their offsets and the spaces around them trim.
+ * A cell with a code span is left whole: a tag written in code is text the reader sees.
+ */
+const withoutTags = (raw: string): string => (raw.includes("`") ? raw : raw.replace(HTML_TAG, (tag) => " ".repeat(tag.length)));
+
+const labelCell = (line: Line, cell: { readonly raw: string; readonly from: number }): HeaderCell[] => {
+  const { from } = cell;
+  const raw = withoutTags(cell.raw);
   const trimmed = raw.trim();
   const unwrapped = unwrap(trimmed);
   const written = unwrapped.replace(INNER_MARKS, "").trim();

package/src/detectors/technical-docs.ts

@@ -6,6 +6,7 @@ import { codeFences, codeSpans, type CodeFence } from "./code-fences.ts";
 import { opensAsNounPhrase } from "./nominal-step.ts";
 import { quoteAt } from "./structure-tree.ts";
 import { escapeRegExp } from "../orthography.ts";
+import { isFormatTemplate } from "../format-template.ts";
 import type { Detector, Finding, ProseDocument, Span, Token } from "../plugin.ts";
 
 const patternsOf = (doc: ProseDocument, id: string): string[] => (doc.lexicons[id] ?? []).map((entry) => entry.pattern);
@@ -25,6 +26,9 @@ const finding = (doc: ProseDocument, rule: string, offset: number, values: Reado
   values: { ...values, offset },
 });
 
+/** A fence that should name a language: an untagged format template (<MAJOR>.<MINOR>) is in none, so it is left out. */
+const wantsLanguage = (fence: CodeFence): boolean => fence.language !== "" || !isFormatTemplate(fence.lines);
+
 /** Fences without a language, when they are the fewer kind: a document that mostly leaves the language out is not reported. */
 export const untaggedFewer = (fences: readonly CodeFence[], limit: number): readonly CodeFence[] => {
   const untagged = fences.filter((fence) => fence.language === "");
@@ -33,7 +37,7 @@ export const untaggedFewer = (fences: readonly CodeFence[], limit: number): read
 };
 
 export const codeFenceLanguage: Detector = (doc, options): Finding[] => {
-  const fences = codeFences(doc.source);
+  const fences = codeFences(doc.source).filter(wantsLanguage);
   const tagged = fences.filter((fence) => fence.language !== "").length;
   return untaggedFewer(fences, options.limit).map((fence) =>
     finding(doc, "code-fence-language-mix", fence.start, { tagged, untagged: fences.length - tagged }),

package/src/detectors/template-sections.ts

@@ -0,0 +1,37 @@
+import type { Lexicon, MarkupHeading } from "../plugin.ts";
+import { escapeRegExp } from "../orthography.ts";
+
+type Span = { readonly start: number; readonly end: number };
+
+/** Lexicon group of placeholder-word: a word that, in a heading, says what follows is a template (Template, 雛形). */
+export const TEMPLATE = "template";
+
+const LATIN = /\p{Script=Latin}/u;
+
+/** Does the character beside a match end the word? A Latin edge needs a non-letter next to it; a CJK word stands anywhere. */
+const boundsWord = (edge: string, beside: string | undefined): boolean => !LATIN.test(edge) || beside === undefined || !/[\p{L}\p{N}]/u.test(beside);
+
+const holdsWordAt = (text: string, pattern: string, index: number): boolean =>
+  boundsWord(pattern[0] ?? "", text[index - 1]) && boundsWord(pattern.at(-1) ?? "", text[index + pattern.length]);
+
+/** The heading names a template: one of the words appears in it as a whole word, in any case. */
+export const namesTemplate = (heading: string, words: Lexicon): boolean => {
+  const text = heading.toLowerCase();
+  return words.some((word) => {
+    const pattern = word.pattern.toLowerCase();
+    return [...text.matchAll(new RegExp(escapeRegExp(pattern), "gu"))].some((match) => holdsWordAt(text, pattern, match.index));
+  });
+};
+
+/** A heading's section runs from the heading to the next one at its depth or above; text before a template title stays checked. */
+const sectionOf = (headings: readonly MarkupHeading[], index: number, documentEnd: number): Span => {
+  const heading = headings[index];
+  const next = headings.slice(index + 1).find((later) => later.depth <= (heading?.depth ?? 0));
+  return { start: heading?.start ?? 0, end: next?.start ?? documentEnd };
+};
+
+/** The parts of a document under a heading that names a template: their blanks are the point, not a slip. */
+export const templateSections = (headings: readonly MarkupHeading[], documentEnd: number, words: Lexicon): Span[] =>
+  words.length === 0 ? [] : headings.flatMap((heading, index) => (namesTemplate(heading.text, words) ? [sectionOf(headings, index, documentEnd)] : []));
+
+export const isInside = (spans: readonly Span[], offset: number): boolean => spans.some((span) => span.start <= offset && offset < span.end);

package/src/detectors/term-mention-sentence.ts

@@ -0,0 +1,31 @@
+import { escapeRegExp } from "../orthography.ts";
+import type { Sentence } from "../plugin.ts";
+
+// An entry of a definitions list opens with the term it defines, in quotes ("Capital Contributions" The capital
+// contribution to the Venture …). The words after the quoted name are the definition's own body, not uses of the term.
+// Pure.
+
+/** The ending a quoted name may carry past the term: a plural or a possessive (Contributions, Contribution's). */
+const ENDING = String.raw`(?:e?s|['’]s)?`;
+/** The label of a list item before the entry (a., (b), 1., iv)). */
+const ITEM_LABEL = String.raw`(?:\(?[\p{L}\p{N}]{1,4}[.)]\s+)?`;
+/** The quotes a term is named in, each with its own closing mark. */
+const QUOTE_PAIRS = [
+  ['"', '"'],
+  ["“", "”"],
+  ["「", "」"],
+  ["『", "』"],
+] as const;
+
+const entryHead = (term: string): RegExp => {
+  const quoted = QUOTE_PAIRS.map(([open, close]) => `${open}${escapeRegExp(term)}${ENDING}${close}`).join("|");
+  return new RegExp(`^\\s*${ITEM_LABEL}(?:${quoted})`, "u");
+};
+
+/** Whether the sentence holding offset is a definitions entry that opens with the term in quotes, and offset is after that name. */
+export const inEntryOfTerm = (sentences: readonly Sentence[], term: string, offset: number): boolean => {
+  const sentence = sentences.find((candidate) => candidate.span.start <= offset && offset < candidate.span.end);
+  if (sentence === undefined) return false;
+  const head = entryHead(term).exec(sentence.text);
+  return head !== null && offset - sentence.span.start >= head[0].length;
+};

package/src/detectors/titled-run.ts

@@ -0,0 +1,48 @@
+import type { Token } from "../plugin.ts";
+import { quotedSpans } from "../quoted-span.ts";
+
+/** 題の付いたものの名前を閉じる語（語彙表 title-ending: 〜要綱、〜試験、〜事業、〜法）。 */
+export type TitleWords = { readonly endings: readonly string[] };
+
+/** 題を囲む括弧。鉤括弧と、見出しや分類の名前を囲む隅付き括弧。 */
+const TITLE_MARKS = { "「": "」", "『": "』", "【": "】" };
+/**
+ * 名前のすぐ後ろ（同じ行の空白を挟んでもよい）に付く、その名前の補い（就職氷河期世代、＜創業型＞）、略称（以下「奨励金」という。）、
+ * 日付や番号、ファイルの種類と大きさ（[PDFファイル／333KB]）の括弧。角括弧は同じ行で閉じ、Markdown のリンク（[文字](…)）でないもの。
+ * 半角の < は HTML の札と見分けられないので読まない。
+ */
+const QUALIFIER = /^[ \t\u3000]*(?:[（(＜〈［]|\[[^\]\n]*\](?![([]))/u;
+/** 閉じる語の前に要る字数。1 字なら、題ではなく普通の語（新法、本事業）。 */
+const MIN_BODY_LENGTH = 2;
+const BLANK = /^\s*$/u;
+
+const surfaceOf = (tokens: readonly Token[]): string => tokens.map((token) => token.surface).join("");
+
+/**
+ * 連なりの語（run: 隙間なく並ぶ語）が、題の名前を閉じる語で終わるか。閉じる語は語の切れ目で始まる
+ * （地方自治法の「法」は語、解決方法の「法」は方法の一部）。閉じる語の前に 2 字以上の語が要る。
+ */
+export const endsInTitle = (run: readonly Token[], words: TitleWords): boolean =>
+  run.some((_, at) => {
+    const tail = surfaceOf(run.slice(at));
+    return words.endings.includes(tail) && [...surfaceOf(run.slice(0, at))].length >= MIN_BODY_LENGTH;
+  });
+
+/**
+ * 文（text）の start から end までの連なりが、題の名前の置き方をしているか。
+ * 括弧の中身の終わりで閉じる（「令和8年度高度統計人材育成強化拠点形成事業」）、すぐ後ろに補いの括弧が付く
+ * （国家公務員中途採用者選考試験（就職氷河期世代）、持続化補助金＜創業型＞、登録申込書 [PDFファイル／333KB]）、
+ * または文がその連なりだけ（要綱の題を並べた行）。
+ */
+export const isTitlePosition = (text: string, start: number, end: number): boolean => {
+  const closesBracket = quotedSpans(text, TITLE_MARKS).some((span) => span.end === end && span.start <= start);
+  const qualified = QUALIFIER.test(text.slice(end));
+  const alone = BLANK.test(text.slice(0, start)) && BLANK.test(text.slice(end));
+  return closesBracket || qualified || alone;
+};
+
+/**
+ * 文書の中で題として置かれた名前（titled: 題の置き方をした連なりと、見出しの中の漢字の連なり）に、
+ * この連なりの字（text）が名前の終わりとして含まれるか。同じ名前は、文の中で使っても名前（認定職業訓練実施奨励金の支給額）。
+ */
+export const isTitledName = (text: string, titled: readonly string[]): boolean => titled.some((name) => name.endsWith(text));

package/src/detectors/token-shape.ts

@@ -1,10 +1,12 @@
 import { hasParticle, isClosed, lastContent } from "../sentence-shape.ts";
 import { isWithinAny, quotedIn } from "../quoted-span.ts";
+import { chronologyRows } from "../chronology-rows.ts";
+import { externalLinkTitleSpans } from "../link-title.ts";
 import type { Detector, Finding, Span, Token } from "../plugin.ts";
 
 /**
  * 文末が名詞で終わる（体言止め）。箇条書きでは普通だが、本文では文が途中で切れて読める。
- * 実文書で測ったら、指摘の 9 割が箇条書きの項目だった。箇条書きは除く。
+ * 実文書で測ったら、指摘の 9 割が箇条書きの項目だった。箇条書きは除く。年表の行（1950年4月 設立。）も名詞で止める書き方なので除く。
  */
 const NOMINAL = new Set(["NOUN", "PROPN"]);
 
@@ -14,8 +16,9 @@ const isDependent = (token: Token): boolean => token.features?.["NounType"] ===
 const inside = (span: Span, spans: readonly Span[]): boolean => spans.some((other) => span.start >= other.start && span.start < other.end);
 
 export const nounEnding: Detector = (doc, options): Finding[] => {
+  const rows = chronologyRows(doc);
   const ended = doc.sentences.filter((sentence) => {
-    if (inside(sentence.span, doc.listSpans) || !isClosed(sentence) || !hasParticle(sentence)) return false;
+    if (inside(sentence.span, doc.listSpans) || rows.has(sentence) || !isClosed(sentence) || !hasParticle(sentence)) return false;
     const last = lastContent(sentence);
     return last !== undefined && NOMINAL.has(last.pos) && !isDependent(last);
   });
@@ -62,19 +65,25 @@ const BREAK: Run = { surface: "", tokens: [] };
  *
  * 鉤括弧で引いたもの（括弧ごと）は、名詞と同じく連なりの中身として飛ばす。引いた題名（「食品中のウイルスの制御のための…ガイドライン」）や
  * 発言は書き手が言い換えられないので、中の助詞は数えない。題名も名前なので、外の連なり（弊社の「新製品」の販売の計画）は続けて数える。
+ * 外のページの題名を引いたリンクの文字（- [サービスの料金の計算の方法](https://…)）も、引いた題名と同じに読む。
  */
-const runsOf = (tokens: readonly Token[], nesting: readonly string[], quoted: readonly Span[]): Run[] =>
+const runsOf = (tokens: readonly Token[], nesting: readonly string[], breaks: readonly string[], quoted: readonly Span[]): Run[] =>
   tokens.reduce<Run[]>((acc, token) => {
     if (isWithinAny(quoted, token.span)) return acc;
     if (nesting.includes(token.surface)) return extend(acc, token);
-    if (token.pos === "PUNCT" || PARTICLE.has(token.pos)) acc.push(BREAK);
+    if (token.pos === "PUNCT" || PARTICLE.has(token.pos) || breaks.includes(token.surface)) acc.push(BREAK);
     return acc;
   }, []);
 
+/** 句と句の間の記号（制限=タグの制限、A→B）。半角の = や : は名詞として読まれ、句読点のようには切れない。 */
+const RUN_BREAK_LEXICON = "particle-run-break";
+
 export const doubledParticle: Detector = (doc, options): Finding[] => {
   const nesting = (options.lexicon ?? []).map((entry) => entry.pattern);
+  const titles = externalLinkTitleSpans(doc.source, doc.links);
+  const breaks = (doc.lexicons[RUN_BREAK_LEXICON] ?? []).map((entry) => entry.pattern);
   return doc.sentences.flatMap((sentence) =>
-    runsOf(sentence.tokens ?? [], nesting, quotedIn(sentence))
+    runsOf(sentence.tokens ?? [], nesting, breaks, [...quotedIn(sentence), ...titles])
       .filter((run) => run.tokens.length > options.limit)
       .map((run) => ({
         rule: "no-doubled-joshi",

package/src/detectors/unbalanced-bracket.ts

@@ -5,6 +5,7 @@ import { quoteAround } from "./quote-around.ts";
 import { mergeSpans } from "../span-merge.ts";
 import { firstEndingAfter } from "../soft-break.ts";
 import { isLabelClose, isLabelInAside } from "./bracket-label.ts";
+import { isInchMark } from "./inch-mark.ts";
 
 /**
  * 括弧の組。どの言語でも同じ字の決まりなので、語彙表ではなくここに置く。family は形（丸・角）で、全角と半角は同じ形の別の幅。
@@ -93,6 +94,8 @@ export const bracketProblems = (text: string): BracketProblem[] => {
   const scan: Scan = { open: [], problems: [] };
   const heads = paragraphHeads(text);
   [...text.matchAll(MARKS)].forEach((match) => {
+    const quoteOpen = scan.open.some((entry) => entry.bracket.family === "quote");
+    if (isInchMark(text, match.index, quoteOpen)) return;
     const opening = BY_OPEN.get(match[0]);
     const closing = BY_CLOSE.get(match[0]);
     if (opening !== undefined) openWith(scan, opening, match.index, heads.has(match.index));

package/src/detectors/undefined-term.ts

@@ -91,6 +91,8 @@ const findingOf = (doc: ProseDocument, use: UndefinedTerm): Finding => ({
 });
 
 const NAME_LISTS = ["proper-name-word", "place-region", "calendar-name", "month-name", "company-form", "party-role"] as const;
+/** The lists whose words end a name right after a comma (Post Holdings, Inc.; Canton, Ohio). */
+const FORM_ENDS = ["company-form", "place-region"] as const;
 
 /** Capitalised terms, for a language whose package lists the words that join one name (name-joiner); Japanese has none. */
 const capitalisedUses = (doc: ProseDocument, sentences: readonly (readonly Token[])[], defined: readonly Defined[]): UndefinedTerm[] => {
@@ -99,6 +101,12 @@ const capitalisedUses = (doc: ProseDocument, sentences: readonly (readonly Token
     names: NAME_LISTS.flatMap((id) => patternsOf(doc, id)),
     divisions: patternsOf(doc, "numbered-division"),
     joiners: patternsOf(doc, "name-joiner"),
+    nameEnds: patternsOf(doc, "proper-name-word"),
+    formEnds: FORM_ENDS.flatMap((id) => patternsOf(doc, id)),
+    definitionMarkers: patternsOf(doc, "definition-marker"),
+    definedElsewhere: patternsOf(doc, "defined-elsewhere"),
+    capacityMarkers: patternsOf(doc, "capacity-marker"),
+    articles: patternsOf(doc, "article"),
   };
   const marked = [...(doc.markup?.headings ?? []), ...doc.links, ...setApartSpans(doc.source)];
   return capitalisedTerms({ source: doc.source, readable: readableText(doc.source), sentences, defined, named: namedWords(doc.source), marked, words });

package/src/detectors/unfilled-placeholder.ts

@@ -0,0 +1,10 @@
+import type { Detector } from "../plugin.ts";
+import { unfilledPlaceholder } from "./ai-phrasing.ts";
+import { TEMPLATE, isInside, templateSections } from "./template-sections.ts";
+
+/** A blank under a heading that names a template (Template, 雛形) is what the template is for, so it is not reported. */
+export const unfilledPlaceholderOutsideTemplates: Detector = (doc, options) => {
+  const words = (options.lexicon ?? []).filter((word) => word.group === TEMPLATE);
+  const templates = templateSections(doc.markup?.headings ?? [], doc.source.length, words);
+  return unfilledPlaceholder(doc, options).filter((finding) => !isInside(templates, Number(finding.values["offset"])));
+};

package/src/detectors/vague-density.ts

@@ -0,0 +1,84 @@
+// Vague words dense for the document's length (vague-word-density). Each sentence counts the vague words it holds once:
+// a word inside a longer one ("timely" in "in a timely manner") is that longer word, and a word inside a term of art of the
+// document's genre ("reasonably" in a contract, "adequately compensate" as the test for an injunction) is not vague
+// there. The rule speaks once per sentence, about the document's density. Pure: the words come from the language's
+// lexicons vague-word and vague-word-term-of-art, the genre from the options.
+import { entriesForGenre } from "../lexicon-genres.ts";
+import { escapeRegExp } from "../orthography.ts";
+import type { Detector, Finding, Lexicon, LexiconEntry, Sentence } from "../plugin.ts";
+import { comparableText, comparableWords, entryRanges } from "./lexicon-match.ts";
+import { rateOver } from "./lexicon.ts";
+import { specifiedByBracket } from "./bracket-specified.ts";
+
+const TERMS_OF_ART = "vague-word-term-of-art";
+/** The words that start an aside giving a name (以下「HR」という, hereinafter): such an aside pins nothing down. */
+const NAME_MARKERS = "definition-marker";
+
+type Place = { readonly start: number; readonly end: number };
+
+type Match = { readonly pattern: string; readonly place: Place };
+
+/** Token spans are places in the document and text places are in the sentence, so one sentence uses one kind for every entry. */
+const readsTokens = (sentence: Sentence, entries: Lexicon): boolean =>
+  sentence.tokens !== undefined && entries.every((entry) => (entry.tokens?.length ?? 0) > 0);
+
+/** Where an entry stands in the sentence, as character spans: its tokens, or the comparable text. */
+const placesOf = (sentence: Sentence, entry: LexiconEntry, byTokens: boolean): Place[] => {
+  const tokens = sentence.tokens;
+  if (byTokens && tokens !== undefined)
+    return entryRanges(sentence, entry).map((range) => ({ start: tokens[range.start]?.span.start ?? 0, end: tokens[range.end - 1]?.span.end ?? 0 }));
+  const word = comparableWords(entry.pattern);
+  if (word === "") return [];
+  return [...comparableText(sentence).matchAll(new RegExp(escapeRegExp(word), "gu"))].map((found) => ({ start: found.index, end: found.index + word.length }));
+};
+
+const matchesOf = (sentence: Sentence, lexicon: Lexicon, byTokens: boolean): Match[] =>
+  lexicon.flatMap((entry) => placesOf(sentence, entry, byTokens).map((place) => ({ pattern: entry.pattern, place })));
+
+const covers = (outer: Place, inner: Place): boolean => outer.start <= inner.start && inner.end <= outer.end;
+
+const isLonger = (outer: Match, inner: Match): boolean => outer.place.end - outer.place.start > inner.place.end - inner.place.start;
+
+/** Whether a bracket right after the match pins the word down; token places are in the document, text places in the comparable text. */
+const pinnedDown = (sentence: Sentence, match: Match, byTokens: boolean, nameMarkers: readonly string[]): boolean =>
+  byTokens
+    ? specifiedByBracket(sentence.text, match.place.end - sentence.span.start, nameMarkers)
+    : specifiedByBracket(comparableText(sentence), match.place.end, nameMarkers);
+
+/**
+ * The vague words of one sentence, in the order written, each once: none inside a longer vague word or a term of art,
+ * and none pinned down by the bracket right after it (一定の事項（…）). nameMarkers: the words that start a name in an aside.
+ */
+export const vagueWordsIn = (sentence: Sentence, vague: Lexicon, termsOfArt: Lexicon, nameMarkers: readonly string[] = []): string[] => {
+  const byTokens = readsTokens(sentence, [...vague, ...termsOfArt]);
+  const matches = matchesOf(sentence, vague, byTokens);
+  const terms = matchesOf(sentence, termsOfArt, byTokens);
+  const kept = matches.filter(
+    (match) =>
+      !matches.some((other) => isLonger(other, match) && covers(other.place, match.place)) &&
+      !terms.some((term) => covers(term.place, match.place)) &&
+      !pinnedDown(sentence, match, byTokens, nameMarkers),
+  );
+  return [...new Set(kept.toSorted((left, right) => left.place.start - right.place.start).map((match) => match.pattern))];
+};
+
+type SentenceHits = { readonly sentence: Sentence; readonly words: readonly string[] };
+
+export const vagueDensity: Detector = (doc, options): Finding[] => {
+  const termsOfArt = entriesForGenre(doc.lexicons[TERMS_OF_ART] ?? [], options.genre);
+  const nameMarkers = (doc.lexicons[NAME_MARKERS] ?? []).map((entry) => entry.pattern);
+  const hits: SentenceHits[] = doc.sentences
+    .map((sentence) => ({ sentence, words: vagueWordsIn(sentence, options.lexicon ?? [], termsOfArt, nameMarkers) }))
+    .filter((hit) => hit.words.length > 0);
+  const count = hits.reduce((sum, hit) => sum + hit.words.length, 0);
+  const density = rateOver(doc, count, options.limit);
+  if (density === undefined) return [];
+  return hits.map(({ sentence, words }) => ({
+    rule: "vague-word-density",
+    severity: "warning",
+    line: 0,
+    column: 0,
+    quote: sentence.text.trim(),
+    values: { matched: words[0] ?? "", count, density, limit: options.limit, offset: sentence.span.start },
+  }));
+};

package/src/document-paragraphs.ts

@@ -0,0 +1,27 @@
+import { documentLineParagraphs } from "./line-paragraphs.ts";
+import { listItemParagraphs } from "./list-item-paragraphs.ts";
+import type { Paragraph, Sentence, Span } from "./plugin.ts";
+
+/** sentences（並び順）の中で、start が offset 以上の最初の添字。1 行 1 段落の何万行でも、段落ごとに全部をなめない。 */
+const firstStartingAt = (sentences: readonly Sentence[], offset: number): number => {
+  const search = (low: number, high: number): number => {
+    if (low >= high) return low;
+    const middle = (low + high) >> 1;
+    return (sentences[middle]?.span.start ?? Number.POSITIVE_INFINITY) >= offset ? search(low, middle) : search(middle + 1, high);
+  };
+  return search(0, sentences.length);
+};
+
+const sentencesWithin = (sentences: readonly Sentence[], span: Span): Sentence[] =>
+  sentences.slice(firstStartingAt(sentences, span.start), firstStartingAt(sentences, span.end));
+
+/** 文書の段落と、その中の文。spans は Markdown の段落、listSpans は箇条書きの項目と同じに読む範囲。 */
+export const paragraphsOf = (prose: string, spans: readonly Span[], sentences: readonly Sentence[], listSpans: readonly Span[]): Paragraph[] => {
+  // 箇条書きの中の段落は「段落」として数えない。項目 1 つを 1 段落と読むと、
+  // 段落あたりの文数も長さのばらつきも、箇条書きの多い文書で壊れる。
+  const outsideLists = spans.filter((span) => !listSpans.some((list) => span.start >= list.start && span.start < list.end));
+  const withSentences = outsideLists.map((span) => ({ span, sentences: sentencesWithin(sentences, span).map((sentence) => sentence.span) }));
+  // 1 行 1 段落の書き方かは、行頭の印の項目で割る前の段落で決める。割った後の段落は、その手がかりにならない。
+  const items = documentLineParagraphs(prose, withSentences).flatMap((span) => listItemParagraphs(span, sentencesWithin(sentences, span)));
+  return items.map((span) => ({ span, sentences: sentencesWithin(sentences, span) }));
+};

package/src/document.ts

@@ -4,7 +4,7 @@ import { hasTitle } from "./heading-title.ts";
 import { maskSpans } from "./mask.ts";
 import { spansWithin, unmaskedSoftBreaks } from "./soft-break.ts";
 import { segmentJoined } from "./joined-view.ts";
-import { documentLineParagraphs } from "./line-paragraphs.ts";
+import { paragraphsOf } from "./document-paragraphs.ts";
 import { closingRun, standaloneLines, subheadingPieces, type StandsAlone } from "./subheading-line.ts";
 import { isLinkLine } from "./link-line.ts";
 import { speakerLabels } from "./speaker-labels.ts";
@@ -26,7 +26,7 @@ import { markdownFigures } from "./text-figures.ts";
 import { documentMarkup } from "./markup.ts";
 import { BARE_URL } from "./bare-url.ts";
 import { lazyCells, tableCellsOf } from "./table-cells.ts";
-import type { BulletList, LanguageAdapter, Markup, Paragraph, ProseDocument, Sentence, Span, StructureNode, DocumentProfile } from "./plugin.ts";
+import type { BulletList, LanguageAdapter, Markup, ProseDocument, Sentence, Span, StructureNode, DocumentProfile } from "./plugin.ts";
 
 /**
  * 本文として数えないもの。
@@ -234,30 +234,10 @@ const sentencesOf = (prose: string, paragraphs: readonly Span[], adapter: Langua
         ...(sentence.tokens === undefined ? {} : { tokens: sentence.tokens.map((token) => ({ ...token, span: shift(token.span, paragraph.start) })) }),
         ...(sentence.wrapBreaks === undefined ? {} : { wrapBreaks: sentence.wrapBreaks.map((span) => shift(span, paragraph.start)) }),
         ...(sentence.embeddedLanguage === undefined ? {} : { embeddedLanguage: sentence.embeddedLanguage }),
+        ...(sentence.listItem === undefined ? {} : { listItem: sentence.listItem }),
       })),
   );
 
-/** sentences（並び順）の中で、start が offset 以上の最初の添字。1 行 1 段落の何万行でも、段落ごとに全部をなめない。 */
-const firstStartingAt = (sentences: readonly Sentence[], offset: number): number => {
-  const search = (low: number, high: number): number => {
-    if (low >= high) return low;
-    const middle = (low + high) >> 1;
-    return (sentences[middle]?.span.start ?? Number.POSITIVE_INFINITY) >= offset ? search(low, middle) : search(middle + 1, high);
-  };
-  return search(0, sentences.length);
-};
-
-const sentencesWithin = (sentences: readonly Sentence[], span: Span): Sentence[] =>
-  sentences.slice(firstStartingAt(sentences, span.start), firstStartingAt(sentences, span.end));
-
-const paragraphsOf = (prose: string, spans: readonly Span[], sentences: readonly Sentence[], listSpans: readonly Span[]): Paragraph[] => {
-  // 箇条書きの中の段落は「段落」として数えない。項目 1 つを 1 段落と読むと、
-  // 段落あたりの文数も長さのばらつきも、箇条書きの多い文書で壊れる。
-  const outsideLists = spans.filter((span) => !listSpans.some((list) => span.start >= list.start && span.start < list.end));
-  const withSentences = outsideLists.map((span) => ({ span, sentences: sentencesWithin(sentences, span).map((sentence) => sentence.span) }));
-  return documentLineParagraphs(prose, withSentences).map((span) => ({ span, sentences: sentencesWithin(sentences, span) }));
-};
-
 /** 箇条書きは list ノードの直下の項目を数える。入れ子の項目は内側の list のものとして数える。目次のような案内だけの箇条書きは数えない。 */
 const listsOf = (root: Node, source: string, anchors: InPageAnchors): BulletList[] => {
   const found: BulletList[] = [];

package/src/facts/fact-scope.ts

@@ -118,20 +118,69 @@ const headCount = (heads: ReadonlyMap<string, readonly number[]>, from: number,
 
 type Lines = { readonly lines: readonly Line[]; readonly breakers: readonly number[]; readonly heads: ReadonlyMap<string, readonly number[]> };
 
-const scopeOf = (fact: Fact, innermost: StructureNode | undefined, at: Lines): { scope: string; record: boolean } => {
+/** 表の、見出しを繰り返した何番目の並びの升か（table-facts の columnGroup）。並びごとに別の範囲。 */
+type PlacedFact = Fact & { readonly columnGroup?: number };
+
+const scopeOf = (fact: PlacedFact, innermost: StructureNode | undefined, at: Lines): { scope: string; record: boolean } => {
   const index = lineIndexAt(at.lines, fact.value.start);
   const from = innermost === undefined ? 0 : lineIndexAt(at.lines, innermost.span.start);
   const to = innermost === undefined ? at.lines.length : lineIndexAt(at.lines, innermost.span.end - 1) + 1;
-  const scope = [innermost?.address ?? "doc", (at.breakers[index] ?? 0) - (at.breakers[from] ?? 0), parentItem(at.lines, index)].join("#");
+  const place = [innermost?.address ?? "doc", (at.breakers[index] ?? 0) - (at.breakers[from] ?? 0), parentItem(at.lines, index)];
+  const scope = [...place, ...(fact.columnGroup === undefined ? [] : [`g${fact.columnGroup}`])].join("#");
   return { scope, record: headCount(at.heads, from, to, fact.key) >= RECORD_REPEATS };
 };
 
+const MAX_PARAGRAPH_HEAD = 40;
+/** 文の終わりか続きの印。これで終わる行は小見出しではない。 */
+const SENTENCE_END = /[.!?。．！？:：;；,、，]$/u;
+const NOT_PLAIN_LINE = /^[ \t]*(?:[#>|]|```|~~~)/u;
+/** setext の見出しの下線。上の行は見出しで、節として木が分ける。 */
+const SETEXT_UNDERLINE = /^[ \t]*(?:=+|-+)[ \t]*$/u;
+
+/**
+ * 段落の頭の、短い名前だけの行（「Example」「Element 1 (standard rated portion)」「例」）。文の終わりの印も値も無く、項目や表の行でもない。
+ * 例や場合ごとに名前を付けた段落の頭で、続く値はその例の値。
+ */
+const isNameOnly = (line: string): boolean => {
+  const text = line.trim();
+  if (text === "" || text.length > MAX_PARAGRAPH_HEAD) return false;
+  return /\p{L}/u.test(text) && !SENTENCE_END.test(text) && !LIST_ITEM.test(line) && !NOT_PLAIN_LINE.test(line);
+};
+
+const isParagraphHead = (lines: readonly Line[], index: number, valueLines: ReadonlySet<number>): boolean => {
+  const startsParagraph = (lines[index - 1]?.text.trim() ?? "") === "";
+  const isSetextHeading = SETEXT_UNDERLINE.test(lines[index + 1]?.text ?? "");
+  return startsParagraph && !isSetextHeading && !valueLines.has(index) && isNameOnly(lines[index]?.text ?? "");
+};
+
+/** 小見出しの代わりの行の数に、段落の頭の名前だけの行も数える。 */
+const withParagraphHeads = (lines: readonly Line[], facts: readonly Fact[]): number[] => {
+  const valueLines = new Set(facts.map((fact) => lineIndexAt(lines, fact.value.start)));
+  return lines.reduce<number[]>(
+    (counts, line, index) => {
+      counts.push((counts[index] ?? 0) + (isBreaker(line) || isParagraphHead(lines, index, valueLines) ? 1 : 0));
+      return counts;
+    },
+    [0],
+  );
+};
+
+/** paragraphHeads: 段落の頭の名前だけの行（Example、例）でも範囲を分ける。 */
+export type ScopeReading = { readonly paragraphHeads?: boolean };
+
 /** 事実ごとに、比べてよい範囲と、文書のどの部分かを付ける。 */
-export const scopedFacts = (facts: readonly Fact[], tree: StructureNode, source: string, summaryWords: readonly string[]): ScopedFact[] => {
+export const scopedFacts = (
+  facts: readonly PlacedFact[],
+  tree: StructureNode,
+  source: string,
+  summaryWords: readonly string[],
+  reading: ScopeReading = {},
+): ScopedFact[] => {
   if (facts.length === 0) return [];
   const nodes = inDocumentOrder(tree).filter((node) => CONTAINERS.has(node.kind));
   const lines = linesOf(source);
-  const at: Lines = { lines, breakers: breakerCounts(lines), heads: headLines(lines) };
+  const breakers = reading.paragraphHeads === true ? withParagraphHeads(lines, facts) : breakerCounts(lines);
+  const at: Lines = { lines, breakers, heads: headLines(lines) };
   return facts
     .toSorted((left, right) => left.value.start - right.value.start)
     .map((fact) => {