chaffjs 0.35.0 → 0.37.0

package/src/structure/universal.ts

@@ -5,7 +5,7 @@ import { numberInSentence } from "./number-in-sentence.ts";
  * 「4.2 設定」「3.1.4 Scope」のような通し番号。数字と点だけで書くので言語を問わない。
  * 各部分を 3 桁・深さを 6 までに抑え、後戻りが爆発しない形にしてある。
  */
-const DOTTED = /^[ \t]{0,3}(?<number>\d{1,3}(?:\.\d{1,3}){0,5})(?<closing>[.．])?[ \t\u3000]+(?<rest>\S.*)$/u;
+const DOTTED = /^[ \t]{0,3}(?<number>\d{1,3}(?:\.\d{1,3}){0,5})(?<closing>[.．])?(?<gap>[ \t\u3000]+)(?<rest>\S.*)$/u;
 /**
  * 見出しで、点で閉じた番号に題を詰めて書いたもの（「7.委託」「7．委託」「2.Overview」）。後ろが数字なら小数（「1.5万人」）、
  * 小文字なら版（「2.x 系」）なので読まない。
@@ -26,8 +26,11 @@ const isTopSection = (line: string, rest: string): boolean => TOP_SECTION.test(l
 const dottedGroups = (line: string, context: NumberingContext): Readonly<Record<string, string | undefined>> | undefined =>
   DOTTED.exec(line)?.groups ?? (context.isHeading ? TIGHT_HEADING.exec(line)?.groups : undefined);
 
-/** closedByDot: 番号を点で閉じたか（「5. 」）。「1.5 万人」の 1.5 は閉じていない。 */
-type DottedLine = NumberedLine & { readonly closedByDot: boolean };
+/**
+ * closedByDot: 番号を点で閉じたか（「5. 」）。「1.5 万人」の 1.5 は閉じていない。wideGap: 番号と題の間に全角の空白を置いたか
+ * （「9（全角空白）年間収入階級」）。どちらも番号の札の書き方で、数量はそう書かない。
+ */
+type DottedLine = NumberedLine & { readonly closedByDot: boolean; readonly wideGap: boolean };
 
 /**
  * 本文の「1. 」は箇条書きで、見出しの「1. 」は章番号。点を含まない番号は、見出しの行か、テキストの章見出しの形でだけ読む。
@@ -41,20 +44,41 @@ export const dottedNumber = (line: string, context: NumberingContext, plainText
   const rest = groups?.["rest"]?.trim() ?? "";
   if (depth < 2 && !context.isHeading && !(plainText && isTopSection(line, rest))) return undefined;
   const closedByDot = groups?.["closing"] !== undefined;
-  return { kind: "article", depth, number, absolute: true, label: number, heading: rest, rest, ordinal: Number(number.split(".").at(-1)), closedByDot };
+  const wideGap = groups?.["gap"]?.includes("\u3000") === true;
+  return {
+    kind: "article",
+    depth,
+    number,
+    absolute: true,
+    label: number,
+    heading: rest,
+    rest,
+    ordinal: Number(number.split(".").at(-1)),
+    closedByDot,
+    wideGap,
+  };
 };
 
 /**
- * 番号の後ろが単位なら数量（「1.5 万人」「2.5 days」）。点で閉じた番号（「5. ページ自身の通信」「2. Days off」）は札で、
- * 数量はそう書かないので、後ろに単位の語が来ても数量にしない。
+ * 番号の後ろが単位なら数量（「1.5 万人」「2.5 days」）。点で閉じた番号（「5. ページ自身の通信」「2. Days off」）と、全角の空白を
+ * 置いた番号（「9（全角空白）年間収入階級」）は札で、数量はそう書かないので、後ろに単位の語が来ても数量にしない。
  */
 export const isAmount = (dotted: DottedLine, countedAfter: StructurePatterns["countedAfter"]): boolean =>
-  !dotted.closedByDot && countedAfter?.(dotted.number, dotted.rest) === true;
+  !dotted.closedByDot && !dotted.wideGap && countedAfter?.(dotted.number, dotted.rest) === true;
+
+/** 年まで書いた日付の値（2022-12-08）。年の無い「2 May」は、番号 2 の題「May」とも読める。 */
+const WITH_YEAR = /^\d{4}-/u;
+
+/** 番号が年まで書いた日付の頭（「8 December 2022」の 8）か。日付を言語の dates が読む。年表の見出しの日は番号の並びではない。 */
+export const startsDate = (text: string, dates: StructurePatterns["dates"]): boolean => {
+  const at = text.length - text.trimStart().length;
+  return (dates?.(text) ?? []).some((date) => date.start <= at && at < date.end && WITH_YEAR.test(String(date.attrs["value"] ?? "")));
+};
 
-/** 言語を問わない通し番号。後ろが単位なら数量、本文の文の続きなら文の中の数なので、番号にしない。 */
+/** 言語を問わない通し番号。後ろが単位なら数量、日付なら日、本文の文の続きなら文の中の数なので、番号にしない。 */
 export const universalNumber = (patterns: StructurePatterns, text: string, context: NumberingContext, plainText: boolean): NumberedLine | undefined => {
   const dotted = dottedNumber(text, context, plainText);
-  if (dotted === undefined || isAmount(dotted, patterns.countedAfter)) return undefined;
+  if (dotted === undefined || isAmount(dotted, patterns.countedAfter) || startsDate(text, patterns.dates)) return undefined;
   const numbered = { number: dotted.number, rest: dotted.rest, isHeading: context.isHeading };
   return numberInSentence(numbered, patterns.continuesSentence) ? undefined : dotted;
 };

package/src/structure/unspaced-heading.ts

@@ -0,0 +1,27 @@
+import type { Heading } from "../document.ts";
+import type { NumberingContext, StructurePatterns } from "../plugin.ts";
+import type { Line } from "./lines.ts";
+
+/** Hashes with no space before the words: 「##第一四条 （存続条項）」. CommonMark reads the line as a paragraph. */
+const UNSPACED = /^ {0,3}(?<hashes>#{1,6})(?<words>[^#\s].*)$/u;
+
+const ON_HEADING: NumberingContext = { open: [], isHeading: true };
+
+/**
+ * Lines (by number) written as a heading without the space after the hashes, kept only when the language reads the words as an article or
+ * a chapter. The writer's neighbours are headings of the same shape, so reading this one as a paragraph puts a gap in the
+ * article numbers that is not in the document. 「#タグ」 and 「#1 の件」 stay text.
+ */
+export const unspacedHeadings = (lines: readonly Line[], headed: ReadonlyMap<number, Heading>, patterns: StructurePatterns): [number, Heading][] =>
+  lines.flatMap((line) => {
+    if (headed.has(line.number)) return [];
+    const groups = UNSPACED.exec(line.text)?.groups;
+    const hashes = groups?.["hashes"];
+    const words = groups?.["words"]?.trim();
+    if (hashes === undefined || words === undefined) return [];
+    const kind = patterns.numbered(words, ON_HEADING)?.kind;
+    if (kind !== "article" && kind !== "chapter") return [];
+    const heading: Heading = { depth: hashes.length, text: words, start: line.start, end: line.start + line.text.length };
+    const entry: [number, Heading] = [line.number, heading];
+    return [entry];
+  });

package/src/structure/written-list-path.ts

@@ -0,0 +1,46 @@
+// 箇条書きで書いた小見出し。「- (b) Composition: …」の下に字下げして「- i. one representative …」と書いた文書は、
+// 「Section 8(b)(i)」の (b) と i を本文の括弧ではなく行の頭の札で書く。札の入れ子は字下げで読む。
+
+/** 行の頭の札。indent は札の前の字下げの幅（タブは 4）。 */
+export type ListLabel = { readonly label: string; readonly indent: number };
+
+/** 字下げ、箇条書きの記号（-、*、+）、札、空白、本文。 */
+const LIST_LINE = /^(?<indent>[ \t]*)(?:[-*+][ \t]+)?(?<mark>\S+)[ \t]+\S/u;
+/** 札「(b)」「i.」「2)」。 */
+const MARK = /^\(?(?<label>[a-zA-Z]{1,4}|\d{1,3})[.)]$/u;
+
+/** 札の中身。開き括弧で始まる札は閉じ括弧で終わる（「(b.」は札ではない）。 */
+const labelOf = (mark: string): string | undefined => {
+  const label = MARK.exec(mark)?.groups?.["label"];
+  return mark.startsWith("(") && !mark.endsWith(")") ? undefined : label;
+};
+const TAB_WIDTH = 4;
+
+const widthOf = (indent: string): number => [...indent].reduce((width, char) => width + (char === "\t" ? TAB_WIDTH : 1), 0);
+
+/** 本文の行の頭に書かれた札を、文書の順に。Pure. */
+export const listLabels = (body: string): ListLabel[] =>
+  body.split("\n").flatMap((line) => {
+    const groups = LIST_LINE.exec(line)?.groups;
+    const label = labelOf(groups?.["mark"] ?? "");
+    return label === undefined ? [] : [{ label, indent: widthOf(groups?.["indent"] ?? "") }];
+  });
+
+/** 札 index の下（次に同じか浅い字下げの札が来るまで）の終わり。 */
+const scopeEnd = (items: readonly ListLabel[], index: number, end: number): number => {
+  const depth = items[index]?.indent ?? 0;
+  const next = items.findIndex((item, at) => at > index && at < end && item.indent <= depth);
+  return next === -1 ? end : next;
+};
+
+/** items[from, end) の一番浅い字下げの札（直下の札）から、labels を入れ子の順にたどれるか。深い札を飛ばして親にしない。 */
+const pathFrom = (items: readonly ListLabel[], labels: readonly string[], from: number, end: number): boolean => {
+  const [first, ...rest] = labels;
+  if (first === undefined) return true;
+  const scope = items.slice(from, end);
+  const top = Math.min(...scope.map((item) => item.indent));
+  return scope.some((item, at) => item.indent === top && item.label === first && pathFrom(items, rest, from + at + 1, scopeEnd(items, from + at, end)));
+};
+
+/** 札「b」「iv」が、行の頭の札として「(b)」の下に「iv.」と入れ子で書かれているか。札が無ければ false。Pure. */
+export const hasListPath = (items: readonly ListLabel[], labels: readonly string[]): boolean => labels.length > 0 && pathFrom(items, labels, 0, items.length);

package/src/structure/written-resolve.ts

@@ -1,5 +1,6 @@
 import type { StructureNode } from "../plugin.ts";
 import { headingRegions, parentAndLabels, writtenHeadings, writtenSubdivisions, type WrittenHeading } from "./written-headings.ts";
+import { hasListPath, listLabels } from "./written-list-path.ts";
 
 // 木が番地にしない平文の見出しと、本文の中に書いた括弧の小見出し（「6.1 … that (i) the Software …」）を、参照の行き先として引く。
 
@@ -31,27 +32,33 @@ const treeRegions = (nodes: readonly StructureNode[], headings: readonly Written
   return regions;
 };
 
+/** 範囲の本文を read で読んだ結果。範囲ごとに、要るときに一度だけ読む。 */
+const perRegion = <T>(source: string, read: (body: string) => T): ((region: Region) => T) => {
+  const cache = new Map<string, T>();
+  return ([start, end]) => {
+    const key = `${String(start)}:${String(end)}`;
+    const found = cache.get(key) ?? read(source.slice(start, end));
+    cache.set(key, found);
+    return found;
+  };
+};
+
 /**
  * 平文の見出しの番地と、参照（Section 6.1(i)）の親（6.1）が木の条か平文の見出しとしてあり、その範囲の本文に札「(i)」が
- * 書かれているかを引く関数。範囲ごとの札は、要るときに一度だけ読む。Pure.
+ * 書かれているか（札を行の頭に入れ子で書いた箇条書きも読む）を引く関数。Pure.
  */
 export const writtenReading = (nodes: readonly StructureNode[], source: string): WrittenReading => {
   const references = nodes.filter((node) => node.kind === "reference").map((node) => ({ start: node.span.start, end: node.span.end, target: targetOf(node) }));
   const headings = writtenHeadings(references, source);
   const written = headingRegions(headings, source.length);
   const tree = treeRegions(nodes, headings);
-  const labels = new Map<string, ReadonlySet<string>>();
-  const labelsIn = ([start, end]: Region): ReadonlySet<string> => {
-    const key = `${String(start)}:${String(end)}`;
-    const found = labels.get(key) ?? writtenSubdivisions(source.slice(start, end));
-    labels.set(key, found);
-    return found;
-  };
+  const labelsIn = perRegion(source, writtenSubdivisions);
+  const listIn = perRegion(source, listLabels);
   const regionsOf = (address: string): Region[] => [...(tree.get(address) ?? []), ...(written.get(address) ?? [])];
   const subdivision = (reference: StructureNode): boolean => {
     const wanted = parentAndLabels(targetOf(reference), String(reference.attrs["label"] ?? ""));
     if (wanted === undefined) return false;
-    return regionsOf(wanted.parent).some((region) => wanted.labels.every((label) => labelsIn(region).has(label)));
+    return regionsOf(wanted.parent).some((region) => wanted.labels.every((label) => labelsIn(region).has(label)) || hasListPath(listIn(region), wanted.labels));
   };
   return { addresses: headings.map((heading) => heading.address), subdivision };
 };

package/src/title-line.ts

@@ -0,0 +1,53 @@
+// A title written on a line of its own, as a plain-text heading (Exhibit A to Merger Agreement, ANNEX 1): a few words with
+// no full stop, and a blank line or the document's edge on both sides. Pure.
+
+/**
+ * How a reader tells a title from a sentence. dangling-figure-reference reads a caption of any labelled thing; duplicate-
+ * definition reads where an attached instrument starts. Their notions differ, so each says how.
+ */
+export type TitleNotion = {
+  /** A title ending in one of these is a sentence or runs on. duplicate-definition also refuses a trailing `: ; ,`. */
+  readonly sentenceEnd: RegExp;
+  /** Every word longer than three letters starts with a capital. dangling-figure-reference only: Figure 3 shows the trend. */
+  readonly capitalised: boolean;
+  /**
+   * "any-whitespace": the document's edge may be reached across any whitespace (a line of U+3000 or NBSP before the
+   * first line). "blank-line": only lines of spaces and tabs count. dangling-figure-reference takes the first.
+   */
+  readonly edge: "any-whitespace" | "blank-line";
+};
+
+/** A title runs at most this many words after its label and number. */
+const TITLE_MAX_WORDS = 6;
+/** A word this long or shorter may stay in lower case in a title (to, of, the, and). */
+const TITLE_SMALL_WORD = 3;
+const BLANK_LINE = /\n[ \t\r]*\n[ \t\r]*$/u;
+const ENDS_LINE = /^[ \t\r]*\n/u;
+const BLANK = /^[ \t\r]*$/u;
+const LOWER_START = /^\p{Ll}/u;
+
+/** rest, the words after a label and its number, are a short title by notion's measure. */
+export const isShortTitle = (rest: string, notion: TitleNotion): boolean => {
+  const title = rest.trim();
+  const words = title.split(/\s+/u).filter((word) => word !== "");
+  return (
+    words.length <= TITLE_MAX_WORDS &&
+    !notion.sentenceEnd.test(title) &&
+    (!notion.capitalised || words.every((word) => word.length <= TITLE_SMALL_WORD || !LOWER_START.test(word)))
+  );
+};
+
+const isAloneBefore = (before: string, notion: TitleNotion): boolean =>
+  before === "" || BLANK_LINE.test(`\n${before}`) || (notion.edge === "any-whitespace" && before.trim() === "");
+
+const isAloneAfter = (after: string, isLastLine: boolean, notion: TitleNotion): boolean =>
+  isLastLine || ENDS_LINE.test(after) || BLANK.test(after) || (notion.edge === "any-whitespace" && after.trim() === "");
+
+/**
+ * The line from lineStart to lineEnd (-1: the last line) is a title: rest, the words after its label and number, are a
+ * short title, and a blank line or the document's edge is on both sides.
+ */
+export const isTitleLine = (source: string, lineStart: number, lineEnd: number, rest: string, notion: TitleNotion): boolean =>
+  isShortTitle(rest, notion) &&
+  isAloneBefore(source.slice(0, lineStart), notion) &&
+  isAloneAfter(lineEnd === -1 ? "" : source.slice(lineEnd + 1), lineEnd === -1, notion);

package/src/topic-clause.ts

@@ -0,0 +1,109 @@
+// Where a sentence's topic (〜は) gets its own predicate when that is not the end of the sentence: the topic is said to be
+// something and the sentence goes on with another clause (この道行きは安易なものではありませんが、…). Pure: tokens in, an index out.
+import type { Token } from "./plugin.ts";
+
+const COMMA = new Set(["、", "，"]);
+const NOMINAL = new Set(["NOUN", "PRON", "PROPN"]);
+const PREDICATE = new Set(["VERB", "AUX", "ADJ", "SCONJ"]);
+/** The nouns a restrictive であって leads to: 失業者であって、支援の必要がある者. */
+const RESTRICTED_HEADS = new Set(["者", "もの", "物", "方"]);
+/** Verbs that report what others say or think (〜と言われている). とされる is a rule's wording, not a report. */
+const REPORTING = new Set(["言う", "思う", "考える", "見る", "伝える"]);
+
+/**
+ * The copula that says what the topic is: で (ものである; read as a particle in 事業ではなく) and です. な before a noun is not a
+ * predicate.
+ */
+const isCopula = (token: Token | undefined): boolean =>
+  token?.surface === "で" ? token.pos === "AUX" || token.pos === "ADP" : token?.pos === "AUX" && token.lemma === "です";
+
+/** The words a copula is finished with before its conjunction: あり, ござい, ます, ませ, ん, the は of ではありません. */
+const COPULA_TAIL = new Set(["ある", "ござる", "ます", "ん", "ぬ", "は"]);
+
+const isCopulaTail = (token: Token): boolean => COPULA_TAIL.has(token.lemma ?? token.surface) || COPULA_TAIL.has(token.surface);
+
+/** Whether the words just before end are a copula and its tail (ではありません, でございます, である). */
+/** The first word at or before at that is not a copula's tail. Walks back only over the tail, so each call is short. */
+const beforeTail = (tokens: readonly Token[], at: number): number => {
+  const token = tokens[at];
+  return token !== undefined && isCopulaTail(token) ? beforeTail(tokens, at - 1) : at;
+};
+
+const copulaBefore = (tokens: readonly Token[], end: number): boolean => isCopula(tokens[beforeTail(tokens, end - 1)]);
+
+const isComma = (token: Token | undefined): boolean => COMMA.has(token?.surface ?? "");
+
+/** A noun of its own, not a nominaliser (こと, の, ところ) that only turns the clause before it into a noun. */
+const isOwnNoun = (token: Token | undefined): boolean => NOMINAL.has(token?.pos ?? "") && token?.features?.["NounType"] !== "Dependent";
+
+const isParticle = (token: Token, surface: string): boolean => token.surface === surface && token.pos === "ADP";
+
+/**
+ * The last place a subject of its own stands before the sentence's last word: a new topic (法律はありませんが) anywhere, or a
+ * noun with が that the final predicate takes (no comma after it: 動きが見られ始めている). A noun with が inside a condition
+ * (債務者が請求した場合、) is not one. -1 when there is none. A clause has a subject of its own after it when this is past it.
+ */
+const lastOwnSubject = (tokens: readonly Token[], last: number): number => {
+  const lastComma = tokens.findLastIndex((token, at) => at < last && isComma(token));
+  return tokens.findLastIndex(
+    (token, at) => at < last && isOwnNoun(tokens[at - 1]) && (isParticle(token, "は") || (isParticle(token, "が") && at > lastComma)),
+  );
+};
+
+/** The words of the clause after a comma at from, up to the next clause stopped by a comma, or the sentence's last word. */
+const nextClause = (tokens: readonly Token[], from: number, last: number): readonly Token[] => {
+  const stop = tokens.findIndex((token, at) => at > from && at < last && isComma(token) && PREDICATE.has(tokens[at - 1]?.pos ?? ""));
+  return tokens.slice(from + 1, stop === -1 ? last : stop);
+};
+
+/** であって that narrows a class to a head noun after it (失業者であって、支援の必要がある者の…): the topic's predicate is still to come. */
+const narrowsToHead = (tokens: readonly Token[], comma: number, last: number): boolean =>
+  nextClause(tokens, comma, last).some((token, at, clause) => token.pos === "NOUN" && RESTRICTED_HEADS.has(token.surface) && clause[at + 1]?.surface !== "は");
+
+/** A copula with が or て and a comma (〜ではありませんが、, 〜でございますが、, 〜であって、 that does not narrow to a head noun). */
+const closesCopula = (tokens: readonly Token[], at: number, last: number): boolean => {
+  const token = tokens[at];
+  if (token?.pos !== "SCONJ" || (token.surface !== "が" && token.surface !== "て")) return false;
+  if (!isComma(tokens[at + 1]) || !copulaBefore(tokens, at)) return false;
+  return token.surface === "が" || !narrowsToHead(tokens, at + 1, last);
+};
+
+/** A quotation the sentence reports in the passive (〜だと言われているが), whose topic is inside the quotation. */
+const closesReport = (tokens: readonly Token[], at: number): boolean => {
+  const token = tokens[at];
+  return token?.surface === "と" && token.pos === "ADP" && REPORTING.has(tokens[at + 1]?.lemma ?? "") && tokens[at + 2]?.features?.["Voice"] === "Pass";
+};
+
+/** A clause the topic's predicate closes for certain. */
+const closesSurely = (tokens: readonly Token[], at: number, last: number): boolean => closesCopula(tokens, at, last) || closesReport(tokens, at);
+
+/** 〜にあるところ、 / 〜したところ、: a clause the nominaliser ところ closes after a predicate. */
+const closesWithTokoro = (tokens: readonly Token[], at: number): boolean =>
+  tokens[at]?.surface === "ところ" && tokens[at]?.features?.["NounType"] === "Dependent" && PREDICATE.has(tokens[at - 1]?.pos ?? "");
+
+/** 〜であるならば、: a copula's conditional. */
+const closesWithCondition = (tokens: readonly Token[], at: number): boolean =>
+  tokens[at]?.surface === "ば" && tokens[at]?.pos === "SCONJ" && tokens[at - 1]?.surface === "なら" && tokens[at - 1]?.lemma === "だ";
+
+/**
+ * A clause that may close the topic's predicate, when a new subject follows, each stopped by a comma: a copula after ない or
+ * ため (〜ではなく、, 〜であるため、), a copula's condition (〜であるならば、), ところ (〜にあるところ、), and a continuing verb
+ * (〜を続けており、).
+ */
+const closesIfNewSubject = (tokens: readonly Token[], at: number): boolean => {
+  const token = tokens[at];
+  if (!isComma(tokens[at + 1])) return false;
+  if (token?.surface === "ため" || token?.lemma === "ない") return copulaBefore(tokens, at);
+  if (closesWithTokoro(tokens, at) || closesWithCondition(tokens, at)) return true;
+  return token?.pos === "VERB" && token.lemma === "おる" && token.features?.["Bound"] === "Yes";
+};
+
+/**
+ * The index of the word that ends the topic's own clause, between the topic and the sentence's last word, or undefined when the
+ * topic's predicate is the one the sentence ends on.
+ */
+export const topicClauseEnd = (tokens: readonly Token[], topic: number, last: number): number | undefined => {
+  const subject = lastOwnSubject(tokens, last);
+  const end = tokens.findIndex((_, at) => at > topic && at < last && (closesSurely(tokens, at, last) || (closesIfNewSubject(tokens, at) && subject > at)));
+  return end === -1 ? undefined : end;
+};

package/src/uncounted-particles.ts

@@ -0,0 +1,72 @@
+import type { Lexicon, Token } from "./plugin.ts";
+
+/** The words that decide which particle-looking tokens are not links in a chain of particles. */
+export type UncountedWords = {
+  /** The particles a chain is made of (の). */
+  readonly particles: readonly string[];
+  /** Words written as a number: every character one of these, or a decimal digit. */
+  readonly numeralChars: ReadonlySet<string>;
+  /** Words that number a provision and may be followed by a branch number (第70条の２: 条). */
+  readonly counters: readonly string[];
+};
+
+const DECIMAL_DIGITS = /^\p{Nd}+$/u;
+
+export const isNumeral = (surface: string, numeralChars: ReadonlySet<string>): boolean =>
+  DECIMAL_DIGITS.test(surface) || (surface !== "" && [...surface].every((char) => numeralChars.has(char)));
+
+const numeralAt = (tokens: readonly Token[], at: number, words: UncountedWords): boolean => {
+  const token = tokens[at];
+  return token !== undefined && isNumeral(token.surface, words.numeralChars);
+};
+
+/** 第70条, 第３項: a counter right after a number. */
+const numberedCounterAt = (tokens: readonly Token[], at: number, words: UncountedWords): boolean => {
+  const token = tokens[at];
+  return token !== undefined && words.counters.includes(token.surface) && numeralAt(tokens, at - 1, words);
+};
+
+const NOMINAL = new Set(["NOUN", "PROPN"]);
+const LETTER = /\p{L}/u;
+
+/** Where the run of number tokens that starts at `at` ends (三十 may be two tokens). */
+const numeralRunEnd = (tokens: readonly Token[], at: number, words: UncountedWords): number => {
+  const end = tokens.findIndex((token, index) => index >= at && !isNumeral(token.surface, words.numeralChars));
+  return end === -1 ? tokens.length : end;
+};
+
+/**
+ * A word after the number makes it a quantity (資料２－１の４ページ: page 4 of it), not a branch number. A word that opens
+ * the next number (第 of 第70条の２第１項) is not one.
+ */
+const isWordAt = (tokens: readonly Token[], at: number, words: UncountedWords): boolean => {
+  const token = tokens[at];
+  return token !== undefined && NOMINAL.has(token.pos) && LETTER.test(token.surface) && !numeralAt(tokens, at + 1, words);
+};
+
+/** 第70条の２, 70の２－８の２, 第２の２: the particle joins a branch number to the number before it, so it is part of the number. */
+const isNumberingConnector = (tokens: readonly Token[], at: number, words: UncountedWords): boolean =>
+  numeralAt(tokens, at + 1, words) &&
+  !isWordAt(tokens, numeralRunEnd(tokens, at + 1, words), words) &&
+  (numeralAt(tokens, at - 1, words) || numberedCounterAt(tokens, at - 1, words));
+
+/** The particle tokens of a sentence that join the parts of a number (第70条の２) and are not links of a chain. */
+export const uncountedParticles = (tokens: readonly Token[], words: UncountedWords): ReadonlySet<Token> =>
+  new Set(tokens.filter((token, at) => words.particles.includes(token.surface) && isNumberingConnector(tokens, at, words)));
+
+const COUNTER_LEXICON = "numbering-counter";
+const NUMBER_LEXICON = "number-word";
+const NUMERAL_GROUPS: readonly (string | undefined)[] = ["digit", "place"];
+
+/** The words for `uncountedParticles`, read from the document's lexicons. */
+export const uncountedWordsOf = (lexicons: Readonly<Record<string, Lexicon>>, particles: readonly string[]): UncountedWords => ({
+  particles,
+  numeralChars: new Set((lexicons[NUMBER_LEXICON] ?? []).filter((entry) => NUMERAL_GROUPS.includes(entry.group)).map((entry) => entry.pattern)),
+  counters: (lexicons[COUNTER_LEXICON] ?? []).map((entry) => entry.pattern),
+});
+
+/** The tokens of a sentence without the particles that join the parts of a number. */
+export const countedTokens = (tokens: readonly Token[], words: UncountedWords): readonly Token[] => {
+  const uncounted = uncountedParticles(tokens, words);
+  return uncounted.size === 0 ? tokens : tokens.filter((token) => !uncounted.has(token));
+};