@chaffjs/lang-en 0.12.0 → 0.13.0

package/src/closing-quote.ts

@@ -0,0 +1,52 @@
+import type { Span } from "chaffjs/plugin";
+import { NEXT_SENTENCE_HEAD } from "./quoted-stop.ts";
+
+/**
+ * sentence-splitter は疑問符・感嘆符の後の曲がった閉じ引用符（“Is it done?” Nobody …）と直線の一重引用符の手前で文を切る。
+ * 次の文が「”」で始まり、前の文は引用符が開いたまま終わる。
+ *
+ * 前の文が句点で終わり、次の文がそこから間を置かずに閉じ引用符（閉じ括弧も）で始まるとき、それを前の文の末尾へ戻す。
+ * 戻した後に空白と大文字が続けば文は二つのまま、何も続かなければ前の文だけ、
+ * 小文字・数字・句読点が続けば引用は文の途中にあるので一つの文につなぐ（直線の二重引用符で分割器がそうするのと同じ）。
+ * 開き引用符（“ ‘）は閉じの分類に入らない。語頭のアポストロフィ（’Tis ’90s）は後ろに字が続くので閉じと読まない。
+ */
+const CLOSING_RUN = /^[\p{Pf}\p{Pe}"']+/u;
+const WORD_CHARACTER = /^[\p{L}\p{N}]/u;
+const STOP_AT_END = /[.?!]$/u;
+// 語頭のアポストロフィの後の大文字（’Tis）も次の文の頭。
+const NEXT_SENTENCE = new RegExp(String.raw`^(?:${NEXT_SENTENCE_HEAD}|\s+’\p{Lu})`, "u");
+const LEADING_SPACE = /^\s*/u;
+
+/** 文頭の閉じ引用符の並びの長さ。後ろに字が続けば開き引用符かアポストロフィなので 0。 */
+export const closingRunLength = (sentence: string): number => {
+  const run = CLOSING_RUN.exec(sentence)?.[0].length ?? 0;
+  return run > 0 && !WORD_CHARACTER.test(sentence.slice(run)) ? run : 0;
+};
+
+const follows = (text: string, previous: Span, span: Span): boolean =>
+  previous.end === span.start && STOP_AT_END.test(text.slice(previous.start, previous.end));
+
+/** previous と span を置き換える文。閉じ引用符が戻らなければ undefined。 */
+const reattached = (text: string, previous: Span, span: Span): Span[] | undefined => {
+  const run = follows(text, previous, span) ? closingRunLength(text.slice(span.start, span.end)) : 0;
+  if (run === 0) return undefined;
+  const quoteEnd = span.start + run;
+  const rest = text.slice(quoteEnd, span.end);
+  if (rest.trim() === "") return [{ start: previous.start, end: quoteEnd }];
+  if (!NEXT_SENTENCE.test(rest)) return [{ start: previous.start, end: span.end }];
+  const next = quoteEnd + (LEADING_SPACE.exec(rest)?.[0].length ?? 0);
+  return [
+    { start: previous.start, end: quoteEnd },
+    { start: next, end: span.end },
+  ];
+};
+
+/** 文の span の並び（text 上の位置）で、文頭に取り残された閉じ引用符を前の文へ戻したもの。 */
+export const reattachClosingQuotes = (text: string, spans: readonly Span[]): Span[] =>
+  spans.reduce<Span[]>((sentences, span) => {
+    const previous = sentences.at(-1);
+    const replaced = previous === undefined ? undefined : reattached(text, previous, span);
+    if (replaced === undefined) sentences.push(span);
+    else sentences.splice(-1, 1, ...replaced);
+    return sentences;
+  }, []);

package/src/index.ts

@@ -1,6 +1,8 @@
 import { loadLexicons } from "./lexicons.ts";
 import { unmarkNumberStops } from "./number-stop.ts";
 import { sentenceSpans } from "./sentence-split.ts";
+import { splitAtQuotedStops } from "./quoted-stop.ts";
+import { reattachClosingQuotes } from "./closing-quote.ts";
 import { structure } from "./structure.ts";
 import { isReady, prepare, tokenize } from "./pos.ts";
 import type { AdapterNeeds, LanguageAdapter, Segmentation, Sentence } from "chaffjs/plugin";
@@ -25,7 +27,7 @@ const withTokens = (sentence: Sentence): Sentence => {
 
 /**
  * 英語は sentence-splitter の既定にほぼ任せる。"Dr." "e.g." "U.S." "$3.50" を
- * いずれも文末と誤認しない。前処理は、行の途中の番号を箇条書きと読ませないことだけ。spec §7.2。
+ * いずれも文末と誤認しない。前処理は行の途中の番号を箇条書きと読ませること、後処理は閉じ引用符の内側で閉じた文を切ることと、文頭に取り残された閉じ引用符を前の文へ戻すこと。spec §7.2。
  */
 export const adapter: LanguageAdapter = {
   kind: "language",
@@ -52,7 +54,8 @@ export const adapter: LanguageAdapter = {
   lexicons: loadLexicons(),
   structure,
   segment: (text: string): Segmentation => {
-    const sentences: Sentence[] = sentenceSpans(unmarkNumberStops(text)).map((span) => ({ span, text: text.slice(span.start, span.end) }));
+    const quotedStops = sentenceSpans(unmarkNumberStops(text)).flatMap((span) => splitAtQuotedStops(text, span));
+    const sentences: Sentence[] = reattachClosingQuotes(text, quotedStops).map((span) => ({ span, text: text.slice(span.start, span.end) }));
     return { sentences: isReady() ? sentences.map(withTokens) : sentences };
   },
 };

package/src/item-style.ts

@@ -0,0 +1,130 @@
+import type { NumberedLine, NumberingContext } from "chaffjs/plugin";
+import { parseRoman } from "./roman.ts";
+
+/**
+ * An amendment inserts a subsection between two others and numbers it "(A1)" or "(2A)". It is a subsection, written
+ * outside the sequence: it has no ordinal, so "(1)" after "(A1)" is still the first.
+ */
+export const INSERTED = "\\d{1,3}[A-Z]{1,2}|[A-Z]{1,2}\\d{1,3}";
+export const IS_INSERTED = new RegExp(`^(?:${INSERTED})$`, "u");
+const MULTI_ROMAN = /^(?:ii|iii|iv|vi|vii|viii|ix)$/u;
+const AMBIGUOUS = /^[ivx]$/u;
+const DIGITS = /^\d+$/u;
+const CAPITALS = /^[A-Z]+$/u;
+const ONE_CAPITAL = /^[A-Z]$/u;
+
+export type Style = "letter" | "roman" | "digit" | "capital" | "capital-roman";
+
+/** 大文字で書いた同じ並び。"(B)" は英字の、"(II)" はローマ数字の大文字。 */
+const CAPITAL_OF: Readonly<Record<"letter" | "roman", Style>> = { letter: "capital", roman: "capital-roman" };
+
+/** 小文字にした番号を形で読む。一文字の "i" は、開いたときに付けた並びの位置で見分ける。ローマ数字なら 1、英字なら 9。 */
+const shapeOf = (lower: string, ordinal: number | undefined): "letter" | "roman" => {
+  if (MULTI_ROMAN.test(lower)) return "roman";
+  return AMBIGUOUS.test(lower) && ordinal === parseRoman(lower) ? "roman" : "letter";
+};
+
+/** 開いている項目の書き方。"(ii)" はローマ数字、"(b)" は英字、"(B)" は大文字の英字。 */
+export const styleOfOpen = (open: NumberedLine): Style | undefined => {
+  const inner = /^\((?<n>[A-Za-z0-9]{1,5})\)$/u.exec(open.label)?.groups?.["n"];
+  if (inner === undefined) return undefined;
+  if (DIGITS.test(inner) || IS_INSERTED.test(inner)) return "digit";
+  const shape = shapeOf(inner.toLowerCase(), open.ordinal);
+  return CAPITALS.test(inner) ? CAPITAL_OF[shape] : shape;
+};
+
+const LETTER_BEFORE_A = "a".charCodeAt(0) - 1;
+
+/** "(b)" と "(B)" は 2 番目、"(ii)" と "(II)" も 2 番目。二文字以上の英字（"(aa)"）は並びが決まらないので付けない。 */
+export const ordinalOf = (raw: string, style: Style): number | undefined => {
+  if (style === "digit") return IS_INSERTED.test(raw) ? undefined : Number(raw);
+  if (style === "roman" || style === "capital-roman") return parseRoman(raw);
+  return raw.length === 1 ? raw.toLowerCase().charCodeAt(0) - LETTER_BEFORE_A : undefined;
+};
+
+const styles = (context: NumberingContext): (Style | undefined)[] => context.open.map(styleOfOpen);
+
+/** "(h)" の次の "(i)"、"(H)" の次の "(I)" は英字。同じ書き方で開いている英字の次の文字なら、ローマ数字とは読まない。 */
+const follows = (raw: string, style: "letter" | "capital", context: NumberingContext): boolean =>
+  context.open.some((open, index) => styles(context)[index] === style && open.number.charCodeAt(0) + 1 === raw.charCodeAt(0));
+
+/**
+ * 英字の並びは (a) から始まるので、"(a)" や "(1)" のすぐ下に来た "(i)" は一段深いローマ数字。
+ * 米国の規則は (a)(1)(i) の順に下る。見出しのすぐ下の "(i)" は、どちらとも決まらないので英字。
+ */
+const OPENS_ROMAN: ReadonlySet<Style | undefined> = new Set(["letter", "digit"]);
+
+/** "(I)" は米国の法典で (i) の一段下の大文字ローマ数字。"(H)" の次なら英字。 */
+const capitalRoman = (raw: string, open: readonly (Style | undefined)[], context: NumberingContext): boolean => {
+  const lower = raw.toLowerCase();
+  if (MULTI_ROMAN.test(lower)) return open.includes("capital-roman");
+  if (!AMBIGUOUS.test(lower) || follows(raw, "capital", context)) return false;
+  return open.includes("capital-roman") || (raw === "I" && open.at(-1) === "roman");
+};
+
+/**
+ * 米国の規則は (a)(1)(i)(A) と下る。大文字の "(A)" は、開いているローマ数字のすぐ下で始まるか、開いている大文字の続きのときだけ項目。
+ * 英国の法令や契約書が一段目に使う "(A)" は、何の下とも決まらないので本文のまま読む。
+ */
+const capitalStyleOf = (raw: string, context: NumberingContext): Style | undefined => {
+  const open = styles(context);
+  if (capitalRoman(raw, open, context)) return "capital-roman";
+  if (!ONE_CAPITAL.test(raw)) return undefined;
+  return open.includes("capital") || (raw === "A" && open.at(-1) === "roman") ? "capital" : undefined;
+};
+
+/**
+ * "(i)" is a roman numeral right under "(a)" or "(1)", or when a roman list is already open; the letter i otherwise.
+ * A capital "(A)" is an item only where a US regulation puts it, under a roman item; undefined elsewhere.
+ */
+export const styleOf = (raw: string, context: NumberingContext): Style | undefined => {
+  if (DIGITS.test(raw) || IS_INSERTED.test(raw)) return "digit";
+  if (CAPITALS.test(raw)) return capitalStyleOf(raw, context);
+  if (MULTI_ROMAN.test(raw)) return "roman";
+  if (!AMBIGUOUS.test(raw) || follows(raw, "letter", context)) return "letter";
+  const open = styles(context);
+  return open.includes("roman") || OPENS_ROMAN.has(open.at(-1)) ? "roman" : "letter";
+};
+
+/** The nearest open item written the same way: "(b)" closes "(i)" and sits beside "(a)". */
+const siblingOf = (style: Style, context: NumberingContext): NumberedLine | undefined => {
+  const open = styles(context);
+  return context.open.findLast((_item, index) => open[index] === style);
+};
+
+const IS_CAPITAL: ReadonlySet<Style | undefined> = new Set(["capital", "capital-roman"]);
+
+/**
+ * "(A)" の下の "(1)" や "(i)"、"(i)" の下の "(A)" は、上に開いている同じ書き方の項目の続きではなく、一段深い並びの始まり。
+ * 米国の規則は (a)(1)(i)(A) の下を、もう一度 (1) や (i) で数える。
+ */
+const startsBelow = (style: Style, ordinal: number | undefined, innermost: Style | undefined): boolean =>
+  ordinal === 1 && style !== innermost && (IS_CAPITAL.has(innermost) || (IS_CAPITAL.has(style) && innermost === "roman"));
+
+/**
+ * A sibling has the depth of the open item written the same way. A new way of numbering goes one deeper than whatever
+ * is open, and so does a first item right under a capital, or a first capital right under a roman item.
+ */
+export const depthFor = (style: Style, context: NumberingContext, ordinal?: number): number => {
+  const innermost = context.open.at(-1);
+  const deeper = (innermost?.depth ?? 0) + 1;
+  if (innermost !== undefined && startsBelow(style, ordinal, styleOfOpen(innermost))) return deeper;
+  return siblingOf(style, context)?.depth ?? deeper;
+};
+
+/**
+ * 参照の "(A)" や "(I)" は、ローマ数字の "(iii)" の次に書かれたときだけ番地の続き。木も、大文字はローマ数字の下でしか読まない。
+ * "section 5(A)" の "(A)" は番地に入れず、これまでどおり "section 5" を引く。"(h)(i)" や番号のすぐ後の "(i)" は、木と同じく英字と読む。
+ */
+export const continuesAddress = (part: string, before: readonly string[]): boolean => {
+  if (!CAPITALS.test(part)) return true;
+  const [previous, earlier] = [before.at(-1) ?? "", before.at(-2) ?? ""];
+  if (MULTI_ROMAN.test(previous)) return true;
+  return AMBIGUOUS.test(previous) && earlier !== "" && earlier.charCodeAt(0) + 1 !== previous.charCodeAt(0);
+};
+
+/** 番号だけの行 "(5)" は、開いている "(4)" の次のときだけ項目。本文は次の行から始まる。 */
+export const continuesOpen = (style: Style, ordinal: number | undefined, context: NumberingContext): boolean => {
+  const previous = siblingOf(style, context)?.ordinal;
+  return ordinal !== undefined && previous !== undefined && ordinal === previous + 1;
+};

package/src/long-runs.ts

@@ -0,0 +1,6 @@
+/**
+ * 空白の無いまま limit 文字を超える並びを、同じ長さの空白で覆う。語ではなく（base64、ハッシュ、区切りの無い記号の列）、
+ * 解析器（wink）はその長さの二乗で遅くなる。長さを変えないので、残りの語の位置はそのまま。
+ */
+export const blankLongRuns = (text: string, limit: number): string =>
+  text.replace(new RegExp(`\\S{${String(limit + 1)},}`, "gu"), (run) => " ".repeat(run.length));

package/src/pos.ts

@@ -1,7 +1,8 @@
 import { createRequire } from "node:module";
 import type { Token } from "chaffjs/plugin";
 import { loadLexicons } from "./lexicons.ts";
-import { properNounChecked } from "./proper-noun.ts";
+import { blankLongRuns } from "./long-runs.ts";
+import { lowercasedAt, properNounChecked, rereadAt, sentenceInitialCommonWord } from "./proper-noun.ts";
 import { isStativeParticiple, stativeVocabulary } from "./stative-participle.ts";
 
 const require = createRequire(import.meta.url);
@@ -157,18 +158,39 @@ const determinerFeatures = (entry: Tagged): Features => {
   return entry.pos === "DT" && ARTICLES.has(entry.value.toLowerCase()) ? { features: { PronType: "Art" } } : {};
 };
 
+/** 複数形の名詞は UD の Number=Plur。数の語がそれを数えていれば（five minutes）、one of のような言い回しではなく量。 */
+const PLURAL_TAG = new Set(["NNS", "NNPS"]);
+
+const nounOrDeterminerFeatures = (entry: Tagged): Features => (PLURAL_TAG.has(entry.pos) ? { features: { Number: "Plur" } } : determinerFeatures(entry));
+
 /** 過去分詞は VerbForm=Part。Based on the review, のような分詞の導入句を、命令形の並び（fix the parser, ship it）と見分ける。 */
 const featuresOf = (tagged: readonly Tagged[], at: number): Features => {
   const entry = tagged[at];
   if (entry === undefined) return {};
-  if (entry.pos !== "VBN") return determinerFeatures(entry);
+  if (entry.pos !== "VBN") return nounOrDeterminerFeatures(entry);
   return { features: isPassive(tagged, at) ? { VerbForm: "Part", Voice: "Pass" } : { VerbForm: "Part" } };
 };
 
-const state: { ready: Tagger | undefined } = { ready: undefined };
+/** 解析器が引く語彙（語 → Penn Treebank の品詞の並び）。wink-pos-tagger が自分の依存から読むものと同じ一つを、同じ場所から読む。 */
+type Vocabulary = (word: string) => readonly string[] | undefined;
+
+const isTags = (value: unknown): value is readonly string[] => Array.isArray(value) && value.every((tag) => typeof tag === "string");
+
+const buildVocabulary = (): Vocabulary => {
+  const words: unknown = createRequire(require.resolve("wink-pos-tagger"))("wink-lexicon/src/lexicon.js");
+  if (!isRecord(words)) throw new Error("wink-lexicon の語彙が object ではありません");
+  return (word) => {
+    const tags = Object.hasOwn(words, word) ? words[word] : undefined;
+    return isTags(tags) ? tags : undefined;
+  };
+};
+
+const state: { ready: Tagger | undefined; vocabulary: Vocabulary } = { ready: undefined, vocabulary: () => undefined };
 
 export const prepare = (): void => {
-  state.ready ??= build();
+  if (state.ready !== undefined) return;
+  state.ready = build();
+  state.vocabulary = buildVocabulary();
 };
 
 export const isReady = (): boolean => state.ready !== undefined;
@@ -195,8 +217,21 @@ const locate = (text: string, tagged: readonly Tagged[]): Token[] =>
     { tokens: [], cursor: 0 },
   ).tokens;
 
+/** 英語の語はこれより長くならない。超える並びは語として読まない。 */
+const RUN_LIMIT = 1000;
+
+const tagged = (tagger: Tagger, text: string): readonly Tagged[] => toArray(callMethod(tagger, "tagSentence", [text])).filter(isTagged);
+
+/** 文頭で大文字になっただけの普通の語を、小文字で書いたときの品詞に戻す。前後の語による判断も効くよう、文ごと解析し直す。 */
+const withSentenceInitialCase = (tagger: Tagger, text: string, entries: readonly Tagged[]): readonly Tagged[] => {
+  const at = sentenceInitialCommonWord(entries, state.vocabulary);
+  const lowered = lowercasedAt(text, entries[at]?.value);
+  return lowered === undefined ? entries : rereadAt(entries, at, tagged(tagger, lowered));
+};
+
 export const tokenize = (text: string): Token[] | undefined => {
   const tagger = state.ready;
   if (tagger === undefined) return undefined;
-  return locate(text, toArray(callMethod(tagger, "tagSentence", [text])).filter(isTagged));
+  const words = blankLongRuns(text, RUN_LIMIT);
+  return locate(words, withSentenceInitialCase(tagger, words, tagged(tagger, words)));
 };

package/src/proper-noun.ts

@@ -19,3 +19,46 @@ export const properNounChecked = (surface: string, pos: string): string => {
   if (!LETTER.test(surface)) return nonWord(surface);
   return SMALL.test(surface) && !CAPITAL.test(surface) ? "NOUN" : "PROPN";
 };
+
+/** 解析器が返す 1 語。pos は Penn Treebank。 */
+type TaggedWord = { readonly value: string; readonly pos: string; readonly lemma?: string };
+
+const PROPER_TAG = new Set(["NNP", "NNPS"]);
+
+/** 頭の一字だけが大文字の語（Containers、Such）。文頭の大文字はこの形になる。API や iPhone は名前の書き方なので含めない。 */
+const CAPITALISED = /^\p{Lu}\p{Ll}+$/u;
+
+/** 解析器の語彙に、その語が普通の語として載っているか。一度でも固有名詞として載っていれば（may の May）、名前かもしれない。 */
+const isCommonWord = (tags: readonly string[] | undefined): boolean => tags !== undefined && tags.length > 0 && !tags.some((tag) => PROPER_TAG.has(tag));
+
+/**
+ * 文頭で大文字になっただけの普通の語の位置。無ければ -1。
+ * wink は大文字で始まる名詞と形容詞をすべて固有名詞にするので、Containers are … の Containers も Traditional servers … の Traditional も固有名詞になる。
+ * 文頭の大文字は英語の書き方で、名前の印ではない。解析器の語彙が小文字の形を普通の語として知っていれば、名前ではなくその語として読む。
+ * 語彙に無い語（Kubernetes、Congress）と、文の途中の大文字の語は名前のまま。
+ */
+export const sentenceInitialCommonWord = (tagged: readonly TaggedWord[], tagsOf: (word: string) => readonly string[] | undefined): number => {
+  const at = tagged.findIndex((entry) => LETTER.test(entry.value));
+  const first = tagged[at];
+  if (first === undefined || !PROPER_TAG.has(first.pos) || !CAPITALISED.test(first.value)) return -1;
+  return isCommonWord(tagsOf(first.value.toLowerCase())) ? at : -1;
+};
+
+/** 文の中の最初の word を小文字にした文。word が文に無ければ undefined。 */
+export const lowercasedAt = (text: string, word: string | undefined): string | undefined => {
+  const start = word === undefined ? -1 : text.indexOf(word);
+  if (word === undefined || start === -1) return undefined;
+  return `${text.slice(0, start)}${word.toLowerCase()}${text.slice(start + word.length)}`;
+};
+
+/**
+ * at の語の品詞と原形を、小文字にして解析し直した結果（again）から取る。表層は元のまま。
+ * 解析し直した文の同じ位置に、同じ語の小文字が無ければ、語の切り方が変わったので元のまま。
+ */
+export const rereadAt = (entries: readonly TaggedWord[], at: number, again: readonly TaggedWord[]): readonly TaggedWord[] => {
+  const entry = entries[at];
+  const reread = again[at];
+  if (entry === undefined || reread === undefined || reread.value !== entry.value.toLowerCase()) return entries;
+  const word: TaggedWord = { value: entry.value, pos: reread.pos, ...(reread.lemma === undefined ? {} : { lemma: reread.lemma }) };
+  return entries.map((original, index) => (index === at ? word : original));
+};

package/src/quoted-stop.ts

@@ -0,0 +1,52 @@
+import type { Span } from "chaffjs/plugin";
+import { isAbbreviation } from "./sentence-split.ts";
+
+/**
+ * 米国式の引用は、文末のピリオドを閉じ引用符の内側に置く（…called it "a fair trial." Plainly, …）。
+ * sentence-splitter は引用符の中の句点で文を切らず、引用符が閉じた後の空白でも切らないので、次の文が前の文につながる。
+ *
+ * 分割器が返した一つの文を、文末の記号 + 閉じ引用符 + 空白 + 次の文の頭（大文字。開き括弧・引用符の後の大文字も）で切る。
+ * 引用符の中の語が略語・頭文字（"U.S." "Dr." "J."）のときと、括弧が開いたままのときは切らない。
+ */
+/** 次の文の頭。空白の後の大文字。開き括弧・引用符の後の大文字も文頭である。 */
+export const NEXT_SENTENCE_HEAD = String.raw`\s+[\p{Ps}\p{Pi}"']*\p{Lu}`;
+const QUOTED_STOP = new RegExp(String.raw`[.?!]["'”’]+(?=${NEXT_SENTENCE_HEAD})`, "gu");
+// ピリオドの前の語は、空白・開き引用符・開き括弧の後から数える。
+const WORD_START = /[\s\p{Ps}\p{Pi}"']/u;
+// 一字ずつピリオドを打った語（J. / U.S. / e.g.）は、略語の一覧に無くても略語。
+const INITIALS = /^(?:\p{L}\.)+$/u;
+// 分割器が一組として読む括弧（( [ { （ ［ ｛ 【 《 「 『 …）は、Unicode の開き・閉じ括弧の分類にすべて入る。
+const OPENER = /\p{Ps}/u;
+const CLOSER = /\p{Pe}/u;
+
+const endsWithAbbreviation = (sentence: string, stop: number): boolean => {
+  if (sentence[stop] !== ".") return false;
+  const word = `${sentence.slice(0, stop).split(WORD_START).at(-1) ?? ""}.`;
+  return isAbbreviation(word) || INITIALS.test(word);
+};
+
+// 対の無い閉じ括弧（「1) 最初の項目」の番号）は、後から開く括弧を打ち消さない。
+const depthAfter = (depth: number, char: string): number => {
+  if (OPENER.test(char)) return depth + 1;
+  return CLOSER.test(char) ? Math.max(0, depth - 1) : depth;
+};
+
+const bracketOpenAt = (sentence: string, index: number): boolean => Array.from(sentence.slice(0, index)).reduce(depthAfter, 0) > 0;
+
+type Cut = { readonly end: number; readonly next: number };
+
+const cutsIn = (sentence: string): Cut[] =>
+  [...sentence.matchAll(QUOTED_STOP)]
+    .filter((match) => !endsWithAbbreviation(sentence, match.index) && !bracketOpenAt(sentence, match.index))
+    .map((match) => {
+      const end = match.index + match[0].length;
+      return { end, next: end + (/^\s+/u.exec(sentence.slice(end))?.[0].length ?? 0) };
+    });
+
+/** span の文を、閉じ引用符の内側で閉じた文ごとに分けた span。切れ目が無ければ span そのもの一つ。 */
+export const splitAtQuotedStops = (text: string, span: Span): Span[] => {
+  const cuts = cutsIn(text.slice(span.start, span.end));
+  const starts = [0, ...cuts.map((cut) => cut.next)];
+  const ends = [...cuts.map((cut) => cut.end), span.end - span.start];
+  return starts.map((start, index) => ({ start: span.start + start, end: span.start + (ends[index] ?? start) }));
+};

package/src/sentence-split.ts

@@ -45,6 +45,8 @@ const { language } = DefaultAbbrMarkerOptions;
 const ABBREVIATIONS = new Set(
   [...language.ABBREVIATIONS, ...language.PREPOSITIVE_ABBREVIATIONS, ...language.EXCLAMATION_WORDS].map((word) => word.toLowerCase()),
 );
+/** 分割器が略語と読む語（ピリオドまで含めて渡す）。「Dr.」「U.S.」「Jan.」。 */
+export const isAbbreviation = (word: string): boolean => ABBREVIATIONS.has(word.toLowerCase());
 // 「J. Smith」のような語は前の語を見て略語か決まる。前の語が切れ目の向こうにあると判定が変わる。
 const CAPITAL_DOT = /\p{Lu}\./gu;
 
@@ -76,7 +78,7 @@ const indexesOf = (text: string, pattern: RegExp): number[] => [...text.matchAll
 /** 句点の並びの直後。和文は直前が仮名・漢字の句点、英文は略語でない語のピリオド。 */
 const stopEnds = (text: string): number[] => {
   const japanese = [...text.matchAll(JAPANESE_STOPS)].filter((match) => isCJK(text[match.index - 1]));
-  const english = [...text.matchAll(ENGLISH_STOP)].filter((match) => !ABBREVIATIONS.has(match[0].toLowerCase()));
+  const english = [...text.matchAll(ENGLISH_STOP)].filter((match) => !isAbbreviation(match[0]));
   return [...japanese, ...english].map((match) => match.index + match[0].length).sort((a, b) => a - b);
 };
 

package/src/structure.ts

@@ -1,10 +1,11 @@
 import type { Mention, NumberedLine, NumberingContext, StructurePatterns } from "chaffjs/plugin";
-import { citedDocumentAfter, citedDocumentBefore } from "./citation.ts";
+import { citedDocumentAfter, citedDocumentBefore, hyphenatedTagAround } from "./citation.ts";
 import { membersAfter } from "./reference-list.ts";
 import { parseRoman } from "./roman.ts";
 import { dates } from "./dates.ts";
 import { definitionScopeDepth, definitions, opensDefinitionScope } from "./definitions.ts";
 import { CHAPTER_DEPTH, PART_DEPTH } from "./depth.ts";
+import { continuesAddress, continuesOpen, depthFor, INSERTED, ordinalOf, styleOf } from "./item-style.ts";
 
 // Contracts, specifications and statutes in English. core nests what this reads; it does not know
 // how English numbers its articles.
@@ -28,14 +29,8 @@ const titleOf = (rest: string): string | undefined => {
 
 const ARTICLE = /^\s{0,3}(?:ARTICLE|Article)\s+(?<n>\d{1,3}|[IVXLC]{1,7})\b(?<rest>.*)$/u;
 const SECTION = /^\s{0,3}(?:SECTION|Section|§)\s*(?<n>\d{1,3}(?:\.\d{1,3}){0,5})\b(?<rest>.*)$/u;
-/**
- * An amendment inserts a subsection between two others and numbers it "(A1)" or "(2A)". It is a subsection, written
- * outside the sequence: it has no ordinal, so "(1)" after "(A1)" is still the first.
- */
-const INSERTED = "\\d{1,3}[A-Z]{1,2}|[A-Z]{1,2}\\d{1,3}";
-const LETTERED = new RegExp(`^\\s{0,6}\\((?<n>[a-z]{1,4}|\\d{1,3}|${INSERTED})\\)\\s+(?<rest>\\S.*)$`, "u");
-const IS_INSERTED = new RegExp(`^(?:${INSERTED})$`, "u");
-const MULTI_ROMAN = /^(?:ii|iii|iv|vi|vii|viii|ix)$/u;
+/** "(a) text", "(ii) text", "(B) text", or the label alone on its line with the text below it. */
+const LETTERED = new RegExp(`^\\s{0,6}\\((?<n>[a-z]{1,4}|[A-Z]{1,4}|\\d{1,3}|${INSERTED})\\)(?:\\s+(?<rest>\\S.*))?\\s*$`, "u");
 
 const headed = (pattern: RegExp, line: string, numbering: string, label: (n: string) => string): NumberedLine | undefined => {
   const groups = pattern.exec(line)?.groups;
@@ -56,70 +51,24 @@ const headed = (pattern: RegExp, line: string, numbering: string, label: (n: str
   };
 };
 
-type Style = "letter" | "roman" | "digit";
-
-/**
- * 開いている項目の書き方。"(ii)" はローマ数字、"(b)" は英字。一文字の "(i)" はどちらにも読めるので、
- * 開いたときの読みを覚えておく代わりに、一つ上に英字が開いていたかで決め直す。
- */
-const styleOfLabel = (open: NumberedLine, index: number, all: readonly NumberedLine[]): Style | undefined => {
-  const inner = /^\((?<n>[A-Za-z0-9]{1,5})\)$/u.exec(open.label)?.groups?.["n"];
-  if (inner === undefined) return undefined;
-  if (/^\d+$/u.test(inner) || IS_INSERTED.test(inner)) return "digit";
-  if (MULTI_ROMAN.test(inner)) return "roman";
-  const above = all[index - 1];
-  return /^[ivx]$/u.test(inner) && above !== undefined && styleOfLabel(above, index - 1, all) === "letter" && above.depth < open.depth ? "roman" : "letter";
-};
-
-const styles = (context: NumberingContext): (Style | undefined)[] => context.open.map((open, index, all) => styleOfLabel(open, index, all));
-
-/** "(h)" の次の "(i)" は英字。開いている英字の次の文字なら、ローマ数字とは読まない。 */
-const followsLetter = (raw: string, context: NumberingContext): boolean =>
-  context.open.some((open, index) => styles(context)[index] === "letter" && open.number.charCodeAt(0) + 1 === raw.charCodeAt(0));
-
-/** "(i)" is a roman numeral right under "(a)", or when a roman list is already open; the letter i otherwise. */
-const styleOf = (raw: string, context: NumberingContext): Style => {
-  if (/^\d+$/u.test(raw) || IS_INSERTED.test(raw)) return "digit";
-  if (MULTI_ROMAN.test(raw)) return "roman";
-  if (!/^[ivx]$/u.test(raw) || followsLetter(raw, context)) return "letter";
-  const open = styles(context);
-  return open.includes("roman") || open.at(-1) === "letter" ? "roman" : "letter";
-};
-
-/**
- * A sibling has the depth of the open item written the same way: "(b)" closes "(i)" and sits beside "(a)".
- * A new way of numbering goes one deeper than whatever is open.
- */
-const depthFor = (style: Style, context: NumberingContext): number => {
-  const open = styles(context);
-  const sibling = [...context.open].reverse().find((_item, reversed) => open[context.open.length - 1 - reversed] === style);
-  return sibling?.depth ?? (context.open.at(-1)?.depth ?? 0) + 1;
-};
-
-const LETTER_BEFORE_A = "a".charCodeAt(0) - 1;
-
-/** "(b)" は 2 番目、"(ii)" も 2 番目。二文字以上の英字（"(aa)"）は並びが決まらないので付けない。 */
-const ordinalOf = (raw: string, style: Style): number | undefined => {
-  if (style === "digit") return IS_INSERTED.test(raw) ? undefined : Number(raw);
-  if (style === "roman") return parseRoman(raw);
-  return raw.length === 1 ? raw.charCodeAt(0) - LETTER_BEFORE_A : undefined;
-};
-
 const lettered = (line: string, context: NumberingContext): NumberedLine | undefined => {
   const groups = LETTERED.exec(line)?.groups;
   const raw = groups?.["n"];
   if (groups === undefined || raw === undefined) return undefined;
   const style = styleOf(raw, context);
+  const ordinal = style === undefined ? undefined : ordinalOf(raw, style);
+  const rest = groups["rest"];
+  if (style === undefined || (rest === undefined && !continuesOpen(style, ordinal, context))) return undefined;
   // 番地は書かれたままの "ii" を使う。参照「Section 4.2(a)(ii)」も同じ形で書かれるので、そのまま引ける。
   return {
     kind: "item",
-    depth: depthFor(style, context),
-    ordinal: ordinalOf(raw, style),
+    depth: depthFor(style, context, ordinal),
+    ordinal,
     number: raw,
     absolute: false,
     label: `(${raw})`,
     heading: "",
-    rest: groups["rest"]?.trim() ?? "",
+    rest: rest?.trim() ?? "",
   };
 };
 
@@ -144,13 +93,13 @@ const numbered = (line: string, context: NumberingContext): NumberedLine | undef
   lettered(line, context);
 
 const REFERENCE = /(?<word>\b[Ss]ections?|\b[Aa]rticles?|§) ?(?<n>\d{1,3}(?:\.\d{1,3}){0,5}|[IVXLC]{1,7})\b/gu;
-/** "(a)", "(ii)", "(3)", and an inserted "(A1)" or "(2A)": the same labels the tree reads. */
-const SUBDIVISION = new RegExp(`^\\((?<p>[a-z0-9]{1,4}|${INSERTED})\\)`, "u");
+/** "(a)", "(ii)", "(3)", "(B)", and an inserted "(A1)" or "(2A)": the same labels the tree reads. */
+const SUBDIVISION = new RegExp(`^\\((?<p>[a-z0-9]{1,4}|[A-Z]{1,4}|${INSERTED})\\)`, "u");
 /** The longest label, with its parentheses: "(ZZ999)". */
 const MAX_SUBDIVISION_LENGTH = 7;
 
-/** "(a)(ii)(3)" is as deep as a reference goes; more parentheses are text, not a deeper address. */
-const MAX_SUBDIVISIONS = 4;
+/** "(a)(1)(i)(A)(1)", a US regulation's deepest paragraph, is as deep as a reference goes; more parentheses are text. */
+const MAX_SUBDIVISIONS = 5;
 
 /**
  * "(a)(ii)" のような続きの括弧を、正規表現を複雑にせずに一つずつ読む。
@@ -161,7 +110,7 @@ const subdivisions = (text: string, from: number): { readonly parts: readonly st
   let end = from;
   while (parts.length < MAX_SUBDIVISIONS) {
     const part = SUBDIVISION.exec(text.slice(end, end + MAX_SUBDIVISION_LENGTH))?.groups?.["p"];
-    if (part === undefined) break;
+    if (part === undefined || !continuesAddress(part, parts)) break;
     parts.push(part);
     end += part.length + 2;
   }
@@ -191,6 +140,13 @@ const glossedDocument = (gloss: Gloss): string | undefined => {
   return anchor !== undefined && gloss.depth > anchor.depth ? anchor.document : undefined;
 };
 
+/** The other document a reference names, or else a bracketed tag that the core checks against the document's list. */
+const citation = (text: string, start: number, end: number, document: string | undefined): Readonly<Record<string, string>> => {
+  if (document !== undefined) return { document };
+  const citedTag = hyphenatedTagAround(text, start, end);
+  return citedTag === undefined ? {} : { citedTag };
+};
+
 /**
  * "Section 4.2(a)" → 4.2.a, "Article III" → 3. The same addresses the tree gives.
  * "Section 9 of the Master Agreement" carries the other document's name, and is not looked up in this tree.
@@ -207,7 +163,7 @@ const references = (text: string): Mention[] => {
     gloss.scanned = Math.max(gloss.scanned, end);
     if (cited !== undefined) gloss.anchors.push({ document: cited, depth: gloss.depth });
     const numbering = /^[Aa]/u.test(match.groups?.["word"] ?? "") ? "article" : "section";
-    const shared = { numbering, ...(document === undefined ? {} : { document }) };
+    const shared = { numbering, ...citation(text, match.index, end, document) };
     const first = { start: match.index, end, attrs: { target: [main, ...parts].join("."), label: text.slice(match.index, end), ...shared } };
     const [plural, roman] = [/s$/u.test(match.groups?.["word"] ?? ""), /^[IVXLC]+$/u.test(match.groups?.["n"] ?? "")];
     return [first, ...membersAfter(text, end, [main, ...parts], shared, plural, roman)];