@chaffjs/lang-en 0.6.0 → 0.8.0

package/src/roman.ts

@@ -0,0 +1,9 @@
+const ROMAN: Readonly<Record<string, number>> = { i: 1, v: 5, x: 10, l: 50, c: 100 };
+
+/** "IV" → 4, "xii" → 12. Undefined for anything that is not a roman numeral. */
+export const parseRoman = (text: string): number | undefined => {
+  const values = [...text.toLowerCase()].map((char) => ROMAN[char]);
+  if (values.length === 0 || values.some((value) => value === undefined)) return undefined;
+  const known = values.filter((value) => value !== undefined);
+  return known.reduce((total, value, index) => ((known[index + 1] ?? 0) > value ? total - value : total + value), 0);
+};

package/src/sentence-split.ts

@@ -0,0 +1,131 @@
+import { DefaultAbbrMarkerOptions, split, SentenceSplitterSyntax } from "sentence-splitter";
+import type { Span } from "chaffjs/plugin";
+
+/**
+ * sentence-splitter に長い段落を一度に渡すと、文の数の 2 乗で遅くなる。
+ * 文を閉じるたびに、それまでに閉じた括弧を全部並べ直すため。
+ *
+ * そこで分割器の状態が空に戻る所（文が閉じ、括弧も開いていない所）で切り、切れ目ごとに渡す。
+ * 切ってよいかは分割器の規則を写して決める。写し方が狂うと文が変わるので、迷う所では切らない。
+ */
+
+// PairMaker と同じ組。開き括弧を鍵にする。
+const PAIRS: readonly (readonly [string, string])[] = [
+  ['"', '"'],
+  ["[", "]"],
+  ["(", ")"],
+  ["{", "}"],
+  ["「", "」"],
+  ["（", "）"],
+  ["『", "』"],
+  ["｛", "｝"],
+  ["［", "］"],
+  ["〚", "〛"],
+  ["【", "】"],
+  ["《", "》"],
+];
+const KEY_OF = new Map<string, string>(
+  PAIRS.flatMap(([open, close]): [string, string][] => [
+    [close, open],
+    [open, open],
+  ]),
+);
+const CLOSE_OF = new Map<string, string>(PAIRS.map(([open, close]): [string, string] => [open, close]));
+
+// AbbrMarker の語の区切り。分割器は UTF-16 の 1 単位ずつ読むので、こちらも 1 単位で判定する。
+const CJK = /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Hangul}]/u;
+const isCJK = (unit: string | undefined): boolean => unit !== undefined && CJK.test(unit);
+const isSpace = (unit: string | undefined): boolean => unit !== undefined && /\s/.test(unit);
+
+// 和文の句点の並び。直前が仮名・漢字なら、略語の判定がこの句点に掛かることはない。
+const JAPANESE_STOPS = /[。．？！]+/g;
+// 英語は「空白の後の英字だけの語 + ピリオド + 空白」。略語の一覧に無い語だけを切れ目にする。
+const ENGLISH_STOP = /(?<=^|\s)[A-Za-z]{2,}\.(?=\s)/g;
+const { language } = DefaultAbbrMarkerOptions;
+const ABBREVIATIONS = new Set(
+  [...language.ABBREVIATIONS, ...language.PREPOSITIVE_ABBREVIATIONS, ...language.EXCLAMATION_WORDS].map((word) => word.toLowerCase()),
+);
+// 「J. Smith」のような語は前の語を見て略語か決まる。前の語が切れ目の向こうにあると判定が変わる。
+const CAPITAL_DOT = /\p{Lu}\./gu;
+
+/** 各位置の手前で、開いたままの括弧があるか。PairMaker と同じく種類ごとに 1 つまで開く。 */
+const pairOpenBefore = (text: string): boolean[] => {
+  const open = new Set<string>();
+  const before = [false];
+  text.split("").forEach((unit) => {
+    const key = KEY_OF.get(unit);
+    if (key !== undefined && !open.has(key) && unit === key) open.add(key);
+    else if (key !== undefined && open.has(key) && unit === CLOSE_OF.get(key)) open.delete(key);
+    before.push(open.size > 0);
+  });
+  return before;
+};
+
+/** sorted の中で from 以上の最初の値。無ければ fallback。 */
+const firstAtOrAfter = (sorted: readonly number[], from: number, fallback: number): number => {
+  const search = (low: number, high: number): number => {
+    if (low >= high) return sorted[low] ?? fallback;
+    const middle = Math.floor((low + high) / 2);
+    return (sorted[middle] ?? fallback) >= from ? search(low, middle) : search(middle + 1, high);
+  };
+  return search(0, sorted.length);
+};
+
+const indexesOf = (text: string, pattern: RegExp): number[] => [...text.matchAll(pattern)].map((match) => match.index);
+
+/** 句点の並びの直後。和文は直前が仮名・漢字の句点、英文は略語でない語のピリオド。 */
+const stopEnds = (text: string): number[] => {
+  const japanese = [...text.matchAll(JAPANESE_STOPS)].filter((match) => isCJK(text[match.index - 1]));
+  const english = [...text.matchAll(ENGLISH_STOP)].filter((match) => !ABBREVIATIONS.has(match[0].toLowerCase()));
+  return [...japanese, ...english].map((match) => match.index + match[0].length).sort((a, b) => a - b);
+};
+
+const skipSpaces = (text: string, from: number): number => {
+  const spaces = /\s*/y;
+  spaces.lastIndex = from;
+  return from + (spaces.exec(text)?.[0].length ?? 0);
+};
+
+/**
+ * 句点の直後から次の文の頭へ。空白を挟むか、仮名・漢字が続くときだけ切れる。
+ * それ以外の文字が続くと、分割器はその文字を前の語の続きとして読む。
+ */
+const chunkStart = (text: string, stopEnd: number): number | undefined => {
+  const next = text[stopEnd];
+  if (isSpace(next)) return skipSpaces(text, stopEnd);
+  return isCJK(next) ? stopEnd : undefined;
+};
+
+type Landmarks = { readonly open: readonly boolean[]; readonly spaces: readonly number[]; readonly capitalDots: readonly number[] };
+
+/**
+ * 切れ目の後ろ、最初の 2 つの語の塊に「J.」の形が無いこと。
+ * 分割器はその形の語だけ、空白を越えて前の語を読み返す。
+ */
+const noLookBack = (text: string, start: number, marks: Landmarks): boolean => {
+  const secondRun = skipSpaces(text, firstAtOrAfter(marks.spaces, start, text.length));
+  return firstAtOrAfter(marks.capitalDots, start, Infinity) > secondRun;
+};
+
+const cutPoints = (text: string): number[] => {
+  const marks: Landmarks = { open: pairOpenBefore(text), spaces: indexesOf(text, /\s/g), capitalDots: indexesOf(text, CAPITAL_DOT) };
+  return stopEnds(text).flatMap((stopEnd) => {
+    if (marks.open[stopEnd] === true) return [];
+    const start = chunkStart(text, stopEnd);
+    return start !== undefined && start < text.length && noLookBack(text, start, marks) ? [start] : [];
+  });
+};
+
+const spansOf = (text: string, offset: number): Span[] =>
+  split(text)
+    .filter((node) => node.type === SentenceSplitterSyntax.Sentence)
+    .map((node) => ({ start: offset + node.range[0], end: offset + node.range[1] }));
+
+/** 分割器に別々に渡してよい区間。つなぐと text 全体になる。 */
+export const chunksOf = (text: string): Span[] => {
+  const starts = [0, ...cutPoints(text)];
+  return starts.map((start, index) => ({ start, end: starts[index + 1] ?? text.length }));
+};
+
+/** split(text) の文の span と同じものを返す。区間ごとに分けて渡すだけ。 */
+export const sentenceSpans = (text: string): Span[] => chunksOf(text).flatMap((chunk) => spansOf(text.slice(chunk.start, chunk.end), chunk.start));

package/src/structure.ts

@@ -1,19 +1,14 @@
 import type { Mention, NumberedLine, NumberingContext, StructurePatterns } from "chaffjs/plugin";
 import { citedDocumentAfter } from "./citation.ts";
+import { membersAfter } from "./reference-list.ts";
+import { parseRoman } from "./roman.ts";
+import { dates } from "./dates.ts";
+import { definitionScopeDepth, definitions, opensDefinitionScope } from "./definitions.ts";
+import { CHAPTER_DEPTH, PART_DEPTH } from "./depth.ts";
 
 // Contracts, specifications and statutes in English. core nests what this reads; it does not know
 // how English numbers its articles.
 
-const ROMAN: Readonly<Record<string, number>> = { i: 1, v: 5, x: 10, l: 50, c: 100 };
-
-/** "IV" → 4, "xii" → 12. Undefined for anything that is not a roman numeral. */
-export const parseRoman = (text: string): number | undefined => {
-  const values = [...text.toLowerCase()].map((char) => ROMAN[char]);
-  if (values.length === 0 || values.some((value) => value === undefined)) return undefined;
-  const known = values.filter((value) => value !== undefined);
-  return known.reduce((total, value, index) => ((known[index + 1] ?? 0) > value ? total - value : total + value), 0);
-};
-
 const numberOf = (text: string | undefined): string | undefined => {
   if (text === undefined) return undefined;
   if (/^\d{1,3}(?:\.\d{1,3}){0,5}$/u.test(text)) return text;
@@ -33,10 +28,16 @@ const titleOf = (rest: string): string | undefined => {
 
 const ARTICLE = /^\s{0,3}(?:ARTICLE|Article)\s+(?<n>\d{1,3}|[IVXLC]{1,7})\b(?<rest>.*)$/u;
 const SECTION = /^\s{0,3}(?:SECTION|Section|§)\s*(?<n>\d{1,3}(?:\.\d{1,3}){0,5})\b(?<rest>.*)$/u;
-const LETTERED = /^\s{0,6}\((?<n>[a-z]{1,4}|\d{1,3})\)\s+(?<rest>\S.*)$/u;
+/**
+ * An amendment inserts a subsection between two others and numbers it "(A1)" or "(2A)". It is a subsection, written
+ * outside the sequence: it has no ordinal, so "(1)" after "(A1)" is still the first.
+ */
+const INSERTED = "\\d{1,3}[A-Z]{1,2}|[A-Z]{1,2}\\d{1,3}";
+const LETTERED = new RegExp(`^\\s{0,6}\\((?<n>[a-z]{1,4}|\\d{1,3}|${INSERTED})\\)\\s+(?<rest>\\S.*)$`, "u");
+const IS_INSERTED = new RegExp(`^(?:${INSERTED})$`, "u");
 const MULTI_ROMAN = /^(?:ii|iii|iv|vi|vii|viii|ix)$/u;
 
-const headed = (pattern: RegExp, line: string, label: (n: string) => string): NumberedLine | undefined => {
+const headed = (pattern: RegExp, line: string, numbering: string, label: (n: string) => string): NumberedLine | undefined => {
   const groups = pattern.exec(line)?.groups;
   const number = numberOf(groups?.["n"]);
   const heading = titleOf(groups?.["rest"] ?? "");
@@ -51,6 +52,7 @@ const headed = (pattern: RegExp, line: string, label: (n: string) => string): Nu
     heading,
     rest: heading,
     ordinal: Number(parts.at(-1)),
+    numbering,
   };
 };
 
@@ -61,9 +63,9 @@ type Style = "letter" | "roman" | "digit";
  * 開いたときの読みを覚えておく代わりに、一つ上に英字が開いていたかで決め直す。
  */
 const styleOfLabel = (open: NumberedLine, index: number, all: readonly NumberedLine[]): Style | undefined => {
-  const inner = /^\((?<n>[a-z0-9]{1,4})\)$/u.exec(open.label)?.groups?.["n"];
+  const inner = /^\((?<n>[A-Za-z0-9]{1,5})\)$/u.exec(open.label)?.groups?.["n"];
   if (inner === undefined) return undefined;
-  if (/^\d+$/u.test(inner)) return "digit";
+  if (/^\d+$/u.test(inner) || IS_INSERTED.test(inner)) return "digit";
   if (MULTI_ROMAN.test(inner)) return "roman";
   const above = all[index - 1];
   return /^[ivx]$/u.test(inner) && above !== undefined && styleOfLabel(above, index - 1, all) === "letter" && above.depth < open.depth ? "roman" : "letter";
@@ -77,7 +79,7 @@ const followsLetter = (raw: string, context: NumberingContext): boolean =>
 
 /** "(i)" is a roman numeral right under "(a)", or when a roman list is already open; the letter i otherwise. */
 const styleOf = (raw: string, context: NumberingContext): Style => {
-  if (/^\d+$/u.test(raw)) return "digit";
+  if (/^\d+$/u.test(raw) || IS_INSERTED.test(raw)) return "digit";
   if (MULTI_ROMAN.test(raw)) return "roman";
   if (!/^[ivx]$/u.test(raw) || followsLetter(raw, context)) return "letter";
   const open = styles(context);
@@ -98,7 +100,7 @@ const LETTER_BEFORE_A = "a".charCodeAt(0) - 1;
 
 /** "(b)" は 2 番目、"(ii)" も 2 番目。二文字以上の英字（"(aa)"）は並びが決まらないので付けない。 */
 const ordinalOf = (raw: string, style: Style): number | undefined => {
-  if (style === "digit") return Number(raw);
+  if (style === "digit") return IS_INSERTED.test(raw) ? undefined : Number(raw);
   if (style === "roman") return parseRoman(raw);
   return raw.length === 1 ? raw.charCodeAt(0) - LETTER_BEFORE_A : undefined;
 };
@@ -135,33 +137,17 @@ const chapter = (pattern: RegExp, line: string, depth: number, prefix: string, w
 };
 
 const numbered = (line: string, context: NumberingContext): NumberedLine | undefined =>
-  chapter(PART, line, -2, "pt", "Part") ??
-  chapter(CHAPTER, line, -1, "ch", "Chapter") ??
-  headed(ARTICLE, line, (n) => `Article ${n}`) ??
-  headed(SECTION, line, (n) => `Section ${n}`) ??
+  chapter(PART, line, PART_DEPTH, "pt", "Part") ??
+  chapter(CHAPTER, line, CHAPTER_DEPTH, "ch", "Chapter") ??
+  headed(ARTICLE, line, "article", (n) => `Article ${n}`) ??
+  headed(SECTION, line, "section", (n) => `Section ${n}`) ??
   lettered(line, context);
 
-const mentions = (
-  pattern: RegExp,
-  text: string,
-  attrs: (groups: Readonly<Record<string, string | undefined>>, whole: string) => Mention["attrs"] | undefined,
-): Mention[] =>
-  [...text.matchAll(pattern)].flatMap((match) => {
-    const found = attrs(match.groups ?? {}, match[0]);
-    return found === undefined ? [] : [{ start: match.index, end: match.index + match[0].length, attrs: found }];
-  });
-
-const DEFINITIONS = [
-  /["“](?<term>[^"”\n]{1,60})["”] (?:means|shall mean|refers to|has the meaning)\b/gu,
-  /\((?:the |hereinafter )?["“](?<term>[^"”\n]{1,60})["”]\)/gu,
-  /\(hereinafter referred to as ["“](?<term>[^"”\n]{1,60})["”]\)/gu,
-];
-
-const definitions = (text: string): Mention[] =>
-  DEFINITIONS.flatMap((pattern) => mentions(pattern, text, (groups) => (groups["term"] === undefined ? undefined : { term: groups["term"] })));
-
-const REFERENCE = /\b(?:Sections?|Articles?|§) ?(?<n>\d{1,3}(?:\.\d{1,3}){0,5}|[IVXLC]{1,7})\b/gu;
-const SUBDIVISION = /^\((?<p>[a-z0-9]{1,4})\)/u;
+const REFERENCE = /(?<word>\b[Ss]ections?|\b[Aa]rticles?|§) ?(?<n>\d{1,3}(?:\.\d{1,3}){0,5}|[IVXLC]{1,7})\b/gu;
+/** "(a)", "(ii)", "(3)", and an inserted "(A1)" or "(2A)": the same labels the tree reads. */
+const SUBDIVISION = new RegExp(`^\\((?<p>[a-z0-9]{1,4}|${INSERTED})\\)`, "u");
+/** The longest label, with its parentheses: "(ZZ999)". */
+const MAX_SUBDIVISION_LENGTH = 7;
 
 /** "(a)(ii)(3)" is as deep as a reference goes; more parentheses are text, not a deeper address. */
 const MAX_SUBDIVISIONS = 4;
@@ -174,7 +160,7 @@ const subdivisions = (text: string, from: number): { readonly parts: readonly st
   const parts: string[] = [];
   let end = from;
   while (parts.length < MAX_SUBDIVISIONS) {
-    const part = SUBDIVISION.exec(text.slice(end, end + 6))?.groups?.["p"];
+    const part = SUBDIVISION.exec(text.slice(end, end + MAX_SUBDIVISION_LENGTH))?.groups?.["p"];
     if (part === undefined) break;
     parts.push(part);
     end += part.length + 2;
@@ -182,19 +168,51 @@ const subdivisions = (text: string, from: number): { readonly parts: readonly st
   return { parts, end };
 };
 
+/** Parentheses opened and not yet closed in `between`. */
+const PAREN_STEP: Readonly<Record<string, number>> = { "(": 1, ")": -1 };
+
+/**
+ * "section 120(3) of the Communications Act 2003 (conditions under section 120 …)": a gloss in parentheses after
+ * a reference into another document describes that document, so the references in it are into it too — until the
+ * parenthesis the reference stood in closes. One pass over the line, however many references it holds.
+ */
+type Gloss = { depth: number; scanned: number; readonly anchors: { readonly document: string; readonly depth: number }[] };
+
+const advance = (gloss: Gloss, text: string, to: number): void => {
+  for (let index = gloss.scanned; index < to; index += 1) {
+    gloss.depth += PAREN_STEP[text[index] ?? ""] ?? 0;
+    while ((gloss.anchors.at(-1)?.depth ?? -Infinity) > gloss.depth) gloss.anchors.pop();
+  }
+  gloss.scanned = Math.max(gloss.scanned, to);
+};
+
+const glossedDocument = (gloss: Gloss): string | undefined => {
+  const anchor = gloss.anchors.at(-1);
+  return anchor !== undefined && gloss.depth > anchor.depth ? anchor.document : undefined;
+};
+
 /**
  * "Section 4.2(a)" → 4.2.a, "Article III" → 3. The same addresses the tree gives.
  * "Section 9 of the Master Agreement" carries the other document's name, and is not looked up in this tree.
  */
-const references = (text: string): Mention[] =>
-  [...text.matchAll(REFERENCE)].flatMap((match) => {
+const references = (text: string): Mention[] => {
+  const gloss: Gloss = { depth: 0, scanned: 0, anchors: [] };
+  return [...text.matchAll(REFERENCE)].flatMap((match) => {
     const main = numberOf(match.groups?.["n"]);
     if (main === undefined) return [];
     const { parts, end } = subdivisions(text, match.index + match[0].length);
-    const document = citedDocumentAfter(text, end);
-    const attrs = { target: [main, ...parts].join("."), label: text.slice(match.index, end), ...(document === undefined ? {} : { document }) };
-    return [{ start: match.index, end, attrs }];
+    advance(gloss, text, match.index);
+    const cited = citedDocumentAfter(text, end);
+    const document = cited ?? glossedDocument(gloss);
+    gloss.scanned = Math.max(gloss.scanned, end);
+    if (cited !== undefined) gloss.anchors.push({ document: cited, depth: gloss.depth });
+    const numbering = /^[Aa]/u.test(match.groups?.["word"] ?? "") ? "article" : "section";
+    const shared = { numbering, ...(document === undefined ? {} : { document }) };
+    const first = { start: match.index, end, attrs: { target: [main, ...parts].join("."), label: text.slice(match.index, end), ...shared } };
+    const [plural, roman] = [/s$/u.test(match.groups?.["word"] ?? ""), /^[IVXLC]+$/u.test(match.groups?.["n"] ?? "")];
+    return [first, ...membersAfter(text, end, [main, ...parts], shared, plural, roman)];
   });
+};
 
 /** Longest first, and never inside a word: "shall not" is not also "shall", "mayor" is not "may". */
 const MARKERS: readonly (readonly [string, "must" | "must-not" | "may"])[] = [
@@ -304,63 +322,17 @@ const quantities = (text: string): Mention[] =>
     return unit === undefined || Number.isNaN(value) || isWordChar(text[match.index - 1]) ? [] : [{ start: match.index, end, attrs: { value, unit } }];
   });
 
-const MONTHS = ["january", "february", "march", "april", "may", "june", "july", "august", "september", "october", "november", "december"];
-const MONTH_WORD = /\b(?<month>[A-Z][a-z]{2,8})\b/gu;
-const ISO_DATE = /\b(?<y>\d{4})-(?<m>\d{2})-(?<d>\d{2})\b/gu;
-const DAY_BEFORE = /(?<d>\d{1,2})(?:st|nd|rd|th)? $/u;
-const DAY_YEAR_AFTER = /^ (?<d>\d{1,2})(?:st|nd|rd|th)?,? (?<y>\d{4})\b/u;
-const YEAR_AFTER = /^,? (?<y>\d{4})\b/u;
-
-const pad = (value: string): string => value.padStart(2, "0");
-
-/** "1 April 2024": the day written before the month. */
-const dayBefore = (text: string, at: number): { readonly day: string; readonly start: number } | undefined => {
-  const found = DAY_BEFORE.exec(text.slice(Math.max(0, at - 6), at));
-  const day = found?.groups?.["d"];
-  return found === null || day === undefined ? undefined : { day, start: at - found[0].length };
-};
-
-/** "April 1, 2024" → 2024-04-01. */
-const monthDayYear = (text: string, at: number, end: number, month: number): Mention | undefined => {
-  const found = DAY_YEAR_AFTER.exec(text.slice(end, end + 16));
-  if (found?.groups === undefined) return undefined;
-  const value = `${found.groups["y"] ?? ""}-${pad(String(month))}-${pad(found.groups["d"] ?? "")}`;
-  return { start: at, end: end + found[0].length, attrs: { value } };
-};
-
-/** "1 April 2024" → 2024-04-01, "April 2024" → 2024-04. */
-const monthYear = (text: string, at: number, end: number, month: number): Mention | undefined => {
-  const found = YEAR_AFTER.exec(text.slice(end, end + 8));
-  const year = found?.groups?.["y"];
-  if (found === null || year === undefined) return undefined;
-  const before = dayBefore(text, at);
-  const value = [year, pad(String(month)), ...(before === undefined ? [] : [pad(before.day)])].join("-");
-  return { start: before?.start ?? at, end: end + found[0].length, attrs: { value } };
-};
-
-/**
- * A month name alone is not a date: "May" is also the modal verb, so it counts only with a year beside it.
- * The month is found first and its neighbours read with anchored patterns, never one long alternation.
- */
-const namedDate = (text: string, match: RegExpExecArray): Mention | undefined => {
-  const month = MONTHS.indexOf((match.groups?.["month"] ?? "").toLowerCase()) + 1;
-  if (month === 0) return undefined;
-  const end = match.index + match[0].length;
-  return monthDayYear(text, match.index, end, month) ?? monthYear(text, match.index, end, month);
-};
-
-const isoDate = (match: RegExpExecArray): Mention => ({
-  start: match.index,
-  end: match.index + match[0].length,
-  attrs: { value: `${match.groups?.["y"] ?? ""}-${match.groups?.["m"] ?? ""}-${match.groups?.["d"] ?? ""}` },
-});
-
-const dates = (text: string): Mention[] =>
-  [...[...text.matchAll(MONTH_WORD)].flatMap((match) => namedDate(text, match) ?? []), ...[...text.matchAll(ISO_DATE)].map(isoDate)].sort(
-    (left, right) => left.start - right.start,
-  );
-
 /** "2.5 days" and "1.5 times" are amounts, not section 2.5 titled "days". */
 const countedAfter = (_number: string, rest: string): boolean => unitAfter(` ${rest}`, 0) !== undefined;
 
-export const structure: StructurePatterns = { numbered, definitions, references, obligations, quantities, dates, countedAfter };
+export const structure: StructurePatterns = {
+  numbered,
+  definitions,
+  references,
+  obligations,
+  quantities,
+  dates,
+  countedAfter,
+  opensDefinitionScope,
+  definitionScopeDepth,
+};