@chaffjs/lang-en 0.12.0 → 0.14.0

package/src/quoted-stop.ts

@@ -0,0 +1,52 @@
+import type { Span } from "chaffjs/plugin";
+import { isAbbreviation } from "./sentence-split.ts";
+
+/**
+ * 米国式の引用は、文末のピリオドを閉じ引用符の内側に置く（…called it "a fair trial." Plainly, …）。
+ * sentence-splitter は引用符の中の句点で文を切らず、引用符が閉じた後の空白でも切らないので、次の文が前の文につながる。
+ *
+ * 分割器が返した一つの文を、文末の記号 + 閉じ引用符 + 空白 + 次の文の頭（大文字。開き括弧・引用符の後の大文字も）で切る。
+ * 引用符の中の語が略語・頭文字（"U.S." "Dr." "J."）のときと、括弧が開いたままのときは切らない。
+ */
+/** 次の文の頭。空白の後の大文字。開き括弧・引用符の後の大文字も文頭である。 */
+export const NEXT_SENTENCE_HEAD = String.raw`\s+[\p{Ps}\p{Pi}"']*\p{Lu}`;
+const QUOTED_STOP = new RegExp(String.raw`[.?!]["'”’]+(?=${NEXT_SENTENCE_HEAD})`, "gu");
+// ピリオドの前の語は、空白・開き引用符・開き括弧の後から数える。
+const WORD_START = /[\s\p{Ps}\p{Pi}"']/u;
+// 一字ずつピリオドを打った語（J. / U.S. / e.g.）は、略語の一覧に無くても略語。
+const INITIALS = /^(?:\p{L}\.)+$/u;
+// 分割器が一組として読む括弧（( [ { （ ［ ｛ 【 《 「 『 …）は、Unicode の開き・閉じ括弧の分類にすべて入る。
+const OPENER = /\p{Ps}/u;
+const CLOSER = /\p{Pe}/u;
+
+const endsWithAbbreviation = (sentence: string, stop: number): boolean => {
+  if (sentence[stop] !== ".") return false;
+  const word = `${sentence.slice(0, stop).split(WORD_START).at(-1) ?? ""}.`;
+  return isAbbreviation(word) || INITIALS.test(word);
+};
+
+// 対の無い閉じ括弧（「1) 最初の項目」の番号）は、後から開く括弧を打ち消さない。
+const depthAfter = (depth: number, char: string): number => {
+  if (OPENER.test(char)) return depth + 1;
+  return CLOSER.test(char) ? Math.max(0, depth - 1) : depth;
+};
+
+const bracketOpenAt = (sentence: string, index: number): boolean => Array.from(sentence.slice(0, index)).reduce(depthAfter, 0) > 0;
+
+type Cut = { readonly end: number; readonly next: number };
+
+const cutsIn = (sentence: string): Cut[] =>
+  [...sentence.matchAll(QUOTED_STOP)]
+    .filter((match) => !endsWithAbbreviation(sentence, match.index) && !bracketOpenAt(sentence, match.index))
+    .map((match) => {
+      const end = match.index + match[0].length;
+      return { end, next: end + (/^\s+/u.exec(sentence.slice(end))?.[0].length ?? 0) };
+    });
+
+/** span の文を、閉じ引用符の内側で閉じた文ごとに分けた span。切れ目が無ければ span そのもの一つ。 */
+export const splitAtQuotedStops = (text: string, span: Span): Span[] => {
+  const cuts = cutsIn(text.slice(span.start, span.end));
+  const starts = [0, ...cuts.map((cut) => cut.next)];
+  const ends = [...cuts.map((cut) => cut.end), span.end - span.start];
+  return starts.map((start, index) => ({ start: span.start + start, end: span.start + (ends[index] ?? start) }));
+};

package/src/sentence-split.ts

@@ -45,6 +45,8 @@ const { language } = DefaultAbbrMarkerOptions;
 const ABBREVIATIONS = new Set(
   [...language.ABBREVIATIONS, ...language.PREPOSITIVE_ABBREVIATIONS, ...language.EXCLAMATION_WORDS].map((word) => word.toLowerCase()),
 );
+/** 分割器が略語と読む語（ピリオドまで含めて渡す）。「Dr.」「U.S.」「Jan.」。 */
+export const isAbbreviation = (word: string): boolean => ABBREVIATIONS.has(word.toLowerCase());
 // 「J. Smith」のような語は前の語を見て略語か決まる。前の語が切れ目の向こうにあると判定が変わる。
 const CAPITAL_DOT = /\p{Lu}\./gu;
 
@@ -76,8 +78,8 @@ const indexesOf = (text: string, pattern: RegExp): number[] => [...text.matchAll
 /** 句点の並びの直後。和文は直前が仮名・漢字の句点、英文は略語でない語のピリオド。 */
 const stopEnds = (text: string): number[] => {
   const japanese = [...text.matchAll(JAPANESE_STOPS)].filter((match) => isCJK(text[match.index - 1]));
-  const english = [...text.matchAll(ENGLISH_STOP)].filter((match) => !ABBREVIATIONS.has(match[0].toLowerCase()));
-  return [...japanese, ...english].map((match) => match.index + match[0].length).sort((a, b) => a - b);
+  const english = [...text.matchAll(ENGLISH_STOP)].filter((match) => !isAbbreviation(match[0]));
+  return [...japanese, ...english].map((match) => match.index + match[0].length).toSorted((a, b) => a - b);
 };
 
 const skipSpaces = (text: string, from: number): number => {

package/src/structure.ts

@@ -1,10 +1,11 @@
 import type { Mention, NumberedLine, NumberingContext, StructurePatterns } from "chaffjs/plugin";
-import { citedDocumentAfter, citedDocumentBefore } from "./citation.ts";
+import { citedDocumentAfter, citedDocumentBefore, hyphenatedTagAround } from "./citation.ts";
 import { membersAfter } from "./reference-list.ts";
 import { parseRoman } from "./roman.ts";
 import { dates } from "./dates.ts";
 import { definitionScopeDepth, definitions, opensDefinitionScope } from "./definitions.ts";
 import { CHAPTER_DEPTH, PART_DEPTH } from "./depth.ts";
+import { continuesAddress, continuesOpen, depthFor, INSERTED, ordinalOf, styleOf } from "./item-style.ts";
 
 // Contracts, specifications and statutes in English. core nests what this reads; it does not know
 // how English numbers its articles.
@@ -28,14 +29,8 @@ const titleOf = (rest: string): string | undefined => {
 
 const ARTICLE = /^\s{0,3}(?:ARTICLE|Article)\s+(?<n>\d{1,3}|[IVXLC]{1,7})\b(?<rest>.*)$/u;
 const SECTION = /^\s{0,3}(?:SECTION|Section|§)\s*(?<n>\d{1,3}(?:\.\d{1,3}){0,5})\b(?<rest>.*)$/u;
-/**
- * An amendment inserts a subsection between two others and numbers it "(A1)" or "(2A)". It is a subsection, written
- * outside the sequence: it has no ordinal, so "(1)" after "(A1)" is still the first.
- */
-const INSERTED = "\\d{1,3}[A-Z]{1,2}|[A-Z]{1,2}\\d{1,3}";
-const LETTERED = new RegExp(`^\\s{0,6}\\((?<n>[a-z]{1,4}|\\d{1,3}|${INSERTED})\\)\\s+(?<rest>\\S.*)$`, "u");
-const IS_INSERTED = new RegExp(`^(?:${INSERTED})$`, "u");
-const MULTI_ROMAN = /^(?:ii|iii|iv|vi|vii|viii|ix)$/u;
+/** "(a) text", "(ii) text", "(B) text", or the label alone on its line with the text below it. */
+const LETTERED = new RegExp(`^\\s{0,6}\\((?<n>[a-z]{1,4}|[A-Z]{1,4}|\\d{1,3}|${INSERTED})\\)(?:\\s+(?<rest>\\S.*))?\\s*$`, "u");
 
 const headed = (pattern: RegExp, line: string, numbering: string, label: (n: string) => string): NumberedLine | undefined => {
   const groups = pattern.exec(line)?.groups;
@@ -56,70 +51,24 @@ const headed = (pattern: RegExp, line: string, numbering: string, label: (n: str
   };
 };
 
-type Style = "letter" | "roman" | "digit";
-
-/**
- * 開いている項目の書き方。"(ii)" はローマ数字、"(b)" は英字。一文字の "(i)" はどちらにも読めるので、
- * 開いたときの読みを覚えておく代わりに、一つ上に英字が開いていたかで決め直す。
- */
-const styleOfLabel = (open: NumberedLine, index: number, all: readonly NumberedLine[]): Style | undefined => {
-  const inner = /^\((?<n>[A-Za-z0-9]{1,5})\)$/u.exec(open.label)?.groups?.["n"];
-  if (inner === undefined) return undefined;
-  if (/^\d+$/u.test(inner) || IS_INSERTED.test(inner)) return "digit";
-  if (MULTI_ROMAN.test(inner)) return "roman";
-  const above = all[index - 1];
-  return /^[ivx]$/u.test(inner) && above !== undefined && styleOfLabel(above, index - 1, all) === "letter" && above.depth < open.depth ? "roman" : "letter";
-};
-
-const styles = (context: NumberingContext): (Style | undefined)[] => context.open.map((open, index, all) => styleOfLabel(open, index, all));
-
-/** "(h)" の次の "(i)" は英字。開いている英字の次の文字なら、ローマ数字とは読まない。 */
-const followsLetter = (raw: string, context: NumberingContext): boolean =>
-  context.open.some((open, index) => styles(context)[index] === "letter" && open.number.charCodeAt(0) + 1 === raw.charCodeAt(0));
-
-/** "(i)" is a roman numeral right under "(a)", or when a roman list is already open; the letter i otherwise. */
-const styleOf = (raw: string, context: NumberingContext): Style => {
-  if (/^\d+$/u.test(raw) || IS_INSERTED.test(raw)) return "digit";
-  if (MULTI_ROMAN.test(raw)) return "roman";
-  if (!/^[ivx]$/u.test(raw) || followsLetter(raw, context)) return "letter";
-  const open = styles(context);
-  return open.includes("roman") || open.at(-1) === "letter" ? "roman" : "letter";
-};
-
-/**
- * A sibling has the depth of the open item written the same way: "(b)" closes "(i)" and sits beside "(a)".
- * A new way of numbering goes one deeper than whatever is open.
- */
-const depthFor = (style: Style, context: NumberingContext): number => {
-  const open = styles(context);
-  const sibling = [...context.open].reverse().find((_item, reversed) => open[context.open.length - 1 - reversed] === style);
-  return sibling?.depth ?? (context.open.at(-1)?.depth ?? 0) + 1;
-};
-
-const LETTER_BEFORE_A = "a".charCodeAt(0) - 1;
-
-/** "(b)" は 2 番目、"(ii)" も 2 番目。二文字以上の英字（"(aa)"）は並びが決まらないので付けない。 */
-const ordinalOf = (raw: string, style: Style): number | undefined => {
-  if (style === "digit") return IS_INSERTED.test(raw) ? undefined : Number(raw);
-  if (style === "roman") return parseRoman(raw);
-  return raw.length === 1 ? raw.charCodeAt(0) - LETTER_BEFORE_A : undefined;
-};
-
 const lettered = (line: string, context: NumberingContext): NumberedLine | undefined => {
   const groups = LETTERED.exec(line)?.groups;
   const raw = groups?.["n"];
   if (groups === undefined || raw === undefined) return undefined;
   const style = styleOf(raw, context);
+  const ordinal = style === undefined ? undefined : ordinalOf(raw, style);
+  const rest = groups["rest"];
+  if (style === undefined || (rest === undefined && !continuesOpen(style, ordinal, context))) return undefined;
   // 番地は書かれたままの "ii" を使う。参照「Section 4.2(a)(ii)」も同じ形で書かれるので、そのまま引ける。
   return {
     kind: "item",
-    depth: depthFor(style, context),
-    ordinal: ordinalOf(raw, style),
+    depth: depthFor(style, context, ordinal),
+    ordinal,
     number: raw,
     absolute: false,
     label: `(${raw})`,
     heading: "",
-    rest: groups["rest"]?.trim() ?? "",
+    rest: rest?.trim() ?? "",
   };
 };
 
@@ -144,13 +93,13 @@ const numbered = (line: string, context: NumberingContext): NumberedLine | undef
   lettered(line, context);
 
 const REFERENCE = /(?<word>\b[Ss]ections?|\b[Aa]rticles?|§) ?(?<n>\d{1,3}(?:\.\d{1,3}){0,5}|[IVXLC]{1,7})\b/gu;
-/** "(a)", "(ii)", "(3)", and an inserted "(A1)" or "(2A)": the same labels the tree reads. */
-const SUBDIVISION = new RegExp(`^\\((?<p>[a-z0-9]{1,4}|${INSERTED})\\)`, "u");
+/** "(a)", "(ii)", "(3)", "(B)", and an inserted "(A1)" or "(2A)": the same labels the tree reads. */
+const SUBDIVISION = new RegExp(`^\\((?<p>[a-z0-9]{1,4}|[A-Z]{1,4}|${INSERTED})\\)`, "u");
 /** The longest label, with its parentheses: "(ZZ999)". */
 const MAX_SUBDIVISION_LENGTH = 7;
 
-/** "(a)(ii)(3)" is as deep as a reference goes; more parentheses are text, not a deeper address. */
-const MAX_SUBDIVISIONS = 4;
+/** "(a)(1)(i)(A)(1)", a US regulation's deepest paragraph, is as deep as a reference goes; more parentheses are text. */
+const MAX_SUBDIVISIONS = 5;
 
 /**
  * "(a)(ii)" のような続きの括弧を、正規表現を複雑にせずに一つずつ読む。
@@ -161,7 +110,7 @@ const subdivisions = (text: string, from: number): { readonly parts: readonly st
   let end = from;
   while (parts.length < MAX_SUBDIVISIONS) {
     const part = SUBDIVISION.exec(text.slice(end, end + MAX_SUBDIVISION_LENGTH))?.groups?.["p"];
-    if (part === undefined) break;
+    if (part === undefined || !continuesAddress(part, parts)) break;
     parts.push(part);
     end += part.length + 2;
   }
@@ -191,6 +140,13 @@ const glossedDocument = (gloss: Gloss): string | undefined => {
   return anchor !== undefined && gloss.depth > anchor.depth ? anchor.document : undefined;
 };
 
+/** The other document a reference names, or else a bracketed tag that the core checks against the document's list. */
+const citation = (text: string, start: number, end: number, document: string | undefined): Readonly<Record<string, string>> => {
+  if (document !== undefined) return { document };
+  const citedTag = hyphenatedTagAround(text, start, end);
+  return citedTag === undefined ? {} : { citedTag };
+};
+
 /**
  * "Section 4.2(a)" → 4.2.a, "Article III" → 3. The same addresses the tree gives.
  * "Section 9 of the Master Agreement" carries the other document's name, and is not looked up in this tree.
@@ -207,9 +163,9 @@ const references = (text: string): Mention[] => {
     gloss.scanned = Math.max(gloss.scanned, end);
     if (cited !== undefined) gloss.anchors.push({ document: cited, depth: gloss.depth });
     const numbering = /^[Aa]/u.test(match.groups?.["word"] ?? "") ? "article" : "section";
-    const shared = { numbering, ...(document === undefined ? {} : { document }) };
+    const shared = { numbering, ...citation(text, match.index, end, document) };
     const first = { start: match.index, end, attrs: { target: [main, ...parts].join("."), label: text.slice(match.index, end), ...shared } };
-    const [plural, roman] = [/s$/u.test(match.groups?.["word"] ?? ""), /^[IVXLC]+$/u.test(match.groups?.["n"] ?? "")];
+    const [plural, roman] = [(match.groups?.["word"] ?? "").endsWith("s"), /^[IVXLC]+$/u.test(match.groups?.["n"] ?? "")];
     return [first, ...membersAfter(text, end, [main, ...parts], shared, plural, roman)];
   });
 };
@@ -263,7 +219,7 @@ const obligations = (text: string): Mention[] => {
       kept.push({ start, end, attrs: { marker, type } });
     });
   });
-  return kept.sort((left, right) => left.start - right.start);
+  return kept.toSorted((left, right) => left.start - right.start);
 };
 
 /** Find the number first, then look at what is right before (a currency) and right after (a unit). */