@chaffjs/lang-en 0.15.0 → 0.16.0

package/src/citation.ts

@@ -52,17 +52,18 @@ export const listMembers = (rest: string, plural: boolean): ListMember[] =>
  */
 const TAG = "(?<tag>[A-Z][A-Z0-9]{1,30})";
 /**
- * "[HTTP-CACHING]" is written like a contract's placeholder "[BUYER-1]". Only the document tells them apart, by listing
- * the tag or not, so this tag is a candidate that the core checks against the document (attrs.citedTag).
+ * "[HTTP-CACHING]" is written like a contract's placeholder "[BUYER-1]", and a numbered citation "[19]" like a blank to
+ * fill in. Only the document tells them apart, by listing the tag or not, so this tag is a candidate that the core checks
+ * against the document (attrs.citedTag).
  */
-const HYPHENATED_TAG = "(?<tag>[A-Z][A-Z0-9]{0,30}(?:-[A-Z0-9]{1,30}){1,4})";
+const LISTED_TAG = "(?<tag>\\d{1,3}|[A-Z][A-Z0-9]{0,30}(?:-[A-Z0-9]{1,30}){1,4})";
 const tagAfter = (tag: string): RegExp => new RegExp(`^\\[${tag}\\]`, "u");
 /** "[HTTP], Section 12.1": the tag written just before the reference. */
 const tagBefore = (tag: string): RegExp => new RegExp(`\\[${tag}\\],?\\s?$`, "u");
 const TAG_AFTER = tagAfter(TAG);
 const TAG_BEFORE = tagBefore(TAG);
-const HYPHENATED_AFTER = tagAfter(HYPHENATED_TAG);
-const HYPHENATED_BEFORE = tagBefore(HYPHENATED_TAG);
+const LISTED_AFTER = tagAfter(LISTED_TAG);
+const LISTED_BEFORE = tagBefore(LISTED_TAG);
 const TAG_REACH = 40;
 
 const tagEndingAt = (pattern: RegExp, text: string, start: number): string | undefined =>
@@ -120,9 +121,12 @@ export const citedDocumentAfter = (text: string, end: number): string | undefine
   return words.length === 1 && SELF.has(name) ? undefined : name;
 };
 
-/** A hyphenated tag right after a reference ("Section 4.2.3 of [HTTP-CACHING]") or just before it ("[HTTP-CACHING], Section 4"). */
-export const hyphenatedTagAround = (text: string, start: number, end: number): string | undefined => {
+/**
+ * A tag that names another document only if this document lists it, right after a reference ("Section 4.2.3 of
+ * [HTTP-CACHING]", "Section 4.2.2.17 of [19]") or just before it ("[HTTP-CACHING], Section 4", "[23], Section 2.17").
+ */
+export const listedTagAround = (text: string, start: number, end: number): string | undefined => {
   const named = afterOf(text, end);
-  const following = named === undefined ? undefined : HYPHENATED_AFTER.exec(named)?.groups?.["tag"];
-  return following ?? tagEndingAt(HYPHENATED_BEFORE, text, start);
+  const following = named === undefined ? undefined : LISTED_AFTER.exec(named)?.groups?.["tag"];
+  return following ?? tagEndingAt(LISTED_BEFORE, text, start);
 };

package/src/code-citation.ts

@@ -0,0 +1,52 @@
+import type { Lexicon, LexiconEntry } from "chaffjs/plugin";
+import { escapeRegExp } from "./regexp.ts";
+
+// "35 CFR §122", "42 U.S.C. § 1983", "RFC 1122, Section 3.3.4.2": a section of a code or a numbered document named
+// right before the reference, not of this document. Which names are codes is the lexicon's (document-kind).
+
+/** Built once from the lexicon; each pattern is undefined when the lexicon names no code of its shape. */
+export type CodeVocabulary = {
+  readonly before: RegExp | undefined;
+  readonly numberedBefore: RegExp | undefined;
+  /** The name of a code that takes a title number, at the start of the text and ending at a word boundary. */
+  readonly titled: RegExp | undefined;
+};
+
+/** The code's title number ("35 CFR"), then the name, then at most a second "§" ("§§ 1981 and 1983") up to the reference. */
+const TITLE_NUMBER = String.raw`(?:\d{1,3}\s+)?`;
+const SECOND_SIGN = String.raw`\s*(?:§\s*)?$`;
+/** The document's own number after its name, then a comma, an opening parenthesis or a space up to the reference. */
+const OWN_NUMBER = String.raw`\s+\d{1,5}`;
+const NUMBER_TO_REFERENCE = String.raw`(?:,\s*|\s*\(\s*|\s+)(?:§\s*)?$`;
+const NOT_INSIDE_A_WORD = String.raw`(?<![\p{L}\p{N}_.])`;
+const WORD_ENDS = String.raw`(?![\p{L}\p{N}_])`;
+
+/** "35 CFR " is a few characters. Only this much before a reference is read, however long the line. */
+const REACH = 40;
+
+const namesOf = (entries: readonly LexiconEntry[]): string | undefined => {
+  const names = entries.map((entry) => entry.pattern).toSorted((left, right) => right.length - left.length);
+  return names.length === 0 ? undefined : `(?:${names.map(escapeRegExp).join("|")})`;
+};
+
+/** An entry with position "before" is a name written before its own number ("RFC 1122"); the others follow a title number. */
+export const codeVocabulary = (lexicons: Readonly<Record<string, Lexicon>>): CodeVocabulary => {
+  const entries = lexicons["document-kind"] ?? [];
+  const titled = namesOf(entries.filter((entry) => entry.position !== "before"));
+  const numbered = namesOf(entries.filter((entry) => entry.position === "before"));
+  return {
+    before: titled === undefined ? undefined : new RegExp(String.raw`${NOT_INSIDE_A_WORD}(?<code>${TITLE_NUMBER}${titled})${SECOND_SIGN}`, "u"),
+    numberedBefore:
+      numbered === undefined ? undefined : new RegExp(String.raw`${NOT_INSIDE_A_WORD}(?<code>${numbered}${OWN_NUMBER})${NUMBER_TO_REFERENCE}`, "u"),
+    titled: titled === undefined ? undefined : new RegExp(String.raw`^${titled}${WORD_ENDS}`, "u"),
+  };
+};
+
+/** The code named right before the reference at `start`, or undefined when none is. */
+export const citedCodeBefore = (text: string, start: number, vocabulary: CodeVocabulary): string | undefined => {
+  const before = text.slice(Math.max(0, start - REACH), start);
+  return vocabulary.before?.exec(before)?.groups?.["code"] ?? vocabulary.numberedBefore?.exec(before)?.groups?.["code"];
+};
+
+/** "40 CFR § 163.25" at the start of a heading: the text after the number starts with a code named after a title number. */
+export const titledCodeAt = (rest: string, vocabulary: CodeVocabulary): boolean => vocabulary.titled?.test(rest) === true;

package/src/index.ts

@@ -1,14 +1,19 @@
 import { loadLexicons } from "./lexicons.ts";
 import { unmarkNumberStops } from "./number-stop.ts";
+import { labelStops, unmarkLabelStops } from "./label-stop.ts";
 import { sentenceSpans } from "./sentence-split.ts";
 import { splitAtQuotedStops } from "./quoted-stop.ts";
 import { reattachClosingQuotes } from "./closing-quote.ts";
 import { structure } from "./structure.ts";
 import { isReady, prepare, tokenize } from "./pos.ts";
-import type { AdapterNeeds, LanguageAdapter, Segmentation, Sentence } from "chaffjs/plugin";
+import { isJapaneseRun } from "./japanese-run.ts";
+import type { AdapterNeeds, EmbeddedLanguage, LanguageAdapter, Segmentation, Sentence, Span } from "chaffjs/plugin";
 
 // chaff からは型だけを取る。実行時の値依存を作らない。アダプタは単体で動く。
 
+const LEXICONS = loadLexicons();
+const LABEL_STOPS = labelStops((LEXICONS["abbreviated-label"] ?? []).map((entry) => entry.pattern));
+
 const LATIN_LETTER = /[a-z]/giu;
 const COUNTABLE = /\S/gu;
 
@@ -25,9 +30,13 @@ const withTokens = (sentence: Sentence): Sentence => {
   };
 };
 
+const JAPANESE: EmbeddedLanguage = { id: "ja", lengthUnit: "char" };
+
+const withLanguage = (text: string, span: Span): Sentence => (isJapaneseRun(text) ? { span, text, embeddedLanguage: JAPANESE } : { span, text });
+
 /**
  * 英語は sentence-splitter の既定にほぼ任せる。"Dr." "e.g." "U.S." "$3.50" を
- * いずれも文末と誤認しない。前処理は行の途中の番号を箇条書きと読ませること、後処理は閉じ引用符の内側で閉じた文を切ることと、文頭に取り残された閉じ引用符を前の文へ戻すこと。spec §7.2。
+ * いずれも文末と誤認しない。前処理は行の途中の番号を箇条書きと読ませることと、番号の前の略した名前（FIG. 1、Vol. XLIII）の点で切らないこと、後処理は閉じ引用符の内側で閉じた文を切ることと、文頭に取り残された閉じ引用符を前の文へ戻すこと。spec §7.2。
  */
 export const adapter: LanguageAdapter = {
   kind: "language",
@@ -51,11 +60,11 @@ export const adapter: LanguageAdapter = {
     if (total === 0) return 0;
     return [...source.matchAll(LATIN_LETTER)].length / total;
   },
-  lexicons: loadLexicons(),
+  lexicons: LEXICONS,
   structure,
   segment: (text: string): Segmentation => {
-    const quotedStops = sentenceSpans(unmarkNumberStops(text)).flatMap((span) => splitAtQuotedStops(text, span));
-    const sentences: Sentence[] = reattachClosingQuotes(text, quotedStops).map((span) => ({ span, text: text.slice(span.start, span.end) }));
+    const quotedStops = sentenceSpans(unmarkNumberStops(unmarkLabelStops(text, LABEL_STOPS))).flatMap((span) => splitAtQuotedStops(text, span));
+    const sentences: Sentence[] = reattachClosingQuotes(text, quotedStops).map((span) => withLanguage(text.slice(span.start, span.end), span));
     return { sentences: isReady() ? sentences.map(withTokens) : sentences };
   },
 };

package/src/japanese-run.ts

@@ -0,0 +1,9 @@
+/**
+ * A Japanese sentence in an English document (a quoted notice, a bilingual title): it has kana, and kana or kanji make
+ * up at least half of its letters (digits and symbols are not counted). Kanji alone do not count, as a Chinese name in English text is not Japanese.
+ */
+const KANA = /[\p{Script=Hiragana}\p{Script=Katakana}]/u;
+const JAPANESE = /[\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Han}]/gu;
+const LETTER = /\p{L}/gu;
+
+export const isJapaneseRun = (text: string): boolean => KANA.test(text) && [...text.matchAll(JAPANESE)].length * 2 >= [...text.matchAll(LETTER)].length;

package/src/label-stop.ts

@@ -0,0 +1,25 @@
+import { escapeRegExp } from "./regexp.ts";
+
+/**
+ * sentence-splitter ends a sentence at the full stop of a label before its number: "FIG." and "1 illustrates …" were two
+ * sentences, and so were "Vol." and "XLIII (1979)". Where a number follows, that full stop is replaced with a letter
+ * before splitting. The length does not change, so the spans fit the original text.
+ * A lone "I" after a label is the pronoun ("He said No. I left."), and a word ("the last Fig. The tree") is not a number.
+ */
+
+/** Built once from the lexicon; undefined when no label ends in a full stop. */
+export type LabelStops = RegExp | undefined;
+
+const NUMBER_AFTER = String.raw`(?=\s+(?:\d|(?:[IVXLCDM]{2,}|[VXLCDM])(?![\p{L}\p{N}_])))`;
+const PLAIN_LETTER = "n";
+
+/** The labels as written and in capitals, without their full stop ("Fig." → Fig, FIG). */
+export const labelStops = (labels: readonly string[]): LabelStops => {
+  const stems = labels.filter((label) => label.endsWith(".")).flatMap((label) => [label.slice(0, -1), label.slice(0, -1).toUpperCase()]);
+  if (stems.length === 0) return undefined;
+  return new RegExp(String.raw`(?<![\p{L}\p{N}_.])(?:${[...new Set(stems)].map(escapeRegExp).join("|")})\.${NUMBER_AFTER}`, "gu");
+};
+
+/** The text with the full stop of every label before a number replaced by a letter. */
+export const unmarkLabelStops = (text: string, stops: LabelStops): string =>
+  stops === undefined ? text : text.replace(stops, (label: string) => `${label.slice(0, -1)}${PLAIN_LETTER}`);

package/src/pos.ts

@@ -100,9 +100,22 @@ const BE = new Set(["be", "am", "is", "are", "was", "were", "been", "being"]);
 
 const isBe = (entry: Tagged): boolean => BE.has(entry.lemma ?? entry.value.toLowerCase()) || BE.has(entry.value.toLowerCase());
 
+/**
+ * end（0 以上）より前で test に合う最後の位置。無ければ -1。過去分詞の多い長い文で、分詞ごとに文の頭から写すと語数の二乗になるので、後ろから探す。
+ */
+const lastIndexBefore = (tagged: readonly Tagged[], end: number, test: (entry: Tagged) => boolean): number => {
+  let at = Math.min(end, tagged.length);
+  while (at > 0) {
+    at -= 1;
+    const entry = tagged[at];
+    if (entry !== undefined && test(entry)) return at;
+  }
+  return -1;
+};
+
 /** 過去分詞の前の be の位置。無ければ -1。 */
 const beBefore = (tagged: readonly Tagged[], at: number): number => {
-  const head = tagged.slice(0, at).findLastIndex((entry) => !SKIPPABLE.has(entry.pos));
+  const head = lastIndexBefore(tagged, at, (entry) => !SKIPPABLE.has(entry.pos));
   const entry = tagged[head];
   return entry !== undefined && isBe(entry) ? head : -1;
 };
@@ -125,11 +138,11 @@ const NOMINAL_TAG = new Set(["NN", "NNS", "NNP", "NNPS", "PRP", "CD", "DT"]);
 const RELATIVE_TAG = new Set(["WDT", "WP"]);
 
 const inRelativeClause = (tagged: readonly Tagged[], be: number): boolean => {
-  const lead = tagged.slice(0, be).findLastIndex((entry) => !isAuxiliary(entry));
+  const lead = lastIndexBefore(tagged, be, (entry) => !isAuxiliary(entry));
   const relative = tagged[lead];
   if (relative === undefined || !RELATIVE_TAG.has(relative.pos)) return false;
   // 文頭の That was decided. / Which was chosen? は、前に指す名詞が無いので述語。
-  const antecedent = tagged.slice(0, lead).findLast((entry) => entry.pos !== ",");
+  const antecedent = tagged[lastIndexBefore(tagged, lead, (entry) => entry.pos !== ",")];
   return antecedent !== undefined && NOMINAL_TAG.has(antecedent.pos);
 };
 
@@ -230,23 +243,26 @@ const withEmphasis = (token: Token): Token =>
  * wink は位置を返さないので、表層を順に照合して復元する。
  * 見つからないものは飛ばし、カーソルは進めない。位置の当てずっぽうを下流に流さない。
  */
-const locate = (text: string, tagged: readonly Tagged[]): Token[] =>
-  tagged.reduce<{ tokens: Token[]; cursor: number }>(
-    (acc, entry, at) => {
-      const start = text.indexOf(entry.value, acc.cursor);
-      if (start === -1) return acc;
-      const end = start + entry.value.length;
-      const token = {
-        span: { start, end },
-        surface: entry.value,
-        pos: properNounChecked(entry.value, upos(entry.pos)),
-        ...(entry.lemma === undefined ? {} : { lemma: entry.lemma }),
-        ...featuresOf(tagged, at),
-      };
-      return { tokens: [...acc.tokens, withEmphasis(token)], cursor: end };
-    },
-    { tokens: [], cursor: 0 },
-  ).tokens;
+const locate = (text: string, tagged: readonly Tagged[]): Token[] => {
+  // 語を足すたびに並びを作り直すと、長い文で語数の二乗になる。一つの並びに足していく。
+  const tokens: Token[] = [];
+  let cursor = 0;
+  tagged.forEach((entry, at) => {
+    const start = text.indexOf(entry.value, cursor);
+    if (start === -1) return;
+    const end = start + entry.value.length;
+    const token = {
+      span: { start, end },
+      surface: entry.value,
+      pos: properNounChecked(entry.value, upos(entry.pos)),
+      ...(entry.lemma === undefined ? {} : { lemma: entry.lemma }),
+      ...featuresOf(tagged, at),
+    };
+    tokens.push(withEmphasis(token));
+    cursor = end;
+  });
+  return tokens;
+};
 
 /** 英語の語はこれより長くならない。超える並びは語として読まない。 */
 const RUN_LIMIT = 1000;

package/src/regexp.ts

@@ -0,0 +1,2 @@
+/** A word from a lexicon, matched as written: a sign in it is that character, not a pattern. */
+export const escapeRegExp = (text: string): string => text.replace(/[.*+?^${}()|[\]\\]/gu, String.raw`\$&`);

package/src/structure.ts

@@ -1,5 +1,7 @@
 import type { Mention, NumberedLine, NumberingContext, StructurePatterns } from "chaffjs/plugin";
-import { citedDocumentAfter, citedDocumentBefore, hyphenatedTagAround } from "./citation.ts";
+import { citedDocumentAfter, citedDocumentBefore, listedTagAround } from "./citation.ts";
+import { citedCodeBefore, codeVocabulary, titledCodeAt } from "./code-citation.ts";
+import { loadLexicons } from "./lexicons.ts";
 import { membersAfter } from "./reference-list.ts";
 import { parseRoman } from "./roman.ts";
 import { dates } from "./dates.ts";
@@ -140,16 +142,19 @@ const glossedDocument = (gloss: Gloss): string | undefined => {
   return anchor !== undefined && gloss.depth > anchor.depth ? anchor.document : undefined;
 };
 
+const LEXICONS = loadLexicons();
+const CODES = codeVocabulary(LEXICONS);
+
 /** The other document a reference names, or else a bracketed tag that the core checks against the document's list. */
 const citation = (text: string, start: number, end: number, document: string | undefined): Readonly<Record<string, string>> => {
   if (document !== undefined) return { document };
-  const citedTag = hyphenatedTagAround(text, start, end);
+  const citedTag = listedTagAround(text, start, end);
   return citedTag === undefined ? {} : { citedTag };
 };
 
 /**
  * "Section 4.2(a)" → 4.2.a, "Article III" → 3. The same addresses the tree gives.
- * "Section 9 of the Master Agreement" carries the other document's name, and is not looked up in this tree.
+ * "Section 9 of the Master Agreement" and "35 CFR §122" carry the other document's name, and are not looked up in this tree.
  */
 const references = (text: string): Mention[] => {
   const gloss: Gloss = { depth: 0, scanned: 0, anchors: [] };
@@ -158,7 +163,7 @@ const references = (text: string): Mention[] => {
     if (main === undefined) return [];
     const { parts, end } = subdivisions(text, match.index + match[0].length);
     advance(gloss, text, match.index);
-    const cited = citedDocumentAfter(text, end) ?? citedDocumentBefore(text, match.index);
+    const cited = citedDocumentAfter(text, end) ?? citedDocumentBefore(text, match.index) ?? citedCodeBefore(text, match.index, CODES);
     const document = cited ?? glossedDocument(gloss);
     gloss.scanned = Math.max(gloss.scanned, end);
     if (cited !== undefined) gloss.anchors.push({ document: cited, depth: gloss.depth });
@@ -278,8 +283,19 @@ const quantities = (text: string): Mention[] =>
     return unit === undefined || Number.isNaN(value) || isWordChar(text[match.index - 1]) ? [] : [{ start: match.index, end, attrs: { value, unit } }];
   });
 
-/** "2.5 days" and "1.5 times" are amounts, not section 2.5 titled "days". */
-const countedAfter = (_number: string, rest: string): boolean => unitAfter(` ${rest}`, 0) !== undefined;
+const MEASURE_UNITS = (LEXICONS["measure-unit"] ?? []).map((entry) => entry.pattern);
+
+/** A letter, digit or hyphen right after the symbol makes it the start of a word: "2.1 mmap", "5.2.2.4 min-fresh". */
+const CONTINUES_WORD = /^[\p{Script=Latin}\p{Nd}_-]/u;
+
+const startsWithMeasureUnit = (rest: string): boolean => MEASURE_UNITS.some((unit) => rest.startsWith(unit) && !CONTINUES_WORD.test(rest.slice(unit.length)));
+
+/**
+ * "2.5 days", "1.5 times" and "1.5 mM in each" are amounts, not section 2.5 titled "days". "40 CFR § 163.25" is title 40 of
+ * another code, not section 40 of this document.
+ */
+const countedAfter = (_number: string, rest: string): boolean =>
+  unitAfter(` ${rest}`, 0) !== undefined || startsWithMeasureUnit(rest) || titledCodeAt(rest, CODES);
 
 export const structure: StructurePatterns = {
   numbered,