@chaffjs/lang-en 0.15.0 → 0.17.0

package/src/index.ts

@@ -1,14 +1,19 @@
 import { loadLexicons } from "./lexicons.ts";
 import { unmarkNumberStops } from "./number-stop.ts";
+import { labelStops, unmarkLabelStops } from "./label-stop.ts";
 import { sentenceSpans } from "./sentence-split.ts";
 import { splitAtQuotedStops } from "./quoted-stop.ts";
 import { reattachClosingQuotes } from "./closing-quote.ts";
 import { structure } from "./structure.ts";
 import { isReady, prepare, tokenize } from "./pos.ts";
-import type { AdapterNeeds, LanguageAdapter, Segmentation, Sentence } from "chaffjs/plugin";
+import { isJapaneseRun } from "./japanese-run.ts";
+import type { AdapterNeeds, EmbeddedLanguage, LanguageAdapter, Segmentation, Sentence, Span } from "chaffjs/plugin";
 
 // chaff からは型だけを取る。実行時の値依存を作らない。アダプタは単体で動く。
 
+const LEXICONS = loadLexicons();
+const LABEL_STOPS = labelStops((LEXICONS["abbreviated-label"] ?? []).map((entry) => entry.pattern));
+
 const LATIN_LETTER = /[a-z]/giu;
 const COUNTABLE = /\S/gu;
 
@@ -25,9 +30,13 @@ const withTokens = (sentence: Sentence): Sentence => {
   };
 };
 
+const JAPANESE: EmbeddedLanguage = { id: "ja", lengthUnit: "char" };
+
+const withLanguage = (text: string, span: Span): Sentence => (isJapaneseRun(text) ? { span, text, embeddedLanguage: JAPANESE } : { span, text });
+
 /**
  * 英語は sentence-splitter の既定にほぼ任せる。"Dr." "e.g." "U.S." "$3.50" を
- * いずれも文末と誤認しない。前処理は行の途中の番号を箇条書きと読ませること、後処理は閉じ引用符の内側で閉じた文を切ることと、文頭に取り残された閉じ引用符を前の文へ戻すこと。spec §7.2。
+ * いずれも文末と誤認しない。前処理は行の途中の番号を箇条書きと読ませることと、番号の前の略した名前（FIG. 1、Vol. XLIII）の点で切らないこと、後処理は閉じ引用符の内側で閉じた文を切ることと、文頭に取り残された閉じ引用符を前の文へ戻すこと。spec §7.2。
  */
 export const adapter: LanguageAdapter = {
   kind: "language",
@@ -51,11 +60,11 @@ export const adapter: LanguageAdapter = {
     if (total === 0) return 0;
     return [...source.matchAll(LATIN_LETTER)].length / total;
   },
-  lexicons: loadLexicons(),
+  lexicons: LEXICONS,
   structure,
   segment: (text: string): Segmentation => {
-    const quotedStops = sentenceSpans(unmarkNumberStops(text)).flatMap((span) => splitAtQuotedStops(text, span));
-    const sentences: Sentence[] = reattachClosingQuotes(text, quotedStops).map((span) => ({ span, text: text.slice(span.start, span.end) }));
+    const quotedStops = sentenceSpans(unmarkNumberStops(unmarkLabelStops(text, LABEL_STOPS))).flatMap((span) => splitAtQuotedStops(text, span));
+    const sentences: Sentence[] = reattachClosingQuotes(text, quotedStops).map((span) => withLanguage(text.slice(span.start, span.end), span));
     return { sentences: isReady() ? sentences.map(withTokens) : sentences };
   },
 };

package/src/japanese-run.ts

@@ -0,0 +1,9 @@
+/**
+ * A Japanese sentence in an English document (a quoted notice, a bilingual title): it has kana, and kana or kanji make
+ * up at least half of its letters (digits and symbols are not counted). Kanji alone do not count, as a Chinese name in English text is not Japanese.
+ */
+const KANA = /[\p{Script=Hiragana}\p{Script=Katakana}]/u;
+const JAPANESE = /[\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Han}]/gu;
+const LETTER = /\p{L}/gu;
+
+export const isJapaneseRun = (text: string): boolean => KANA.test(text) && [...text.matchAll(JAPANESE)].length * 2 >= [...text.matchAll(LETTER)].length;

package/src/label-stop.ts

@@ -0,0 +1,25 @@
+import { escapeRegExp } from "./regexp.ts";
+
+/**
+ * sentence-splitter ends a sentence at the full stop of a label before its number: "FIG." and "1 illustrates …" were two
+ * sentences, and so were "Vol." and "XLIII (1979)". Where a number follows, that full stop is replaced with a letter
+ * before splitting. The length does not change, so the spans fit the original text.
+ * A lone "I" after a label is the pronoun ("He said No. I left."), and a word ("the last Fig. The tree") is not a number.
+ */
+
+/** Built once from the lexicon; undefined when no label ends in a full stop. */
+export type LabelStops = RegExp | undefined;
+
+const NUMBER_AFTER = String.raw`(?=\s+(?:\d|(?:[IVXLCDM]{2,}|[VXLCDM])(?![\p{L}\p{N}_])))`;
+const PLAIN_LETTER = "n";
+
+/** The labels as written and in capitals, without their full stop ("Fig." → Fig, FIG). */
+export const labelStops = (labels: readonly string[]): LabelStops => {
+  const stems = labels.filter((label) => label.endsWith(".")).flatMap((label) => [label.slice(0, -1), label.slice(0, -1).toUpperCase()]);
+  if (stems.length === 0) return undefined;
+  return new RegExp(String.raw`(?<![\p{L}\p{N}_.])(?:${[...new Set(stems)].map(escapeRegExp).join("|")})\.${NUMBER_AFTER}`, "gu");
+};
+
+/** The text with the full stop of every label before a number replaced by a letter. */
+export const unmarkLabelStops = (text: string, stops: LabelStops): string =>
+  stops === undefined ? text : text.replace(stops, (label: string) => `${label.slice(0, -1)}${PLAIN_LETTER}`);

package/src/lexicons.ts

@@ -14,11 +14,15 @@ const toEntry = (raw: unknown): LexiconEntry | undefined => {
   if (!isRecord(raw) || typeof raw["pattern"] !== "string") return undefined;
   const weight = raw["weight"];
   const instead = raw["instead_of"];
+  const rewrite = raw["rewrite"];
+  const group = raw["group"];
   return {
     pattern: raw["pattern"],
     weight: typeof weight === "number" ? weight : undefined,
     instead_of: typeof instead === "string" ? instead : undefined,
     position: POSITIONS.find((position) => position === raw["position"]),
+    rewrite: typeof rewrite === "string" ? rewrite : undefined,
+    ...(typeof group === "string" ? { group } : {}),
   };
 };
 

package/src/pos.ts

@@ -100,9 +100,22 @@ const BE = new Set(["be", "am", "is", "are", "was", "were", "been", "being"]);
 
 const isBe = (entry: Tagged): boolean => BE.has(entry.lemma ?? entry.value.toLowerCase()) || BE.has(entry.value.toLowerCase());
 
+/**
+ * end（0 以上）より前で test に合う最後の位置。無ければ -1。過去分詞の多い長い文で、分詞ごとに文の頭から写すと語数の二乗になるので、後ろから探す。
+ */
+const lastIndexBefore = (tagged: readonly Tagged[], end: number, test: (entry: Tagged) => boolean): number => {
+  let at = Math.min(end, tagged.length);
+  while (at > 0) {
+    at -= 1;
+    const entry = tagged[at];
+    if (entry !== undefined && test(entry)) return at;
+  }
+  return -1;
+};
+
 /** 過去分詞の前の be の位置。無ければ -1。 */
 const beBefore = (tagged: readonly Tagged[], at: number): number => {
-  const head = tagged.slice(0, at).findLastIndex((entry) => !SKIPPABLE.has(entry.pos));
+  const head = lastIndexBefore(tagged, at, (entry) => !SKIPPABLE.has(entry.pos));
   const entry = tagged[head];
   return entry !== undefined && isBe(entry) ? head : -1;
 };
@@ -125,11 +138,11 @@ const NOMINAL_TAG = new Set(["NN", "NNS", "NNP", "NNPS", "PRP", "CD", "DT"]);
 const RELATIVE_TAG = new Set(["WDT", "WP"]);
 
 const inRelativeClause = (tagged: readonly Tagged[], be: number): boolean => {
-  const lead = tagged.slice(0, be).findLastIndex((entry) => !isAuxiliary(entry));
+  const lead = lastIndexBefore(tagged, be, (entry) => !isAuxiliary(entry));
   const relative = tagged[lead];
   if (relative === undefined || !RELATIVE_TAG.has(relative.pos)) return false;
   // 文頭の That was decided. / Which was chosen? は、前に指す名詞が無いので述語。
-  const antecedent = tagged.slice(0, lead).findLast((entry) => entry.pos !== ",");
+  const antecedent = tagged[lastIndexBefore(tagged, lead, (entry) => entry.pos !== ",")];
   return antecedent !== undefined && NOMINAL_TAG.has(antecedent.pos);
 };
 
@@ -230,23 +243,26 @@ const withEmphasis = (token: Token): Token =>
  * wink は位置を返さないので、表層を順に照合して復元する。
  * 見つからないものは飛ばし、カーソルは進めない。位置の当てずっぽうを下流に流さない。
  */
-const locate = (text: string, tagged: readonly Tagged[]): Token[] =>
-  tagged.reduce<{ tokens: Token[]; cursor: number }>(
-    (acc, entry, at) => {
-      const start = text.indexOf(entry.value, acc.cursor);
-      if (start === -1) return acc;
-      const end = start + entry.value.length;
-      const token = {
-        span: { start, end },
-        surface: entry.value,
-        pos: properNounChecked(entry.value, upos(entry.pos)),
-        ...(entry.lemma === undefined ? {} : { lemma: entry.lemma }),
-        ...featuresOf(tagged, at),
-      };
-      return { tokens: [...acc.tokens, withEmphasis(token)], cursor: end };
-    },
-    { tokens: [], cursor: 0 },
-  ).tokens;
+const locate = (text: string, tagged: readonly Tagged[]): Token[] => {
+  // 語を足すたびに並びを作り直すと、長い文で語数の二乗になる。一つの並びに足していく。
+  const tokens: Token[] = [];
+  let cursor = 0;
+  tagged.forEach((entry, at) => {
+    const start = text.indexOf(entry.value, cursor);
+    if (start === -1) return;
+    const end = start + entry.value.length;
+    const token = {
+      span: { start, end },
+      surface: entry.value,
+      pos: properNounChecked(entry.value, upos(entry.pos)),
+      ...(entry.lemma === undefined ? {} : { lemma: entry.lemma }),
+      ...featuresOf(tagged, at),
+    };
+    tokens.push(withEmphasis(token));
+    cursor = end;
+  });
+  return tokens;
+};
 
 /** 英語の語はこれより長くならない。超える並びは語として読まない。 */
 const RUN_LIMIT = 1000;

package/src/regexp.ts

@@ -0,0 +1,2 @@
+/** A word from a lexicon, matched as written: a sign in it is that character, not a pattern. */
+export const escapeRegExp = (text: string): string => text.replace(/[.*+?^${}()|[\]\\]/gu, String.raw`\$&`);

package/src/structure.ts

@@ -1,5 +1,7 @@
 import type { Mention, NumberedLine, NumberingContext, StructurePatterns } from "chaffjs/plugin";
-import { citedDocumentAfter, citedDocumentBefore, hyphenatedTagAround } from "./citation.ts";
+import { citedDocumentAfter, citedDocumentBefore, listedTagAround } from "./citation.ts";
+import { citedCodeBefore, codeVocabulary, titledCodeAt } from "./code-citation.ts";
+import { loadLexicons } from "./lexicons.ts";
 import { membersAfter } from "./reference-list.ts";
 import { parseRoman } from "./roman.ts";
 import { dates } from "./dates.ts";
@@ -140,16 +142,19 @@ const glossedDocument = (gloss: Gloss): string | undefined => {
   return anchor !== undefined && gloss.depth > anchor.depth ? anchor.document : undefined;
 };
 
+const LEXICONS = loadLexicons();
+const CODES = codeVocabulary(LEXICONS);
+
 /** The other document a reference names, or else a bracketed tag that the core checks against the document's list. */
 const citation = (text: string, start: number, end: number, document: string | undefined): Readonly<Record<string, string>> => {
   if (document !== undefined) return { document };
-  const citedTag = hyphenatedTagAround(text, start, end);
+  const citedTag = listedTagAround(text, start, end);
   return citedTag === undefined ? {} : { citedTag };
 };
 
 /**
  * "Section 4.2(a)" → 4.2.a, "Article III" → 3. The same addresses the tree gives.
- * "Section 9 of the Master Agreement" carries the other document's name, and is not looked up in this tree.
+ * "Section 9 of the Master Agreement" and "35 CFR §122" carry the other document's name, and are not looked up in this tree.
  */
 const references = (text: string): Mention[] => {
   const gloss: Gloss = { depth: 0, scanned: 0, anchors: [] };
@@ -158,7 +163,7 @@ const references = (text: string): Mention[] => {
     if (main === undefined) return [];
     const { parts, end } = subdivisions(text, match.index + match[0].length);
     advance(gloss, text, match.index);
-    const cited = citedDocumentAfter(text, end) ?? citedDocumentBefore(text, match.index);
+    const cited = citedDocumentAfter(text, end) ?? citedDocumentBefore(text, match.index) ?? citedCodeBefore(text, match.index, CODES);
     const document = cited ?? glossedDocument(gloss);
     gloss.scanned = Math.max(gloss.scanned, end);
     if (cited !== undefined) gloss.anchors.push({ document: cited, depth: gloss.depth });
@@ -278,8 +283,19 @@ const quantities = (text: string): Mention[] =>
     return unit === undefined || Number.isNaN(value) || isWordChar(text[match.index - 1]) ? [] : [{ start: match.index, end, attrs: { value, unit } }];
   });
 
-/** "2.5 days" and "1.5 times" are amounts, not section 2.5 titled "days". */
-const countedAfter = (_number: string, rest: string): boolean => unitAfter(` ${rest}`, 0) !== undefined;
+const MEASURE_UNITS = (LEXICONS["measure-unit"] ?? []).map((entry) => entry.pattern);
+
+/** A letter, digit or hyphen right after the symbol makes it the start of a word: "2.1 mmap", "5.2.2.4 min-fresh". */
+const CONTINUES_WORD = /^[\p{Script=Latin}\p{Nd}_-]/u;
+
+const startsWithMeasureUnit = (rest: string): boolean => MEASURE_UNITS.some((unit) => rest.startsWith(unit) && !CONTINUES_WORD.test(rest.slice(unit.length)));
+
+/**
+ * "2.5 days", "1.5 times" and "1.5 mM in each" are amounts, not section 2.5 titled "days". "40 CFR § 163.25" is title 40 of
+ * another code, not section 40 of this document.
+ */
+const countedAfter = (_number: string, rest: string): boolean =>
+  unitAfter(` ${rest}`, 0) !== undefined || startsWithMeasureUnit(rest) || titledCodeAt(rest, CODES);
 
 export const structure: StructurePatterns = {
   numbered,