@chaffjs/lang-en 0.14.0 → 0.16.0

package/src/pos.ts

@@ -5,6 +5,7 @@ import { loadLexicons } from "./lexicons.ts";
 import { blankLongRuns } from "./long-runs.ts";
 import { lowercasedAt, properNounChecked, rereadAt, sentenceInitialCommonWord } from "./proper-noun.ts";
 import { isStativeParticiple, stativeVocabulary } from "./stative-participle.ts";
+import { isEmphasisedAdverb } from "./emphasis.ts";
 
 const require = createRequire(import.meta.url);
 
@@ -99,9 +100,22 @@ const BE = new Set(["be", "am", "is", "are", "was", "were", "been", "being"]);
 
 const isBe = (entry: Tagged): boolean => BE.has(entry.lemma ?? entry.value.toLowerCase()) || BE.has(entry.value.toLowerCase());
 
+/**
+ * end（0 以上）より前で test に合う最後の位置。無ければ -1。過去分詞の多い長い文で、分詞ごとに文の頭から写すと語数の二乗になるので、後ろから探す。
+ */
+const lastIndexBefore = (tagged: readonly Tagged[], end: number, test: (entry: Tagged) => boolean): number => {
+  let at = Math.min(end, tagged.length);
+  while (at > 0) {
+    at -= 1;
+    const entry = tagged[at];
+    if (entry !== undefined && test(entry)) return at;
+  }
+  return -1;
+};
+
 /** 過去分詞の前の be の位置。無ければ -1。 */
 const beBefore = (tagged: readonly Tagged[], at: number): number => {
-  const head = tagged.slice(0, at).findLastIndex((entry) => !SKIPPABLE.has(entry.pos));
+  const head = lastIndexBefore(tagged, at, (entry) => !SKIPPABLE.has(entry.pos));
   const entry = tagged[head];
   return entry !== undefined && isBe(entry) ? head : -1;
 };
@@ -124,11 +138,11 @@ const NOMINAL_TAG = new Set(["NN", "NNS", "NNP", "NNPS", "PRP", "CD", "DT"]);
 const RELATIVE_TAG = new Set(["WDT", "WP"]);
 
 const inRelativeClause = (tagged: readonly Tagged[], be: number): boolean => {
-  const lead = tagged.slice(0, be).findLastIndex((entry) => !isAuxiliary(entry));
+  const lead = lastIndexBefore(tagged, be, (entry) => !isAuxiliary(entry));
   const relative = tagged[lead];
   if (relative === undefined || !RELATIVE_TAG.has(relative.pos)) return false;
   // 文頭の That was decided. / Which was chosen? は、前に指す名詞が無いので述語。
-  const antecedent = tagged.slice(0, lead).findLast((entry) => entry.pos !== ",");
+  const antecedent = tagged[lastIndexBefore(tagged, lead, (entry) => entry.pos !== ",")];
   return antecedent !== undefined && NOMINAL_TAG.has(antecedent.pos);
 };
 
@@ -164,6 +178,27 @@ const PLURAL_TAG = new Set(["NNS", "NNPS"]);
 
 const nounOrDeterminerFeatures = (entry: Tagged): Features => (PLURAL_TAG.has(entry.pos) ? { features: { Number: "Plur" } } : determinerFeatures(entry));
 
+const COMMON_NOUN_TAG = new Set(["NN", "NNS"]);
+const NOUN_TAG = new Set(["NN", "NNS", "NNP", "NNPS"]);
+const ADJECTIVE_TAG = new Set(["JJ", "JJR", "JJS"]);
+
+/**
+ * 解析器が一つに決めた読みの、ほかの読み。名詞と付けた語が動詞にもなる（works / report）なら AlsoVerb=Yes、
+ * 形容詞と付けた語が名詞にもなる（individual / key）なら AlsoNoun=Yes。an individual works は名詞と動詞とも読める。
+ * 語彙に無い名詞・形容詞（tribunal / stimuli）は、品詞も単数・複数も解析器が形から当てたものなので Guess=Yes。
+ */
+const otherReading = (entry: Tagged): Readonly<Record<string, string>> => {
+  const tags = state.vocabulary(entry.value.toLowerCase());
+  if (tags === undefined) return NOUN_TAG.has(entry.pos) || ADJECTIVE_TAG.has(entry.pos) ? { Guess: "Yes" } : {};
+  if (COMMON_NOUN_TAG.has(entry.pos) && tags.some((tag) => tag.startsWith("VB"))) return { AlsoVerb: "Yes" };
+  return ADJECTIVE_TAG.has(entry.pos) && tags.some((tag) => COMMON_NOUN_TAG.has(tag)) ? { AlsoNoun: "Yes" } : {};
+};
+
+const withOtherReading = (entry: Tagged, found: Features): Features => {
+  const other = otherReading(entry);
+  return Object.keys(other).length === 0 ? found : { features: { ...found.features, ...other } };
+};
+
 /**
  * 過去分詞は VerbForm=Part。Based on the review, のような分詞の導入句を、命令形の並び（fix the parser, ship it）と見分ける。
  * -ing 形は VerbForm=Ger。解析器は動名詞と現在分詞を分けないので、過去分詞を見る判断（Part）には混ぜない。
@@ -172,7 +207,7 @@ const featuresOf = (tagged: readonly Tagged[], at: number): Features => {
   const entry = tagged[at];
   if (entry === undefined) return {};
   if (entry.pos === "VBG") return { features: { VerbForm: "Ger" } };
-  if (entry.pos !== "VBN") return nounOrDeterminerFeatures(entry);
+  if (entry.pos !== "VBN") return withOtherReading(entry, nounOrDeterminerFeatures(entry));
   return { features: isPassive(tagged, at) ? { VerbForm: "Part", Voice: "Pass" } : { VerbForm: "Part" } };
 };
 
@@ -200,27 +235,34 @@ export const prepare = (): void => {
 
 export const isReady = (): boolean => state.ready !== undefined;
 
+/** 大文字で強調した副詞（NEVER）は、解析器が名前と付けても副詞。Emph=Yes は、略語ではないと detector に伝える。 */
+const withEmphasis = (token: Token): Token =>
+  isEmphasisedAdverb(token.surface, state.vocabulary) ? { ...token, pos: "ADV", lemma: token.surface.toLowerCase(), features: { Emph: "Yes" } } : token;
+
 /**
  * wink は位置を返さないので、表層を順に照合して復元する。
  * 見つからないものは飛ばし、カーソルは進めない。位置の当てずっぽうを下流に流さない。
  */
-const locate = (text: string, tagged: readonly Tagged[]): Token[] =>
-  tagged.reduce<{ tokens: Token[]; cursor: number }>(
-    (acc, entry, at) => {
-      const start = text.indexOf(entry.value, acc.cursor);
-      if (start === -1) return acc;
-      const end = start + entry.value.length;
-      const token = {
-        span: { start, end },
-        surface: entry.value,
-        pos: properNounChecked(entry.value, upos(entry.pos)),
-        ...(entry.lemma === undefined ? {} : { lemma: entry.lemma }),
-        ...featuresOf(tagged, at),
-      };
-      return { tokens: [...acc.tokens, token], cursor: end };
-    },
-    { tokens: [], cursor: 0 },
-  ).tokens;
+const locate = (text: string, tagged: readonly Tagged[]): Token[] => {
+  // 語を足すたびに並びを作り直すと、長い文で語数の二乗になる。一つの並びに足していく。
+  const tokens: Token[] = [];
+  let cursor = 0;
+  tagged.forEach((entry, at) => {
+    const start = text.indexOf(entry.value, cursor);
+    if (start === -1) return;
+    const end = start + entry.value.length;
+    const token = {
+      span: { start, end },
+      surface: entry.value,
+      pos: properNounChecked(entry.value, upos(entry.pos)),
+      ...(entry.lemma === undefined ? {} : { lemma: entry.lemma }),
+      ...featuresOf(tagged, at),
+    };
+    tokens.push(withEmphasis(token));
+    cursor = end;
+  });
+  return tokens;
+};
 
 /** 英語の語はこれより長くならない。超える並びは語として読まない。 */
 const RUN_LIMIT = 1000;

package/src/regexp.ts

@@ -0,0 +1,2 @@
+/** A word from a lexicon, matched as written: a sign in it is that character, not a pattern. */
+export const escapeRegExp = (text: string): string => text.replace(/[.*+?^${}()|[\]\\]/gu, String.raw`\$&`);

package/src/structure.ts

@@ -1,5 +1,7 @@
 import type { Mention, NumberedLine, NumberingContext, StructurePatterns } from "chaffjs/plugin";
-import { citedDocumentAfter, citedDocumentBefore, hyphenatedTagAround } from "./citation.ts";
+import { citedDocumentAfter, citedDocumentBefore, listedTagAround } from "./citation.ts";
+import { citedCodeBefore, codeVocabulary, titledCodeAt } from "./code-citation.ts";
+import { loadLexicons } from "./lexicons.ts";
 import { membersAfter } from "./reference-list.ts";
 import { parseRoman } from "./roman.ts";
 import { dates } from "./dates.ts";
@@ -140,16 +142,19 @@ const glossedDocument = (gloss: Gloss): string | undefined => {
   return anchor !== undefined && gloss.depth > anchor.depth ? anchor.document : undefined;
 };
 
+const LEXICONS = loadLexicons();
+const CODES = codeVocabulary(LEXICONS);
+
 /** The other document a reference names, or else a bracketed tag that the core checks against the document's list. */
 const citation = (text: string, start: number, end: number, document: string | undefined): Readonly<Record<string, string>> => {
   if (document !== undefined) return { document };
-  const citedTag = hyphenatedTagAround(text, start, end);
+  const citedTag = listedTagAround(text, start, end);
   return citedTag === undefined ? {} : { citedTag };
 };
 
 /**
  * "Section 4.2(a)" → 4.2.a, "Article III" → 3. The same addresses the tree gives.
- * "Section 9 of the Master Agreement" carries the other document's name, and is not looked up in this tree.
+ * "Section 9 of the Master Agreement" and "35 CFR §122" carry the other document's name, and are not looked up in this tree.
  */
 const references = (text: string): Mention[] => {
   const gloss: Gloss = { depth: 0, scanned: 0, anchors: [] };
@@ -158,7 +163,7 @@ const references = (text: string): Mention[] => {
     if (main === undefined) return [];
     const { parts, end } = subdivisions(text, match.index + match[0].length);
     advance(gloss, text, match.index);
-    const cited = citedDocumentAfter(text, end) ?? citedDocumentBefore(text, match.index);
+    const cited = citedDocumentAfter(text, end) ?? citedDocumentBefore(text, match.index) ?? citedCodeBefore(text, match.index, CODES);
     const document = cited ?? glossedDocument(gloss);
     gloss.scanned = Math.max(gloss.scanned, end);
     if (cited !== undefined) gloss.anchors.push({ document: cited, depth: gloss.depth });
@@ -278,8 +283,19 @@ const quantities = (text: string): Mention[] =>
     return unit === undefined || Number.isNaN(value) || isWordChar(text[match.index - 1]) ? [] : [{ start: match.index, end, attrs: { value, unit } }];
   });
 
-/** "2.5 days" and "1.5 times" are amounts, not section 2.5 titled "days". */
-const countedAfter = (_number: string, rest: string): boolean => unitAfter(` ${rest}`, 0) !== undefined;
+const MEASURE_UNITS = (LEXICONS["measure-unit"] ?? []).map((entry) => entry.pattern);
+
+/** A letter, digit or hyphen right after the symbol makes it the start of a word: "2.1 mmap", "5.2.2.4 min-fresh". */
+const CONTINUES_WORD = /^[\p{Script=Latin}\p{Nd}_-]/u;
+
+const startsWithMeasureUnit = (rest: string): boolean => MEASURE_UNITS.some((unit) => rest.startsWith(unit) && !CONTINUES_WORD.test(rest.slice(unit.length)));
+
+/**
+ * "2.5 days", "1.5 times" and "1.5 mM in each" are amounts, not section 2.5 titled "days". "40 CFR § 163.25" is title 40 of
+ * another code, not section 40 of this document.
+ */
+const countedAfter = (_number: string, rest: string): boolean =>
+  unitAfter(` ${rest}`, 0) !== undefined || startsWithMeasureUnit(rest) || titledCodeAt(rest, CODES);
 
 export const structure: StructurePatterns = {
   numbered,