chaffjs 0.30.0 → 0.32.0

package/src/table-names.ts

@@ -1,6 +1,7 @@
 import { isNearWord, nameKey, type NameMention } from "./name-variants.ts";
 import type { TableCell, Token } from "./plugin.ts";
 import { withoutEdgeMarks } from "./facts/trim-marks.ts";
+import { NO_CHAR_READINGS, readingOfWords, type CharReadings } from "./name-char-reading.ts";
 
 // 表の升に書いた名前と、本文の名前の書き分け（本文は Sofia Mendes、担当者の表は Sofia Mendez）。表の升は品詞解析を通らないので、
 // 名前の形をした升だけを読み、本文で読めた名前と比べる。比べるのは姓と名のように二語以上の名前で、一語だけが違うときに限る。
@@ -22,20 +23,19 @@ const CJK_NAME = /^[\p{Script=Han}\p{Script=Katakana}ー々]+(?:[ \u3000][\p{Scr
 const SPACE = /[ \u3000]/u;
 const LOWER = /\p{Ll}/u;
 
-/** 範囲の中の語の読みをつないだもの。記号と空白の語は読まない。読めない語があれば無い。 */
-export const readingWithin = (tokens: readonly Token[], start: number, end: number): string | undefined => {
+/** 範囲の中の語の読みをつないだもの。記号と空白の語は読まない。読めない語があれば無い。charReadings は解析器が読めない名前の字の読み。 */
+export const readingWithin = (tokens: readonly Token[], start: number, end: number, charReadings = NO_CHAR_READINGS): string | undefined => {
   const words = tokens.filter((token) => token.span.start >= start && token.span.end <= end && token.pos !== "PUNCT" && token.surface.trim() !== "");
-  const readings = words.map((token) => token.reading);
-  return words.length > 0 && readings.every((reading) => reading !== undefined && reading !== "") ? readings.join("") : undefined;
+  return readingOfWords(words, charReadings);
 };
 
 /** 名前の形をした升。升の字の前後の空白と強調の印は名前の外。 */
-export const cellNamesIn = (cells: readonly TableCell[]): CellName[] =>
+export const cellNamesIn = (cells: readonly TableCell[], charReadings: CharReadings = NO_CHAR_READINGS): CellName[] =>
   cells.flatMap((cell) => {
     const surface = withoutEdgeMarks(cell.text.trim());
     if (!(LATIN_NAME.test(surface) && LOWER.test(surface)) && !CJK_NAME.test(surface)) return [];
     const offset = cell.start + cell.text.indexOf(surface);
-    return [{ surface, offset, words: surface.split(SPACE), reading: readingWithin(cell.tokens ?? [], offset, offset + surface.length) }];
+    return [{ surface, offset, words: surface.split(SPACE), reading: readingWithin(cell.tokens ?? [], offset, offset + surface.length, charReadings) }];
   });
 
 /**

package/src/unlabelled-references.ts

@@ -0,0 +1,83 @@
+// A document's own code (an item code, GP-0512) labelled once — after a label in the prose, or in a table under a label —
+// and written elsewhere, with no label, with two neighbours swapped (GP-0521). Pure: reads the document's text.
+import type { LexiconEntry } from "./plugin.ts";
+import { swapsNeighbours, withoutUrls } from "./phone-numbers.ts";
+import { linesOf } from "./structure/lines.ts";
+import { cellsOf, tablesOf, type Cell } from "./facts/table-facts.ts";
+import { withoutEdgeMarks } from "./facts/trim-marks.ts";
+import { charactersOf, MAX_CHARACTERS, MIN_DIGITS, REFERENCE_SHAPE, type ReferenceNumber, type ReferenceVariant } from "./reference-numbers.ts";
+
+const SEPARATOR = /[-‐−－]/u;
+const LATIN_LETTER = /[A-Z]/u;
+const DIGIT = /\d/gu;
+
+/** A code: letters, a hyphen and four digits or more (GP-0512). A plain number (48215, 2026-118) is never one. */
+const isCode = (written: string): boolean => {
+  const characters = charactersOf(written);
+  return (
+    SEPARATOR.test(written) && LATIN_LETTER.test(characters) && (characters.match(DIGIT)?.length ?? 0) >= MIN_DIGITS && characters.length <= MAX_CHARACTERS
+  );
+};
+
+const keyOf = (text: string): string => withoutEdgeMarks(text).normalize("NFKC").toLowerCase();
+
+/** The group of a cell that is a label and nothing else (| 品番 |, | Item code |). */
+const labelOfCell = (cell: Cell | undefined, labels: readonly LexiconEntry[]): string | undefined => {
+  if (cell === undefined) return undefined;
+  const entry = labels.find((label) => keyOf(label.pattern) === keyOf(cell.text));
+  return entry === undefined ? undefined : (entry.group ?? entry.pattern);
+};
+
+/** A cell holding one code and nothing else, as a reference number of the group. */
+const codeOfCell = (cell: Cell | undefined, group: string): ReferenceNumber[] => {
+  if (cell === undefined) return [];
+  const written = withoutEdgeMarks(cell.text);
+  if (!isCode(written) || [...written.matchAll(REFERENCE_SHAPE)].length !== 1) return [];
+  return [{ offset: cell.start + cell.text.indexOf(written), written, characters: charactersOf(written), label: group }];
+};
+
+/**
+ * The codes a table labels: in a row whose first cell is a label (| 品番 | GP-0512 |), and in a column whose heading is
+ * one (| 品番 | 品名 | over | GP-0512 | …). Pass the text with its tables (proseAndTablesOf).
+ */
+export const tableLabelledCodes = (text: string, labels: readonly LexiconEntry[]): ReferenceNumber[] =>
+  tablesOf(linesOf(text)).flatMap((table) => {
+    const headings = cellsOf(table.header).map((cell) => labelOfCell(cell, labels));
+    return table.rows.flatMap((row) => {
+      const cells = cellsOf(row);
+      const rowGroup = labelOfCell(cells[0], labels);
+      const byRow = rowGroup === undefined ? [] : codeOfCell(cells[1], rowGroup);
+      const byColumn = cells.flatMap((cell, index) => {
+        const group = headings[index];
+        return group === undefined ? [] : codeOfCell(cell, group);
+      });
+      return [...byRow, ...byColumn];
+    });
+  });
+
+/** The letters and digits of every code a cell holds. */
+const codesInCell = (cell: Cell): string[] =>
+  [...cell.text.matchAll(REFERENCE_SHAPE)].filter((match) => isCode(match[0])).map((match) => charactersOf(match[0]));
+
+/** Every code written anywhere in a table, its heading row too, labelled or not: a code a table lists is an item of its own, never a slip. */
+export const tableCodes = (text: string): Set<string> =>
+  new Set(tablesOf(linesOf(text)).flatMap((table) => [table.header, ...table.rows].flatMap((row) => cellsOf(row).flatMap(codesInCell))));
+
+/**
+ * Every code of the prose written with no label that is one labelled code with two neighbours swapped. Only when exactly
+ * one labelled code swaps to it, and when the writing is not itself a code the document labels or lists in a table: a
+ * catalog's AB-1234 and AB-1243 are two items.
+ */
+export const unlabelledVariants = (prose: string, labelled: readonly ReferenceNumber[], known: ReadonlySet<string>): ReferenceVariant[] => {
+  const labelledAt = new Set(labelled.map((number) => number.offset));
+  const anchors = [...new Map(labelled.map((number) => [number.characters, number])).values()];
+  return [...withoutUrls(prose).matchAll(REFERENCE_SHAPE)].flatMap((match) => {
+    if (labelledAt.has(match.index) || !isCode(match[0])) return [];
+    const characters = charactersOf(match[0]);
+    if (known.has(characters) || labelled.some((number) => number.characters === characters)) return [];
+    const swapped = anchors.filter((anchor) => swapsNeighbours(anchor.characters, characters));
+    const [other] = swapped;
+    if (swapped.length !== 1 || other === undefined) return [];
+    return [{ number: { offset: match.index, written: match[0], characters, label: other.label }, other }];
+  });
+};

package/src/vehicle-plates.ts

@@ -0,0 +1,142 @@
+// A vehicle's plate written two ways in one document (ひばり野 330 ね 48-15 and 48-75). Pure: reads the document's text; the
+// labels come from the reference-label lexicon and the plate's shapes from the vehicle-plate lexicon, each a pattern whose
+// named parts are the plate's parts. The part named serial is the one a slip changes; the others must be the same.
+import type { LexiconEntry } from "./plugin.ts";
+import { escapeRegExp } from "./orthography.ts";
+import { withoutUrls } from "./phone-numbers.ts";
+import type { ReferenceNumber } from "./reference-numbers.ts";
+import { boundedMatches } from "./custom/bounded-match.ts";
+
+/**
+ * A plate as written, in its shape's group. fixed is its parts but the serial; opensSentence whether its label opens its
+ * sentence (no word before it says whose plate it is, as 代車の does); onField whether its line holds only the label and the
+ * plate (登録番号：ひばり野 330 ね 48-15), the vehicle the document is about.
+ */
+export type VehiclePlate = ReferenceNumber & {
+  readonly fixed: string;
+  readonly serial: string;
+  readonly opensSentence: boolean;
+  readonly onField: boolean;
+};
+
+/** A plate that is the document's own with one character of its serial changed, and the document's own plate. */
+export type PlateVariant = { readonly number: VehiclePlate; readonly other: VehiclePlate };
+
+type LabelHit = { readonly start: number; readonly end: number };
+
+const SERIAL_PART = "serial";
+const PART_JOIN = "\u0000";
+const PLATE_GAP = /[\s\-‐−－・･.]/gu;
+/** Only spaces and a colon, comma or opening bracket between a label and its plate, and nothing alphanumeric right after it. */
+const BEFORE_PLATE = String.raw`^([\s:：,，、#＃(（「『[［]{0,4})`;
+const PLATE_END = String.raw`(?![0-9０-９A-Za-zＡ-Ｚａ-ｚ\-‐−－])`;
+const SENTENCE_END = /[。！？!?]|\.(?=\s)/u;
+const LATIN_START = /^[a-z]/u;
+const LETTER = /\p{Script=Latin}/u;
+const FLAGS = "u";
+
+/** Every place a label is written; a Latin label only as a whole word. */
+const labelHits = (text: string, labels: readonly LexiconEntry[]): LabelHit[] => {
+  const lowered = text.toLowerCase();
+  const isWhole = (label: string, at: number): boolean =>
+    !LATIN_START.test(label) || (!LETTER.test(lowered[at - 1] ?? "") && !LETTER.test(lowered[at + label.length] ?? ""));
+  return labels.flatMap((entry) => {
+    const label = entry.pattern.toLowerCase();
+    if (label === "") return [];
+    return [...lowered.matchAll(new RegExp(escapeRegExp(label), "gu"))]
+      .filter((match) => isWhole(label, match.index))
+      .map((match) => ({ start: match.index, end: match.index + label.length }));
+  });
+};
+
+const lineEnd = (text: string, offset: number): number => {
+  const end = text.indexOf("\n", offset);
+  return end === -1 ? text.length : end;
+};
+
+const plateCharacters = (written: string): string => written.normalize("NFKC").replaceAll(PLATE_GAP, "").toUpperCase();
+
+/** The parts of a plate but its serial, each evened out, so two plates of one vehicle's region, class and kana meet. */
+const fixedParts = (parts: Record<string, string | undefined>): string =>
+  Object.entries(parts)
+    .flatMap(([name, part]) => (name === SERIAL_PART || part === undefined ? [] : [plateCharacters(part)]))
+    .join(PART_JOIN);
+
+/** A shape's pattern read right after a label, or nothing when it is not a valid expression. */
+const compiledShape = (entry: LexiconEntry): RegExp | undefined => {
+  try {
+    return new RegExp(`${BEFORE_PLATE}(?:${entry.pattern})${PLATE_END}`, FLAGS);
+  } catch {
+    return undefined;
+  }
+};
+
+/** The plate a shape matched right after a label: matched is the label's separators and the plate. */
+const plateOf = (text: string, label: LabelHit, matched: string, shape: RegExp, group: string): VehiclePlate | undefined => {
+  const match = shape.exec(matched);
+  const serial = match?.groups?.[SERIAL_PART];
+  if (match === null || match.groups === undefined || serial === undefined) return undefined;
+  const offset = label.end + (match[1] ?? "").length;
+  const written = matched.slice((match[1] ?? "").length);
+  const lineStart = text.lastIndexOf("\n", label.start - 1) + 1;
+  const head = text.slice(lineStart, label.start);
+  const opensSentence = (head.split(SENTENCE_END).at(-1) ?? "").trim() === "";
+  const onField = head.trim() === "" && text.slice(offset + written.length, lineEnd(text, offset)).trim() === "";
+  return {
+    offset,
+    written,
+    characters: plateCharacters(written),
+    label: group,
+    fixed: fixedParts(match.groups),
+    serial: plateCharacters(serial),
+    opensSentence,
+    onField,
+  };
+};
+
+/**
+ * What one shape reads right after each label, in the labels' order. A shape comes from a lexicon a team may write, so it is
+ * matched with a time limit (boundedMatches), and one that is not a valid expression reads nothing.
+ */
+const readsOfShape = (text: string, hits: readonly LabelHit[], afters: readonly string[], entry: LexiconEntry): (VehiclePlate | undefined)[] => {
+  const shape = compiledShape(entry);
+  if (shape === undefined) return hits.map(() => undefined);
+  const matches = boundedMatches(shape.source, FLAGS, afters);
+  return hits.map((hit, index) => {
+    const matched = matches[index]?.[0];
+    return matched?.index === 0 ? plateOf(text, hit, matched.text, shape, entry.group ?? entry.pattern) : undefined;
+  });
+};
+
+/** The plates of a text, each right after a reference label in one of the plate shapes (the first that reads one wins). */
+export const labelledPlates = (source: string, labels: readonly LexiconEntry[], shapeEntries: readonly LexiconEntry[]): VehiclePlate[] => {
+  if (shapeEntries.length === 0) return [];
+  const text = withoutUrls(source);
+  // The longest label is read last, so the plate after 自動車登録番号 is read with it, not with 登録番号 inside it.
+  const hits = labelHits(text, labels).toSorted((a, b) => a.end - a.start - (b.end - b.start));
+  if (hits.length === 0) return [];
+  const afters = hits.map((hit) => text.slice(hit.end, lineEnd(text, hit.end)));
+  const readsByShape = shapeEntries.map((entry) => readsOfShape(text, hits, afters, entry));
+  const plates = hits.flatMap((_, index) => readsByShape.map((reads) => reads[index]).find((plate) => plate !== undefined) ?? []);
+  return [...new Map(plates.map((plate) => [plate.offset, plate])).values()].toSorted((a, b) => a.offset - b.offset);
+};
+
+/** Whether two serials of one length differ in one character only. */
+const oneCharacterApart = (a: string, b: string): boolean => a.length === b.length && [...a].filter((character, index) => character !== b[index]).length === 1;
+
+/**
+ * The writings of a plate that is the document's own with one character of its serial changed (48-15 and 48-75), the rest
+ * of the plate the same. Only when the document names one vehicle: its plate on a field line, one other plate and no third,
+ * the other never on a field line and its label always opening the sentence. Two different numbers one character apart are
+ * common, so nothing else is read.
+ */
+export const plateVariants = (plates: readonly VehiclePlate[]): PlateVariant[] => {
+  const distinct = new Set(plates.map((plate) => `${plate.label}${PART_JOIN}${plate.characters}`));
+  const fields = plates.filter((plate) => plate.onField);
+  const own = fields[0];
+  if (distinct.size !== 2 || own === undefined || fields.some((field) => field.characters !== own.characters)) return [];
+  const others = plates.filter((plate) => plate.characters !== own.characters);
+  const other = others[0];
+  if (other === undefined || other.label !== own.label || other.fixed !== own.fixed || !oneCharacterApart(other.serial, own.serial)) return [];
+  return others.every((plate) => plate.opensSentence) ? others.map((number) => ({ number, other: own })) : [];
+};

package/src/whole-word-runs.ts

@@ -0,0 +1,38 @@
+/** A word as the tokenizer gives it: what is written and where. */
+export type SpannedSurface = { readonly surface: string; readonly span: { readonly start: number; readonly end: number } };
+
+/** Where a listed word lies in the tokens (from the first token to the last, both included) and the word as written. */
+export type WholeWordRun = { readonly first: number; readonly last: number; readonly word: string };
+
+/** The listed words, as a set or keyed to what they map to. */
+export type ListedWords = ReadonlySet<string> | ReadonlyMap<string, string>;
+
+/** お + 嬢 + さま is the longest cut seen; a listed word is not looked for across more tokens than this. */
+export const MAX_WHOLE_WORD_PARTS = 3;
+
+const joinedSurface = (tokens: readonly SpannedSurface[], first: number, last: number): string =>
+  tokens
+    .slice(first, last + 1)
+    .map((token) => token.surface)
+    .join("");
+
+/** The longest listed word written by tokens from `first` on, each touching the one before. */
+const longestFrom = (tokens: readonly SpannedSurface[], first: number, words: ListedWords): WholeWordRun | undefined => {
+  const lasts = Array.from({ length: MAX_WHOLE_WORD_PARTS }, (_, extra) => first + extra).filter((last) => last < tokens.length);
+  const touching = lasts.filter((last) => tokens.slice(first + 1, last + 1).every((token, at) => tokens[first + at]?.span.end === token.span.start));
+  const written = touching.map((last) => ({ first, last, word: joinedSurface(tokens, first, last) }));
+  return written.filter(({ word }) => words.has(word)).at(-1);
+};
+
+/**
+ * The listed words in a sentence's tokens, each one run, however the tokenizer cut it: お客様 is one token and お客さま is お客 and
+ * さま, and the さま must not be read as the さま written after a name. Runs do not overlap; the earlier one wins.
+ */
+export const wholeWordRuns = (tokens: readonly SpannedSurface[], words: ListedWords): WholeWordRun[] =>
+  words.size === 0
+    ? []
+    : tokens.reduce<WholeWordRun[]>((runs, _, first) => {
+        if ((runs.at(-1)?.last ?? -1) >= first) return runs;
+        const run = longestFrom(tokens, first, words);
+        return run === undefined ? runs : [...runs, run];
+      }, []);