@wirecat/cli-core 0.401.0 → 0.403.0

package/dist/store/sqlite/transcript-query.js

@@ -1,5 +1,5 @@
+import { dictionaryLanguages, knownWord } from "../../search/dictionary.js";
 import { isNumberWord, widenWords } from "../../search/garbled.js";
-import { knownRussianWord } from "../../search/russian-dictionary.js";
 import { createStemmer, DEFAULT_STEMMERS, STEM_SEPARATOR } from "../../search/stem.js";
 import { normalize } from "../normalize.js";
 import { ftsMatch, quoted, SPEECH, stemMatch } from "./lucene-fields.js";
@@ -9,6 +9,13 @@ const predicates = (node) => node.kind === "predicate" ? [node] : node.clauses.f
 const readsTranscripts = (node) => (node.operator === "term" || node.operator === "phrase") &&
     /[\p{L}\p{N}]/u.test(normalize(node.value)) &&
     (node.field === "transcript" || (node.implicit === true && (node.field === "text" || node.field === "exact")));
+/** `isNumberWord` knows Russian numbers only; a widened number is a wrong fact in any language. */
+const LATIN_NUMBERS = new Set(`zero one two three four five six seven eight nine ten eleven twelve twenty thirty forty fifty sixty seventy
+  eighty ninety hundred thousand million billion first second third fourth fifth sixth seventh eighth ninth tenth
+  half quarter cero uno una dos tres cuatro cinco seis siete ocho nueve diez once doce veinte treinta cuarenta
+  cincuenta sesenta setenta ochenta noventa cien ciento mil millon primero segundo tercero cuarto quinto medio`.split(/\s+/));
+/** A word widens when its alphabet has a dictionary to tell a garbled spelling from another real word. */
+const widens = (word, covered) => covered(word) && !isNumberWord(word) && !LATIN_NUMBERS.has(word);
 /**
  * Each leaf that reads transcripts, with what it matches there. A transcript always widens, not only when nothing
  * matched as typed text does: a correct and a garbled spelling sit in one chat. Only a single word widens —
@@ -20,7 +27,9 @@ export const transcriptTerms = (database, root, item) => {
     const saved = savedStemmers(database);
     if (item !== "message" || !leaves.length || saved === null)
         return terms;
-    const stemmer = createStemmer(saved ?? DEFAULT_STEMMERS);
+    const stemmers = saved ?? DEFAULT_STEMMERS;
+    const stemmer = createStemmer(stemmers);
+    const dictionary = knownWord(dictionaryLanguages(stemmers));
     const stemsOf = (word) => [...new Set(stemmer.explain(word).map((part) => part.stem))];
     const widening = [];
     for (const [node, one] of terms) {
@@ -31,8 +40,9 @@ export const transcriptTerms = (database, root, item) => {
         if (stems)
             Object.assign(one, { stems: node.slop === undefined ? stems : `(${stems}) NOT ${quoted(STEM_SEPARATOR)}` });
         const words = normalize(node.value).match(/[\p{L}\p{N}]+/gu) ?? [];
-        if (node.operator === "term" && words.length === 1 && !isNumberWord(words[0]))
-            widening.push({ node, literal: words[0], stems: stemsOf(node.value) });
+        const [word] = words;
+        if (node.operator === "term" && word !== undefined && words.length === 1 && widens(word, dictionary.covers))
+            widening.push({ node, literal: word, stems: stemsOf(node.value) });
     }
     if (!widening.length)
         return terms;
@@ -40,7 +50,7 @@ export const transcriptTerms = (database, root, item) => {
         .prepare("SELECT term, doc FROM transcript_words_vocab WHERE col = 'normalized_text'")
         .all()
         .map((row) => ({ term: String(row.term), docs: Number(row.doc) }));
-    widenWords(widening, vocabulary, stemsOf, knownRussianWord).forEach((words, at) => {
+    widenWords(widening, vocabulary, stemsOf, dictionary.known).forEach((words, at) => {
         const leaf = widening[at];
         if (leaf)
             Object.assign(terms.get(leaf.node), { widened: words });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@wirecat/cli-core",
-  "version": "0.401.0",
+  "version": "0.403.0",
   "description": "The messenger-neutral half of a messaging CLI: domain model, message locators, rendering, name resolution, the SQLite seam, the command skeleton and shared read commands, the send guard, run records and the message store",
   "license": "Apache-2.0",
   "author": "Viacheslav Ptsarev",
@@ -173,6 +173,8 @@
     "@xmldom/xmldom": "0.9.12",
     "chardet": "2.2.0",
     "defuddle": "0.19.5",
+    "dictionary-en": "4.0.0",
+    "dictionary-es": "4.0.0",
     "dictionary-ru": "3.0.0",
     "echarts": "6.1.0",
     "fflate": "0.8.3",