@whiteslove/parsing-lexicon 0.9.6 → 0.9.8

package/README.md

@@ -52,6 +52,8 @@ Lexical aliases belong here. Consumers keep source adapters, persistence, rankin
 
 The package must remain dependency-light and must never require a runtime lexicon HTTP service, Redis or a message broker.
 
+Location entries expose their alias-matching regex through a lazy `re` getter, compiled on first match rather than at merge/import time. Merge helpers must copy entry fields without using object spread on a full entry (`{ ...entry }`), since spread invokes getters and would compile every alias regex eagerly during module load. See `AUDIT.md` for the history of this constraint.
+
 ## Usage
 
 ```js

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@whiteslove/parsing-lexicon",
-  "version": "0.9.6",
+  "version": "0.9.8",
   "description": "Shared deterministic multilingual parsing lexicon for WhitesLove housing and hiring services",
   "repository": {
     "type": "git",
@@ -16,6 +16,7 @@
     },
     "./lexicon-core": "./src/lexicon-core.js",
     "./normalization": "./src/normalization.js",
+    "./alias-prefilter": "./src/alias-prefilter.js",
     "./parser-core": {
       "types": "./src/parser-core.d.ts",
       "import": "./src/parser-core.js"

package/src/alias-prefilter.js

@@ -0,0 +1,145 @@
+// Candidate prefilter for alias-backed location/entity lists.
+//
+// A country dictionary holds tens of thousands of entries, each carrying a
+// lazily compiled `re` built by aliasesToRegex(). Scanning a list with
+// `entries.find((entry) => entry.re.test(text))` therefore runs one large
+// alternation regex per entry — ~14.5k of them for a single Uzbek listing,
+// which costs seconds per call and dwarfs every other parsing step.
+//
+// Almost all of that work is provably wasted. aliasPattern() only relaxes
+// *separators* between alias words; the letters and digits of an alias are
+// matched literally (modulo the Karakalpak Latin equivalence classes and
+// case). So the longest run of letters/digits inside an alias must appear as a
+// contiguous substring of the text for that alias to match at all. Indexing
+// entries by the first few characters of that run lets a text's own character
+// n-grams select the handful of entries worth testing.
+//
+// The index is a filter, never a verdict: every surviving candidate is still
+// matched with its real `re`, in the list's original order, so results are
+// identical to a full scan. Folding is deliberately more aggressive than the
+// pattern's own equivalences (Cyrillic ё/ў/қ/ғ… are collapsed): over-merging
+// can only admit extra candidates, never discard a real match.
+
+import { normalizeUnicode } from './normalization.js';
+
+const GRAM = 4;
+
+const PREFILTER_FOLD = Object.freeze({
+  // Karakalpak Latin equivalences aliasPattern() encodes as character classes.
+  á: 'a', ǵ: 'g', ı: 'i', ń: 'n', ó: 'o', ú: 'u',
+  // Safe over-merging: these only widen the candidate set.
+  ё: 'е', ў: 'у', қ: 'к', ғ: 'г', ҳ: 'х', ә: 'а', і: 'и',
+  ң: 'н', ө: 'о', ұ: 'у', ү: 'у', һ: 'х', є: 'е', ї: 'и', ґ: 'г',
+});
+
+const NON_ALNUM_RE = /[^\p{L}\p{N}]+/u;
+
+function fold(value) {
+  let out = '';
+  for (const char of normalizeUnicode(value).toLocaleLowerCase()) {
+    out += PREFILTER_FOLD[char] ?? char;
+  }
+  return out;
+}
+
+/** The first GRAM characters of an alias's longest literal run, or null. */
+function aliasGram(alias) {
+  let longest = '';
+  for (const run of fold(alias).split(NON_ALNUM_RE)) {
+    if (run.length > longest.length) longest = run;
+  }
+  return longest.length >= GRAM ? longest.slice(0, GRAM) : null;
+}
+
+/** Every GRAM-length window of the folded text. */
+function textGrams(text) {
+  const folded = fold(text);
+  const grams = new Set();
+  for (let i = 0; i + GRAM <= folded.length; i += 1) {
+    grams.add(folded.slice(i, i + GRAM));
+  }
+  return grams;
+}
+
+function buildIndex(entries) {
+  const byGram = new Map();
+  // Entries whose every alias is shorter than GRAM cannot be indexed, so they
+  // are always tested. In practice this is a tiny tail.
+  const always = [];
+
+  for (let i = 0; i < entries.length; i += 1) {
+    const entry = entries[i];
+    const aliases = entry?.aliases?.length ? entry.aliases : [entry?.name].filter(Boolean);
+    const grams = [];
+    let indexable = aliases.length > 0;
+
+    for (const alias of aliases) {
+      const gram = aliasGram(alias);
+      if (!gram) {
+        indexable = false;
+        break;
+      }
+      grams.push(gram);
+    }
+
+    if (!indexable) {
+      always.push(i);
+      continue;
+    }
+    for (const gram of new Set(grams)) {
+      const bucket = byGram.get(gram);
+      if (bucket) bucket.push(i);
+      else byGram.set(gram, [i]);
+    }
+  }
+
+  return { byGram, always };
+}
+
+const INDEX_CACHE = new WeakMap();
+
+function indexFor(entries) {
+  let index = INDEX_CACHE.get(entries);
+  if (!index) {
+    index = buildIndex(entries);
+    INDEX_CACHE.set(entries, index);
+  }
+  return index;
+}
+
+/**
+ * First entry in `entries` whose alias regex matches `text`.
+ *
+ * Equivalent to `entries.find((entry) => entry.re.test(text))`, including the
+ * "first in list order wins" tie-break, but only compiles and runs the regexes
+ * of entries the text could plausibly contain.
+ *
+ * `accept(entry, matchedText)` optionally vets each hit before it wins. It
+ * receives the substring the alias regex actually matched, which is what a
+ * caller needs to tell a genuine name from an alias that merely repeats some
+ * other place's name; rejecting a hit continues the scan rather than ending it.
+ */
+export function matchFirstEntry(entries, text, accept) {
+  if (!Array.isArray(entries) || !entries.length) return undefined;
+  const value = String(text || '');
+  if (!value) return undefined;
+
+  const { byGram, always } = indexFor(entries);
+  const candidates = new Set(always);
+  for (const gram of textGrams(value)) {
+    const bucket = byGram.get(gram);
+    if (!bucket) continue;
+    for (const i of bucket) candidates.add(i);
+  }
+  if (!candidates.size) return undefined;
+
+  for (const i of [...candidates].sort((a, b) => a - b)) {
+    const entry = entries[i];
+    // `re` carries no /g flag, so exec() is stateless and safe to reuse here.
+    const match = entry?.re?.exec(value);
+    if (!match) continue;
+    if (accept && !accept(entry, match[0])) continue;
+    return entry;
+  }
+  return undefined;
+}

package/src/housing-intent.js

@@ -9,6 +9,15 @@ const UZ_CONTEXTUAL_RENT_OUT_RE = /(?:^|[^\p{L}\p{N}_])(?:ijaraga|ижарага
 const UZ_PER_DAY_RE = /(?:^|[^\p{L}\p{N}_])(?:kuniga|кунига)(?=$|[^\p{L}\p{N}_])/iu;
 const UZ_DAILY_RENT_PRICE_RE = /(?:narx|нарх|ijara|ижара|to['’`]?lov|т[ўу]лов|оплата)[^.!?\r\n]{0,48}(?:kuniga|кунига)|(?:kuniga|кунига)[^.!?\r\n]{0,48}(?:narx|нарх|ijara|ижара|to['’`]?lov|т[ўу]лов|оплата)/iu;
 
+// Short-stay wording is heavily inflected in RU/UK ("подобова оренда",
+// "посуточной аренды") and routinely abbreviated on catalogue cards
+// ("посут/почас", "подобово-погодинно"). HOUSING_DEAL_TYPES keeps the canonical
+// dictionary forms; enumerating every inflection there would be brittle, so
+// stems and established abbreviations are matched by pattern instead. A bare
+// day-rate mention ("сутки/суток") counts too: it outranks a source's generic
+// long-rent default even when nothing else resolves to shortRent.
+const EXPLICIT_SHORT_STAY_RE = /(?:^|[^\p{L}\p{N}_])(?:сут(?:ки|ок)|посуточн\p{L}*|почасов\p{L}*|подобов\p{L}*|погодинн\p{L}*|подобу|посут|почас)(?=$|[^\p{L}\p{N}_])/iu;
+
 // "ищу квартиру" / "шукаю квартиру" style aliases only match a literal,
 // adjacent phrase. Real posts routinely insert a room count or adjective
 // between the search verb and the housing noun ("Ищу 2-комнатную квартиру"),
@@ -115,7 +124,7 @@ export const HOUSING_DEAL_TYPES = Object.freeze([
   group('shortRent', {
     ru: ['посуточно', 'посуточная аренда', 'на сутки', 'на час', 'почасово', 'краткосрочно'],
     en: ['daily rent', 'short term', 'short-term rent', 'per day', 'hourly'],
-    uk: ['подобово', 'погодинно', 'на добу', 'на годину', 'короткострокова оренда'],
+    uk: ['подобово', 'подобова оренда', 'погодинно', 'на добу', 'на годину', 'короткострокова оренда'],
     ro: ['regim hotelier', 'pe zi', 'zilnic', 'pe noapte', 'închiriere pe termen scurt', 'inchiriere pe termen scurt'],
     uzLatn: ['kunlik', 'sutkaga', 'sutkalik', 'soatlik'],
     uzCyrl: ['кунлик', 'суткага', 'суткалик', 'соатлик'],
@@ -141,11 +150,19 @@ export function resolveHousingIntent(value) {
   const durationDeal = findCanonical(text, HOUSING_DEAL_TYPES, { partial: true });
   const hasContextualUzPerDay = UZ_PER_DAY_RE.test(text)
     && ((action === 'rentOut' || action === 'rentIn') || UZ_DAILY_RENT_PRICE_RE.test(text));
+  // An inflected/abbreviated short-stay stem is as authoritative as a
+  // dictionary alias, and outranks a longRent alias in the same text: a card
+  // reading "подобова оренда" is a day rental that happens to use the generic
+  // word for renting, not a long-term lease.
+  const hasShortStayStem = EXPLICIT_SHORT_STAY_RE.test(text);
+  const shortStay = durationDeal?.canonical === 'shortRent'
+    || hasContextualUzPerDay
+    || hasShortStayStem;
 
   if (action) {
     const base = HOUSING_ACTION_MAP[action];
     let dealType = base.dealType;
-    if ((durationDeal?.canonical === 'shortRent' || hasContextualUzPerDay) && (action === 'rentOut' || action === 'rentIn')) {
+    if (shortStay && (action === 'rentOut' || action === 'rentIn')) {
       dealType = 'shortRent';
     }
     return Object.freeze({
@@ -155,11 +172,11 @@ export function resolveHousingIntent(value) {
     });
   }
 
-  if (!durationDeal && !hasContextualUzPerDay) return null;
+  if (!durationDeal && !shortStay) return null;
   return Object.freeze({
     action: null,
     listingKind: null,
-    dealType: durationDeal?.canonical || 'shortRent',
+    dealType: shortStay ? 'shortRent' : durationDeal.canonical,
   });
 }
 
@@ -178,11 +195,6 @@ export function classifyHousingDealType(value) {
   return null;
 }
 
-// A bare "сутки/суток" (day-rate) mention outranks a source's generic
-// long-rent default, even when the rest of the text does not otherwise
-// resolve to shortRent.
-const EXPLICIT_SHORT_STAY_RE = /(?:^|[^\p{L}\p{N}_])сут(?:ки|ок)(?=$|[^\p{L}\p{N}_])/iu;
-
 export function looksExplicitDailyRentalMention(value) {
   return EXPLICIT_SHORT_STAY_RE.test(String(value || ''));
 }