@whiteslove/parsing-lexicon 0.9.0 → 0.9.3

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@whiteslove/parsing-lexicon",
-  "version": "0.9.0",
+  "version": "0.9.3",
   "description": "Shared deterministic multilingual parsing lexicon for WhitesLove housing and hiring services",
   "repository": {
     "type": "git",
@@ -197,6 +197,10 @@
       "types": "./src/semantic-spans.d.ts",
       "import": "./src/semantic-spans.js"
     },
+    "./address-grammar": {
+      "types": "./src/address-grammar.d.ts",
+      "import": "./src/address-grammar.js"
+    },
     "./hiring-professions": "./src/hiring-professions.js",
     "./hiring-profession-display": {
       "types": "./src/hiring-profession-display.d.ts",

package/src/address-grammar.d.ts

@@ -0,0 +1,13 @@
+export type AddressGrammarLanguage = 'ru' | 'uk' | 'ro' | 'en' | 'kk' | 'ky' | 'uzLatn' | 'uzCyrl';
+export type AddressGrammarMarker = Readonly<{ pattern: string; lang: AddressGrammarLanguage }>;
+
+export const STREET_PREFIX_MARKERS: readonly AddressGrammarMarker[];
+export const STREET_POSTFIX_MARKERS: readonly AddressGrammarMarker[];
+export const STREET_TYPE_MARKERS: readonly AddressGrammarMarker[];
+export const HOUSE_MARKERS: readonly AddressGrammarMarker[];
+export const BUILDING_MARKERS: readonly AddressGrammarMarker[];
+
+export const COUNTRY_ADDRESS_LANGUAGES: Readonly<Record<string, readonly AddressGrammarLanguage[]>>;
+
+export function combinedMarkerPattern(markers: readonly AddressGrammarMarker[]): string;
+export function matchesCountryAddressLanguage(text: unknown, country: unknown, markers: readonly AddressGrammarMarker[]): boolean;

package/src/address-grammar.js

@@ -0,0 +1,139 @@
+/**
+ * Country/language-scoped address grammar data, extracted from
+ * housing-address.js's combined marker regexes. This is data, not logic:
+ * housing-address.js still runs one shared parser engine and still builds
+ * the exact same combined patterns from these fragments (verified
+ * byte-identical to the pre-extraction hardcoded regex source in
+ * address-grammar.test.js) — nothing about matching behavior changes.
+ *
+ * The value of splitting it out is making "which markers belong to which
+ * language/country" an explicit, inspectable data structure instead of
+ * only implicit in one long alternation, so a caller that already knows
+ * the listing's country can use STREET_GRAMMAR_LANGUAGES_BY_COUNTRY as
+ * scoring evidence (e.g. "this address's markers are consistent with the
+ * supplied country") — a prior, per the plan, never a rigid requirement,
+ * since messy marketplace text routinely mixes languages regardless of
+ * country.
+ */
+
+// Each marker fragment is tagged by the language it belongs to (not the
+// country) because that's what's actually true of the text — "ул./улица"
+// is a Russian-language marker that shows up in UZ, KZ, KG and UA listings
+// alike, not a marker exclusive to one country.
+export const STREET_PREFIX_MARKERS = Object.freeze([
+  { pattern: 'ул(?:ица)?', lang: 'ru' },
+  { pattern: 'вул(?:иця)?', lang: 'uk' },
+  { pattern: 'пр', lang: 'ru' },
+  { pattern: 'просп(?:ект)?', lang: 'ru' },
+  { pattern: 'пр-т', lang: 'ru' },
+  { pattern: 'переул(?:ок)?', lang: 'ru' },
+  { pattern: 'пров(?:улок)?', lang: 'uk' },
+  { pattern: 'проезд', lang: 'ru' },
+  { pattern: 'наб(?:ережная)?', lang: 'ru' },
+  { pattern: 'шоссе', lang: 'ru' },
+  { pattern: "str(?:ada)?", lang: 'ro' },
+  { pattern: 'street', lang: 'en' },
+  { pattern: 'st', lang: 'en' },
+  { pattern: 'avenue', lang: 'en' },
+  { pattern: 'ave', lang: 'en' },
+  { pattern: 'road', lang: 'en' },
+  { pattern: 'rd', lang: 'en' },
+  { pattern: 'көше', lang: 'kk' },
+]);
+
+export const STREET_POSTFIX_MARKERS = Object.freeze([
+  { pattern: 'ko[\'’ʼ\\u02bc]?cha(?:si)?', lang: 'uzLatn' },
+  { pattern: 'кўча(?:си)?', lang: 'uzCyrl' },
+  { pattern: 'коча(?:си)?', lang: 'uzCyrl' },
+  { pattern: 'kocha(?:si)?', lang: 'uzLatn' },
+  { pattern: 'көше(?:сі)?', lang: 'kk' },
+  { pattern: 'көшесі', lang: 'kk' },
+  { pattern: 'көчө(?:сү)?', lang: 'ky' },
+]);
+
+export const STREET_TYPE_MARKERS = Object.freeze([
+  { pattern: 'вулиця', lang: 'uk' },
+  { pattern: 'улица', lang: 'ru' },
+  { pattern: 'провулок', lang: 'uk' },
+  { pattern: 'переулок', lang: 'ru' },
+  { pattern: 'проспект', lang: 'ru' },
+  { pattern: 'бульвар', lang: 'ru' },
+  { pattern: 'набережна', lang: 'uk' },
+  { pattern: 'набережная', lang: 'ru' },
+  { pattern: 'шосе', lang: 'uk' },
+  { pattern: 'шоссе', lang: 'ru' },
+  { pattern: 'площа', lang: 'uk' },
+  { pattern: 'площадь', lang: 'ru' },
+  { pattern: 'узвіз', lang: 'uk' },
+  { pattern: 'спуск', lang: 'ru' },
+  { pattern: 'алея', lang: 'uk' },
+  { pattern: 'аллея', lang: 'ru' },
+  { pattern: 'дорога', lang: 'ru' },
+  { pattern: 'тупик', lang: 'ru' },
+  { pattern: 'көше(?:сі)?', lang: 'kk' },
+  { pattern: 'көшесі', lang: 'kk' },
+  { pattern: 'көчө(?:сү)?', lang: 'ky' },
+]);
+
+export const HOUSE_MARKERS = Object.freeze([
+  { pattern: 'дом', lang: 'ru' },
+  { pattern: 'д\\.', lang: 'ru' },
+  { pattern: 'будинок', lang: 'uk' },
+  { pattern: 'буд\\.', lang: 'uk' },
+  { pattern: 'house', lang: 'en' },
+  { pattern: 'h\\.', lang: 'en' },
+  { pattern: 'uy', lang: 'uzLatn' },
+  { pattern: 'уй', lang: 'uzCyrl' },
+  { pattern: 'үй', lang: 'kk' },
+  { pattern: 'nr\\.?', lang: 'ro' },
+  { pattern: 'no\\.?', lang: 'en' },
+  { pattern: '№', lang: 'ru' },
+]);
+
+export const BUILDING_MARKERS = Object.freeze([
+  { pattern: 'корп(?:ус)?\\.?', lang: 'ru' },
+  { pattern: 'к\\.', lang: 'ru' },
+  { pattern: 'строен(?:ие)?', lang: 'ru' },
+  { pattern: 'стр\\.', lang: 'ru' },
+  { pattern: 'будова', lang: 'uk' },
+  { pattern: 'секц(?:ия|ія)?', lang: 'ru' },
+  { pattern: 'bloc', lang: 'ro' },
+  { pattern: 'corp', lang: 'ro' },
+  { pattern: 'building', lang: 'en' },
+  { pattern: 'bldg\\.?', lang: 'en' },
+  { pattern: 'korpus', lang: 'uzLatn' },
+  { pattern: 'bino', lang: 'uzLatn' },
+  { pattern: 'bina', lang: 'uzLatn' },
+  { pattern: 'бино', lang: 'uzCyrl' },
+]);
+
+// Which languages a given priority country's listings commonly mix, in the
+// order they're worth checking first. Deliberately not exhaustive or
+// exclusive — messy marketplace text can and does use any language
+// regardless of country, so this is scoring-prior data, not a filter.
+export const COUNTRY_ADDRESS_LANGUAGES = Object.freeze({
+  UZ: Object.freeze(['uzLatn', 'uzCyrl', 'ru']),
+  KZ: Object.freeze(['kk', 'ru']),
+  KG: Object.freeze(['ky', 'ru']),
+  UA: Object.freeze(['uk', 'ru']),
+  RO: Object.freeze(['ro']),
+});
+
+function joinMarkerGroup(markers) {
+  return markers.map((marker) => marker.pattern).join('|');
+}
+
+/** Rebuild the exact combined alternation source housing-address.js uses internally, from the tagged marker data above. */
+export function combinedMarkerPattern(markers) {
+  return joinMarkerGroup(markers);
+}
+
+/** True when `text` contains at least one marker tagged for one of `country`'s priority languages. */
+export function matchesCountryAddressLanguage(text, country, markers) {
+  const languages = COUNTRY_ADDRESS_LANGUAGES[String(country || '').toUpperCase()];
+  if (!languages?.length) return false;
+  const relevant = markers.filter((marker) => languages.includes(marker.lang));
+  if (!relevant.length) return false;
+  const pattern = new RegExp(`(?:${joinMarkerGroup(relevant)})`, 'iu');
+  return pattern.test(String(text || ''));
+}

package/src/central-asia-locations.js

@@ -1,5 +1,5 @@
 import { LOCATION_DICTIONARIES } from './locations-runtime.js';
-import { LOCATION_LIST_KEYS } from './location-merge.js';
+import { isMapDataEntry, LOCATION_LIST_KEYS } from './location-merge.js';
 import { CITIES_BY_COUNTRY, canonicalCity } from './geography.js';
 import { aliasesOf, aliasesToRegex, normalizeForMatch } from './normalization.js';
 import { KZ_AMBIGUOUS_LOCAL_NAMES, KZ_SEARCH_CLUSTERS } from './kz-location-extensions.js';
@@ -227,12 +227,24 @@ function publicMatch(candidate) {
   return Object.freeze(result);
 }
 
-function findEntryMatches(text, cityName, data) {
+function mapDataMatch(value, normalizedValue, item) {
+  const paddedValue = ` ${normalizedValue} `;
+  for (const alias of item.aliases || []) {
+    const normalizedAlias = normalizeForMatch(alias);
+    const index = paddedValue.indexOf(` ${normalizedAlias} `);
+    if (normalizedAlias && index >= 0) return { 0: normalizedAlias, index };
+  }
+  return null;
+}
+
+function findEntryMatches(text, cityName, data, { includeMapData = false } = {}) {
   const value = String(text || '');
+  const normalizedValue = normalizeForMatch(value);
   const raw = [];
   for (const key of LOCATION_LIST_KEYS) {
     for (const item of data?.[key] || []) {
-      const match = value.match(item?.re)
+      if (isMapDataEntry(item) && !includeMapData) continue;
+      const match = (isMapDataEntry(item) ? mapDataMatch(value, normalizedValue, item) : value.match(item?.re))
         || (key === 'residentialComplexes' ? markedResidentialMatch(value, item) : null);
       if (!match) continue;
       const { start, end } = semanticBounds(value, match);
@@ -319,7 +331,7 @@ export function matchCentralAsiaLocationEntities(text, countryCode, preferredCit
   const scopedCity = preferred && country[preferred] ? preferred : explicit && country[explicit] ? explicit : null;
 
   if (scopedCity) {
-    const matches = findEntryMatches(text, scopedCity, country[scopedCity]);
+    const matches = findEntryMatches(text, scopedCity, country[scopedCity], { includeMapData: true });
     const clusters = clusterMatches(matches, countryCode);
     return Object.freeze({ city: scopedCity, matches: Object.freeze(matches), searchClusters: Object.freeze(clusters), candidates: Object.freeze([]) });
   }

package/src/housing-address.js

@@ -5,14 +5,28 @@ import {
   matchTashkentNumberedArea,
 } from './tashkent-housing-geography.js';
 import { detectNonAddressSpans } from './semantic-spans.js';
+import {
+  BUILDING_MARKERS,
+  HOUSE_MARKERS,
+  STREET_POSTFIX_MARKERS,
+  STREET_PREFIX_MARKERS,
+  STREET_TYPE_MARKERS,
+  combinedMarkerPattern,
+} from './address-grammar.js';
 
 const PHONE_RUN_RE = /\+?\d[\d\s().-]{7,}\d/gu;
 const ADDRESS_LABEL_RE = /(?:адрес|адреса|адресація|адресация|manzil|address|adresă|adresa)\s*[:=\-–—]\s*/iu;
-const PREFIX_STREET_MARKER = String.raw`(?:(?:ул(?:ица)?|вул(?:иця)?|пр|просп(?:ект)?|пр-т|переул(?:ок)?|пров(?:улок)?|проезд|наб(?:ережная)?|шоссе|str(?:ada)?|street|st|avenue|ave|road|rd|көше)\.?)`;
-const POSTFIX_STREET_MARKER = String.raw`(?:ko['’ʼ\u02bc]?cha(?:si)?|кўча(?:си)?|коча(?:си)?|kocha(?:si)?|көше(?:сі)?|көшесі|көчө(?:сү)?)`;
-const POSTFIX_STREET_TYPE = String.raw`(?:вулиця|улица|провулок|переулок|проспект|бульвар|набережна|набережная|шосе|шоссе|площа|площадь|узвіз|спуск|алея|аллея|дорога|тупик|көше(?:сі)?|көшесі|көчө(?:сү)?)`;
-const HOUSE_MARKER = String.raw`(?:дом|д\.|будинок|буд\.|house|h\.|uy|уй|үй|nr\.?|no\.?|№)`;
-const BUILDING_MARKER = String.raw`(?:корп(?:ус)?\.?|к\.|строен(?:ие)?|стр\.|будова|секц(?:ия|ія)?|bloc|corp|building|bldg\.?|korpus|bino|bina|бино)`;
+// These marker alternations are built from address-grammar.js's tagged,
+// per-language fragment data (one shared parser engine, country/language
+// grammar as data — see that file) rather than hardcoded here. The
+// combined pattern strings are verified byte-identical to the original
+// hardcoded regexes in address-grammar.test.js, so this is a pure data
+// extraction with no behavior change.
+const PREFIX_STREET_MARKER = String.raw`(?:(?:${combinedMarkerPattern(STREET_PREFIX_MARKERS)})\.?)`;
+const POSTFIX_STREET_MARKER = `(?:${combinedMarkerPattern(STREET_POSTFIX_MARKERS)})`;
+const POSTFIX_STREET_TYPE = `(?:${combinedMarkerPattern(STREET_TYPE_MARKERS)})`;
+const HOUSE_MARKER = `(?:${combinedMarkerPattern(HOUSE_MARKERS)})`;
+const BUILDING_MARKER = `(?:${combinedMarkerPattern(BUILDING_MARKERS)})`;
 const NUMBER_TOKEN = String.raw`\d{1,5}(?:[-\/]?[\p{L}]\d{0,4})?(?:[\/-]\d{1,4}(?:[-\/]?[\p{L}]\d{0,4})?){0,2}`;
 const STREET_WORD = String.raw`[\p{L}'’.-]{2,48}`;
 // Common post-Soviet street names lead with a bare numeral ("8 Марта",