@whiteslove/parsing-lexicon 0.10.2 → 0.10.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/country-context.d.ts +1 -0
- package/src/country-context.js +11 -0
- package/src/housing-commercial.js +1 -1
- package/src/housing-listing-enrichment.js +1 -1
- package/src/housing-listing-fields.js +2 -2
- package/src/housing-money.js +35 -2
- package/src/housing-structured.js +10 -2
- package/src/housing.js +2 -2
- package/src/normalization.js +39 -10
- package/src/tashkent-housing-geography.js +1 -1
package/package.json
CHANGED
package/src/country-context.d.ts
CHANGED
|
@@ -8,3 +8,4 @@ export type CountryContext = Readonly<{
|
|
|
8
8
|
export function countryContext(value: unknown): CountryContext | null;
|
|
9
9
|
export function countryCurrency(value: unknown): string | null;
|
|
10
10
|
export function countryPhoneHint(value: unknown): string | null;
|
|
11
|
+
export function countryCodeForCity(city: unknown): string;
|
package/src/country-context.js
CHANGED
|
@@ -1,4 +1,6 @@
|
|
|
1
1
|
import { canonicalCountryCode, countryByCode } from './countries.js';
|
|
2
|
+
import { CITIES } from './geography.js';
|
|
3
|
+
import { findCanonical } from './normalization.js';
|
|
2
4
|
|
|
3
5
|
export function countryContext(value) {
|
|
4
6
|
const code = canonicalCountryCode(value);
|
|
@@ -20,3 +22,12 @@ export function countryCurrency(value) {
|
|
|
20
22
|
export function countryPhoneHint(value) {
|
|
21
23
|
return countryContext(value)?.phoneCountry || null;
|
|
22
24
|
}
|
|
25
|
+
|
|
26
|
+
// A context that names only the city ("Tashkent") still fixes the country and
|
|
27
|
+
// with it the default currency. Without this a Tashkent listing parsed with
|
|
28
|
+
// `{ city }` alone had no fallback currency and consumers filled the gap with
|
|
29
|
+
// their own default (KZT), mislabelling so'm prices.
|
|
30
|
+
export function countryCodeForCity(city) {
|
|
31
|
+
if (!city) return '';
|
|
32
|
+
return findCanonical(String(city), CITIES)?.country || '';
|
|
33
|
+
}
|
|
@@ -50,7 +50,7 @@ export function looksParkingOnly(value) {
|
|
|
50
50
|
// classifyTelegramAgency in whiteslove.me-backend-platform's
|
|
51
51
|
// apps/flats/src/scrapers/telegram.js). If you're tempted to merge these,
|
|
52
52
|
// check that call site's expectations first.
|
|
53
|
-
const DIRECT_OWNER_RE = /(
|
|
53
|
+
const DIRECT_OWNER_RE = /(?:(?:makler|vositachi)(?:lar)?\s+bilan\s+ishla(?:n)?may\p{L}*|(?:makler|vositachi)(?:lar)?\s+(?:bezovta|bezota)\s+qilma\p{L}*|(?:с\s+)?(?:ри[еэ]лтор|маклер|агент(?:ств)?|посредник)\p{L}*\s+(?:просьба\s+)?не\s+(?:беспокоить|работа\p{L}*|звонить)|без\s+(?:макл(?:ер[а-яё]*)?|посредник[а-яё]*|ри[еэ]?лтор[а-яё]*|агент[а-яё]*)|от\s+(?:собственник[а-яё]*|хозяин[а-яё]*)|від\s+(?:власник[а-яіїґ]*|власниц[яії][а-яіїґ]*|господар[а-яіїґ]*)|без\s+(?:посередник[а-яіїґ]*|рі[єе]лтор[а-яіїґ]*|агент[а-яіїґ]*)|прямо\s+від\s+(?:власник[а-яіїґ]*|власниц[яії][а-яіїґ]*|господар[а-яіїґ]*)|(?:власник|власниця)\s+(?:зда[єе]|прода[єе])|no\s+(?:agency|broker|realtor|agent)|owner\s+direct|direct\s+from\s+(?:owner|landlord)|f(?:ă|a)r(?:ă|a)\s+(?:agen(?:ț|t)ie|intermediar\w*)|direct\s+(?:de\s+la\s+)?proprietar|makler\s*[- ]?siz|maklersiz|bez\s*makler(?:a|ov)?|bezmakler(?:a|ov)?|vositachi\s*[- ]?siz|vositachisiz|egasidan|uy\s+egasidan|делдалсыз|делдал\s*жоқ|иесінен|үй\s+иесінен)/iu;
|
|
54
54
|
|
|
55
55
|
// Cross-language "no commission" signals. Direct-owner language implies zero
|
|
56
56
|
// commission even without an explicit "no commission" phrase.
|
|
@@ -33,7 +33,7 @@ const APPLIANCE_PATTERNS = Object.freeze([
|
|
|
33
33
|
const FIRST_RENT_UZ_RE = /(?:hali\s+hech\s+kim\s+(?:yashamagan|turmagan)|ҳали\s+ҳеч\s+ким\s+(?:яшамаган|турмаган))/iu;
|
|
34
34
|
const LANDLORD_PRESENT_RE = /(?:xozaykali|hojaykali|xo['’]?jaykali|с\s+хозяйк(?:ой|ой\s+в\s+квартире)|хозяйк\p{L}*\s+(?:жив[её]т|прожива\p{L}*)|with\s+(?:the\s+)?(?:landlord|owner)\s+(?:present|living\s+in)|cu\s+proprietar(?:ul)?\s+în\s+cas(?:ă|a)|үй\s*иесі\s+(?:тұрады|бірге\s+тұрады))/iu;
|
|
35
35
|
const STUDENT_RE = /(?:studentlar\s+uchun|talabalar\s+uchun|(?<![\p{L}\p{N}_])(?:student(?:ka)?|talaba)(?:lar)?(?:ga)?\s+(?:qiz|yigit|o['’ʻʼ‘`]?g['’ʻʼ‘`]?il\s+bola|bola)(?:lar)?(?:ga)?(?![\p{L}\p{N}_])|(?<![\p{L}\p{N}_])(?:studentlar|talabalar)ga(?![\p{L}\p{N}_])|студент(?:ам|ы|ок|ов)?\s+(?:можно|для)|для\s+студент|students?\s+(?:only|welcome)|for\s+students|pentru\s+studen[țt]i|studen[țt]i(?:lor)?|студенттерге|студенттер\s+үшін|(?:oila|oyla)(?:ga|lar|li)?\s+yoki\s+\d{1,2}\s+ta\s+bola(?:lar)?(?:ga)?\s+(?:ijara(?:ga)?\s+)?(?:beril|topshiril))/iu;
|
|
36
|
-
const NO_BROKER_RE = /(?:bez\s*makler|maklersiz|vositachisiz|без\s+(?:маклер|посредник|риелтор|риэлтор|комисси)|no\s+(?:broker|agent|commission|agency\s+fee)|f[ăa]r[ăa]\s+(?:comision|agen[țt]ie|intermediari)|делдалсыз|комиссиясыз)/iu;
|
|
36
|
+
const NO_BROKER_RE = /(?:(?:makler|vositachi)(?:lar)?\s+bilan\s+ishla(?:n)?may\p{L}*|(?:makler|vositachi)(?:lar)?\s+(?:bezovta|bezota)\s+qilma\p{L}*|(?:с\s+)?(?:ри[еэ]лтор|маклер|агент(?:ств)?|посредник)\p{L}*\s+(?:просьба\s+)?не\s+(?:беспокоить|работа\p{L}*|звонить)|bez\s*makler|maklersiz|vositachisiz|без\s+(?:маклер|посредник|риелтор|риэлтор|комисси)|no\s+(?:broker|agent|commission|agency\s+fee)|f[ăa]r[ăa]\s+(?:comision|agen[țt]ie|intermediari)|делдалсыз|комиссиясыз)/iu;
|
|
37
37
|
const BROKER_RE = /(?:makler|vositachi|макл(?:ер[а-яё]*)?|ри[еэ]лтор[а-яё]*|агентств[а-яё]*|комисси[а-яё]*|broker|realtor|commission|comision(?:ul)?|agen[țt]ie|delda[lл]\p{L}*|делдал\p{L}*)/iu;
|
|
38
38
|
const MEN_RE = /(?:o['’ʻʼ‘`]?g['’ʻʼ‘`]?il\s+bola(?:lar)?(?:ga)?|ogil\s+bola(?:lar)?(?:ga)?|sherik\s+bola|эркак(?:лар)?|erkak(?:lar)?(?:ga)?|только\s+(?:мужчин|парн)|\bmen\s+only\b|b[ăa]rba[țt]i(?:lor)?|b[ăa]ie[țt]i(?:lor)?|жігіттерге|жігіттер(?:ге)?|хлопц(?:ям|і|ів)?|чоловік(?:ам|и)?)/iu;
|
|
39
39
|
const WOMEN_RE = /(?:qiz(?:lar)?(?:ga)?|ayol(?:lar)?(?:ga|ni)?|киз(?:лар)?(?:га)?|аёл(?:лар)?(?:га|ни)?|девушк\p{L}*|женщин\p{L}*|girls?\s+only|women\s+only|fete(?:lor)?|femei(?:lor)?|қыздарға|қыздар(?:ға)?|дівчат(?:ам|а|ок)?|жінк(?:ам|и)?)/iu;
|
|
@@ -10,7 +10,7 @@ const bool = (text, positive, negative = null) => {
|
|
|
10
10
|
};
|
|
11
11
|
|
|
12
12
|
const REFRIGERATOR_RE = /(?:холодильник\p{L}*|haladelnik|xolodilnik|muzlatgich|refrigerator|fridge)/iu;
|
|
13
|
-
const WASHING_MACHINE_RE = /(?:кир\s*машин\p{L}*|кирмошин\p{L}*|стиральн\p{L}*\s+машин\p{L}*|washing\s+machine|kir\s*
|
|
13
|
+
const WASHING_MACHINE_RE = /(?:кир\s*машин\p{L}*|кирмошин\p{L}*|стиральн\p{L}*\s+машин\p{L}*|washing\s+machine|kir\s*m[ao]shin\p{L}*|kirmoshina|kir\s*yuvish\s+mashin\p{L}*)/iu;
|
|
14
14
|
|
|
15
15
|
function listedUzbekAbsence(text, appliance) {
|
|
16
16
|
// Marketplace Uzbek frequently puts a single "yo'q" after a short,
|
|
@@ -173,7 +173,7 @@ export function parseHousingListingFields(value, { country = '', dealType = null
|
|
|
173
173
|
const washingMachine = applianceState(
|
|
174
174
|
text,
|
|
175
175
|
WASHING_MACHINE_RE,
|
|
176
|
-
/(?:без|нет)\s+(?:кир\s*машин\p{L}*|кирмошин\p{L}*|стиральн\p{L}*\s+машин\p{L}*|kir\s*
|
|
176
|
+
/(?:без|нет)\s+(?:кир\s*машин\p{L}*|кирмошин\p{L}*|стиральн\p{L}*\s+машин\p{L}*|kir\s*m[ao]shin\p{L}*|kirmoshina)|(?:кир\s*машин\p{L}*|кирмошин\p{L}*|стиральн\p{L}*\s+машин\p{L}*|kir\s*m[ao]shin\p{L}*|kirmoshina)\s+(?:нет|yo['’ʻʼ`]?q|йўқ)|no\s+washing\s+machine/iu,
|
|
177
177
|
);
|
|
178
178
|
|
|
179
179
|
return deepFreeze({
|
package/src/housing-money.js
CHANGED
|
@@ -10,6 +10,7 @@ import {
|
|
|
10
10
|
import { maskPhoneLikeSpans } from './contact.js';
|
|
11
11
|
import { DEPOSIT_TERMS, SELLER_TERMS } from './housing.js';
|
|
12
12
|
import { COUNTRIES, canonicalCountryCode, countryByCode } from './countries.js';
|
|
13
|
+
import { countryCodeForCity } from './country-context.js';
|
|
13
14
|
import { extractTemporalCandidates } from './temporal.js';
|
|
14
15
|
import {
|
|
15
16
|
classifyHousingSingleMSpans,
|
|
@@ -92,7 +93,7 @@ const PRICE_AMOUNT_BEFORE_CURRENCY_RE = new RegExp(`${PRICE_CURRENCY_BEFORE_NUMB
|
|
|
92
93
|
|
|
93
94
|
function moneyParsingContext(value = '') {
|
|
94
95
|
if (value && typeof value === 'object' && !Array.isArray(value)) {
|
|
95
|
-
const country = canonicalCountryCode(value.country) ||
|
|
96
|
+
const country = canonicalCountryCode(value.country) || countryCodeForCity(value.city);
|
|
96
97
|
const currency = String(value.currency || value.fallbackCurrency || countryByCode(country)?.currency || '')
|
|
97
98
|
.trim()
|
|
98
99
|
.toUpperCase();
|
|
@@ -102,6 +103,14 @@ function moneyParsingContext(value = '') {
|
|
|
102
103
|
}
|
|
103
104
|
|
|
104
105
|
const currency = String(value || '').trim().toUpperCase();
|
|
106
|
+
// A bare country code ("UZ") is not a currency. Treat it as the country so
|
|
107
|
+
// the fallback becomes its currency rather than the literal string "UZ".
|
|
108
|
+
const namedCountry = currency && !COUNTRIES.some((item) => item.currency === currency)
|
|
109
|
+
? canonicalCountryCode(value)
|
|
110
|
+
: null;
|
|
111
|
+
if (namedCountry) {
|
|
112
|
+
return { country: namedCountry, currency: countryByCode(namedCountry)?.currency || '', dealType: null };
|
|
113
|
+
}
|
|
105
114
|
const countryMatches = currency
|
|
106
115
|
? COUNTRIES.filter((item) => item.currency === currency)
|
|
107
116
|
: [];
|
|
@@ -406,6 +415,25 @@ export function parseHousingPricePerSqm(value, context = '') {
|
|
|
406
415
|
});
|
|
407
416
|
}
|
|
408
417
|
|
|
418
|
+
// An unlabelled amount far below any real local price is a USD price written
|
|
419
|
+
// without its sign: "Цена 500" is 500 USD, never 500 KZT or 500 UAH. The floors
|
|
420
|
+
// sit well under the cheapest genuine local rent/sale so a real tenge or
|
|
421
|
+
// hryvnia price is never reclassified. UZS has its own rule below.
|
|
422
|
+
const LOCAL_PRICE_FLOORS = Object.freeze({
|
|
423
|
+
KZT: Object.freeze({ sale: 1_000_000, daily: 1_000, rent: 10_000 }),
|
|
424
|
+
UAH: Object.freeze({ sale: 100_000, daily: null, rent: 1_000 }),
|
|
425
|
+
});
|
|
426
|
+
const DAILY_RENT_RE = /(?:посуточн|суточн|сутк|за\s+ночь|подобов|доба|tunlik|kunlik|sutkaga|тәулік|per\s+night|daily)/iu;
|
|
427
|
+
|
|
428
|
+
function lowLocalAmountCurrency(amount, currency, dealType, text) {
|
|
429
|
+
const floors = LOCAL_PRICE_FLOORS[currency];
|
|
430
|
+
if (!floors || amount == null) return currency;
|
|
431
|
+
const floor = dealType === 'sale' ? floors.sale
|
|
432
|
+
: dealType === 'shortRent' || DAILY_RENT_RE.test(text) ? floors.daily
|
|
433
|
+
: floors.rent;
|
|
434
|
+
return floor != null && amount > 0 && amount < floor ? 'USD' : currency;
|
|
435
|
+
}
|
|
436
|
+
|
|
409
437
|
export function parseHousingPrice(value, context = '') {
|
|
410
438
|
const { country, currency: fallbackCurrency, dealType } = moneyParsingContext(context);
|
|
411
439
|
const original = String(value || '');
|
|
@@ -427,9 +455,12 @@ export function parseHousingPrice(value, context = '') {
|
|
|
427
455
|
// regional formats below remain as deterministic fallbacks until each is
|
|
428
456
|
// represented by a richer candidate extractor.
|
|
429
457
|
if (preferredCandidate && preferredCandidate.confidence >= 0.65) {
|
|
458
|
+
const candidateCurrency = preferredCandidate.currency || fallbackCurrency || '';
|
|
430
459
|
const result = {
|
|
431
460
|
amount: preferredCandidate.amount,
|
|
432
|
-
currency: preferredCandidate.
|
|
461
|
+
currency: preferredCandidate.explicitCurrency
|
|
462
|
+
? candidateCurrency
|
|
463
|
+
: lowLocalAmountCurrency(preferredCandidate.amount, candidateCurrency, dealType, priceText),
|
|
433
464
|
approximate: preferredCandidate.approximate,
|
|
434
465
|
};
|
|
435
466
|
if (preferredCandidate.range) result.range = preferredCandidate.range;
|
|
@@ -606,6 +637,8 @@ export function parseHousingPrice(value, context = '') {
|
|
|
606
637
|
// turned "800 000" per person into an 800,000 USD rent.
|
|
607
638
|
const monthlyRent = dealType === 'longRent' && price >= 100_000;
|
|
608
639
|
currency = price >= 1_000_000 || monthlyRent || (dailyUzbek && price >= 10_000) ? 'UZS' : 'USD';
|
|
640
|
+
} else if (!explicit && currency === fallbackCurrency) {
|
|
641
|
+
currency = lowLocalAmountCurrency(price, currency, dealType, priceText);
|
|
609
642
|
}
|
|
610
643
|
|
|
611
644
|
return Object.freeze({ amount: price, currency, approximate: price != null && APPROXIMATE_RE.test(priceText) });
|
|
@@ -6,7 +6,7 @@ import { DEPOSIT_TERMS, SELLER_TERMS, UTILITY_TERMS } from './housing.js';
|
|
|
6
6
|
import { GENERIC_LANDMARK_TERMS } from './landmarks.js';
|
|
7
7
|
import { LOCATION_RELATIONS, parseHousingContext } from './housing-context.js';
|
|
8
8
|
import { isHousingCommercialAd, resolveHousingIntent } from './housing-intent.js';
|
|
9
|
-
import { countryCurrency, countryPhoneHint } from './country-context.js';
|
|
9
|
+
import { countryCodeForCity, countryCurrency, countryPhoneHint } from './country-context.js';
|
|
10
10
|
import { findTelegramContacts, maskPhoneLikeSpans, parsePhoneNumbers } from './contact.js';
|
|
11
11
|
import { parseHousingAddress } from './housing-address.js';
|
|
12
12
|
import { parseHousingListingFields } from './housing-listing-fields.js';
|
|
@@ -81,6 +81,14 @@ export function parseHousingRoomCount(value) {
|
|
|
81
81
|
if (rooms != null && rooms >= 1 && rooms <= 20) return rooms;
|
|
82
82
|
}
|
|
83
83
|
|
|
84
|
+
// "1в2 комнатная" / "2 в 3-х комн.": a flat replanned from one room count
|
|
85
|
+
// into another is listed as the resulting layout.
|
|
86
|
+
const replanned = text.match(/(?:^|[^\p{L}\p{N}])\d{1,2}\s*(?:-\s*х\s*)?(?:к\.?\s*)?в\s*(\d{1,2})\s*(?:-\s*(?:х|ти|и|ух|[её]х)\s*)?(?:комн\p{L}*|к(?:\.|\b)|xona)/iu);
|
|
87
|
+
if (replanned) {
|
|
88
|
+
const rooms = toNumber(replanned[1]);
|
|
89
|
+
if (rooms != null && rooms >= 1 && rooms <= 20) return rooms;
|
|
90
|
+
}
|
|
91
|
+
|
|
84
92
|
const describedRooms = [...text.matchAll(/(?:^|[^\p{L}\p{N}_])(\d{1,2})\s*ta\s+(?:katta|kichkina)\s+xona(?=$|[^\p{L}\p{N}_])/giu)];
|
|
85
93
|
if (describedRooms.length > 1) {
|
|
86
94
|
const total = describedRooms.reduce((sum, match) => sum + Number(match[1]), 0);
|
|
@@ -414,7 +422,7 @@ export function parseHousingStructured(value, options = {}) {
|
|
|
414
422
|
const original = String(value ?? '');
|
|
415
423
|
const sourcePost = parseHousingSourcePost(original, { source: options.source });
|
|
416
424
|
const text = sourcePost.text;
|
|
417
|
-
const country = String(options.country || '').trim();
|
|
425
|
+
const country = String(options.country || '').trim() || countryCodeForCity(options.city);
|
|
418
426
|
const fallbackCurrency = options.fallbackCurrency || countryCurrency(country) || '';
|
|
419
427
|
const phoneCountry = options.phoneCountry || countryPhoneHint(country) || null;
|
|
420
428
|
const intent = resolveHousingIntent(text);
|
package/src/housing.js
CHANGED
|
@@ -120,8 +120,8 @@ export const CURRENCIES = Object.freeze([
|
|
|
120
120
|
|
|
121
121
|
export const SELLER_TERMS = Object.freeze({
|
|
122
122
|
owner: group('owner', {
|
|
123
|
-
ru: ['собственник', 'хозяин', 'от хозяина', 'без посредников', 'без посредника', 'без риелтора', 'без риэлтора', 'без маклера', 'без маклер', 'без агента'], en: ['owner', 'direct owner', 'no agent', 'no broker', 'no realtor'], uk: ['власник', 'власниця', 'від власника', 'без посередників'], ro: ['proprietar', 'direct proprietar', 'fără agenție', 'fara agentie', 'fără intermediari'],
|
|
124
|
-
uzLatn: ['egasi', 'uy egasi', 'mulkdor', 'maklersiz', 'vositachisiz'], uzCyrl: ['эгаси', 'уй эгаси', 'мулкдор', 'маклерсиз', 'воситачисиз', 'без маклер'], kk: ['иесі', 'үй иесі', 'меншік иесі', 'делдалсыз'],
|
|
123
|
+
ru: ['собственник', 'хозяин', 'от хозяина', 'без посредников', 'без посредника', 'без риелтора', 'без риэлтора', 'без маклера', 'без маклер', 'без агента', 'с риелторами не работаю', 'с риэлторами не работаю', 'с маклерами не работаю', 'с агентствами не работаю', 'риелторам не беспокоить', 'риэлторам не беспокоить', 'агентствам не беспокоить'], en: ['owner', 'direct owner', 'no agent', 'no broker', 'no realtor'], uk: ['власник', 'власниця', 'від власника', 'без посередників'], ro: ['proprietar', 'direct proprietar', 'fără agenție', 'fara agentie', 'fără intermediari'],
|
|
124
|
+
uzLatn: ['egasi', 'uy egasi', 'mulkdor', 'maklersiz', 'vositachisiz', 'bezmakler', 'bez makler', 'maklerlar bilan ishlanmaydi', 'maklerlar bilan ishlamaymiz', 'makler bilan ishlanmaydi', 'maklerlar bezovta qilmasin'], uzCyrl: ['эгаси', 'уй эгаси', 'мулкдор', 'маклерсиз', 'воситачисиз', 'без маклер', 'маклерлар билан ишланмайди', 'маклерлар безовта қилмасин'], kk: ['иесі', 'үй иесі', 'меншік иесі', 'делдалсыз'],
|
|
125
125
|
}),
|
|
126
126
|
agency: group('agency', {
|
|
127
127
|
ru: ['агентство', 'агент', 'риелтор', 'риэлтор', 'маклер', 'посредник'], en: ['agency', 'agent', 'realtor', 'broker'], uk: ['агентство', 'агент', 'рієлтор', 'риелтор', 'посередник'], ro: ['agenție', 'agentie', 'agent imobiliar', 'agenție imobiliară', 'agentie imobiliara', 'broker'],
|
package/src/normalization.js
CHANGED
|
@@ -3,16 +3,39 @@ import { LEXICON_LANGUAGES } from './lexicon-core.js';
|
|
|
3
3
|
const APOSTROPHES_RE = /[’‘ʻʼ`´]/g;
|
|
4
4
|
const DASHES_RE = /[‐‑‒–—―]/g;
|
|
5
5
|
|
|
6
|
+
// Parsing one vacancy or listing runs dozens of matchers over the same text,
|
|
7
|
+
// and each normalizes it again from scratch: profiling the workforce
|
|
8
|
+
// enrichment showed normalization among the top costs at ~170-490 ms per
|
|
9
|
+
// vacancy. These functions are pure, so each keeps a small cache of its
|
|
10
|
+
// recent long inputs. Short strings (dictionary aliases, single words) are
|
|
11
|
+
// cheaper to normalize than to cache and are left alone; the cap bounds
|
|
12
|
+
// memory to a few recent texts per function.
|
|
13
|
+
const MEMO_MIN_LENGTH = 200;
|
|
14
|
+
const MEMO_MAX_ENTRIES = 32;
|
|
15
|
+
|
|
16
|
+
function memoizeLongText(fn) {
|
|
17
|
+
const cache = new Map();
|
|
18
|
+
return (value) => {
|
|
19
|
+
if (typeof value !== 'string' || value.length < MEMO_MIN_LENGTH) return fn(value);
|
|
20
|
+
const hit = cache.get(value);
|
|
21
|
+
if (hit !== undefined) return hit;
|
|
22
|
+
const result = fn(value);
|
|
23
|
+
if (cache.size >= MEMO_MAX_ENTRIES) cache.delete(cache.keys().next().value);
|
|
24
|
+
cache.set(value, result);
|
|
25
|
+
return result;
|
|
26
|
+
};
|
|
27
|
+
}
|
|
28
|
+
|
|
6
29
|
/** Normalize Unicode and punctuation variants without destroying letters. */
|
|
7
|
-
export function normalizeUnicode(value) {
|
|
30
|
+
export const normalizeUnicode = memoizeLongText(function normalizeUnicode(value) {
|
|
8
31
|
return String(value ?? '')
|
|
9
32
|
.normalize('NFKC')
|
|
10
33
|
.replace(APOSTROPHES_RE, "'")
|
|
11
34
|
.replace(DASHES_RE, '-');
|
|
12
|
-
}
|
|
35
|
+
});
|
|
13
36
|
|
|
14
37
|
/** Stable comparison form for parser dictionaries. */
|
|
15
|
-
export function normalizeForMatch(value) {
|
|
38
|
+
export const normalizeForMatch = memoizeLongText(function normalizeForMatch(value) {
|
|
16
39
|
return normalizeUnicode(value)
|
|
17
40
|
.toLocaleLowerCase()
|
|
18
41
|
.replace(/ё/g, 'е')
|
|
@@ -20,10 +43,10 @@ export function normalizeForMatch(value) {
|
|
|
20
43
|
.replace(/[^\p{L}\p{N}]+/gu, ' ')
|
|
21
44
|
.replace(/\s+/g, ' ')
|
|
22
45
|
.trim();
|
|
23
|
-
}
|
|
46
|
+
});
|
|
24
47
|
|
|
25
48
|
/** Search-only apostrophe compaction for Uzbek/Karakalpak variants. */
|
|
26
|
-
function normalizeCompactApostropheForMatch(value) {
|
|
49
|
+
const normalizeCompactApostropheForMatch = memoizeLongText(function normalizeCompactApostropheForMatch(value) {
|
|
27
50
|
return normalizeUnicode(value)
|
|
28
51
|
.toLocaleLowerCase()
|
|
29
52
|
.replace(/ё/g, 'е')
|
|
@@ -32,7 +55,7 @@ function normalizeCompactApostropheForMatch(value) {
|
|
|
32
55
|
.replace(/[^\p{L}\p{N}]+/gu, ' ')
|
|
33
56
|
.replace(/\s+/g, ' ')
|
|
34
57
|
.trim();
|
|
35
|
-
}
|
|
58
|
+
});
|
|
36
59
|
|
|
37
60
|
export const CYRILLIC_SEARCH_MAP = Object.freeze({
|
|
38
61
|
а: 'a', б: 'b', в: 'v', г: 'g', д: 'd', е: 'e', ё: 'e', ж: 'zh', з: 'z',
|
|
@@ -65,21 +88,21 @@ const KARAKALPAK_LATIN_PATTERN_EQUIVALENCE = Object.freeze({
|
|
|
65
88
|
});
|
|
66
89
|
|
|
67
90
|
/** Search-oriented Cyrillic folding; canonical identity still comes from explicit aliases. */
|
|
68
|
-
export function foldCyrillicForSearch(value) {
|
|
91
|
+
export const foldCyrillicForSearch = memoizeLongText(function foldCyrillicForSearch(value) {
|
|
69
92
|
return normalizeUnicode(value)
|
|
70
93
|
.toLocaleLowerCase()
|
|
71
94
|
.split('')
|
|
72
95
|
.map((char) => CYRILLIC_SEARCH_MAP[char] ?? char)
|
|
73
96
|
.join('');
|
|
74
|
-
}
|
|
97
|
+
});
|
|
75
98
|
|
|
76
|
-
function foldKazakhForSearch(value) {
|
|
99
|
+
const foldKazakhForSearch = memoizeLongText(function foldKazakhForSearch(value) {
|
|
77
100
|
return normalizeUnicode(value)
|
|
78
101
|
.toLocaleLowerCase()
|
|
79
102
|
.split('')
|
|
80
103
|
.map((char) => KAZAKH_SEARCH_EQUIVALENCE[char] ?? char)
|
|
81
104
|
.join('');
|
|
82
|
-
}
|
|
105
|
+
});
|
|
83
106
|
|
|
84
107
|
function foldKarakalpakLatinForSearch(value) {
|
|
85
108
|
return normalizeUnicode(value)
|
|
@@ -287,6 +310,8 @@ function matcherFor(entries, { transliteration = true } = {}) {
|
|
|
287
310
|
* Return every canonical match with original-text offsets. Colliding aliases are
|
|
288
311
|
* deliberately returned as multiple matches instead of silently choosing one.
|
|
289
312
|
*/
|
|
313
|
+
const UZBEK_APOSTROPHE_CONTINUATION_RE = /^[og]['’ʻʼ‘`]\p{L}/iu;
|
|
314
|
+
|
|
290
315
|
export function findAllCanonical(value, entries, { transliteration = true } = {}) {
|
|
291
316
|
const text = normalizeUnicode(value ?? '');
|
|
292
317
|
if (!text || !entries?.length) return [];
|
|
@@ -300,6 +325,10 @@ export function findAllCanonical(value, entries, { transliteration = true } = {}
|
|
|
300
325
|
const alias = match[0];
|
|
301
326
|
const start = match.index ?? 0;
|
|
302
327
|
const end = start + alias.length;
|
|
328
|
+
// Uzbek Latin writes oʻ/gʻ with an apostrophe inside the word, so an alias
|
|
329
|
+
// that stops on o/g right before one ("bog" in "bog'laning", "contact us")
|
|
330
|
+
// has matched only the front of a longer word.
|
|
331
|
+
if (UZBEK_APOSTROPHE_CONTINUATION_RE.test(text.slice(end - 1, end + 2))) continue;
|
|
303
332
|
const keys = normalizedAliasKeys(alias, { transliteration });
|
|
304
333
|
const candidates = [];
|
|
305
334
|
for (const key of keys) {
|
|
@@ -213,7 +213,7 @@ export function matchTashkentNumberedArea(value, canonical) {
|
|
|
213
213
|
if (!text) return null;
|
|
214
214
|
const alternatives = normalizedAlternatives(aliases);
|
|
215
215
|
let match = text.match(new RegExp(
|
|
216
|
-
`(?:^|\\s)(?:${alternatives})(?:\\s+${NUMBERED_CONTEXT})?\\s+(\\d{1,2})(?:\\s*([adад]))?(?:\\s+${NUMBERED_SUFFIX_CONTEXT})*(?:\\s|$)`,
|
|
216
|
+
`(?:^|\\s)(?:${alternatives})(?:\\s+${NUMBERED_CONTEXT})?\\s+(\\d{1,2})(?:\\s*([adад]))?(?:\\s*(?:кв|kv)(?=\\s|$))?(?:\\s+${NUMBERED_SUFFIX_CONTEXT})*(?:\\s|$)`,
|
|
217
217
|
'iu',
|
|
218
218
|
));
|
|
219
219
|
|