@whiteslove/parsing-lexicon 0.8.2 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +8 -2
- package/src/contact.js +9 -1
- package/src/housing-address.d.ts +11 -0
- package/src/housing-address.js +54 -8
- package/src/housing-intent.d.ts +8 -0
- package/src/housing-intent.js +40 -0
- package/src/housing-money.js +20 -0
- package/src/housing-poi-relations.js +4 -4
- package/src/housing-structured.d.ts +1 -0
- package/src/housing-structured.js +22 -11
- package/src/landmarks.js +2 -0
- package/src/location-data.js +5 -0
- package/src/locations.js +65 -1
- package/src/semantic-spans.d.ts +19 -0
- package/src/semantic-spans.js +83 -0
- package/src/temporal-calendar-date.js +72 -0
- package/src/temporal-clock.js +46 -0
- package/src/temporal-duration.js +44 -0
- package/src/temporal-relative-date.js +38 -0
- package/src/temporal-schedule.js +65 -0
- package/src/temporal-shared.js +54 -0
- package/src/temporal-weekday.js +17 -0
- package/src/temporal.js +46 -223
- package/src/uz-bukhara-location-extensions.js +15 -0
- package/src/uz-map-data-location-extensions.js +74377 -0
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@whiteslove/parsing-lexicon",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.9.0",
|
|
4
4
|
"description": "Shared deterministic multilingual parsing lexicon for WhitesLove housing and hiring services",
|
|
5
5
|
"repository": {
|
|
6
6
|
"type": "git",
|
|
@@ -193,6 +193,10 @@
|
|
|
193
193
|
"types": "./src/housing-money.d.ts",
|
|
194
194
|
"import": "./src/housing-money.js"
|
|
195
195
|
},
|
|
196
|
+
"./semantic-spans": {
|
|
197
|
+
"types": "./src/semantic-spans.d.ts",
|
|
198
|
+
"import": "./src/semantic-spans.js"
|
|
199
|
+
},
|
|
196
200
|
"./hiring-professions": "./src/hiring-professions.js",
|
|
197
201
|
"./hiring-profession-display": {
|
|
198
202
|
"types": "./src/hiring-profession-display.d.ts",
|
|
@@ -226,7 +230,9 @@
|
|
|
226
230
|
"node": ">=20"
|
|
227
231
|
},
|
|
228
232
|
"scripts": {
|
|
229
|
-
"test": "node --test"
|
|
233
|
+
"test": "node --test",
|
|
234
|
+
"sync:geo-map-data": "node ./scripts/sync-geo-map-data-lexicon.js",
|
|
235
|
+
"audit:geo-map-data": "node ./scripts/audit-geo-map-data-lexicon.js"
|
|
230
236
|
},
|
|
231
237
|
"publishConfig": {
|
|
232
238
|
"access": "public"
|
package/src/contact.js
CHANGED
|
@@ -111,11 +111,19 @@ export function maskPhoneLikeSpans(value, replacement = ' ', options = {}) {
|
|
|
111
111
|
const spans = findPhoneLikeSpans(text, options);
|
|
112
112
|
if (!spans.length) return text;
|
|
113
113
|
|
|
114
|
+
// The replacement must fill the exact span length. A single-character
|
|
115
|
+
// replacement collapsing a whole multi-digit phone span down to one
|
|
116
|
+
// character silently shifts every character offset after it, which is
|
|
117
|
+
// harmless to a caller that only reads back matched substrings but
|
|
118
|
+
// corrupts any offset (start/end) computed against this masked text and
|
|
119
|
+
// later compared to the original, unmasked string.
|
|
120
|
+
const fill = String(replacement || ' ') || ' ';
|
|
114
121
|
let out = '';
|
|
115
122
|
let cursor = 0;
|
|
116
123
|
for (const span of spans) {
|
|
117
124
|
out += text.slice(cursor, span.start);
|
|
118
|
-
|
|
125
|
+
const spanLength = span.end - span.start;
|
|
126
|
+
out += fill.repeat(Math.ceil(spanLength / fill.length)).slice(0, spanLength);
|
|
119
127
|
cursor = span.end;
|
|
120
128
|
}
|
|
121
129
|
return out + text.slice(cursor);
|
package/src/housing-address.d.ts
CHANGED
|
@@ -71,3 +71,14 @@ export function composeHousingAddress(parts?: Readonly<{
|
|
|
71
71
|
houseNumber?: unknown;
|
|
72
72
|
building?: unknown;
|
|
73
73
|
}>): string | null;
|
|
74
|
+
|
|
75
|
+
export type HousingAddressCandidate = Readonly<{
|
|
76
|
+
address: string | null;
|
|
77
|
+
street: string | null;
|
|
78
|
+
houseNumber: string | null;
|
|
79
|
+
building: string | null;
|
|
80
|
+
confidence: number;
|
|
81
|
+
score: number;
|
|
82
|
+
}>;
|
|
83
|
+
|
|
84
|
+
export function extractHousingAddressCandidates(value: unknown): readonly HousingAddressCandidate[];
|
package/src/housing-address.js
CHANGED
|
@@ -4,6 +4,7 @@ import {
|
|
|
4
4
|
matchTashkentHousingMetro,
|
|
5
5
|
matchTashkentNumberedArea,
|
|
6
6
|
} from './tashkent-housing-geography.js';
|
|
7
|
+
import { detectNonAddressSpans } from './semantic-spans.js';
|
|
7
8
|
|
|
8
9
|
const PHONE_RUN_RE = /\+?\d[\d\s().-]{7,}\d/gu;
|
|
9
10
|
const ADDRESS_LABEL_RE = /(?:адрес|адреса|адресація|адресация|manzil|address|adresă|adresa)\s*[:=\-–—]\s*/iu;
|
|
@@ -83,6 +84,11 @@ function compactStreet(value) {
|
|
|
83
84
|
// "проживания" -> "оживания").
|
|
84
85
|
.replace(new RegExp(`^${PREFIX_STREET_MARKER}(?!\\p{L})\\s*`, 'iu'), '')
|
|
85
86
|
.replace(new RegExp(`\\s+${POSTFIX_STREET_MARKER}$`, 'iu'), '')
|
|
87
|
+
// A generic street-word capture has no stop-word list of its own, so a
|
|
88
|
+
// trailing relation marker ("недалеко", "рядом") from prose describing
|
|
89
|
+
// a *different* nearby location can get swept into the street name
|
|
90
|
+
// itself (e.g. "ул. Первого Мая недалеко"). Trim it and anything after.
|
|
91
|
+
.replace(new RegExp(`\\s+(?:${LOCATION_RELATION_RE.source})(?:\\s+.*)?$`, 'iu'), '')
|
|
86
92
|
// OCR frequently substitutes “оя” for the Ukrainian/Russian “ля” in
|
|
87
93
|
// “шлях”; correct the street-token typo before canonical lookup.
|
|
88
94
|
.replace(/(?<!\p{L})шоях(?!\p{L})/giu, 'шлях')
|
|
@@ -424,11 +430,23 @@ function addressCandidateLine(line) {
|
|
|
424
430
|
const markerIndex = text.search(new RegExp(`${PREFIX_STREET_MARKER}|${POSTFIX_STREET_MARKER}`, 'iu'));
|
|
425
431
|
const searchStart = markerIndex >= 0 ? markerIndex : 0;
|
|
426
432
|
const tail = text.slice(searchStart);
|
|
427
|
-
const
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
433
|
+
const stopMatch = tail.match(ADDRESS_FIELD_STOP_RE);
|
|
434
|
+
const stopAt = stopMatch ? searchStart + (stopMatch.index ?? 0) : Infinity;
|
|
435
|
+
// A money/contact/temporal span overlapping the street phrase (e.g.
|
|
436
|
+
// "99 1881919", "100$ депозит", "от 1 месяца") is at least as strong
|
|
437
|
+
// evidence that this text belongs to another domain as the local
|
|
438
|
+
// stop-word list below — reuse the shared classifier instead of growing
|
|
439
|
+
// another ad-hoc stop-word list here for every new case found. The
|
|
440
|
+
// classifier already excludes calendar-date-shaped temporal spans
|
|
441
|
+
// ("8 Марта", "9 Января" — a real Soviet-legacy street-naming
|
|
442
|
+
// convention) from its TEMPORAL results, so this stays safe for those.
|
|
443
|
+
const nonAddressSpan = detectNonAddressSpans(text)
|
|
444
|
+
.find((span) => span.start >= searchStart && span.start < stopAt);
|
|
445
|
+
const cutAt = nonAddressSpan ? Math.min(stopAt, nonAddressSpan.start) : stopAt;
|
|
446
|
+
return cutAt < Infinity ? clean(text.slice(0, cutAt)) : line;
|
|
447
|
+
}
|
|
448
|
+
|
|
449
|
+
function collectExplicitStreetCandidates(text) {
|
|
432
450
|
const lines = text
|
|
433
451
|
.split(/[\r\n|]/u)
|
|
434
452
|
.map((part) => clean(part).slice(0, 1200))
|
|
@@ -513,7 +531,7 @@ function explicitStreetAddress(text) {
|
|
|
513
531
|
// Listing text often names a nearby street before the actual postal
|
|
514
532
|
// address. Keep alternatives long enough to rank component evidence rather
|
|
515
533
|
// than returning whichever regex happened to run first.
|
|
516
|
-
return candidates
|
|
534
|
+
return Object.freeze(candidates
|
|
517
535
|
.map((value, index) => ({
|
|
518
536
|
value,
|
|
519
537
|
index,
|
|
@@ -521,7 +539,27 @@ function explicitStreetAddress(text) {
|
|
|
521
539
|
+ (value.houseNumber ? 0.18 : 0)
|
|
522
540
|
+ (value.building ? 0.03 : 0),
|
|
523
541
|
}))
|
|
524
|
-
.sort((a, b) => b.score - a.score || a.index - b.index)
|
|
542
|
+
.sort((a, b) => b.score - a.score || a.index - b.index)
|
|
543
|
+
.map(({ value, score }) => Object.freeze({ ...value, score: Number(score.toFixed(2)) })));
|
|
544
|
+
}
|
|
545
|
+
|
|
546
|
+
function explicitStreetAddress(text) {
|
|
547
|
+
const { score, ...winner } = collectExplicitStreetCandidates(text)[0] || {};
|
|
548
|
+
return winner.street ? Object.freeze(winner) : null;
|
|
549
|
+
}
|
|
550
|
+
|
|
551
|
+
/**
|
|
552
|
+
* Expose every plausible street/house candidate found in free-form listing
|
|
553
|
+
* text, ranked by evidence score, instead of only the single winner
|
|
554
|
+
* parseHousingAddress() commits to. Useful when the caller wants to inspect
|
|
555
|
+
* or re-rank competing parses (e.g. a nearby street mentioned before the
|
|
556
|
+
* actual postal address).
|
|
557
|
+
*/
|
|
558
|
+
export function extractHousingAddressCandidates(value) {
|
|
559
|
+
const text = clean(value);
|
|
560
|
+
if (!text) return Object.freeze([]);
|
|
561
|
+
const addressText = stripSecondaryComponents(text) || text;
|
|
562
|
+
return collectExplicitStreetCandidates(addressText);
|
|
525
563
|
}
|
|
526
564
|
|
|
527
565
|
function knownStreetAddress(text, knownStreet) {
|
|
@@ -626,7 +664,15 @@ function bareAddress(text) {
|
|
|
626
664
|
const cleaned = clean(text);
|
|
627
665
|
if (!cleaned || PROPERTY_AREA_LINE_RE.test(cleaned) || NON_ADDRESS_BARE_RE.test(cleaned)) return null;
|
|
628
666
|
const stopMatch = cleaned.match(ADDRESS_FIELD_STOP_RE);
|
|
629
|
-
const
|
|
667
|
+
const stopAt = stopMatch ? stopMatch.index : Infinity;
|
|
668
|
+
// allowBare trusts the caller's claim that this whole field is an
|
|
669
|
+
// address, so it has no street-marker anchor to lean on the way
|
|
670
|
+
// addressCandidateLine() does — a money/contact/temporal amount is the
|
|
671
|
+
// only guard against source data that mislabels e.g. a rental-duration
|
|
672
|
+
// field ("от 1 месяца") as an address field.
|
|
673
|
+
const nonAddressSpan = detectNonAddressSpans(cleaned).find((span) => span.start < stopAt);
|
|
674
|
+
const cutAt = nonAddressSpan ? Math.min(stopAt, nonAddressSpan.start) : stopAt;
|
|
675
|
+
const truncated = cutAt < Infinity ? clean(cleaned.slice(0, cutAt)) : cleaned;
|
|
630
676
|
if (!truncated) return null;
|
|
631
677
|
const tail = splitAddressTail(truncated);
|
|
632
678
|
if (!tail) return null;
|
package/src/housing-intent.d.ts
CHANGED
|
@@ -9,4 +9,12 @@ export const HOUSING_ACTION_MAP: Readonly<Record<HousingAction, Readonly<{ listi
|
|
|
9
9
|
export function resolveHousingIntent(value: unknown): HousingIntentResult | null;
|
|
10
10
|
export declare function classifyHousingDealType(value: unknown): 'sale' | 'longRent' | 'shortRent' | null;
|
|
11
11
|
export declare function looksExplicitDailyRentalMention(value: unknown): boolean;
|
|
12
|
+
export type HousingCommercialAdSignals = Readonly<{
|
|
13
|
+
multipleBusinessNames: boolean;
|
|
14
|
+
repeatedContactBlocks: boolean;
|
|
15
|
+
manyPriceMentions: boolean;
|
|
16
|
+
promotionalText: boolean;
|
|
17
|
+
}>;
|
|
18
|
+
export declare function detectHousingCommercialAdSignals(value: unknown): HousingCommercialAdSignals;
|
|
19
|
+
export declare function isHousingCommercialAd(value: unknown): boolean;
|
|
12
20
|
|
package/src/housing-intent.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { findCanonical } from './normalization.js';
|
|
2
2
|
import { lexiconEntity } from './lexicon-core.js';
|
|
3
|
+
import { findPhoneLikeSpans } from './contact.js';
|
|
3
4
|
|
|
4
5
|
const group = (canonical, aliases, extra = {}) => lexiconEntity(canonical, aliases, extra);
|
|
5
6
|
const KK_RENT_OUT_ALIASES = Object.freeze(['жалға беремін', 'жалға беріледі']);
|
|
@@ -161,3 +162,42 @@ export function looksExplicitDailyRentalMention(value) {
|
|
|
161
162
|
return EXPLICIT_SHORT_STAY_RE.test(String(value || ''));
|
|
162
163
|
}
|
|
163
164
|
|
|
165
|
+
// A commercial hotel-group advertisement lists several distinct business
|
|
166
|
+
// names, several unrelated phone numbers and a scattering of prices — none
|
|
167
|
+
// of which cohere into one describable property the way a single owner
|
|
168
|
+
// listing does. Each signal alone is common in ordinary listings (a broker
|
|
169
|
+
// might list two contact numbers; a listing might mention one hotel by
|
|
170
|
+
// name as a nearby landmark), so only their co-occurrence is meaningful.
|
|
171
|
+
const HOTEL_BUSINESS_MARKER_RE = /(?:\bhotel\b|гостиниц\p{L}*|отел[ья]?\p{L}*|\bhostel\b|хостел\p{L}*|mehmonxona\p{L}*|guest\s*house|\bb\s*&\s*b\b|bed\s+and\s+breakfast)/giu;
|
|
172
|
+
const PROMOTIONAL_MARKER_RE = /(?:скидк\p{L}*|\bакция\b|бронируйте|бронирование|звоните\s+прямо\s+сейчас|call\s+now|book\s+now|chegirma|eng\s+arzon\s+narx|лучшие\s+цены|top\s+prices|специальное\s+предложение)/iu;
|
|
173
|
+
const PRICE_LIKE_RE = /\d{2,6}\s*(?:\$|usd|сум|сўм|so['’ʻʼ]?m|som|сом|тенге|kzt)/giu;
|
|
174
|
+
|
|
175
|
+
/**
|
|
176
|
+
* Weak-signal evidence for a commercial/multi-property advertisement rather
|
|
177
|
+
* than a single owner's listing. Exposed separately from
|
|
178
|
+
* isHousingCommercialAd() so callers can inspect which signals fired.
|
|
179
|
+
*/
|
|
180
|
+
export function detectHousingCommercialAdSignals(value) {
|
|
181
|
+
const text = String(value || '');
|
|
182
|
+
const hotelNameCount = new Set((text.match(HOTEL_BUSINESS_MARKER_RE) || []).map((match) => match.toLowerCase())).size;
|
|
183
|
+
const phoneCount = findPhoneLikeSpans(text).length;
|
|
184
|
+
const priceCount = (text.match(PRICE_LIKE_RE) || []).length;
|
|
185
|
+
return Object.freeze({
|
|
186
|
+
multipleBusinessNames: hotelNameCount >= 2,
|
|
187
|
+
repeatedContactBlocks: phoneCount >= 3,
|
|
188
|
+
manyPriceMentions: priceCount >= 3,
|
|
189
|
+
promotionalText: PROMOTIONAL_MARKER_RE.test(text),
|
|
190
|
+
});
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
/**
|
|
194
|
+
* True when at least two independent weak signals of a commercial/
|
|
195
|
+
* multi-property advertisement co-occur. A single signal (one hotel name
|
|
196
|
+
* mentioned as a landmark, two phone numbers on a broker listing) is
|
|
197
|
+
* common in ordinary single-property ads and must not trigger this alone.
|
|
198
|
+
*/
|
|
199
|
+
export function isHousingCommercialAd(value) {
|
|
200
|
+
const signals = detectHousingCommercialAdSignals(value);
|
|
201
|
+
return Object.values(signals).filter(Boolean).length >= 2;
|
|
202
|
+
}
|
|
203
|
+
|
package/src/housing-money.js
CHANGED
|
@@ -10,6 +10,7 @@ import {
|
|
|
10
10
|
import { maskPhoneLikeSpans } from './contact.js';
|
|
11
11
|
import { DEPOSIT_TERMS, SELLER_TERMS } from './housing.js';
|
|
12
12
|
import { COUNTRIES, canonicalCountryCode, countryByCode } from './countries.js';
|
|
13
|
+
import { extractTemporalCandidates } from './temporal.js';
|
|
13
14
|
import {
|
|
14
15
|
classifyHousingSingleMSpans,
|
|
15
16
|
HOUSING_NUMERIC_SPAN_TYPES,
|
|
@@ -94,6 +95,21 @@ function parsedMoneyAmount(numberValue, scaleValue) {
|
|
|
94
95
|
|
|
95
96
|
const APPROXIMATE_RE = /около|примерно|~|≈/iu;
|
|
96
97
|
|
|
98
|
+
// A number already claimed by a duration/date/schedule reading ("на 1200
|
|
99
|
+
// дней", "с 2027 года") is not a plausible bare price — it is at least as
|
|
100
|
+
// likely to be that temporal value's own count/year as a price digit that
|
|
101
|
+
// happens to sit near it. This is only used as a last-resort exclusion in
|
|
102
|
+
// the bare-amount fallback below, once every explicit currency/scale/
|
|
103
|
+
// keyword path has already failed to find a price. Only reasonably
|
|
104
|
+
// confident temporal candidates disqualify a span, so an ambiguous bare
|
|
105
|
+
// number elsewhere in the text is unaffected. Calls temporal.js directly
|
|
106
|
+
// rather than semantic-spans.js: that module itself depends on this file's
|
|
107
|
+
// extractHousingMoneyCandidates(), so importing it here would cycle back.
|
|
108
|
+
function overlapsConfidentTemporalSpan(text, start, end) {
|
|
109
|
+
return extractTemporalCandidates(text).some((item) => (Number(item.confidence) || 0) >= 0.5
|
|
110
|
+
&& start < item.end && item.start < end);
|
|
111
|
+
}
|
|
112
|
+
|
|
97
113
|
function perSquareMeterMatches(text, fallbackCurrency = '') {
|
|
98
114
|
const matches = [];
|
|
99
115
|
const seen = new Set();
|
|
@@ -561,6 +577,10 @@ export function parseHousingPrice(value, context = '') {
|
|
|
561
577
|
const hasPriceEvidence = PRICE_KEYWORD_RE.test(window) || Boolean(moneyCurrencyFromText(window, ''));
|
|
562
578
|
if (!hasPriceEvidence) continue;
|
|
563
579
|
}
|
|
580
|
+
// A number already read as a duration/date/schedule value ("\u043D\u0430 1200
|
|
581
|
+
// \u0434\u043D\u0435\u0439") is not a plausible bare price either \u2014 e.g. "\u0441\u0434\u0430\u044E \u043D\u0430 1200
|
|
582
|
+
// \u0434\u043D\u0435\u0439" must not report 1200 as the rent.
|
|
583
|
+
if (overlapsConfidentTemporalSpan(priceText, start, start + raw.length)) continue;
|
|
564
584
|
if (best == null || amount > best) best = amount;
|
|
565
585
|
}
|
|
566
586
|
price = best;
|
|
@@ -8,10 +8,10 @@ const TYPE_MARKERS = Object.freeze([
|
|
|
8
8
|
['poi.kindergarten', /(?:kindergarten|childcare|bogcha|bog['’ʻʼ`]cha|детск(?:ий|ого)\s+сад|садик|балабақша)/iu],
|
|
9
9
|
['poi.hospital', /(?:hospital|shifoxona|больниц\p{L}*|госпитал\p{L}*)/iu],
|
|
10
10
|
['poi.clinic', /(?:clinic|polyclinic|medical\s+cent(?:er|re)|klinika|поликлиник\p{L}*|клиник\p{L}*|медицинск\p{L}*\s+центр)/iu],
|
|
11
|
-
['poi.airport', /(?:airport|aeroport
|
|
12
|
-
['poi.railway_station', /(?:railway\s+station|train\s+station|railway|temir\s+
|
|
13
|
-
['poi.bus_station', /(?:bus\s+station|coach\s+station
|
|
14
|
-
['poi.parking', /(?:parking|парковк\p{L}*|паркинг|автостоянк\p{L}
|
|
11
|
+
['poi.airport', /(?:airport|aeroport|аэропорт|аеропорт|әуежай)/iu],
|
|
12
|
+
['poi.railway_station', /(?:railway\s+station|train\s+station|railway|gar[ăa]|temir\s+yo['’ʻʼ`]?l|темир\s+йўл|темір\s*жол|залізничн\p{L}*\s+вокзал|вокзал|ж\.?д\.?\s*вокзал|железнодорожн\p{L}*\s+вокзал)/iu],
|
|
13
|
+
['poi.bus_station', /(?:bus\s+station|coach\s+station|автовокзал|автостанц\p{L}*|avtovokzal|avtostansiya|autogar[ăa]|bus\s+terminal)/iu],
|
|
14
|
+
['poi.parking', /(?:parking|car\s+park|парковк\p{L}*|паркінг|паркуван\p{L}*|паркинг|автостоянк\p{L}*|avtoturargoh|автотұрақ|көлік\s+тұра)/iu],
|
|
15
15
|
['poi.shopping_mall', /(?:shopping\s+(?:mall|cent(?:er|re))|mall\b|т[цр]\b|savdo\s+markaz)/iu],
|
|
16
16
|
['poi.supermarket', /(?:supermarket|супермаркет|гипермаркет|магазин)/iu],
|
|
17
17
|
['poi.market', /(?:market|bazaar|bozor|базар|рынок)/iu],
|
|
@@ -76,6 +76,7 @@ export type HousingStructuredResult = Readonly<{
|
|
|
76
76
|
text: string;
|
|
77
77
|
source: Readonly<{ platform: string | null; contact: string | null }>;
|
|
78
78
|
intent: Readonly<{ action: string | null; listingKind: string | null; dealType: string }> | null;
|
|
79
|
+
isCommercialAd: boolean;
|
|
79
80
|
context: Readonly<Record<string, unknown>>;
|
|
80
81
|
rooms: number | null;
|
|
81
82
|
floor: Readonly<{ floor: number | null; totalFloors: number | null }>;
|
|
@@ -5,7 +5,7 @@ import { moneyCurrencyFromText } from './money-core.js';
|
|
|
5
5
|
import { DEPOSIT_TERMS, SELLER_TERMS, UTILITY_TERMS } from './housing.js';
|
|
6
6
|
import { GENERIC_LANDMARK_TERMS } from './landmarks.js';
|
|
7
7
|
import { LOCATION_RELATIONS, parseHousingContext } from './housing-context.js';
|
|
8
|
-
import { resolveHousingIntent } from './housing-intent.js';
|
|
8
|
+
import { isHousingCommercialAd, resolveHousingIntent } from './housing-intent.js';
|
|
9
9
|
import { countryCurrency, countryPhoneHint } from './country-context.js';
|
|
10
10
|
import { findTelegramContacts, maskPhoneLikeSpans, parsePhoneNumbers } from './contact.js';
|
|
11
11
|
import { parseHousingAddress } from './housing-address.js';
|
|
@@ -425,6 +425,12 @@ export function parseHousingStructured(value, options = {}) {
|
|
|
425
425
|
? deepFreeze({ ...parsedPayments, utilities: null })
|
|
426
426
|
: parsedPayments;
|
|
427
427
|
const numericOptions = { country, dealType: effectiveDealType };
|
|
428
|
+
// A commercial/multi-property advertisement (hotel group, several
|
|
429
|
+
// listed businesses) has no single coherent address or price to extract
|
|
430
|
+
// — whichever one a naive parse picked would be arbitrary and misleading
|
|
431
|
+
// rather than downranked-but-plausible. Suppress both instead of
|
|
432
|
+
// returning a confident-looking but meaningless single value.
|
|
433
|
+
const isCommercialAd = isHousingCommercialAd(text);
|
|
428
434
|
|
|
429
435
|
return deepFreeze({
|
|
430
436
|
text,
|
|
@@ -433,20 +439,25 @@ export function parseHousingStructured(value, options = {}) {
|
|
|
433
439
|
contact: sourcePost.contact,
|
|
434
440
|
},
|
|
435
441
|
intent,
|
|
442
|
+
isCommercialAd,
|
|
436
443
|
context: parseHousingContext(text),
|
|
437
444
|
rooms: parseHousingRoomCount(text),
|
|
438
445
|
floor: parseHousingFloor(text),
|
|
439
446
|
area: parseHousingAreas(text, numericOptions),
|
|
440
|
-
price:
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
447
|
+
price: isCommercialAd
|
|
448
|
+
? deepFreeze({ amount: null, currency: fallbackCurrency, approximate: false })
|
|
449
|
+
: parseHousingPrice(text, {
|
|
450
|
+
country,
|
|
451
|
+
currency: fallbackCurrency,
|
|
452
|
+
dealType: effectiveDealType,
|
|
453
|
+
}),
|
|
454
|
+
address: isCommercialAd
|
|
455
|
+
? deepFreeze({ address: null, street: null, houseNumber: null, building: null, confidence: 0 })
|
|
456
|
+
: parseHousingAddress(text, {
|
|
457
|
+
knownStreet: options.knownStreet || null,
|
|
458
|
+
allowDelimitedBare: options.allowDelimitedBareAddress === true,
|
|
459
|
+
allowBare: options.allowBareAddress === true,
|
|
460
|
+
}),
|
|
450
461
|
residentialComplex: parseHousingResidentialComplex(text),
|
|
451
462
|
amenities: parseHousingAmenities(text),
|
|
452
463
|
listingFields,
|
package/src/landmarks.js
CHANGED
|
@@ -25,4 +25,6 @@ export const GENERIC_LANDMARK_TERMS = Object.freeze([
|
|
|
25
25
|
group('Church', { ru: ['церковь', 'храм'], en: ['church', 'cathedral'], uk: ['церква', 'храм', 'собор'], ro: ['biserică', 'biserica', 'catedrală', 'catedrala'], uzLatn: ['cherkov', 'ibodatxona'], uzCyrl: ['черков'], kk: ['шіркеу', 'собор'] }),
|
|
26
26
|
group('Railway station', { ru: ['вокзал', 'железнодорожный вокзал'], en: ['railway station', 'train station'], uk: ['вокзал', 'залізничний вокзал'], ro: ['gară', 'gara', 'gară feroviară', 'gara feroviara'], uzLatn: ['vokzal', 'temir yol vokzali', "temir yo'l vokzali"], uzCyrl: ['вокзал', 'темир йўл вокзали'], kk: ['теміржол вокзалы', 'вокзал'] }),
|
|
27
27
|
group('Airport', { ru: ['аэропорт'], en: ['airport'], uk: ['аеропорт'], ro: ['aeroport'], uzLatn: ['aeroport'], uzCyrl: ['аэропорт'], kk: ['әуежай', 'аэропорт'] }),
|
|
28
|
+
group('Bus station', { ru: ['автовокзал', 'автостанция'], en: ['bus station', 'bus terminal', 'coach station'], uk: ['автовокзал', 'автостанція'], ro: ['autogară', 'autogara', 'stație de autobuz', 'statie de autobuz'], uzLatn: ['avtovokzal', 'avtostansiya', 'avtobus vokzali'], uzCyrl: ['автовокзал', 'автостанция', 'автобус вокзали'], kk: ['автовокзал', 'автостанция', 'автобус вокзалы'] }),
|
|
29
|
+
group('Parking', { ru: ['парковка', 'паркинг', 'автостоянка'], en: ['parking', 'car park', 'parking lot'], uk: ['парковка', 'паркінг', 'автостоянка'], ro: ['parcare', 'loc de parcare'], uzLatn: ['parking', 'avtoturargoh', 'mashina turargohi'], uzCyrl: ['паркинг', 'автотураргоҳ', 'машина тураргоҳи'], kk: ['тұрақ', 'автотұрақ', 'көлік тұрағы'] }),
|
|
28
30
|
]);
|
package/src/location-data.js
CHANGED
|
@@ -177,6 +177,11 @@ export const LOCATION_DICTIONARIES = Object.freeze({
|
|
|
177
177
|
['Gara de Nord', 'București Nord', 'Северный вокзал Бухареста'], ['Piața Romană', 'Piata Romana'], ['Piața Victoriei', 'Piata Victoriei'],
|
|
178
178
|
]),
|
|
179
179
|
}),
|
|
180
|
+
Otopeni: Object.freeze({
|
|
181
|
+
landmarks: entries([
|
|
182
|
+
['Bucharest Henri Coandă International Airport', 'Henri Coandă Airport', 'Henri Coanda Airport', 'Otopeni Airport', 'Aeroportul Internațional Henri Coandă', 'Aeroportul Otopeni', 'Аэропорт Отопень', 'OTP', 'LROP'],
|
|
183
|
+
]),
|
|
184
|
+
}),
|
|
180
185
|
Brasov: Object.freeze({
|
|
181
186
|
microdistricts: entries([
|
|
182
187
|
['Tractorul'], ['Coresi'], ['Astra'], ['Racadau', 'Răcădău'], ['Bartolomeu'], ['Noua'], ['Darste', 'Dârste'], ['Schei', 'Șchei'],
|
package/src/locations.js
CHANGED
|
@@ -10,6 +10,7 @@ import { mergeLocationCountries } from './location-merge.js';
|
|
|
10
10
|
import { aliasesToRegex } from './normalization.js';
|
|
11
11
|
import { KZ_LOCATION_EXTENSIONS } from './kz-location-extensions.js';
|
|
12
12
|
import { UZ_LOCATION_EXTENSIONS } from './uz-location-extensions.js';
|
|
13
|
+
import { UZ_MAP_DATA_LOCATION_EXTENSIONS } from './uz-map-data-location-extensions.js';
|
|
13
14
|
import { UA_MAJOR_LOCATION_EXTENSIONS } from './ua-location-extensions-major.js';
|
|
14
15
|
import { UA_REGIONAL_LOCATION_EXTENSIONS } from './ua-location-extensions-regional.js';
|
|
15
16
|
import { UA_SECONDARY_LOCATION_EXTENSIONS } from './ua-secondary-cities.js';
|
|
@@ -606,9 +607,72 @@ function normalizeUaSemanticLocations(country) {
|
|
|
606
607
|
});
|
|
607
608
|
}
|
|
608
609
|
|
|
610
|
+
// UZ_MAP_DATA_LOCATION_EXTENSIONS is bulk auto-imported/transliterated data.
|
|
611
|
+
// mergeLocationEntries() only dedupes entries that share the exact same
|
|
612
|
+
// canonical `name`, so a handful of its entries create a second, competing
|
|
613
|
+
// entity for a real street that already has a reviewed canonical under a
|
|
614
|
+
// different (often English) name — one that already lists the map-data
|
|
615
|
+
// entry's own transliteration as an alias. Whichever of the two entries a
|
|
616
|
+
// lookup happens to match first then wins by array position, which is
|
|
617
|
+
// fragile and can surface the auto-transliterated name instead of the
|
|
618
|
+
// reviewed one.
|
|
619
|
+
//
|
|
620
|
+
// Exclude each known duplicate by city + canonical name rather than
|
|
621
|
+
// changing the shared merge algorithm: a generic alias-overlap merge (and
|
|
622
|
+
// a generic "drop any map-data entry whose canonical exactly matches an
|
|
623
|
+
// existing alias anywhere in the reviewed data") were both tried and
|
|
624
|
+
// reverted, since either one also affected genuinely distinct entries that
|
|
625
|
+
// happen to share one incidental alias elsewhere in this large dataset —
|
|
626
|
+
// this hand-verified list only touches the specific streets confirmed to
|
|
627
|
+
// be pure duplicates.
|
|
628
|
+
const UZ_MAP_DATA_KNOWN_DUPLICATES = Object.freeze({
|
|
629
|
+
// Duplicates the reviewed "Shimoliy Olmazor Street" (merged later, in
|
|
630
|
+
// locations-runtime.js's UZ_TASHKENT_REVIEWED_STREET_EXTENSIONS), which
|
|
631
|
+
// already lists this exact transliteration as an alias.
|
|
632
|
+
Tashkent: new Set(["Shimoliy Olmazor ko'chasi"]),
|
|
633
|
+
// Duplicates the reviewed "University Boulevard", "Gagarin Street",
|
|
634
|
+
// "Spitamen Avenue", "Rudakiy Street" and "Mirzo Ulugbek Street"
|
|
635
|
+
// (uz-samarkand-context-extensions.js).
|
|
636
|
+
Samarkand: new Set([
|
|
637
|
+
'Universitet bulvari',
|
|
638
|
+
"Gagarin ko'chasi",
|
|
639
|
+
"Spitamen shoh ko'chasi",
|
|
640
|
+
"Rudakiy ko'chasi",
|
|
641
|
+
"Mirzo Ulug'bek ko'chasi",
|
|
642
|
+
]),
|
|
643
|
+
// Duplicates the reviewed "Amir Temur Street" and "Bunyodkor Street"
|
|
644
|
+
// (uz-location-extensions.js).
|
|
645
|
+
Angren: new Set(["Amir Temur ko'chasi", "Bunyodkor ko'chasi"]),
|
|
646
|
+
});
|
|
647
|
+
|
|
648
|
+
// A bare number ("2", "5", "9") is never a legitimate standalone street/POI
|
|
649
|
+
// name on its own — these are OSM import artifacts (a house number or
|
|
650
|
+
// building tag mistakenly carried through as the object's name) that
|
|
651
|
+
// otherwise resolve as a "street" matching any bare house-number-shaped
|
|
652
|
+
// text (e.g. the "5" in "..., дом 5"). Unlike the named-duplicate cases
|
|
653
|
+
// above, this is safe to apply everywhere rather than per city: a real
|
|
654
|
+
// street/POI canonical is never purely numeric.
|
|
655
|
+
const BARE_NUMERIC_NAME_RE = /^\d+$/u;
|
|
656
|
+
|
|
657
|
+
function withoutKnownMapDataDuplicates(extensions) {
|
|
658
|
+
return Object.freeze(Object.fromEntries(Object.entries(extensions).map(([city, data]) => {
|
|
659
|
+
const excluded = UZ_MAP_DATA_KNOWN_DUPLICATES[city];
|
|
660
|
+
return [city, Object.freeze(Object.fromEntries(Object.entries(data).map(([key, entries]) => [
|
|
661
|
+
key,
|
|
662
|
+
Array.isArray(entries)
|
|
663
|
+
? Object.freeze(entries.filter((entry) => !BARE_NUMERIC_NAME_RE.test(String(entry?.name ?? '').trim())
|
|
664
|
+
&& !(excluded && excluded.has(entry?.name))))
|
|
665
|
+
: entries,
|
|
666
|
+
])))];
|
|
667
|
+
})));
|
|
668
|
+
}
|
|
669
|
+
|
|
670
|
+
const UZ_MAP_DATA_LOCATION_EXTENSIONS_CURATED = withoutKnownMapDataDuplicates(UZ_MAP_DATA_LOCATION_EXTENSIONS);
|
|
671
|
+
|
|
609
672
|
const UZ_LOCATION_DICTIONARIES = normalizeUzSemanticLocations(mergeLocationCountries(
|
|
610
673
|
UZ_BASE_LOCATION_DICTIONARIES,
|
|
611
674
|
UZ_LOCATION_EXTENSIONS,
|
|
675
|
+
UZ_MAP_DATA_LOCATION_EXTENSIONS_CURATED,
|
|
612
676
|
));
|
|
613
677
|
|
|
614
678
|
const UA_SEMANTIC_LOCATION_EXTENSIONS = normalizeUaSemanticLocations(UA_MAJOR_LOCATION_EXTENSIONS);
|
|
@@ -642,4 +706,4 @@ export {
|
|
|
642
706
|
UA_SECONDARY_CITIES,
|
|
643
707
|
matchUkraineRegion,
|
|
644
708
|
matchUkraineSecondaryCity,
|
|
645
|
-
};
|
|
709
|
+
};
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
export type NonAddressSpanType = 'contact' | 'money' | 'temporal';
|
|
2
|
+
|
|
3
|
+
export const NON_ADDRESS_SPAN_TYPE: Readonly<{
|
|
4
|
+
CONTACT: 'contact';
|
|
5
|
+
MONEY: 'money';
|
|
6
|
+
TEMPORAL: 'temporal';
|
|
7
|
+
}>;
|
|
8
|
+
|
|
9
|
+
export type NonAddressSpan = Readonly<{
|
|
10
|
+
type: NonAddressSpanType;
|
|
11
|
+
start: number;
|
|
12
|
+
end: number;
|
|
13
|
+
}>;
|
|
14
|
+
|
|
15
|
+
export function detectNonAddressSpans(
|
|
16
|
+
value: unknown,
|
|
17
|
+
context?: Readonly<Record<string, unknown>> & { types?: readonly NonAddressSpanType[] },
|
|
18
|
+
): readonly NonAddressSpan[];
|
|
19
|
+
export function overlapsAnySpan(start: number, end: number, spans: readonly NonAddressSpan[]): boolean;
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
import { findPhoneLikeSpans } from './contact.js';
|
|
2
|
+
import { extractHousingMoneyCandidates } from './housing-money.js';
|
|
3
|
+
import { extractTemporalCandidates } from './temporal.js';
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* A shared, reusable classification of text spans that already belong to
|
|
7
|
+
* another semantic domain (money, contact, temporal) before any address/geo
|
|
8
|
+
* parsing runs. This deliberately does not reimplement money/date/contact
|
|
9
|
+
* detection — it orchestrates the existing domain extractors so a span
|
|
10
|
+
* claimed by one domain (e.g. "от 1 месяца", "99 1881919", "100$ депозит")
|
|
11
|
+
* can be recognized as NOT_ADDRESS evidence by any consumer, instead of each
|
|
12
|
+
* parser independently growing its own local guard against the same class
|
|
13
|
+
* of cross-domain contamination.
|
|
14
|
+
*/
|
|
15
|
+
export const NON_ADDRESS_SPAN_TYPE = Object.freeze({
|
|
16
|
+
CONTACT: 'contact',
|
|
17
|
+
MONEY: 'money',
|
|
18
|
+
TEMPORAL: 'temporal',
|
|
19
|
+
});
|
|
20
|
+
|
|
21
|
+
function moneySpans(text, context) {
|
|
22
|
+
// A bare unlabelled number (a plausible house number) must not be treated
|
|
23
|
+
// as money evidence — only candidates with an explicit currency marker or
|
|
24
|
+
// a recognized price keyword/scale are strong enough non-address evidence.
|
|
25
|
+
return extractHousingMoneyCandidates(text, context)
|
|
26
|
+
.filter((candidate) => candidate.explicitCurrency || candidate.priceKeyword || candidate.scale)
|
|
27
|
+
.map((candidate) => ({ type: NON_ADDRESS_SPAN_TYPE.MONEY, start: candidate.start, end: candidate.end }));
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
function contactSpans(text, context) {
|
|
31
|
+
return findPhoneLikeSpans(text, context)
|
|
32
|
+
.map((span) => ({ type: NON_ADDRESS_SPAN_TYPE.CONTACT, start: span.start, end: span.end }));
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
// Calendar dates built from a literal "<day> <month-name>" pattern
|
|
36
|
+
// (temporal.calendar.month-name/month-first/month-end) are the one temporal
|
|
37
|
+
// shape that collides with real street names: many Soviet-legacy streets
|
|
38
|
+
// ("8 Марта", "9 Января", "1 Мая") are themselves day-plus-month-name
|
|
39
|
+
// phrases. Everything else temporal.js recognizes — durations, schedules,
|
|
40
|
+
// clock times, relative wording ("завтра", "через 3 дня") — has no such
|
|
41
|
+
// collision risk and is safe to treat as non-address evidence.
|
|
42
|
+
const CALENDAR_MONTH_NAME_PARSERS = new Set([
|
|
43
|
+
'temporal.calendar.month-name',
|
|
44
|
+
'temporal.calendar.month-first',
|
|
45
|
+
'temporal.calendar.month-end',
|
|
46
|
+
]);
|
|
47
|
+
|
|
48
|
+
function temporalSpans(text, context) {
|
|
49
|
+
return extractTemporalCandidates(text, context)
|
|
50
|
+
.filter((candidate) => (Number(candidate.confidence) || 0) >= 0.5 && !CALENDAR_MONTH_NAME_PARSERS.has(candidate.parser))
|
|
51
|
+
.map((candidate) => ({ type: NON_ADDRESS_SPAN_TYPE.TEMPORAL, start: candidate.start, end: candidate.end }));
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
const ALL_SPAN_TYPES = Object.freeze(Object.values(NON_ADDRESS_SPAN_TYPE));
|
|
55
|
+
|
|
56
|
+
/**
|
|
57
|
+
* Detect spans in free-form listing/vacancy text that already belong to the
|
|
58
|
+
* money, contact or temporal domain. Each span carries its type, start and
|
|
59
|
+
* end offset (into the original string) so a consumer can mask or reject an
|
|
60
|
+
* overlapping candidate from an unrelated parser (typically address/geo).
|
|
61
|
+
*
|
|
62
|
+
* Pass `context.types` (an array of NON_ADDRESS_SPAN_TYPE values) to skip
|
|
63
|
+
* running the other extractors entirely — useful when a caller only cares
|
|
64
|
+
* about one domain (e.g. money/contact but not temporal, since calendar-
|
|
65
|
+
* date-shaped street names like "8 Марта" make TEMPORAL a poor address-line
|
|
66
|
+
* exclusion signal) and wants to avoid the unused extractor's cost.
|
|
67
|
+
*/
|
|
68
|
+
export function detectNonAddressSpans(value, context = {}) {
|
|
69
|
+
const text = String(value || '');
|
|
70
|
+
if (!text) return Object.freeze([]);
|
|
71
|
+
const types = Array.isArray(context.types) && context.types.length ? context.types : ALL_SPAN_TYPES;
|
|
72
|
+
const spans = [
|
|
73
|
+
...(types.includes(NON_ADDRESS_SPAN_TYPE.CONTACT) ? contactSpans(text, context) : []),
|
|
74
|
+
...(types.includes(NON_ADDRESS_SPAN_TYPE.MONEY) ? moneySpans(text, context) : []),
|
|
75
|
+
...(types.includes(NON_ADDRESS_SPAN_TYPE.TEMPORAL) ? temporalSpans(text, context) : []),
|
|
76
|
+
];
|
|
77
|
+
return Object.freeze(spans.sort((a, b) => a.start - b.start || a.end - b.end));
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/** True when [start, end) overlaps any span in the given (sorted or unsorted) span list. */
|
|
81
|
+
export function overlapsAnySpan(start, end, spans) {
|
|
82
|
+
return spans.some((span) => start < span.end && span.start < end);
|
|
83
|
+
}
|