@whiteslove/parsing-lexicon 0.10.1 → 0.10.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/geo.js +8 -4
- package/src/geography-detection.js +18 -3
- package/src/hiring-professions.js +19 -3
- package/src/hiring-source-aliases.js +14 -7
- package/src/kg-location-extensions.js +5 -1
- package/src/kz-address-extensions.js +5 -1
- package/src/kz-almaty-cleaned-residential-extensions.js +5 -1
- package/src/kz-almaty-street-extensions.js +5 -1
- package/src/kz-astana-cleaned-residential-extensions.js +5 -1
- package/src/kz-location-extensions.js +12 -1
- package/src/kz-primary-address-extensions.js +5 -1
- package/src/kz-regional-address-extensions.js +5 -1
- package/src/kz-regional-residential-extensions.js +5 -1
- package/src/kz-residential-extensions.js +5 -1
- package/src/kz-shymkent-cleaned-address-extensions.js +5 -1
- package/src/location-data.js +23 -2
- package/src/locations.js +10 -2
- package/src/normalization.js +33 -10
- package/src/odesa-metropolitan.js +5 -1
- package/src/tashkent-gastronomic-streets.js +5 -1
- package/src/tashkent-housing-geography.js +10 -2
- package/src/tashkent-pois.js +5 -1
- package/src/tashkent-residential-complexes.js +5 -1
- package/src/ua-chernihiv-residential-extensions.js +5 -1
- package/src/ua-residential-extensions.js +5 -1
- package/src/uz-location-extensions.js +12 -1
- package/src/uz-samarkand-context-extensions.js +5 -1
- package/src/uz-tashkent-context-extensions.js +10 -2
package/package.json
CHANGED
package/src/geo.js
CHANGED
|
@@ -28,17 +28,21 @@ export function canonicalTashkentDistrict(value) {
|
|
|
28
28
|
}
|
|
29
29
|
|
|
30
30
|
function station(name, ru, en, line, aliases = []) {
|
|
31
|
-
const allAliases = [...new Set([name, ru, en, ...aliases])];
|
|
32
|
-
|
|
31
|
+
const allAliases = Object.freeze([...new Set([name, ru, en, ...aliases])]);
|
|
32
|
+
let re = null;
|
|
33
|
+
return Object.freeze({
|
|
33
34
|
canonical: name,
|
|
34
35
|
name,
|
|
35
36
|
type: 'metro',
|
|
36
37
|
country: 'UZ',
|
|
37
38
|
city: 'Tashkent',
|
|
38
39
|
line,
|
|
39
|
-
labels: { ru, en },
|
|
40
|
+
labels: Object.freeze({ ru, en }),
|
|
40
41
|
aliases: allAliases,
|
|
41
|
-
re
|
|
42
|
+
get re() {
|
|
43
|
+
re ||= aliasesToRegex(allAliases);
|
|
44
|
+
return re;
|
|
45
|
+
},
|
|
42
46
|
});
|
|
43
47
|
}
|
|
44
48
|
|
|
@@ -14,10 +14,18 @@ function cityInflectionRegex(aliases) {
|
|
|
14
14
|
|
|
15
15
|
const CITY_MATCHERS = CITIES.map((item) => {
|
|
16
16
|
const aliases = [item.canonical, ...aliasesOf(item)];
|
|
17
|
+
let exact = null;
|
|
18
|
+
let inflected;
|
|
17
19
|
return Object.freeze({
|
|
18
20
|
item,
|
|
19
|
-
exact
|
|
20
|
-
|
|
21
|
+
get exact() {
|
|
22
|
+
exact ||= aliasesToRegex(aliases);
|
|
23
|
+
return exact;
|
|
24
|
+
},
|
|
25
|
+
get inflected() {
|
|
26
|
+
if (inflected === undefined) inflected = cityInflectionRegex(aliases);
|
|
27
|
+
return inflected;
|
|
28
|
+
},
|
|
21
29
|
});
|
|
22
30
|
});
|
|
23
31
|
|
|
@@ -27,7 +35,14 @@ const COUNTRY_MATCHERS = COUNTRIES.map((item) => {
|
|
|
27
35
|
// are still handled by canonicalCountryCode().
|
|
28
36
|
const aliases = [item.canonical, ...aliasesOf(item)]
|
|
29
37
|
.filter((alias) => normalizeForMatch(alias).replace(/\s/g, '').length > 2);
|
|
30
|
-
|
|
38
|
+
let re = null;
|
|
39
|
+
return Object.freeze({
|
|
40
|
+
item,
|
|
41
|
+
get re() {
|
|
42
|
+
re ||= aliasesToRegex(aliases);
|
|
43
|
+
return re;
|
|
44
|
+
},
|
|
45
|
+
});
|
|
31
46
|
});
|
|
32
47
|
|
|
33
48
|
const US_STATE_RE = /\b[A-Z][A-Za-z.'-]+(?:\s+[A-Z][A-Za-z.'-]+){0,3}\s*,?\s+(?:AL|AK|AZ|AR|CA|CO|CT|DE|FL|GA|HI|ID|IL|IN|IA|KS|KY|LA|ME|MD|MA|MI|MN|MS|MO|MT|NE|NV|NH|NJ|NM|NY|NC|ND|OH|OK|OR|PA|RI|SC|SD|TN|TX|UT|VT|VA|WA|WV|WI|WY|DC)\b/;
|
|
@@ -331,6 +331,8 @@ function profession(canonical, group, strongAliases, weakAliases = []) {
|
|
|
331
331
|
const meta = GROUP_META.get(group) || { family: 'other', aliases: [] };
|
|
332
332
|
const strong = [...new Set([...strongAliases, ...(ROMANIAN_PROFESSION_ALIASES[canonical] || [])].filter(Boolean))];
|
|
333
333
|
const weak = [...new Set(weakAliases.filter(Boolean))];
|
|
334
|
+
let strongRe = null;
|
|
335
|
+
let weakRe;
|
|
334
336
|
return Object.freeze({
|
|
335
337
|
id: canonical,
|
|
336
338
|
taxonomyId: `profession.${group}.${canonical}`,
|
|
@@ -340,8 +342,14 @@ function profession(canonical, group, strongAliases, weakAliases = []) {
|
|
|
340
342
|
aliases: Object.freeze(strong),
|
|
341
343
|
strongAliases: Object.freeze(strong),
|
|
342
344
|
weakAliases: Object.freeze(weak),
|
|
343
|
-
strongRe
|
|
344
|
-
|
|
345
|
+
get strongRe() {
|
|
346
|
+
strongRe ||= aliasesToRegex(strong);
|
|
347
|
+
return strongRe;
|
|
348
|
+
},
|
|
349
|
+
get weakRe() {
|
|
350
|
+
if (weakRe === undefined) weakRe = weak.length ? aliasesToRegex(weak) : null;
|
|
351
|
+
return weakRe;
|
|
352
|
+
},
|
|
345
353
|
});
|
|
346
354
|
}
|
|
347
355
|
|
|
@@ -369,7 +377,15 @@ export const SENIORITY_LEVELS = Object.freeze([
|
|
|
369
377
|
{ canonical: 'chief', aliases: ['chief officer','chief executive','chief technology','chief product','chief financial','chief marketing','chief operating'], score: 1 },
|
|
370
378
|
].map((entry) => {
|
|
371
379
|
const aliases = [...new Set([...entry.aliases, ...(ROMANIAN_SENIORITY_ALIASES[entry.canonical] || [])])];
|
|
372
|
-
|
|
380
|
+
let re = null;
|
|
381
|
+
return Object.freeze({
|
|
382
|
+
...entry,
|
|
383
|
+
aliases: Object.freeze(aliases),
|
|
384
|
+
get re() {
|
|
385
|
+
re ||= aliasesToRegex(aliases);
|
|
386
|
+
return re;
|
|
387
|
+
},
|
|
388
|
+
});
|
|
373
389
|
}));
|
|
374
390
|
|
|
375
391
|
export function matchProfessions(value, { limit = 5, allowWeak = true } = {}) {
|
|
@@ -1,13 +1,20 @@
|
|
|
1
1
|
import { aliasesToRegex } from './normalization.js';
|
|
2
2
|
import { matchProfessions } from './hiring-professions.js';
|
|
3
3
|
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
4
|
+
function sourceRole(canonical, label, aliases, group = 'other') {
|
|
5
|
+
const all = Object.freeze([...new Set(aliases)]);
|
|
6
|
+
let re = null;
|
|
7
|
+
return Object.freeze({
|
|
8
|
+
canonical,
|
|
9
|
+
label,
|
|
10
|
+
group,
|
|
11
|
+
aliases: all,
|
|
12
|
+
get re() {
|
|
13
|
+
re ||= aliasesToRegex(all);
|
|
14
|
+
return re;
|
|
15
|
+
},
|
|
16
|
+
});
|
|
17
|
+
}
|
|
11
18
|
|
|
12
19
|
/**
|
|
13
20
|
* Source spellings seen in real CV feeds which are too source-specific or
|
|
@@ -2,6 +2,7 @@ import { aliasesToRegex } from './normalization.js';
|
|
|
2
2
|
|
|
3
3
|
function entry(name, aliases = [], meta = {}) {
|
|
4
4
|
const all = Object.freeze([...new Set([name, ...aliases].filter(Boolean))]);
|
|
5
|
+
let re = null;
|
|
5
6
|
return Object.freeze({
|
|
6
7
|
...meta,
|
|
7
8
|
canonical: name,
|
|
@@ -10,7 +11,10 @@ function entry(name, aliases = [], meta = {}) {
|
|
|
10
11
|
entityType: meta.entityType || meta.type,
|
|
11
12
|
country: 'KG',
|
|
12
13
|
aliases: all,
|
|
13
|
-
re
|
|
14
|
+
get re() {
|
|
15
|
+
re ||= aliasesToRegex(all);
|
|
16
|
+
return re;
|
|
17
|
+
},
|
|
14
18
|
});
|
|
15
19
|
}
|
|
16
20
|
|
|
@@ -2,6 +2,7 @@ import { aliasesToRegex } from './normalization.js';
|
|
|
2
2
|
|
|
3
3
|
function entry(name, aliases, entityType) {
|
|
4
4
|
const all = Object.freeze([...new Set([name, ...aliases].filter(Boolean))]);
|
|
5
|
+
let re = null;
|
|
5
6
|
return Object.freeze({
|
|
6
7
|
canonical: name,
|
|
7
8
|
name,
|
|
@@ -9,7 +10,10 @@ function entry(name, aliases, entityType) {
|
|
|
9
10
|
entityType,
|
|
10
11
|
country: 'KZ',
|
|
11
12
|
aliases: all,
|
|
12
|
-
re
|
|
13
|
+
get re() {
|
|
14
|
+
re ||= aliasesToRegex(all);
|
|
15
|
+
return re;
|
|
16
|
+
},
|
|
13
17
|
});
|
|
14
18
|
}
|
|
15
19
|
|
|
@@ -2,6 +2,7 @@ import { aliasesToRegex } from './normalization.js';
|
|
|
2
2
|
|
|
3
3
|
function residential(name, aliases = []) {
|
|
4
4
|
const all = Object.freeze([...new Set([name, ...aliases].filter(Boolean))]);
|
|
5
|
+
let re = null;
|
|
5
6
|
return Object.freeze({
|
|
6
7
|
canonical: name,
|
|
7
8
|
name,
|
|
@@ -9,7 +10,10 @@ function residential(name, aliases = []) {
|
|
|
9
10
|
entityType: 'residential_complex',
|
|
10
11
|
country: 'KZ',
|
|
11
12
|
aliases: all,
|
|
12
|
-
re
|
|
13
|
+
get re() {
|
|
14
|
+
re ||= aliasesToRegex(all);
|
|
15
|
+
return re;
|
|
16
|
+
},
|
|
13
17
|
});
|
|
14
18
|
}
|
|
15
19
|
|
|
@@ -2,6 +2,7 @@ import { aliasesToRegex } from './normalization.js';
|
|
|
2
2
|
|
|
3
3
|
function street(name, aliases = []) {
|
|
4
4
|
const all = Object.freeze([...new Set([name, ...aliases].filter(Boolean))]);
|
|
5
|
+
let re = null;
|
|
5
6
|
return Object.freeze({
|
|
6
7
|
canonical: name,
|
|
7
8
|
name,
|
|
@@ -9,7 +10,10 @@ function street(name, aliases = []) {
|
|
|
9
10
|
entityType: 'street',
|
|
10
11
|
country: 'KZ',
|
|
11
12
|
aliases: all,
|
|
12
|
-
re
|
|
13
|
+
get re() {
|
|
14
|
+
re ||= aliasesToRegex(all);
|
|
15
|
+
return re;
|
|
16
|
+
},
|
|
13
17
|
});
|
|
14
18
|
}
|
|
15
19
|
|
|
@@ -2,6 +2,7 @@ import { aliasesToRegex } from './normalization.js';
|
|
|
2
2
|
|
|
3
3
|
function residential(name, aliases = []) {
|
|
4
4
|
const all = Object.freeze([...new Set([name, ...aliases].filter(Boolean))]);
|
|
5
|
+
let re = null;
|
|
5
6
|
return Object.freeze({
|
|
6
7
|
canonical: name,
|
|
7
8
|
name,
|
|
@@ -9,7 +10,10 @@ function residential(name, aliases = []) {
|
|
|
9
10
|
entityType: 'residential_complex',
|
|
10
11
|
country: 'KZ',
|
|
11
12
|
aliases: all,
|
|
12
|
-
re
|
|
13
|
+
get re() {
|
|
14
|
+
re ||= aliasesToRegex(all);
|
|
15
|
+
return re;
|
|
16
|
+
},
|
|
13
17
|
});
|
|
14
18
|
}
|
|
15
19
|
|
|
@@ -2,7 +2,18 @@ import { aliasesToRegex } from './normalization.js';
|
|
|
2
2
|
|
|
3
3
|
function entry(name, aliases = [], meta = {}) {
|
|
4
4
|
const all = [...new Set([name, ...aliases].filter(Boolean))];
|
|
5
|
-
|
|
5
|
+
let re = null;
|
|
6
|
+
return Object.freeze({
|
|
7
|
+
...meta,
|
|
8
|
+
canonical: name,
|
|
9
|
+
name,
|
|
10
|
+
type: meta.type || meta.entityType,
|
|
11
|
+
aliases: Object.freeze(all),
|
|
12
|
+
get re() {
|
|
13
|
+
re ||= aliasesToRegex(all);
|
|
14
|
+
return re;
|
|
15
|
+
},
|
|
16
|
+
});
|
|
6
17
|
}
|
|
7
18
|
|
|
8
19
|
function entries(rows = [], defaults = {}) {
|
|
@@ -2,6 +2,7 @@ import { aliasesToRegex } from './normalization.js';
|
|
|
2
2
|
|
|
3
3
|
function entry(name, aliases, entityType) {
|
|
4
4
|
const all = Object.freeze([...new Set([name, ...aliases].filter(Boolean))]);
|
|
5
|
+
let re = null;
|
|
5
6
|
return Object.freeze({
|
|
6
7
|
canonical: name,
|
|
7
8
|
name,
|
|
@@ -9,7 +10,10 @@ function entry(name, aliases, entityType) {
|
|
|
9
10
|
entityType,
|
|
10
11
|
country: 'KZ',
|
|
11
12
|
aliases: all,
|
|
12
|
-
re
|
|
13
|
+
get re() {
|
|
14
|
+
re ||= aliasesToRegex(all);
|
|
15
|
+
return re;
|
|
16
|
+
},
|
|
13
17
|
});
|
|
14
18
|
}
|
|
15
19
|
|
|
@@ -2,6 +2,7 @@ import { aliasesToRegex } from './normalization.js';
|
|
|
2
2
|
|
|
3
3
|
function street(name, aliases = []) {
|
|
4
4
|
const all = Object.freeze([...new Set([name, ...aliases].filter(Boolean))]);
|
|
5
|
+
let re = null;
|
|
5
6
|
return Object.freeze({
|
|
6
7
|
canonical: name,
|
|
7
8
|
name,
|
|
@@ -9,7 +10,10 @@ function street(name, aliases = []) {
|
|
|
9
10
|
entityType: 'street',
|
|
10
11
|
country: 'KZ',
|
|
11
12
|
aliases: all,
|
|
12
|
-
re
|
|
13
|
+
get re() {
|
|
14
|
+
re ||= aliasesToRegex(all);
|
|
15
|
+
return re;
|
|
16
|
+
},
|
|
13
17
|
});
|
|
14
18
|
}
|
|
15
19
|
|
|
@@ -2,6 +2,7 @@ import { aliasesToRegex } from './normalization.js';
|
|
|
2
2
|
|
|
3
3
|
function residential(name, aliases = []) {
|
|
4
4
|
const all = Object.freeze([...new Set([name, ...aliases].filter(Boolean))]);
|
|
5
|
+
let re = null;
|
|
5
6
|
return Object.freeze({
|
|
6
7
|
canonical: name,
|
|
7
8
|
name,
|
|
@@ -9,7 +10,10 @@ function residential(name, aliases = []) {
|
|
|
9
10
|
entityType: 'residential_complex',
|
|
10
11
|
country: 'KZ',
|
|
11
12
|
aliases: all,
|
|
12
|
-
re
|
|
13
|
+
get re() {
|
|
14
|
+
re ||= aliasesToRegex(all);
|
|
15
|
+
return re;
|
|
16
|
+
},
|
|
13
17
|
});
|
|
14
18
|
}
|
|
15
19
|
|
|
@@ -2,6 +2,7 @@ import { aliasesToRegex } from './normalization.js';
|
|
|
2
2
|
|
|
3
3
|
function residential(name, aliases = []) {
|
|
4
4
|
const all = Object.freeze([...new Set([name, ...aliases].filter(Boolean))]);
|
|
5
|
+
let re = null;
|
|
5
6
|
return Object.freeze({
|
|
6
7
|
canonical: name,
|
|
7
8
|
name,
|
|
@@ -9,7 +10,10 @@ function residential(name, aliases = []) {
|
|
|
9
10
|
entityType: 'residential_complex',
|
|
10
11
|
country: 'KZ',
|
|
11
12
|
aliases: all,
|
|
12
|
-
re
|
|
13
|
+
get re() {
|
|
14
|
+
re ||= aliasesToRegex(all);
|
|
15
|
+
return re;
|
|
16
|
+
},
|
|
13
17
|
});
|
|
14
18
|
}
|
|
15
19
|
|
|
@@ -2,6 +2,7 @@ import { aliasesToRegex } from './normalization.js';
|
|
|
2
2
|
|
|
3
3
|
function entry(name, aliases = [], entityType) {
|
|
4
4
|
const all = Object.freeze([...new Set([name, ...aliases].filter(Boolean))]);
|
|
5
|
+
let re = null;
|
|
5
6
|
return Object.freeze({
|
|
6
7
|
canonical: name,
|
|
7
8
|
name,
|
|
@@ -9,7 +10,10 @@ function entry(name, aliases = [], entityType) {
|
|
|
9
10
|
entityType,
|
|
10
11
|
country: 'KZ',
|
|
11
12
|
aliases: all,
|
|
12
|
-
re
|
|
13
|
+
get re() {
|
|
14
|
+
re ||= aliasesToRegex(all);
|
|
15
|
+
return re;
|
|
16
|
+
},
|
|
13
17
|
});
|
|
14
18
|
}
|
|
15
19
|
|
package/src/location-data.js
CHANGED
|
@@ -5,14 +5,35 @@ import { UA_REGIONS as UA_REGION_LEXICON } from './geography.js';
|
|
|
5
5
|
function entries(rows) {
|
|
6
6
|
return Object.freeze(rows.map(([name, ...aliases]) => {
|
|
7
7
|
const all = [...new Set([name, ...aliases].filter(Boolean))];
|
|
8
|
-
|
|
8
|
+
let re = null;
|
|
9
|
+
return Object.freeze({
|
|
10
|
+
canonical: name,
|
|
11
|
+
name,
|
|
12
|
+
aliases: Object.freeze(all),
|
|
13
|
+
get re() {
|
|
14
|
+
re ||= aliasesToRegex(all);
|
|
15
|
+
return re;
|
|
16
|
+
},
|
|
17
|
+
});
|
|
9
18
|
}));
|
|
10
19
|
}
|
|
11
20
|
|
|
12
21
|
function lexiconEntries(items) {
|
|
13
22
|
return Object.freeze((items || []).map((item) => {
|
|
14
23
|
const all = [...new Set([item.canonical, ...aliasesOf(item)].filter(Boolean))];
|
|
15
|
-
|
|
24
|
+
let re = null;
|
|
25
|
+
return Object.freeze({
|
|
26
|
+
canonical: item.canonical,
|
|
27
|
+
name: item.canonical,
|
|
28
|
+
type: item.type,
|
|
29
|
+
country: item.country,
|
|
30
|
+
city: item.city,
|
|
31
|
+
aliases: Object.freeze(all),
|
|
32
|
+
get re() {
|
|
33
|
+
re ||= aliasesToRegex(all);
|
|
34
|
+
return re;
|
|
35
|
+
},
|
|
36
|
+
});
|
|
16
37
|
}));
|
|
17
38
|
}
|
|
18
39
|
|
package/src/locations.js
CHANGED
|
@@ -115,6 +115,7 @@ const UZ_BASE_LOCATION_DICTIONARIES = Object.freeze({
|
|
|
115
115
|
|
|
116
116
|
function semanticEntry(entry, name, aliases, entityType) {
|
|
117
117
|
const all = Object.freeze([...new Set([name, ...aliases].filter(Boolean))]);
|
|
118
|
+
let re = null;
|
|
118
119
|
return Object.freeze({
|
|
119
120
|
...entry,
|
|
120
121
|
canonical: name,
|
|
@@ -122,7 +123,10 @@ function semanticEntry(entry, name, aliases, entityType) {
|
|
|
122
123
|
type: entityType,
|
|
123
124
|
entityType,
|
|
124
125
|
aliases: all,
|
|
125
|
-
re
|
|
126
|
+
get re() {
|
|
127
|
+
re ||= aliasesToRegex(all);
|
|
128
|
+
return re;
|
|
129
|
+
},
|
|
126
130
|
});
|
|
127
131
|
}
|
|
128
132
|
|
|
@@ -179,13 +183,17 @@ function normalizeUzSemanticLocations(country) {
|
|
|
179
183
|
'Корасув дахаси',
|
|
180
184
|
'Карасу даха',
|
|
181
185
|
])]);
|
|
186
|
+
let qorasuvRe = null;
|
|
182
187
|
const qorasuvArea = Object.freeze({
|
|
183
188
|
...qorasuv,
|
|
184
189
|
type: 'local_area',
|
|
185
190
|
entityType: 'local_area',
|
|
186
191
|
parent: 'Mirzo Ulugbek',
|
|
187
192
|
aliases: qorasuvAliases,
|
|
188
|
-
re
|
|
193
|
+
get re() {
|
|
194
|
+
qorasuvRe ||= aliasesToRegex(qorasuvAliases);
|
|
195
|
+
return qorasuvRe;
|
|
196
|
+
},
|
|
189
197
|
});
|
|
190
198
|
|
|
191
199
|
normalized = Object.freeze({
|
package/src/normalization.js
CHANGED
|
@@ -3,16 +3,39 @@ import { LEXICON_LANGUAGES } from './lexicon-core.js';
|
|
|
3
3
|
const APOSTROPHES_RE = /[’‘ʻʼ`´]/g;
|
|
4
4
|
const DASHES_RE = /[‐‑‒–—―]/g;
|
|
5
5
|
|
|
6
|
+
// Parsing one vacancy or listing runs dozens of matchers over the same text,
|
|
7
|
+
// and each normalizes it again from scratch: profiling the workforce
|
|
8
|
+
// enrichment showed normalization among the top costs at ~170-490 ms per
|
|
9
|
+
// vacancy. These functions are pure, so each keeps a small cache of its
|
|
10
|
+
// recent long inputs. Short strings (dictionary aliases, single words) are
|
|
11
|
+
// cheaper to normalize than to cache and are left alone; the cap bounds
|
|
12
|
+
// memory to a few recent texts per function.
|
|
13
|
+
const MEMO_MIN_LENGTH = 200;
|
|
14
|
+
const MEMO_MAX_ENTRIES = 32;
|
|
15
|
+
|
|
16
|
+
function memoizeLongText(fn) {
|
|
17
|
+
const cache = new Map();
|
|
18
|
+
return (value) => {
|
|
19
|
+
if (typeof value !== 'string' || value.length < MEMO_MIN_LENGTH) return fn(value);
|
|
20
|
+
const hit = cache.get(value);
|
|
21
|
+
if (hit !== undefined) return hit;
|
|
22
|
+
const result = fn(value);
|
|
23
|
+
if (cache.size >= MEMO_MAX_ENTRIES) cache.delete(cache.keys().next().value);
|
|
24
|
+
cache.set(value, result);
|
|
25
|
+
return result;
|
|
26
|
+
};
|
|
27
|
+
}
|
|
28
|
+
|
|
6
29
|
/** Normalize Unicode and punctuation variants without destroying letters. */
|
|
7
|
-
export function normalizeUnicode(value) {
|
|
30
|
+
export const normalizeUnicode = memoizeLongText(function normalizeUnicode(value) {
|
|
8
31
|
return String(value ?? '')
|
|
9
32
|
.normalize('NFKC')
|
|
10
33
|
.replace(APOSTROPHES_RE, "'")
|
|
11
34
|
.replace(DASHES_RE, '-');
|
|
12
|
-
}
|
|
35
|
+
});
|
|
13
36
|
|
|
14
37
|
/** Stable comparison form for parser dictionaries. */
|
|
15
|
-
export function normalizeForMatch(value) {
|
|
38
|
+
export const normalizeForMatch = memoizeLongText(function normalizeForMatch(value) {
|
|
16
39
|
return normalizeUnicode(value)
|
|
17
40
|
.toLocaleLowerCase()
|
|
18
41
|
.replace(/ё/g, 'е')
|
|
@@ -20,10 +43,10 @@ export function normalizeForMatch(value) {
|
|
|
20
43
|
.replace(/[^\p{L}\p{N}]+/gu, ' ')
|
|
21
44
|
.replace(/\s+/g, ' ')
|
|
22
45
|
.trim();
|
|
23
|
-
}
|
|
46
|
+
});
|
|
24
47
|
|
|
25
48
|
/** Search-only apostrophe compaction for Uzbek/Karakalpak variants. */
|
|
26
|
-
function normalizeCompactApostropheForMatch(value) {
|
|
49
|
+
const normalizeCompactApostropheForMatch = memoizeLongText(function normalizeCompactApostropheForMatch(value) {
|
|
27
50
|
return normalizeUnicode(value)
|
|
28
51
|
.toLocaleLowerCase()
|
|
29
52
|
.replace(/ё/g, 'е')
|
|
@@ -32,7 +55,7 @@ function normalizeCompactApostropheForMatch(value) {
|
|
|
32
55
|
.replace(/[^\p{L}\p{N}]+/gu, ' ')
|
|
33
56
|
.replace(/\s+/g, ' ')
|
|
34
57
|
.trim();
|
|
35
|
-
}
|
|
58
|
+
});
|
|
36
59
|
|
|
37
60
|
export const CYRILLIC_SEARCH_MAP = Object.freeze({
|
|
38
61
|
а: 'a', б: 'b', в: 'v', г: 'g', д: 'd', е: 'e', ё: 'e', ж: 'zh', з: 'z',
|
|
@@ -65,21 +88,21 @@ const KARAKALPAK_LATIN_PATTERN_EQUIVALENCE = Object.freeze({
|
|
|
65
88
|
});
|
|
66
89
|
|
|
67
90
|
/** Search-oriented Cyrillic folding; canonical identity still comes from explicit aliases. */
|
|
68
|
-
export function foldCyrillicForSearch(value) {
|
|
91
|
+
export const foldCyrillicForSearch = memoizeLongText(function foldCyrillicForSearch(value) {
|
|
69
92
|
return normalizeUnicode(value)
|
|
70
93
|
.toLocaleLowerCase()
|
|
71
94
|
.split('')
|
|
72
95
|
.map((char) => CYRILLIC_SEARCH_MAP[char] ?? char)
|
|
73
96
|
.join('');
|
|
74
|
-
}
|
|
97
|
+
});
|
|
75
98
|
|
|
76
|
-
function foldKazakhForSearch(value) {
|
|
99
|
+
const foldKazakhForSearch = memoizeLongText(function foldKazakhForSearch(value) {
|
|
77
100
|
return normalizeUnicode(value)
|
|
78
101
|
.toLocaleLowerCase()
|
|
79
102
|
.split('')
|
|
80
103
|
.map((char) => KAZAKH_SEARCH_EQUIVALENCE[char] ?? char)
|
|
81
104
|
.join('');
|
|
82
|
-
}
|
|
105
|
+
});
|
|
83
106
|
|
|
84
107
|
function foldKarakalpakLatinForSearch(value) {
|
|
85
108
|
return normalizeUnicode(value)
|
|
@@ -2,12 +2,16 @@ import { aliasesToRegex } from './normalization.js';
|
|
|
2
2
|
|
|
3
3
|
function entity(name, type, aliases = [], options = {}) {
|
|
4
4
|
const all = [...new Set([name, ...aliases].filter(Boolean))];
|
|
5
|
+
let re = null;
|
|
5
6
|
return Object.freeze({
|
|
6
7
|
canonical: name,
|
|
7
8
|
name,
|
|
8
9
|
type,
|
|
9
10
|
aliases: Object.freeze(all),
|
|
10
|
-
re
|
|
11
|
+
get re() {
|
|
12
|
+
re ||= aliasesToRegex(all);
|
|
13
|
+
return re;
|
|
14
|
+
},
|
|
11
15
|
parent: options.parent || null,
|
|
12
16
|
cluster: options.cluster || null,
|
|
13
17
|
contextRequired: Boolean(options.contextRequired),
|
|
@@ -2,6 +2,7 @@ import { aliasesToRegex } from './normalization.js';
|
|
|
2
2
|
|
|
3
3
|
function gastronomicStreet(canonical, aliases = [], district = null) {
|
|
4
4
|
const all = Object.freeze([...new Set([canonical, ...aliases].filter(Boolean))]);
|
|
5
|
+
let re = null;
|
|
5
6
|
return Object.freeze({
|
|
6
7
|
canonical,
|
|
7
8
|
name: canonical,
|
|
@@ -11,7 +12,10 @@ function gastronomicStreet(canonical, aliases = [], district = null) {
|
|
|
11
12
|
city: 'Tashkent',
|
|
12
13
|
district,
|
|
13
14
|
aliases: all,
|
|
14
|
-
re
|
|
15
|
+
get re() {
|
|
16
|
+
re ||= aliasesToRegex(all);
|
|
17
|
+
return re;
|
|
18
|
+
},
|
|
15
19
|
});
|
|
16
20
|
}
|
|
17
21
|
|
|
@@ -3,6 +3,7 @@ import { TASHKENT_DISTRICTS, TASHKENT_METRO } from './geo.js';
|
|
|
3
3
|
|
|
4
4
|
function locationEntry(name, category, aliases = [], options = {}) {
|
|
5
5
|
const all = [...new Set([name, ...aliases].filter(Boolean))];
|
|
6
|
+
let re = null;
|
|
6
7
|
return Object.freeze({
|
|
7
8
|
canonical: name,
|
|
8
9
|
name,
|
|
@@ -10,7 +11,10 @@ function locationEntry(name, category, aliases = [], options = {}) {
|
|
|
10
11
|
country: 'UZ',
|
|
11
12
|
city: 'Tashkent',
|
|
12
13
|
aliases: Object.freeze(all),
|
|
13
|
-
re
|
|
14
|
+
get re() {
|
|
15
|
+
re ||= aliasesToRegex(all);
|
|
16
|
+
return re;
|
|
17
|
+
},
|
|
14
18
|
contextRequired: Boolean(options.contextRequired),
|
|
15
19
|
contextRe: options.context ? new RegExp(options.context, 'iu') : null,
|
|
16
20
|
});
|
|
@@ -18,6 +22,7 @@ function locationEntry(name, category, aliases = [], options = {}) {
|
|
|
18
22
|
|
|
19
23
|
const TASHKENT_HOUSING_DISTRICTS = Object.freeze(TASHKENT_DISTRICTS.map((district) => {
|
|
20
24
|
const aliases = [...new Set([district.canonical, ...aliasesOf(district)].filter(Boolean))];
|
|
25
|
+
let re = null;
|
|
21
26
|
return Object.freeze({
|
|
22
27
|
canonical: district.canonical,
|
|
23
28
|
name: district.canonical,
|
|
@@ -25,7 +30,10 @@ const TASHKENT_HOUSING_DISTRICTS = Object.freeze(TASHKENT_DISTRICTS.map((distric
|
|
|
25
30
|
country: 'UZ',
|
|
26
31
|
city: 'Tashkent',
|
|
27
32
|
aliases: Object.freeze(aliases),
|
|
28
|
-
re
|
|
33
|
+
get re() {
|
|
34
|
+
re ||= aliasesToRegex(aliases);
|
|
35
|
+
return re;
|
|
36
|
+
},
|
|
29
37
|
});
|
|
30
38
|
}));
|
|
31
39
|
|
package/src/tashkent-pois.js
CHANGED
|
@@ -2,13 +2,17 @@ import { aliasesToRegex } from './normalization.js';
|
|
|
2
2
|
|
|
3
3
|
function poi(name, category, aliases = [], options = {}) {
|
|
4
4
|
const all = [...new Set([name, ...aliases].filter(Boolean))];
|
|
5
|
+
let re = null;
|
|
5
6
|
return Object.freeze({
|
|
6
7
|
canonical: name,
|
|
7
8
|
name,
|
|
8
9
|
category,
|
|
9
10
|
categories: Object.freeze([...new Set([category, ...(options.categories || [])])]),
|
|
10
11
|
aliases: Object.freeze(all),
|
|
11
|
-
re
|
|
12
|
+
get re() {
|
|
13
|
+
re ||= aliasesToRegex(all);
|
|
14
|
+
return re;
|
|
15
|
+
},
|
|
12
16
|
contextRequired: Boolean(options.contextRequired),
|
|
13
17
|
contextRe: options.context ? new RegExp(options.context, 'iu') : null,
|
|
14
18
|
...(options.parent ? { parent: options.parent } : {}),
|
|
@@ -12,12 +12,16 @@ const AMBIGUOUS_TASHKENT_COMPLEXES = new Set([
|
|
|
12
12
|
|
|
13
13
|
function complex(name, aliases = []) {
|
|
14
14
|
const all = [...new Set([name, ...aliases].filter(Boolean))];
|
|
15
|
+
let re = null;
|
|
15
16
|
return Object.freeze({
|
|
16
17
|
canonical: name,
|
|
17
18
|
name,
|
|
18
19
|
aliases: Object.freeze(all),
|
|
19
20
|
ambiguous: AMBIGUOUS_TASHKENT_COMPLEXES.has(name),
|
|
20
|
-
re
|
|
21
|
+
get re() {
|
|
22
|
+
re ||= aliasesToRegex(all);
|
|
23
|
+
return re;
|
|
24
|
+
},
|
|
21
25
|
});
|
|
22
26
|
}
|
|
23
27
|
|
|
@@ -2,6 +2,7 @@ import { aliasesToRegex } from './normalization.js';
|
|
|
2
2
|
|
|
3
3
|
function residential(name, aliases = []) {
|
|
4
4
|
const all = Object.freeze([...new Set([name, ...aliases].filter(Boolean))]);
|
|
5
|
+
let re = null;
|
|
5
6
|
return Object.freeze({
|
|
6
7
|
canonical: name,
|
|
7
8
|
name,
|
|
@@ -9,7 +10,10 @@ function residential(name, aliases = []) {
|
|
|
9
10
|
entityType: 'residential_complex',
|
|
10
11
|
country: 'UA',
|
|
11
12
|
aliases: all,
|
|
12
|
-
re
|
|
13
|
+
get re() {
|
|
14
|
+
re ||= aliasesToRegex(all);
|
|
15
|
+
return re;
|
|
16
|
+
},
|
|
13
17
|
});
|
|
14
18
|
}
|
|
15
19
|
|
|
@@ -2,6 +2,7 @@ import { aliasesToRegex } from './normalization.js';
|
|
|
2
2
|
|
|
3
3
|
function residential(name, aliases = []) {
|
|
4
4
|
const all = Object.freeze([...new Set([name, ...aliases].filter(Boolean))]);
|
|
5
|
+
let re = null;
|
|
5
6
|
return Object.freeze({
|
|
6
7
|
canonical: name,
|
|
7
8
|
name,
|
|
@@ -9,7 +10,10 @@ function residential(name, aliases = []) {
|
|
|
9
10
|
entityType: 'residential_complex',
|
|
10
11
|
country: 'UA',
|
|
11
12
|
aliases: all,
|
|
12
|
-
re
|
|
13
|
+
get re() {
|
|
14
|
+
re ||= aliasesToRegex(all);
|
|
15
|
+
return re;
|
|
16
|
+
},
|
|
13
17
|
});
|
|
14
18
|
}
|
|
15
19
|
|
|
@@ -2,7 +2,18 @@ import { aliasesToRegex } from './normalization.js';
|
|
|
2
2
|
|
|
3
3
|
function entry(name, aliases = [], meta = {}) {
|
|
4
4
|
const all = [...new Set([name, ...aliases].filter(Boolean))];
|
|
5
|
-
|
|
5
|
+
let re = null;
|
|
6
|
+
return Object.freeze({
|
|
7
|
+
...meta,
|
|
8
|
+
canonical: name,
|
|
9
|
+
name,
|
|
10
|
+
type: meta.type || meta.entityType,
|
|
11
|
+
aliases: Object.freeze(all),
|
|
12
|
+
get re() {
|
|
13
|
+
re ||= aliasesToRegex(all);
|
|
14
|
+
return re;
|
|
15
|
+
},
|
|
16
|
+
});
|
|
6
17
|
}
|
|
7
18
|
|
|
8
19
|
function entries(rows = [], defaults = {}) {
|
|
@@ -2,6 +2,7 @@ import { aliasesToRegex } from './normalization.js';
|
|
|
2
2
|
|
|
3
3
|
function entry(canonical, type, aliases = [], labels = null) {
|
|
4
4
|
const all = Object.freeze([...new Set([canonical, ...aliases].filter(Boolean))]);
|
|
5
|
+
let re = null;
|
|
5
6
|
return Object.freeze({
|
|
6
7
|
canonical,
|
|
7
8
|
name: canonical,
|
|
@@ -11,7 +12,10 @@ function entry(canonical, type, aliases = [], labels = null) {
|
|
|
11
12
|
city: 'Samarkand',
|
|
12
13
|
...(labels ? { labels: Object.freeze(labels) } : {}),
|
|
13
14
|
aliases: all,
|
|
14
|
-
re
|
|
15
|
+
get re() {
|
|
16
|
+
re ||= aliasesToRegex(all);
|
|
17
|
+
return re;
|
|
18
|
+
},
|
|
15
19
|
});
|
|
16
20
|
}
|
|
17
21
|
|
|
@@ -11,6 +11,7 @@ function legacyAreaEntry(district, canonical) {
|
|
|
11
11
|
...(source.aliases || []),
|
|
12
12
|
].filter(Boolean))]);
|
|
13
13
|
|
|
14
|
+
let re = null;
|
|
14
15
|
return Object.freeze({
|
|
15
16
|
canonical: source.name,
|
|
16
17
|
name: source.name,
|
|
@@ -21,12 +22,16 @@ function legacyAreaEntry(district, canonical) {
|
|
|
21
22
|
district,
|
|
22
23
|
parent: district,
|
|
23
24
|
aliases,
|
|
24
|
-
re
|
|
25
|
+
get re() {
|
|
26
|
+
re ||= aliasesToRegex(aliases);
|
|
27
|
+
return re;
|
|
28
|
+
},
|
|
25
29
|
});
|
|
26
30
|
}
|
|
27
31
|
|
|
28
32
|
function residentialEntry(canonical, aliases = []) {
|
|
29
33
|
const all = Object.freeze([...new Set([canonical, ...aliases].filter(Boolean))]);
|
|
34
|
+
let re = null;
|
|
30
35
|
return Object.freeze({
|
|
31
36
|
canonical,
|
|
32
37
|
name: canonical,
|
|
@@ -35,7 +40,10 @@ function residentialEntry(canonical, aliases = []) {
|
|
|
35
40
|
country: 'UZ',
|
|
36
41
|
city: 'Tashkent',
|
|
37
42
|
aliases: all,
|
|
38
|
-
re
|
|
43
|
+
get re() {
|
|
44
|
+
re ||= aliasesToRegex(all);
|
|
45
|
+
return re;
|
|
46
|
+
},
|
|
39
47
|
});
|
|
40
48
|
}
|
|
41
49
|
|