@whiteslove/parsing-lexicon 0.9.9 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/index.d.ts +2 -0
- package/package.json +49 -2
- package/src/alias-prefilter.js +107 -51
- package/src/central-asia-locations.js +26 -7
- package/src/city-hypotheses.js +0 -0
- package/src/cv-employment.d.ts +48 -0
- package/src/cv-employment.js +153 -0
- package/src/cv-sections.d.ts +15 -0
- package/src/cv-sections.js +147 -0
- package/src/cv-skill-experience.d.ts +19 -0
- package/src/cv-skill-experience.js +0 -0
- package/src/evidence-ledger.d.ts +31 -0
- package/src/evidence-ledger.js +0 -0
- package/src/hiring-ats.js +7 -2
- package/src/hiring-requirements.d.ts +3 -3
- package/src/hiring-requirements.js +28 -65
- package/src/hiring-temporal.js +1 -1
- package/src/housing-money.js +32 -19
- package/src/housing-text.js +12 -6
- package/src/index.js +2 -0
- package/src/locations-runtime.js +3 -1
- package/src/micro-grammars.js +144 -0
- package/src/money-core.js +19 -5
- package/src/normalization.js +51 -12
- package/src/parse-document.d.ts +21 -0
- package/src/parse-document.js +79 -0
- package/src/parser-core.d.ts +5 -2
- package/src/parser-core.js +17 -5
- package/src/provenance.d.ts +25 -0
- package/src/provenance.js +73 -0
- package/src/resolver-v2.d.ts +38 -0
- package/src/resolver-v2.js +0 -0
- package/src/resolver-weights.js +123 -0
- package/src/vacancy-blocks.d.ts +21 -0
- package/src/vacancy-blocks.js +134 -0
- package/src/vacancy-requirements.d.ts +36 -0
- package/src/vacancy-requirements.js +0 -0
package/index.d.ts
CHANGED
|
@@ -372,6 +372,8 @@ export function parseExperience(value: unknown): ExperienceParseResult | null;
|
|
|
372
372
|
|
|
373
373
|
export * from './src/housing-context.js';
|
|
374
374
|
export * from './src/housing-address.js';
|
|
375
|
+
// Keep the address grammar as the root owner; the V2 extractor remains on its subpath.
|
|
376
|
+
export { extractHousingAddressCandidates } from './src/housing-address.js';
|
|
375
377
|
export * from './src/parser-core.js';
|
|
376
378
|
export * from './src/temporal.js';
|
|
377
379
|
export * from './src/hiring-context.js';
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@whiteslove/parsing-lexicon",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.10.0",
|
|
4
4
|
"description": "Shared deterministic multilingual parsing lexicon for WhitesLove housing and hiring services",
|
|
5
5
|
"repository": {
|
|
6
6
|
"type": "git",
|
|
@@ -17,6 +17,38 @@
|
|
|
17
17
|
"./lexicon-core": "./src/lexicon-core.js",
|
|
18
18
|
"./normalization": "./src/normalization.js",
|
|
19
19
|
"./alias-prefilter": "./src/alias-prefilter.js",
|
|
20
|
+
"./provenance": {
|
|
21
|
+
"types": "./src/provenance.d.ts",
|
|
22
|
+
"import": "./src/provenance.js"
|
|
23
|
+
},
|
|
24
|
+
"./vacancy-requirements": {
|
|
25
|
+
"types": "./src/vacancy-requirements.d.ts",
|
|
26
|
+
"import": "./src/vacancy-requirements.js"
|
|
27
|
+
},
|
|
28
|
+
"./vacancy-blocks": {
|
|
29
|
+
"types": "./src/vacancy-blocks.d.ts",
|
|
30
|
+
"import": "./src/vacancy-blocks.js"
|
|
31
|
+
},
|
|
32
|
+
"./cv-skill-experience": {
|
|
33
|
+
"types": "./src/cv-skill-experience.d.ts",
|
|
34
|
+
"import": "./src/cv-skill-experience.js"
|
|
35
|
+
},
|
|
36
|
+
"./cv-employment": {
|
|
37
|
+
"types": "./src/cv-employment.d.ts",
|
|
38
|
+
"import": "./src/cv-employment.js"
|
|
39
|
+
},
|
|
40
|
+
"./cv-sections": {
|
|
41
|
+
"types": "./src/cv-sections.d.ts",
|
|
42
|
+
"import": "./src/cv-sections.js"
|
|
43
|
+
},
|
|
44
|
+
"./resolver-v2": {
|
|
45
|
+
"types": "./src/resolver-v2.d.ts",
|
|
46
|
+
"import": "./src/resolver-v2.js"
|
|
47
|
+
},
|
|
48
|
+
"./evidence-ledger": {
|
|
49
|
+
"types": "./src/evidence-ledger.d.ts",
|
|
50
|
+
"import": "./src/evidence-ledger.js"
|
|
51
|
+
},
|
|
20
52
|
"./parser-core": {
|
|
21
53
|
"types": "./src/parser-core.d.ts",
|
|
22
54
|
"import": "./src/parser-core.js"
|
|
@@ -223,6 +255,19 @@
|
|
|
223
255
|
"./hiring-safety": {
|
|
224
256
|
"types": "./src/hiring-safety.d.ts",
|
|
225
257
|
"import": "./src/hiring-safety.js"
|
|
258
|
+
},
|
|
259
|
+
"./parse-document": {
|
|
260
|
+
"types": "./src/parse-document.d.ts",
|
|
261
|
+
"import": "./src/parse-document.js"
|
|
262
|
+
},
|
|
263
|
+
"./city-hypotheses": {
|
|
264
|
+
"import": "./src/city-hypotheses.js"
|
|
265
|
+
},
|
|
266
|
+
"./micro-grammars": {
|
|
267
|
+
"import": "./src/micro-grammars.js"
|
|
268
|
+
},
|
|
269
|
+
"./resolver-weights": {
|
|
270
|
+
"import": "./src/resolver-weights.js"
|
|
226
271
|
}
|
|
227
272
|
},
|
|
228
273
|
"files": [
|
|
@@ -237,7 +282,9 @@
|
|
|
237
282
|
"scripts": {
|
|
238
283
|
"test": "node --test",
|
|
239
284
|
"sync:geo-map-data": "node ./scripts/sync-geo-map-data-lexicon.js",
|
|
240
|
-
"audit:geo-map-data": "node ./scripts/audit-geo-map-data-lexicon.js"
|
|
285
|
+
"audit:geo-map-data": "node ./scripts/audit-geo-map-data-lexicon.js",
|
|
286
|
+
"benchmark:parser": "node benchmarks/parser-baseline.js",
|
|
287
|
+
"calibrate:resolver": "node ./scripts/calibrate-resolver-weights.js"
|
|
241
288
|
},
|
|
242
289
|
"publishConfig": {
|
|
243
290
|
"access": "public"
|
package/src/alias-prefilter.js
CHANGED
|
@@ -9,20 +9,31 @@
|
|
|
9
9
|
// Almost all of that work is provably wasted. aliasPattern() only relaxes
|
|
10
10
|
// *separators* between alias words; the letters and digits of an alias are
|
|
11
11
|
// matched literally (modulo the Karakalpak Latin equivalence classes and
|
|
12
|
-
// case). So
|
|
13
|
-
// contiguous substring of the text for that alias to match at all.
|
|
14
|
-
//
|
|
15
|
-
// n-grams select the handful of entries worth testing.
|
|
12
|
+
// case). So every maximal run of letters/digits inside an alias must appear as
|
|
13
|
+
// a contiguous substring of the text for that alias to match at all. That is
|
|
14
|
+
// the invariant the whole index rests on.
|
|
16
15
|
//
|
|
17
16
|
// The index is a filter, never a verdict: every surviving candidate is still
|
|
18
17
|
// matched with its real `re`, in the list's original order, so results are
|
|
19
18
|
// identical to a full scan. Folding is deliberately more aggressive than the
|
|
20
19
|
// pattern's own equivalences (Cyrillic ё/ў/қ/ғ… are collapsed): over-merging
|
|
21
20
|
// can only admit extra candidates, never discard a real match.
|
|
21
|
+
//
|
|
22
|
+
// Indexing is per alias, not per entry. An entry whose aliases include one
|
|
23
|
+
// short string used to fall out of the index entirely and be tested against
|
|
24
|
+
// every text; now each alias is indexed on its own terms, so a single short
|
|
25
|
+
// alias costs only that alias, never the whole entry. Bucket keys are chosen
|
|
26
|
+
// by rarity: of the grams an alias could be filed under, the one appearing in
|
|
27
|
+
// the fewest aliases wins, which keeps buckets small where the vocabulary is
|
|
28
|
+
// dense. Surviving candidates are then verified by exact substring containment
|
|
29
|
+
// of every run, which is cheap and admits almost nothing spurious.
|
|
22
30
|
|
|
23
31
|
import { normalizeUnicode } from './normalization.js';
|
|
24
32
|
|
|
25
33
|
const GRAM = 4;
|
|
34
|
+
/** Runs shorter than GRAM are looked up against every text substring of the
|
|
35
|
+
* same length, so this bounds how many such sets a query builds. */
|
|
36
|
+
const MAX_SHORT = GRAM - 1;
|
|
26
37
|
|
|
27
38
|
const PREFILTER_FOLD = Object.freeze({
|
|
28
39
|
// Karakalpak Latin equivalences aliasPattern() encodes as character classes.
|
|
@@ -42,58 +53,59 @@ function fold(value) {
|
|
|
42
53
|
return out;
|
|
43
54
|
}
|
|
44
55
|
|
|
45
|
-
/**
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
56
|
+
/** Maximal literal runs of an alias, folded. Each must appear verbatim in the
|
|
57
|
+
* text for the alias to match, which is what makes them safe filter keys. */
|
|
58
|
+
function aliasRuns(alias) {
|
|
59
|
+
return fold(alias).split(NON_ALNUM_RE).filter(Boolean);
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
function* windows(run) {
|
|
63
|
+
for (let i = 0; i + GRAM <= run.length; i += 1) yield run.slice(i, i + GRAM);
|
|
52
64
|
}
|
|
53
65
|
|
|
54
|
-
/** Every GRAM-length window of the folded text
|
|
55
|
-
|
|
66
|
+
/** Every GRAM-length window of the folded text, plus the shorter substrings a
|
|
67
|
+
* short alias needs. Both are needed because an alias run of two characters
|
|
68
|
+
* cannot be found in a set of four-character windows. */
|
|
69
|
+
function textIndex(text) {
|
|
56
70
|
const folded = fold(text);
|
|
57
71
|
const grams = new Set();
|
|
58
|
-
for (let i = 0; i + GRAM <= folded.length; i += 1)
|
|
59
|
-
|
|
72
|
+
for (let i = 0; i + GRAM <= folded.length; i += 1) grams.add(folded.slice(i, i + GRAM));
|
|
73
|
+
const shorts = new Set();
|
|
74
|
+
for (let size = 1; size <= MAX_SHORT; size += 1) {
|
|
75
|
+
for (let i = 0; i + size <= folded.length; i += 1) shorts.add(folded.slice(i, i + size));
|
|
60
76
|
}
|
|
61
|
-
return grams;
|
|
77
|
+
return { folded, grams, shorts };
|
|
62
78
|
}
|
|
63
79
|
|
|
64
80
|
function buildIndex(entries) {
|
|
65
|
-
|
|
66
|
-
//
|
|
67
|
-
|
|
68
|
-
const
|
|
69
|
-
|
|
81
|
+
// Pass one counts how many aliases could be filed under each gram, so pass
|
|
82
|
+
// two can file every alias under its rarest option.
|
|
83
|
+
const frequency = new Map();
|
|
84
|
+
const prepared = [];
|
|
70
85
|
for (let i = 0; i < entries.length; i += 1) {
|
|
71
86
|
const entry = entries[i];
|
|
72
87
|
const aliases = entry?.aliases?.length ? entry.aliases : [entry?.name].filter(Boolean);
|
|
73
|
-
const grams = [];
|
|
74
|
-
let indexable = aliases.length > 0;
|
|
75
|
-
|
|
76
88
|
for (const alias of aliases) {
|
|
77
|
-
const
|
|
78
|
-
if (!
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
}
|
|
84
|
-
|
|
85
|
-
if (!indexable) {
|
|
86
|
-
always.push(i);
|
|
87
|
-
continue;
|
|
88
|
-
}
|
|
89
|
-
for (const gram of new Set(grams)) {
|
|
90
|
-
const bucket = byGram.get(gram);
|
|
91
|
-
if (bucket) bucket.push(i);
|
|
92
|
-
else byGram.set(gram, [i]);
|
|
89
|
+
const runs = aliasRuns(alias);
|
|
90
|
+
if (!runs.length) { prepared.push({ index: i, runs: null }); continue; }
|
|
91
|
+
const longest = runs.reduce((best, run) => (run.length > best.length ? run : best), '');
|
|
92
|
+
const options = longest.length >= GRAM ? [...windows(longest)] : [longest];
|
|
93
|
+
for (const option of options) frequency.set(option, (frequency.get(option) ?? 0) + 1);
|
|
94
|
+
prepared.push({ index: i, runs, options, short: longest.length < GRAM });
|
|
93
95
|
}
|
|
94
96
|
}
|
|
95
97
|
|
|
96
|
-
|
|
98
|
+
const byGram = new Map(); const byShort = new Map(); const always = new Set();
|
|
99
|
+
for (const alias of prepared) {
|
|
100
|
+
if (!alias.runs) { always.add(alias.index); continue; }
|
|
101
|
+
let key = alias.options[0];
|
|
102
|
+
for (const option of alias.options) if (frequency.get(option) < frequency.get(key)) key = option;
|
|
103
|
+
const target = alias.short ? byShort : byGram;
|
|
104
|
+
const bucket = target.get(key);
|
|
105
|
+
const ref = { index: alias.index, runs: alias.runs };
|
|
106
|
+
if (bucket) bucket.push(ref); else target.set(key, [ref]);
|
|
107
|
+
}
|
|
108
|
+
return { byGram, byShort, always: [...always].sort((a, b) => a - b) };
|
|
97
109
|
}
|
|
98
110
|
|
|
99
111
|
const INDEX_CACHE = new WeakMap();
|
|
@@ -107,6 +119,59 @@ function indexFor(entries) {
|
|
|
107
119
|
return index;
|
|
108
120
|
}
|
|
109
121
|
|
|
122
|
+
/**
|
|
123
|
+
* Text-side index for `text`, precomputed once for reuse across lists. The
|
|
124
|
+
* returned value is opaque: pass it straight back to `candidateEntries`.
|
|
125
|
+
*/
|
|
126
|
+
export function computeTextGrams(text) {
|
|
127
|
+
return textIndex(String(text || ''));
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
/** Accepts an opaque index or, for older callers, a bare Set of grams. */
|
|
131
|
+
function resolveTextIndex(value, text) {
|
|
132
|
+
if (value && value.grams instanceof Set) return value;
|
|
133
|
+
if (value instanceof Set) return { ...textIndex(text), grams: value };
|
|
134
|
+
return textIndex(text);
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
function candidateIndices(entries, value, provided) {
|
|
138
|
+
const { byGram, byShort, always } = indexFor(entries);
|
|
139
|
+
const { folded, grams, shorts } = resolveTextIndex(provided, value);
|
|
140
|
+
const candidates = new Set(always);
|
|
141
|
+
const consider = (refs) => {
|
|
142
|
+
if (!refs) return;
|
|
143
|
+
for (const ref of refs) {
|
|
144
|
+
if (candidates.has(ref.index)) continue;
|
|
145
|
+
// Exact containment of every run. The bucket only proposed this alias;
|
|
146
|
+
// this is what makes the proposal almost always correct.
|
|
147
|
+
let ok = true;
|
|
148
|
+
for (const run of ref.runs) if (!folded.includes(run)) { ok = false; break; }
|
|
149
|
+
if (ok) candidates.add(ref.index);
|
|
150
|
+
}
|
|
151
|
+
};
|
|
152
|
+
for (const gram of grams) consider(byGram.get(gram));
|
|
153
|
+
if (byShort.size) for (const short of shorts) consider(byShort.get(short));
|
|
154
|
+
return [...candidates].sort((a, b) => a - b);
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
/**
|
|
158
|
+
* Every entry in `entries` (original list order) the text could plausibly
|
|
159
|
+
* contain, per the index — a filter, not a verdict. Callers still run their
|
|
160
|
+
* own verification (regex, exact-token match, ...) on what comes back; see the
|
|
161
|
+
* module doc for why this is safe even for non-regex verifiers.
|
|
162
|
+
*
|
|
163
|
+
* `grams`, from `computeTextGrams()`, lets a caller scanning the same text
|
|
164
|
+
* against many lists compute the O(text length) text pass once instead of
|
|
165
|
+
* once per list.
|
|
166
|
+
*/
|
|
167
|
+
export function candidateEntries(entries, text, grams) {
|
|
168
|
+
if (!Array.isArray(entries) || !entries.length) return [];
|
|
169
|
+
const value = String(text || '');
|
|
170
|
+
if (!value) return [];
|
|
171
|
+
|
|
172
|
+
return candidateIndices(entries, value, grams).map((i) => entries[i]);
|
|
173
|
+
}
|
|
174
|
+
|
|
110
175
|
/**
|
|
111
176
|
* First entry in `entries` whose alias regex matches `text`.
|
|
112
177
|
*
|
|
@@ -124,16 +189,7 @@ export function matchFirstEntry(entries, text, accept) {
|
|
|
124
189
|
const value = String(text || '');
|
|
125
190
|
if (!value) return undefined;
|
|
126
191
|
|
|
127
|
-
const
|
|
128
|
-
const candidates = new Set(always);
|
|
129
|
-
for (const gram of textGrams(value)) {
|
|
130
|
-
const bucket = byGram.get(gram);
|
|
131
|
-
if (!bucket) continue;
|
|
132
|
-
for (const i of bucket) candidates.add(i);
|
|
133
|
-
}
|
|
134
|
-
if (!candidates.size) return undefined;
|
|
135
|
-
|
|
136
|
-
for (const i of [...candidates].sort((a, b) => a - b)) {
|
|
192
|
+
for (const i of candidateIndices(entries, value)) {
|
|
137
193
|
const entry = entries[i];
|
|
138
194
|
// `re` carries no /g flag, so exec() is stateless and safe to reuse here.
|
|
139
195
|
const match = entry?.re?.exec(value);
|
|
@@ -2,6 +2,8 @@ import { LOCATION_DICTIONARIES } from './locations-runtime.js';
|
|
|
2
2
|
import { isMapDataEntry, LOCATION_LIST_KEYS } from './location-merge.js';
|
|
3
3
|
import { CITIES_BY_COUNTRY, canonicalCity } from './geography.js';
|
|
4
4
|
import { aliasesOf, aliasesToRegex, normalizeForMatch } from './normalization.js';
|
|
5
|
+
import { candidateEntries, computeTextGrams } from './alias-prefilter.js';
|
|
6
|
+
import { scoreCityHypotheses } from './city-hypotheses.js';
|
|
5
7
|
import { KZ_AMBIGUOUS_LOCAL_NAMES, KZ_SEARCH_CLUSTERS } from './kz-location-extensions.js';
|
|
6
8
|
import { UZ_AMBIGUOUS_LOCAL_NAMES } from './uz-location-extensions.js';
|
|
7
9
|
|
|
@@ -237,12 +239,13 @@ function mapDataMatch(value, normalizedValue, item) {
|
|
|
237
239
|
return null;
|
|
238
240
|
}
|
|
239
241
|
|
|
240
|
-
function findEntryMatches(text, cityName, data, { includeMapData = false } = {}) {
|
|
242
|
+
function findEntryMatches(text, cityName, data, { includeMapData = false, grams = null } = {}) {
|
|
241
243
|
const value = String(text || '');
|
|
242
244
|
const normalizedValue = normalizeForMatch(value);
|
|
245
|
+
const textGrams = grams || computeTextGrams(value);
|
|
243
246
|
const raw = [];
|
|
244
247
|
for (const key of LOCATION_LIST_KEYS) {
|
|
245
|
-
for (const item of data?.[key] || []) {
|
|
248
|
+
for (const item of candidateEntries(data?.[key] || [], value, textGrams)) {
|
|
246
249
|
if (isMapDataEntry(item) && !includeMapData) continue;
|
|
247
250
|
const match = (isMapDataEntry(item) ? mapDataMatch(value, normalizedValue, item) : value.match(item?.re))
|
|
248
251
|
|| (key === 'residentialComplexes' ? markedResidentialMatch(value, item) : null);
|
|
@@ -329,23 +332,37 @@ export function matchCentralAsiaLocationEntities(text, countryCode, preferredCit
|
|
|
329
332
|
const preferred = canonicalCity(preferredCity, countryCode) || preferredCity;
|
|
330
333
|
const explicit = explicitCityFromText(text, countryCode);
|
|
331
334
|
const scopedCity = preferred && country[preferred] ? preferred : explicit && country[explicit] ? explicit : null;
|
|
335
|
+
const grams = computeTextGrams(text);
|
|
332
336
|
|
|
333
337
|
if (scopedCity) {
|
|
334
|
-
const matches = findEntryMatches(text, scopedCity, country[scopedCity], { includeMapData: true });
|
|
338
|
+
const matches = findEntryMatches(text, scopedCity, country[scopedCity], { includeMapData: true, grams });
|
|
335
339
|
const clusters = clusterMatches(matches, countryCode);
|
|
336
|
-
|
|
340
|
+
// The city is already decided here; the hypothesis is reported so callers
|
|
341
|
+
// get one shape either way, and can see the evidence behind the scope.
|
|
342
|
+
const scoped = scoreCityHypotheses([{ city: scopedCity, matches }], {
|
|
343
|
+
explicitCity: explicit, preferredCity: preferred, cityNames: Object.keys(country),
|
|
344
|
+
isAmbiguous: (match) => isAmbiguousMatch(match, countryCode),
|
|
345
|
+
});
|
|
346
|
+
return Object.freeze({ city: scopedCity, matches: Object.freeze(matches), searchClusters: Object.freeze(clusters), candidates: Object.freeze([]), hypotheses: scoped.hypotheses });
|
|
337
347
|
}
|
|
338
348
|
|
|
339
349
|
const byCity = [];
|
|
340
350
|
for (const [cityName, data] of Object.entries(country)) {
|
|
341
|
-
const matches = findEntryMatches(text, cityName, data);
|
|
351
|
+
const matches = findEntryMatches(text, cityName, data, { grams });
|
|
342
352
|
if (matches.length) byCity.push({ city: cityName, matches });
|
|
343
353
|
}
|
|
344
354
|
|
|
345
355
|
if (!byCity.length) {
|
|
346
|
-
return Object.freeze({ city: null, matches: Object.freeze([]), searchClusters: Object.freeze([]), candidates: Object.freeze([]) });
|
|
356
|
+
return Object.freeze({ city: null, matches: Object.freeze([]), searchClusters: Object.freeze([]), candidates: Object.freeze([]), hypotheses: Object.freeze([]) });
|
|
347
357
|
}
|
|
348
358
|
|
|
359
|
+
// Scored view of the same per-city matches. Additive: the selection rules
|
|
360
|
+
// below are unchanged, so every existing caller sees what it always saw.
|
|
361
|
+
const scored = scoreCityHypotheses(byCity, {
|
|
362
|
+
explicitCity: explicit, preferredCity: preferred, cityNames: Object.keys(country),
|
|
363
|
+
isAmbiguous: (match) => isAmbiguousMatch(match, countryCode),
|
|
364
|
+
});
|
|
365
|
+
|
|
349
366
|
// Numeric microdistricts and common names such as Samal/Center occur in many
|
|
350
367
|
// cities. Without an explicit/structured city we must not silently assign a
|
|
351
368
|
// parent. Prefer a city only when it owns at least one non-ambiguous match
|
|
@@ -363,11 +380,13 @@ export function matchCentralAsiaLocationEntities(text, countryCode, preferredCit
|
|
|
363
380
|
matches: Object.freeze([]),
|
|
364
381
|
searchClusters: Object.freeze([]),
|
|
365
382
|
candidates: Object.freeze(byCity.map((candidate) => Object.freeze({ city: candidate.city, matches: Object.freeze(candidate.matches) }))),
|
|
383
|
+
hypotheses: scored.hypotheses,
|
|
384
|
+
unresolvedReasons: scored.unresolvedReasons,
|
|
366
385
|
});
|
|
367
386
|
}
|
|
368
387
|
|
|
369
388
|
const clusters = clusterMatches(selected.matches, countryCode);
|
|
370
|
-
return Object.freeze({ city: selected.city, matches: Object.freeze(selected.matches), searchClusters: Object.freeze(clusters), candidates: Object.freeze([]) });
|
|
389
|
+
return Object.freeze({ city: selected.city, matches: Object.freeze(selected.matches), searchClusters: Object.freeze(clusters), candidates: Object.freeze([]), hypotheses: scored.hypotheses });
|
|
371
390
|
}
|
|
372
391
|
|
|
373
392
|
export function matchCentralAsiaLocationEntity(text, countryCode, preferredCity = null, type = null) {
|
|
Binary file
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
import type { CvSection } from './cv-sections.js';
|
|
2
|
+
|
|
3
|
+
/** Precision records what the text said. A bare "2019" is year precision; the
|
|
4
|
+
* month is absent rather than guessed. */
|
|
5
|
+
export type ParsedDate = Readonly<{ year: number; month?: number; precision: 'year' | 'month' }>;
|
|
6
|
+
|
|
7
|
+
export type EmploymentRange = Readonly<{
|
|
8
|
+
start: ParsedDate;
|
|
9
|
+
end: ParsedDate;
|
|
10
|
+
ongoing: boolean;
|
|
11
|
+
startIndex: number;
|
|
12
|
+
endIndex: number;
|
|
13
|
+
durationMonths: number;
|
|
14
|
+
raw: string;
|
|
15
|
+
range: Readonly<{ start: number; end: number }>;
|
|
16
|
+
}>;
|
|
17
|
+
|
|
18
|
+
export type EmploymentPeriod = Readonly<{
|
|
19
|
+
company?: string;
|
|
20
|
+
role?: string;
|
|
21
|
+
/** The CV section this entry was read from; undefined when the document had
|
|
22
|
+
* no headings and the whole text was scanned. */
|
|
23
|
+
section?: CvSection;
|
|
24
|
+
start: ParsedDate;
|
|
25
|
+
end: ParsedDate;
|
|
26
|
+
ongoing: boolean;
|
|
27
|
+
durationMonths: number;
|
|
28
|
+
sectionRange: Readonly<{ start: number; end: number }>;
|
|
29
|
+
skills: readonly string[];
|
|
30
|
+
startIndex: number;
|
|
31
|
+
endIndex: number;
|
|
32
|
+
}>;
|
|
33
|
+
|
|
34
|
+
export type MergedEmploymentPeriod = Readonly<{
|
|
35
|
+
startIndex: number;
|
|
36
|
+
endIndex: number;
|
|
37
|
+
ongoing: boolean;
|
|
38
|
+
durationMonths: number;
|
|
39
|
+
companies: readonly string[];
|
|
40
|
+
}>;
|
|
41
|
+
|
|
42
|
+
export type EmploymentOptions = { referenceDate?: Date; sections?: readonly CvSection[] };
|
|
43
|
+
|
|
44
|
+
export function parseEmploymentDate(value: unknown): ParsedDate | null;
|
|
45
|
+
export function findEmploymentRanges(value: unknown, options?: EmploymentOptions): readonly EmploymentRange[];
|
|
46
|
+
export function parseCvEmploymentPeriods(value: unknown, options?: EmploymentOptions): readonly EmploymentPeriod[];
|
|
47
|
+
export function mergeEmploymentPeriods(periods: readonly EmploymentPeriod[]): readonly MergedEmploymentPeriod[];
|
|
48
|
+
export function totalEmploymentMonths(periods: readonly EmploymentPeriod[]): number;
|
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
import { HIRING_MONTHS } from './hiring-temporal.js';
|
|
2
|
+
import { detectCvSections } from './cv-sections.js';
|
|
3
|
+
import { extractSkillNames } from './hiring-skills.js';
|
|
4
|
+
import { matchProfession } from './hiring-professions.js';
|
|
5
|
+
import { escapeRegex } from './normalization.js';
|
|
6
|
+
|
|
7
|
+
/** Structured employment chronology for CVs. This is not another date parser:
|
|
8
|
+
* month vocabulary comes from `hiring-temporal`, section spans from
|
|
9
|
+
* `cv-sections`, skills from `hiring-skills` and roles from
|
|
10
|
+
* `hiring-professions`. What is new here is pairing two dates into a period,
|
|
11
|
+
* attaching the employer and role that introduced it, and keeping the source
|
|
12
|
+
* range so a caller can point back at the text. */
|
|
13
|
+
|
|
14
|
+
const MONTH_SRC = Object.keys(HIRING_MONTHS).sort((a, b) => b.length - a.length).map(escapeRegex).join('|');
|
|
15
|
+
const YEAR_SRC = '(?:19|20)\\d{2}';
|
|
16
|
+
// Longest alternative first, and the trailing digit guard on DATE_SRC below:
|
|
17
|
+
// without both, "2021-12" matches as "2021-1" and silently loses 11 months.
|
|
18
|
+
const MONTH_NUMBER_SRC = '(?:1[0-2]|0?[1-9])';
|
|
19
|
+
const PRESENT_SRC = '(?:present|current|now|to date|ongoing|по\\s+настоящее\\s+время|настоящее\\s+время|наст\\.?\\s*время|н\\.\\s*в\\.|по\\s+сей\\s+день|сейчас|теперь|донині|до\\s+тепер|дотепер|теперішній\\s+час|зараз|hozirgi\\s+kunga\\s+qadar|hozirgacha|hozir|ҳозирги\\s+кунгача|ҳозиргача|ҳозир|қазірге\\s+дейін|қазір|осы\\s+күнге\\s+дейін|în\\s+prezent|in\\s+prezent|prezent|pân[ăa]\\s+în\\s+prezent)';
|
|
20
|
+
/** Written longest-first so "March 2019" is never truncated to "2019". */
|
|
21
|
+
const DATE_SRC = `(?:(?:${MONTH_SRC})\\.?\\s+${YEAR_SRC}|${YEAR_SRC}\\s*[-/.]\\s*${MONTH_NUMBER_SRC}|${MONTH_NUMBER_SRC}\\s*[/.]\\s*${YEAR_SRC}|${YEAR_SRC})(?!\\d)`;
|
|
22
|
+
const SEPARATOR_SRC = '(?:\\s*(?:[-–—]|\\.{2,})\\s*|\\s+(?:to|till|until|по|до|gacha|pân[ăa]|pana)\\s+)';
|
|
23
|
+
|
|
24
|
+
const RANGE_RE = new RegExp(`(${DATE_SRC})${SEPARATOR_SRC}(${DATE_SRC}|${PRESENT_SRC})`, 'giu');
|
|
25
|
+
const MONTH_YEAR_RE = new RegExp(`^(${MONTH_SRC})\\.?\\s+(${YEAR_SRC})$`, 'iu');
|
|
26
|
+
const YEAR_MONTH_RE = new RegExp(`^(${YEAR_SRC})\\s*[-/.]\\s*(${MONTH_NUMBER_SRC})$`, 'iu');
|
|
27
|
+
const MONTH_SLASH_YEAR_RE = new RegExp(`^(${MONTH_NUMBER_SRC})\\s*[/.]\\s*(${YEAR_SRC})$`, 'iu');
|
|
28
|
+
const YEAR_ONLY_RE = new RegExp(`^(${YEAR_SRC})$`, 'iu');
|
|
29
|
+
const PRESENT_RE = new RegExp(`^${PRESENT_SRC}$`, 'iu');
|
|
30
|
+
/** Employer and role sit on the same line as the dates, separated by commas,
|
|
31
|
+
* pipes, bullets or a spaced dash. "at"/"в" join a role to its employer. */
|
|
32
|
+
const ENTRY_SPLIT_RE = /\s*(?:[,|•·;]|\s[—–-]\s|\s+(?:at|@|in|в|у|da)\s+)\s*/iu;
|
|
33
|
+
const COMPANY_MARKER_RE = /\b(?:llc|ltd|inc|gmbh|corp|corporation|company|co|plc|ag|sa|srl|bv|oy|ab|as|llp|studio|labs?|group|holding|bank|university|ооо|зао|оао|пао|ип|тоо|ао|мчж|mchj|xk|аж|жшс)\b\.?|["«»“”]/iu;
|
|
34
|
+
|
|
35
|
+
const monthIndex = (year, month) => year * 12 + Math.max(1, Math.min(12, month)) - 1;
|
|
36
|
+
const parsedDate = (year, month, precision) => Object.freeze(month === undefined ? { year, precision } : { year, month, precision });
|
|
37
|
+
|
|
38
|
+
/** A ParsedDate records what the text actually said. "2019" is year precision;
|
|
39
|
+
* inventing a month for it would fabricate detail the CV never gave. */
|
|
40
|
+
export function parseEmploymentDate(value) {
|
|
41
|
+
const text = String(value ?? '').trim();
|
|
42
|
+
if (!text) return null;
|
|
43
|
+
let match = MONTH_YEAR_RE.exec(text);
|
|
44
|
+
if (match) return parsedDate(Number(match[2]), HIRING_MONTHS[match[1].toLowerCase()] + 1, 'month');
|
|
45
|
+
match = YEAR_MONTH_RE.exec(text);
|
|
46
|
+
if (match) return parsedDate(Number(match[1]), Number(match[2]), 'month');
|
|
47
|
+
match = MONTH_SLASH_YEAR_RE.exec(text);
|
|
48
|
+
if (match) return parsedDate(Number(match[2]), Number(match[1]), 'month');
|
|
49
|
+
match = YEAR_ONLY_RE.exec(text);
|
|
50
|
+
if (match) return parsedDate(Number(match[1]), undefined, 'year');
|
|
51
|
+
return null;
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
const startIndexOf = date => monthIndex(date.year, date.month ?? 1);
|
|
55
|
+
const endIndexOf = date => monthIndex(date.year, date.month ?? 12);
|
|
56
|
+
|
|
57
|
+
/** Date ranges with their offsets in `text`. Shared by the period builder and
|
|
58
|
+
* by the legacy total-years helper, so both read the same chronology. */
|
|
59
|
+
export function findEmploymentRanges(value, options = {}) {
|
|
60
|
+
const text = String(value ?? '');
|
|
61
|
+
const reference = options.referenceDate ?? new Date();
|
|
62
|
+
const ranges = [];
|
|
63
|
+
RANGE_RE.lastIndex = 0;
|
|
64
|
+
for (const match of text.matchAll(RANGE_RE)) {
|
|
65
|
+
const start = parseEmploymentDate(match[1]);
|
|
66
|
+
if (!start) continue;
|
|
67
|
+
const ongoing = PRESENT_RE.test(match[2].trim());
|
|
68
|
+
const end = ongoing ? parsedDate(reference.getUTCFullYear(), reference.getUTCMonth() + 1, 'month') : parseEmploymentDate(match[2]);
|
|
69
|
+
if (!end) continue;
|
|
70
|
+
const startIndex = startIndexOf(start);
|
|
71
|
+
const endIndex = ongoing ? monthIndex(end.year, end.month) : endIndexOf(end);
|
|
72
|
+
if (endIndex < startIndex || endIndex - startIndex > 12 * 50) continue;
|
|
73
|
+
ranges.push(Object.freeze({ start, end, ongoing, startIndex, endIndex, durationMonths: endIndex - startIndex + 1, raw: match[0], range: Object.freeze({ start: match.index, end: match.index + match[0].length }) }));
|
|
74
|
+
}
|
|
75
|
+
return Object.freeze(ranges);
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
function labelsFrom(line, rangeText) {
|
|
79
|
+
const remainder = line.replace(rangeText, ' ');
|
|
80
|
+
const segments = remainder.split(ENTRY_SPLIT_RE).map(part => part.replace(/^[\s\-–—:|()]+|[\s\-–—:|()]+$/gu, '')).filter(part => part.length > 1);
|
|
81
|
+
let role, company;
|
|
82
|
+
for (const segment of segments) {
|
|
83
|
+
const isRole = Boolean(matchProfession(segment));
|
|
84
|
+
if (isRole && role === undefined) { role = segment; continue; }
|
|
85
|
+
if (!isRole && company === undefined && COMPANY_MARKER_RE.test(segment)) { company = segment; continue; }
|
|
86
|
+
}
|
|
87
|
+
for (const segment of segments) {
|
|
88
|
+
if (segment === role || segment === company) continue;
|
|
89
|
+
if (company === undefined) company = segment;
|
|
90
|
+
else if (role === undefined) role = segment;
|
|
91
|
+
}
|
|
92
|
+
return { role, company };
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
/** One period per employment entry: the line carrying the dates plus the lines
|
|
96
|
+
* under it, which is where that entry's skills live. */
|
|
97
|
+
export function parseCvEmploymentPeriods(value, options = {}) {
|
|
98
|
+
const text = String(value ?? '');
|
|
99
|
+
const wanted = options.sections ?? ['experience', 'projects'];
|
|
100
|
+
const spans = detectCvSections(text).filter(span => wanted.includes(span.section));
|
|
101
|
+
const scopes = spans.length ? spans.map(span => ({ start: span.contentStart, end: span.end, section: span.section })) : [{ start: 0, end: text.length, section: undefined }];
|
|
102
|
+
const periods = [];
|
|
103
|
+
for (const scope of scopes) {
|
|
104
|
+
const body = text.slice(scope.start, scope.end);
|
|
105
|
+
const found = findEmploymentRanges(body, options);
|
|
106
|
+
if (!found.length) continue;
|
|
107
|
+
const lines = [];
|
|
108
|
+
let offset = 0;
|
|
109
|
+
for (const raw of body.split('\n')) { lines.push({ start: offset, end: offset + raw.replace(/\r$/, '').length, text: raw.trim() }); offset += raw.length + 1; }
|
|
110
|
+
const headers = found.map(range => ({ range, line: lines.find(line => range.range.start >= line.start && range.range.start <= line.end) ?? lines[0] }));
|
|
111
|
+
headers.forEach((header, index) => {
|
|
112
|
+
const next = headers.slice(index + 1).find(other => other.line.start > header.line.start);
|
|
113
|
+
const entryEnd = next ? next.line.start : scope.end - scope.start;
|
|
114
|
+
const { role, company } = labelsFrom(header.line.text, header.range.raw);
|
|
115
|
+
periods.push(Object.freeze({
|
|
116
|
+
company, role, section: scope.section,
|
|
117
|
+
start: header.range.start, end: header.range.end, ongoing: header.range.ongoing,
|
|
118
|
+
durationMonths: header.range.durationMonths,
|
|
119
|
+
sectionRange: Object.freeze({ start: scope.start + header.line.start, end: scope.start + entryEnd }),
|
|
120
|
+
skills: Object.freeze([...new Set(extractSkillNames(body.slice(header.line.start, entryEnd)))]),
|
|
121
|
+
startIndex: header.range.startIndex, endIndex: header.range.endIndex,
|
|
122
|
+
}));
|
|
123
|
+
});
|
|
124
|
+
}
|
|
125
|
+
return Object.freeze(periods);
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
/** Overlapping or adjacent periods collapse into one interval. Concurrent jobs
|
|
129
|
+
* must not be counted twice, so this merges across employers and keeps every
|
|
130
|
+
* contributing company name. */
|
|
131
|
+
export function mergeEmploymentPeriods(periods) {
|
|
132
|
+
const ordered = [...periods].sort((a, b) => a.startIndex - b.startIndex || a.endIndex - b.endIndex);
|
|
133
|
+
const merged = [];
|
|
134
|
+
for (const period of ordered) {
|
|
135
|
+
const last = merged[merged.length - 1];
|
|
136
|
+
if (last && period.startIndex <= last.endIndex + 1) {
|
|
137
|
+
last.endIndex = Math.max(last.endIndex, period.endIndex);
|
|
138
|
+
last.ongoing ||= period.ongoing;
|
|
139
|
+
if (period.company) last.companies.add(period.company);
|
|
140
|
+
continue;
|
|
141
|
+
}
|
|
142
|
+
merged.push({ startIndex: period.startIndex, endIndex: period.endIndex, ongoing: period.ongoing, companies: new Set(period.company ? [period.company] : []) });
|
|
143
|
+
}
|
|
144
|
+
return Object.freeze(merged.map(entry => Object.freeze({
|
|
145
|
+
startIndex: entry.startIndex, endIndex: entry.endIndex, ongoing: entry.ongoing,
|
|
146
|
+
durationMonths: entry.endIndex - entry.startIndex + 1,
|
|
147
|
+
companies: Object.freeze([...entry.companies]),
|
|
148
|
+
})));
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
export function totalEmploymentMonths(periods) {
|
|
152
|
+
return mergeEmploymentPeriods(periods).reduce((sum, entry) => sum + entry.durationMonths, 0);
|
|
153
|
+
}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
export type CvSection = 'profile' | 'experience' | 'projects' | 'skills' | 'education' | 'languages' | 'certifications' | 'contact' | 'additional';
|
|
2
|
+
export const CV_SECTIONS: readonly CvSection[];
|
|
3
|
+
|
|
4
|
+
export type CvSectionSpan = Readonly<{
|
|
5
|
+
section: CvSection | 'preamble';
|
|
6
|
+
heading: string | null;
|
|
7
|
+
headingRange: Readonly<{ start: number; end: number }> | null;
|
|
8
|
+
start: number;
|
|
9
|
+
contentStart: number;
|
|
10
|
+
end: number;
|
|
11
|
+
}>;
|
|
12
|
+
|
|
13
|
+
export function classifyCvSectionHeading(value: unknown): CvSection | null;
|
|
14
|
+
export function detectCvSections(value: unknown): readonly CvSectionSpan[];
|
|
15
|
+
export function extractCvSectionText(value: unknown, wanted: CvSection): string;
|