@stll/anonymize-wasm 1.1.1 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/address-boundaries.mjs +113 -5
- package/dist/address-stop-keywords.mjs +137 -0
- package/dist/address-stop-keywords.mjs.map +1 -0
- package/dist/address-stopwords.mjs +81 -0
- package/dist/address-stopwords.mjs.map +1 -0
- package/dist/address-street-types.mjs +67 -4
- package/dist/clause-noun-heads.mjs +75 -0
- package/dist/clause-noun-heads.mjs.map +1 -0
- package/dist/coreference.es.mjs +22 -0
- package/dist/coreference.es.mjs.map +1 -0
- package/dist/coreference.fr.mjs +32 -0
- package/dist/coreference.fr.mjs.map +1 -0
- package/dist/coreference.it.mjs +27 -0
- package/dist/coreference.it.mjs.map +1 -0
- package/dist/coreference.pl.mjs +27 -0
- package/dist/coreference.pl.mjs.map +1 -0
- package/dist/coreference.pt-br.mjs +14 -0
- package/dist/coreference.pt-br.mjs.map +1 -0
- package/dist/coreference.sk.mjs +22 -5
- package/dist/generic-roles.mjs +179 -1
- package/dist/hotword-rules.mjs +7 -3
- package/dist/legal-forms.mjs +3 -0
- package/dist/legal-role-heads.cs.mjs +42 -0
- package/dist/legal-role-heads.cs.mjs.map +1 -0
- package/dist/legal-role-heads.de.mjs +33 -0
- package/dist/legal-role-heads.de.mjs.map +1 -0
- package/dist/legal-role-heads.en.mjs +37 -0
- package/dist/legal-role-heads.en.mjs.map +1 -0
- package/dist/legal-role-heads.es.mjs +54 -0
- package/dist/legal-role-heads.es.mjs.map +1 -0
- package/dist/legal-role-heads.fr.mjs +72 -0
- package/dist/legal-role-heads.fr.mjs.map +1 -0
- package/dist/legal-role-heads.it.mjs +68 -0
- package/dist/legal-role-heads.it.mjs.map +1 -0
- package/dist/legal-role-heads.pl.mjs +84 -0
- package/dist/legal-role-heads.pl.mjs.map +1 -0
- package/dist/legal-role-heads.pt-br.mjs +63 -0
- package/dist/legal-role-heads.pt-br.mjs.map +1 -0
- package/dist/legal-role-heads.sk.mjs +80 -0
- package/dist/legal-role-heads.sk.mjs.map +1 -0
- package/dist/manifest.mjs +21 -8
- package/dist/names-exclusions.mjs +21 -1
- package/dist/person-stopwords.mjs +121 -1
- package/dist/section-headings.mjs +15 -3
- package/dist/sentence-verb-indicators.mjs +223 -0
- package/dist/sentence-verb-indicators.mjs.map +1 -0
- package/dist/triggers.cs.mjs +107 -3
- package/dist/triggers.es.mjs +43 -4
- package/dist/triggers.fr.mjs +206 -7
- package/dist/triggers.it.mjs +26 -4
- package/dist/triggers.pl.mjs +191 -4
- package/dist/triggers.pt-br.mjs +193 -0
- package/dist/triggers.pt-br.mjs.map +1 -0
- package/dist/triggers.sk.mjs +493 -0
- package/dist/vite.mjs.map +1 -1
- package/dist/wasm.d.mts +100 -8
- package/dist/wasm.mjs +2520 -1360
- package/dist/wasm.mjs.map +1 -1
- package/package.json +2 -2
package/dist/wasm.mjs
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { TextSearch } from "@stll/text-search-wasm";
|
|
2
|
-
import { at, be, bg, cy, cz, de, dk, ee, es, fi, fr, gb, gr, hr, hu, ie, it, lt, lu, lv, mt, nl, pl, pt, ro, se, si, sk } from "@stll/stdnum";
|
|
2
|
+
import { at, be, bg, br, cy, cz, de, dk, ee, es, fi, fr, gb, gr, hr, hu, ie, it, lt, lu, lv, mt, nl, pl, pt, ro, se, si, sk } from "@stll/stdnum";
|
|
3
3
|
import { toRegex } from "@stll/stdnum/patterns";
|
|
4
4
|
//#region src/search-engine.ts
|
|
5
5
|
let _TextSearch;
|
|
@@ -66,11 +66,15 @@ const DEFAULT_ENTITY_LABELS = [
|
|
|
66
66
|
"iban",
|
|
67
67
|
"tax identification number",
|
|
68
68
|
"identity card number",
|
|
69
|
+
"birth number",
|
|
70
|
+
"national identification number",
|
|
71
|
+
"social security number",
|
|
69
72
|
"registration number",
|
|
70
73
|
"credit card number",
|
|
71
74
|
"passport number",
|
|
72
75
|
"monetary amount",
|
|
73
|
-
"land parcel"
|
|
76
|
+
"land parcel",
|
|
77
|
+
"misc"
|
|
74
78
|
];
|
|
75
79
|
const isLegalFormsEnabled = (config) => config.enableLegalForms !== false;
|
|
76
80
|
//#endregion
|
|
@@ -95,6 +99,8 @@ const createPipelineContext = () => ({
|
|
|
95
99
|
allowListPromise: null,
|
|
96
100
|
personStopwords: null,
|
|
97
101
|
personStopwordsPromise: null,
|
|
102
|
+
addressStopwords: null,
|
|
103
|
+
addressStopwordsPromise: null,
|
|
98
104
|
firstNameExclusions: null,
|
|
99
105
|
firstNameExclusionCorpusLen: 0,
|
|
100
106
|
genericRoles: null,
|
|
@@ -151,6 +157,7 @@ const LOADER_REGISTRIES = {
|
|
|
151
157
|
hu: () => import("./triggers.hu.mjs"),
|
|
152
158
|
it: () => import("./triggers.it.mjs"),
|
|
153
159
|
pl: () => import("./triggers.pl.mjs"),
|
|
160
|
+
"pt-br": () => import("./triggers.pt-br.mjs"),
|
|
154
161
|
ro: () => import("./triggers.ro.mjs"),
|
|
155
162
|
sk: () => import("./triggers.sk.mjs"),
|
|
156
163
|
sv: () => import("./triggers.sv.mjs")
|
|
@@ -159,7 +166,23 @@ const LOADER_REGISTRIES = {
|
|
|
159
166
|
cs: () => import("./coreference.cs.mjs"),
|
|
160
167
|
de: () => import("./coreference.de.mjs"),
|
|
161
168
|
en: () => import("./coreference.en.mjs"),
|
|
169
|
+
es: () => import("./coreference.es.mjs"),
|
|
170
|
+
fr: () => import("./coreference.fr.mjs"),
|
|
171
|
+
it: () => import("./coreference.it.mjs"),
|
|
172
|
+
pl: () => import("./coreference.pl.mjs"),
|
|
173
|
+
"pt-br": () => import("./coreference.pt-br.mjs"),
|
|
162
174
|
sk: () => import("./coreference.sk.mjs")
|
|
175
|
+
},
|
|
176
|
+
legalRoleHeads: {
|
|
177
|
+
cs: () => import("./legal-role-heads.cs.mjs"),
|
|
178
|
+
de: () => import("./legal-role-heads.de.mjs"),
|
|
179
|
+
en: () => import("./legal-role-heads.en.mjs"),
|
|
180
|
+
es: () => import("./legal-role-heads.es.mjs"),
|
|
181
|
+
fr: () => import("./legal-role-heads.fr.mjs"),
|
|
182
|
+
it: () => import("./legal-role-heads.it.mjs"),
|
|
183
|
+
pl: () => import("./legal-role-heads.pl.mjs"),
|
|
184
|
+
"pt-br": () => import("./legal-role-heads.pt-br.mjs"),
|
|
185
|
+
sk: () => import("./legal-role-heads.sk.mjs")
|
|
163
186
|
}
|
|
164
187
|
};
|
|
165
188
|
const FALLBACK_LANGUAGES = {
|
|
@@ -172,6 +195,7 @@ const FALLBACK_LANGUAGES = {
|
|
|
172
195
|
"hu",
|
|
173
196
|
"it",
|
|
174
197
|
"pl",
|
|
198
|
+
"pt-br",
|
|
175
199
|
"ro",
|
|
176
200
|
"sk",
|
|
177
201
|
"sv"
|
|
@@ -180,6 +204,22 @@ const FALLBACK_LANGUAGES = {
|
|
|
180
204
|
"cs",
|
|
181
205
|
"de",
|
|
182
206
|
"en",
|
|
207
|
+
"es",
|
|
208
|
+
"fr",
|
|
209
|
+
"it",
|
|
210
|
+
"pl",
|
|
211
|
+
"pt-br",
|
|
212
|
+
"sk"
|
|
213
|
+
],
|
|
214
|
+
legalRoleHeads: [
|
|
215
|
+
"cs",
|
|
216
|
+
"de",
|
|
217
|
+
"en",
|
|
218
|
+
"es",
|
|
219
|
+
"fr",
|
|
220
|
+
"it",
|
|
221
|
+
"pl",
|
|
222
|
+
"pt-br",
|
|
183
223
|
"sk"
|
|
184
224
|
]
|
|
185
225
|
};
|
|
@@ -226,838 +266,64 @@ const loadLanguageConfigs = async (configType, mapFn) => {
|
|
|
226
266
|
await Promise.all(loads);
|
|
227
267
|
return results.filter((r) => r !== void 0);
|
|
228
268
|
};
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
console.warn("[anonymize] coreference: no definition patterns loaded; coreference detection will be inactive");
|
|
288
|
-
}
|
|
289
|
-
return patterns;
|
|
290
|
-
}
|
|
291
|
-
ctx.corefPatterns = patterns;
|
|
292
|
-
return patterns;
|
|
293
|
-
};
|
|
294
|
-
const SEARCH_WINDOW = 200;
|
|
295
|
-
/**
|
|
296
|
-
* Check whether an alias has textual similarity to
|
|
297
|
-
* the source entity. Prevents roles and structural
|
|
298
|
-
* terms from being treated as name aliases.
|
|
299
|
-
*
|
|
300
|
-
* Three checks (any passes → similar):
|
|
301
|
-
* 1. Word overlap: a word in the alias appears in the
|
|
302
|
-
* entity (case-insensitive, min 2 chars)
|
|
303
|
-
* 2. Initials: alias letters match first letters of
|
|
304
|
-
* entity words ("TB" ↔ "Tomas Bata")
|
|
305
|
-
* 3. Substring: alias is a substring of the entity
|
|
306
|
-
* or vice versa (min 3 chars)
|
|
307
|
-
*/
|
|
308
|
-
const hasEntitySimilarity = (alias, entityText) => {
|
|
309
|
-
const aliasLower = alias.toLowerCase();
|
|
310
|
-
const entityLower = entityText.toLowerCase();
|
|
311
|
-
if (aliasLower.length >= 3 && entityLower.includes(aliasLower)) return true;
|
|
312
|
-
if (entityLower.length >= 3 && aliasLower.includes(entityLower)) return true;
|
|
313
|
-
const aliasWords = aliasLower.split(/[\s.,;:'"()/-]+/).filter((w) => w.length >= 2);
|
|
314
|
-
const entityWords = entityLower.split(/[\s.,;:'"()/-]+/).filter((w) => w.length >= 2);
|
|
315
|
-
const entityWordSet = new Set(entityWords);
|
|
316
|
-
for (const word of aliasWords) if (entityWordSet.has(word)) return true;
|
|
317
|
-
if (/^[\p{Lu}]+$/u.test(alias) && alias.length >= 2 && alias.length <= entityWords.length) {
|
|
318
|
-
for (let start = 0; start <= entityWords.length - alias.length; start++) if (entityWords.slice(start, start + alias.length).map((w) => w.charAt(0)).join("") === aliasLower) return true;
|
|
319
|
-
}
|
|
320
|
-
return false;
|
|
321
|
-
};
|
|
322
|
-
/**
|
|
323
|
-
* Scan for defined-term patterns near known entities.
|
|
324
|
-
*
|
|
325
|
-
* Legal documents universally follow:
|
|
326
|
-
* "Dr. Heinrich Muller (hereinafter 'the Seller')..."
|
|
327
|
-
*
|
|
328
|
-
* After NER detects the entity, this function scans for
|
|
329
|
-
* definitional patterns within +/-200 chars and extracts
|
|
330
|
-
* the alias. Returns alias + label pairs that can be added
|
|
331
|
-
* to the gazetteer for a full-text re-scan.
|
|
332
|
-
*/
|
|
333
|
-
/**
|
|
334
|
-
* Labels that can be the source of a coreference alias.
|
|
335
|
-
* Only parties (person, organization) have defined-term
|
|
336
|
-
* aliases in legal text. Dates, addresses, IDs do not.
|
|
337
|
-
*/
|
|
338
|
-
const COREF_SOURCE_LABELS = new Set(["person", "organization"]);
|
|
339
|
-
const extractDefinedTerms = async (fullText, entities, ctx = defaultContext) => {
|
|
340
|
-
const [definitionPatterns, roleStops] = await Promise.all([getDefinitionPatterns(ctx), getRoleStopSet(ctx)]);
|
|
341
|
-
const terms = [];
|
|
342
|
-
const seen = /* @__PURE__ */ new Set();
|
|
343
|
-
const sorted = [...entities].sort((a, b) => a.start - b.start);
|
|
344
|
-
for (const { pattern } of definitionPatterns) {
|
|
345
|
-
pattern.lastIndex = 0;
|
|
346
|
-
for (let match = pattern.exec(fullText); match !== null; match = pattern.exec(fullText)) {
|
|
347
|
-
const alias = match[1]?.trim();
|
|
348
|
-
if (!alias || alias.length < 2) continue;
|
|
349
|
-
if (roleStops.has(alias.toLowerCase())) continue;
|
|
350
|
-
const defPos = match.index;
|
|
351
|
-
let bestEntity = null;
|
|
352
|
-
for (let i = sorted.length - 1; i >= 0; i--) {
|
|
353
|
-
const e = sorted[i];
|
|
354
|
-
if (e === void 0) continue;
|
|
355
|
-
if (e.end > defPos) continue;
|
|
356
|
-
if (defPos - e.end > SEARCH_WINDOW) break;
|
|
357
|
-
if (!COREF_SOURCE_LABELS.has(e.label)) continue;
|
|
358
|
-
bestEntity = e;
|
|
359
|
-
break;
|
|
360
|
-
}
|
|
361
|
-
if (bestEntity === null) continue;
|
|
362
|
-
const gapText = fullText.slice(bestEntity.end, defPos);
|
|
363
|
-
if (/(?:;|\.(?=\s*(?:["'„‚(]*\p{Lu}|$)))/u.test(gapText)) continue;
|
|
364
|
-
if (!hasEntitySimilarity(alias, bestEntity.text)) continue;
|
|
365
|
-
const key = `${alias.toLowerCase()}::${bestEntity.label}`;
|
|
366
|
-
if (seen.has(key)) continue;
|
|
367
|
-
seen.add(key);
|
|
368
|
-
terms.push({
|
|
369
|
-
alias,
|
|
370
|
-
label: bestEntity.label,
|
|
371
|
-
definitionStart: defPos,
|
|
372
|
-
sourceText: bestEntity.text
|
|
373
|
-
});
|
|
374
|
-
}
|
|
375
|
-
}
|
|
376
|
-
return terms;
|
|
377
|
-
};
|
|
378
|
-
/**
|
|
379
|
-
* Check if a character is a Unicode word character
|
|
380
|
-
* (letter, digit, or combining mark). Used for word
|
|
381
|
-
* boundary checks in coreference matching.
|
|
382
|
-
*/
|
|
383
|
-
const isWordChar = (ch) => {
|
|
384
|
-
if (ch === void 0) return false;
|
|
385
|
-
return /[\p{L}\p{M}\p{N}]/u.test(ch);
|
|
386
|
-
};
|
|
387
|
-
/**
|
|
388
|
-
* Find all occurrences of defined-term aliases in the
|
|
389
|
-
* full text. Returns Entity spans for each match.
|
|
390
|
-
*
|
|
391
|
-
* Respects word boundaries: "Kupující" must not match
|
|
392
|
-
* inside "Kupujícímu". A match is valid only if the
|
|
393
|
-
* character before the start and after the end are NOT
|
|
394
|
-
* word characters (letter/digit).
|
|
395
|
-
*
|
|
396
|
-
* Populates `ctx.corefSourceMap` with entries linking
|
|
397
|
-
* each coref entity to its source entity text, for
|
|
398
|
-
* consistent placeholder numbering.
|
|
399
|
-
*/
|
|
400
|
-
const findCoreferenceSpans = (fullText, terms, ctx = defaultContext) => {
|
|
401
|
-
const results = [];
|
|
402
|
-
for (const term of terms) {
|
|
403
|
-
let searchFrom = 0;
|
|
404
|
-
while (searchFrom < fullText.length) {
|
|
405
|
-
const idx = fullText.indexOf(term.alias, searchFrom);
|
|
406
|
-
if (idx === -1) break;
|
|
407
|
-
const matchEnd = idx + term.alias.length;
|
|
408
|
-
const charBefore = idx > 0 ? fullText[idx - 1] : void 0;
|
|
409
|
-
const charAfter = fullText[matchEnd];
|
|
410
|
-
if (isWordChar(charBefore) || isWordChar(charAfter)) {
|
|
411
|
-
searchFrom = idx + 1;
|
|
412
|
-
continue;
|
|
413
|
-
}
|
|
414
|
-
const entity = {
|
|
415
|
-
start: idx,
|
|
416
|
-
end: matchEnd,
|
|
417
|
-
label: term.label,
|
|
418
|
-
text: term.alias,
|
|
419
|
-
score: .95,
|
|
420
|
-
source: DETECTION_SOURCES.COREFERENCE
|
|
421
|
-
};
|
|
422
|
-
ctx.corefSourceMap.set(corefKey(entity), term.sourceText);
|
|
423
|
-
results.push(entity);
|
|
424
|
-
searchFrom = matchEnd;
|
|
425
|
-
}
|
|
426
|
-
}
|
|
427
|
-
return results;
|
|
428
|
-
};
|
|
429
|
-
//#endregion
|
|
430
|
-
//#region src/detectors/gazetteer.ts
|
|
431
|
-
const MAX_EDIT_DISTANCE = 2;
|
|
432
|
-
const MIN_FUZZY_LENGTH = 4;
|
|
433
|
-
const MAX_PREFIX_OVERSHOOT = 7;
|
|
434
|
-
/**
|
|
435
|
-
* Collect all searchable strings (canonical + variants)
|
|
436
|
-
* from gazetteer entries, mapped to their labels and
|
|
437
|
-
* entry IDs.
|
|
438
|
-
*/
|
|
439
|
-
const buildSearchTerms = (entries) => {
|
|
440
|
-
const terms = /* @__PURE__ */ new Map();
|
|
441
|
-
for (const entry of entries) {
|
|
442
|
-
const meta = {
|
|
443
|
-
label: entry.label,
|
|
444
|
-
entryId: entry.id
|
|
445
|
-
};
|
|
446
|
-
terms.set(entry.canonical, meta);
|
|
447
|
-
for (const variant of entry.variants) terms.set(variant, meta);
|
|
448
|
-
}
|
|
449
|
-
return terms;
|
|
450
|
-
};
|
|
451
|
-
/**
|
|
452
|
-
* Build TextSearch-compatible patterns from gazetteer
|
|
453
|
-
* entries. Returns:
|
|
454
|
-
* - Exact literal patterns for all terms
|
|
455
|
-
* - Fuzzy patterns (distance: 2) for terms >= 4 chars
|
|
456
|
-
* - Parallel metadata arrays for post-processing
|
|
457
|
-
*
|
|
458
|
-
* Patterns are ordered: all exact first, then all
|
|
459
|
-
* fuzzy. The isFuzzy array marks which are which.
|
|
460
|
-
*/
|
|
461
|
-
const buildGazetteerPatterns = (entries) => {
|
|
462
|
-
const terms = buildSearchTerms(entries);
|
|
463
|
-
const patterns = [];
|
|
464
|
-
const labels = [];
|
|
465
|
-
const isFuzzy = [];
|
|
466
|
-
for (const [term, meta] of terms) {
|
|
467
|
-
patterns.push({
|
|
468
|
-
pattern: term,
|
|
469
|
-
literal: true,
|
|
470
|
-
wholeWords: false
|
|
471
|
-
});
|
|
472
|
-
labels.push(meta.label);
|
|
473
|
-
isFuzzy.push(false);
|
|
474
|
-
}
|
|
475
|
-
for (const [term, meta] of terms) {
|
|
476
|
-
if (term.length < MIN_FUZZY_LENGTH) continue;
|
|
477
|
-
patterns.push({
|
|
478
|
-
pattern: term,
|
|
479
|
-
distance: MAX_EDIT_DISTANCE
|
|
480
|
-
});
|
|
481
|
-
labels.push(meta.label);
|
|
482
|
-
isFuzzy.push(true);
|
|
483
|
-
}
|
|
484
|
-
return {
|
|
485
|
-
patterns,
|
|
486
|
-
data: {
|
|
487
|
-
labels,
|
|
488
|
-
isFuzzy
|
|
489
|
-
}
|
|
490
|
-
};
|
|
491
|
-
};
|
|
492
|
-
/**
|
|
493
|
-
* Process gazetteer matches from the unified literal
|
|
494
|
-
* search. Receives all matches; filters to the
|
|
495
|
-
* gazetteer slice via sliceStart/sliceEnd.
|
|
496
|
-
*
|
|
497
|
-
* Exact matches get score 0.9; fuzzy matches get
|
|
498
|
-
* 0.85. Fuzzy matches that overlap an exact match
|
|
499
|
-
* are dropped.
|
|
500
|
-
*
|
|
501
|
-
* For exact matches, attempts prefix extension for
|
|
502
|
-
* legal suffixes ("a.s.", "GmbH", "s.r.o." after
|
|
503
|
-
* the matched term).
|
|
504
|
-
*/
|
|
505
|
-
const processGazetteerMatches = (allMatches, sliceStart, sliceEnd, fullText, data) => {
|
|
506
|
-
const results = [];
|
|
507
|
-
const exactSpans = [];
|
|
508
|
-
for (const match of allMatches) {
|
|
509
|
-
const idx = match.pattern;
|
|
510
|
-
if (idx < sliceStart || idx >= sliceEnd) continue;
|
|
511
|
-
const localIdx = idx - sliceStart;
|
|
512
|
-
if (data.isFuzzy[localIdx]) continue;
|
|
513
|
-
const label = data.labels[localIdx];
|
|
514
|
-
if (!label) continue;
|
|
515
|
-
const extended = tryPrefixExtension(fullText, match.start, match.end);
|
|
516
|
-
const end = extended?.end ?? match.end;
|
|
517
|
-
const text = extended?.text ?? fullText.slice(match.start, match.end);
|
|
518
|
-
exactSpans.push({
|
|
519
|
-
start: match.start,
|
|
520
|
-
end
|
|
521
|
-
});
|
|
522
|
-
results.push({
|
|
523
|
-
start: match.start,
|
|
524
|
-
end,
|
|
525
|
-
label,
|
|
526
|
-
text,
|
|
527
|
-
score: .9,
|
|
528
|
-
source: DETECTION_SOURCES.GAZETTEER
|
|
529
|
-
});
|
|
530
|
-
}
|
|
531
|
-
for (const match of allMatches) {
|
|
532
|
-
const idx = match.pattern;
|
|
533
|
-
if (idx < sliceStart || idx >= sliceEnd) continue;
|
|
534
|
-
const localIdx = idx - sliceStart;
|
|
535
|
-
if (!data.isFuzzy[localIdx]) continue;
|
|
536
|
-
if (match.distance === 0) continue;
|
|
537
|
-
const label = data.labels[localIdx];
|
|
538
|
-
if (!label) continue;
|
|
539
|
-
if (exactSpans.some((e) => match.start < e.end && match.end > e.start)) continue;
|
|
540
|
-
const matchText = fullText.slice(match.start, match.end);
|
|
541
|
-
results.push({
|
|
542
|
-
start: match.start,
|
|
543
|
-
end: match.end,
|
|
544
|
-
label,
|
|
545
|
-
text: matchText,
|
|
546
|
-
score: .85,
|
|
547
|
-
source: DETECTION_SOURCES.GAZETTEER
|
|
548
|
-
});
|
|
549
|
-
}
|
|
550
|
-
return results;
|
|
551
|
-
};
|
|
552
|
-
/**
|
|
553
|
-
* Try to extend an exact match to capture one
|
|
554
|
-
* trailing token (max 6 chars) that may be a legal
|
|
555
|
-
* entity suffix (e.g., "a.s.", "GmbH", "s.r.o.").
|
|
556
|
-
*
|
|
557
|
-
* Does not validate the token against a legal-forms
|
|
558
|
-
* list; false extensions are filtered by mergeAndDedup
|
|
559
|
-
* when a legal-form detector produces a competing
|
|
560
|
-
* entity with the correct span.
|
|
561
|
-
*/
|
|
562
|
-
const tryPrefixExtension = (fullText, start, end) => {
|
|
563
|
-
const maxEnd = Math.min(end + MAX_PREFIX_OVERSHOOT, fullText.length);
|
|
564
|
-
if (maxEnd <= end + 1) return null;
|
|
565
|
-
const after = fullText.slice(end, maxEnd);
|
|
566
|
-
if (!after.startsWith(" ")) return null;
|
|
567
|
-
const nextSpace = after.indexOf(" ", 1);
|
|
568
|
-
const suffixEnd = nextSpace !== -1 ? nextSpace : after.length;
|
|
569
|
-
if (suffixEnd <= 1) return null;
|
|
570
|
-
const newEnd = end + suffixEnd;
|
|
571
|
-
return {
|
|
572
|
-
end: newEnd,
|
|
573
|
-
text: fullText.slice(start, newEnd)
|
|
574
|
-
};
|
|
575
|
-
};
|
|
576
|
-
//#endregion
|
|
577
|
-
//#region src/util/text.ts
|
|
578
|
-
/**
|
|
579
|
-
* Shared text utilities for detectors.
|
|
580
|
-
*
|
|
581
|
-
* Extracted from names.ts and deny-list.ts to avoid
|
|
582
|
-
* duplicating regex constants and helper functions.
|
|
583
|
-
*/
|
|
584
|
-
/** Matches a string that starts with an uppercase letter. */
|
|
585
|
-
const UPPER_START_RE = /^\p{Lu}/u;
|
|
586
|
-
/** Matches a string consisting entirely of uppercase letters. */
|
|
587
|
-
const ALL_UPPER_RE = /^\p{Lu}+$/u;
|
|
588
|
-
const SENTENCE_END_RE = /[.!?]/;
|
|
589
|
-
/**
|
|
590
|
-
* Detect whether a position is at the start of a sentence.
|
|
591
|
-
* Looks backward past whitespace for sentence-ending
|
|
592
|
-
* punctuation (.!?). Position 0 and positions preceded
|
|
593
|
-
* only by whitespace are considered sentence starts.
|
|
594
|
-
*/
|
|
595
|
-
const isSentenceStart = (text, pos) => {
|
|
596
|
-
if (pos === 0) return true;
|
|
597
|
-
let i = pos - 1;
|
|
598
|
-
while (i >= 0 && /\s/.test(text[i] ?? "")) i--;
|
|
599
|
-
if (i < 0) return true;
|
|
600
|
-
return SENTENCE_END_RE.test(text[i] ?? "");
|
|
601
|
-
};
|
|
602
|
-
//#endregion
|
|
603
|
-
//#region src/detectors/names.ts
|
|
604
|
-
const getCorpus = (ctx) => ctx.nameCorpus;
|
|
605
|
-
const getNameCorpusFirstNames = (ctx = defaultContext) => ctx.nameCorpus?.firstNamesList ?? [];
|
|
606
|
-
const getNameCorpusSurnames = (ctx = defaultContext) => ctx.nameCorpus?.surnamesList ?? [];
|
|
607
|
-
const getNameCorpusTitles = (ctx = defaultContext) => ctx.nameCorpus?.titlesList ?? [];
|
|
608
|
-
/**
|
|
609
|
-
* Load name corpus data from injected dictionaries
|
|
610
|
-
* and legacy config files. Merges all sources.
|
|
611
|
-
*
|
|
612
|
-
* Safe to call multiple times; only loads once per
|
|
613
|
-
* context. Must be called before detectNameCorpus or
|
|
614
|
-
* the getNameCorpus*() accessors are used.
|
|
615
|
-
*
|
|
616
|
-
* @param dictionaries Optional pre-loaded dictionaries
|
|
617
|
-
* with per-language first names and surnames. When
|
|
618
|
-
* omitted, only legacy config files are used.
|
|
619
|
-
*/
|
|
620
|
-
const initNameCorpus = (ctx = defaultContext, dictionaries) => {
|
|
621
|
-
if (ctx.nameCorpusPromise) return ctx.nameCorpusPromise;
|
|
622
|
-
const promise = (async () => {
|
|
623
|
-
try {
|
|
624
|
-
const [legacyFirstMod, legacySurnameMod, titleMod, exclusionMod] = await Promise.all([
|
|
625
|
-
import("./names-first.mjs"),
|
|
626
|
-
import("./names-surnames.mjs"),
|
|
627
|
-
import("./names-title-tokens.mjs"),
|
|
628
|
-
import("./names-exclusions.mjs")
|
|
629
|
-
]);
|
|
630
|
-
const firstNames = [...legacyFirstMod.default.names];
|
|
631
|
-
if (dictionaries?.firstNames) for (const names of Object.values(dictionaries.firstNames)) for (const name of names) firstNames.push(name);
|
|
632
|
-
const surnames = [...legacySurnameMod.default.names];
|
|
633
|
-
if (dictionaries?.surnames) for (const names of Object.values(dictionaries.surnames)) for (const name of names) surnames.push(name);
|
|
634
|
-
const dedup = (arr) => {
|
|
635
|
-
const seen = /* @__PURE__ */ new Set();
|
|
636
|
-
const result = [];
|
|
637
|
-
for (const item of arr) {
|
|
638
|
-
if (seen.has(item)) continue;
|
|
639
|
-
seen.add(item);
|
|
640
|
-
result.push(item);
|
|
641
|
-
}
|
|
642
|
-
return result;
|
|
643
|
-
};
|
|
644
|
-
const dedupFirst = dedup(firstNames);
|
|
645
|
-
const dedupSurnames = dedup(surnames);
|
|
646
|
-
const titles = titleMod.default.tokens;
|
|
647
|
-
const exclusions = exclusionMod.default.words;
|
|
648
|
-
ctx.nameCorpus = {
|
|
649
|
-
firstNames: Object.freeze(new Set(dedupFirst)),
|
|
650
|
-
surnames: Object.freeze(new Set(dedupSurnames)),
|
|
651
|
-
titleTokens: Object.freeze(new Set(titles)),
|
|
652
|
-
excludedWords: Object.freeze(new Set(exclusions)),
|
|
653
|
-
firstNamesList: Object.freeze(dedupFirst),
|
|
654
|
-
surnamesList: Object.freeze(dedupSurnames),
|
|
655
|
-
titlesList: Object.freeze(titles),
|
|
656
|
-
excludedList: Object.freeze(exclusions)
|
|
657
|
-
};
|
|
658
|
-
} catch (err) {
|
|
659
|
-
ctx.nameCorpusPromise = null;
|
|
660
|
-
console.warn("[anonymize] Failed to load name corpus JSON — name detection disabled:", err);
|
|
661
|
-
}
|
|
662
|
-
})();
|
|
663
|
-
ctx.nameCorpusPromise = promise;
|
|
664
|
-
return promise;
|
|
665
|
-
};
|
|
666
|
-
const INFLECTION_SUFFIXES = [
|
|
667
|
-
"ovi",
|
|
668
|
-
"em",
|
|
669
|
-
"om",
|
|
670
|
-
"ou",
|
|
671
|
-
"é",
|
|
672
|
-
"a",
|
|
673
|
-
"u"
|
|
674
|
-
];
|
|
675
|
-
/**
|
|
676
|
-
* Strip common Czech/Slovak case suffixes from a token.
|
|
677
|
-
* Returns candidate base forms if stripping produces a
|
|
678
|
-
* plausible name (capitalised, length >= 3).
|
|
679
|
-
*
|
|
680
|
-
* For the "-ou" instrumental feminine suffix, also yields
|
|
681
|
-
* base + "a" (e.g., "Editou" → "Edit" and "Edita")
|
|
682
|
-
* because Czech feminine names decline -a → -ou.
|
|
683
|
-
*/
|
|
684
|
-
const stripInflection = (token) => {
|
|
685
|
-
const candidates = [];
|
|
686
|
-
for (const suffix of INFLECTION_SUFFIXES) if (token.length > suffix.length + 2 && token.endsWith(suffix)) {
|
|
687
|
-
const base = token.slice(0, -suffix.length);
|
|
688
|
-
if (/^\p{Lu}/u.test(base)) {
|
|
689
|
-
candidates.push(base);
|
|
690
|
-
if (suffix === "ou" || suffix === "é" || suffix === "u") candidates.push(`${base}a`);
|
|
691
|
-
}
|
|
692
|
-
}
|
|
693
|
-
return candidates;
|
|
694
|
-
};
|
|
695
|
-
const TOKEN_TYPE = {
|
|
696
|
-
NAME: "name",
|
|
697
|
-
SURNAME: "surname",
|
|
698
|
-
TITLE: "title",
|
|
699
|
-
ABBREVIATION: "abbreviation",
|
|
700
|
-
CAPITALIZED: "capitalized",
|
|
701
|
-
OTHER: "other"
|
|
702
|
-
};
|
|
703
|
-
const PERSON_CHAIN_BREAK_RE$1 = /[!?;:]/u;
|
|
704
|
-
const isInitialContinuationGap$1 = (text, gap) => /^\p{Lu}$/u.test(text) && /^\.[^\S\n]{1,2}$/u.test(gap) || /^[^\S\n]{1,2}(?:\p{Lu}\.[^\S\n]{1,2})+$/u.test(gap);
|
|
705
|
-
/**
|
|
706
|
-
* Check if a token is in the first-name set, either
|
|
707
|
-
* directly or after stripping Czech/Slovak inflection.
|
|
708
|
-
*/
|
|
709
|
-
const isFirstNameToken = (token, corpus) => {
|
|
710
|
-
if (corpus.firstNames.has(token)) return true;
|
|
711
|
-
return stripInflection(token).some((b) => corpus.firstNames.has(b));
|
|
712
|
-
};
|
|
713
|
-
/**
|
|
714
|
-
* Check if a token is in the surname set, either
|
|
715
|
-
* directly or after stripping Czech/Slovak inflection.
|
|
716
|
-
*/
|
|
717
|
-
const isSurnameToken = (token, corpus) => {
|
|
718
|
-
if (corpus.surnames.has(token)) return true;
|
|
719
|
-
return stripInflection(token).some((b) => corpus.surnames.has(b));
|
|
720
|
-
};
|
|
721
|
-
/**
|
|
722
|
-
* Check if a token looks like a single-letter
|
|
723
|
-
* abbreviation: "J.", "M.", etc.
|
|
724
|
-
*/
|
|
725
|
-
const isAbbreviation = (token) => token.length === 2 && /^\p{Lu}$/u.test(token[0] ?? "") && token[1] === ".";
|
|
726
|
-
const segmenter$1 = new Intl.Segmenter(void 0, { granularity: "word" });
|
|
727
|
-
/**
|
|
728
|
-
* Split text into word segments using Intl.Segmenter.
|
|
729
|
-
* Only returns segments flagged as words.
|
|
730
|
-
*/
|
|
731
|
-
const segmentWords = (fullText) => {
|
|
732
|
-
const words = [];
|
|
733
|
-
for (const seg of segmenter$1.segment(fullText)) if (seg.isWordLike) words.push({
|
|
734
|
-
text: seg.segment,
|
|
735
|
-
start: seg.index,
|
|
736
|
-
end: seg.index + seg.segment.length
|
|
737
|
-
});
|
|
738
|
-
return words;
|
|
739
|
-
};
|
|
740
|
-
/** NAME or SURNAME — both represent corpus-matched tokens */
|
|
741
|
-
const isCorpusMatch = (type) => type === TOKEN_TYPE.NAME || type === TOKEN_TYPE.SURNAME;
|
|
742
|
-
const classifyToken = (word, corpus) => {
|
|
743
|
-
const { text, start, end } = word;
|
|
744
|
-
const lower = text.toLowerCase();
|
|
745
|
-
const stripped = text.endsWith(".") ? text.slice(0, -1).toLowerCase() : lower;
|
|
746
|
-
if (corpus.titleTokens.has(stripped)) return {
|
|
747
|
-
text,
|
|
748
|
-
type: TOKEN_TYPE.TITLE,
|
|
749
|
-
start,
|
|
750
|
-
end
|
|
751
|
-
};
|
|
752
|
-
if (isAbbreviation(text)) return {
|
|
753
|
-
text,
|
|
754
|
-
type: TOKEN_TYPE.ABBREVIATION,
|
|
755
|
-
start,
|
|
756
|
-
end
|
|
757
|
-
};
|
|
758
|
-
if (corpus.excludedWords.has(lower)) return {
|
|
759
|
-
text,
|
|
760
|
-
type: TOKEN_TYPE.OTHER,
|
|
761
|
-
start,
|
|
762
|
-
end
|
|
763
|
-
};
|
|
764
|
-
if (text.length < 3) return {
|
|
765
|
-
text,
|
|
766
|
-
type: TOKEN_TYPE.OTHER,
|
|
767
|
-
start,
|
|
768
|
-
end
|
|
769
|
-
};
|
|
770
|
-
if (text.length > 3 && ALL_UPPER_RE.test(text)) return {
|
|
771
|
-
text,
|
|
772
|
-
type: TOKEN_TYPE.OTHER,
|
|
773
|
-
start,
|
|
774
|
-
end
|
|
775
|
-
};
|
|
776
|
-
if (!UPPER_START_RE.test(text)) return {
|
|
777
|
-
text,
|
|
778
|
-
type: TOKEN_TYPE.OTHER,
|
|
779
|
-
start,
|
|
780
|
-
end
|
|
781
|
-
};
|
|
782
|
-
if (isFirstNameToken(text, corpus)) return {
|
|
783
|
-
text,
|
|
784
|
-
type: TOKEN_TYPE.NAME,
|
|
785
|
-
start,
|
|
786
|
-
end
|
|
787
|
-
};
|
|
788
|
-
if (isSurnameToken(text, corpus)) return {
|
|
789
|
-
text,
|
|
790
|
-
type: TOKEN_TYPE.SURNAME,
|
|
791
|
-
start,
|
|
792
|
-
end
|
|
793
|
-
};
|
|
794
|
-
return {
|
|
795
|
-
text,
|
|
796
|
-
type: TOKEN_TYPE.CAPITALIZED,
|
|
797
|
-
start,
|
|
798
|
-
end
|
|
799
|
-
};
|
|
800
|
-
};
|
|
801
|
-
/**
|
|
802
|
-
* Detect person names by looking up tokens against the
|
|
803
|
-
* name corpus, then chaining adjacent name-like tokens.
|
|
804
|
-
*
|
|
805
|
-
* Requires initNameCorpus() to have been called first.
|
|
806
|
-
* If not initialized, returns an empty array.
|
|
807
|
-
*
|
|
808
|
-
* Scoring:
|
|
809
|
-
* TITLE + NAME/SURNAME → 0.95
|
|
810
|
-
* NAME + NAME/SURNAME → 0.9
|
|
811
|
-
* SURNAME + NAME/SURNAME → 0.9
|
|
812
|
-
* NAME + CAPITALIZED → 0.7
|
|
813
|
-
* ABBREVIATION + NAME → 0.7
|
|
814
|
-
* Standalone NAME → 0.5 (low confidence)
|
|
815
|
-
* Standalone SURNAME → skip (too ambiguous)
|
|
816
|
-
*/
|
|
817
|
-
const detectNameCorpus = (fullText, ctx = defaultContext) => {
|
|
818
|
-
const corpus = getCorpus(ctx);
|
|
819
|
-
if (!corpus) return [];
|
|
820
|
-
const tokens = segmentWords(fullText).map((w) => classifyToken(w, corpus));
|
|
821
|
-
const entities = [];
|
|
822
|
-
const consumed = /* @__PURE__ */ new Set();
|
|
823
|
-
for (let i = 0; i < tokens.length; i++) {
|
|
824
|
-
if (consumed.has(i)) continue;
|
|
825
|
-
const token = tokens[i];
|
|
826
|
-
if (!token) continue;
|
|
827
|
-
if (token.type !== TOKEN_TYPE.TITLE && token.type !== TOKEN_TYPE.NAME && token.type !== TOKEN_TYPE.SURNAME && token.type !== TOKEN_TYPE.ABBREVIATION) continue;
|
|
828
|
-
const MAX_CHAIN = 5;
|
|
829
|
-
const chain = [token];
|
|
830
|
-
let j = i + 1;
|
|
831
|
-
while (j < tokens.length && chain.length < MAX_CHAIN) {
|
|
832
|
-
const next = tokens[j];
|
|
833
|
-
if (!next) break;
|
|
834
|
-
const prev = chain.at(-1);
|
|
835
|
-
if (prev) {
|
|
836
|
-
const gap = fullText.slice(prev.end, next.start);
|
|
837
|
-
const breaksOnPeriod = gap.includes(".") && !isInitialContinuationGap$1(prev.text, gap);
|
|
838
|
-
if (gap.includes("\n") || PERSON_CHAIN_BREAK_RE$1.test(gap) || breaksOnPeriod) break;
|
|
839
|
-
}
|
|
840
|
-
if (next.type === TOKEN_TYPE.NAME || next.type === TOKEN_TYPE.SURNAME || next.type === TOKEN_TYPE.TITLE || next.type === TOKEN_TYPE.ABBREVIATION || next.type === TOKEN_TYPE.CAPITALIZED) {
|
|
841
|
-
chain.push(next);
|
|
842
|
-
j++;
|
|
843
|
-
} else break;
|
|
844
|
-
}
|
|
845
|
-
const hasTitle = chain.some((t) => t.type === TOKEN_TYPE.TITLE);
|
|
846
|
-
const hasCorpusName = chain.some((t) => isCorpusMatch(t.type));
|
|
847
|
-
const hasFirstName = chain.some((t) => t.type === TOKEN_TYPE.NAME);
|
|
848
|
-
const hasAbbreviation = chain.some((t) => t.type === TOKEN_TYPE.ABBREVIATION);
|
|
849
|
-
const corpusCount = chain.filter((t) => isCorpusMatch(t.type)).length;
|
|
850
|
-
const capitalizedCount = chain.filter((t) => t.type === TOKEN_TYPE.CAPITALIZED).length;
|
|
851
|
-
let score = 0;
|
|
852
|
-
if (hasTitle && hasCorpusName) score = .95;
|
|
853
|
-
else if (corpusCount >= 2) score = .9;
|
|
854
|
-
else if (hasCorpusName && capitalizedCount > 0) score = .7;
|
|
855
|
-
else if (hasAbbreviation && hasCorpusName) score = .7;
|
|
856
|
-
else if (hasFirstName && chain.length === 1) {
|
|
857
|
-
if (isSentenceStart(fullText, token.start)) continue;
|
|
858
|
-
score = .5;
|
|
859
|
-
} else if (!hasFirstName && chain.length === 1 && chain[0]?.type === TOKEN_TYPE.SURNAME) continue;
|
|
860
|
-
else if (hasTitle && chain.length === 1) continue;
|
|
861
|
-
else {
|
|
862
|
-
if (!hasCorpusName) continue;
|
|
863
|
-
score = .5;
|
|
864
|
-
}
|
|
865
|
-
const first = chain.at(0);
|
|
866
|
-
const last = chain.at(-1);
|
|
867
|
-
if (!first || !last) continue;
|
|
868
|
-
const start = first.start;
|
|
869
|
-
const end = last.end;
|
|
870
|
-
const text = fullText.slice(start, end);
|
|
871
|
-
for (let k = i; k < i + chain.length; k++) consumed.add(k);
|
|
872
|
-
entities.push({
|
|
873
|
-
start,
|
|
874
|
-
end,
|
|
875
|
-
label: "person",
|
|
876
|
-
text,
|
|
877
|
-
score,
|
|
878
|
-
source: DETECTION_SOURCES.REGEX
|
|
879
|
-
});
|
|
880
|
-
}
|
|
881
|
-
return entities;
|
|
882
|
-
};
|
|
883
|
-
//#endregion
|
|
884
|
-
//#region src/config/titles.ts
|
|
885
|
-
/**
|
|
886
|
-
* Academic and professional title prefixes.
|
|
887
|
-
* Plain text; the detector auto-escapes for regex.
|
|
888
|
-
* Sorted longest-first at build time.
|
|
889
|
-
*/
|
|
890
|
-
const TITLE_PREFIXES = [
|
|
891
|
-
"Ing.",
|
|
892
|
-
"Mgr.",
|
|
893
|
-
"MgA.",
|
|
894
|
-
"Bc.",
|
|
895
|
-
"BcA.",
|
|
896
|
-
"JUDr.",
|
|
897
|
-
"MUDr.",
|
|
898
|
-
"MVDr.",
|
|
899
|
-
"MDDr.",
|
|
900
|
-
"PhDr.",
|
|
901
|
-
"RNDr.",
|
|
902
|
-
"PaedDr.",
|
|
903
|
-
"ThDr.",
|
|
904
|
-
"ThLic.",
|
|
905
|
-
"ICDr.",
|
|
906
|
-
"RSDr.",
|
|
907
|
-
"PharmDr.",
|
|
908
|
-
"artD.",
|
|
909
|
-
"akad.",
|
|
910
|
-
"doc.",
|
|
911
|
-
"prof.",
|
|
912
|
-
"ao. Univ.-Prof.",
|
|
913
|
-
"o. Univ.-Prof.",
|
|
914
|
-
"Univ.-Prof.",
|
|
915
|
-
"Hon.-Prof.",
|
|
916
|
-
"em. Prof.",
|
|
917
|
-
"Dr. med. dent.",
|
|
918
|
-
"Dr. med. vet.",
|
|
919
|
-
"Dr. med.",
|
|
920
|
-
"Dr. rer. nat.",
|
|
921
|
-
"Dr. rer. soc.",
|
|
922
|
-
"Dr. rer. pol.",
|
|
923
|
-
"Dr. sc. tech.",
|
|
924
|
-
"Dr. sc. nat.",
|
|
925
|
-
"Dr. sc. hum.",
|
|
926
|
-
"Dr. iur.",
|
|
927
|
-
"Dr. jur.",
|
|
928
|
-
"Dr. theol.",
|
|
929
|
-
"Dr. oec.",
|
|
930
|
-
"Dr. techn.",
|
|
931
|
-
"Dr. h. c.",
|
|
932
|
-
"Dr. phil.",
|
|
933
|
-
"Dr.-Ing.",
|
|
934
|
-
"Dr. Ing.",
|
|
935
|
-
"Dr.",
|
|
936
|
-
"Dipl.-Wirt.-Ing.",
|
|
937
|
-
"Dipl.-Betriebsw.",
|
|
938
|
-
"Dipl.-Inform.",
|
|
939
|
-
"Dipl.-Volksw.",
|
|
940
|
-
"Dipl.-Psych.",
|
|
941
|
-
"Dipl.-Phys.",
|
|
942
|
-
"Dipl.-Chem.",
|
|
943
|
-
"Dipl.-Biol.",
|
|
944
|
-
"Dipl.-Math.",
|
|
945
|
-
"Dipl.-Päd.",
|
|
946
|
-
"Dipl.-Soz.",
|
|
947
|
-
"Dipl.-Kfm.",
|
|
948
|
-
"Dipl.-Jur.",
|
|
949
|
-
"Dipl. Ing.",
|
|
950
|
-
"Dipl.-Ing.",
|
|
951
|
-
"Mag. rer. soc. oec.",
|
|
952
|
-
"Mag. rer. nat.",
|
|
953
|
-
"Mag. phil.",
|
|
954
|
-
"Mag. iur.",
|
|
955
|
-
"Mag. arch.",
|
|
956
|
-
"Mag. pharm.",
|
|
957
|
-
"Mag. (FH)",
|
|
958
|
-
"Mag.",
|
|
959
|
-
"Bakk. rer. nat.",
|
|
960
|
-
"Bakk. techn.",
|
|
961
|
-
"Bakk. phil.",
|
|
962
|
-
"Bakk.",
|
|
963
|
-
"Lic. phil.",
|
|
964
|
-
"Lic. iur.",
|
|
965
|
-
"Lic. oec.",
|
|
966
|
-
"Lic. theol.",
|
|
967
|
-
"Lic.",
|
|
968
|
-
"Priv.-Doz.",
|
|
969
|
-
"PD",
|
|
970
|
-
"RA"
|
|
971
|
-
];
|
|
972
|
-
/**
|
|
973
|
-
* Courtesy/honorific titles that precede a person's
|
|
974
|
-
* name. Sorted alphabetically. The detector escapes
|
|
975
|
-
* dots and adds \b for entries in HONORIFIC_BOUNDARY.
|
|
976
|
-
*/
|
|
977
|
-
const HONORIFICS = [
|
|
978
|
-
"Avv.",
|
|
979
|
-
"Dame",
|
|
980
|
-
"Doamna",
|
|
981
|
-
"Domnul",
|
|
982
|
-
"Don",
|
|
983
|
-
"Doña",
|
|
984
|
-
"Dott.",
|
|
985
|
-
"Judge",
|
|
986
|
-
"Justice",
|
|
987
|
-
"Lady",
|
|
988
|
-
"Lord",
|
|
989
|
-
"M.",
|
|
990
|
-
"Madame",
|
|
991
|
-
"Mademoiselle",
|
|
992
|
-
"Maître",
|
|
993
|
-
"Me",
|
|
994
|
-
"Messrs",
|
|
995
|
-
"Miss",
|
|
996
|
-
"Mlle",
|
|
997
|
-
"Mme",
|
|
998
|
-
"Monsieur",
|
|
999
|
-
"Mr",
|
|
1000
|
-
"Mrs",
|
|
1001
|
-
"Ms",
|
|
1002
|
-
"President",
|
|
1003
|
-
"Señor",
|
|
1004
|
-
"Señora",
|
|
1005
|
-
"Sig.",
|
|
1006
|
-
"Sig.ra",
|
|
1007
|
-
"Signor",
|
|
1008
|
-
"Signora",
|
|
1009
|
-
"Signorina",
|
|
1010
|
-
"Sir",
|
|
1011
|
-
"Sr.",
|
|
1012
|
-
"Sra."
|
|
1013
|
-
];
|
|
1014
|
-
/**
|
|
1015
|
-
* Honorifics that need \b word-boundary anchors
|
|
1016
|
-
* (short or common words that could match mid-word).
|
|
1017
|
-
*/
|
|
1018
|
-
const HONORIFIC_BOUNDARY = new Set([
|
|
1019
|
-
"Don",
|
|
1020
|
-
"Doña",
|
|
1021
|
-
"M.",
|
|
1022
|
-
"Me",
|
|
1023
|
-
"Señor",
|
|
1024
|
-
"Señora"
|
|
1025
|
-
]);
|
|
1026
|
-
/**
|
|
1027
|
-
* Post-nominal degrees (comma or space separated after name).
|
|
1028
|
-
* Plain text; the detector auto-escapes for regex.
|
|
1029
|
-
*/
|
|
1030
|
-
const POST_NOMINALS = [
|
|
1031
|
-
"Ph.D.",
|
|
1032
|
-
"Ph.D",
|
|
1033
|
-
"CSc.",
|
|
1034
|
-
"DrSc.",
|
|
1035
|
-
"ArtD.",
|
|
1036
|
-
"D.Phil.",
|
|
1037
|
-
"DPhil.",
|
|
1038
|
-
"MPhil.",
|
|
1039
|
-
"MBA",
|
|
1040
|
-
"MPA",
|
|
1041
|
-
"LL.M.",
|
|
1042
|
-
"LL.B.",
|
|
1043
|
-
"M.Sc.",
|
|
1044
|
-
"B.Sc.",
|
|
1045
|
-
"MSc.",
|
|
1046
|
-
"BSc.",
|
|
1047
|
-
"M.Eng.",
|
|
1048
|
-
"B.Eng.",
|
|
1049
|
-
"M.A.",
|
|
1050
|
-
"B.A.",
|
|
1051
|
-
"JCD",
|
|
1052
|
-
"JD",
|
|
1053
|
-
"DiS.",
|
|
1054
|
-
"ACCA",
|
|
1055
|
-
"FCCA",
|
|
1056
|
-
"CIPM",
|
|
1057
|
-
"CIPT",
|
|
1058
|
-
"CIPP/E",
|
|
1059
|
-
"CIPP"
|
|
1060
|
-
];
|
|
269
|
+
const LEGAL_SUFFIXES = [...[
|
|
270
|
+
"spol. s r.o.",
|
|
271
|
+
"s.r.o.",
|
|
272
|
+
"s. r. o.",
|
|
273
|
+
"a.s.",
|
|
274
|
+
"a. s.",
|
|
275
|
+
"v.o.s.",
|
|
276
|
+
"v. o. s.",
|
|
277
|
+
"k.s.",
|
|
278
|
+
"k. s.",
|
|
279
|
+
"z.s.",
|
|
280
|
+
"z. s.",
|
|
281
|
+
"z.ú.",
|
|
282
|
+
"z. ú.",
|
|
283
|
+
"o.p.s.",
|
|
284
|
+
"o. p. s.",
|
|
285
|
+
"s.p.",
|
|
286
|
+
"s. p.",
|
|
287
|
+
"GmbH",
|
|
288
|
+
"AG",
|
|
289
|
+
"SE",
|
|
290
|
+
"KG",
|
|
291
|
+
"OHG",
|
|
292
|
+
"Ltd.",
|
|
293
|
+
"Ltd",
|
|
294
|
+
"LLC",
|
|
295
|
+
"LLP",
|
|
296
|
+
"Inc.",
|
|
297
|
+
"Corp.",
|
|
298
|
+
"Corporation",
|
|
299
|
+
"Co.",
|
|
300
|
+
"LP",
|
|
301
|
+
"L.P.",
|
|
302
|
+
"PLC",
|
|
303
|
+
"plc",
|
|
304
|
+
"N.A.",
|
|
305
|
+
"N.V.",
|
|
306
|
+
"B.V.",
|
|
307
|
+
"Pty Ltd.",
|
|
308
|
+
"Pty Ltd",
|
|
309
|
+
"S.A.",
|
|
310
|
+
"SA",
|
|
311
|
+
"SAS",
|
|
312
|
+
"SARL",
|
|
313
|
+
"S.p.A.",
|
|
314
|
+
"Sp. z o.o.",
|
|
315
|
+
"Sp. k.",
|
|
316
|
+
"Sp. j.",
|
|
317
|
+
"Ltda.",
|
|
318
|
+
"LTDA.",
|
|
319
|
+
"Ltda",
|
|
320
|
+
"LTDA",
|
|
321
|
+
"S/A",
|
|
322
|
+
"EIRELI",
|
|
323
|
+
"EPP",
|
|
324
|
+
"ME",
|
|
325
|
+
"MEI"
|
|
326
|
+
]].sort((a, b) => b.length - a.length);
|
|
1061
327
|
//#endregion
|
|
1062
328
|
//#region src/data/char-groups.json
|
|
1063
329
|
var char_groups_default = {
|
|
@@ -1397,255 +663,1536 @@ var char_groups_default = {
|
|
|
1397
663
|
"code": "U+A789"
|
|
1398
664
|
},
|
|
1399
665
|
{
|
|
1400
|
-
"char": "∶",
|
|
1401
|
-
"name": "ratio",
|
|
1402
|
-
"code": "U+2236"
|
|
666
|
+
"char": "∶",
|
|
667
|
+
"name": "ratio",
|
|
668
|
+
"code": "U+2236"
|
|
669
|
+
},
|
|
670
|
+
{
|
|
671
|
+
"char": ":",
|
|
672
|
+
"name": "full-width colon",
|
|
673
|
+
"code": "U+FF1A"
|
|
674
|
+
},
|
|
675
|
+
{
|
|
676
|
+
"char": "፥",
|
|
677
|
+
"name": "Ethiopic colon",
|
|
678
|
+
"code": "U+1365"
|
|
679
|
+
}
|
|
680
|
+
]
|
|
681
|
+
},
|
|
682
|
+
"comma": {
|
|
683
|
+
"description": "Comma-like characters",
|
|
684
|
+
"chars": [
|
|
685
|
+
{
|
|
686
|
+
"char": ",",
|
|
687
|
+
"name": "comma",
|
|
688
|
+
"code": "U+002C"
|
|
689
|
+
},
|
|
690
|
+
{
|
|
691
|
+
"char": "‚",
|
|
692
|
+
"name": "single low-9 quotation mark (used as comma)",
|
|
693
|
+
"code": "U+201A"
|
|
694
|
+
},
|
|
695
|
+
{
|
|
696
|
+
"char": "،",
|
|
697
|
+
"name": "Arabic comma",
|
|
698
|
+
"code": "U+060C"
|
|
699
|
+
},
|
|
700
|
+
{
|
|
701
|
+
"char": "、",
|
|
702
|
+
"name": "ideographic comma",
|
|
703
|
+
"code": "U+3001"
|
|
704
|
+
},
|
|
705
|
+
{
|
|
706
|
+
"char": ",",
|
|
707
|
+
"name": "full-width comma",
|
|
708
|
+
"code": "U+FF0C"
|
|
709
|
+
},
|
|
710
|
+
{
|
|
711
|
+
"char": "﹐",
|
|
712
|
+
"name": "small comma",
|
|
713
|
+
"code": "U+FE50"
|
|
714
|
+
},
|
|
715
|
+
{
|
|
716
|
+
"char": "՝",
|
|
717
|
+
"name": "Armenian comma",
|
|
718
|
+
"code": "U+055D"
|
|
719
|
+
}
|
|
720
|
+
]
|
|
721
|
+
},
|
|
722
|
+
"ellipsis": {
|
|
723
|
+
"description": "Ellipsis characters",
|
|
724
|
+
"chars": [{
|
|
725
|
+
"char": "…",
|
|
726
|
+
"name": "horizontal ellipsis",
|
|
727
|
+
"code": "U+2026"
|
|
728
|
+
}]
|
|
729
|
+
},
|
|
730
|
+
"question-mark": {
|
|
731
|
+
"description": "Question mark variants",
|
|
732
|
+
"chars": [
|
|
733
|
+
{
|
|
734
|
+
"char": "?",
|
|
735
|
+
"name": "question mark",
|
|
736
|
+
"code": "U+003F"
|
|
737
|
+
},
|
|
738
|
+
{
|
|
739
|
+
"char": "?",
|
|
740
|
+
"name": "full-width question mark",
|
|
741
|
+
"code": "U+FF1F"
|
|
742
|
+
},
|
|
743
|
+
{
|
|
744
|
+
"char": "⁇",
|
|
745
|
+
"name": "double question mark",
|
|
746
|
+
"code": "U+2047"
|
|
1403
747
|
},
|
|
1404
748
|
{
|
|
1405
|
-
"char": "
|
|
1406
|
-
"name": "
|
|
1407
|
-
"code": "U+
|
|
749
|
+
"char": "⁈",
|
|
750
|
+
"name": "question exclamation mark",
|
|
751
|
+
"code": "U+2048"
|
|
1408
752
|
},
|
|
1409
753
|
{
|
|
1410
|
-
"char": "
|
|
1411
|
-
"name": "
|
|
1412
|
-
"code": "U+
|
|
754
|
+
"char": "؟",
|
|
755
|
+
"name": "Arabic question mark",
|
|
756
|
+
"code": "U+061F"
|
|
757
|
+
},
|
|
758
|
+
{
|
|
759
|
+
"char": "‽",
|
|
760
|
+
"name": "interrobang",
|
|
761
|
+
"code": "U+203D"
|
|
1413
762
|
}
|
|
1414
763
|
]
|
|
1415
764
|
},
|
|
1416
|
-
"
|
|
1417
|
-
"description": "
|
|
765
|
+
"exclamation-mark": {
|
|
766
|
+
"description": "Exclamation mark variants",
|
|
1418
767
|
"chars": [
|
|
1419
768
|
{
|
|
1420
|
-
"char": "
|
|
1421
|
-
"name": "
|
|
1422
|
-
"code": "U+
|
|
769
|
+
"char": "!",
|
|
770
|
+
"name": "exclamation mark",
|
|
771
|
+
"code": "U+0021"
|
|
1423
772
|
},
|
|
1424
773
|
{
|
|
1425
|
-
"char": "
|
|
1426
|
-
"name": "
|
|
1427
|
-
"code": "U+
|
|
774
|
+
"char": "!",
|
|
775
|
+
"name": "full-width exclamation mark",
|
|
776
|
+
"code": "U+FF01"
|
|
1428
777
|
},
|
|
1429
778
|
{
|
|
1430
|
-
"char": "
|
|
1431
|
-
"name": "
|
|
1432
|
-
"code": "U+
|
|
779
|
+
"char": "‼",
|
|
780
|
+
"name": "double exclamation mark",
|
|
781
|
+
"code": "U+203C"
|
|
1433
782
|
},
|
|
1434
783
|
{
|
|
1435
|
-
"char": "
|
|
1436
|
-
"name": "
|
|
1437
|
-
"code": "U+
|
|
784
|
+
"char": "¡",
|
|
785
|
+
"name": "inverted exclamation mark",
|
|
786
|
+
"code": "U+00A1"
|
|
787
|
+
}
|
|
788
|
+
]
|
|
789
|
+
},
|
|
790
|
+
"semicolon": {
|
|
791
|
+
"description": "Semicolon variants",
|
|
792
|
+
"chars": [
|
|
793
|
+
{
|
|
794
|
+
"char": ";",
|
|
795
|
+
"name": "semicolon",
|
|
796
|
+
"code": "U+003B"
|
|
1438
797
|
},
|
|
1439
798
|
{
|
|
1440
|
-
"char": "
|
|
1441
|
-
"name": "
|
|
1442
|
-
"code": "U+
|
|
799
|
+
"char": "؛",
|
|
800
|
+
"name": "Arabic semicolon",
|
|
801
|
+
"code": "U+061B"
|
|
1443
802
|
},
|
|
1444
803
|
{
|
|
1445
|
-
"char": "
|
|
1446
|
-
"name": "
|
|
1447
|
-
"code": "U+
|
|
804
|
+
"char": ";",
|
|
805
|
+
"name": "full-width semicolon",
|
|
806
|
+
"code": "U+FF1B"
|
|
1448
807
|
},
|
|
1449
808
|
{
|
|
1450
|
-
"char": "
|
|
1451
|
-
"name": "
|
|
1452
|
-
"code": "U+
|
|
809
|
+
"char": "፤",
|
|
810
|
+
"name": "Ethiopic semicolon",
|
|
811
|
+
"code": "U+1364"
|
|
812
|
+
}
|
|
813
|
+
]
|
|
814
|
+
},
|
|
815
|
+
"underscore": {
|
|
816
|
+
"description": "Underscore variants",
|
|
817
|
+
"chars": [{
|
|
818
|
+
"char": "_",
|
|
819
|
+
"name": "low line",
|
|
820
|
+
"code": "U+005F"
|
|
821
|
+
}, {
|
|
822
|
+
"char": "_",
|
|
823
|
+
"name": "full-width low line",
|
|
824
|
+
"code": "U+FF3F"
|
|
825
|
+
}]
|
|
826
|
+
}
|
|
827
|
+
}
|
|
828
|
+
};
|
|
829
|
+
//#endregion
|
|
830
|
+
//#region src/util/char-groups.ts
|
|
831
|
+
/**
|
|
832
|
+
* Centralized Unicode character equivalence groups.
|
|
833
|
+
*
|
|
834
|
+
* Centralized Unicode character equivalence groups.
|
|
835
|
+
*
|
|
836
|
+
* Provides helpers to build regex character classes that
|
|
837
|
+
* match all typographic variants of a character type
|
|
838
|
+
* (dashes, spaces, quotes, etc.).
|
|
839
|
+
*
|
|
840
|
+
* The JSON is statically imported so the bundler inlines
|
|
841
|
+
* it, avoiding a runtime require() that breaks browsers.
|
|
842
|
+
*/
|
|
843
|
+
/** Chars that need escaping inside a regex char class. */
|
|
844
|
+
const REGEX_CLASS_SPECIAL = /[\\\]^-]/;
|
|
845
|
+
const escapeForCharClass = (ch) => REGEX_CLASS_SPECIAL.test(ch) ? `\\${ch}` : ch;
|
|
846
|
+
const config = char_groups_default;
|
|
847
|
+
/**
|
|
848
|
+
* Get the raw characters for a named group.
|
|
849
|
+
* Throws if the group does not exist.
|
|
850
|
+
*/
|
|
851
|
+
const charSet = (group) => {
|
|
852
|
+
const g = config.groups[group];
|
|
853
|
+
if (!g) throw new Error(`Unknown char group: "${group}"`);
|
|
854
|
+
return g.chars.map((entry) => entry.char);
|
|
855
|
+
};
|
|
856
|
+
/**
|
|
857
|
+
* Build a regex character class string for a named
|
|
858
|
+
* group. E.g., charClass("dash") returns a string
|
|
859
|
+
* like "[-\u2013\u2014\u2010\u2011\u2212\u2043]".
|
|
860
|
+
*
|
|
861
|
+
* The hyphen-minus is placed first so it is treated
|
|
862
|
+
* as a literal, not a range indicator.
|
|
863
|
+
*/
|
|
864
|
+
const charClass = (group) => {
|
|
865
|
+
return `[${[...charSet(group)].sort((a, b) => {
|
|
866
|
+
if (a === "-") return -1;
|
|
867
|
+
if (b === "-") return 1;
|
|
868
|
+
return a.localeCompare(b);
|
|
869
|
+
}).map(escapeForCharClass).join("")}]`;
|
|
870
|
+
};
|
|
871
|
+
/**
|
|
872
|
+
* Return the inner content of a character class (without
|
|
873
|
+
* the surrounding brackets). Useful for embedding a group
|
|
874
|
+
* inside a larger character class, e.g.:
|
|
875
|
+
* `[\\s&,.${charClassInner("dash")}]`
|
|
876
|
+
*/
|
|
877
|
+
const charClassInner = (group) => {
|
|
878
|
+
return [...charSet(group)].sort((a, b) => {
|
|
879
|
+
if (a === "-") return -1;
|
|
880
|
+
if (b === "-") return 1;
|
|
881
|
+
return a.localeCompare(b);
|
|
882
|
+
}).map(escapeForCharClass).join("");
|
|
883
|
+
};
|
|
884
|
+
/** Regex char class matching all dash variants. */
|
|
885
|
+
const DASH = charClass("dash");
|
|
886
|
+
/**
|
|
887
|
+
* Inner content of the dash char class (no brackets).
|
|
888
|
+
* For embedding inside a larger character class:
|
|
889
|
+
* `[\\s&,.${DASH_INNER}]`
|
|
890
|
+
*/
|
|
891
|
+
const DASH_INNER = charClassInner("dash");
|
|
892
|
+
charClass("space");
|
|
893
|
+
charClass("quote-double");
|
|
894
|
+
/** Inner content of the double-quote char class (no brackets). */
|
|
895
|
+
const QUOTE_DOUBLE_INNER = charClassInner("quote-double");
|
|
896
|
+
charClass("quote-single");
|
|
897
|
+
/** Inner content of the single-quote char class (no brackets). */
|
|
898
|
+
const QUOTE_SINGLE_INNER = charClassInner("quote-single");
|
|
899
|
+
/**
|
|
900
|
+
* Inline opening-bracket characters that frequently introduce
|
|
901
|
+
* a defined-term parenthetical immediately after an address or
|
|
902
|
+
* organization span (e.g. `…GA 30326, USA (the "Premises")`).
|
|
903
|
+
* Hardcoded rather than data-driven because the set is tiny,
|
|
904
|
+
* regex-special, and not language-dependent.
|
|
905
|
+
*/
|
|
906
|
+
const OPENING_BRACKETS_INNER = "(\\[{";
|
|
907
|
+
charClass("slash");
|
|
908
|
+
charClass("dot");
|
|
909
|
+
charClass("colon");
|
|
910
|
+
//#endregion
|
|
911
|
+
//#region src/detectors/legal-forms.ts
|
|
912
|
+
const SENTENCE_VERB_INDICATORS_SEED = new Set([
|
|
913
|
+
"je",
|
|
914
|
+
"jsou",
|
|
915
|
+
"is",
|
|
916
|
+
"are",
|
|
917
|
+
"ist",
|
|
918
|
+
"sind"
|
|
919
|
+
]);
|
|
920
|
+
let sentenceVerbIndicatorsCache = null;
|
|
921
|
+
let sentenceVerbIndicatorsPromise = null;
|
|
922
|
+
const loadSentenceVerbIndicators = async () => {
|
|
923
|
+
if (sentenceVerbIndicatorsCache) return sentenceVerbIndicatorsCache;
|
|
924
|
+
if (sentenceVerbIndicatorsPromise) return sentenceVerbIndicatorsPromise;
|
|
925
|
+
sentenceVerbIndicatorsPromise = (async () => {
|
|
926
|
+
let data = {};
|
|
927
|
+
try {
|
|
928
|
+
const mod = await import("./sentence-verb-indicators.mjs");
|
|
929
|
+
data = mod.default ?? mod;
|
|
930
|
+
} catch (err) {
|
|
931
|
+
console.warn("[anonymize] legal-forms: failed to load sentence-verb-indicators.json, falling back to seed list:", err);
|
|
932
|
+
}
|
|
933
|
+
const all = new Set(SENTENCE_VERB_INDICATORS_SEED);
|
|
934
|
+
for (const [key, value] of Object.entries(data)) {
|
|
935
|
+
if (key.startsWith("_")) continue;
|
|
936
|
+
if (!Array.isArray(value)) continue;
|
|
937
|
+
for (const verb of value) {
|
|
938
|
+
if (typeof verb !== "string" || verb.length === 0) continue;
|
|
939
|
+
all.add(verb.toLowerCase());
|
|
940
|
+
}
|
|
941
|
+
}
|
|
942
|
+
sentenceVerbIndicatorsCache = all;
|
|
943
|
+
return all;
|
|
944
|
+
})();
|
|
945
|
+
return sentenceVerbIndicatorsPromise;
|
|
946
|
+
};
|
|
947
|
+
const getSentenceVerbIndicatorsSync = () => sentenceVerbIndicatorsCache ?? SENTENCE_VERB_INDICATORS_SEED;
|
|
948
|
+
const UPPER = "A-ZÁČĎÉĚÍŇÓŘŠŤÚŮÝŽÄÖÜÀÂÆÇÈÊËÎÏÔÙÛŸÑĄĆĘŁŃŚŹŻ\\u0130";
|
|
949
|
+
const LOWER = "a-záčďéěíňóřšťúůýžäöüßàâæçèêëîïôùûÿñąćęłńśźż\\u0131";
|
|
950
|
+
const CAP_WORD = `(?:[${UPPER}]{2,}|[${UPPER}][${LOWER}${UPPER}]+)`;
|
|
951
|
+
const ALLCAP_WORD = `[${UPPER}]{2,}`;
|
|
952
|
+
const ROMAN_NUMERAL_RE = /^(?=[IVXLCDM])M{0,3}(?:CM|CD|D?C{0,3})(?:XC|XL|L?X{0,3})(?:IX|IV|V?I{0,3})$/;
|
|
953
|
+
let legalRoleHeadsCache = null;
|
|
954
|
+
let legalRoleHeadsPromise = null;
|
|
955
|
+
const loadLegalRoleHeads = async () => {
|
|
956
|
+
if (legalRoleHeadsCache) return legalRoleHeadsCache;
|
|
957
|
+
if (legalRoleHeadsPromise) return legalRoleHeadsPromise;
|
|
958
|
+
legalRoleHeadsPromise = (async () => {
|
|
959
|
+
const sets = await loadLanguageConfigs("legalRoleHeads", (mod) => {
|
|
960
|
+
return mod.default ?? mod;
|
|
961
|
+
});
|
|
962
|
+
const all = /* @__PURE__ */ new Set();
|
|
963
|
+
for (const entry of sets) {
|
|
964
|
+
if (!entry || !Array.isArray(entry.words)) continue;
|
|
965
|
+
for (const word of entry.words) if (typeof word === "string" && word.length > 0) all.add(word.toLowerCase());
|
|
966
|
+
}
|
|
967
|
+
legalRoleHeadsCache = all;
|
|
968
|
+
return all;
|
|
969
|
+
})();
|
|
970
|
+
return legalRoleHeadsPromise;
|
|
971
|
+
};
|
|
972
|
+
const getLegalRoleHeadsSync = () => legalRoleHeadsCache ?? /* @__PURE__ */ new Set();
|
|
973
|
+
const warmLegalRoleHeads = async () => {
|
|
974
|
+
await Promise.all([
|
|
975
|
+
loadLegalRoleHeads(),
|
|
976
|
+
loadAllLegalSuffixes(),
|
|
977
|
+
loadSentenceVerbIndicators(),
|
|
978
|
+
loadClauseNounHeads()
|
|
979
|
+
]);
|
|
980
|
+
};
|
|
981
|
+
let allLegalSuffixesCache = null;
|
|
982
|
+
let allLegalSuffixesPromise = null;
|
|
983
|
+
const loadAllLegalSuffixes = async () => {
|
|
984
|
+
if (allLegalSuffixesCache) return allLegalSuffixesCache;
|
|
985
|
+
if (allLegalSuffixesPromise) return allLegalSuffixesPromise;
|
|
986
|
+
allLegalSuffixesPromise = (async () => {
|
|
987
|
+
let data = {};
|
|
988
|
+
try {
|
|
989
|
+
data = (await import("./legal-forms.mjs")).default;
|
|
990
|
+
} catch {
|
|
991
|
+
data = {};
|
|
992
|
+
}
|
|
993
|
+
const seen = /* @__PURE__ */ new Set();
|
|
994
|
+
const out = [];
|
|
995
|
+
for (const list of Object.values(data)) for (const form of list) {
|
|
996
|
+
if (typeof form !== "string" || form.length === 0) continue;
|
|
997
|
+
if (seen.has(form)) continue;
|
|
998
|
+
seen.add(form);
|
|
999
|
+
out.push(form);
|
|
1000
|
+
}
|
|
1001
|
+
for (const form of LEGAL_SUFFIXES) if (!seen.has(form)) {
|
|
1002
|
+
seen.add(form);
|
|
1003
|
+
out.push(form);
|
|
1004
|
+
}
|
|
1005
|
+
out.sort((a, b) => b.length - a.length);
|
|
1006
|
+
allLegalSuffixesCache = out;
|
|
1007
|
+
return out;
|
|
1008
|
+
})();
|
|
1009
|
+
return allLegalSuffixesPromise;
|
|
1010
|
+
};
|
|
1011
|
+
const getAllLegalSuffixesSync = () => allLegalSuffixesCache ?? LEGAL_SUFFIXES;
|
|
1012
|
+
/**
|
|
1013
|
+
* Sync accessor for the full legal-form vocabulary
|
|
1014
|
+
* (`data/legal-forms.json` plus `LEGAL_SUFFIXES`,
|
|
1015
|
+
* longest-first). Falls back to `LEGAL_SUFFIXES` when
|
|
1016
|
+
* `warmLegalRoleHeads()` has not run yet. Exposed so the
|
|
1017
|
+
* trailing-period strip in `sanitizeEntities` can keep
|
|
1018
|
+
* pace with the detector vocabulary rather than only the
|
|
1019
|
+
* smaller `LEGAL_SUFFIXES` propagation list.
|
|
1020
|
+
*/
|
|
1021
|
+
const getKnownLegalSuffixes = getAllLegalSuffixesSync;
|
|
1022
|
+
const CLAUSE_NOUN_HEADS_SEED = new Set(["agreement", "contract"]);
|
|
1023
|
+
let clauseNounHeadsCache = null;
|
|
1024
|
+
let clauseNounHeadsPromise = null;
|
|
1025
|
+
const loadClauseNounHeads = async () => {
|
|
1026
|
+
if (clauseNounHeadsCache) return clauseNounHeadsCache;
|
|
1027
|
+
if (clauseNounHeadsPromise) return clauseNounHeadsPromise;
|
|
1028
|
+
clauseNounHeadsPromise = (async () => {
|
|
1029
|
+
let data = {};
|
|
1030
|
+
try {
|
|
1031
|
+
const mod = await import("./clause-noun-heads.mjs");
|
|
1032
|
+
data = mod.default ?? mod;
|
|
1033
|
+
} catch (err) {
|
|
1034
|
+
console.warn("[anonymize] legal-forms: failed to load clause-noun-heads.json, falling back to seed list:", err);
|
|
1035
|
+
}
|
|
1036
|
+
const all = new Set(CLAUSE_NOUN_HEADS_SEED);
|
|
1037
|
+
for (const [key, value] of Object.entries(data)) {
|
|
1038
|
+
if (key.startsWith("_")) continue;
|
|
1039
|
+
if (!Array.isArray(value)) continue;
|
|
1040
|
+
for (const noun of value) {
|
|
1041
|
+
if (typeof noun !== "string" || noun.length === 0) continue;
|
|
1042
|
+
all.add(noun.toLowerCase());
|
|
1043
|
+
}
|
|
1044
|
+
}
|
|
1045
|
+
clauseNounHeadsCache = all;
|
|
1046
|
+
return all;
|
|
1047
|
+
})();
|
|
1048
|
+
return clauseNounHeadsPromise;
|
|
1049
|
+
};
|
|
1050
|
+
const getClauseNounHeadsSync = () => clauseNounHeadsCache ?? CLAUSE_NOUN_HEADS_SEED;
|
|
1051
|
+
const escapeForRegex = (form) => form.replace(/[.*+?^${}()|[\]\\]/g, "\\$&").replace(/\s+/g, "\\s+").replace(/\\\./g, "\\.[^\\S\\n]?");
|
|
1052
|
+
const isShortForm = (form) => form.replace(/[.\s]/g, "").length <= 3 && !form.includes(" ");
|
|
1053
|
+
const buildPatternString = (forms) => {
|
|
1054
|
+
if (forms.length === 0) return null;
|
|
1055
|
+
const alt = forms.toSorted((a, b) => b.length - a.length).map(escapeForRegex).join("|");
|
|
1056
|
+
const HSPACE = "[^\\S\\n]";
|
|
1057
|
+
const LOWER_CONNECTOR = `${HSPACE}+(?:a|and|und|et|e|y|i)${HSPACE}+(?=[${LOWER}])`;
|
|
1058
|
+
const SIMPLE_SEP = `(?:${HSPACE}|[&,.${DASH_INNER}]){1,4}`;
|
|
1059
|
+
const CAP_OR_NUM_WORD = `(?:${CAP_WORD}|\\d{1,4})`;
|
|
1060
|
+
const LOWER_WORD = `(?:(?!(?:and|und|et)(?![${UPPER}${LOWER}]))[${LOWER}][${LOWER}${UPPER}]+)`;
|
|
1061
|
+
const ANY_WORD_TAIL = `(?:(?!(?:and|und|et)(?![${UPPER}${LOWER}]))[${UPPER}${LOWER}][${LOWER}${UPPER}]+|[${UPPER}]{2,3}|\\d{1,4})`;
|
|
1062
|
+
return `${`(?:${`(?:${CAP_WORD})(?:${SIMPLE_SEP}(?:${CAP_OR_NUM_WORD})){0,10}`})(?:${`${SIMPLE_SEP}(?:${LOWER_WORD})(?:(?:${LOWER_CONNECTOR}|${SIMPLE_SEP})(?:${ANY_WORD_TAIL})){0,10}`})?`}(?:\\s+|,\\s*)(?:${alt})(?![${LOWER}])`;
|
|
1063
|
+
};
|
|
1064
|
+
/**
|
|
1065
|
+
* Build legal form regex pattern strings.
|
|
1066
|
+
* Returns an array of regex strings for the unified
|
|
1067
|
+
* TextSearch builder. Empty if data package is not
|
|
1068
|
+
* installed.
|
|
1069
|
+
*/
|
|
1070
|
+
const buildLegalFormPatterns = async () => {
|
|
1071
|
+
let data = {};
|
|
1072
|
+
try {
|
|
1073
|
+
data = (await import("./legal-forms.mjs")).default;
|
|
1074
|
+
} catch {
|
|
1075
|
+
return [];
|
|
1076
|
+
}
|
|
1077
|
+
const allForms = [];
|
|
1078
|
+
const seen = /* @__PURE__ */ new Set();
|
|
1079
|
+
for (const forms of Object.values(data)) for (const form of forms) if (!seen.has(form)) {
|
|
1080
|
+
seen.add(form);
|
|
1081
|
+
allForms.push(form);
|
|
1082
|
+
}
|
|
1083
|
+
for (const form of LEGAL_SUFFIXES) if (!seen.has(form)) {
|
|
1084
|
+
seen.add(form);
|
|
1085
|
+
allForms.push(form);
|
|
1086
|
+
}
|
|
1087
|
+
const patterns = [];
|
|
1088
|
+
const longPattern = buildPatternString(allForms.filter((f) => !isShortForm(f)));
|
|
1089
|
+
if (longPattern) patterns.push(longPattern);
|
|
1090
|
+
const shortPattern = buildPatternString(allForms.filter(isShortForm));
|
|
1091
|
+
if (shortPattern) patterns.push(shortPattern);
|
|
1092
|
+
const allcapPrefix = `(?:${ALLCAP_WORD})(?:[ \\t&,.${DASH_INNER}]{1,4}(?:${ALLCAP_WORD})){0,2}`;
|
|
1093
|
+
const allcapAlt = allForms.toSorted((a, b) => b.length - a.length).map(escapeForRegex).join("|");
|
|
1094
|
+
patterns.push(`${allcapPrefix}(?:[ \\t]+|,[ \\t]*)(?:${allcapAlt})(?![${LOWER}])`);
|
|
1095
|
+
return patterns;
|
|
1096
|
+
};
|
|
1097
|
+
const CONNECTOR_RE = /^(?:a|and|und|et|e|y|i|&)$/i;
|
|
1098
|
+
const AND_TYPE_CONNECTOR_RE = /^(?:and|und|et)$/i;
|
|
1099
|
+
const UPPER_LETTER_RE = /^\p{Lu}/u;
|
|
1100
|
+
const COMPANY_SUFFIX_WORDS_RE = /^(?:Company|Co|Bank|Brothers|Bros|Sons|Group|Holdings|Trust|Partners|Associates|Corporation|Industries|Enterprises|Solutions|Systems|Services|Foundation|Institute)$/;
|
|
1101
|
+
const IN_NAME_PREPOSITION_RE = /^(?:of|the)$/i;
|
|
1102
|
+
const ENTITY_HEAD_WORD_RE = /^[\p{L}\p{M}&]+/u;
|
|
1103
|
+
const LEADING_CLAUSE_RE = /(?:^|\s)(?:by\s+and\s+between|is\s+between)\s+/giu;
|
|
1104
|
+
/**
|
|
1105
|
+
* Find the word ending just before `pos` in `text`,
|
|
1106
|
+
* skipping any whitespace (not newlines).
|
|
1107
|
+
* Returns null if no word is found (e.g., at start
|
|
1108
|
+
* of text, or preceded by non-word chars like ".").
|
|
1109
|
+
*/
|
|
1110
|
+
const findWordBefore = (text, pos) => {
|
|
1111
|
+
let scan = pos - 1;
|
|
1112
|
+
while (scan >= 0) {
|
|
1113
|
+
const ch = text.charAt(scan);
|
|
1114
|
+
if (ch === "\n" || !/\s/.test(ch)) break;
|
|
1115
|
+
scan--;
|
|
1116
|
+
}
|
|
1117
|
+
if (scan < 0 || text.charAt(scan) === "\n") return null;
|
|
1118
|
+
const wordEnd = scan + 1;
|
|
1119
|
+
while (scan >= 0 && /[\p{L}\p{M}&]/u.test(text.charAt(scan))) scan--;
|
|
1120
|
+
const wordStart = scan + 1;
|
|
1121
|
+
const word = text.slice(wordStart, wordEnd);
|
|
1122
|
+
if (word.length === 0) return null;
|
|
1123
|
+
return {
|
|
1124
|
+
word,
|
|
1125
|
+
start: wordStart
|
|
1126
|
+
};
|
|
1127
|
+
};
|
|
1128
|
+
/**
|
|
1129
|
+
* Count consecutive uppercase-starting words immediately
|
|
1130
|
+
* before `pos`. Stops at the first non-upper word or at
|
|
1131
|
+
* text/line start. Used to disambiguate "<First> <Last>
|
|
1132
|
+
* and <ORG>" from "<Multi-word Org> and <Continuation>".
|
|
1133
|
+
*/
|
|
1134
|
+
const countUpperWordsBefore = (fullText, pos) => {
|
|
1135
|
+
let count = 0;
|
|
1136
|
+
let scan = pos;
|
|
1137
|
+
while (scan > 0) {
|
|
1138
|
+
const found = findWordBefore(fullText, scan);
|
|
1139
|
+
if (!found) break;
|
|
1140
|
+
if (!UPPER_LETTER_RE.test(found.word)) break;
|
|
1141
|
+
count++;
|
|
1142
|
+
scan = found.start;
|
|
1143
|
+
}
|
|
1144
|
+
return count;
|
|
1145
|
+
};
|
|
1146
|
+
/**
|
|
1147
|
+
* Extend a match backward through uppercase words and
|
|
1148
|
+
* lowercase connectors. Stops at start of text,
|
|
1149
|
+
* newline, or a word that doesn't qualify.
|
|
1150
|
+
*
|
|
1151
|
+
* Connectors (a, and, und, et, ...) are only consumed
|
|
1152
|
+
* when there is a valid word before them — a trailing
|
|
1153
|
+
* connector at an entity boundary is not consumed.
|
|
1154
|
+
* For multi-char "and"-type connectors we additionally
|
|
1155
|
+
* refuse to cross when exactly two uppercase words
|
|
1156
|
+
* precede them ("First Last and ORG, Inc." shape) —
|
|
1157
|
+
* unless the match itself begins with a known company-
|
|
1158
|
+
* suffix word ("…and Company, Inc."), in which case
|
|
1159
|
+
* the chain belongs to one organisation. In that
|
|
1160
|
+
* suffix-mode we also cross in-name prepositions
|
|
1161
|
+
* ("Bank of America and Trust Company, Inc.").
|
|
1162
|
+
*/
|
|
1163
|
+
const extendBackward = (fullText, matchStart, options = {}) => {
|
|
1164
|
+
const headWord = ENTITY_HEAD_WORD_RE.exec(fullText.slice(matchStart))?.[0] ?? "";
|
|
1165
|
+
const suffixMode = options.forceSuffixMode === true || COMPANY_SUFFIX_WORDS_RE.test(headWord);
|
|
1166
|
+
let pos = matchStart;
|
|
1167
|
+
while (pos > 0) {
|
|
1168
|
+
const found = findWordBefore(fullText, pos);
|
|
1169
|
+
if (!found) break;
|
|
1170
|
+
const { word, start: wordStart } = found;
|
|
1171
|
+
const isUpper = UPPER_LETTER_RE.test(word);
|
|
1172
|
+
const isConnector = CONNECTOR_RE.test(word);
|
|
1173
|
+
const isInNamePrep = suffixMode && IN_NAME_PREPOSITION_RE.test(word);
|
|
1174
|
+
if (isUpper) pos = wordStart;
|
|
1175
|
+
else if (isConnector) {
|
|
1176
|
+
if (!suffixMode && AND_TYPE_CONNECTOR_RE.test(word) && countUpperWordsBefore(fullText, wordStart) === 2) break;
|
|
1177
|
+
const prev = findWordBefore(fullText, wordStart);
|
|
1178
|
+
if (!prev) break;
|
|
1179
|
+
if (!UPPER_LETTER_RE.test(prev.word)) break;
|
|
1180
|
+
pos = prev.start;
|
|
1181
|
+
} else if (isInNamePrep) {
|
|
1182
|
+
const prev = findWordBefore(fullText, wordStart);
|
|
1183
|
+
if (!prev) break;
|
|
1184
|
+
if (!UPPER_LETTER_RE.test(prev.word)) break;
|
|
1185
|
+
pos = prev.start;
|
|
1186
|
+
} else break;
|
|
1187
|
+
}
|
|
1188
|
+
return pos;
|
|
1189
|
+
};
|
|
1190
|
+
const trimLeadingClause = (text) => {
|
|
1191
|
+
let cut = -1;
|
|
1192
|
+
for (const match of text.matchAll(LEADING_CLAUSE_RE)) cut = match.index + match[0].length;
|
|
1193
|
+
if (cut <= 0) return {
|
|
1194
|
+
offset: 0,
|
|
1195
|
+
text
|
|
1196
|
+
};
|
|
1197
|
+
const trimmed = text.slice(cut);
|
|
1198
|
+
const leadingWs = trimmed.match(/^\s*/u)?.[0].length ?? 0;
|
|
1199
|
+
return {
|
|
1200
|
+
offset: cut + leadingWs,
|
|
1201
|
+
text: trimmed.slice(leadingWs)
|
|
1202
|
+
};
|
|
1203
|
+
};
|
|
1204
|
+
/**
|
|
1205
|
+
* Process legal form matches from the unified search.
|
|
1206
|
+
* Receives all matches; filters to the legal forms
|
|
1207
|
+
* slice via sliceStart/sliceEnd.
|
|
1208
|
+
*
|
|
1209
|
+
* The role-head trimming step reads per-language data from
|
|
1210
|
+
* a cache that `runPipeline` warms via `warmLegalRoleHeads()`
|
|
1211
|
+
* before calling this. Callers that invoke
|
|
1212
|
+
* `processLegalFormMatches` directly (without going through
|
|
1213
|
+
* `runPipeline`) must `await warmLegalRoleHeads()` first;
|
|
1214
|
+
* otherwise the trim falls back to a no-op and sentence-
|
|
1215
|
+
* fragment fixes do not apply.
|
|
1216
|
+
*/
|
|
1217
|
+
const processLegalFormMatches = (allMatches, sliceStart, sliceEnd, fullText) => {
|
|
1218
|
+
const results = [];
|
|
1219
|
+
for (const match of allMatches) {
|
|
1220
|
+
const idx = match.pattern;
|
|
1221
|
+
if (idx < sliceStart || idx >= sliceEnd) continue;
|
|
1222
|
+
const text = match.text.trimEnd();
|
|
1223
|
+
if (text.length < 5) continue;
|
|
1224
|
+
const roleHeads = getLegalRoleHeadsSync();
|
|
1225
|
+
const firstWordMatch = /^[\p{L}\p{M}]+(?:-[\p{L}\p{M}]+)*/u.exec(text);
|
|
1226
|
+
let processedStart = match.start;
|
|
1227
|
+
let processedText = text;
|
|
1228
|
+
let trimmed = false;
|
|
1229
|
+
const firstWordText = firstWordMatch?.[0] ?? "";
|
|
1230
|
+
const firstWordLeading = /^[\p{L}\p{M}]+/u.exec(firstWordText)?.[0] ?? "";
|
|
1231
|
+
if (firstWordMatch !== null && (roleHeads.has(firstWordText.toLowerCase()) || firstWordLeading.length > 0 && roleHeads.has(firstWordLeading.toLowerCase()))) {
|
|
1232
|
+
let suffixOffset = -1;
|
|
1233
|
+
for (const suffix of getAllLegalSuffixesSync()) {
|
|
1234
|
+
const idx = text.lastIndexOf(suffix);
|
|
1235
|
+
if (idx !== -1 && idx + suffix.length >= text.length - 1) {
|
|
1236
|
+
suffixOffset = idx;
|
|
1237
|
+
break;
|
|
1453
1238
|
}
|
|
1454
|
-
|
|
1455
|
-
|
|
1456
|
-
|
|
1457
|
-
|
|
1458
|
-
|
|
1459
|
-
|
|
1460
|
-
|
|
1461
|
-
|
|
1462
|
-
|
|
1463
|
-
|
|
1464
|
-
|
|
1465
|
-
|
|
1466
|
-
|
|
1467
|
-
|
|
1468
|
-
"char": "?",
|
|
1469
|
-
"name": "question mark",
|
|
1470
|
-
"code": "U+003F"
|
|
1471
|
-
},
|
|
1472
|
-
{
|
|
1473
|
-
"char": "?",
|
|
1474
|
-
"name": "full-width question mark",
|
|
1475
|
-
"code": "U+FF1F"
|
|
1476
|
-
},
|
|
1477
|
-
{
|
|
1478
|
-
"char": "⁇",
|
|
1479
|
-
"name": "double question mark",
|
|
1480
|
-
"code": "U+2047"
|
|
1481
|
-
},
|
|
1482
|
-
{
|
|
1483
|
-
"char": "⁈",
|
|
1484
|
-
"name": "question exclamation mark",
|
|
1485
|
-
"code": "U+2048"
|
|
1486
|
-
},
|
|
1487
|
-
{
|
|
1488
|
-
"char": "؟",
|
|
1489
|
-
"name": "Arabic question mark",
|
|
1490
|
-
"code": "U+061F"
|
|
1491
|
-
},
|
|
1492
|
-
{
|
|
1493
|
-
"char": "‽",
|
|
1494
|
-
"name": "interrobang",
|
|
1495
|
-
"code": "U+203D"
|
|
1239
|
+
}
|
|
1240
|
+
if (suffixOffset < 0) {} else {
|
|
1241
|
+
const midStart = firstWordMatch[0].length;
|
|
1242
|
+
const midEnd = suffixOffset;
|
|
1243
|
+
const midSection = text.slice(midStart, midEnd);
|
|
1244
|
+
const verbIndicators = getSentenceVerbIndicatorsSync();
|
|
1245
|
+
let lastVerbEndInMid = -1;
|
|
1246
|
+
for (const match of midSection.matchAll(/(?<![\p{L}\p{N}])[\p{L}\p{M}]+/gu)) if (match[0] !== void 0 && match.index !== void 0 && verbIndicators.has(match[0].toLowerCase())) lastVerbEndInMid = match.index + match[0].length;
|
|
1247
|
+
const digitAfterRole = /^\s+\d+(?:\.|\b)/u.test(midSection);
|
|
1248
|
+
let appositiveRoleHead = false;
|
|
1249
|
+
if (!digitAfterRole && lastVerbEndInMid === -1 && fullText) {
|
|
1250
|
+
const before = fullText.slice(Math.max(0, match.start - 40), match.start);
|
|
1251
|
+
const prevWord = /(?<![\p{L}\p{N}])(\p{L}[\p{L}\p{M}]*)\s*$/u.exec(before);
|
|
1252
|
+
if (prevWord !== null && getSentenceVerbIndicatorsSync().has(prevWord[1].toLowerCase())) appositiveRoleHead = true;
|
|
1496
1253
|
}
|
|
1497
|
-
|
|
1498
|
-
|
|
1499
|
-
|
|
1500
|
-
|
|
1501
|
-
|
|
1502
|
-
|
|
1503
|
-
|
|
1504
|
-
|
|
1505
|
-
|
|
1506
|
-
|
|
1507
|
-
|
|
1508
|
-
|
|
1509
|
-
|
|
1510
|
-
|
|
1511
|
-
|
|
1512
|
-
|
|
1513
|
-
|
|
1514
|
-
"name": "double exclamation mark",
|
|
1515
|
-
"code": "U+203C"
|
|
1516
|
-
},
|
|
1517
|
-
{
|
|
1518
|
-
"char": "¡",
|
|
1519
|
-
"name": "inverted exclamation mark",
|
|
1520
|
-
"code": "U+00A1"
|
|
1254
|
+
if (lastVerbEndInMid !== -1 || digitAfterRole || appositiveRoleHead) {
|
|
1255
|
+
const scanStart = lastVerbEndInMid !== -1 ? midStart + lastVerbEndInMid : midStart;
|
|
1256
|
+
const capRe = /(?<![\p{L}\p{N}])\p{Lu}[\p{L}\p{M}\p{N}]*/gu;
|
|
1257
|
+
capRe.lastIndex = scanStart;
|
|
1258
|
+
const clauseNouns = getClauseNounHeadsSync();
|
|
1259
|
+
let capMatch = null;
|
|
1260
|
+
for (let next = capRe.exec(text); next !== null; next = capRe.exec(text)) {
|
|
1261
|
+
if (next.index >= suffixOffset) break;
|
|
1262
|
+
const lc = next[0].toLowerCase();
|
|
1263
|
+
if (roleHeads.has(lc) || clauseNouns.has(lc)) continue;
|
|
1264
|
+
capMatch = next;
|
|
1265
|
+
break;
|
|
1266
|
+
}
|
|
1267
|
+
if (capMatch === null) continue;
|
|
1268
|
+
processedStart = match.start + capMatch.index;
|
|
1269
|
+
processedText = text.slice(capMatch.index);
|
|
1270
|
+
trimmed = true;
|
|
1521
1271
|
}
|
|
1522
|
-
|
|
1523
|
-
}
|
|
1524
|
-
"
|
|
1525
|
-
|
|
1526
|
-
|
|
1527
|
-
|
|
1528
|
-
|
|
1529
|
-
|
|
1530
|
-
|
|
1531
|
-
|
|
1532
|
-
|
|
1533
|
-
|
|
1534
|
-
|
|
1535
|
-
|
|
1536
|
-
|
|
1537
|
-
|
|
1538
|
-
|
|
1539
|
-
|
|
1540
|
-
|
|
1541
|
-
|
|
1542
|
-
|
|
1543
|
-
|
|
1544
|
-
|
|
1545
|
-
|
|
1272
|
+
}
|
|
1273
|
+
}
|
|
1274
|
+
if (processedText.includes("\n")) continue;
|
|
1275
|
+
let entityStart = processedStart;
|
|
1276
|
+
let entityText = processedText;
|
|
1277
|
+
if (fullText && !trimmed) {
|
|
1278
|
+
const extended = extendBackward(fullText, processedStart);
|
|
1279
|
+
if (extended < processedStart) {
|
|
1280
|
+
entityStart = extended;
|
|
1281
|
+
entityText = fullText.slice(extended, processedStart + processedText.length).trimEnd();
|
|
1282
|
+
}
|
|
1283
|
+
}
|
|
1284
|
+
const clauseTrim = trimLeadingClause(entityText);
|
|
1285
|
+
if (clauseTrim.offset > 0) {
|
|
1286
|
+
entityStart += clauseTrim.offset;
|
|
1287
|
+
entityText = clauseTrim.text;
|
|
1288
|
+
}
|
|
1289
|
+
const getPrefixInfo = (value) => {
|
|
1290
|
+
const prefixEnd = value.lastIndexOf(",") !== -1 ? value.lastIndexOf(",") : value.lastIndexOf(" ");
|
|
1291
|
+
return {
|
|
1292
|
+
prefixEnd,
|
|
1293
|
+
prefixPart: prefixEnd > 0 ? value.slice(0, prefixEnd).replace(/[^a-zA-ZÀ-ž]/g, "") : value.replace(/[^a-zA-ZÀ-ž]/g, "")
|
|
1294
|
+
};
|
|
1295
|
+
};
|
|
1296
|
+
let { prefixEnd, prefixPart } = getPrefixInfo(entityText);
|
|
1297
|
+
let isAllCapsMatch = prefixPart.length > 2 && prefixPart === prefixPart.toUpperCase();
|
|
1298
|
+
if (isAllCapsMatch && fullText) {
|
|
1299
|
+
const lineStart = fullText.lastIndexOf("\n", entityStart);
|
|
1300
|
+
const lineEnd = fullText.indexOf("\n", entityStart + entityText.length);
|
|
1301
|
+
const lineLetters = fullText.slice(lineStart + 1, lineEnd === -1 ? fullText.length : lineEnd).replace(/[^a-zA-ZÀ-ž]/g, "");
|
|
1302
|
+
const upperCount = [...lineLetters].filter((c) => c === c.toUpperCase()).length;
|
|
1303
|
+
if (lineLetters.length > 5 && upperCount / lineLetters.length >= .95) continue;
|
|
1304
|
+
if ((prefixPart.length > 0 ? entityText.slice(0, prefixEnd > 0 ? prefixEnd : entityText.length).trim().split(/\s+/).length : 0) > 3) {
|
|
1305
|
+
entityStart = match.start;
|
|
1306
|
+
entityText = text;
|
|
1307
|
+
({prefixEnd, prefixPart} = getPrefixInfo(entityText));
|
|
1308
|
+
isAllCapsMatch = prefixPart.length > 2 && prefixPart === prefixPart.toUpperCase();
|
|
1309
|
+
}
|
|
1310
|
+
} else if (isAllCapsMatch) continue;
|
|
1311
|
+
const lastSpace = entityText.lastIndexOf(" ");
|
|
1312
|
+
const rawSuffix = lastSpace !== -1 ? entityText.slice(lastSpace + 1) : "";
|
|
1313
|
+
const suffixClean = rawSuffix.replace(/[.,]/g, "");
|
|
1314
|
+
if (suffixClean.length > 0 && ROMAN_NUMERAL_RE.test(suffixClean)) continue;
|
|
1315
|
+
if (suffixClean.length <= 2 && !/\./.test(rawSuffix) && /[^\x00-\x7F]/.test(entityText.slice(0, lastSpace !== -1 ? lastSpace : entityText.length))) continue;
|
|
1316
|
+
results.push({
|
|
1317
|
+
start: entityStart,
|
|
1318
|
+
end: entityStart + entityText.length,
|
|
1319
|
+
label: "organization",
|
|
1320
|
+
text: entityText,
|
|
1321
|
+
score: .95,
|
|
1322
|
+
source: DETECTION_SOURCES.LEGAL_FORM
|
|
1323
|
+
});
|
|
1324
|
+
}
|
|
1325
|
+
return results;
|
|
1326
|
+
};
|
|
1327
|
+
//#endregion
|
|
1328
|
+
//#region src/detectors/coreference.ts
|
|
1329
|
+
/**
|
|
1330
|
+
* Case-insensitive set of every known legal-form suffix
|
|
1331
|
+
* with whitespace normalized away. Used by the
|
|
1332
|
+
* coreference scan to reject parenthetical define-term
|
|
1333
|
+
* captures whose alias is only a legal-form
|
|
1334
|
+
* abbreviation (e.g. "(« SAS »)" after "ACME SAS"),
|
|
1335
|
+
* which would otherwise propagate generic suffix
|
|
1336
|
+
* mentions as document-wide org aliases.
|
|
1337
|
+
*/
|
|
1338
|
+
const isLegalFormAlias = (alias) => {
|
|
1339
|
+
const normalized = alias.replace(/\s+/g, "").toLowerCase();
|
|
1340
|
+
if (normalized.length === 0) return false;
|
|
1341
|
+
for (const suffix of getKnownLegalSuffixes()) if (suffix.replace(/\s+/g, "").toLowerCase() === normalized) return true;
|
|
1342
|
+
return false;
|
|
1343
|
+
};
|
|
1344
|
+
/**
|
|
1345
|
+
* Load coreference definition patterns from per-language
|
|
1346
|
+
* JSON configs in src/data/. Uses the
|
|
1347
|
+
* language manifest for auto-discovery.
|
|
1348
|
+
*/
|
|
1349
|
+
const loadDefinitionPatterns = async () => {
|
|
1350
|
+
const patterns = [];
|
|
1351
|
+
const allRows = await loadLanguageConfigs("coreference", (mod) => {
|
|
1352
|
+
return mod.default ?? mod;
|
|
1353
|
+
});
|
|
1354
|
+
for (const rows of allRows) {
|
|
1355
|
+
if (!Array.isArray(rows)) {
|
|
1356
|
+
console.warn("[anonymize] coreference: unexpected config shape, skipping");
|
|
1357
|
+
continue;
|
|
1358
|
+
}
|
|
1359
|
+
for (const row of rows) try {
|
|
1360
|
+
patterns.push({ pattern: new RegExp(row.pattern, row.flags) });
|
|
1361
|
+
} catch (err) {
|
|
1362
|
+
console.warn(`[anonymize] coreference: invalid regex "${row.pattern}":`, err);
|
|
1363
|
+
}
|
|
1364
|
+
}
|
|
1365
|
+
return patterns;
|
|
1366
|
+
};
|
|
1367
|
+
/**
|
|
1368
|
+
* Load generic role terms that should NOT be treated
|
|
1369
|
+
* as PII coreferences. "Prodávající" (Seller),
|
|
1370
|
+
* "Kupující" (Buyer), etc. are legal roles, not
|
|
1371
|
+
* identifying information.
|
|
1372
|
+
*/
|
|
1373
|
+
const getRoleStopSet = async (ctx) => {
|
|
1374
|
+
if (ctx.roleStopSet) return ctx.roleStopSet;
|
|
1375
|
+
if (ctx.roleStopSetPromise) return ctx.roleStopSetPromise;
|
|
1376
|
+
const promise = (async () => {
|
|
1377
|
+
let result;
|
|
1378
|
+
try {
|
|
1379
|
+
const mod = await import("./generic-roles.mjs");
|
|
1380
|
+
const data = mod.default ?? mod;
|
|
1381
|
+
result = new Set(data.roles.map((r) => r.toLowerCase()));
|
|
1382
|
+
} catch {
|
|
1383
|
+
result = /* @__PURE__ */ new Set();
|
|
1384
|
+
}
|
|
1385
|
+
ctx.roleStopSet = result;
|
|
1386
|
+
return result;
|
|
1387
|
+
})();
|
|
1388
|
+
ctx.roleStopSetPromise = promise;
|
|
1389
|
+
return promise;
|
|
1390
|
+
};
|
|
1391
|
+
const getDefinitionPatterns = async (ctx) => {
|
|
1392
|
+
if (ctx.corefPatterns) return ctx.corefPatterns;
|
|
1393
|
+
if (ctx.corefPatternsPromise) return ctx.corefPatternsPromise;
|
|
1394
|
+
ctx.corefPatternsPromise = loadDefinitionPatterns();
|
|
1395
|
+
const patterns = await ctx.corefPatternsPromise;
|
|
1396
|
+
if (patterns.length === 0) {
|
|
1397
|
+
ctx.corefPatterns = patterns;
|
|
1398
|
+
if (!ctx.corefLoadAttempted) {
|
|
1399
|
+
ctx.corefLoadAttempted = true;
|
|
1400
|
+
console.warn("[anonymize] coreference: no definition patterns loaded; coreference detection will be inactive");
|
|
1401
|
+
}
|
|
1402
|
+
return patterns;
|
|
1403
|
+
}
|
|
1404
|
+
ctx.corefPatterns = patterns;
|
|
1405
|
+
return patterns;
|
|
1406
|
+
};
|
|
1407
|
+
const SEARCH_WINDOW = 200;
|
|
1408
|
+
/**
|
|
1409
|
+
* Check whether an alias has textual similarity to
|
|
1410
|
+
* the source entity. Prevents roles and structural
|
|
1411
|
+
* terms from being treated as name aliases.
|
|
1412
|
+
*
|
|
1413
|
+
* Three checks (any passes → similar):
|
|
1414
|
+
* 1. Word overlap: a word in the alias appears in the
|
|
1415
|
+
* entity (case-insensitive, min 2 chars)
|
|
1416
|
+
* 2. Initials: alias letters match first letters of
|
|
1417
|
+
* entity words ("TB" ↔ "Tomas Bata")
|
|
1418
|
+
* 3. Substring: alias is a substring of the entity
|
|
1419
|
+
* or vice versa (min 3 chars)
|
|
1420
|
+
*/
|
|
1421
|
+
const hasEntitySimilarity = (alias, entityText) => {
|
|
1422
|
+
const aliasLower = alias.toLowerCase();
|
|
1423
|
+
const entityLower = entityText.toLowerCase();
|
|
1424
|
+
if (aliasLower.length >= 3 && entityLower.includes(aliasLower)) return true;
|
|
1425
|
+
if (entityLower.length >= 3 && aliasLower.includes(entityLower)) return true;
|
|
1426
|
+
const aliasWords = aliasLower.split(/[\s.,;:'"()/-]+/).filter((w) => w.length >= 2);
|
|
1427
|
+
const entityWords = entityLower.split(/[\s.,;:'"()/-]+/).filter((w) => w.length >= 2);
|
|
1428
|
+
const entityWordSet = new Set(entityWords);
|
|
1429
|
+
for (const word of aliasWords) if (entityWordSet.has(word)) return true;
|
|
1430
|
+
if (/^[\p{Lu}]+$/u.test(alias) && alias.length >= 2 && alias.length <= entityWords.length) {
|
|
1431
|
+
for (let start = 0; start <= entityWords.length - alias.length; start++) if (entityWords.slice(start, start + alias.length).map((w) => w.charAt(0)).join("") === aliasLower) return true;
|
|
1432
|
+
}
|
|
1433
|
+
return false;
|
|
1434
|
+
};
|
|
1435
|
+
/**
|
|
1436
|
+
* Scan for defined-term patterns near known entities.
|
|
1437
|
+
*
|
|
1438
|
+
* Legal documents universally follow:
|
|
1439
|
+
* "Dr. Heinrich Muller (hereinafter 'the Seller')..."
|
|
1440
|
+
*
|
|
1441
|
+
* After NER detects the entity, this function scans for
|
|
1442
|
+
* definitional patterns within +/-200 chars and extracts
|
|
1443
|
+
* the alias. Returns alias + label pairs that can be added
|
|
1444
|
+
* to the gazetteer for a full-text re-scan.
|
|
1445
|
+
*/
|
|
1446
|
+
/**
|
|
1447
|
+
* Labels that can be the source of a coreference alias.
|
|
1448
|
+
* Only parties (person, organization) have defined-term
|
|
1449
|
+
* aliases in legal text. Dates, addresses, IDs do not.
|
|
1450
|
+
*/
|
|
1451
|
+
const COREF_SOURCE_LABELS = new Set(["person", "organization"]);
|
|
1452
|
+
const extractDefinedTerms = async (fullText, entities, ctx = defaultContext) => {
|
|
1453
|
+
const [definitionPatterns, roleStops] = await Promise.all([getDefinitionPatterns(ctx), getRoleStopSet(ctx)]);
|
|
1454
|
+
const terms = [];
|
|
1455
|
+
const seen = /* @__PURE__ */ new Set();
|
|
1456
|
+
const sorted = [...entities].sort((a, b) => a.start - b.start);
|
|
1457
|
+
for (const { pattern } of definitionPatterns) {
|
|
1458
|
+
pattern.lastIndex = 0;
|
|
1459
|
+
for (let match = pattern.exec(fullText); match !== null; match = pattern.exec(fullText)) {
|
|
1460
|
+
const alias = match[1]?.trim();
|
|
1461
|
+
if (!alias || alias.length < 2) continue;
|
|
1462
|
+
if (roleStops.has(alias.toLowerCase())) continue;
|
|
1463
|
+
if (isLegalFormAlias(alias)) continue;
|
|
1464
|
+
const defPos = match.index;
|
|
1465
|
+
let bestEntity = null;
|
|
1466
|
+
for (let i = sorted.length - 1; i >= 0; i--) {
|
|
1467
|
+
const e = sorted[i];
|
|
1468
|
+
if (e === void 0) continue;
|
|
1469
|
+
if (e.end > defPos) continue;
|
|
1470
|
+
if (defPos - e.end > SEARCH_WINDOW) break;
|
|
1471
|
+
if (!COREF_SOURCE_LABELS.has(e.label)) continue;
|
|
1472
|
+
bestEntity = e;
|
|
1473
|
+
break;
|
|
1474
|
+
}
|
|
1475
|
+
if (bestEntity === null) continue;
|
|
1476
|
+
const gapText = fullText.slice(bestEntity.end, defPos);
|
|
1477
|
+
if (/(?:;|\.(?=\s*(?:["'„‚(]*\p{Lu}|$)))/u.test(gapText)) continue;
|
|
1478
|
+
if (!hasEntitySimilarity(alias, bestEntity.text)) continue;
|
|
1479
|
+
const key = `${alias.toLowerCase()}::${bestEntity.label}`;
|
|
1480
|
+
if (seen.has(key)) continue;
|
|
1481
|
+
seen.add(key);
|
|
1482
|
+
terms.push({
|
|
1483
|
+
alias,
|
|
1484
|
+
label: bestEntity.label,
|
|
1485
|
+
definitionStart: defPos,
|
|
1486
|
+
sourceText: bestEntity.text
|
|
1487
|
+
});
|
|
1488
|
+
}
|
|
1489
|
+
}
|
|
1490
|
+
return terms;
|
|
1491
|
+
};
|
|
1492
|
+
/**
|
|
1493
|
+
* Check if a character is a Unicode word character
|
|
1494
|
+
* (letter, digit, or combining mark). Used for word
|
|
1495
|
+
* boundary checks in coreference matching.
|
|
1496
|
+
*/
|
|
1497
|
+
const isWordChar = (ch) => {
|
|
1498
|
+
if (ch === void 0) return false;
|
|
1499
|
+
return /[\p{L}\p{M}\p{N}]/u.test(ch);
|
|
1500
|
+
};
|
|
1501
|
+
/**
|
|
1502
|
+
* Find all occurrences of defined-term aliases in the
|
|
1503
|
+
* full text. Returns Entity spans for each match.
|
|
1504
|
+
*
|
|
1505
|
+
* Respects word boundaries: "Kupující" must not match
|
|
1506
|
+
* inside "Kupujícímu". A match is valid only if the
|
|
1507
|
+
* character before the start and after the end are NOT
|
|
1508
|
+
* word characters (letter/digit).
|
|
1509
|
+
*
|
|
1510
|
+
* Populates `ctx.corefSourceMap` with entries linking
|
|
1511
|
+
* each coref entity to its source entity text, for
|
|
1512
|
+
* consistent placeholder numbering.
|
|
1513
|
+
*/
|
|
1514
|
+
const findCoreferenceSpans = (fullText, terms, ctx = defaultContext) => {
|
|
1515
|
+
const results = [];
|
|
1516
|
+
for (const term of terms) {
|
|
1517
|
+
let searchFrom = 0;
|
|
1518
|
+
while (searchFrom < fullText.length) {
|
|
1519
|
+
const idx = fullText.indexOf(term.alias, searchFrom);
|
|
1520
|
+
if (idx === -1) break;
|
|
1521
|
+
const matchEnd = idx + term.alias.length;
|
|
1522
|
+
const charBefore = idx > 0 ? fullText[idx - 1] : void 0;
|
|
1523
|
+
const charAfter = fullText[matchEnd];
|
|
1524
|
+
if (isWordChar(charBefore) || isWordChar(charAfter)) {
|
|
1525
|
+
searchFrom = idx + 1;
|
|
1526
|
+
continue;
|
|
1527
|
+
}
|
|
1528
|
+
const entity = {
|
|
1529
|
+
start: idx,
|
|
1530
|
+
end: matchEnd,
|
|
1531
|
+
label: term.label,
|
|
1532
|
+
text: term.alias,
|
|
1533
|
+
score: .95,
|
|
1534
|
+
source: DETECTION_SOURCES.COREFERENCE
|
|
1535
|
+
};
|
|
1536
|
+
ctx.corefSourceMap.set(corefKey(entity), term.sourceText);
|
|
1537
|
+
results.push(entity);
|
|
1538
|
+
searchFrom = matchEnd;
|
|
1539
|
+
}
|
|
1540
|
+
}
|
|
1541
|
+
return results;
|
|
1542
|
+
};
|
|
1543
|
+
//#endregion
|
|
1544
|
+
//#region src/detectors/gazetteer.ts
|
|
1545
|
+
const MAX_EDIT_DISTANCE = 2;
|
|
1546
|
+
const MIN_FUZZY_LENGTH = 4;
|
|
1547
|
+
const MAX_PREFIX_OVERSHOOT = 7;
|
|
1548
|
+
/**
|
|
1549
|
+
* Collect all searchable strings (canonical + variants)
|
|
1550
|
+
* from gazetteer entries, mapped to their labels and
|
|
1551
|
+
* entry IDs.
|
|
1552
|
+
*/
|
|
1553
|
+
const buildSearchTerms = (entries) => {
|
|
1554
|
+
const terms = /* @__PURE__ */ new Map();
|
|
1555
|
+
for (const entry of entries) {
|
|
1556
|
+
const meta = {
|
|
1557
|
+
label: entry.label,
|
|
1558
|
+
entryId: entry.id
|
|
1559
|
+
};
|
|
1560
|
+
terms.set(entry.canonical, meta);
|
|
1561
|
+
for (const variant of entry.variants) terms.set(variant, meta);
|
|
1562
|
+
}
|
|
1563
|
+
return terms;
|
|
1564
|
+
};
|
|
1565
|
+
/**
|
|
1566
|
+
* Build TextSearch-compatible patterns from gazetteer
|
|
1567
|
+
* entries. Returns:
|
|
1568
|
+
* - Exact literal patterns for all terms
|
|
1569
|
+
* - Fuzzy patterns (distance: 2) for terms >= 4 chars
|
|
1570
|
+
* - Parallel metadata arrays for post-processing
|
|
1571
|
+
*
|
|
1572
|
+
* Patterns are ordered: all exact first, then all
|
|
1573
|
+
* fuzzy. The isFuzzy array marks which are which.
|
|
1574
|
+
*/
|
|
1575
|
+
const buildGazetteerPatterns = (entries) => {
|
|
1576
|
+
const terms = buildSearchTerms(entries);
|
|
1577
|
+
const patterns = [];
|
|
1578
|
+
const labels = [];
|
|
1579
|
+
const isFuzzy = [];
|
|
1580
|
+
for (const [term, meta] of terms) {
|
|
1581
|
+
patterns.push({
|
|
1582
|
+
pattern: term,
|
|
1583
|
+
literal: true,
|
|
1584
|
+
wholeWords: false
|
|
1585
|
+
});
|
|
1586
|
+
labels.push(meta.label);
|
|
1587
|
+
isFuzzy.push(false);
|
|
1588
|
+
}
|
|
1589
|
+
for (const [term, meta] of terms) {
|
|
1590
|
+
if (term.length < MIN_FUZZY_LENGTH) continue;
|
|
1591
|
+
patterns.push({
|
|
1592
|
+
pattern: term,
|
|
1593
|
+
distance: MAX_EDIT_DISTANCE
|
|
1594
|
+
});
|
|
1595
|
+
labels.push(meta.label);
|
|
1596
|
+
isFuzzy.push(true);
|
|
1597
|
+
}
|
|
1598
|
+
return {
|
|
1599
|
+
patterns,
|
|
1600
|
+
data: {
|
|
1601
|
+
labels,
|
|
1602
|
+
isFuzzy
|
|
1603
|
+
}
|
|
1604
|
+
};
|
|
1605
|
+
};
|
|
1606
|
+
/**
|
|
1607
|
+
* Process gazetteer matches from the unified literal
|
|
1608
|
+
* search. Receives all matches; filters to the
|
|
1609
|
+
* gazetteer slice via sliceStart/sliceEnd.
|
|
1610
|
+
*
|
|
1611
|
+
* Exact matches get score 0.9; fuzzy matches get
|
|
1612
|
+
* 0.85. Fuzzy matches that overlap an exact match
|
|
1613
|
+
* are dropped.
|
|
1614
|
+
*
|
|
1615
|
+
* For exact matches, attempts prefix extension for
|
|
1616
|
+
* legal suffixes ("a.s.", "GmbH", "s.r.o." after
|
|
1617
|
+
* the matched term).
|
|
1618
|
+
*/
|
|
1619
|
+
const processGazetteerMatches = (allMatches, sliceStart, sliceEnd, fullText, data) => {
|
|
1620
|
+
const results = [];
|
|
1621
|
+
const exactSpans = [];
|
|
1622
|
+
for (const match of allMatches) {
|
|
1623
|
+
const idx = match.pattern;
|
|
1624
|
+
if (idx < sliceStart || idx >= sliceEnd) continue;
|
|
1625
|
+
const localIdx = idx - sliceStart;
|
|
1626
|
+
if (data.isFuzzy[localIdx]) continue;
|
|
1627
|
+
const label = data.labels[localIdx];
|
|
1628
|
+
if (!label) continue;
|
|
1629
|
+
const extended = tryPrefixExtension(fullText, match.start, match.end);
|
|
1630
|
+
const end = extended?.end ?? match.end;
|
|
1631
|
+
const text = extended?.text ?? fullText.slice(match.start, match.end);
|
|
1632
|
+
exactSpans.push({
|
|
1633
|
+
start: match.start,
|
|
1634
|
+
end
|
|
1635
|
+
});
|
|
1636
|
+
results.push({
|
|
1637
|
+
start: match.start,
|
|
1638
|
+
end,
|
|
1639
|
+
label,
|
|
1640
|
+
text,
|
|
1641
|
+
score: .9,
|
|
1642
|
+
source: DETECTION_SOURCES.GAZETTEER
|
|
1643
|
+
});
|
|
1644
|
+
}
|
|
1645
|
+
for (const match of allMatches) {
|
|
1646
|
+
const idx = match.pattern;
|
|
1647
|
+
if (idx < sliceStart || idx >= sliceEnd) continue;
|
|
1648
|
+
const localIdx = idx - sliceStart;
|
|
1649
|
+
if (!data.isFuzzy[localIdx]) continue;
|
|
1650
|
+
if (match.distance === 0) continue;
|
|
1651
|
+
const label = data.labels[localIdx];
|
|
1652
|
+
if (!label) continue;
|
|
1653
|
+
if (exactSpans.some((e) => match.start < e.end && match.end > e.start)) continue;
|
|
1654
|
+
const matchText = fullText.slice(match.start, match.end);
|
|
1655
|
+
results.push({
|
|
1656
|
+
start: match.start,
|
|
1657
|
+
end: match.end,
|
|
1658
|
+
label,
|
|
1659
|
+
text: matchText,
|
|
1660
|
+
score: .85,
|
|
1661
|
+
source: DETECTION_SOURCES.GAZETTEER
|
|
1662
|
+
});
|
|
1663
|
+
}
|
|
1664
|
+
return results;
|
|
1665
|
+
};
|
|
1666
|
+
/**
|
|
1667
|
+
* Try to extend an exact match to capture one
|
|
1668
|
+
* trailing token (max 6 chars) that may be a legal
|
|
1669
|
+
* entity suffix (e.g., "a.s.", "GmbH", "s.r.o.").
|
|
1670
|
+
*
|
|
1671
|
+
* Does not validate the token against a legal-forms
|
|
1672
|
+
* list; false extensions are filtered by mergeAndDedup
|
|
1673
|
+
* when a legal-form detector produces a competing
|
|
1674
|
+
* entity with the correct span.
|
|
1675
|
+
*/
|
|
1676
|
+
const tryPrefixExtension = (fullText, start, end) => {
|
|
1677
|
+
const maxEnd = Math.min(end + MAX_PREFIX_OVERSHOOT, fullText.length);
|
|
1678
|
+
if (maxEnd <= end + 1) return null;
|
|
1679
|
+
const after = fullText.slice(end, maxEnd);
|
|
1680
|
+
if (!after.startsWith(" ")) return null;
|
|
1681
|
+
const nextSpace = after.indexOf(" ", 1);
|
|
1682
|
+
const suffixEnd = nextSpace !== -1 ? nextSpace : after.length;
|
|
1683
|
+
if (suffixEnd <= 1) return null;
|
|
1684
|
+
const newEnd = end + suffixEnd;
|
|
1685
|
+
return {
|
|
1686
|
+
end: newEnd,
|
|
1687
|
+
text: fullText.slice(start, newEnd)
|
|
1688
|
+
};
|
|
1689
|
+
};
|
|
1690
|
+
//#endregion
|
|
1691
|
+
//#region src/util/text.ts
|
|
1692
|
+
/**
|
|
1693
|
+
* Shared text utilities for detectors.
|
|
1694
|
+
*
|
|
1695
|
+
* Extracted from names.ts and deny-list.ts to avoid
|
|
1696
|
+
* duplicating regex constants and helper functions.
|
|
1697
|
+
*/
|
|
1698
|
+
/** Matches a string that starts with an uppercase letter. */
|
|
1699
|
+
const UPPER_START_RE = /^\p{Lu}/u;
|
|
1700
|
+
/** Matches a string consisting entirely of uppercase letters. */
|
|
1701
|
+
const ALL_UPPER_RE = /^\p{Lu}+$/u;
|
|
1702
|
+
const SENTENCE_END_RE = /[.!?]/;
|
|
1703
|
+
/**
|
|
1704
|
+
* Detect whether a position is at the start of a sentence.
|
|
1705
|
+
* Looks backward past whitespace for sentence-ending
|
|
1706
|
+
* punctuation (.!?). Position 0 and positions preceded
|
|
1707
|
+
* only by whitespace are considered sentence starts.
|
|
1708
|
+
*/
|
|
1709
|
+
const isSentenceStart = (text, pos) => {
|
|
1710
|
+
if (pos === 0) return true;
|
|
1711
|
+
let i = pos - 1;
|
|
1712
|
+
while (i >= 0 && /\s/.test(text[i] ?? "")) i--;
|
|
1713
|
+
if (i < 0) return true;
|
|
1714
|
+
return SENTENCE_END_RE.test(text[i] ?? "");
|
|
1715
|
+
};
|
|
1716
|
+
//#endregion
|
|
1717
|
+
//#region src/detectors/names.ts
|
|
1718
|
+
const getCorpus = (ctx) => ctx.nameCorpus;
|
|
1719
|
+
const getNameCorpusFirstNames = (ctx = defaultContext) => ctx.nameCorpus?.firstNamesList ?? [];
|
|
1720
|
+
const getNameCorpusSurnames = (ctx = defaultContext) => ctx.nameCorpus?.surnamesList ?? [];
|
|
1721
|
+
const getNameCorpusTitles = (ctx = defaultContext) => ctx.nameCorpus?.titlesList ?? [];
|
|
1722
|
+
/**
|
|
1723
|
+
* Load name corpus data from injected dictionaries
|
|
1724
|
+
* and legacy config files. Merges all sources.
|
|
1725
|
+
*
|
|
1726
|
+
* Safe to call multiple times; only loads once per
|
|
1727
|
+
* context. Must be called before detectNameCorpus or
|
|
1728
|
+
* the getNameCorpus*() accessors are used.
|
|
1729
|
+
*
|
|
1730
|
+
* @param dictionaries Optional pre-loaded dictionaries
|
|
1731
|
+
* with per-language first names and surnames. When
|
|
1732
|
+
* omitted, only legacy config files are used.
|
|
1733
|
+
*/
|
|
1734
|
+
const initNameCorpus = (ctx = defaultContext, dictionaries) => {
|
|
1735
|
+
if (ctx.nameCorpusPromise) return ctx.nameCorpusPromise;
|
|
1736
|
+
const promise = (async () => {
|
|
1737
|
+
try {
|
|
1738
|
+
const [legacyFirstMod, legacySurnameMod, titleMod, exclusionMod] = await Promise.all([
|
|
1739
|
+
import("./names-first.mjs"),
|
|
1740
|
+
import("./names-surnames.mjs"),
|
|
1741
|
+
import("./names-title-tokens.mjs"),
|
|
1742
|
+
import("./names-exclusions.mjs")
|
|
1743
|
+
]);
|
|
1744
|
+
const firstNames = [...legacyFirstMod.default.names];
|
|
1745
|
+
if (dictionaries?.firstNames) for (const names of Object.values(dictionaries.firstNames)) for (const name of names) firstNames.push(name);
|
|
1746
|
+
const surnames = [...legacySurnameMod.default.names];
|
|
1747
|
+
if (dictionaries?.surnames) for (const names of Object.values(dictionaries.surnames)) for (const name of names) surnames.push(name);
|
|
1748
|
+
const dedup = (arr) => {
|
|
1749
|
+
const seen = /* @__PURE__ */ new Set();
|
|
1750
|
+
const result = [];
|
|
1751
|
+
for (const item of arr) {
|
|
1752
|
+
if (seen.has(item)) continue;
|
|
1753
|
+
seen.add(item);
|
|
1754
|
+
result.push(item);
|
|
1546
1755
|
}
|
|
1547
|
-
|
|
1548
|
-
|
|
1549
|
-
|
|
1550
|
-
|
|
1551
|
-
|
|
1552
|
-
|
|
1553
|
-
|
|
1554
|
-
|
|
1555
|
-
|
|
1556
|
-
|
|
1557
|
-
|
|
1558
|
-
|
|
1559
|
-
|
|
1756
|
+
return result;
|
|
1757
|
+
};
|
|
1758
|
+
const dedupFirst = dedup(firstNames);
|
|
1759
|
+
const dedupSurnames = dedup(surnames);
|
|
1760
|
+
const titles = titleMod.default.tokens;
|
|
1761
|
+
const exclusions = exclusionMod.default.words;
|
|
1762
|
+
ctx.nameCorpus = {
|
|
1763
|
+
firstNames: Object.freeze(new Set(dedupFirst)),
|
|
1764
|
+
surnames: Object.freeze(new Set(dedupSurnames)),
|
|
1765
|
+
titleTokens: Object.freeze(new Set(titles)),
|
|
1766
|
+
excludedWords: Object.freeze(new Set(exclusions)),
|
|
1767
|
+
firstNamesList: Object.freeze(dedupFirst),
|
|
1768
|
+
surnamesList: Object.freeze(dedupSurnames),
|
|
1769
|
+
titlesList: Object.freeze(titles),
|
|
1770
|
+
excludedList: Object.freeze(exclusions)
|
|
1771
|
+
};
|
|
1772
|
+
} catch (err) {
|
|
1773
|
+
ctx.nameCorpusPromise = null;
|
|
1774
|
+
console.warn("[anonymize] Failed to load name corpus JSON — name detection disabled:", err);
|
|
1775
|
+
}
|
|
1776
|
+
})();
|
|
1777
|
+
ctx.nameCorpusPromise = promise;
|
|
1778
|
+
return promise;
|
|
1779
|
+
};
|
|
1780
|
+
const INFLECTION_SUFFIXES = [
|
|
1781
|
+
"ovi",
|
|
1782
|
+
"em",
|
|
1783
|
+
"om",
|
|
1784
|
+
"ou",
|
|
1785
|
+
"é",
|
|
1786
|
+
"a",
|
|
1787
|
+
"u"
|
|
1788
|
+
];
|
|
1789
|
+
/**
|
|
1790
|
+
* Strip common Czech/Slovak case suffixes from a token.
|
|
1791
|
+
* Returns candidate base forms if stripping produces a
|
|
1792
|
+
* plausible name (capitalised, length >= 3).
|
|
1793
|
+
*
|
|
1794
|
+
* For the "-ou" instrumental feminine suffix, also yields
|
|
1795
|
+
* base + "a" (e.g., "Editou" → "Edit" and "Edita")
|
|
1796
|
+
* because Czech feminine names decline -a → -ou.
|
|
1797
|
+
*/
|
|
1798
|
+
const stripInflection = (token) => {
|
|
1799
|
+
const candidates = [];
|
|
1800
|
+
for (const suffix of INFLECTION_SUFFIXES) if (token.length > suffix.length + 2 && token.endsWith(suffix)) {
|
|
1801
|
+
const base = token.slice(0, -suffix.length);
|
|
1802
|
+
if (/^\p{Lu}/u.test(base)) {
|
|
1803
|
+
candidates.push(base);
|
|
1804
|
+
if (suffix === "ou" || suffix === "é" || suffix === "u") candidates.push(`${base}a`);
|
|
1560
1805
|
}
|
|
1561
1806
|
}
|
|
1807
|
+
return candidates;
|
|
1808
|
+
};
|
|
1809
|
+
const TOKEN_TYPE = {
|
|
1810
|
+
NAME: "name",
|
|
1811
|
+
SURNAME: "surname",
|
|
1812
|
+
TITLE: "title",
|
|
1813
|
+
ABBREVIATION: "abbreviation",
|
|
1814
|
+
CAPITALIZED: "capitalized",
|
|
1815
|
+
OTHER: "other"
|
|
1816
|
+
};
|
|
1817
|
+
const PERSON_CHAIN_BREAK_RE$1 = /[!?;:]/u;
|
|
1818
|
+
const isInitialContinuationGap$1 = (text, gap) => /^\p{Lu}$/u.test(text) && /^\.[^\S\n]{1,2}$/u.test(gap) || /^[^\S\n]{1,2}(?:\p{Lu}\.[^\S\n]{1,2})+$/u.test(gap);
|
|
1819
|
+
/**
|
|
1820
|
+
* Check if a token is in the first-name set, either
|
|
1821
|
+
* directly or after stripping Czech/Slovak inflection.
|
|
1822
|
+
*/
|
|
1823
|
+
const isFirstNameToken = (token, corpus) => {
|
|
1824
|
+
if (corpus.firstNames.has(token)) return true;
|
|
1825
|
+
return stripInflection(token).some((b) => corpus.firstNames.has(b));
|
|
1826
|
+
};
|
|
1827
|
+
/**
|
|
1828
|
+
* Check if a token is in the surname set, either
|
|
1829
|
+
* directly or after stripping Czech/Slovak inflection.
|
|
1830
|
+
*/
|
|
1831
|
+
const isSurnameToken = (token, corpus) => {
|
|
1832
|
+
if (corpus.surnames.has(token)) return true;
|
|
1833
|
+
return stripInflection(token).some((b) => corpus.surnames.has(b));
|
|
1834
|
+
};
|
|
1835
|
+
/**
|
|
1836
|
+
* Check if a token looks like a single-letter
|
|
1837
|
+
* abbreviation: "J.", "M.", etc.
|
|
1838
|
+
*/
|
|
1839
|
+
const isAbbreviation = (token) => token.length === 2 && /^\p{Lu}$/u.test(token[0] ?? "") && token[1] === ".";
|
|
1840
|
+
const segmenter$1 = new Intl.Segmenter(void 0, { granularity: "word" });
|
|
1841
|
+
/**
|
|
1842
|
+
* Split text into word segments using Intl.Segmenter.
|
|
1843
|
+
* Only returns segments flagged as words.
|
|
1844
|
+
*/
|
|
1845
|
+
const segmentWords = (fullText) => {
|
|
1846
|
+
const words = [];
|
|
1847
|
+
for (const seg of segmenter$1.segment(fullText)) if (seg.isWordLike) words.push({
|
|
1848
|
+
text: seg.segment,
|
|
1849
|
+
start: seg.index,
|
|
1850
|
+
end: seg.index + seg.segment.length
|
|
1851
|
+
});
|
|
1852
|
+
return words;
|
|
1853
|
+
};
|
|
1854
|
+
/** NAME or SURNAME — both represent corpus-matched tokens */
|
|
1855
|
+
const isCorpusMatch = (type) => type === TOKEN_TYPE.NAME || type === TOKEN_TYPE.SURNAME;
|
|
1856
|
+
const classifyToken = (word, corpus) => {
|
|
1857
|
+
const { text, start, end } = word;
|
|
1858
|
+
const lower = text.toLowerCase();
|
|
1859
|
+
const stripped = text.endsWith(".") ? text.slice(0, -1).toLowerCase() : lower;
|
|
1860
|
+
if (corpus.titleTokens.has(stripped)) return {
|
|
1861
|
+
text,
|
|
1862
|
+
type: TOKEN_TYPE.TITLE,
|
|
1863
|
+
start,
|
|
1864
|
+
end
|
|
1865
|
+
};
|
|
1866
|
+
if (isAbbreviation(text)) return {
|
|
1867
|
+
text,
|
|
1868
|
+
type: TOKEN_TYPE.ABBREVIATION,
|
|
1869
|
+
start,
|
|
1870
|
+
end
|
|
1871
|
+
};
|
|
1872
|
+
if (corpus.excludedWords.has(lower)) return {
|
|
1873
|
+
text,
|
|
1874
|
+
type: TOKEN_TYPE.OTHER,
|
|
1875
|
+
start,
|
|
1876
|
+
end
|
|
1877
|
+
};
|
|
1878
|
+
if (text.length < 3) return {
|
|
1879
|
+
text,
|
|
1880
|
+
type: TOKEN_TYPE.OTHER,
|
|
1881
|
+
start,
|
|
1882
|
+
end
|
|
1883
|
+
};
|
|
1884
|
+
if (text.length > 3 && ALL_UPPER_RE.test(text)) return {
|
|
1885
|
+
text,
|
|
1886
|
+
type: TOKEN_TYPE.OTHER,
|
|
1887
|
+
start,
|
|
1888
|
+
end
|
|
1889
|
+
};
|
|
1890
|
+
if (!UPPER_START_RE.test(text)) return {
|
|
1891
|
+
text,
|
|
1892
|
+
type: TOKEN_TYPE.OTHER,
|
|
1893
|
+
start,
|
|
1894
|
+
end
|
|
1895
|
+
};
|
|
1896
|
+
if (isFirstNameToken(text, corpus)) return {
|
|
1897
|
+
text,
|
|
1898
|
+
type: TOKEN_TYPE.NAME,
|
|
1899
|
+
start,
|
|
1900
|
+
end
|
|
1901
|
+
};
|
|
1902
|
+
if (isSurnameToken(text, corpus)) return {
|
|
1903
|
+
text,
|
|
1904
|
+
type: TOKEN_TYPE.SURNAME,
|
|
1905
|
+
start,
|
|
1906
|
+
end
|
|
1907
|
+
};
|
|
1908
|
+
return {
|
|
1909
|
+
text,
|
|
1910
|
+
type: TOKEN_TYPE.CAPITALIZED,
|
|
1911
|
+
start,
|
|
1912
|
+
end
|
|
1913
|
+
};
|
|
1562
1914
|
};
|
|
1563
|
-
//#endregion
|
|
1564
|
-
//#region src/util/char-groups.ts
|
|
1565
1915
|
/**
|
|
1566
|
-
*
|
|
1567
|
-
*
|
|
1568
|
-
* Centralized Unicode character equivalence groups.
|
|
1916
|
+
* Detect person names by looking up tokens against the
|
|
1917
|
+
* name corpus, then chaining adjacent name-like tokens.
|
|
1569
1918
|
*
|
|
1570
|
-
*
|
|
1571
|
-
*
|
|
1572
|
-
* (dashes, spaces, quotes, etc.).
|
|
1919
|
+
* Requires initNameCorpus() to have been called first.
|
|
1920
|
+
* If not initialized, returns an empty array.
|
|
1573
1921
|
*
|
|
1574
|
-
*
|
|
1575
|
-
*
|
|
1922
|
+
* Scoring:
|
|
1923
|
+
* TITLE + NAME/SURNAME → 0.95
|
|
1924
|
+
* NAME + NAME/SURNAME → 0.9
|
|
1925
|
+
* SURNAME + NAME/SURNAME → 0.9
|
|
1926
|
+
* NAME + CAPITALIZED → 0.7
|
|
1927
|
+
* ABBREVIATION + NAME → 0.7
|
|
1928
|
+
* Standalone NAME → 0.5 (low confidence)
|
|
1929
|
+
* Standalone SURNAME → skip (too ambiguous)
|
|
1576
1930
|
*/
|
|
1577
|
-
|
|
1578
|
-
const
|
|
1579
|
-
|
|
1580
|
-
const
|
|
1931
|
+
const detectNameCorpus = (fullText, ctx = defaultContext) => {
|
|
1932
|
+
const corpus = getCorpus(ctx);
|
|
1933
|
+
if (!corpus) return [];
|
|
1934
|
+
const tokens = segmentWords(fullText).map((w) => classifyToken(w, corpus));
|
|
1935
|
+
const entities = [];
|
|
1936
|
+
const consumed = /* @__PURE__ */ new Set();
|
|
1937
|
+
for (let i = 0; i < tokens.length; i++) {
|
|
1938
|
+
if (consumed.has(i)) continue;
|
|
1939
|
+
const token = tokens[i];
|
|
1940
|
+
if (!token) continue;
|
|
1941
|
+
if (token.type !== TOKEN_TYPE.TITLE && token.type !== TOKEN_TYPE.NAME && token.type !== TOKEN_TYPE.SURNAME && token.type !== TOKEN_TYPE.ABBREVIATION) continue;
|
|
1942
|
+
const MAX_CHAIN = 5;
|
|
1943
|
+
const chain = [token];
|
|
1944
|
+
let j = i + 1;
|
|
1945
|
+
while (j < tokens.length && chain.length < MAX_CHAIN) {
|
|
1946
|
+
const next = tokens[j];
|
|
1947
|
+
if (!next) break;
|
|
1948
|
+
const prev = chain.at(-1);
|
|
1949
|
+
if (prev) {
|
|
1950
|
+
const gap = fullText.slice(prev.end, next.start);
|
|
1951
|
+
const breaksOnPeriod = gap.includes(".") && !isInitialContinuationGap$1(prev.text, gap);
|
|
1952
|
+
if (gap.includes("\n") || PERSON_CHAIN_BREAK_RE$1.test(gap) || breaksOnPeriod) break;
|
|
1953
|
+
}
|
|
1954
|
+
if (next.type === TOKEN_TYPE.NAME || next.type === TOKEN_TYPE.SURNAME || next.type === TOKEN_TYPE.TITLE || next.type === TOKEN_TYPE.ABBREVIATION || next.type === TOKEN_TYPE.CAPITALIZED) {
|
|
1955
|
+
chain.push(next);
|
|
1956
|
+
j++;
|
|
1957
|
+
} else break;
|
|
1958
|
+
}
|
|
1959
|
+
const hasTitle = chain.some((t) => t.type === TOKEN_TYPE.TITLE);
|
|
1960
|
+
const hasCorpusName = chain.some((t) => isCorpusMatch(t.type));
|
|
1961
|
+
const hasFirstName = chain.some((t) => t.type === TOKEN_TYPE.NAME);
|
|
1962
|
+
const hasAbbreviation = chain.some((t) => t.type === TOKEN_TYPE.ABBREVIATION);
|
|
1963
|
+
const corpusCount = chain.filter((t) => isCorpusMatch(t.type)).length;
|
|
1964
|
+
const capitalizedCount = chain.filter((t) => t.type === TOKEN_TYPE.CAPITALIZED).length;
|
|
1965
|
+
let score = 0;
|
|
1966
|
+
if (hasTitle && hasCorpusName) score = .95;
|
|
1967
|
+
else if (corpusCount >= 2) score = .9;
|
|
1968
|
+
else if (hasCorpusName && capitalizedCount > 0) score = .7;
|
|
1969
|
+
else if (hasAbbreviation && hasCorpusName) score = .7;
|
|
1970
|
+
else if (hasFirstName && chain.length === 1) {
|
|
1971
|
+
if (isSentenceStart(fullText, token.start)) continue;
|
|
1972
|
+
score = .5;
|
|
1973
|
+
} else if (!hasFirstName && chain.length === 1 && chain[0]?.type === TOKEN_TYPE.SURNAME) continue;
|
|
1974
|
+
else if (hasTitle && chain.length === 1) continue;
|
|
1975
|
+
else {
|
|
1976
|
+
if (!hasCorpusName) continue;
|
|
1977
|
+
score = .5;
|
|
1978
|
+
}
|
|
1979
|
+
const first = chain.at(0);
|
|
1980
|
+
const last = chain.at(-1);
|
|
1981
|
+
if (!first || !last) continue;
|
|
1982
|
+
const start = first.start;
|
|
1983
|
+
const end = last.end;
|
|
1984
|
+
const text = fullText.slice(start, end);
|
|
1985
|
+
for (let k = i; k < i + chain.length; k++) consumed.add(k);
|
|
1986
|
+
entities.push({
|
|
1987
|
+
start,
|
|
1988
|
+
end,
|
|
1989
|
+
label: "person",
|
|
1990
|
+
text,
|
|
1991
|
+
score,
|
|
1992
|
+
source: DETECTION_SOURCES.REGEX
|
|
1993
|
+
});
|
|
1994
|
+
}
|
|
1995
|
+
return entities;
|
|
1996
|
+
};
|
|
1997
|
+
//#endregion
|
|
1998
|
+
//#region src/config/titles.ts
|
|
1581
1999
|
/**
|
|
1582
|
-
*
|
|
1583
|
-
*
|
|
2000
|
+
* Academic and professional title prefixes.
|
|
2001
|
+
* Plain text; the detector auto-escapes for regex.
|
|
2002
|
+
* Sorted longest-first at build time.
|
|
1584
2003
|
*/
|
|
1585
|
-
const
|
|
1586
|
-
|
|
1587
|
-
|
|
1588
|
-
|
|
1589
|
-
|
|
2004
|
+
const TITLE_PREFIXES = [
|
|
2005
|
+
"Ing.",
|
|
2006
|
+
"Mgr.",
|
|
2007
|
+
"MgA.",
|
|
2008
|
+
"Bc.",
|
|
2009
|
+
"BcA.",
|
|
2010
|
+
"JUDr.",
|
|
2011
|
+
"MUDr.",
|
|
2012
|
+
"MVDr.",
|
|
2013
|
+
"MDDr.",
|
|
2014
|
+
"PhDr.",
|
|
2015
|
+
"RNDr.",
|
|
2016
|
+
"PaedDr.",
|
|
2017
|
+
"ThDr.",
|
|
2018
|
+
"ThLic.",
|
|
2019
|
+
"ICDr.",
|
|
2020
|
+
"RSDr.",
|
|
2021
|
+
"PharmDr.",
|
|
2022
|
+
"artD.",
|
|
2023
|
+
"akad.",
|
|
2024
|
+
"doc.",
|
|
2025
|
+
"prof.",
|
|
2026
|
+
"ao. Univ.-Prof.",
|
|
2027
|
+
"o. Univ.-Prof.",
|
|
2028
|
+
"Univ.-Prof.",
|
|
2029
|
+
"Hon.-Prof.",
|
|
2030
|
+
"em. Prof.",
|
|
2031
|
+
"Dr. med. dent.",
|
|
2032
|
+
"Dr. med. vet.",
|
|
2033
|
+
"Dr. med.",
|
|
2034
|
+
"Dr. rer. nat.",
|
|
2035
|
+
"Dr. rer. soc.",
|
|
2036
|
+
"Dr. rer. pol.",
|
|
2037
|
+
"Dr. sc. tech.",
|
|
2038
|
+
"Dr. sc. nat.",
|
|
2039
|
+
"Dr. sc. hum.",
|
|
2040
|
+
"Dr. iur.",
|
|
2041
|
+
"Dr. jur.",
|
|
2042
|
+
"Dr. theol.",
|
|
2043
|
+
"Dr. oec.",
|
|
2044
|
+
"Dr. techn.",
|
|
2045
|
+
"Dr. h. c.",
|
|
2046
|
+
"Dr. phil.",
|
|
2047
|
+
"Dr.-Ing.",
|
|
2048
|
+
"Dr. Ing.",
|
|
2049
|
+
"Dr.",
|
|
2050
|
+
"Dipl.-Wirt.-Ing.",
|
|
2051
|
+
"Dipl.-Betriebsw.",
|
|
2052
|
+
"Dipl.-Inform.",
|
|
2053
|
+
"Dipl.-Volksw.",
|
|
2054
|
+
"Dipl.-Psych.",
|
|
2055
|
+
"Dipl.-Phys.",
|
|
2056
|
+
"Dipl.-Chem.",
|
|
2057
|
+
"Dipl.-Biol.",
|
|
2058
|
+
"Dipl.-Math.",
|
|
2059
|
+
"Dipl.-Päd.",
|
|
2060
|
+
"Dipl.-Soz.",
|
|
2061
|
+
"Dipl.-Kfm.",
|
|
2062
|
+
"Dipl.-Jur.",
|
|
2063
|
+
"Dipl. Ing.",
|
|
2064
|
+
"Dipl.-Ing.",
|
|
2065
|
+
"Mag. rer. soc. oec.",
|
|
2066
|
+
"Mag. rer. nat.",
|
|
2067
|
+
"Mag. phil.",
|
|
2068
|
+
"Mag. iur.",
|
|
2069
|
+
"Mag. arch.",
|
|
2070
|
+
"Mag. pharm.",
|
|
2071
|
+
"Mag. (FH)",
|
|
2072
|
+
"Mag.",
|
|
2073
|
+
"Bakk. rer. nat.",
|
|
2074
|
+
"Bakk. techn.",
|
|
2075
|
+
"Bakk. phil.",
|
|
2076
|
+
"Bakk.",
|
|
2077
|
+
"Lic. phil.",
|
|
2078
|
+
"Lic. iur.",
|
|
2079
|
+
"Lic. oec.",
|
|
2080
|
+
"Lic. theol.",
|
|
2081
|
+
"Lic.",
|
|
2082
|
+
"Priv.-Doz.",
|
|
2083
|
+
"PD",
|
|
2084
|
+
"RA"
|
|
2085
|
+
];
|
|
1590
2086
|
/**
|
|
1591
|
-
*
|
|
1592
|
-
*
|
|
1593
|
-
*
|
|
1594
|
-
*
|
|
1595
|
-
* The hyphen-minus is placed first so it is treated
|
|
1596
|
-
* as a literal, not a range indicator.
|
|
2087
|
+
* Courtesy/honorific titles that precede a person's
|
|
2088
|
+
* name. Sorted alphabetically. The detector escapes
|
|
2089
|
+
* dots and adds \b for entries in HONORIFIC_BOUNDARY.
|
|
1597
2090
|
*/
|
|
1598
|
-
const
|
|
1599
|
-
|
|
1600
|
-
|
|
1601
|
-
|
|
1602
|
-
|
|
1603
|
-
|
|
1604
|
-
|
|
2091
|
+
const HONORIFICS = [
|
|
2092
|
+
"Avv.",
|
|
2093
|
+
"Dame",
|
|
2094
|
+
"Doamna",
|
|
2095
|
+
"Domnul",
|
|
2096
|
+
"Don",
|
|
2097
|
+
"Doña",
|
|
2098
|
+
"Dott.",
|
|
2099
|
+
"Judge",
|
|
2100
|
+
"Justice",
|
|
2101
|
+
"Lady",
|
|
2102
|
+
"Lord",
|
|
2103
|
+
"M.",
|
|
2104
|
+
"Madame",
|
|
2105
|
+
"Mademoiselle",
|
|
2106
|
+
"Maître",
|
|
2107
|
+
"Me",
|
|
2108
|
+
"Messrs",
|
|
2109
|
+
"Miss",
|
|
2110
|
+
"Mlle",
|
|
2111
|
+
"Mme",
|
|
2112
|
+
"Monsieur",
|
|
2113
|
+
"Mr",
|
|
2114
|
+
"Mrs",
|
|
2115
|
+
"Ms",
|
|
2116
|
+
"Pr",
|
|
2117
|
+
"Pr.",
|
|
2118
|
+
"President",
|
|
2119
|
+
"Señor",
|
|
2120
|
+
"Señora",
|
|
2121
|
+
"Sig.",
|
|
2122
|
+
"Sig.ra",
|
|
2123
|
+
"Signor",
|
|
2124
|
+
"Signora",
|
|
2125
|
+
"Signorina",
|
|
2126
|
+
"Sir",
|
|
2127
|
+
"Sr.",
|
|
2128
|
+
"Sra."
|
|
2129
|
+
];
|
|
1605
2130
|
/**
|
|
1606
|
-
*
|
|
1607
|
-
*
|
|
1608
|
-
* inside a larger character class, e.g.:
|
|
1609
|
-
* `[\\s&,.${charClassInner("dash")}]`
|
|
2131
|
+
* Honorifics that need \b word-boundary anchors
|
|
2132
|
+
* (short or common words that could match mid-word).
|
|
1610
2133
|
*/
|
|
1611
|
-
const
|
|
1612
|
-
|
|
1613
|
-
|
|
1614
|
-
|
|
1615
|
-
|
|
1616
|
-
|
|
1617
|
-
|
|
1618
|
-
|
|
1619
|
-
|
|
2134
|
+
const HONORIFIC_BOUNDARY = new Set([
|
|
2135
|
+
"Don",
|
|
2136
|
+
"Doña",
|
|
2137
|
+
"M.",
|
|
2138
|
+
"Me",
|
|
2139
|
+
"Pr",
|
|
2140
|
+
"Pr.",
|
|
2141
|
+
"Señor",
|
|
2142
|
+
"Señora"
|
|
2143
|
+
]);
|
|
1620
2144
|
/**
|
|
1621
|
-
*
|
|
1622
|
-
*
|
|
1623
|
-
* `[\\s&,.${DASH_INNER}]`
|
|
2145
|
+
* Post-nominal degrees (comma or space separated after name).
|
|
2146
|
+
* Plain text; the detector auto-escapes for regex.
|
|
1624
2147
|
*/
|
|
1625
|
-
const
|
|
1626
|
-
|
|
1627
|
-
|
|
1628
|
-
|
|
1629
|
-
|
|
1630
|
-
|
|
1631
|
-
|
|
2148
|
+
const POST_NOMINALS = [
|
|
2149
|
+
"Ph.D.",
|
|
2150
|
+
"Ph.D",
|
|
2151
|
+
"CSc.",
|
|
2152
|
+
"DrSc.",
|
|
2153
|
+
"ArtD.",
|
|
2154
|
+
"D.Phil.",
|
|
2155
|
+
"DPhil.",
|
|
2156
|
+
"MPhil.",
|
|
2157
|
+
"MBA",
|
|
2158
|
+
"MPA",
|
|
2159
|
+
"LL.M.",
|
|
2160
|
+
"LL.B.",
|
|
2161
|
+
"M.Sc.",
|
|
2162
|
+
"B.Sc.",
|
|
2163
|
+
"MSc.",
|
|
2164
|
+
"BSc.",
|
|
2165
|
+
"M.Eng.",
|
|
2166
|
+
"B.Eng.",
|
|
2167
|
+
"M.A.",
|
|
2168
|
+
"B.A.",
|
|
2169
|
+
"JCD",
|
|
2170
|
+
"JD",
|
|
2171
|
+
"DiS.",
|
|
2172
|
+
"ACCA",
|
|
2173
|
+
"FCCA",
|
|
2174
|
+
"CIPM",
|
|
2175
|
+
"CIPT",
|
|
2176
|
+
"CIPP/E",
|
|
2177
|
+
"CIPP"
|
|
2178
|
+
];
|
|
1632
2179
|
//#endregion
|
|
1633
2180
|
//#region src/detectors/regex.ts
|
|
1634
2181
|
const MIN_PHONE_LENGTH = 7;
|
|
1635
2182
|
const MIN_MONTH_NAME_LENGTH = 3;
|
|
1636
2183
|
const escapeTitle = (title) => title.replace(/[.*+?^${}()|[\]\\]/g, "\\$&").replace(/\s+/g, "\\s*");
|
|
1637
2184
|
/** Escape for use inside a regex alternation. */
|
|
1638
|
-
const escapeRegex$
|
|
2185
|
+
const escapeRegex$2 = (s) => s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
1639
2186
|
/** Escape for use inside a regex character class. */
|
|
1640
2187
|
const escapeCharClass = (s) => s.replace(/[\]\\^-]/g, "\\$&");
|
|
1641
2188
|
const TITLE_PREFIX = TITLE_PREFIXES.toSorted((a, b) => b.length - a.length).map(escapeTitle).join("|");
|
|
1642
2189
|
const POST_NOMINAL = POST_NOMINALS.toSorted((a, b) => b.length - a.length).map(escapeTitle).join("|");
|
|
1643
|
-
const NAME_WORD =
|
|
2190
|
+
const NAME_WORD = `\\p{Lu}\\p{Ll}+`;
|
|
1644
2191
|
const PARTICLE = "(?:van der|van den|de la|della|von|van|dos|ibn|ben|bin|del|zum|zur|ten|ter|da|de|di|al|el|le|la|zu|af|av)";
|
|
1645
2192
|
const SP = "[^\\S\\n\\t]";
|
|
1646
2193
|
/** Honorific alternation built from titles.ts config. */
|
|
1647
2194
|
const HONORIFIC_ALT = [...HONORIFICS].toSorted((a, b) => b.length - a.length).map((h) => {
|
|
1648
|
-
const escaped = escapeRegex$
|
|
2195
|
+
const escaped = escapeRegex$2(h);
|
|
1649
2196
|
return HONORIFIC_BOUNDARY.has(h) ? `\\b${escaped}` : escaped;
|
|
1650
2197
|
}).join("|");
|
|
1651
2198
|
const toEntry = (validator, label, score) => {
|
|
@@ -1670,8 +2217,6 @@ const toEntry = (validator, label, score) => {
|
|
|
1670
2217
|
const STDNUM_ENTRIES = [
|
|
1671
2218
|
toEntry(hu.vat, "tax identification number", .95),
|
|
1672
2219
|
toEntry(it.codiceFiscale, "national identification number", .95),
|
|
1673
|
-
toEntry(es.dni, "national identification number", .9),
|
|
1674
|
-
toEntry(es.nie, "national identification number", .95),
|
|
1675
2220
|
toEntry(se.personnummer, "national identification number", .9),
|
|
1676
2221
|
toEntry(ro.cnp, "national identification number", .95),
|
|
1677
2222
|
toEntry(fr.nir, "social security number", .9),
|
|
@@ -1697,7 +2242,6 @@ const STDNUM_ENTRIES = [
|
|
|
1697
2242
|
toEntry(fi.ytunnus, "registration number", .9),
|
|
1698
2243
|
toEntry(bg.vat, "tax identification number", .95),
|
|
1699
2244
|
toEntry(sk.dic, "tax identification number", .95),
|
|
1700
|
-
toEntry(es.cif, "registration number", .95),
|
|
1701
2245
|
toEntry(es.vat, "tax identification number", .95),
|
|
1702
2246
|
toEntry(es.nss, "social security number", .9),
|
|
1703
2247
|
toEntry(fr.tva, "tax identification number", .95),
|
|
@@ -1719,7 +2263,9 @@ const STDNUM_ENTRIES = [
|
|
|
1719
2263
|
toEntry(ee.ik, "national identification number", .9),
|
|
1720
2264
|
toEntry(cy.vat, "tax identification number", .95),
|
|
1721
2265
|
toEntry(mt.vat, "tax identification number", .95),
|
|
1722
|
-
toEntry(lu.vat, "tax identification number", .95)
|
|
2266
|
+
toEntry(lu.vat, "tax identification number", .95),
|
|
2267
|
+
toEntry(br.cpf, "tax identification number", .95),
|
|
2268
|
+
toEntry(br.cnpj, "tax identification number", .95)
|
|
1723
2269
|
].filter((e) => e !== null);
|
|
1724
2270
|
const TITLED_PERSON = {
|
|
1725
2271
|
pattern: `(?:${TITLE_PREFIX})(?:${SP}+(?:${TITLE_PREFIX}))*${SP}+(?:${NAME_WORD})(?:${SP}{1,4}(?:${PARTICLE}${SP}+)?${NAME_WORD}){1,3}(?:,?${SP}+(?:${POST_NOMINAL})(?:,?${SP}+(?:${POST_NOMINAL}))*)?`,
|
|
@@ -1768,7 +2314,7 @@ const CREDIT_CARD = {
|
|
|
1768
2314
|
};
|
|
1769
2315
|
const CZ_BIRTH_NUMBER = {
|
|
1770
2316
|
pattern: `\\b\\d{6}/\\d{3,4}\\b`,
|
|
1771
|
-
label: "
|
|
2317
|
+
label: "birth number",
|
|
1772
2318
|
score: 1,
|
|
1773
2319
|
validator: cz.rc
|
|
1774
2320
|
};
|
|
@@ -1802,6 +2348,51 @@ const CZ_POSTAL = {
|
|
|
1802
2348
|
label: "address",
|
|
1803
2349
|
score: .7
|
|
1804
2350
|
};
|
|
2351
|
+
const ES_POSTAL = {
|
|
2352
|
+
pattern: "\\b(?:C\\.?P\\.?|[Cc][óo]digo[^\\S\\n]+postal)[^\\S\\n]{0,3}:?[^\\S\\n]{0,3}\\d{5}\\b",
|
|
2353
|
+
label: "address",
|
|
2354
|
+
score: .7
|
|
2355
|
+
};
|
|
2356
|
+
const ES_DNI = {
|
|
2357
|
+
pattern: `\\b\\d{8}-?[A-Za-z]\\b`,
|
|
2358
|
+
label: "national identification number",
|
|
2359
|
+
score: .9,
|
|
2360
|
+
validator: es.dni
|
|
2361
|
+
};
|
|
2362
|
+
const ES_NIE = {
|
|
2363
|
+
pattern: `\\b[XYZxyz]-?\\d{7}-?[A-Za-z]\\b`,
|
|
2364
|
+
label: "national identification number",
|
|
2365
|
+
score: .95,
|
|
2366
|
+
validator: es.nie
|
|
2367
|
+
};
|
|
2368
|
+
const ES_CIF = {
|
|
2369
|
+
pattern: `\\b[A-HJNP-SUVWa-hjnp-suvw]-?\\d{7}-?[0-9A-Ja-j]\\b`,
|
|
2370
|
+
label: "registration number",
|
|
2371
|
+
score: .95,
|
|
2372
|
+
validator: es.cif
|
|
2373
|
+
};
|
|
2374
|
+
const BR_CPF_FORMATTED = {
|
|
2375
|
+
pattern: `\\b\\d{3}\\.\\d{3}\\.\\d{3}${DASH}\\d{2}\\b`,
|
|
2376
|
+
label: "tax identification number",
|
|
2377
|
+
score: .95,
|
|
2378
|
+
validator: br.cpf
|
|
2379
|
+
};
|
|
2380
|
+
const BR_CNPJ_FORMATTED = {
|
|
2381
|
+
pattern: `\\b\\d{2}\\.\\d{3}\\.\\d{3}/\\d{4}${DASH}\\d{2}\\b`,
|
|
2382
|
+
label: "tax identification number",
|
|
2383
|
+
score: .95,
|
|
2384
|
+
validator: br.cnpj
|
|
2385
|
+
};
|
|
2386
|
+
const BR_RG_WITH_SSP = {
|
|
2387
|
+
pattern: `\\b\\d{1,3}\\.?\\d{3}\\.?\\d{3}(?:${DASH}[0-9A-Za-z])?[^\\S\\n]+SSP(?:/[A-Z]{2})?\\b`,
|
|
2388
|
+
label: "national identification number",
|
|
2389
|
+
score: .95
|
|
2390
|
+
};
|
|
2391
|
+
const BR_OAB = {
|
|
2392
|
+
pattern: "\\bOAB/[A-Z]{2}[^\\S\\n]+(?:n[º°.][^\\S\\n]*)?(?:\\d{1,3}(?:\\.\\d{3})+|\\d{4,6})\\b",
|
|
2393
|
+
label: "registration number",
|
|
2394
|
+
score: .95
|
|
2395
|
+
};
|
|
1805
2396
|
const URL = {
|
|
1806
2397
|
pattern: "(?:https?://|https?:(?=[^\\s])|www\\.)[\\w\\-]+(?:\\.[\\w\\-]+)+(?::\\d+)?(?:[/?#][^\\s)\\]>]*[^\\s.,;:!?)\\]>])?",
|
|
1807
2398
|
label: "url",
|
|
@@ -1843,6 +2434,14 @@ const ALL_REGEX_DEFS = [
|
|
|
1843
2434
|
CZ_BANK_ACCOUNT,
|
|
1844
2435
|
HU_LANDLINE,
|
|
1845
2436
|
CZ_POSTAL,
|
|
2437
|
+
ES_POSTAL,
|
|
2438
|
+
ES_DNI,
|
|
2439
|
+
ES_NIE,
|
|
2440
|
+
ES_CIF,
|
|
2441
|
+
BR_CPF_FORMATTED,
|
|
2442
|
+
BR_CNPJ_FORMATTED,
|
|
2443
|
+
BR_RG_WITH_SSP,
|
|
2444
|
+
BR_OAB,
|
|
1846
2445
|
URL,
|
|
1847
2446
|
{
|
|
1848
2447
|
pattern: "\\b(?:[0-9a-fA-F]{1,4}:){7}[0-9a-fA-F]{1,4}\\b|\\b(?:[0-9a-fA-F]{1,4}:){1,7}:\\b|::(?:[0-9a-fA-F]{1,4}:){0,6}[0-9a-fA-F]{1,4}",
|
|
@@ -1894,7 +2493,7 @@ const buildMonthAlternation = (months) => {
|
|
|
1894
2493
|
if (clean.length >= MIN_MONTH_NAME_LENGTH) seen.add(clean);
|
|
1895
2494
|
}
|
|
1896
2495
|
}
|
|
1897
|
-
return [...seen].toSorted((a, b) => b.length - a.length).map(escapeRegex$
|
|
2496
|
+
return [...seen].toSorted((a, b) => b.length - a.length).map(escapeRegex$2).join("|");
|
|
1898
2497
|
};
|
|
1899
2498
|
/**
|
|
1900
2499
|
* Build date patterns from a month-name alternation.
|
|
@@ -1951,11 +2550,11 @@ const buildCurrencyPatterns = (data) => {
|
|
|
1951
2550
|
const isAsciiAlpha = /^[a-zA-Z\s]+$/;
|
|
1952
2551
|
const parts = data.codes.map((code) => ({
|
|
1953
2552
|
len: code.length,
|
|
1954
|
-
alt: escapeRegex$
|
|
2553
|
+
alt: escapeRegex$2(code)
|
|
1955
2554
|
}));
|
|
1956
2555
|
const MIN_CI_LENGTH = 3;
|
|
1957
2556
|
if (data.localNames) for (const name of data.localNames) {
|
|
1958
|
-
const escaped = escapeRegex$
|
|
2557
|
+
const escaped = escapeRegex$2(name);
|
|
1959
2558
|
if (isAsciiAlpha.test(name) && name.length >= MIN_CI_LENGTH) parts.push({
|
|
1960
2559
|
len: name.length,
|
|
1961
2560
|
alt: `(?i:${escaped})`
|
|
@@ -1967,7 +2566,7 @@ const buildCurrencyPatterns = (data) => {
|
|
|
1967
2566
|
}
|
|
1968
2567
|
if (symbols) for (const ch of data.symbols) parts.push({
|
|
1969
2568
|
len: ch.length,
|
|
1970
|
-
alt: escapeRegex$
|
|
2569
|
+
alt: escapeRegex$2(ch)
|
|
1971
2570
|
});
|
|
1972
2571
|
const trailingAlt = parts.toSorted((a, b) => b.len - a.len).map((p) => p.alt).join("|");
|
|
1973
2572
|
if (!symbols && !trailingAlt) return [];
|
|
@@ -2021,19 +2620,21 @@ const processRegexMatches = (allMatches, sliceStart, sliceEnd, meta_) => {
|
|
|
2021
2620
|
if (idx < sliceStart || idx >= sliceEnd) continue;
|
|
2022
2621
|
const meta = meta_[idx - sliceStart];
|
|
2023
2622
|
if (!meta) continue;
|
|
2024
|
-
if (meta.label === "phone number" && match.text.length < MIN_PHONE_LENGTH) continue;
|
|
2623
|
+
if (meta.sourceDetail !== "custom-regex" && meta.label === "phone number" && match.text.length < MIN_PHONE_LENGTH) continue;
|
|
2025
2624
|
if (meta.validator) {
|
|
2026
2625
|
const compacted = meta.validator.compact(match.text);
|
|
2027
2626
|
if (!meta.validator.validate(compacted).valid) continue;
|
|
2028
2627
|
}
|
|
2029
|
-
|
|
2628
|
+
const entity = {
|
|
2030
2629
|
start: match.start,
|
|
2031
2630
|
end: match.end,
|
|
2032
2631
|
label: meta.label,
|
|
2033
2632
|
text: match.text,
|
|
2034
2633
|
score: meta.score,
|
|
2035
2634
|
source: DETECTION_SOURCES.REGEX
|
|
2036
|
-
}
|
|
2635
|
+
};
|
|
2636
|
+
if (meta.sourceDetail) entity.sourceDetail = meta.sourceDetail;
|
|
2637
|
+
results.push(entity);
|
|
2037
2638
|
}
|
|
2038
2639
|
return results;
|
|
2039
2640
|
};
|
|
@@ -2063,244 +2664,24 @@ const SIGNING_CLAUSE_META = {
|
|
|
2063
2664
|
label: "address",
|
|
2064
2665
|
score: .9
|
|
2065
2666
|
};
|
|
2066
|
-
let signingPatternPromise = null;
|
|
2067
|
-
const loadSigningPatterns = async () => {
|
|
2068
|
-
const mod = await import("./signing-clauses.mjs");
|
|
2069
|
-
return buildSigningClausePatterns(mod.default ?? mod);
|
|
2070
|
-
};
|
|
2071
|
-
const getSigningClausePatterns = () => {
|
|
2072
|
-
if (!signingPatternPromise) signingPatternPromise = loadSigningPatterns().catch((err) => {
|
|
2073
|
-
signingPatternPromise = null;
|
|
2074
|
-
throw err;
|
|
2075
|
-
});
|
|
2076
|
-
return signingPatternPromise;
|
|
2077
|
-
};
|
|
2078
|
-
//#endregion
|
|
2079
|
-
//#region src/detectors/legal-forms.ts
|
|
2080
|
-
const UPPER = "A-ZÁČĎÉĚÍŇÓŘŠŤÚŮÝŽÄÖÜÀÂÆÇÈÊËÎÏÔÙÛŸÑ\\u0130";
|
|
2081
|
-
const LOWER = "a-záčďéěíňóřšťúůýžäöüßàâæçèêëîïôùûÿñ\\u0131";
|
|
2082
|
-
const CAP_WORD = `(?:[${UPPER}]{2,}|[${UPPER}][${LOWER}${UPPER}]+)`;
|
|
2083
|
-
const ANY_WORD = `(?:[${UPPER}${LOWER}][${LOWER}${UPPER}]+|[${UPPER}]{2,3}|\\d{1,4})`;
|
|
2084
|
-
const ALLCAP_WORD = `[${UPPER}]{2,}`;
|
|
2085
|
-
const ROMAN_NUMERAL_RE = /^(?=[IVXLCDM])M{0,3}(?:CM|CD|D?C{0,3})(?:XC|XL|L?X{0,3})(?:IX|IV|V?I{0,3})$/;
|
|
2086
|
-
const escapeForRegex = (form) => form.replace(/[.*+?^${}()|[\]\\]/g, "\\$&").replace(/\s+/g, "\\s+").replace(/\\\./g, "\\.[^\\S\\n]?");
|
|
2087
|
-
const isShortForm = (form) => form.replace(/[.\s]/g, "").length <= 3 && !form.includes(" ");
|
|
2088
|
-
const buildPatternString = (forms) => {
|
|
2089
|
-
if (forms.length === 0) return null;
|
|
2090
|
-
const alt = forms.toSorted((a, b) => b.length - a.length).map(escapeForRegex).join("|");
|
|
2091
|
-
return `${`(?:${CAP_WORD})(?:${`(?:[\\s&,.${DASH_INNER}]{1,4}|${`\\s+(?:a|and|und|et|e|y|i)\\s+(?=[${LOWER}])`})`}(?:${ANY_WORD})){0,10}`}(?:\\s+|,\\s*)(?:${alt})(?![${LOWER}])`;
|
|
2092
|
-
};
|
|
2093
|
-
/**
|
|
2094
|
-
* Build legal form regex pattern strings.
|
|
2095
|
-
* Returns an array of regex strings for the unified
|
|
2096
|
-
* TextSearch builder. Empty if data package is not
|
|
2097
|
-
* installed.
|
|
2098
|
-
*/
|
|
2099
|
-
const buildLegalFormPatterns = async () => {
|
|
2100
|
-
let data = {};
|
|
2101
|
-
try {
|
|
2102
|
-
data = (await import("./legal-forms.mjs")).default;
|
|
2103
|
-
} catch {
|
|
2104
|
-
return [];
|
|
2105
|
-
}
|
|
2106
|
-
const allForms = [];
|
|
2107
|
-
const seen = /* @__PURE__ */ new Set();
|
|
2108
|
-
for (const forms of Object.values(data)) for (const form of forms) {
|
|
2109
|
-
const key = form.toLowerCase();
|
|
2110
|
-
if (!seen.has(key)) {
|
|
2111
|
-
seen.add(key);
|
|
2112
|
-
allForms.push(form);
|
|
2113
|
-
}
|
|
2114
|
-
}
|
|
2115
|
-
const patterns = [];
|
|
2116
|
-
const longPattern = buildPatternString(allForms.filter((f) => !isShortForm(f)));
|
|
2117
|
-
if (longPattern) patterns.push(longPattern);
|
|
2118
|
-
const shortPattern = buildPatternString(allForms.filter(isShortForm));
|
|
2119
|
-
if (shortPattern) patterns.push(shortPattern);
|
|
2120
|
-
const allcapPrefix = `(?:${ALLCAP_WORD})(?:[\\s&,.${DASH_INNER}]{1,4}(?:${ALLCAP_WORD})){0,2}`;
|
|
2121
|
-
const allcapAlt = allForms.toSorted((a, b) => b.length - a.length).map(escapeForRegex).join("|");
|
|
2122
|
-
patterns.push(`${allcapPrefix}(?:\\s+|,\\s*)(?:${allcapAlt})(?![${LOWER}])`);
|
|
2123
|
-
return patterns;
|
|
2124
|
-
};
|
|
2125
|
-
const CONNECTOR_RE = /^(?:a|and|und|et|e|y|i|&)$/i;
|
|
2126
|
-
const LEADING_CLAUSE_RE = /(?:^|\s)(?:by\s+and\s+between|is\s+between)\s+/giu;
|
|
2127
|
-
/**
|
|
2128
|
-
* Find the word ending just before `pos` in `text`,
|
|
2129
|
-
* skipping any whitespace (not newlines).
|
|
2130
|
-
* Returns null if no word is found (e.g., at start
|
|
2131
|
-
* of text, or preceded by non-word chars like ".").
|
|
2132
|
-
*/
|
|
2133
|
-
const findWordBefore = (text, pos) => {
|
|
2134
|
-
let scan = pos - 1;
|
|
2135
|
-
while (scan >= 0) {
|
|
2136
|
-
const ch = text.charAt(scan);
|
|
2137
|
-
if (ch === "\n" || !/\s/.test(ch)) break;
|
|
2138
|
-
scan--;
|
|
2139
|
-
}
|
|
2140
|
-
if (scan < 0 || text.charAt(scan) === "\n") return null;
|
|
2141
|
-
const wordEnd = scan + 1;
|
|
2142
|
-
while (scan >= 0 && /[\p{L}\p{M}&]/u.test(text.charAt(scan))) scan--;
|
|
2143
|
-
const wordStart = scan + 1;
|
|
2144
|
-
const word = text.slice(wordStart, wordEnd);
|
|
2145
|
-
if (word.length === 0) return null;
|
|
2146
|
-
return {
|
|
2147
|
-
word,
|
|
2148
|
-
start: wordStart
|
|
2149
|
-
};
|
|
2150
|
-
};
|
|
2151
|
-
/**
|
|
2152
|
-
* Extend a match backward through uppercase words and
|
|
2153
|
-
* lowercase connectors. Stops at start of text,
|
|
2154
|
-
* newline, or a word that doesn't qualify.
|
|
2155
|
-
*
|
|
2156
|
-
* Connectors (a, and, und, et, ...) are only consumed
|
|
2157
|
-
* when there is a valid word before them — a trailing
|
|
2158
|
-
* connector at an entity boundary is not consumed.
|
|
2159
|
-
*/
|
|
2160
|
-
const extendBackward = (fullText, matchStart) => {
|
|
2161
|
-
let pos = matchStart;
|
|
2162
|
-
while (pos > 0) {
|
|
2163
|
-
const found = findWordBefore(fullText, pos);
|
|
2164
|
-
if (!found) break;
|
|
2165
|
-
const { word, start: wordStart } = found;
|
|
2166
|
-
const isUpper = /^\p{Lu}/u.test(word);
|
|
2167
|
-
const isConnector = CONNECTOR_RE.test(word);
|
|
2168
|
-
if (isUpper) pos = wordStart;
|
|
2169
|
-
else if (isConnector) {
|
|
2170
|
-
const prev = findWordBefore(fullText, wordStart);
|
|
2171
|
-
if (!prev) break;
|
|
2172
|
-
if (!/^\p{Lu}/u.test(prev.word)) break;
|
|
2173
|
-
pos = prev.start;
|
|
2174
|
-
} else break;
|
|
2175
|
-
}
|
|
2176
|
-
return pos;
|
|
2177
|
-
};
|
|
2178
|
-
const trimLeadingClause = (text) => {
|
|
2179
|
-
let cut = -1;
|
|
2180
|
-
for (const match of text.matchAll(LEADING_CLAUSE_RE)) cut = match.index + match[0].length;
|
|
2181
|
-
if (cut <= 0) return {
|
|
2182
|
-
offset: 0,
|
|
2183
|
-
text
|
|
2184
|
-
};
|
|
2185
|
-
const trimmed = text.slice(cut);
|
|
2186
|
-
const leadingWs = trimmed.match(/^\s*/u)?.[0].length ?? 0;
|
|
2187
|
-
return {
|
|
2188
|
-
offset: cut + leadingWs,
|
|
2189
|
-
text: trimmed.slice(leadingWs)
|
|
2190
|
-
};
|
|
2191
|
-
};
|
|
2192
|
-
/**
|
|
2193
|
-
* Process legal form matches from the unified search.
|
|
2194
|
-
* Receives all matches; filters to the legal forms
|
|
2195
|
-
* slice via sliceStart/sliceEnd.
|
|
2196
|
-
*/
|
|
2197
|
-
const processLegalFormMatches = (allMatches, sliceStart, sliceEnd, fullText) => {
|
|
2198
|
-
const results = [];
|
|
2199
|
-
for (const match of allMatches) {
|
|
2200
|
-
const idx = match.pattern;
|
|
2201
|
-
if (idx < sliceStart || idx >= sliceEnd) continue;
|
|
2202
|
-
const text = match.text.trimEnd();
|
|
2203
|
-
if (text.length < 5) continue;
|
|
2204
|
-
if (text.includes("\n")) continue;
|
|
2205
|
-
let entityStart = match.start;
|
|
2206
|
-
let entityText = text;
|
|
2207
|
-
if (fullText) {
|
|
2208
|
-
const extended = extendBackward(fullText, match.start);
|
|
2209
|
-
if (extended < match.start) {
|
|
2210
|
-
entityStart = extended;
|
|
2211
|
-
entityText = fullText.slice(extended, match.start + text.length).trimEnd();
|
|
2212
|
-
}
|
|
2213
|
-
}
|
|
2214
|
-
const clauseTrim = trimLeadingClause(entityText);
|
|
2215
|
-
if (clauseTrim.offset > 0) {
|
|
2216
|
-
entityStart += clauseTrim.offset;
|
|
2217
|
-
entityText = clauseTrim.text;
|
|
2218
|
-
}
|
|
2219
|
-
const getPrefixInfo = (value) => {
|
|
2220
|
-
const prefixEnd = value.lastIndexOf(",") !== -1 ? value.lastIndexOf(",") : value.lastIndexOf(" ");
|
|
2221
|
-
return {
|
|
2222
|
-
prefixEnd,
|
|
2223
|
-
prefixPart: prefixEnd > 0 ? value.slice(0, prefixEnd).replace(/[^a-zA-ZÀ-ž]/g, "") : value.replace(/[^a-zA-ZÀ-ž]/g, "")
|
|
2224
|
-
};
|
|
2225
|
-
};
|
|
2226
|
-
let { prefixEnd, prefixPart } = getPrefixInfo(entityText);
|
|
2227
|
-
let isAllCapsMatch = prefixPart.length > 2 && prefixPart === prefixPart.toUpperCase();
|
|
2228
|
-
if (isAllCapsMatch && fullText) {
|
|
2229
|
-
const lineStart = fullText.lastIndexOf("\n", entityStart);
|
|
2230
|
-
const lineEnd = fullText.indexOf("\n", entityStart + entityText.length);
|
|
2231
|
-
const lineLetters = fullText.slice(lineStart + 1, lineEnd === -1 ? fullText.length : lineEnd).replace(/[^a-zA-ZÀ-ž]/g, "");
|
|
2232
|
-
const upperCount = [...lineLetters].filter((c) => c === c.toUpperCase()).length;
|
|
2233
|
-
if (lineLetters.length > 5 && upperCount / lineLetters.length >= .95) continue;
|
|
2234
|
-
if ((prefixPart.length > 0 ? entityText.slice(0, prefixEnd > 0 ? prefixEnd : entityText.length).trim().split(/\s+/).length : 0) > 3) {
|
|
2235
|
-
entityStart = match.start;
|
|
2236
|
-
entityText = text;
|
|
2237
|
-
({prefixEnd, prefixPart} = getPrefixInfo(entityText));
|
|
2238
|
-
isAllCapsMatch = prefixPart.length > 2 && prefixPart === prefixPart.toUpperCase();
|
|
2239
|
-
}
|
|
2240
|
-
} else if (isAllCapsMatch) continue;
|
|
2241
|
-
const lastSpace = entityText.lastIndexOf(" ");
|
|
2242
|
-
const rawSuffix = lastSpace !== -1 ? entityText.slice(lastSpace + 1) : "";
|
|
2243
|
-
const suffixClean = rawSuffix.replace(/[.,]/g, "");
|
|
2244
|
-
if (suffixClean.length > 0 && ROMAN_NUMERAL_RE.test(suffixClean)) continue;
|
|
2245
|
-
if (suffixClean.length <= 2 && !/\./.test(rawSuffix) && /[^\x00-\x7F]/.test(entityText.slice(0, lastSpace !== -1 ? lastSpace : entityText.length))) continue;
|
|
2246
|
-
results.push({
|
|
2247
|
-
start: entityStart,
|
|
2248
|
-
end: entityStart + entityText.length,
|
|
2249
|
-
label: "organization",
|
|
2250
|
-
text: entityText,
|
|
2251
|
-
score: .95,
|
|
2252
|
-
source: DETECTION_SOURCES.LEGAL_FORM
|
|
2253
|
-
});
|
|
2254
|
-
}
|
|
2255
|
-
return results;
|
|
2256
|
-
};
|
|
2257
|
-
//#endregion
|
|
2258
|
-
//#region src/config/legal-forms.ts
|
|
2259
|
-
/**
|
|
2260
|
-
* Known legal form suffixes. Shared between trigger
|
|
2261
|
-
* detection (reclassification) and org-propagation
|
|
2262
|
-
* (suffix stripping). Within each family of related
|
|
2263
|
-
* forms, longer variants come first so that
|
|
2264
|
-
* "spol. s r.o." matches before "s.r.o." and
|
|
2265
|
-
* "s. r. o." matches before "s.r.o.".
|
|
2266
|
-
*/
|
|
2267
|
-
const LEGAL_SUFFIXES = [
|
|
2268
|
-
"spol. s r.o.",
|
|
2269
|
-
"s.r.o.",
|
|
2270
|
-
"s. r. o.",
|
|
2271
|
-
"a.s.",
|
|
2272
|
-
"a. s.",
|
|
2273
|
-
"v.o.s.",
|
|
2274
|
-
"v. o. s.",
|
|
2275
|
-
"k.s.",
|
|
2276
|
-
"k. s.",
|
|
2277
|
-
"z.s.",
|
|
2278
|
-
"z. s.",
|
|
2279
|
-
"z.ú.",
|
|
2280
|
-
"z. ú.",
|
|
2281
|
-
"o.p.s.",
|
|
2282
|
-
"o. p. s.",
|
|
2283
|
-
"s.p.",
|
|
2284
|
-
"s. p.",
|
|
2285
|
-
"GmbH",
|
|
2286
|
-
"AG",
|
|
2287
|
-
"SE",
|
|
2288
|
-
"KG",
|
|
2289
|
-
"OHG",
|
|
2290
|
-
"Ltd.",
|
|
2291
|
-
"Ltd",
|
|
2292
|
-
"LLC",
|
|
2293
|
-
"LLP",
|
|
2294
|
-
"Inc.",
|
|
2295
|
-
"S.A.",
|
|
2296
|
-
"SA",
|
|
2297
|
-
"SAS",
|
|
2298
|
-
"SARL",
|
|
2299
|
-
"Sp. z o.o.",
|
|
2300
|
-
"S.p.A."
|
|
2301
|
-
];
|
|
2667
|
+
let signingPatternPromise = null;
|
|
2668
|
+
const loadSigningPatterns = async () => {
|
|
2669
|
+
const mod = await import("./signing-clauses.mjs");
|
|
2670
|
+
return buildSigningClausePatterns(mod.default ?? mod);
|
|
2671
|
+
};
|
|
2672
|
+
const getSigningClausePatterns = () => {
|
|
2673
|
+
if (!signingPatternPromise) signingPatternPromise = loadSigningPatterns().catch((err) => {
|
|
2674
|
+
signingPatternPromise = null;
|
|
2675
|
+
throw err;
|
|
2676
|
+
});
|
|
2677
|
+
return signingPatternPromise;
|
|
2678
|
+
};
|
|
2302
2679
|
//#endregion
|
|
2303
2680
|
//#region src/detectors/triggers.ts
|
|
2681
|
+
const VALID_ID_VALIDATORS = {
|
|
2682
|
+
"br.cpf": br.cpf,
|
|
2683
|
+
"br.cnpj": br.cnpj
|
|
2684
|
+
};
|
|
2304
2685
|
const TRIGGER_SCORE = .95;
|
|
2305
2686
|
const WHITESPACE_RE$1 = /\s+/;
|
|
2306
2687
|
const LETTER_RE = /\p{L}/u;
|
|
@@ -2315,12 +2696,23 @@ const DECIMAL_COMMA_RE = new RegExp(`^,(?:\\d|${DASH}{1,2})`);
|
|
|
2315
2696
|
* etc.), skip the comma and degree, then continue.
|
|
2316
2697
|
*/
|
|
2317
2698
|
const POST_NOMINAL_RE = new RegExp(`^,\\s*(?:${POST_NOMINALS.toSorted((a, b) => b.length - a.length).map((d) => d.replace(/[.*+?^${}()|[\]\\]/g, "\\$&").replace(/\\\./g, "\\.\\s*")).join("|")})\\.?`, "i");
|
|
2318
|
-
const
|
|
2319
|
-
const
|
|
2320
|
-
|
|
2321
|
-
|
|
2322
|
-
|
|
2323
|
-
|
|
2699
|
+
const buildLegalFormCheckRe = (forms) => {
|
|
2700
|
+
const parts = forms.map((f) => {
|
|
2701
|
+
const escaped = f.replace(/[.*+?^${}()|[\]\\]/g, "\\$&").replace(/\\\./g, "\\.\\s*");
|
|
2702
|
+
return /^[\p{L}\p{M}]+$/u.test(f) ? `(?<![\\p{L}\\p{N}])${escaped}(?![\\p{L}\\p{N}])` : escaped;
|
|
2703
|
+
});
|
|
2704
|
+
return new RegExp(parts.join("|"), "u");
|
|
2705
|
+
};
|
|
2706
|
+
let cachedLegalFormCheckRe = null;
|
|
2707
|
+
let cachedLegalFormCheckSource = null;
|
|
2708
|
+
const getLegalFormCheckRe = () => {
|
|
2709
|
+
const source = getKnownLegalSuffixes();
|
|
2710
|
+
if (cachedLegalFormCheckSource !== source || cachedLegalFormCheckRe === null) {
|
|
2711
|
+
cachedLegalFormCheckSource = source;
|
|
2712
|
+
cachedLegalFormCheckRe = buildLegalFormCheckRe(source);
|
|
2713
|
+
}
|
|
2714
|
+
return cachedLegalFormCheckRe;
|
|
2715
|
+
};
|
|
2324
2716
|
const compileValidations = (validations) => validations.map((v) => {
|
|
2325
2717
|
switch (v.type) {
|
|
2326
2718
|
case "starts-uppercase": return {
|
|
@@ -2347,6 +2739,17 @@ const compileValidations = (validations) => validations.map((v) => {
|
|
|
2347
2739
|
type: "matches-pattern",
|
|
2348
2740
|
re: new RegExp(v.pattern, (v.flags ?? "").replace(/[gy]/g, ""))
|
|
2349
2741
|
};
|
|
2742
|
+
case "valid-id": {
|
|
2743
|
+
const validator = VALID_ID_VALIDATORS[v.validator];
|
|
2744
|
+
if (!validator) throw new Error(`Unknown valid-id validator: ${JSON.stringify(v.validator)}`);
|
|
2745
|
+
return {
|
|
2746
|
+
type: "valid-id",
|
|
2747
|
+
check: (value) => {
|
|
2748
|
+
const compact = value.replace(/[\s.\-/]/g, "");
|
|
2749
|
+
return validator.validate(compact).valid;
|
|
2750
|
+
}
|
|
2751
|
+
};
|
|
2752
|
+
}
|
|
2350
2753
|
default: throw new Error(`Unknown validation type: ${JSON.stringify(v)}`);
|
|
2351
2754
|
}
|
|
2352
2755
|
});
|
|
@@ -2370,6 +2773,9 @@ const applyValidations = (text, validations) => {
|
|
|
2370
2773
|
case "matches-pattern":
|
|
2371
2774
|
if (!v.re.test(text)) return false;
|
|
2372
2775
|
break;
|
|
2776
|
+
case "valid-id":
|
|
2777
|
+
if (!v.check(text)) return false;
|
|
2778
|
+
break;
|
|
2373
2779
|
default: throw new Error(`Unknown compiled validation type: ${JSON.stringify(v)}`);
|
|
2374
2780
|
}
|
|
2375
2781
|
return true;
|
|
@@ -2464,8 +2870,10 @@ const buildTriggerPatterns = async () => {
|
|
|
2464
2870
|
strategy: rule.strategy.type
|
|
2465
2871
|
});
|
|
2466
2872
|
}
|
|
2873
|
+
const patterns = rules.map((r) => r.trigger.toLowerCase());
|
|
2874
|
+
await loadAddressStopKeywords();
|
|
2467
2875
|
return {
|
|
2468
|
-
patterns
|
|
2876
|
+
patterns,
|
|
2469
2877
|
rules
|
|
2470
2878
|
};
|
|
2471
2879
|
};
|
|
@@ -2486,14 +2894,103 @@ const stripQuotes = (value) => {
|
|
|
2486
2894
|
const COMMA_STOP_CHARS = new Set([
|
|
2487
2895
|
"\n",
|
|
2488
2896
|
"(",
|
|
2489
|
-
" "
|
|
2897
|
+
" ",
|
|
2898
|
+
";"
|
|
2490
2899
|
]);
|
|
2491
2900
|
/**
|
|
2901
|
+
* Sentence-terminator detection: a period that genuinely ends
|
|
2902
|
+
* one clause and starts another. Used by `to-next-comma` so
|
|
2903
|
+
* governing-law clauses ("…State of New York. SECTION 2…")
|
|
2904
|
+
* don't sweep across the period.
|
|
2905
|
+
*
|
|
2906
|
+
* Three positive signals (any one terminates):
|
|
2907
|
+
* 1. Long lowercase tail before the dot (>= 5 letters) —
|
|
2908
|
+
* catches "…construction. SECTION 2…", "…vykonává.".
|
|
2909
|
+
* 2. Currency/amount tail (zł, Kč, USD, €) — catches
|
|
2910
|
+
* "…w kwocie 1000 zł. Termin płatności…".
|
|
2911
|
+
* 3. Proper-noun head: a capitalized word of >= 4 letters
|
|
2912
|
+
* ending in lowercase, with no internal uppercase, AND
|
|
2913
|
+
* a substantial next clause (Capital + >= 2 lowercase).
|
|
2914
|
+
* Catches short city names that the lowercase-tail rule
|
|
2915
|
+
* misses: "…z siedzibą w Łódź. Kapitał zakładowy…",
|
|
2916
|
+
* "…seat in Brno. Section 2…".
|
|
2917
|
+
*
|
|
2918
|
+
* The rule must NOT fire on:
|
|
2919
|
+
* - title abbreviations: "Mr.", "Mrs.", "Dr.", "Hon.",
|
|
2920
|
+
* "Sr.", "Jr." (head <= 3 chars or insufficient Ll tail)
|
|
2921
|
+
* - degree abbreviations: "Ph.D.", "RNDr.", "MUDr.",
|
|
2922
|
+
* "Ing." (internal periods or internal uppercase block
|
|
2923
|
+
* the proper-noun pattern)
|
|
2924
|
+
* - street-type abbreviations: "Ste.", "Ave.", "Inc.",
|
|
2925
|
+
* "ul.", "al.", "nábř." (lowercase initials or
|
|
2926
|
+
* insufficient Ll tail)
|
|
2927
|
+
* - small-word lowercase abbreviations: "prof.", "inż.",
|
|
2928
|
+
* "hab." (no leading uppercase, so the proper-noun rule
|
|
2929
|
+
* can't fire)
|
|
2930
|
+
*/
|
|
2931
|
+
const NEXT_IS_SENTENCE_START_RE = /^\.(?:\s+\p{Lu}|\s*$)/u;
|
|
2932
|
+
const SENTENCE_TAIL_RE = /\p{Ll}{5,}$/u;
|
|
2933
|
+
/**
|
|
2934
|
+
* Proper-noun tail: capital letter + >= 3 lowercase letters,
|
|
2935
|
+
* preceded by a non-letter and non-period (so we don't slice
|
|
2936
|
+
* into the middle of an acronym or a multi-dot abbreviation).
|
|
2937
|
+
* The 4-character minimum excludes 2–3-char titles ("Mr",
|
|
2938
|
+
* "Mrs", "Dr", "Inc", "Ste"); the all-lowercase tail
|
|
2939
|
+
* excludes mixed-case degrees ("RNDr", "MUDr").
|
|
2940
|
+
*/
|
|
2941
|
+
const PROPER_NOUN_HEAD_RE = /(?:^|[^\p{L}.])\p{Lu}\p{Ll}{3,}$/u;
|
|
2942
|
+
/**
|
|
2943
|
+
* Next clause begins with a real word: capital + >= 2
|
|
2944
|
+
* lowercase letters. Filters cases where a capitalized
|
|
2945
|
+
* abbreviation (e.g., "Smith Inc.") follows a proper noun,
|
|
2946
|
+
* which would otherwise look sentence-like.
|
|
2947
|
+
*/
|
|
2948
|
+
const NEXT_IS_REAL_SENTENCE_RE = /^\.\s+\p{Lu}\p{Ll}{2,}/u;
|
|
2949
|
+
/**
|
|
2950
|
+
* Short currency-abbreviation tail (zł, Kč, gr, Ft, kr,
|
|
2951
|
+
* лв, USD, PLN, EUR, …). When a period follows one of
|
|
2952
|
+
* these and the next token starts uppercase, treat it as
|
|
2953
|
+
* a sentence boundary even though the tail is too short
|
|
2954
|
+
* to satisfy `SENTENCE_TAIL_RE`. Without this, amount
|
|
2955
|
+
* triggers using `to-next-comma` swallow the following
|
|
2956
|
+
* clause: `"w kwocie 1000 zł. Termin płatności…"`.
|
|
2957
|
+
*
|
|
2958
|
+
* Currency codes are typically uppercase (`USD`, `PLN`)
|
|
2959
|
+
* but appear lowercased in informal writing (`pln`, `eur`);
|
|
2960
|
+
* local names mix case (`zł`, `Kč`, `Ft`); symbols
|
|
2961
|
+
* (`€`, `$`, `£`) appear after the amount as well. The
|
|
2962
|
+
* negative lookbehind on a letter ensures the abbreviation
|
|
2963
|
+
* is matched only as a standalone token; symbols are
|
|
2964
|
+
* matched unconditionally since they are not letters.
|
|
2965
|
+
*/
|
|
2966
|
+
const CURRENCY_TAIL_RE = /(?:(?<![\p{L}])(?:zł|Kč|gr|Ft|kr|лв|USD|PLN|EUR|CZK|GBP|CHF|HUF|RON|SEK|NOK|DKK)|[€$£])$/iu;
|
|
2967
|
+
const NUMERIC_SENTENCE_TAIL_RE = /\d$/;
|
|
2968
|
+
const isSentenceTerminator = (text, periodIndex) => {
|
|
2969
|
+
const tail = text.slice(periodIndex);
|
|
2970
|
+
if (!NEXT_IS_SENTENCE_START_RE.test(tail)) return false;
|
|
2971
|
+
const head = text.slice(0, periodIndex);
|
|
2972
|
+
if (SENTENCE_TAIL_RE.test(head) || CURRENCY_TAIL_RE.test(head) || NUMERIC_SENTENCE_TAIL_RE.test(head)) return true;
|
|
2973
|
+
return PROPER_NOUN_HEAD_RE.test(head) && NEXT_IS_REAL_SENTENCE_RE.test(tail);
|
|
2974
|
+
};
|
|
2975
|
+
/**
|
|
2492
2976
|
* Field-label keywords that terminate address scanning.
|
|
2493
2977
|
* When a comma in the address strategy is followed by
|
|
2494
2978
|
* one of these, the address stops before the keyword.
|
|
2979
|
+
*
|
|
2980
|
+
* The list is sourced from
|
|
2981
|
+
* `data/address-stop-keywords.json` (per-language so new
|
|
2982
|
+
* languages can drop in their own labels without
|
|
2983
|
+
* touching this file). `loadAddressStopKeywords` unions
|
|
2984
|
+
* every language into a single longest-first array;
|
|
2985
|
+
* `getAddressStopKeywordsSync` returns the cached union
|
|
2986
|
+
* and falls back to a seed list so the strategy keeps
|
|
2987
|
+
* working before the warmup promise resolves.
|
|
2988
|
+
*
|
|
2989
|
+
* The address strategy doesn't know the document
|
|
2990
|
+
* language, so a flat union is intentional: any
|
|
2991
|
+
* language's labels can appear in any address.
|
|
2495
2992
|
*/
|
|
2496
|
-
const
|
|
2993
|
+
const ADDRESS_STOP_KEYWORDS_SEED = [
|
|
2497
2994
|
"číslo účtu",
|
|
2498
2995
|
"registrační",
|
|
2499
2996
|
"zastoupen",
|
|
@@ -2516,6 +3013,52 @@ const ADDRESS_STOP_KEYWORDS = [
|
|
|
2516
3013
|
"bic",
|
|
2517
3014
|
"ič"
|
|
2518
3015
|
];
|
|
3016
|
+
let addressStopKeywordsCache = null;
|
|
3017
|
+
let addressStopKeywordsPromise = null;
|
|
3018
|
+
const loadAddressStopKeywords = async () => {
|
|
3019
|
+
if (addressStopKeywordsCache) return addressStopKeywordsCache;
|
|
3020
|
+
if (addressStopKeywordsPromise) return addressStopKeywordsPromise;
|
|
3021
|
+
addressStopKeywordsPromise = (async () => {
|
|
3022
|
+
let data = {};
|
|
3023
|
+
try {
|
|
3024
|
+
const mod = await import("./address-stop-keywords.mjs");
|
|
3025
|
+
data = mod.default ?? mod;
|
|
3026
|
+
} catch (err) {
|
|
3027
|
+
console.warn("[anonymize] triggers: failed to load address-stop-keywords.json, falling back to seed list:", err);
|
|
3028
|
+
}
|
|
3029
|
+
const seen = /* @__PURE__ */ new Set();
|
|
3030
|
+
const out = [];
|
|
3031
|
+
const addAll = (list) => {
|
|
3032
|
+
for (const kw of list) {
|
|
3033
|
+
if (typeof kw !== "string" || kw.length === 0) continue;
|
|
3034
|
+
const lower = kw.toLowerCase();
|
|
3035
|
+
if (seen.has(lower)) continue;
|
|
3036
|
+
seen.add(lower);
|
|
3037
|
+
out.push(lower);
|
|
3038
|
+
}
|
|
3039
|
+
};
|
|
3040
|
+
for (const [key, value] of Object.entries(data)) {
|
|
3041
|
+
if (key.startsWith("_")) continue;
|
|
3042
|
+
if (!Array.isArray(value)) continue;
|
|
3043
|
+
addAll(value);
|
|
3044
|
+
}
|
|
3045
|
+
addAll(ADDRESS_STOP_KEYWORDS_SEED);
|
|
3046
|
+
out.sort((a, b) => b.length - a.length);
|
|
3047
|
+
addressStopKeywordsCache = out;
|
|
3048
|
+
return out;
|
|
3049
|
+
})();
|
|
3050
|
+
return addressStopKeywordsPromise;
|
|
3051
|
+
};
|
|
3052
|
+
const getAddressStopKeywordsSync = () => addressStopKeywordsCache ?? ADDRESS_STOP_KEYWORDS_SEED;
|
|
3053
|
+
/**
|
|
3054
|
+
* Warm the address-stop-keywords cache. Pipeline callers
|
|
3055
|
+
* await this before invoking trigger detection so the
|
|
3056
|
+
* synchronous `extractValue` path uses the merged list
|
|
3057
|
+
* instead of the seed fallback.
|
|
3058
|
+
*/
|
|
3059
|
+
const warmAddressStopKeywords = async () => {
|
|
3060
|
+
await loadAddressStopKeywords();
|
|
3061
|
+
};
|
|
2519
3062
|
const extractValue = (text, triggerEnd, strategy, label) => {
|
|
2520
3063
|
const remaining = text.slice(triggerEnd);
|
|
2521
3064
|
const stripped = remaining.replace(/^[\s:;]+/, "");
|
|
@@ -2524,6 +3067,8 @@ const extractValue = (text, triggerEnd, strategy, label) => {
|
|
|
2524
3067
|
if (valueText.length === 0) return null;
|
|
2525
3068
|
switch (strategy.type) {
|
|
2526
3069
|
case "to-next-comma": {
|
|
3070
|
+
const stopWords = strategy.stopWords ?? [];
|
|
3071
|
+
const stopWordRe = stopWords.length > 0 ? new RegExp(`^(?:${stopWords.map((w) => w.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")).join("|")})(?![\\p{L}\\p{N}])`, "iu") : null;
|
|
2527
3072
|
let end = 0;
|
|
2528
3073
|
let foundStop = false;
|
|
2529
3074
|
while (end < valueText.length) {
|
|
@@ -2532,6 +3077,14 @@ const extractValue = (text, triggerEnd, strategy, label) => {
|
|
|
2532
3077
|
foundStop = true;
|
|
2533
3078
|
break;
|
|
2534
3079
|
}
|
|
3080
|
+
if (ch === "." && isSentenceTerminator(valueText, end)) {
|
|
3081
|
+
foundStop = true;
|
|
3082
|
+
break;
|
|
3083
|
+
}
|
|
3084
|
+
if (stopWordRe !== null && (end === 0 || !/[\p{L}\p{N}]/u.test(valueText[end - 1] ?? "")) && stopWordRe.test(valueText.slice(end))) {
|
|
3085
|
+
foundStop = true;
|
|
3086
|
+
break;
|
|
3087
|
+
}
|
|
2535
3088
|
if (ch === ",") {
|
|
2536
3089
|
const afterComma = valueText.slice(end);
|
|
2537
3090
|
if (DECIMAL_COMMA_RE.test(afterComma)) {
|
|
@@ -2560,6 +3113,8 @@ const extractValue = (text, triggerEnd, strategy, label) => {
|
|
|
2560
3113
|
};
|
|
2561
3114
|
}
|
|
2562
3115
|
case "to-end-of-line": {
|
|
3116
|
+
const consumed = remaining.length - valueText.length;
|
|
3117
|
+
if (consumed > 0 && remaining.slice(0, consumed).includes("\n")) return null;
|
|
2563
3118
|
const LINE_STOPS = ["\n"];
|
|
2564
3119
|
let end = valueText.length;
|
|
2565
3120
|
for (const ch of LINE_STOPS) {
|
|
@@ -2580,7 +3135,8 @@ const extractValue = (text, triggerEnd, strategy, label) => {
|
|
|
2580
3135
|
const tabIdx = valueText.indexOf(" ");
|
|
2581
3136
|
const cellText = tabIdx !== -1 ? valueText.slice(0, tabIdx) : valueText;
|
|
2582
3137
|
const PUNCT_ONLY = /^[\p{P}\p{S}]+$/u;
|
|
2583
|
-
const
|
|
3138
|
+
const NUMBER_MARKER = /^(?:n[ºo°.]|№)$/i;
|
|
3139
|
+
const words = cellText.split(WHITESPACE_RE$1).filter((w) => !PUNCT_ONLY.test(w) && !NUMBER_MARKER.test(w)).slice(0, strategy.count);
|
|
2584
3140
|
if (words.length === 0) return null;
|
|
2585
3141
|
const firstWord = words[0];
|
|
2586
3142
|
if (firstWord === void 0) return null;
|
|
@@ -2603,14 +3159,22 @@ const extractValue = (text, triggerEnd, strategy, label) => {
|
|
|
2603
3159
|
}
|
|
2604
3160
|
case "company-id-value": {
|
|
2605
3161
|
const raw = text.slice(triggerEnd);
|
|
2606
|
-
const
|
|
3162
|
+
const triggerLastChar = text[triggerEnd - 1] ?? "";
|
|
3163
|
+
const sepMatch = (triggerLastChar === "°" || triggerLastChar === "º" || triggerLastChar === "№" || triggerLastChar === "#" ? /^(?:\s*:\s*|\s+|)/ : /^(?:\s*:\s*|\s+)/).exec(raw);
|
|
2607
3164
|
if (!sepMatch) return null;
|
|
2608
|
-
|
|
2609
|
-
const
|
|
3165
|
+
let afterSep = raw.slice(sepMatch[0].length);
|
|
3166
|
+
const labelMatch = /^(?:nr\.?|numer|n[ºo°.]|№|no\.?)(?:\s*:\s*|\s+)/i.exec(afterSep);
|
|
3167
|
+
let labelOffset = 0;
|
|
3168
|
+
if (labelMatch) {
|
|
3169
|
+
labelOffset = labelMatch[0].length;
|
|
3170
|
+
afterSep = afterSep.slice(labelOffset);
|
|
3171
|
+
}
|
|
3172
|
+
const idMatch = /^[A-Z]{0,6}\s?(?:[A-Z0-9]{2}\s?)?\d{2}(?:(?<=\d)[A-Z]|[\d\s.\-/]){3,}/i.exec(afterSep);
|
|
2610
3173
|
if (!idMatch) return null;
|
|
2611
|
-
const
|
|
3174
|
+
const trailingLetterMatch = /\d$/.test(idMatch[0]) ? /^[A-Z]/i.exec(afterSep.slice(idMatch[0].length)) : null;
|
|
3175
|
+
const idText = (trailingLetterMatch ? idMatch[0] + trailingLetterMatch[0] : idMatch[0]).trim();
|
|
2612
3176
|
const leadingSpaces = idMatch[0].length - idMatch[0].trimStart().length;
|
|
2613
|
-
const idStart = triggerEnd + sepMatch[0].length + leadingSpaces;
|
|
3177
|
+
const idStart = triggerEnd + sepMatch[0].length + labelOffset + leadingSpaces;
|
|
2614
3178
|
return {
|
|
2615
3179
|
start: idStart,
|
|
2616
3180
|
end: idStart + idText.length,
|
|
@@ -2620,18 +3184,36 @@ const extractValue = (text, triggerEnd, strategy, label) => {
|
|
|
2620
3184
|
case "address": {
|
|
2621
3185
|
const maxLen = strategy.maxChars ?? 120;
|
|
2622
3186
|
const UPPER_RE = /\p{Lu}/u;
|
|
3187
|
+
const stopKeywords = getAddressStopKeywordsSync();
|
|
2623
3188
|
let end = 0;
|
|
2624
3189
|
while (end < valueText.length && end < maxLen) {
|
|
2625
3190
|
const ch = valueText[end];
|
|
2626
3191
|
if (ch === "\n" || ch === "(") break;
|
|
3192
|
+
if (ch === " " || ch === " ") {
|
|
3193
|
+
const afterWs = valueText.slice(end).trimStart().toLowerCase();
|
|
3194
|
+
if (stopKeywords.some((kw) => {
|
|
3195
|
+
if (!afterWs.startsWith(kw)) return false;
|
|
3196
|
+
const next = afterWs[kw.length];
|
|
3197
|
+
return next === void 0 || /[\s:;.,!?()\d]/.test(next);
|
|
3198
|
+
})) break;
|
|
3199
|
+
}
|
|
2627
3200
|
if (ch === ".") {
|
|
2628
3201
|
const next = valueText[end + 1];
|
|
2629
3202
|
const afterNext = valueText[end + 2];
|
|
3203
|
+
const afterPeriod = valueText.slice(end + 1).replace(/^\s+/, "").toLowerCase();
|
|
3204
|
+
if (afterPeriod.length > 0) {
|
|
3205
|
+
if (getAddressStopKeywordsSync().some((kw) => {
|
|
3206
|
+
if (!afterPeriod.startsWith(kw)) return false;
|
|
3207
|
+
const afterKw = afterPeriod[kw.length];
|
|
3208
|
+
return afterKw === void 0 || /[\s:;.,!?()\d]/.test(afterKw);
|
|
3209
|
+
})) break;
|
|
3210
|
+
}
|
|
2630
3211
|
if (next !== void 0 && (/\p{L}/u.test(next) || /\d/.test(next))) {
|
|
2631
3212
|
end++;
|
|
2632
3213
|
continue;
|
|
2633
3214
|
}
|
|
2634
3215
|
if (next === " " && afterNext !== void 0 && (/\p{L}/u.test(afterNext) || /\d/.test(afterNext))) {
|
|
3216
|
+
if (isSentenceTerminator(valueText, end)) break;
|
|
2635
3217
|
end++;
|
|
2636
3218
|
continue;
|
|
2637
3219
|
}
|
|
@@ -2643,7 +3225,7 @@ const extractValue = (text, triggerEnd, strategy, label) => {
|
|
|
2643
3225
|
const peekCh = valueText[peek];
|
|
2644
3226
|
if (peekCh === void 0) break;
|
|
2645
3227
|
const afterComma = valueText.slice(end + 1).trimStart().toLowerCase();
|
|
2646
|
-
if (
|
|
3228
|
+
if (getAddressStopKeywordsSync().some((kw) => {
|
|
2647
3229
|
if (!afterComma.startsWith(kw)) return false;
|
|
2648
3230
|
const next = afterComma[kw.length];
|
|
2649
3231
|
return next === void 0 || /[\s:;.,!?()\d]/.test(next);
|
|
@@ -2670,6 +3252,28 @@ const extractValue = (text, triggerEnd, strategy, label) => {
|
|
|
2670
3252
|
text: extracted
|
|
2671
3253
|
};
|
|
2672
3254
|
}
|
|
3255
|
+
case "match-pattern": {
|
|
3256
|
+
const nlIdx = valueText.indexOf("\n");
|
|
3257
|
+
const searchText = nlIdx === -1 ? valueText : valueText.slice(0, nlIdx);
|
|
3258
|
+
if (searchText.length === 0) return null;
|
|
3259
|
+
const flags = (strategy.flags ?? "").replace(/[gy]/g, "");
|
|
3260
|
+
const anchoredSource = `^(?:${strategy.pattern})`;
|
|
3261
|
+
let re;
|
|
3262
|
+
try {
|
|
3263
|
+
re = new RegExp(anchoredSource, flags);
|
|
3264
|
+
} catch {
|
|
3265
|
+
return null;
|
|
3266
|
+
}
|
|
3267
|
+
const m = re.exec(searchText);
|
|
3268
|
+
if (!m || m[0].length === 0) return null;
|
|
3269
|
+
const matchStart = m.index;
|
|
3270
|
+
const matchEnd = matchStart + m[0].length;
|
|
3271
|
+
return {
|
|
3272
|
+
start: valueStart + matchStart,
|
|
3273
|
+
end: valueStart + matchEnd,
|
|
3274
|
+
text: m[0]
|
|
3275
|
+
};
|
|
3276
|
+
}
|
|
2673
3277
|
default: return null;
|
|
2674
3278
|
}
|
|
2675
3279
|
};
|
|
@@ -2689,7 +3293,8 @@ const processTriggerMatches = (allMatches, sliceStart, sliceEnd, fullText, rules
|
|
|
2689
3293
|
if (match.start > 0 && LETTER_RE.test(fullText[match.start - 1] ?? "")) continue;
|
|
2690
3294
|
const rule = rules[localIdx];
|
|
2691
3295
|
if (!rule) continue;
|
|
2692
|
-
|
|
3296
|
+
const triggerLastChar = rule.trigger.at(-1) ?? "";
|
|
3297
|
+
if (LETTER_RE.test(triggerLastChar) && LETTER_RE.test(fullText[match.end] ?? "")) continue;
|
|
2693
3298
|
const triggerEnd = match.end;
|
|
2694
3299
|
const rawValue = extractValue(fullText, triggerEnd, rule.strategy, rule.label);
|
|
2695
3300
|
const value = rawValue ? stripQuotes(rawValue) : null;
|
|
@@ -2698,7 +3303,7 @@ const processTriggerMatches = (allMatches, sliceStart, sliceEnd, fullText, rules
|
|
|
2698
3303
|
const entityStart = rule.includeTrigger ? match.start : value.start;
|
|
2699
3304
|
const entityEnd = value.end;
|
|
2700
3305
|
const entityText = fullText.slice(entityStart, entityEnd);
|
|
2701
|
-
const effectiveLabel = rule.label === "person" &&
|
|
3306
|
+
const effectiveLabel = rule.label === "person" && getLegalFormCheckRe().test(entityText) ? "organization" : rule.label;
|
|
2702
3307
|
results.push({
|
|
2703
3308
|
start: entityStart,
|
|
2704
3309
|
end: entityEnd,
|
|
@@ -2968,11 +3573,82 @@ const resolveCountries = (regions, countries) => {
|
|
|
2968
3573
|
return result;
|
|
2969
3574
|
};
|
|
2970
3575
|
//#endregion
|
|
3576
|
+
//#region src/util/homoglyphs.ts
|
|
3577
|
+
const CHAR_MAP$1 = new Map([
|
|
3578
|
+
[1040, 65],
|
|
3579
|
+
[1042, 66],
|
|
3580
|
+
[1045, 69],
|
|
3581
|
+
[1050, 75],
|
|
3582
|
+
[1052, 77],
|
|
3583
|
+
[1053, 72],
|
|
3584
|
+
[1054, 79],
|
|
3585
|
+
[1056, 80],
|
|
3586
|
+
[1057, 67],
|
|
3587
|
+
[1058, 84],
|
|
3588
|
+
[1061, 88],
|
|
3589
|
+
[1072, 97],
|
|
3590
|
+
[1077, 101],
|
|
3591
|
+
[1086, 111],
|
|
3592
|
+
[1088, 112],
|
|
3593
|
+
[1089, 99],
|
|
3594
|
+
[1091, 121],
|
|
3595
|
+
[1093, 120],
|
|
3596
|
+
[1110, 105],
|
|
3597
|
+
[1112, 106],
|
|
3598
|
+
[913, 65],
|
|
3599
|
+
[914, 66],
|
|
3600
|
+
[917, 69],
|
|
3601
|
+
[919, 72],
|
|
3602
|
+
[921, 73],
|
|
3603
|
+
[922, 75],
|
|
3604
|
+
[924, 77],
|
|
3605
|
+
[925, 78],
|
|
3606
|
+
[927, 79],
|
|
3607
|
+
[929, 80],
|
|
3608
|
+
[932, 84],
|
|
3609
|
+
[933, 89],
|
|
3610
|
+
[935, 88],
|
|
3611
|
+
[918, 90],
|
|
3612
|
+
[945, 97],
|
|
3613
|
+
[954, 107],
|
|
3614
|
+
[959, 111],
|
|
3615
|
+
[961, 112]
|
|
3616
|
+
]);
|
|
3617
|
+
/** Chunk size for `String.fromCharCode` to avoid hitting
|
|
3618
|
+
* the call-stack limit on very large strings. */
|
|
3619
|
+
const CHUNK_SIZE$1 = 8192;
|
|
3620
|
+
const normalizeHomoglyphs = (text) => {
|
|
3621
|
+
let hasSpecial = false;
|
|
3622
|
+
for (let i = 0; i < text.length; i++) if (CHAR_MAP$1.has(text.charCodeAt(i))) {
|
|
3623
|
+
hasSpecial = true;
|
|
3624
|
+
break;
|
|
3625
|
+
}
|
|
3626
|
+
if (!hasSpecial) return text;
|
|
3627
|
+
const len = text.length;
|
|
3628
|
+
const codes = new Uint16Array(len);
|
|
3629
|
+
for (let i = 0; i < len; i++) {
|
|
3630
|
+
const code = text.charCodeAt(i);
|
|
3631
|
+
codes[i] = CHAR_MAP$1.get(code) ?? code;
|
|
3632
|
+
}
|
|
3633
|
+
if (len <= CHUNK_SIZE$1) return String.fromCharCode(...codes);
|
|
3634
|
+
let result = "";
|
|
3635
|
+
for (let offset = 0; offset < len; offset += CHUNK_SIZE$1) {
|
|
3636
|
+
const end = Math.min(offset + CHUNK_SIZE$1, len);
|
|
3637
|
+
result += String.fromCharCode(...codes.subarray(offset, end));
|
|
3638
|
+
}
|
|
3639
|
+
return result;
|
|
3640
|
+
};
|
|
3641
|
+
//#endregion
|
|
2971
3642
|
//#region src/filters/false-positives.ts
|
|
2972
3643
|
const TEMPLATE_PLACEHOLDER_RE = /^(?:\.{3,}|_{3,}|\[[\w\s]+\]|\{[\w\s]+\})$/;
|
|
2973
3644
|
const POSTAL_CODE_RE = /\d{3}\s?\d{2}/;
|
|
2974
3645
|
const HAS_DIGIT_RE = /\d/;
|
|
2975
|
-
const
|
|
3646
|
+
const ADDRESS_COMPONENT_EXTRA_RE = /(?:^|\s)(?:č\.p\.|č\.ev\.|č\.|sídliště)(?=[\s,./]|$)/i;
|
|
3647
|
+
const BARE_COURS_PROSE_RE = /(?:^|\s)cours(?!\s+\p{Lu})(?=[\s,./]|$)/u;
|
|
3648
|
+
const isOnlyAmbiguousCours = (text) => {
|
|
3649
|
+
if (!BARE_COURS_PROSE_RE.test(text)) return false;
|
|
3650
|
+
return !hasAddressComponent(text.replace(/(?:^|\s)cours(?=[\s,./]|$)/gi, " "));
|
|
3651
|
+
};
|
|
2976
3652
|
const JURISDICTION_RE = /^(?:state|commonwealth|district|territory)\s+of\s+/i;
|
|
2977
3653
|
const MAX_ENTITY_LENGTH = {
|
|
2978
3654
|
organization: 80,
|
|
@@ -3055,6 +3731,42 @@ const loadGenericRoles = (ctx = defaultContext) => {
|
|
|
3055
3731
|
};
|
|
3056
3732
|
/** Sync accessor — returns empty set before init. */
|
|
3057
3733
|
const getGenericRoles = (ctx) => ctx.genericRoles ?? EMPTY_GENERIC_ROLES;
|
|
3734
|
+
const STREET_TYPES_SEED_RE = /(?:^|\s)(?:ul\.|ulice|nám\.|náměstí|tř\.|třída|nábř\.|nábřeží|bulvár)(?=[\s,./]|$)/i;
|
|
3735
|
+
let _streetTypesRe = STREET_TYPES_SEED_RE;
|
|
3736
|
+
let _streetTypesPromise = null;
|
|
3737
|
+
const escapeRegex$1 = (s) => s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
3738
|
+
const loadStreetTypeRegex = async () => {
|
|
3739
|
+
try {
|
|
3740
|
+
const data = (await import("./address-street-types.mjs")).default ?? {};
|
|
3741
|
+
const words = /* @__PURE__ */ new Set();
|
|
3742
|
+
for (const [key, val] of Object.entries(data)) {
|
|
3743
|
+
if (key.startsWith("_")) continue;
|
|
3744
|
+
if (!Array.isArray(val)) continue;
|
|
3745
|
+
for (const w of val) if (typeof w === "string" && w.length > 0) words.add(w.toLowerCase());
|
|
3746
|
+
}
|
|
3747
|
+
if (words.size === 0) {
|
|
3748
|
+
_streetTypesRe = STREET_TYPES_SEED_RE;
|
|
3749
|
+
return;
|
|
3750
|
+
}
|
|
3751
|
+
const alternation = [...words].sort((a, b) => b.length - a.length).map(escapeRegex$1).join("|");
|
|
3752
|
+
_streetTypesRe = new RegExp(`(?:^|\\s)(?:${alternation})(?=[\\s,./]|$)`, "iu");
|
|
3753
|
+
} catch {
|
|
3754
|
+
_streetTypesRe = STREET_TYPES_SEED_RE;
|
|
3755
|
+
}
|
|
3756
|
+
};
|
|
3757
|
+
/** Ensure street-type vocabulary is loaded. */
|
|
3758
|
+
const initAddressComponents = () => {
|
|
3759
|
+
if (!_streetTypesPromise) _streetTypesPromise = loadStreetTypeRegex();
|
|
3760
|
+
return _streetTypesPromise;
|
|
3761
|
+
};
|
|
3762
|
+
/**
|
|
3763
|
+
* True when `text` contains a recognised address
|
|
3764
|
+
* component: either a per-language street word (loaded
|
|
3765
|
+
* from JSON) or a language-agnostic anchor such as a
|
|
3766
|
+
* Czech house number form.
|
|
3767
|
+
*/
|
|
3768
|
+
const hasAddressComponent = (text) => _streetTypesRe.test(text) || ADDRESS_COMPONENT_EXTRA_RE.test(text);
|
|
3769
|
+
const isCallerOwnedEntity$3 = (entity) => entity.sourceDetail === "custom-deny-list" || entity.sourceDetail === "custom-regex";
|
|
3058
3770
|
/**
|
|
3059
3771
|
* Filter out entities that are likely false positives:
|
|
3060
3772
|
* template placeholders, clause/section numbers,
|
|
@@ -3067,9 +3779,17 @@ const filterFalsePositives = (entities, ctx = defaultContext, fullText) => {
|
|
|
3067
3779
|
const filtered = [];
|
|
3068
3780
|
const roles = getGenericRoles(ctx);
|
|
3069
3781
|
for (const entity of entities) {
|
|
3782
|
+
if (isCallerOwnedEntity$3(entity)) {
|
|
3783
|
+
filtered.push(entity);
|
|
3784
|
+
continue;
|
|
3785
|
+
}
|
|
3070
3786
|
const normalized = normalizeEntity(entity);
|
|
3071
3787
|
if (!normalized) continue;
|
|
3072
3788
|
const trimmed = normalized.text;
|
|
3789
|
+
if (isCallerOwnedEntity$3(normalized)) {
|
|
3790
|
+
filtered.push(normalized);
|
|
3791
|
+
continue;
|
|
3792
|
+
}
|
|
3073
3793
|
if (TEMPLATE_PLACEHOLDER_RE.test(trimmed)) continue;
|
|
3074
3794
|
const maxLen = MAX_ENTITY_LENGTH[normalized.label];
|
|
3075
3795
|
if (maxLen && trimmed.length > maxLen && normalized.source !== "legal-form") continue;
|
|
@@ -3080,13 +3800,15 @@ const filterFalsePositives = (entities, ctx = defaultContext, fullText) => {
|
|
|
3080
3800
|
if (normalized.label === "person" && HAS_DIGIT_RE.test(trimmed)) continue;
|
|
3081
3801
|
if (normalized.label === "person") {
|
|
3082
3802
|
const tokens = trimmed.split(/\s+/u);
|
|
3083
|
-
const last = tokens.at(-1)?.replace(/[.,;:!?]+$/u, "")
|
|
3084
|
-
|
|
3803
|
+
const last = tokens.at(-1)?.replace(/[.,;:!?]+$/u, "");
|
|
3804
|
+
const lastFolded = last ? normalizeHomoglyphs(last).toLowerCase() : void 0;
|
|
3805
|
+
if (tokens.length > 1 && lastFolded && PERSON_TRAILING_NOUNS.has(lastFolded)) continue;
|
|
3085
3806
|
}
|
|
3086
|
-
if ((normalized.label === "person" || normalized.label === "organization") && roles.has(trimmed.toLowerCase())) continue;
|
|
3807
|
+
if ((normalized.label === "person" || normalized.label === "organization") && roles.has(normalizeHomoglyphs(trimmed).toLowerCase())) continue;
|
|
3087
3808
|
if (normalized.label === "organization" && normalized.source === "legal-form" && trimmed === trimmed.toUpperCase() && LEGAL_FORM_HEADING_RE.test(trimmed)) continue;
|
|
3088
|
-
if (normalized.label === "address" && trimmed.length > 40 && !POSTAL_CODE_RE.test(trimmed) && !HAS_DIGIT_RE.test(trimmed) && !
|
|
3089
|
-
if (normalized.label === "address" && normalized.source === "trigger" && !HAS_DIGIT_RE.test(trimmed) && !
|
|
3809
|
+
if (normalized.label === "address" && trimmed.length > 40 && !POSTAL_CODE_RE.test(trimmed) && !HAS_DIGIT_RE.test(trimmed) && !hasAddressComponent(trimmed) && !JURISDICTION_RE.test(trimmed)) continue;
|
|
3810
|
+
if (normalized.label === "address" && normalized.source === "trigger" && !HAS_DIGIT_RE.test(trimmed) && !hasAddressComponent(trimmed) && !JURISDICTION_RE.test(trimmed)) continue;
|
|
3811
|
+
if (normalized.label === "address" && normalized.source === "trigger" && !HAS_DIGIT_RE.test(trimmed) && isOnlyAmbiguousCours(trimmed)) continue;
|
|
3090
3812
|
if (normalized.label === "address" && SIGNING_CLAUSE_ADDRESS_RE.test(trimmed)) continue;
|
|
3091
3813
|
filtered.push(normalized);
|
|
3092
3814
|
}
|
|
@@ -3243,7 +3965,124 @@ const loadPersonStopwords = (ctx) => {
|
|
|
3243
3965
|
const EMPTY_PERSON_STOPWORDS = /* @__PURE__ */ new Set();
|
|
3244
3966
|
/** Sync accessor — returns empty set before init. */
|
|
3245
3967
|
const getPersonStopwords = (ctx) => ctx.personStopwords ?? EMPTY_PERSON_STOPWORDS;
|
|
3968
|
+
const loadAddressStopwords = (ctx) => {
|
|
3969
|
+
if (ctx.addressStopwordsPromise) return ctx.addressStopwordsPromise;
|
|
3970
|
+
ctx.addressStopwordsPromise = (async () => {
|
|
3971
|
+
try {
|
|
3972
|
+
const mod = await import("./address-stopwords.mjs");
|
|
3973
|
+
const set = new Set(mod.default?.words ?? []);
|
|
3974
|
+
ctx.addressStopwords = set;
|
|
3975
|
+
return set;
|
|
3976
|
+
} catch {
|
|
3977
|
+
const empty = /* @__PURE__ */ new Set();
|
|
3978
|
+
ctx.addressStopwords = empty;
|
|
3979
|
+
return empty;
|
|
3980
|
+
}
|
|
3981
|
+
})();
|
|
3982
|
+
return ctx.addressStopwordsPromise;
|
|
3983
|
+
};
|
|
3984
|
+
const EMPTY_ADDRESS_STOPWORDS = /* @__PURE__ */ new Set();
|
|
3985
|
+
const getAddressStopwords = (ctx) => ctx.addressStopwords ?? EMPTY_ADDRESS_STOPWORDS;
|
|
3986
|
+
/**
|
|
3987
|
+
* Word characters in unicode property notation. The check is
|
|
3988
|
+
* "single-token" — no internal whitespace — and we keep dashes
|
|
3989
|
+
* tokens like "K-12" out by requiring the surface to be a single
|
|
3990
|
+
* uninterrupted run of letters or marks.
|
|
3991
|
+
*/
|
|
3992
|
+
const SINGLE_WORD_RE = /^\p{L}+$/u;
|
|
3993
|
+
/**
|
|
3994
|
+
* Format-level address signals — structurally numeric or
|
|
3995
|
+
* 2-letter-state patterns, language-agnostic. The street-type
|
|
3996
|
+
* vocabulary is loaded from `address-street-types.json` so new
|
|
3997
|
+
* languages contribute via data, not code.
|
|
3998
|
+
* - `,\s*[A-Z]{2}\b` → US state abbreviation after a comma
|
|
3999
|
+
* - `\b\d{5}(?:-\d{4})?\b` → US ZIP / ZIP+4
|
|
4000
|
+
* - `\b\d{3}\s\d{2}\b` → Czech/Slovak postal block (140 00)
|
|
4001
|
+
* - `\b\d{2}-\d{3}\b` → Polish postal code (00-950)
|
|
4002
|
+
*/
|
|
4003
|
+
const ADDRESS_FORMAT_RE = /,\s*\p{Lu}{2}\b|\b\d{5}(?:-\d{4})?\b|\b\d{3}\s\d{2}\b|\b\d{2}-\d{3}\b/u;
|
|
4004
|
+
let cachedStreetTypeRe = null;
|
|
4005
|
+
let streetTypeReLoaded = false;
|
|
4006
|
+
const loadStreetTypeRe = async () => {
|
|
4007
|
+
if (streetTypeReLoaded) return cachedStreetTypeRe;
|
|
4008
|
+
try {
|
|
4009
|
+
const config = (await import("./address-street-types.mjs")).default ?? {};
|
|
4010
|
+
const words = [];
|
|
4011
|
+
for (const value of Object.values(config)) {
|
|
4012
|
+
if (!Array.isArray(value)) continue;
|
|
4013
|
+
for (const word of value) if (typeof word === "string" && word.length > 0) words.push(word);
|
|
4014
|
+
}
|
|
4015
|
+
if (words.length === 0) cachedStreetTypeRe = null;
|
|
4016
|
+
else {
|
|
4017
|
+
words.sort((a, b) => b.length - a.length);
|
|
4018
|
+
const isLetterDigit = (c) => /[\p{L}\p{N}]/u.test(c);
|
|
4019
|
+
const wordLikeTail = [];
|
|
4020
|
+
const punctTail = [];
|
|
4021
|
+
for (const w of words) {
|
|
4022
|
+
const escaped = w.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
4023
|
+
if (isLetterDigit(w.at(-1) ?? "")) wordLikeTail.push(escaped);
|
|
4024
|
+
else punctTail.push(escaped);
|
|
4025
|
+
}
|
|
4026
|
+
const branches = [];
|
|
4027
|
+
if (wordLikeTail.length > 0) branches.push(`(?:${wordLikeTail.join("|")})(?![\\p{L}\\p{N}])`);
|
|
4028
|
+
if (punctTail.length > 0) branches.push(`(?:${punctTail.join("|")})`);
|
|
4029
|
+
cachedStreetTypeRe = new RegExp(`(?<![\\p{L}\\p{N}])(?:${branches.join("|")})`, "iu");
|
|
4030
|
+
}
|
|
4031
|
+
} catch {
|
|
4032
|
+
cachedStreetTypeRe = null;
|
|
4033
|
+
}
|
|
4034
|
+
streetTypeReLoaded = true;
|
|
4035
|
+
return cachedStreetTypeRe;
|
|
4036
|
+
};
|
|
4037
|
+
const getStreetTypeRe = () => streetTypeReLoaded ? cachedStreetTypeRe : null;
|
|
4038
|
+
const hasAdjacentAddressEvidence = (fullText, start, end) => {
|
|
4039
|
+
const window = fullText.slice(Math.max(0, start - 40), Math.min(fullText.length, end + 40));
|
|
4040
|
+
if (ADDRESS_FORMAT_RE.test(window)) return true;
|
|
4041
|
+
const streetRe = getStreetTypeRe();
|
|
4042
|
+
return streetRe !== null && streetRe.test(window);
|
|
4043
|
+
};
|
|
4044
|
+
/**
|
|
4045
|
+
* Capitalised words that almost never start a person name. When a
|
|
4046
|
+
* single-token surname candidate is immediately followed by one of
|
|
4047
|
+
* these, the "next-word is uppercase" promotion heuristic would
|
|
4048
|
+
* otherwise turn section headings ("Purchase Price↵The Purchaser
|
|
4049
|
+
* undertakes…") into spurious person hits. Kept narrow on purpose;
|
|
4050
|
+
* the surrounding pipeline still chains real names via the deny-list
|
|
4051
|
+
* cascade when both halves are surnames.
|
|
4052
|
+
*/
|
|
4053
|
+
const SENTENCE_STARTER_WORDS = new Set([
|
|
4054
|
+
"The",
|
|
4055
|
+
"This",
|
|
4056
|
+
"These",
|
|
4057
|
+
"Those",
|
|
4058
|
+
"An",
|
|
4059
|
+
"Any",
|
|
4060
|
+
"All",
|
|
4061
|
+
"Each",
|
|
4062
|
+
"Every",
|
|
4063
|
+
"No",
|
|
4064
|
+
"Now",
|
|
4065
|
+
"Whereas",
|
|
4066
|
+
"Whereby",
|
|
4067
|
+
"Wherein",
|
|
4068
|
+
"Whereof",
|
|
4069
|
+
"Notwithstanding",
|
|
4070
|
+
"Subject",
|
|
4071
|
+
"In",
|
|
4072
|
+
"On",
|
|
4073
|
+
"At",
|
|
4074
|
+
"By",
|
|
4075
|
+
"For",
|
|
4076
|
+
"If",
|
|
4077
|
+
"Upon",
|
|
4078
|
+
"Unless",
|
|
4079
|
+
"Until",
|
|
4080
|
+
"Provided",
|
|
4081
|
+
"Pursuant",
|
|
4082
|
+
"Such"
|
|
4083
|
+
]);
|
|
3246
4084
|
const PERSON_CHAIN_BREAK_RE = /[!?;:]|,/u;
|
|
4085
|
+
const WORD_CHAR_RE$1 = /[\p{L}\p{N}]/u;
|
|
3247
4086
|
const isInitialContinuationGap = (text, gap) => /^\p{Lu}$/u.test(text) && /^\.[^\S\n]{1,2}$/u.test(gap) || /^[^\S\n]{1,2}(?:\p{Lu}\.[^\S\n]{1,2})+$/u.test(gap);
|
|
3248
4087
|
/**
|
|
3249
4088
|
* Resolve which dictionaries to load based on country
|
|
@@ -3261,32 +4100,38 @@ const buildDenyList = async (config, ctx = defaultContext) => {
|
|
|
3261
4100
|
loadStopwords(ctx),
|
|
3262
4101
|
loadAllowList(ctx),
|
|
3263
4102
|
loadPersonStopwords(ctx),
|
|
4103
|
+
loadAddressStopwords(ctx),
|
|
4104
|
+
loadStreetTypeRe(),
|
|
3264
4105
|
loadGenericRoles(ctx)
|
|
3265
4106
|
]);
|
|
3266
4107
|
const dictionaries = config.dictionaries;
|
|
3267
4108
|
const hasDenyList = dictionaries?.denyList && dictionaries?.denyListMeta;
|
|
3268
4109
|
const hasCities = dictionaries?.cities && dictionaries.cities.length > 0;
|
|
3269
|
-
|
|
4110
|
+
const hasCustomDenyList = config.customDenyList !== void 0 && config.customDenyList.length > 0;
|
|
4111
|
+
if (!hasDenyList && !hasCities && !hasCustomDenyList) return buildNameCorpusOnly(config, ctx);
|
|
3270
4112
|
const allowedCountries = resolveCountries(config.denyListRegions, config.denyListCountries);
|
|
3271
4113
|
const excluded = config.denyListExcludeCategories;
|
|
3272
4114
|
const excludeCategories = excluded ? new Set(excluded) : /* @__PURE__ */ new Set();
|
|
3273
4115
|
const patternList = [];
|
|
3274
4116
|
const labelList = [];
|
|
4117
|
+
const customLabelList = [];
|
|
3275
4118
|
const sourceList = [];
|
|
3276
4119
|
const patternIndex = /* @__PURE__ */ new Map();
|
|
3277
|
-
const addDenyListEntry = (entry, label) => {
|
|
3278
|
-
const normalized = normalizeForSearch(entry).replace(/[|\\]/g, "");
|
|
4120
|
+
const addDenyListEntry = (entry, label, source = "deny-list") => {
|
|
4121
|
+
const normalized = source === "custom-deny-list" ? normalizeForSearch(entry) : normalizeForSearch(entry).replace(/[|\\]/g, "");
|
|
3279
4122
|
if (normalized.length === 0) return;
|
|
3280
4123
|
const lower = normalized.toLowerCase();
|
|
3281
4124
|
const existing = patternIndex.get(lower);
|
|
3282
4125
|
if (existing !== void 0) {
|
|
3283
4126
|
if (!labelList[existing].includes(label)) labelList[existing].push(label);
|
|
3284
|
-
if (!sourceList[existing].includes(
|
|
4127
|
+
if (!sourceList[existing].includes(source)) sourceList[existing].push(source);
|
|
4128
|
+
if (source === "custom-deny-list" && !customLabelList[existing].includes(label)) customLabelList[existing].push(label);
|
|
3285
4129
|
} else {
|
|
3286
4130
|
patternIndex.set(lower, patternList.length);
|
|
3287
4131
|
patternList.push(normalized);
|
|
3288
4132
|
labelList.push([label]);
|
|
3289
|
-
|
|
4133
|
+
customLabelList.push(source === "custom-deny-list" ? [label] : []);
|
|
4134
|
+
sourceList.push([source]);
|
|
3290
4135
|
}
|
|
3291
4136
|
};
|
|
3292
4137
|
if (hasDenyList) {
|
|
@@ -3304,10 +4149,15 @@ const buildDenyList = async (config, ctx = defaultContext) => {
|
|
|
3304
4149
|
}
|
|
3305
4150
|
}
|
|
3306
4151
|
if (hasCities && !excludeCategories.has("Places")) for (const entry of dictionaries.cities) addDenyListEntry(entry, "address");
|
|
4152
|
+
if (hasCustomDenyList) for (const entry of config.customDenyList) {
|
|
4153
|
+
addDenyListEntry(entry.value, entry.label, "custom-deny-list");
|
|
4154
|
+
for (const variant of entry.variants ?? []) addDenyListEntry(variant, entry.label, "custom-deny-list");
|
|
4155
|
+
}
|
|
3307
4156
|
appendNameCorpusEntries(config, ctx, patternList, labelList, sourceList, patternIndex);
|
|
3308
4157
|
if (patternList.length === 0) return null;
|
|
3309
4158
|
return {
|
|
3310
4159
|
labels: labelList,
|
|
4160
|
+
customLabels: customLabelList,
|
|
3311
4161
|
originals: patternList,
|
|
3312
4162
|
sources: sourceList
|
|
3313
4163
|
};
|
|
@@ -3324,11 +4174,13 @@ const buildNameCorpusOnly = (config, ctx) => {
|
|
|
3324
4174
|
if ((excluded ? new Set(excluded) : /* @__PURE__ */ new Set()).has("Names")) return null;
|
|
3325
4175
|
const patternList = [];
|
|
3326
4176
|
const labelList = [];
|
|
4177
|
+
const customLabelList = [];
|
|
3327
4178
|
const sourceList = [];
|
|
3328
4179
|
appendNameCorpusEntries(config, ctx, patternList, labelList, sourceList, /* @__PURE__ */ new Map());
|
|
3329
4180
|
if (patternList.length === 0) return null;
|
|
3330
4181
|
return {
|
|
3331
4182
|
labels: labelList,
|
|
4183
|
+
customLabels: customLabelList,
|
|
3332
4184
|
originals: patternList,
|
|
3333
4185
|
sources: sourceList
|
|
3334
4186
|
};
|
|
@@ -3374,6 +4226,14 @@ const appendNameCorpusEntries = (config, ctx, patternList, labelList, sourceList
|
|
|
3374
4226
|
}
|
|
3375
4227
|
}
|
|
3376
4228
|
};
|
|
4229
|
+
const customMatchHasValidEdges = (fullText, start, end, pattern) => {
|
|
4230
|
+
if (!WORD_CHAR_RE$1.test(pattern)) return true;
|
|
4231
|
+
const prev = fullText[start - 1] ?? "";
|
|
4232
|
+
const next = fullText[end] ?? "";
|
|
4233
|
+
if (WORD_CHAR_RE$1.test(prev)) return false;
|
|
4234
|
+
if (WORD_CHAR_RE$1.test(next)) return false;
|
|
4235
|
+
return true;
|
|
4236
|
+
};
|
|
3377
4237
|
/**
|
|
3378
4238
|
* Ensure all deny-list support data (stopwords, allow
|
|
3379
4239
|
* list, person stopwords, generic roles) is loaded on
|
|
@@ -3388,6 +4248,8 @@ const ensureDenyListData = async (ctx = defaultContext, dictionaries) => {
|
|
|
3388
4248
|
loadStopwords(ctx),
|
|
3389
4249
|
loadAllowList(ctx),
|
|
3390
4250
|
loadPersonStopwords(ctx),
|
|
4251
|
+
loadAddressStopwords(ctx),
|
|
4252
|
+
loadStreetTypeRe(),
|
|
3391
4253
|
loadGenericRoles(ctx)
|
|
3392
4254
|
]);
|
|
3393
4255
|
};
|
|
@@ -3411,18 +4273,24 @@ const processDenyListMatches = (allMatches, sliceStart, sliceEnd, fullText, data
|
|
|
3411
4273
|
const idx = match.pattern;
|
|
3412
4274
|
if (idx < sliceStart || idx >= sliceEnd) continue;
|
|
3413
4275
|
const localIdx = idx - sliceStart;
|
|
3414
|
-
const
|
|
3415
|
-
if (!UPPER_START_RE.test(sourceChar)) continue;
|
|
4276
|
+
const sources = data.sources[localIdx] ?? [];
|
|
3416
4277
|
const matchText = fullText.slice(match.start, match.end);
|
|
4278
|
+
const sourceChar = fullText[match.start] ?? "";
|
|
3417
4279
|
const keyword = matchText.toLowerCase();
|
|
3418
|
-
if (getStopwords(ctx).has(keyword) || getAllowList(ctx).has(keyword)) continue;
|
|
3419
|
-
if (ALL_UPPER_RE.test(matchText)) continue;
|
|
3420
4280
|
const labels = data.labels[localIdx];
|
|
3421
|
-
|
|
4281
|
+
const pattern = data.originals[localIdx] ?? "";
|
|
4282
|
+
const customPatternLabels = data.customLabels[localIdx] ?? [];
|
|
4283
|
+
const customEdgesAreValid = customMatchHasValidEdges(fullText, match.start, match.end, pattern);
|
|
4284
|
+
const customLabels = customEdgesAreValid ? customPatternLabels : [];
|
|
4285
|
+
if ((!labels || labels.length === 0) && customLabels.length === 0) continue;
|
|
4286
|
+
const curatedLabels = UPPER_START_RE.test(sourceChar) && !getStopwords(ctx).has(keyword) && !getAllowList(ctx).has(keyword) && !ALL_UPPER_RE.test(matchText) ? (labels ?? []).filter((label) => !customPatternLabels.includes(label) && customEdgesAreValid) : [];
|
|
4287
|
+
if (curatedLabels.length === 0 && customLabels.length === 0) continue;
|
|
3422
4288
|
const entry = {
|
|
3423
4289
|
start: match.start,
|
|
3424
4290
|
end: match.end,
|
|
3425
|
-
labels,
|
|
4291
|
+
labels: curatedLabels,
|
|
4292
|
+
customLabels,
|
|
4293
|
+
sources,
|
|
3426
4294
|
text: matchText,
|
|
3427
4295
|
patternIdx: localIdx
|
|
3428
4296
|
};
|
|
@@ -3433,22 +4301,35 @@ const processDenyListMatches = (allMatches, sliceStart, sliceEnd, fullText, data
|
|
|
3433
4301
|
const results = [];
|
|
3434
4302
|
const nameHits = [];
|
|
3435
4303
|
for (const [, matches] of Array.from(matchesByPattern)) {
|
|
3436
|
-
|
|
3437
|
-
|
|
3438
|
-
const hasPerson = first.labels.includes("person");
|
|
3439
|
-
const nonPersonLabels = first.labels.filter((l) => l !== "person");
|
|
3440
|
-
if (hasPerson) {
|
|
3441
|
-
const keyword = first.text.toLowerCase();
|
|
3442
|
-
if (!getPersonStopwords(ctx).has(keyword)) for (const m of matches) nameHits.push(m);
|
|
3443
|
-
}
|
|
3444
|
-
for (const m of matches) for (const label of nonPersonLabels) results.push({
|
|
4304
|
+
if (!matches[0]) continue;
|
|
4305
|
+
for (const m of matches) for (const label of m.customLabels) results.push({
|
|
3445
4306
|
start: m.start,
|
|
3446
4307
|
end: m.end,
|
|
3447
4308
|
label,
|
|
3448
4309
|
text: m.text,
|
|
3449
4310
|
score: .9,
|
|
3450
|
-
source: DETECTION_SOURCES.DENY_LIST
|
|
4311
|
+
source: DETECTION_SOURCES.DENY_LIST,
|
|
4312
|
+
sourceDetail: "custom-deny-list"
|
|
3451
4313
|
});
|
|
4314
|
+
for (const m of matches) {
|
|
4315
|
+
if (m.labels.includes("person")) {
|
|
4316
|
+
const keyword = m.text.toLowerCase();
|
|
4317
|
+
if (!getPersonStopwords(ctx).has(keyword)) nameHits.push(m);
|
|
4318
|
+
}
|
|
4319
|
+
const nonPersonLabels = m.labels.filter((l) => l !== "person");
|
|
4320
|
+
const suppressAddress = SINGLE_WORD_RE.test(m.text) && getAddressStopwords(ctx).has(m.text.toLowerCase()) && !hasAdjacentAddressEvidence(fullText, m.start, m.end);
|
|
4321
|
+
for (const label of nonPersonLabels) {
|
|
4322
|
+
if (label === "address" && suppressAddress) continue;
|
|
4323
|
+
results.push({
|
|
4324
|
+
start: m.start,
|
|
4325
|
+
end: m.end,
|
|
4326
|
+
label,
|
|
4327
|
+
text: m.text,
|
|
4328
|
+
score: .9,
|
|
4329
|
+
source: DETECTION_SOURCES.DENY_LIST
|
|
4330
|
+
});
|
|
4331
|
+
}
|
|
4332
|
+
}
|
|
3452
4333
|
}
|
|
3453
4334
|
nameHits.sort((a, b) => a.start - b.start);
|
|
3454
4335
|
const nameConsumed = /* @__PURE__ */ new Set();
|
|
@@ -3479,6 +4360,8 @@ const processDenyListMatches = (allMatches, sliceStart, sliceEnd, fullText, data
|
|
|
3479
4360
|
const afterEnd = last.end;
|
|
3480
4361
|
const rest = fullText.slice(afterEnd).trimStart();
|
|
3481
4362
|
if (!(rest.length > 1 && /^\p{Lu}\p{Ll}/u.test(rest))) continue;
|
|
4363
|
+
const nextWord = /^\p{L}+/u.exec(rest)?.[0] ?? "";
|
|
4364
|
+
if (SENTENCE_STARTER_WORDS.has(nextWord)) continue;
|
|
3482
4365
|
}
|
|
3483
4366
|
results.push({
|
|
3484
4367
|
start: first.start,
|
|
@@ -3528,6 +4411,7 @@ const TRAILING_WORD_EXCLUSIONS = new Set([
|
|
|
3528
4411
|
const extendCityDistricts = (entities, fullText) => {
|
|
3529
4412
|
for (const entity of entities) {
|
|
3530
4413
|
if (entity.label !== "address") continue;
|
|
4414
|
+
if (entity.sourceDetail === "custom-deny-list") continue;
|
|
3531
4415
|
const afterMatch = fullText.slice(entity.end);
|
|
3532
4416
|
const suffixM = DISTRICT_SUFFIX_RE.exec(afterMatch);
|
|
3533
4417
|
if (suffixM) {
|
|
@@ -3589,6 +4473,15 @@ const extendPersonName = (text, start, end, ctx) => {
|
|
|
3589
4473
|
};
|
|
3590
4474
|
//#endregion
|
|
3591
4475
|
//#region src/detectors/address-seeds.ts
|
|
4476
|
+
/**
|
|
4477
|
+
* Trailing chars that should never end an address span. Combines
|
|
4478
|
+
* structural separators, opening brackets, and typographic quote
|
|
4479
|
+
* variants so a span like `… GA 30326, USA (the "Premises")`
|
|
4480
|
+
* does not pull the parenthetical opener into the address.
|
|
4481
|
+
* The prime (`′`) catches measurement tails (`5′`) that the
|
|
4482
|
+
* cluster occasionally absorbs.
|
|
4483
|
+
*/
|
|
4484
|
+
const ADDRESS_TRAILING_TRIM_RE = new RegExp(`[,;:\\s${OPENING_BRACKETS_INNER}${QUOTE_DOUBLE_INNER}${QUOTE_SINGLE_INNER}′]`, "u");
|
|
3592
4485
|
let cachedBoundaryRe = null;
|
|
3593
4486
|
const loadBoundaryWords = async () => {
|
|
3594
4487
|
try {
|
|
@@ -3597,6 +4490,56 @@ const loadBoundaryWords = async () => {
|
|
|
3597
4490
|
return {};
|
|
3598
4491
|
}
|
|
3599
4492
|
};
|
|
4493
|
+
const BR_CEP_CONTEXT_WINDOW = 200;
|
|
4494
|
+
let cachedBrCepContextRe = null;
|
|
4495
|
+
let cachedBrCepContextPromise = null;
|
|
4496
|
+
const loadBrCueWords = async () => {
|
|
4497
|
+
const sources = await Promise.all([(async () => {
|
|
4498
|
+
try {
|
|
4499
|
+
return (await import("./address-street-types.mjs")).default["pt-br"];
|
|
4500
|
+
} catch {
|
|
4501
|
+
return;
|
|
4502
|
+
}
|
|
4503
|
+
})(), (async () => {
|
|
4504
|
+
try {
|
|
4505
|
+
return (await import("./address-boundaries.mjs")).default["pt-br"];
|
|
4506
|
+
} catch {
|
|
4507
|
+
return;
|
|
4508
|
+
}
|
|
4509
|
+
})()]);
|
|
4510
|
+
const out = [];
|
|
4511
|
+
const seen = /* @__PURE__ */ new Set();
|
|
4512
|
+
for (const entry of sources) {
|
|
4513
|
+
if (!Array.isArray(entry)) continue;
|
|
4514
|
+
for (const word of entry) {
|
|
4515
|
+
if (typeof word !== "string" || word.length === 0) continue;
|
|
4516
|
+
const key = word.toLowerCase();
|
|
4517
|
+
if (seen.has(key)) continue;
|
|
4518
|
+
seen.add(key);
|
|
4519
|
+
out.push(word);
|
|
4520
|
+
}
|
|
4521
|
+
}
|
|
4522
|
+
return out;
|
|
4523
|
+
};
|
|
4524
|
+
const getBrCepContextRe = async () => {
|
|
4525
|
+
if (cachedBrCepContextRe !== null) return cachedBrCepContextRe;
|
|
4526
|
+
if (cachedBrCepContextPromise) return cachedBrCepContextPromise;
|
|
4527
|
+
cachedBrCepContextPromise = (async () => {
|
|
4528
|
+
const words = await loadBrCueWords();
|
|
4529
|
+
if (words.length === 0) return null;
|
|
4530
|
+
const escaped = words.toSorted((a, b) => b.length - a.length).map((w) => w.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"));
|
|
4531
|
+
const re = new RegExp(`(?<![\\p{L}\\p{N}])(?:${escaped.join("|")})(?![\\p{L}\\p{N}])`, "iu");
|
|
4532
|
+
cachedBrCepContextRe = re;
|
|
4533
|
+
return re;
|
|
4534
|
+
})();
|
|
4535
|
+
return cachedBrCepContextPromise;
|
|
4536
|
+
};
|
|
4537
|
+
const hasBrCueNearby = (fullText, start, end, re) => {
|
|
4538
|
+
const windowStart = Math.max(0, start - BR_CEP_CONTEXT_WINDOW);
|
|
4539
|
+
const windowEnd = Math.min(fullText.length, end + BR_CEP_CONTEXT_WINDOW);
|
|
4540
|
+
const window = fullText.slice(windowStart, windowEnd);
|
|
4541
|
+
return new RegExp(re.source, re.flags.replace("g", "")).test(window);
|
|
4542
|
+
};
|
|
3600
4543
|
/**
|
|
3601
4544
|
* Build regex for boundary words. Matches any
|
|
3602
4545
|
* boundary word preceded by a word boundary.
|
|
@@ -3610,7 +4553,7 @@ const getBoundaryRe = async () => {
|
|
|
3610
4553
|
for (const word of entries) words.push(word.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"));
|
|
3611
4554
|
}
|
|
3612
4555
|
words.sort((a, b) => b.length - a.length);
|
|
3613
|
-
cachedBoundaryRe = words.length > 0 ? new RegExp(
|
|
4556
|
+
cachedBoundaryRe = words.length > 0 ? new RegExp(`(?<![\\p{L}\\p{N}])(?:${words.join("|")})(?![\\p{L}\\p{N}])`, "iu") : /(?!)/;
|
|
3614
4557
|
return cachedBoundaryRe;
|
|
3615
4558
|
};
|
|
3616
4559
|
/**
|
|
@@ -3632,7 +4575,7 @@ const buildStreetTypePatterns = async () => {
|
|
|
3632
4575
|
}
|
|
3633
4576
|
return words;
|
|
3634
4577
|
};
|
|
3635
|
-
const collectSeeds = (allMatches, sliceStart, sliceEnd, fullText, existingEntities) => {
|
|
4578
|
+
const collectSeeds = (allMatches, sliceStart, sliceEnd, fullText, existingEntities, brCepContextRe) => {
|
|
3636
4579
|
const seeds = [];
|
|
3637
4580
|
for (const match of allMatches) {
|
|
3638
4581
|
const idx = match.pattern;
|
|
@@ -3646,6 +4589,7 @@ const collectSeeds = (allMatches, sliceStart, sliceEnd, fullText, existingEntiti
|
|
|
3646
4589
|
}
|
|
3647
4590
|
for (const e of existingEntities) {
|
|
3648
4591
|
if (e.label !== "address") continue;
|
|
4592
|
+
if (e.sourceDetail === "custom-deny-list") continue;
|
|
3649
4593
|
if (e.source === "deny-list") seeds.push({
|
|
3650
4594
|
type: "city",
|
|
3651
4595
|
start: e.start,
|
|
@@ -3665,18 +4609,41 @@ const collectSeeds = (allMatches, sliceStart, sliceEnd, fullText, existingEntiti
|
|
|
3665
4609
|
text: e.text
|
|
3666
4610
|
});
|
|
3667
4611
|
}
|
|
3668
|
-
const postalRe = /\b(?:\d{3}\s\d{2}|\d{2}-\d{3})\b/g;
|
|
4612
|
+
const postalRe = /\b(?:\d{3}\s\d{2}|\d{2}-\d{3}|\d{5}-\d{3})\b/g;
|
|
3669
4613
|
let postalMatch;
|
|
3670
4614
|
while ((postalMatch = postalRe.exec(fullText)) !== null) {
|
|
3671
4615
|
const start = postalMatch.index;
|
|
3672
4616
|
const end = start + postalMatch[0].length;
|
|
3673
|
-
if (
|
|
4617
|
+
if (seeds.some((s) => s.start <= start && s.end >= end)) continue;
|
|
4618
|
+
if (/^\d{5}-\d{3}$/.test(postalMatch[0]) && (brCepContextRe === null || !hasBrCueNearby(fullText, start, end, brCepContextRe))) continue;
|
|
4619
|
+
seeds.push({
|
|
3674
4620
|
type: "postal-code",
|
|
3675
4621
|
start,
|
|
3676
4622
|
end,
|
|
3677
4623
|
text: postalMatch[0]
|
|
3678
4624
|
});
|
|
3679
4625
|
}
|
|
4626
|
+
const itCapRe = /\b\d{5}(?=\s+\p{Lu}\p{L}+)/gu;
|
|
4627
|
+
let itCapMatch;
|
|
4628
|
+
while ((itCapMatch = itCapRe.exec(fullText)) !== null) {
|
|
4629
|
+
const start = itCapMatch.index;
|
|
4630
|
+
const end = start + itCapMatch[0].length;
|
|
4631
|
+
if (seeds.some((s) => s.start <= start && s.end >= end)) continue;
|
|
4632
|
+
if (!seeds.some((s) => {
|
|
4633
|
+
if (Math.abs(s.start - start) > 80) return false;
|
|
4634
|
+
if (s.type === "address-trigger") return true;
|
|
4635
|
+
if (s.type === "city") return true;
|
|
4636
|
+
if (s.type === "postal-code") return true;
|
|
4637
|
+
if (s.type === "street-word") return s.text.toLowerCase() !== "via";
|
|
4638
|
+
return false;
|
|
4639
|
+
})) continue;
|
|
4640
|
+
seeds.push({
|
|
4641
|
+
type: "postal-code",
|
|
4642
|
+
start,
|
|
4643
|
+
end,
|
|
4644
|
+
text: itCapMatch[0]
|
|
4645
|
+
});
|
|
4646
|
+
}
|
|
3680
4647
|
const streetNumRe = /\b(\p{Lu}\p{Ll}{2,})\s+(\d{1,5}(?:\/\d{1,5})?)\s*[,\n]/gu;
|
|
3681
4648
|
let streetMatch;
|
|
3682
4649
|
while ((streetMatch = streetNumRe.exec(fullText)) !== null) {
|
|
@@ -3734,6 +4701,10 @@ const scoreCluster = (cluster) => {
|
|
|
3734
4701
|
const NON_ADDRESS_LABELS = new Set([
|
|
3735
4702
|
"registration number",
|
|
3736
4703
|
"tax identification number",
|
|
4704
|
+
"national identification number",
|
|
4705
|
+
"social security number",
|
|
4706
|
+
"birth number",
|
|
4707
|
+
"identity card number",
|
|
3737
4708
|
"person",
|
|
3738
4709
|
"bank account number",
|
|
3739
4710
|
"email address",
|
|
@@ -3770,7 +4741,7 @@ const expandCluster = async (fullText, cluster, existingEntities) => {
|
|
|
3770
4741
|
const doubleNewline = remaining.indexOf("\n\n");
|
|
3771
4742
|
if (doubleNewline !== -1 && doubleNewline < nearestBoundary) nearestBoundary = doubleNewline;
|
|
3772
4743
|
rightPos = end + remaining.slice(0, nearestBoundary).trimEnd().length;
|
|
3773
|
-
while (rightPos > end &&
|
|
4744
|
+
while (rightPos > end && ADDRESS_TRAILING_TRIM_RE.test(fullText[rightPos - 1] ?? "")) rightPos--;
|
|
3774
4745
|
return {
|
|
3775
4746
|
start: Math.min(leftPos, start),
|
|
3776
4747
|
end: Math.max(rightPos, end)
|
|
@@ -3787,7 +4758,7 @@ const expandCluster = async (fullText, cluster, existingEntities) => {
|
|
|
3787
4758
|
* using their output as seed sources.
|
|
3788
4759
|
*/
|
|
3789
4760
|
const processAddressSeeds = async (allMatches, sliceStart, sliceEnd, fullText, existingEntities) => {
|
|
3790
|
-
const clusters = clusterSeeds(collectSeeds(allMatches, sliceStart, sliceEnd, fullText, existingEntities), 150);
|
|
4761
|
+
const clusters = clusterSeeds(collectSeeds(allMatches, sliceStart, sliceEnd, fullText, existingEntities, await getBrCepContextRe()), 150);
|
|
3791
4762
|
const results = [];
|
|
3792
4763
|
for (const cluster of clusters) {
|
|
3793
4764
|
const score = scoreCluster(cluster);
|
|
@@ -3812,6 +4783,8 @@ const processAddressSeeds = async (allMatches, sliceStart, sliceEnd, fullText, e
|
|
|
3812
4783
|
const TRAILING_SEP = /[,\s]+$/;
|
|
3813
4784
|
const WORD_CHAR_RE = /[\p{L}\p{N}]/u;
|
|
3814
4785
|
const ORG_PROPAGATION_SCORE = .9;
|
|
4786
|
+
const ORG_DETERMINER_RE = /(?<![\p{L}\p{N}])(společnost(?:i|í|em|u)?|spolecnost(?:i|em|u)?|the\s+(?:company|corporation|firm)|die\s+(?:gesellschaft|firma)|la\s+(?:société|empresa|sociedad)|el\s+(?:empresa|sociedad))\s+$/iu;
|
|
4787
|
+
const isCallerOwnedEntity$2 = (entity) => entity.sourceDetail === "custom-deny-list" || entity.sourceDetail === "custom-regex";
|
|
3815
4788
|
/**
|
|
3816
4789
|
* After the main detection pass, collect organization
|
|
3817
4790
|
* entities with a legal form suffix, strip the suffix
|
|
@@ -3824,6 +4797,7 @@ const propagateOrgNames = (entities, fullText) => {
|
|
|
3824
4797
|
const seenBases = /* @__PURE__ */ new Set();
|
|
3825
4798
|
for (const e of entities) {
|
|
3826
4799
|
if (e.label !== "organization") continue;
|
|
4800
|
+
if (isCallerOwnedEntity$2(e)) continue;
|
|
3827
4801
|
for (const suffix of LEGAL_SUFFIXES) if (e.text.endsWith(suffix)) {
|
|
3828
4802
|
const base = e.text.slice(0, -suffix.length).replace(TRAILING_SEP, "").trim();
|
|
3829
4803
|
if (base.length >= 3 && !seenBases.has(base)) {
|
|
@@ -3853,16 +4827,25 @@ const propagateOrgNames = (entities, fullText) => {
|
|
|
3853
4827
|
searchFrom = idx + 1;
|
|
3854
4828
|
continue;
|
|
3855
4829
|
}
|
|
3856
|
-
|
|
4830
|
+
let spanStart = idx;
|
|
4831
|
+
const lookbackStart = Math.max(0, idx - 40);
|
|
4832
|
+
const lookback = fullText.slice(lookbackStart, idx);
|
|
4833
|
+
const determinerMatch = ORG_DETERMINER_RE.exec(lookback);
|
|
4834
|
+
if (determinerMatch !== null) {
|
|
4835
|
+
const determiner = determinerMatch[1] ?? "";
|
|
4836
|
+
const offsetInMatch = determinerMatch[0].indexOf(determiner);
|
|
4837
|
+
spanStart = lookbackStart + determinerMatch.index + offsetInMatch;
|
|
4838
|
+
}
|
|
4839
|
+
if (!isOverlapping(spanStart, matchEnd)) {
|
|
3857
4840
|
results.push({
|
|
3858
|
-
start:
|
|
4841
|
+
start: spanStart,
|
|
3859
4842
|
end: matchEnd,
|
|
3860
4843
|
label,
|
|
3861
|
-
text:
|
|
4844
|
+
text: fullText.slice(spanStart, matchEnd),
|
|
3862
4845
|
score: ORG_PROPAGATION_SCORE,
|
|
3863
4846
|
source: DETECTION_SOURCES.COREFERENCE
|
|
3864
4847
|
});
|
|
3865
|
-
covered.push([
|
|
4848
|
+
covered.push([spanStart, matchEnd]);
|
|
3866
4849
|
}
|
|
3867
4850
|
searchFrom = matchEnd;
|
|
3868
4851
|
}
|
|
@@ -3903,6 +4886,7 @@ const NEAR_MISS_BAND = .15;
|
|
|
3903
4886
|
const BOOST_PER_NEIGHBOUR = .05;
|
|
3904
4887
|
const CONTEXT_WINDOW_CHARS = 150;
|
|
3905
4888
|
const HIGH_CONFIDENCE_FLOOR = .9;
|
|
4889
|
+
const isCallerOwnedEntity$1 = (entity) => entity.sourceDetail === "custom-deny-list" || entity.sourceDetail === "custom-regex";
|
|
3906
4890
|
/**
|
|
3907
4891
|
* Boost confidence of near-miss NER entities that appear
|
|
3908
4892
|
* near high-confidence detections (regex, trigger phrase).
|
|
@@ -4011,7 +4995,7 @@ const getStreetAbbrevs = () => _streetAbbrevs ?? /* @__PURE__ */ new Set();
|
|
|
4011
4995
|
*/
|
|
4012
4996
|
const detectStreetPatternsNearAddresses = (fullText, existingEntities) => {
|
|
4013
4997
|
const results = [];
|
|
4014
|
-
const addressEntities = existingEntities.filter((e) => e.label === "address");
|
|
4998
|
+
const addressEntities = existingEntities.filter((entity) => !(entity.label === "address" && isCallerOwnedEntity$1(entity))).filter((e) => e.label === "address");
|
|
4015
4999
|
const headerEnd = Math.floor(fullText.length * HEADER_ZONE_FRACTION);
|
|
4016
5000
|
const houseNumRe = /\b(?:\d{1,4}\/\d+[a-zA-Z]\b|\d{3,4}\/\d+\b|(?:1[3-9]|[2-9]\d)\/\d{3,}\b)/g;
|
|
4017
5001
|
houseNumRe.lastIndex = 0;
|
|
@@ -4107,6 +5091,7 @@ const ORPHAN_STREET_RE = /^\s*(\p{Lu}[\p{Ll}\p{Lu}]+(?:\s+[\p{Lu}\p{Ll}][\p{Ll}]
|
|
|
4107
5091
|
*/
|
|
4108
5092
|
const detectOrphanStreetLines = (fullText, existingEntities) => {
|
|
4109
5093
|
const headerEnd = Math.floor(fullText.length * HEADER_ZONE_FRACTION);
|
|
5094
|
+
const contextEntities = existingEntities.filter((entity) => !(entity.label === "address" && isCallerOwnedEntity$1(entity)));
|
|
4110
5095
|
const results = [];
|
|
4111
5096
|
ORPHAN_STREET_RE.lastIndex = 0;
|
|
4112
5097
|
for (let m = ORPHAN_STREET_RE.exec(fullText); m !== null; m = ORPHAN_STREET_RE.exec(fullText)) {
|
|
@@ -4116,7 +5101,7 @@ const detectOrphanStreetLines = (fullText, existingEntities) => {
|
|
|
4116
5101
|
const end = start + captured.length;
|
|
4117
5102
|
if (start >= headerEnd) continue;
|
|
4118
5103
|
if (existingEntities.some((e) => e.start <= start && e.end >= end)) continue;
|
|
4119
|
-
if (!
|
|
5104
|
+
if (!contextEntities.some((e) => Math.abs(e.start - end) < 200 || Math.abs(e.end - start) < 200)) continue;
|
|
4120
5105
|
results.push({
|
|
4121
5106
|
start,
|
|
4122
5107
|
end,
|
|
@@ -4464,6 +5449,7 @@ const applyHotwordRules = (entities, fullText) => {
|
|
|
4464
5449
|
//#region src/filters/boundary-consistency.ts
|
|
4465
5450
|
/** Max gap (in chars) between entities to merge. */
|
|
4466
5451
|
const MAX_GAP = 3;
|
|
5452
|
+
const hasLockedBoundary$1 = (entity) => entity.sourceDetail === "custom-deny-list" || entity.sourceDetail === "custom-regex";
|
|
4467
5453
|
/**
|
|
4468
5454
|
* Characters allowed in the gap between two adjacent
|
|
4469
5455
|
* same-label entities that should be merged: spaces,
|
|
@@ -4579,6 +5565,11 @@ const mergeAdjacent = (entities, fullText) => {
|
|
|
4579
5565
|
const result = [];
|
|
4580
5566
|
const lastByLabel = /* @__PURE__ */ new Map();
|
|
4581
5567
|
for (const entity of sorted) {
|
|
5568
|
+
if (hasLockedBoundary$1(entity)) {
|
|
5569
|
+
const copy = { ...entity };
|
|
5570
|
+
result.push(copy);
|
|
5571
|
+
continue;
|
|
5572
|
+
}
|
|
4582
5573
|
const prev = lastByLabel.get(entity.label);
|
|
4583
5574
|
if (!prev) {
|
|
4584
5575
|
const copy = { ...entity };
|
|
@@ -4586,7 +5577,7 @@ const mergeAdjacent = (entities, fullText) => {
|
|
|
4586
5577
|
lastByLabel.set(entity.label, copy);
|
|
4587
5578
|
continue;
|
|
4588
5579
|
}
|
|
4589
|
-
if (entity.start < prev.end) {
|
|
5580
|
+
if (!hasLockedBoundary$1(prev) && entity.start < prev.end) {
|
|
4590
5581
|
prev.end = Math.max(prev.end, entity.end);
|
|
4591
5582
|
prev.text = fullText.slice(prev.start, prev.end);
|
|
4592
5583
|
prev.score = Math.max(prev.score, entity.score);
|
|
@@ -4605,7 +5596,7 @@ const mergeAdjacent = (entities, fullText) => {
|
|
|
4605
5596
|
break;
|
|
4606
5597
|
}
|
|
4607
5598
|
}
|
|
4608
|
-
if (!gapOccupied && gap.length <= MAX_GAP && GAP_PATTERN.test(gap)) {
|
|
5599
|
+
if (!hasLockedBoundary$1(prev) && !gapOccupied && gap.length <= MAX_GAP && GAP_PATTERN.test(gap)) {
|
|
4609
5600
|
prev.end = entity.end;
|
|
4610
5601
|
prev.text = fullText.slice(prev.start, prev.end);
|
|
4611
5602
|
prev.score = Math.max(prev.score, entity.score);
|
|
@@ -4637,6 +5628,7 @@ const fixPartialWords = (entities, fullText) => {
|
|
|
4637
5628
|
})).sort((a, b) => a.entity.end - b.entity.end);
|
|
4638
5629
|
const endPositions = byEnd.map((x) => x.entity.end);
|
|
4639
5630
|
return sorted.map((e, eIdx) => {
|
|
5631
|
+
if (hasLockedBoundary$1(e)) return e;
|
|
4640
5632
|
let newStart = wordStartAt(e.start, boundaries, fullText);
|
|
4641
5633
|
let newEnd = wordEndAt(e.end, boundaries, fullText);
|
|
4642
5634
|
let lo = 0;
|
|
@@ -4731,7 +5723,8 @@ const resolveCrossLabelOverlaps = (entities, fullText) => {
|
|
|
4731
5723
|
if (aContainsB || bContainsA) continue;
|
|
4732
5724
|
const aLen = a.end - a.start;
|
|
4733
5725
|
const bLen = b.end - b.start;
|
|
4734
|
-
|
|
5726
|
+
const aLocked = hasLockedBoundary$1(a);
|
|
5727
|
+
if (aLocked !== hasLockedBoundary$1(b) ? aLocked : a.score > b.score || a.score === b.score && aLen >= bLen) {
|
|
4735
5728
|
b.start = a.end;
|
|
4736
5729
|
b.text = fullText.slice(b.start, b.end);
|
|
4737
5730
|
} else {
|
|
@@ -4759,8 +5752,11 @@ const enforceBoundaryConsistency = (entities, fullText) => {
|
|
|
4759
5752
|
};
|
|
4760
5753
|
//#endregion
|
|
4761
5754
|
//#region src/build-unified-search.ts
|
|
5755
|
+
const DEFAULT_CUSTOM_REGEX_SCORE$1 = .9;
|
|
5756
|
+
const ALNUM_RE = /[\p{L}\p{N}]/u;
|
|
4762
5757
|
const buildUnifiedSearch = async (config, gazetteerEntries = [], ctx = defaultContext) => {
|
|
4763
5758
|
const legalFormsEnabled = isLegalFormsEnabled(config);
|
|
5759
|
+
const customRegexes = config.enableRegex ? config.customRegexes ?? [] : [];
|
|
4764
5760
|
const [legalForms, triggers, denyListData, streetTypes, currencyPatterns, datePatterns, signingPatterns] = await Promise.all([
|
|
4765
5761
|
legalFormsEnabled ? buildLegalFormPatterns() : Promise.resolve([]),
|
|
4766
5762
|
config.enableTriggerPhrases ? buildTriggerPatterns() : Promise.resolve({
|
|
@@ -4785,12 +5781,21 @@ const buildUnifiedSearch = async (config, gazetteerEntries = [], ctx = defaultCo
|
|
|
4785
5781
|
...datePatterns.map(() => DATE_PATTERN_META),
|
|
4786
5782
|
...signingPatterns.map(() => SIGNING_CLAUSE_META)
|
|
4787
5783
|
];
|
|
5784
|
+
const customRegexMeta = customRegexes.map((entry) => ({
|
|
5785
|
+
label: entry.label,
|
|
5786
|
+
score: entry.score ?? DEFAULT_CUSTOM_REGEX_SCORE$1,
|
|
5787
|
+
sourceDetail: "custom-regex"
|
|
5788
|
+
}));
|
|
4788
5789
|
let offset = 0;
|
|
4789
5790
|
const regexSlice = {
|
|
4790
5791
|
start: offset,
|
|
4791
5792
|
end: offset + allRegex.length
|
|
4792
5793
|
};
|
|
4793
5794
|
offset = regexSlice.end;
|
|
5795
|
+
const customRegexSlice = {
|
|
5796
|
+
start: 0,
|
|
5797
|
+
end: customRegexes.length
|
|
5798
|
+
};
|
|
4794
5799
|
const legalFormsSlice = {
|
|
4795
5800
|
start: offset,
|
|
4796
5801
|
end: offset + legalForms.length
|
|
@@ -4811,6 +5816,7 @@ const buildUnifiedSearch = async (config, gazetteerEntries = [], ctx = defaultCo
|
|
|
4811
5816
|
...triggerEntries
|
|
4812
5817
|
];
|
|
4813
5818
|
const tsRegex = new (getTextSearch())(regexAllPatterns);
|
|
5819
|
+
const tsCustomRegex = new (getTextSearch())(customRegexes.map((entry) => entry.pattern), { overlapStrategy: "all" });
|
|
4814
5820
|
offset = 0;
|
|
4815
5821
|
const denyListOriginals = denyListData?.originals ?? [];
|
|
4816
5822
|
const denyListSlice = {
|
|
@@ -4829,24 +5835,31 @@ const buildUnifiedSearch = async (config, gazetteerEntries = [], ctx = defaultCo
|
|
|
4829
5835
|
end: offset + (gazResult?.patterns.length ?? 0)
|
|
4830
5836
|
};
|
|
4831
5837
|
offset = gazetteerSlice.end;
|
|
4832
|
-
const wrapWholeWord = (s) => ({
|
|
5838
|
+
const wrapWholeWord = (s, wholeWords) => ({
|
|
4833
5839
|
pattern: s,
|
|
4834
5840
|
literal: true,
|
|
4835
|
-
wholeWords
|
|
5841
|
+
wholeWords
|
|
4836
5842
|
});
|
|
5843
|
+
const customDenyListNeedsWholeWords = (pattern) => {
|
|
5844
|
+
const first = pattern.at(0) ?? "";
|
|
5845
|
+
const last = pattern.at(-1) ?? "";
|
|
5846
|
+
return ALNUM_RE.test(first) && ALNUM_RE.test(last);
|
|
5847
|
+
};
|
|
4837
5848
|
const literalAllPatterns = [
|
|
4838
|
-
...denyListOriginals.map(wrapWholeWord),
|
|
4839
|
-
...streetTypes.map(wrapWholeWord),
|
|
5849
|
+
...denyListOriginals.map((pattern, index) => wrapWholeWord(pattern, (denyListData?.sources[index] ?? []).includes("custom-deny-list") ? customDenyListNeedsWholeWords(pattern) : true)),
|
|
5850
|
+
...streetTypes.map((pattern) => wrapWholeWord(pattern, true)),
|
|
4840
5851
|
...gazResult?.patterns ?? []
|
|
4841
5852
|
];
|
|
4842
5853
|
return {
|
|
4843
5854
|
tsRegex,
|
|
5855
|
+
tsCustomRegex,
|
|
4844
5856
|
tsLiterals: literalAllPatterns.length > 0 ? new (getTextSearch())(literalAllPatterns, {
|
|
4845
5857
|
caseInsensitive: true,
|
|
4846
5858
|
overlapStrategy: "all"
|
|
4847
5859
|
}) : new (getTextSearch())([]),
|
|
4848
5860
|
slices: {
|
|
4849
5861
|
regex: regexSlice,
|
|
5862
|
+
customRegex: customRegexSlice,
|
|
4850
5863
|
legalForms: legalFormsSlice,
|
|
4851
5864
|
triggers: triggersSlice,
|
|
4852
5865
|
denyList: denyListSlice,
|
|
@@ -4854,6 +5867,7 @@ const buildUnifiedSearch = async (config, gazetteerEntries = [], ctx = defaultCo
|
|
|
4854
5867
|
gazetteer: gazetteerSlice
|
|
4855
5868
|
},
|
|
4856
5869
|
regexMeta,
|
|
5870
|
+
customRegexMeta,
|
|
4857
5871
|
triggerRules: triggers.rules,
|
|
4858
5872
|
denyListData,
|
|
4859
5873
|
gazetteerData: gazResult?.data ?? null
|
|
@@ -4863,9 +5877,11 @@ const buildUnifiedSearch = async (config, gazetteerEntries = [], ctx = defaultCo
|
|
|
4863
5877
|
//#region src/unified-search.ts
|
|
4864
5878
|
const runUnifiedSearch = (instance, fullText) => {
|
|
4865
5879
|
const regexMatches = instance.tsRegex.findIter(fullText);
|
|
5880
|
+
const customRegexMatches = instance.tsCustomRegex.findIter(fullText);
|
|
4866
5881
|
const normalized = normalizeForSearch(fullText);
|
|
4867
5882
|
return {
|
|
4868
5883
|
regexMatches,
|
|
5884
|
+
customRegexMatches,
|
|
4869
5885
|
literalMatches: instance.tsLiterals.findIter(normalized)
|
|
4870
5886
|
};
|
|
4871
5887
|
};
|
|
@@ -4978,9 +5994,13 @@ const unmaskNerEntities = (nerEntities, maskResult, fullText) => {
|
|
|
4978
5994
|
* so the containment rule trusts their length.
|
|
4979
5995
|
*/
|
|
4980
5996
|
const LITERAL_SOURCES = new Set(["deny-list", "gazetteer"]);
|
|
5997
|
+
const isCallerOwnedEntity = (entity) => entity.sourceDetail === "custom-deny-list" || entity.sourceDetail === "custom-regex";
|
|
5998
|
+
const hasLockedBoundary = (entity) => isCallerOwnedEntity(entity);
|
|
4981
5999
|
const shouldReplace = (a, b) => {
|
|
4982
6000
|
const aLen = a.end - a.start;
|
|
4983
6001
|
const bLen = b.end - b.start;
|
|
6002
|
+
const aCallerOwned = isCallerOwnedEntity(a);
|
|
6003
|
+
if (aCallerOwned !== isCallerOwnedEntity(b)) return aCallerOwned;
|
|
4984
6004
|
if (a.label === b.label && LITERAL_SOURCES.has(a.source) && a.start <= b.start && a.end >= b.end && aLen > bLen) return true;
|
|
4985
6005
|
if (a.label === b.label && LITERAL_SOURCES.has(b.source) && b.start <= a.start && b.end >= a.end && bLen > aLen) return false;
|
|
4986
6006
|
const aPri = DETECTOR_PRIORITY[a.source] ?? 0;
|
|
@@ -4990,12 +6010,89 @@ const shouldReplace = (a, b) => {
|
|
|
4990
6010
|
};
|
|
4991
6011
|
/** Labels where colons are structurally significant. */
|
|
4992
6012
|
const COLON_LABELS = new Set(["ip address", "mac address"]);
|
|
6013
|
+
/**
|
|
6014
|
+
* Labels whose detectors emit precise, evidence-backed spans. When
|
|
6015
|
+
* one of these fires at the exact same offsets as a fuzzier
|
|
6016
|
+
* `address` hit (city dictionary lookup, address-seed cluster), the
|
|
6017
|
+
* `address` label is almost always a dictionary collision — "March
|
|
6018
|
+
* 15" the date getting tagged as an address because "March" appears
|
|
6019
|
+
* in a city corpus, or "Brent Phillips" emitted as both `person`
|
|
6020
|
+
* and `address` because "Brent" is a UK city. The precise detector
|
|
6021
|
+
* wins.
|
|
6022
|
+
*/
|
|
6023
|
+
const PRECISE_OVER_ADDRESS = new Set([
|
|
6024
|
+
"person",
|
|
6025
|
+
"date",
|
|
6026
|
+
"date of birth",
|
|
6027
|
+
"phone number",
|
|
6028
|
+
"email address",
|
|
6029
|
+
"monetary amount",
|
|
6030
|
+
"iban",
|
|
6031
|
+
"bank account number",
|
|
6032
|
+
"tax identification number",
|
|
6033
|
+
"registration number",
|
|
6034
|
+
"identity card number",
|
|
6035
|
+
"national identification number",
|
|
6036
|
+
"passport number",
|
|
6037
|
+
"credit card number"
|
|
6038
|
+
]);
|
|
6039
|
+
/**
|
|
6040
|
+
* Labels the `person` chain wins against at identical offsets. The
|
|
6041
|
+
* chain carries adjacent-name evidence (deny-list surname plus a
|
|
6042
|
+
* capitalised follow-up) a single-token dictionary collision does
|
|
6043
|
+
* not. Kept narrow: organizations are NOT here — "Morgan Stanley"
|
|
6044
|
+
* legitimately appears in both the org and name dictionaries, and
|
|
6045
|
+
* the existing detector priority is the right tie-breaker there.
|
|
6046
|
+
*/
|
|
6047
|
+
const PERSON_PREFERRED_OVER = new Set(["address", "land parcel"]);
|
|
6048
|
+
const resolveSameSpanLabelConflicts = (entities) => {
|
|
6049
|
+
if (entities.length < 2) return entities;
|
|
6050
|
+
const byOffsets = /* @__PURE__ */ new Map();
|
|
6051
|
+
for (const entity of entities) {
|
|
6052
|
+
const key = `${entity.start}:${entity.end}`;
|
|
6053
|
+
const list = byOffsets.get(key);
|
|
6054
|
+
if (list) list.push(entity);
|
|
6055
|
+
else byOffsets.set(key, [entity]);
|
|
6056
|
+
}
|
|
6057
|
+
const dropped = /* @__PURE__ */ new Set();
|
|
6058
|
+
for (const [, group] of byOffsets) {
|
|
6059
|
+
if (group.length < 2) continue;
|
|
6060
|
+
const labels = new Set(group.map((e) => e.label));
|
|
6061
|
+
if (labels.size < 2) continue;
|
|
6062
|
+
const hasPerson = labels.has("person");
|
|
6063
|
+
const hasPreciseNonAddress = [...labels].some((l) => l !== "address" && PRECISE_OVER_ADDRESS.has(l));
|
|
6064
|
+
let maxPriority = -1;
|
|
6065
|
+
for (const e of group) {
|
|
6066
|
+
if (hasLockedBoundary(e)) continue;
|
|
6067
|
+
const pri = DETECTOR_PRIORITY[e.source] ?? 0;
|
|
6068
|
+
if (pri > maxPriority) maxPriority = pri;
|
|
6069
|
+
}
|
|
6070
|
+
for (const e of group) {
|
|
6071
|
+
if (hasLockedBoundary(e)) continue;
|
|
6072
|
+
if ((DETECTOR_PRIORITY[e.source] ?? 0) < maxPriority) {
|
|
6073
|
+
dropped.add(e);
|
|
6074
|
+
continue;
|
|
6075
|
+
}
|
|
6076
|
+
if (hasPerson && PERSON_PREFERRED_OVER.has(e.label)) {
|
|
6077
|
+
dropped.add(e);
|
|
6078
|
+
continue;
|
|
6079
|
+
}
|
|
6080
|
+
if (hasPreciseNonAddress && e.label === "address") dropped.add(e);
|
|
6081
|
+
}
|
|
6082
|
+
}
|
|
6083
|
+
if (dropped.size === 0) return entities;
|
|
6084
|
+
return entities.filter((e) => !dropped.has(e));
|
|
6085
|
+
};
|
|
4993
6086
|
/** Strip leading/trailing whitespace and punctuation. */
|
|
4994
6087
|
const sanitizeEntities = (entities) => entities.flatMap((e) => {
|
|
6088
|
+
if (hasLockedBoundary(e)) return [e];
|
|
4995
6089
|
const strip = COLON_LABELS.has(e.label) ? /[\s,;]+/ : /[\s:,;]+/;
|
|
4996
6090
|
const leadTrimmed = e.text.replace(/^(?:\.\s)+/, "").replace(new RegExp(`^${strip.source}`, strip.flags), "");
|
|
4997
6091
|
const lead = e.text.length - leadTrimmed.length;
|
|
4998
|
-
|
|
6092
|
+
let cleaned = leadTrimmed.replace(new RegExp(`${strip.source}$`, strip.flags), "");
|
|
6093
|
+
if (e.label === "organization" && cleaned.endsWith(".") && !LITERAL_SOURCES.has(e.source)) {
|
|
6094
|
+
if (!getKnownLegalSuffixes().some((suffix) => cleaned.endsWith(suffix))) cleaned = cleaned.slice(0, -1).trimEnd();
|
|
6095
|
+
}
|
|
4999
6096
|
if (cleaned.length === 0) return [];
|
|
5000
6097
|
if (!/[\p{L}\p{N}]/u.test(cleaned)) return [];
|
|
5001
6098
|
const collapsed = cleaned.replace(/\s*\n\s*/g, " ").replace(/\s{2,}/g, " ");
|
|
@@ -5019,10 +6116,32 @@ const mergeAndDedup = (...layers) => {
|
|
|
5019
6116
|
const entity = sorted[i];
|
|
5020
6117
|
const last = merged[merged.length - 1];
|
|
5021
6118
|
if (!entity || !last) continue;
|
|
5022
|
-
if (last.end <= entity.start)
|
|
5023
|
-
|
|
6119
|
+
if (last.end <= entity.start) {
|
|
6120
|
+
merged.push({ ...entity });
|
|
6121
|
+
continue;
|
|
6122
|
+
}
|
|
6123
|
+
const overlapStart = (() => {
|
|
6124
|
+
for (let j = merged.length - 1; j >= 0; j--) {
|
|
6125
|
+
const existing = merged[j];
|
|
6126
|
+
if (!existing || existing.end <= entity.start) return j + 1;
|
|
6127
|
+
}
|
|
6128
|
+
return 0;
|
|
6129
|
+
})();
|
|
6130
|
+
const overlaps = merged.slice(overlapStart);
|
|
6131
|
+
if (!overlaps.some((existing) => existing.start !== entity.start || existing.end !== entity.end)) {
|
|
6132
|
+
const sameLabelIndex = overlaps.findIndex((existing) => existing.label === entity.label);
|
|
6133
|
+
if (sameLabelIndex === -1) {
|
|
6134
|
+
merged.push({ ...entity });
|
|
6135
|
+
continue;
|
|
6136
|
+
}
|
|
6137
|
+
const actualIndex = overlapStart + sameLabelIndex;
|
|
6138
|
+
const sameLabel = merged[actualIndex];
|
|
6139
|
+
if (sameLabel && shouldReplace(entity, sameLabel)) merged[actualIndex] = { ...entity };
|
|
6140
|
+
continue;
|
|
6141
|
+
}
|
|
6142
|
+
if (overlaps.every((existing) => shouldReplace(entity, existing))) merged.splice(overlapStart, overlaps.length, { ...entity });
|
|
5024
6143
|
}
|
|
5025
|
-
return sanitizeEntities(merged);
|
|
6144
|
+
return resolveSameSpanLabelConflicts(sanitizeEntities(merged));
|
|
5026
6145
|
};
|
|
5027
6146
|
const escapeRegex = (s) => s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
5028
6147
|
let amountWordsRe = null;
|
|
@@ -5039,7 +6158,7 @@ const getAmountWordsRe = async () => {
|
|
|
5039
6158
|
return amountWordsRe;
|
|
5040
6159
|
};
|
|
5041
6160
|
const extendMonetaryAmountWords = (entities, fullText, re) => entities.map((e) => {
|
|
5042
|
-
if (e.label !== "monetary amount") return e;
|
|
6161
|
+
if (e.label !== "monetary amount" || isCallerOwnedEntity(e)) return e;
|
|
5043
6162
|
const after = fullText.slice(e.end);
|
|
5044
6163
|
const m = re.exec(after);
|
|
5045
6164
|
if (!m) return e;
|
|
@@ -5052,22 +6171,34 @@ const extendMonetaryAmountWords = (entities, fullText, re) => entities.map((e) =
|
|
|
5052
6171
|
});
|
|
5053
6172
|
const createAllowedLabelSetFromLabels = (labels) => labels.length > 0 ? new Set(labels) : null;
|
|
5054
6173
|
const createAllowedLabelSet = (config) => createAllowedLabelSetFromLabels(config.labels);
|
|
6174
|
+
const DEFAULT_CUSTOM_REGEX_SCORE = .9;
|
|
5055
6175
|
const filterAllowedLabels = (entities, allowedLabels) => {
|
|
5056
6176
|
if (!allowedLabels) return entities;
|
|
5057
6177
|
return entities.filter((e) => allowedLabels.has(e.label));
|
|
5058
6178
|
};
|
|
5059
6179
|
const labelIsAllowed = (label, allowedLabels) => !allowedLabels || allowedLabels.has(label);
|
|
6180
|
+
const NON_NER_LABELS = new Set(["misc"]);
|
|
5060
6181
|
const getRequestedNerLabels = (config, expandForHotwords = false) => {
|
|
5061
6182
|
const labels = config.labels.length > 0 ? config.labels : DEFAULT_ENTITY_LABELS;
|
|
5062
|
-
return expandForHotwords ? expandLabelsForHotwordRules(labels) : labels;
|
|
6183
|
+
return (expandForHotwords ? expandLabelsForHotwordRules(labels) : labels).filter((label) => !NON_NER_LABELS.has(label));
|
|
5063
6184
|
};
|
|
5064
6185
|
const checkAbort = (signal) => {
|
|
5065
6186
|
if (signal?.aborted) throw new DOMException("Pipeline aborted", "AbortError");
|
|
5066
6187
|
};
|
|
5067
6188
|
const configKey = (config, gazetteerEntries) => {
|
|
5068
6189
|
const legalFormsEnabled = isLegalFormsEnabled(config);
|
|
6190
|
+
const customDenyFingerprint = config.enableDenyList && config.customDenyList ? config.customDenyList.map((entry) => JSON.stringify({
|
|
6191
|
+
label: entry.label,
|
|
6192
|
+
value: entry.value,
|
|
6193
|
+
variants: [...entry.variants ?? []].sort()
|
|
6194
|
+
})).sort().join("\n") : "";
|
|
6195
|
+
const customRegexFingerprint = config.enableRegex && config.customRegexes ? config.customRegexes.map((entry) => JSON.stringify({
|
|
6196
|
+
label: entry.label,
|
|
6197
|
+
pattern: entry.pattern,
|
|
6198
|
+
score: entry.score ?? DEFAULT_CUSTOM_REGEX_SCORE
|
|
6199
|
+
})).sort().join("\n") : "";
|
|
5069
6200
|
const gazFingerprint = config.enableGazetteer && gazetteerEntries.length > 0 ? gazetteerEntries.map((e) => `${e.id}:${e.canonical}:${e.label}:${[...e.variants].sort().join(",")}`).toSorted().join(";") : "";
|
|
5070
|
-
return `${config.enableDenyList}:${config.enableTriggerPhrases}:${legalFormsEnabled}:${config.enableNameCorpus}:${config.denyListCountries?.toSorted().join(",") ?? ""}:${config.denyListRegions?.toSorted().join(",") ?? ""}:${config.denyListExcludeCategories?.toSorted().join(",") ?? ""}:${config.enableGazetteer}:${gazFingerprint}`;
|
|
6201
|
+
return `${config.enableDenyList}:${config.enableTriggerPhrases}:${legalFormsEnabled}:${config.enableNameCorpus}:${config.enableRegex}:${config.denyListCountries?.toSorted().join(",") ?? ""}:${config.denyListRegions?.toSorted().join(",") ?? ""}:${config.denyListExcludeCategories?.toSorted().join(",") ?? ""}:${customDenyFingerprint}:${customRegexFingerprint}:${config.enableGazetteer}:${gazFingerprint}`;
|
|
5071
6202
|
};
|
|
5072
6203
|
/**
|
|
5073
6204
|
* Get or build a cached search instance. Cache state
|
|
@@ -5130,6 +6261,8 @@ const runPipeline = async (options) => {
|
|
|
5130
6261
|
loadGenericRoles(ctx),
|
|
5131
6262
|
initPrepositions(),
|
|
5132
6263
|
initStreetAbbrevs(),
|
|
6264
|
+
initAddressComponents(),
|
|
6265
|
+
warmAddressStopKeywords(),
|
|
5133
6266
|
zoneInit,
|
|
5134
6267
|
hotwordInit
|
|
5135
6268
|
]);
|
|
@@ -5137,6 +6270,8 @@ const runPipeline = async (options) => {
|
|
|
5137
6270
|
loadGenericRoles(ctx),
|
|
5138
6271
|
initPrepositions(),
|
|
5139
6272
|
initStreetAbbrevs(),
|
|
6273
|
+
initAddressComponents(),
|
|
6274
|
+
warmAddressStopKeywords(),
|
|
5140
6275
|
hotwordInit
|
|
5141
6276
|
]);
|
|
5142
6277
|
if (cachedSearch && config.enableDenyList) await ensureDenyListData(ctx, config.dictionaries);
|
|
@@ -5150,14 +6285,17 @@ const runPipeline = async (options) => {
|
|
|
5150
6285
|
const preHotwordAllowedLabels = hotwordsActive ? createAllowedLabelSetFromLabels(expandLabelsForHotwordRules(config.labels)) : allowedLabels;
|
|
5151
6286
|
const search = cachedSearch ?? await getCachedSearch(config, gazetteerEntries, ctx);
|
|
5152
6287
|
checkAbort(signal);
|
|
5153
|
-
const { regexMatches, literalMatches } = runUnifiedSearch(search, fullText);
|
|
6288
|
+
const { regexMatches, customRegexMatches, literalMatches } = runUnifiedSearch(search, fullText);
|
|
5154
6289
|
const { slices } = search;
|
|
5155
6290
|
const rawRegexEntities = config.enableRegex ? processRegexMatches(regexMatches, slices.regex.start, slices.regex.end, search.regexMeta) : [];
|
|
5156
|
-
const
|
|
6291
|
+
const customRegexEntities = filterAllowedLabels(config.enableRegex ? processRegexMatches(customRegexMatches, slices.customRegex.start, slices.customRegex.end, search.customRegexMeta) : [], preHotwordAllowedLabels);
|
|
6292
|
+
const regexEntities = filterAllowedLabels([...rawRegexEntities, ...customRegexEntities], preHotwordAllowedLabels);
|
|
5157
6293
|
if (regexEntities.length > 0) log("regex", `${regexEntities.length} matches`);
|
|
6294
|
+
if (legalFormsEnabled || config.enableTriggerPhrases) await warmLegalRoleHeads();
|
|
5158
6295
|
const rawLegalFormEntities = legalFormsEnabled ? processLegalFormMatches(regexMatches, slices.legalForms.start, slices.legalForms.end, fullText) : [];
|
|
5159
6296
|
const legalFormEntities = filterAllowedLabels(rawLegalFormEntities, preHotwordAllowedLabels);
|
|
5160
6297
|
if (legalFormEntities.length > 0) log("legal-forms", `${legalFormEntities.length} matches`);
|
|
6298
|
+
if (config.enableTriggerPhrases) await warmAddressStopKeywords();
|
|
5161
6299
|
const rawTriggerEntities = config.enableTriggerPhrases ? processTriggerMatches(regexMatches, slices.triggers.start, slices.triggers.end, fullText, search.triggerRules) : [];
|
|
5162
6300
|
const triggerEntities = filterAllowedLabels(rawTriggerEntities, preHotwordAllowedLabels);
|
|
5163
6301
|
if (triggerEntities.length > 0) log("trigger-phrases", `${triggerEntities.length} matches`);
|
|
@@ -5178,20 +6316,36 @@ const runPipeline = async (options) => {
|
|
|
5178
6316
|
const gazetteerEntities = filterAllowedLabels(rawGazetteerEntities, preHotwordAllowedLabels);
|
|
5179
6317
|
if (gazetteerEntities.length > 0) log("gazetteer", `${gazetteerEntities.length} matches`);
|
|
5180
6318
|
checkAbort(signal);
|
|
6319
|
+
const rawCustomDenyListEntities = rawDenyListEntities.filter((entity) => isCallerOwnedEntity(entity));
|
|
6320
|
+
const rawCuratedDenyListEntities = rawDenyListEntities.filter((entity) => !isCallerOwnedEntity(entity));
|
|
6321
|
+
const customDenyListEntities = filterAllowedLabels(rawCustomDenyListEntities, preHotwordAllowedLabels);
|
|
5181
6322
|
const ruleContextEntities = [
|
|
5182
6323
|
...rawTriggerEntities,
|
|
5183
6324
|
...rawRegexEntities,
|
|
6325
|
+
...customRegexEntities,
|
|
6326
|
+
...rawLegalFormEntities,
|
|
6327
|
+
...rawNameCorpusEntities,
|
|
6328
|
+
...rawCuratedDenyListEntities,
|
|
6329
|
+
...customDenyListEntities,
|
|
6330
|
+
...rawGazetteerEntities
|
|
6331
|
+
];
|
|
6332
|
+
const nerMaskEntities = [
|
|
6333
|
+
...rawTriggerEntities,
|
|
6334
|
+
...rawRegexEntities,
|
|
6335
|
+
...customRegexEntities,
|
|
5184
6336
|
...rawLegalFormEntities,
|
|
5185
6337
|
...rawNameCorpusEntities,
|
|
5186
|
-
...
|
|
6338
|
+
...rawCuratedDenyListEntities,
|
|
6339
|
+
...customDenyListEntities,
|
|
5187
6340
|
...rawGazetteerEntities
|
|
5188
6341
|
];
|
|
5189
6342
|
let rawNerEntities = [];
|
|
5190
6343
|
let nerEntities = [];
|
|
5191
|
-
|
|
5192
|
-
|
|
6344
|
+
const requestedNerLabels = getRequestedNerLabels(config, hotwordsActive);
|
|
6345
|
+
if (config.enableNer && nerInference && requestedNerLabels.length > 0) {
|
|
6346
|
+
const maskResult = maskDetectedSpans(fullText, nerMaskEntities);
|
|
5193
6347
|
log("ner", "running inference...");
|
|
5194
|
-
const rawNer = await nerInference(maskResult.maskedText, [...
|
|
6348
|
+
const rawNer = await nerInference(maskResult.maskedText, [...requestedNerLabels], config.threshold, signal);
|
|
5195
6349
|
rawNerEntities = unmaskNerEntities(rawNer, maskResult, fullText);
|
|
5196
6350
|
nerEntities = filterAllowedLabels(rawNerEntities, preHotwordAllowedLabels);
|
|
5197
6351
|
const masked = rawNer.length - rawNerEntities.length;
|
|
@@ -5212,7 +6366,12 @@ const runPipeline = async (options) => {
|
|
|
5212
6366
|
if (addressSeedEntities.length > 0) log("address-seeds", `${addressSeedEntities.length} expanded`);
|
|
5213
6367
|
checkAbort(signal);
|
|
5214
6368
|
const zoneAdjusted = applyZoneAdjustments([...preAddressEntities, ...addressSeedEntities], zones);
|
|
5215
|
-
const preBoostEntities =
|
|
6369
|
+
const preBoostEntities = (() => {
|
|
6370
|
+
if (!hotwordsActive) return zoneAdjusted;
|
|
6371
|
+
const hotwordCandidates = zoneAdjusted.filter((entity) => !isCallerOwnedEntity(entity));
|
|
6372
|
+
const callerOwnedEntities = zoneAdjusted.filter(isCallerOwnedEntity);
|
|
6373
|
+
return [...filterAllowedLabels(applyHotwordRules(hotwordCandidates, fullText), allowedLabels), ...callerOwnedEntities];
|
|
6374
|
+
})();
|
|
5216
6375
|
let allEntities;
|
|
5217
6376
|
if (config.enableConfidenceBoost) {
|
|
5218
6377
|
allEntities = boostNearMissEntities(preBoostEntities, config.threshold);
|
|
@@ -5247,7 +6406,8 @@ const runPipeline = async (options) => {
|
|
|
5247
6406
|
checkAbort(signal);
|
|
5248
6407
|
ctx.corefSourceMap.clear();
|
|
5249
6408
|
if (config.enableCoreference) {
|
|
5250
|
-
|
|
6409
|
+
if (!legalFormsEnabled) await warmLegalRoleHeads();
|
|
6410
|
+
const terms = await extractDefinedTerms(fullText, merged.filter((entity) => !isCallerOwnedEntity(entity)), ctx);
|
|
5251
6411
|
if (terms.length > 0) {
|
|
5252
6412
|
log("coreference", `${terms.length} defined terms`);
|
|
5253
6413
|
const corefSpans = findCoreferenceSpans(fullText, terms, ctx);
|
|
@@ -5287,7 +6447,7 @@ const resolveOperator = (config, label) => config.operators[label] ?? "replace";
|
|
|
5287
6447
|
//#region src/redact.ts
|
|
5288
6448
|
const WHITESPACE_RE = /\s+/g;
|
|
5289
6449
|
const PHONE_NOISE_RE = /[()\s-]/g;
|
|
5290
|
-
const
|
|
6450
|
+
const ID_SEPARATOR_RE = /[\s\-/.]/g;
|
|
5291
6451
|
/**
|
|
5292
6452
|
* Normalize entity text so that surface-form variations
|
|
5293
6453
|
* of the same real-world value map to a single canonical
|
|
@@ -5297,8 +6457,8 @@ const normalizeEntityText = (label, text) => {
|
|
|
5297
6457
|
const upper = label.toUpperCase().replace(WHITESPACE_RE, "_");
|
|
5298
6458
|
if (upper === "EMAIL_ADDRESS" || upper === "EMAIL") return text.toLowerCase().trim();
|
|
5299
6459
|
if (upper === "PHONE_NUMBER" || upper === "PHONE") return text.replace(PHONE_NOISE_RE, "");
|
|
5300
|
-
if (upper === "IBAN" || upper === "BANK_ACCOUNT_NUMBER" || upper === "TAX_IDENTIFICATION_NUMBER" || upper === "REGISTRATION_NUMBER") return text.replace(
|
|
5301
|
-
if (upper === "PERSON" || upper === "ORGANIZATION" || upper === "ADDRESS" || upper === "LAND_PARCEL") return text.replace(WHITESPACE_RE, " ").toLowerCase().trim();
|
|
6460
|
+
if (upper === "IBAN" || upper === "BANK_ACCOUNT_NUMBER" || upper === "TAX_IDENTIFICATION_NUMBER" || upper === "REGISTRATION_NUMBER" || upper === "NATIONAL_IDENTIFICATION_NUMBER" || upper === "SOCIAL_SECURITY_NUMBER" || upper === "BIRTH_NUMBER" || upper === "IDENTITY_CARD_NUMBER" || upper === "PASSPORT_NUMBER" || upper === "CREDIT_CARD_NUMBER") return text.replace(ID_SEPARATOR_RE, "").toUpperCase();
|
|
6461
|
+
if (upper === "PERSON" || upper === "ORGANIZATION" || upper === "ADDRESS" || upper === "LAND_PARCEL" || upper === "MISC") return text.replace(WHITESPACE_RE, " ").toLowerCase().trim();
|
|
5302
6462
|
return text.trim();
|
|
5303
6463
|
};
|
|
5304
6464
|
/**
|
|
@@ -5842,6 +7002,6 @@ const levenshtein = (rawA, rawB) => {
|
|
|
5842
7002
|
//#region src/wasm.ts
|
|
5843
7003
|
initTextSearch(TextSearch);
|
|
5844
7004
|
//#endregion
|
|
5845
|
-
export { CURRENCY_PATTERN_META, DATE_PATTERN_META, DEFAULT_ENTITY_LABELS, DEFAULT_OPERATOR_CONFIG, DETECTION_SOURCES, DETECTOR_PRIORITY, OPERATOR_REGISTRY, OPERATOR_TYPES, REGEX_META, REGEX_PATTERNS, REGIONS, ZONE_SCORE_ADJUSTMENTS, applyHotwordRules, applyZoneAdjustments, boostNearMissEntities, buildDenyList, buildGazetteerPatterns, buildLegalFormPatterns, buildPlaceholderMap, buildStreetTypePatterns, buildTriggerPatterns, buildUnifiedSearch, chunkText, classifyZones, computeChunkOffsets, corefKey, createPipelineContext, deanonymise, decodeSpans, decodeTokenSpans, detectNameCorpus, ensureDenyListData, exportRedactionKey, extractDefinedTerms, filterFalsePositives, findCoreferenceSpans, getCurrencyPatterns, getDatePatterns, initHotwordRules, initNameCorpus, initZoneClassifier, levenshtein, mergeAndDedup, mergeChunkEntities, normalizeForSearch, prepareBatch, processAddressSeeds, processDenyListMatches, processGazetteerMatches, processLegalFormMatches, processRegexMatches, processTriggerMatches, propagateOrgNames, redactText, resolveCountries, resolveOperator, runPipeline, runUnifiedSearch, sanitizeEntities, tokenizeText };
|
|
7005
|
+
export { CURRENCY_PATTERN_META, DATE_PATTERN_META, DEFAULT_ENTITY_LABELS, DEFAULT_OPERATOR_CONFIG, DETECTION_SOURCES, DETECTOR_PRIORITY, OPERATOR_REGISTRY, OPERATOR_TYPES, REGEX_META, REGEX_PATTERNS, REGIONS, ZONE_SCORE_ADJUSTMENTS, applyHotwordRules, applyZoneAdjustments, boostNearMissEntities, buildDenyList, buildGazetteerPatterns, buildLegalFormPatterns, buildPlaceholderMap, buildStreetTypePatterns, buildTriggerPatterns, buildUnifiedSearch, chunkText, classifyZones, computeChunkOffsets, corefKey, createPipelineContext, deanonymise, decodeSpans, decodeTokenSpans, detectNameCorpus, ensureDenyListData, exportRedactionKey, extractDefinedTerms, filterFalsePositives, findCoreferenceSpans, getCurrencyPatterns, getDatePatterns, initAddressComponents, initHotwordRules, initNameCorpus, initZoneClassifier, levenshtein, mergeAndDedup, mergeChunkEntities, normalizeForSearch, prepareBatch, processAddressSeeds, processDenyListMatches, processGazetteerMatches, processLegalFormMatches, processRegexMatches, processTriggerMatches, propagateOrgNames, redactText, resolveCountries, resolveOperator, runPipeline, runUnifiedSearch, sanitizeEntities, tokenizeText, warmLegalRoleHeads };
|
|
5846
7006
|
|
|
5847
7007
|
//# sourceMappingURL=wasm.mjs.map
|