@stll/anonymize-wasm 1.2.0 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/address-boundaries.mjs +113 -5
- package/dist/address-stop-keywords.mjs +137 -0
- package/dist/address-stop-keywords.mjs.map +1 -0
- package/dist/address-stopwords.mjs +81 -0
- package/dist/address-stopwords.mjs.map +1 -0
- package/dist/address-street-types.mjs +67 -4
- package/dist/clause-noun-heads.mjs +75 -0
- package/dist/clause-noun-heads.mjs.map +1 -0
- package/dist/coreference.es.mjs +22 -0
- package/dist/coreference.es.mjs.map +1 -0
- package/dist/coreference.fr.mjs +32 -0
- package/dist/coreference.fr.mjs.map +1 -0
- package/dist/coreference.it.mjs +27 -0
- package/dist/coreference.it.mjs.map +1 -0
- package/dist/coreference.pl.mjs +27 -0
- package/dist/coreference.pl.mjs.map +1 -0
- package/dist/coreference.pt-br.mjs +14 -0
- package/dist/coreference.pt-br.mjs.map +1 -0
- package/dist/coreference.sk.mjs +22 -5
- package/dist/generic-roles.mjs +179 -1
- package/dist/hotword-rules.mjs +7 -3
- package/dist/legal-forms.mjs +3 -0
- package/dist/legal-role-heads.cs.mjs +42 -0
- package/dist/legal-role-heads.cs.mjs.map +1 -0
- package/dist/legal-role-heads.de.mjs +33 -0
- package/dist/legal-role-heads.de.mjs.map +1 -0
- package/dist/legal-role-heads.en.mjs +37 -0
- package/dist/legal-role-heads.en.mjs.map +1 -0
- package/dist/legal-role-heads.es.mjs +54 -0
- package/dist/legal-role-heads.es.mjs.map +1 -0
- package/dist/legal-role-heads.fr.mjs +72 -0
- package/dist/legal-role-heads.fr.mjs.map +1 -0
- package/dist/legal-role-heads.it.mjs +68 -0
- package/dist/legal-role-heads.it.mjs.map +1 -0
- package/dist/legal-role-heads.pl.mjs +84 -0
- package/dist/legal-role-heads.pl.mjs.map +1 -0
- package/dist/legal-role-heads.pt-br.mjs +63 -0
- package/dist/legal-role-heads.pt-br.mjs.map +1 -0
- package/dist/legal-role-heads.sk.mjs +80 -0
- package/dist/legal-role-heads.sk.mjs.map +1 -0
- package/dist/manifest.mjs +21 -8
- package/dist/names-exclusions.mjs +21 -1
- package/dist/person-stopwords.mjs +121 -1
- package/dist/section-headings.mjs +15 -3
- package/dist/sentence-verb-indicators.mjs +223 -0
- package/dist/sentence-verb-indicators.mjs.map +1 -0
- package/dist/triggers.cs.mjs +107 -3
- package/dist/triggers.es.mjs +43 -4
- package/dist/triggers.fr.mjs +206 -7
- package/dist/triggers.it.mjs +26 -4
- package/dist/triggers.pl.mjs +191 -4
- package/dist/triggers.pt-br.mjs +193 -0
- package/dist/triggers.pt-br.mjs.map +1 -0
- package/dist/triggers.sk.mjs +493 -0
- package/dist/vite.mjs.map +1 -1
- package/dist/wasm.d.mts +56 -3
- package/dist/wasm.mjs +2368 -1350
- package/dist/wasm.mjs.map +1 -1
- package/package.json +2 -2
package/dist/wasm.mjs
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { TextSearch } from "@stll/text-search-wasm";
|
|
2
|
-
import { at, be, bg, cy, cz, de, dk, ee, es, fi, fr, gb, gr, hr, hu, ie, it, lt, lu, lv, mt, nl, pl, pt, ro, se, si, sk } from "@stll/stdnum";
|
|
2
|
+
import { at, be, bg, br, cy, cz, de, dk, ee, es, fi, fr, gb, gr, hr, hu, ie, it, lt, lu, lv, mt, nl, pl, pt, ro, se, si, sk } from "@stll/stdnum";
|
|
3
3
|
import { toRegex } from "@stll/stdnum/patterns";
|
|
4
4
|
//#region src/search-engine.ts
|
|
5
5
|
let _TextSearch;
|
|
@@ -66,11 +66,15 @@ const DEFAULT_ENTITY_LABELS = [
|
|
|
66
66
|
"iban",
|
|
67
67
|
"tax identification number",
|
|
68
68
|
"identity card number",
|
|
69
|
+
"birth number",
|
|
70
|
+
"national identification number",
|
|
71
|
+
"social security number",
|
|
69
72
|
"registration number",
|
|
70
73
|
"credit card number",
|
|
71
74
|
"passport number",
|
|
72
75
|
"monetary amount",
|
|
73
|
-
"land parcel"
|
|
76
|
+
"land parcel",
|
|
77
|
+
"misc"
|
|
74
78
|
];
|
|
75
79
|
const isLegalFormsEnabled = (config) => config.enableLegalForms !== false;
|
|
76
80
|
//#endregion
|
|
@@ -95,6 +99,8 @@ const createPipelineContext = () => ({
|
|
|
95
99
|
allowListPromise: null,
|
|
96
100
|
personStopwords: null,
|
|
97
101
|
personStopwordsPromise: null,
|
|
102
|
+
addressStopwords: null,
|
|
103
|
+
addressStopwordsPromise: null,
|
|
98
104
|
firstNameExclusions: null,
|
|
99
105
|
firstNameExclusionCorpusLen: 0,
|
|
100
106
|
genericRoles: null,
|
|
@@ -151,6 +157,7 @@ const LOADER_REGISTRIES = {
|
|
|
151
157
|
hu: () => import("./triggers.hu.mjs"),
|
|
152
158
|
it: () => import("./triggers.it.mjs"),
|
|
153
159
|
pl: () => import("./triggers.pl.mjs"),
|
|
160
|
+
"pt-br": () => import("./triggers.pt-br.mjs"),
|
|
154
161
|
ro: () => import("./triggers.ro.mjs"),
|
|
155
162
|
sk: () => import("./triggers.sk.mjs"),
|
|
156
163
|
sv: () => import("./triggers.sv.mjs")
|
|
@@ -159,7 +166,23 @@ const LOADER_REGISTRIES = {
|
|
|
159
166
|
cs: () => import("./coreference.cs.mjs"),
|
|
160
167
|
de: () => import("./coreference.de.mjs"),
|
|
161
168
|
en: () => import("./coreference.en.mjs"),
|
|
169
|
+
es: () => import("./coreference.es.mjs"),
|
|
170
|
+
fr: () => import("./coreference.fr.mjs"),
|
|
171
|
+
it: () => import("./coreference.it.mjs"),
|
|
172
|
+
pl: () => import("./coreference.pl.mjs"),
|
|
173
|
+
"pt-br": () => import("./coreference.pt-br.mjs"),
|
|
162
174
|
sk: () => import("./coreference.sk.mjs")
|
|
175
|
+
},
|
|
176
|
+
legalRoleHeads: {
|
|
177
|
+
cs: () => import("./legal-role-heads.cs.mjs"),
|
|
178
|
+
de: () => import("./legal-role-heads.de.mjs"),
|
|
179
|
+
en: () => import("./legal-role-heads.en.mjs"),
|
|
180
|
+
es: () => import("./legal-role-heads.es.mjs"),
|
|
181
|
+
fr: () => import("./legal-role-heads.fr.mjs"),
|
|
182
|
+
it: () => import("./legal-role-heads.it.mjs"),
|
|
183
|
+
pl: () => import("./legal-role-heads.pl.mjs"),
|
|
184
|
+
"pt-br": () => import("./legal-role-heads.pt-br.mjs"),
|
|
185
|
+
sk: () => import("./legal-role-heads.sk.mjs")
|
|
163
186
|
}
|
|
164
187
|
};
|
|
165
188
|
const FALLBACK_LANGUAGES = {
|
|
@@ -172,6 +195,7 @@ const FALLBACK_LANGUAGES = {
|
|
|
172
195
|
"hu",
|
|
173
196
|
"it",
|
|
174
197
|
"pl",
|
|
198
|
+
"pt-br",
|
|
175
199
|
"ro",
|
|
176
200
|
"sk",
|
|
177
201
|
"sv"
|
|
@@ -180,6 +204,22 @@ const FALLBACK_LANGUAGES = {
|
|
|
180
204
|
"cs",
|
|
181
205
|
"de",
|
|
182
206
|
"en",
|
|
207
|
+
"es",
|
|
208
|
+
"fr",
|
|
209
|
+
"it",
|
|
210
|
+
"pl",
|
|
211
|
+
"pt-br",
|
|
212
|
+
"sk"
|
|
213
|
+
],
|
|
214
|
+
legalRoleHeads: [
|
|
215
|
+
"cs",
|
|
216
|
+
"de",
|
|
217
|
+
"en",
|
|
218
|
+
"es",
|
|
219
|
+
"fr",
|
|
220
|
+
"it",
|
|
221
|
+
"pl",
|
|
222
|
+
"pt-br",
|
|
183
223
|
"sk"
|
|
184
224
|
]
|
|
185
225
|
};
|
|
@@ -226,838 +266,64 @@ const loadLanguageConfigs = async (configType, mapFn) => {
|
|
|
226
266
|
await Promise.all(loads);
|
|
227
267
|
return results.filter((r) => r !== void 0);
|
|
228
268
|
};
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
console.warn("[anonymize] coreference: no definition patterns loaded; coreference detection will be inactive");
|
|
288
|
-
}
|
|
289
|
-
return patterns;
|
|
290
|
-
}
|
|
291
|
-
ctx.corefPatterns = patterns;
|
|
292
|
-
return patterns;
|
|
293
|
-
};
|
|
294
|
-
const SEARCH_WINDOW = 200;
|
|
295
|
-
/**
|
|
296
|
-
* Check whether an alias has textual similarity to
|
|
297
|
-
* the source entity. Prevents roles and structural
|
|
298
|
-
* terms from being treated as name aliases.
|
|
299
|
-
*
|
|
300
|
-
* Three checks (any passes → similar):
|
|
301
|
-
* 1. Word overlap: a word in the alias appears in the
|
|
302
|
-
* entity (case-insensitive, min 2 chars)
|
|
303
|
-
* 2. Initials: alias letters match first letters of
|
|
304
|
-
* entity words ("TB" ↔ "Tomas Bata")
|
|
305
|
-
* 3. Substring: alias is a substring of the entity
|
|
306
|
-
* or vice versa (min 3 chars)
|
|
307
|
-
*/
|
|
308
|
-
const hasEntitySimilarity = (alias, entityText) => {
|
|
309
|
-
const aliasLower = alias.toLowerCase();
|
|
310
|
-
const entityLower = entityText.toLowerCase();
|
|
311
|
-
if (aliasLower.length >= 3 && entityLower.includes(aliasLower)) return true;
|
|
312
|
-
if (entityLower.length >= 3 && aliasLower.includes(entityLower)) return true;
|
|
313
|
-
const aliasWords = aliasLower.split(/[\s.,;:'"()/-]+/).filter((w) => w.length >= 2);
|
|
314
|
-
const entityWords = entityLower.split(/[\s.,;:'"()/-]+/).filter((w) => w.length >= 2);
|
|
315
|
-
const entityWordSet = new Set(entityWords);
|
|
316
|
-
for (const word of aliasWords) if (entityWordSet.has(word)) return true;
|
|
317
|
-
if (/^[\p{Lu}]+$/u.test(alias) && alias.length >= 2 && alias.length <= entityWords.length) {
|
|
318
|
-
for (let start = 0; start <= entityWords.length - alias.length; start++) if (entityWords.slice(start, start + alias.length).map((w) => w.charAt(0)).join("") === aliasLower) return true;
|
|
319
|
-
}
|
|
320
|
-
return false;
|
|
321
|
-
};
|
|
322
|
-
/**
|
|
323
|
-
* Scan for defined-term patterns near known entities.
|
|
324
|
-
*
|
|
325
|
-
* Legal documents universally follow:
|
|
326
|
-
* "Dr. Heinrich Muller (hereinafter 'the Seller')..."
|
|
327
|
-
*
|
|
328
|
-
* After NER detects the entity, this function scans for
|
|
329
|
-
* definitional patterns within +/-200 chars and extracts
|
|
330
|
-
* the alias. Returns alias + label pairs that can be added
|
|
331
|
-
* to the gazetteer for a full-text re-scan.
|
|
332
|
-
*/
|
|
333
|
-
/**
|
|
334
|
-
* Labels that can be the source of a coreference alias.
|
|
335
|
-
* Only parties (person, organization) have defined-term
|
|
336
|
-
* aliases in legal text. Dates, addresses, IDs do not.
|
|
337
|
-
*/
|
|
338
|
-
const COREF_SOURCE_LABELS = new Set(["person", "organization"]);
|
|
339
|
-
const extractDefinedTerms = async (fullText, entities, ctx = defaultContext) => {
|
|
340
|
-
const [definitionPatterns, roleStops] = await Promise.all([getDefinitionPatterns(ctx), getRoleStopSet(ctx)]);
|
|
341
|
-
const terms = [];
|
|
342
|
-
const seen = /* @__PURE__ */ new Set();
|
|
343
|
-
const sorted = [...entities].sort((a, b) => a.start - b.start);
|
|
344
|
-
for (const { pattern } of definitionPatterns) {
|
|
345
|
-
pattern.lastIndex = 0;
|
|
346
|
-
for (let match = pattern.exec(fullText); match !== null; match = pattern.exec(fullText)) {
|
|
347
|
-
const alias = match[1]?.trim();
|
|
348
|
-
if (!alias || alias.length < 2) continue;
|
|
349
|
-
if (roleStops.has(alias.toLowerCase())) continue;
|
|
350
|
-
const defPos = match.index;
|
|
351
|
-
let bestEntity = null;
|
|
352
|
-
for (let i = sorted.length - 1; i >= 0; i--) {
|
|
353
|
-
const e = sorted[i];
|
|
354
|
-
if (e === void 0) continue;
|
|
355
|
-
if (e.end > defPos) continue;
|
|
356
|
-
if (defPos - e.end > SEARCH_WINDOW) break;
|
|
357
|
-
if (!COREF_SOURCE_LABELS.has(e.label)) continue;
|
|
358
|
-
bestEntity = e;
|
|
359
|
-
break;
|
|
360
|
-
}
|
|
361
|
-
if (bestEntity === null) continue;
|
|
362
|
-
const gapText = fullText.slice(bestEntity.end, defPos);
|
|
363
|
-
if (/(?:;|\.(?=\s*(?:["'„‚(]*\p{Lu}|$)))/u.test(gapText)) continue;
|
|
364
|
-
if (!hasEntitySimilarity(alias, bestEntity.text)) continue;
|
|
365
|
-
const key = `${alias.toLowerCase()}::${bestEntity.label}`;
|
|
366
|
-
if (seen.has(key)) continue;
|
|
367
|
-
seen.add(key);
|
|
368
|
-
terms.push({
|
|
369
|
-
alias,
|
|
370
|
-
label: bestEntity.label,
|
|
371
|
-
definitionStart: defPos,
|
|
372
|
-
sourceText: bestEntity.text
|
|
373
|
-
});
|
|
374
|
-
}
|
|
375
|
-
}
|
|
376
|
-
return terms;
|
|
377
|
-
};
|
|
378
|
-
/**
|
|
379
|
-
* Check if a character is a Unicode word character
|
|
380
|
-
* (letter, digit, or combining mark). Used for word
|
|
381
|
-
* boundary checks in coreference matching.
|
|
382
|
-
*/
|
|
383
|
-
const isWordChar = (ch) => {
|
|
384
|
-
if (ch === void 0) return false;
|
|
385
|
-
return /[\p{L}\p{M}\p{N}]/u.test(ch);
|
|
386
|
-
};
|
|
387
|
-
/**
|
|
388
|
-
* Find all occurrences of defined-term aliases in the
|
|
389
|
-
* full text. Returns Entity spans for each match.
|
|
390
|
-
*
|
|
391
|
-
* Respects word boundaries: "Kupující" must not match
|
|
392
|
-
* inside "Kupujícímu". A match is valid only if the
|
|
393
|
-
* character before the start and after the end are NOT
|
|
394
|
-
* word characters (letter/digit).
|
|
395
|
-
*
|
|
396
|
-
* Populates `ctx.corefSourceMap` with entries linking
|
|
397
|
-
* each coref entity to its source entity text, for
|
|
398
|
-
* consistent placeholder numbering.
|
|
399
|
-
*/
|
|
400
|
-
const findCoreferenceSpans = (fullText, terms, ctx = defaultContext) => {
|
|
401
|
-
const results = [];
|
|
402
|
-
for (const term of terms) {
|
|
403
|
-
let searchFrom = 0;
|
|
404
|
-
while (searchFrom < fullText.length) {
|
|
405
|
-
const idx = fullText.indexOf(term.alias, searchFrom);
|
|
406
|
-
if (idx === -1) break;
|
|
407
|
-
const matchEnd = idx + term.alias.length;
|
|
408
|
-
const charBefore = idx > 0 ? fullText[idx - 1] : void 0;
|
|
409
|
-
const charAfter = fullText[matchEnd];
|
|
410
|
-
if (isWordChar(charBefore) || isWordChar(charAfter)) {
|
|
411
|
-
searchFrom = idx + 1;
|
|
412
|
-
continue;
|
|
413
|
-
}
|
|
414
|
-
const entity = {
|
|
415
|
-
start: idx,
|
|
416
|
-
end: matchEnd,
|
|
417
|
-
label: term.label,
|
|
418
|
-
text: term.alias,
|
|
419
|
-
score: .95,
|
|
420
|
-
source: DETECTION_SOURCES.COREFERENCE
|
|
421
|
-
};
|
|
422
|
-
ctx.corefSourceMap.set(corefKey(entity), term.sourceText);
|
|
423
|
-
results.push(entity);
|
|
424
|
-
searchFrom = matchEnd;
|
|
425
|
-
}
|
|
426
|
-
}
|
|
427
|
-
return results;
|
|
428
|
-
};
|
|
429
|
-
//#endregion
|
|
430
|
-
//#region src/detectors/gazetteer.ts
|
|
431
|
-
const MAX_EDIT_DISTANCE = 2;
|
|
432
|
-
const MIN_FUZZY_LENGTH = 4;
|
|
433
|
-
const MAX_PREFIX_OVERSHOOT = 7;
|
|
434
|
-
/**
|
|
435
|
-
* Collect all searchable strings (canonical + variants)
|
|
436
|
-
* from gazetteer entries, mapped to their labels and
|
|
437
|
-
* entry IDs.
|
|
438
|
-
*/
|
|
439
|
-
const buildSearchTerms = (entries) => {
|
|
440
|
-
const terms = /* @__PURE__ */ new Map();
|
|
441
|
-
for (const entry of entries) {
|
|
442
|
-
const meta = {
|
|
443
|
-
label: entry.label,
|
|
444
|
-
entryId: entry.id
|
|
445
|
-
};
|
|
446
|
-
terms.set(entry.canonical, meta);
|
|
447
|
-
for (const variant of entry.variants) terms.set(variant, meta);
|
|
448
|
-
}
|
|
449
|
-
return terms;
|
|
450
|
-
};
|
|
451
|
-
/**
|
|
452
|
-
* Build TextSearch-compatible patterns from gazetteer
|
|
453
|
-
* entries. Returns:
|
|
454
|
-
* - Exact literal patterns for all terms
|
|
455
|
-
* - Fuzzy patterns (distance: 2) for terms >= 4 chars
|
|
456
|
-
* - Parallel metadata arrays for post-processing
|
|
457
|
-
*
|
|
458
|
-
* Patterns are ordered: all exact first, then all
|
|
459
|
-
* fuzzy. The isFuzzy array marks which are which.
|
|
460
|
-
*/
|
|
461
|
-
const buildGazetteerPatterns = (entries) => {
|
|
462
|
-
const terms = buildSearchTerms(entries);
|
|
463
|
-
const patterns = [];
|
|
464
|
-
const labels = [];
|
|
465
|
-
const isFuzzy = [];
|
|
466
|
-
for (const [term, meta] of terms) {
|
|
467
|
-
patterns.push({
|
|
468
|
-
pattern: term,
|
|
469
|
-
literal: true,
|
|
470
|
-
wholeWords: false
|
|
471
|
-
});
|
|
472
|
-
labels.push(meta.label);
|
|
473
|
-
isFuzzy.push(false);
|
|
474
|
-
}
|
|
475
|
-
for (const [term, meta] of terms) {
|
|
476
|
-
if (term.length < MIN_FUZZY_LENGTH) continue;
|
|
477
|
-
patterns.push({
|
|
478
|
-
pattern: term,
|
|
479
|
-
distance: MAX_EDIT_DISTANCE
|
|
480
|
-
});
|
|
481
|
-
labels.push(meta.label);
|
|
482
|
-
isFuzzy.push(true);
|
|
483
|
-
}
|
|
484
|
-
return {
|
|
485
|
-
patterns,
|
|
486
|
-
data: {
|
|
487
|
-
labels,
|
|
488
|
-
isFuzzy
|
|
489
|
-
}
|
|
490
|
-
};
|
|
491
|
-
};
|
|
492
|
-
/**
|
|
493
|
-
* Process gazetteer matches from the unified literal
|
|
494
|
-
* search. Receives all matches; filters to the
|
|
495
|
-
* gazetteer slice via sliceStart/sliceEnd.
|
|
496
|
-
*
|
|
497
|
-
* Exact matches get score 0.9; fuzzy matches get
|
|
498
|
-
* 0.85. Fuzzy matches that overlap an exact match
|
|
499
|
-
* are dropped.
|
|
500
|
-
*
|
|
501
|
-
* For exact matches, attempts prefix extension for
|
|
502
|
-
* legal suffixes ("a.s.", "GmbH", "s.r.o." after
|
|
503
|
-
* the matched term).
|
|
504
|
-
*/
|
|
505
|
-
const processGazetteerMatches = (allMatches, sliceStart, sliceEnd, fullText, data) => {
|
|
506
|
-
const results = [];
|
|
507
|
-
const exactSpans = [];
|
|
508
|
-
for (const match of allMatches) {
|
|
509
|
-
const idx = match.pattern;
|
|
510
|
-
if (idx < sliceStart || idx >= sliceEnd) continue;
|
|
511
|
-
const localIdx = idx - sliceStart;
|
|
512
|
-
if (data.isFuzzy[localIdx]) continue;
|
|
513
|
-
const label = data.labels[localIdx];
|
|
514
|
-
if (!label) continue;
|
|
515
|
-
const extended = tryPrefixExtension(fullText, match.start, match.end);
|
|
516
|
-
const end = extended?.end ?? match.end;
|
|
517
|
-
const text = extended?.text ?? fullText.slice(match.start, match.end);
|
|
518
|
-
exactSpans.push({
|
|
519
|
-
start: match.start,
|
|
520
|
-
end
|
|
521
|
-
});
|
|
522
|
-
results.push({
|
|
523
|
-
start: match.start,
|
|
524
|
-
end,
|
|
525
|
-
label,
|
|
526
|
-
text,
|
|
527
|
-
score: .9,
|
|
528
|
-
source: DETECTION_SOURCES.GAZETTEER
|
|
529
|
-
});
|
|
530
|
-
}
|
|
531
|
-
for (const match of allMatches) {
|
|
532
|
-
const idx = match.pattern;
|
|
533
|
-
if (idx < sliceStart || idx >= sliceEnd) continue;
|
|
534
|
-
const localIdx = idx - sliceStart;
|
|
535
|
-
if (!data.isFuzzy[localIdx]) continue;
|
|
536
|
-
if (match.distance === 0) continue;
|
|
537
|
-
const label = data.labels[localIdx];
|
|
538
|
-
if (!label) continue;
|
|
539
|
-
if (exactSpans.some((e) => match.start < e.end && match.end > e.start)) continue;
|
|
540
|
-
const matchText = fullText.slice(match.start, match.end);
|
|
541
|
-
results.push({
|
|
542
|
-
start: match.start,
|
|
543
|
-
end: match.end,
|
|
544
|
-
label,
|
|
545
|
-
text: matchText,
|
|
546
|
-
score: .85,
|
|
547
|
-
source: DETECTION_SOURCES.GAZETTEER
|
|
548
|
-
});
|
|
549
|
-
}
|
|
550
|
-
return results;
|
|
551
|
-
};
|
|
552
|
-
/**
|
|
553
|
-
* Try to extend an exact match to capture one
|
|
554
|
-
* trailing token (max 6 chars) that may be a legal
|
|
555
|
-
* entity suffix (e.g., "a.s.", "GmbH", "s.r.o.").
|
|
556
|
-
*
|
|
557
|
-
* Does not validate the token against a legal-forms
|
|
558
|
-
* list; false extensions are filtered by mergeAndDedup
|
|
559
|
-
* when a legal-form detector produces a competing
|
|
560
|
-
* entity with the correct span.
|
|
561
|
-
*/
|
|
562
|
-
const tryPrefixExtension = (fullText, start, end) => {
|
|
563
|
-
const maxEnd = Math.min(end + MAX_PREFIX_OVERSHOOT, fullText.length);
|
|
564
|
-
if (maxEnd <= end + 1) return null;
|
|
565
|
-
const after = fullText.slice(end, maxEnd);
|
|
566
|
-
if (!after.startsWith(" ")) return null;
|
|
567
|
-
const nextSpace = after.indexOf(" ", 1);
|
|
568
|
-
const suffixEnd = nextSpace !== -1 ? nextSpace : after.length;
|
|
569
|
-
if (suffixEnd <= 1) return null;
|
|
570
|
-
const newEnd = end + suffixEnd;
|
|
571
|
-
return {
|
|
572
|
-
end: newEnd,
|
|
573
|
-
text: fullText.slice(start, newEnd)
|
|
574
|
-
};
|
|
575
|
-
};
|
|
576
|
-
//#endregion
|
|
577
|
-
//#region src/util/text.ts
|
|
578
|
-
/**
|
|
579
|
-
* Shared text utilities for detectors.
|
|
580
|
-
*
|
|
581
|
-
* Extracted from names.ts and deny-list.ts to avoid
|
|
582
|
-
* duplicating regex constants and helper functions.
|
|
583
|
-
*/
|
|
584
|
-
/** Matches a string that starts with an uppercase letter. */
|
|
585
|
-
const UPPER_START_RE = /^\p{Lu}/u;
|
|
586
|
-
/** Matches a string consisting entirely of uppercase letters. */
|
|
587
|
-
const ALL_UPPER_RE = /^\p{Lu}+$/u;
|
|
588
|
-
const SENTENCE_END_RE = /[.!?]/;
|
|
589
|
-
/**
|
|
590
|
-
* Detect whether a position is at the start of a sentence.
|
|
591
|
-
* Looks backward past whitespace for sentence-ending
|
|
592
|
-
* punctuation (.!?). Position 0 and positions preceded
|
|
593
|
-
* only by whitespace are considered sentence starts.
|
|
594
|
-
*/
|
|
595
|
-
const isSentenceStart = (text, pos) => {
|
|
596
|
-
if (pos === 0) return true;
|
|
597
|
-
let i = pos - 1;
|
|
598
|
-
while (i >= 0 && /\s/.test(text[i] ?? "")) i--;
|
|
599
|
-
if (i < 0) return true;
|
|
600
|
-
return SENTENCE_END_RE.test(text[i] ?? "");
|
|
601
|
-
};
|
|
602
|
-
//#endregion
|
|
603
|
-
//#region src/detectors/names.ts
|
|
604
|
-
const getCorpus = (ctx) => ctx.nameCorpus;
|
|
605
|
-
const getNameCorpusFirstNames = (ctx = defaultContext) => ctx.nameCorpus?.firstNamesList ?? [];
|
|
606
|
-
const getNameCorpusSurnames = (ctx = defaultContext) => ctx.nameCorpus?.surnamesList ?? [];
|
|
607
|
-
const getNameCorpusTitles = (ctx = defaultContext) => ctx.nameCorpus?.titlesList ?? [];
|
|
608
|
-
/**
|
|
609
|
-
* Load name corpus data from injected dictionaries
|
|
610
|
-
* and legacy config files. Merges all sources.
|
|
611
|
-
*
|
|
612
|
-
* Safe to call multiple times; only loads once per
|
|
613
|
-
* context. Must be called before detectNameCorpus or
|
|
614
|
-
* the getNameCorpus*() accessors are used.
|
|
615
|
-
*
|
|
616
|
-
* @param dictionaries Optional pre-loaded dictionaries
|
|
617
|
-
* with per-language first names and surnames. When
|
|
618
|
-
* omitted, only legacy config files are used.
|
|
619
|
-
*/
|
|
620
|
-
const initNameCorpus = (ctx = defaultContext, dictionaries) => {
|
|
621
|
-
if (ctx.nameCorpusPromise) return ctx.nameCorpusPromise;
|
|
622
|
-
const promise = (async () => {
|
|
623
|
-
try {
|
|
624
|
-
const [legacyFirstMod, legacySurnameMod, titleMod, exclusionMod] = await Promise.all([
|
|
625
|
-
import("./names-first.mjs"),
|
|
626
|
-
import("./names-surnames.mjs"),
|
|
627
|
-
import("./names-title-tokens.mjs"),
|
|
628
|
-
import("./names-exclusions.mjs")
|
|
629
|
-
]);
|
|
630
|
-
const firstNames = [...legacyFirstMod.default.names];
|
|
631
|
-
if (dictionaries?.firstNames) for (const names of Object.values(dictionaries.firstNames)) for (const name of names) firstNames.push(name);
|
|
632
|
-
const surnames = [...legacySurnameMod.default.names];
|
|
633
|
-
if (dictionaries?.surnames) for (const names of Object.values(dictionaries.surnames)) for (const name of names) surnames.push(name);
|
|
634
|
-
const dedup = (arr) => {
|
|
635
|
-
const seen = /* @__PURE__ */ new Set();
|
|
636
|
-
const result = [];
|
|
637
|
-
for (const item of arr) {
|
|
638
|
-
if (seen.has(item)) continue;
|
|
639
|
-
seen.add(item);
|
|
640
|
-
result.push(item);
|
|
641
|
-
}
|
|
642
|
-
return result;
|
|
643
|
-
};
|
|
644
|
-
const dedupFirst = dedup(firstNames);
|
|
645
|
-
const dedupSurnames = dedup(surnames);
|
|
646
|
-
const titles = titleMod.default.tokens;
|
|
647
|
-
const exclusions = exclusionMod.default.words;
|
|
648
|
-
ctx.nameCorpus = {
|
|
649
|
-
firstNames: Object.freeze(new Set(dedupFirst)),
|
|
650
|
-
surnames: Object.freeze(new Set(dedupSurnames)),
|
|
651
|
-
titleTokens: Object.freeze(new Set(titles)),
|
|
652
|
-
excludedWords: Object.freeze(new Set(exclusions)),
|
|
653
|
-
firstNamesList: Object.freeze(dedupFirst),
|
|
654
|
-
surnamesList: Object.freeze(dedupSurnames),
|
|
655
|
-
titlesList: Object.freeze(titles),
|
|
656
|
-
excludedList: Object.freeze(exclusions)
|
|
657
|
-
};
|
|
658
|
-
} catch (err) {
|
|
659
|
-
ctx.nameCorpusPromise = null;
|
|
660
|
-
console.warn("[anonymize] Failed to load name corpus JSON — name detection disabled:", err);
|
|
661
|
-
}
|
|
662
|
-
})();
|
|
663
|
-
ctx.nameCorpusPromise = promise;
|
|
664
|
-
return promise;
|
|
665
|
-
};
|
|
666
|
-
const INFLECTION_SUFFIXES = [
|
|
667
|
-
"ovi",
|
|
668
|
-
"em",
|
|
669
|
-
"om",
|
|
670
|
-
"ou",
|
|
671
|
-
"é",
|
|
672
|
-
"a",
|
|
673
|
-
"u"
|
|
674
|
-
];
|
|
675
|
-
/**
|
|
676
|
-
* Strip common Czech/Slovak case suffixes from a token.
|
|
677
|
-
* Returns candidate base forms if stripping produces a
|
|
678
|
-
* plausible name (capitalised, length >= 3).
|
|
679
|
-
*
|
|
680
|
-
* For the "-ou" instrumental feminine suffix, also yields
|
|
681
|
-
* base + "a" (e.g., "Editou" → "Edit" and "Edita")
|
|
682
|
-
* because Czech feminine names decline -a → -ou.
|
|
683
|
-
*/
|
|
684
|
-
const stripInflection = (token) => {
|
|
685
|
-
const candidates = [];
|
|
686
|
-
for (const suffix of INFLECTION_SUFFIXES) if (token.length > suffix.length + 2 && token.endsWith(suffix)) {
|
|
687
|
-
const base = token.slice(0, -suffix.length);
|
|
688
|
-
if (/^\p{Lu}/u.test(base)) {
|
|
689
|
-
candidates.push(base);
|
|
690
|
-
if (suffix === "ou" || suffix === "é" || suffix === "u") candidates.push(`${base}a`);
|
|
691
|
-
}
|
|
692
|
-
}
|
|
693
|
-
return candidates;
|
|
694
|
-
};
|
|
695
|
-
const TOKEN_TYPE = {
|
|
696
|
-
NAME: "name",
|
|
697
|
-
SURNAME: "surname",
|
|
698
|
-
TITLE: "title",
|
|
699
|
-
ABBREVIATION: "abbreviation",
|
|
700
|
-
CAPITALIZED: "capitalized",
|
|
701
|
-
OTHER: "other"
|
|
702
|
-
};
|
|
703
|
-
const PERSON_CHAIN_BREAK_RE$1 = /[!?;:]/u;
|
|
704
|
-
const isInitialContinuationGap$1 = (text, gap) => /^\p{Lu}$/u.test(text) && /^\.[^\S\n]{1,2}$/u.test(gap) || /^[^\S\n]{1,2}(?:\p{Lu}\.[^\S\n]{1,2})+$/u.test(gap);
|
|
705
|
-
/**
|
|
706
|
-
* Check if a token is in the first-name set, either
|
|
707
|
-
* directly or after stripping Czech/Slovak inflection.
|
|
708
|
-
*/
|
|
709
|
-
const isFirstNameToken = (token, corpus) => {
|
|
710
|
-
if (corpus.firstNames.has(token)) return true;
|
|
711
|
-
return stripInflection(token).some((b) => corpus.firstNames.has(b));
|
|
712
|
-
};
|
|
713
|
-
/**
|
|
714
|
-
* Check if a token is in the surname set, either
|
|
715
|
-
* directly or after stripping Czech/Slovak inflection.
|
|
716
|
-
*/
|
|
717
|
-
const isSurnameToken = (token, corpus) => {
|
|
718
|
-
if (corpus.surnames.has(token)) return true;
|
|
719
|
-
return stripInflection(token).some((b) => corpus.surnames.has(b));
|
|
720
|
-
};
|
|
721
|
-
/**
|
|
722
|
-
* Check if a token looks like a single-letter
|
|
723
|
-
* abbreviation: "J.", "M.", etc.
|
|
724
|
-
*/
|
|
725
|
-
const isAbbreviation = (token) => token.length === 2 && /^\p{Lu}$/u.test(token[0] ?? "") && token[1] === ".";
|
|
726
|
-
const segmenter$1 = new Intl.Segmenter(void 0, { granularity: "word" });
|
|
727
|
-
/**
|
|
728
|
-
* Split text into word segments using Intl.Segmenter.
|
|
729
|
-
* Only returns segments flagged as words.
|
|
730
|
-
*/
|
|
731
|
-
const segmentWords = (fullText) => {
|
|
732
|
-
const words = [];
|
|
733
|
-
for (const seg of segmenter$1.segment(fullText)) if (seg.isWordLike) words.push({
|
|
734
|
-
text: seg.segment,
|
|
735
|
-
start: seg.index,
|
|
736
|
-
end: seg.index + seg.segment.length
|
|
737
|
-
});
|
|
738
|
-
return words;
|
|
739
|
-
};
|
|
740
|
-
/** NAME or SURNAME — both represent corpus-matched tokens */
|
|
741
|
-
const isCorpusMatch = (type) => type === TOKEN_TYPE.NAME || type === TOKEN_TYPE.SURNAME;
|
|
742
|
-
const classifyToken = (word, corpus) => {
|
|
743
|
-
const { text, start, end } = word;
|
|
744
|
-
const lower = text.toLowerCase();
|
|
745
|
-
const stripped = text.endsWith(".") ? text.slice(0, -1).toLowerCase() : lower;
|
|
746
|
-
if (corpus.titleTokens.has(stripped)) return {
|
|
747
|
-
text,
|
|
748
|
-
type: TOKEN_TYPE.TITLE,
|
|
749
|
-
start,
|
|
750
|
-
end
|
|
751
|
-
};
|
|
752
|
-
if (isAbbreviation(text)) return {
|
|
753
|
-
text,
|
|
754
|
-
type: TOKEN_TYPE.ABBREVIATION,
|
|
755
|
-
start,
|
|
756
|
-
end
|
|
757
|
-
};
|
|
758
|
-
if (corpus.excludedWords.has(lower)) return {
|
|
759
|
-
text,
|
|
760
|
-
type: TOKEN_TYPE.OTHER,
|
|
761
|
-
start,
|
|
762
|
-
end
|
|
763
|
-
};
|
|
764
|
-
if (text.length < 3) return {
|
|
765
|
-
text,
|
|
766
|
-
type: TOKEN_TYPE.OTHER,
|
|
767
|
-
start,
|
|
768
|
-
end
|
|
769
|
-
};
|
|
770
|
-
if (text.length > 3 && ALL_UPPER_RE.test(text)) return {
|
|
771
|
-
text,
|
|
772
|
-
type: TOKEN_TYPE.OTHER,
|
|
773
|
-
start,
|
|
774
|
-
end
|
|
775
|
-
};
|
|
776
|
-
if (!UPPER_START_RE.test(text)) return {
|
|
777
|
-
text,
|
|
778
|
-
type: TOKEN_TYPE.OTHER,
|
|
779
|
-
start,
|
|
780
|
-
end
|
|
781
|
-
};
|
|
782
|
-
if (isFirstNameToken(text, corpus)) return {
|
|
783
|
-
text,
|
|
784
|
-
type: TOKEN_TYPE.NAME,
|
|
785
|
-
start,
|
|
786
|
-
end
|
|
787
|
-
};
|
|
788
|
-
if (isSurnameToken(text, corpus)) return {
|
|
789
|
-
text,
|
|
790
|
-
type: TOKEN_TYPE.SURNAME,
|
|
791
|
-
start,
|
|
792
|
-
end
|
|
793
|
-
};
|
|
794
|
-
return {
|
|
795
|
-
text,
|
|
796
|
-
type: TOKEN_TYPE.CAPITALIZED,
|
|
797
|
-
start,
|
|
798
|
-
end
|
|
799
|
-
};
|
|
800
|
-
};
|
|
801
|
-
/**
|
|
802
|
-
* Detect person names by looking up tokens against the
|
|
803
|
-
* name corpus, then chaining adjacent name-like tokens.
|
|
804
|
-
*
|
|
805
|
-
* Requires initNameCorpus() to have been called first.
|
|
806
|
-
* If not initialized, returns an empty array.
|
|
807
|
-
*
|
|
808
|
-
* Scoring:
|
|
809
|
-
* TITLE + NAME/SURNAME → 0.95
|
|
810
|
-
* NAME + NAME/SURNAME → 0.9
|
|
811
|
-
* SURNAME + NAME/SURNAME → 0.9
|
|
812
|
-
* NAME + CAPITALIZED → 0.7
|
|
813
|
-
* ABBREVIATION + NAME → 0.7
|
|
814
|
-
* Standalone NAME → 0.5 (low confidence)
|
|
815
|
-
* Standalone SURNAME → skip (too ambiguous)
|
|
816
|
-
*/
|
|
817
|
-
const detectNameCorpus = (fullText, ctx = defaultContext) => {
|
|
818
|
-
const corpus = getCorpus(ctx);
|
|
819
|
-
if (!corpus) return [];
|
|
820
|
-
const tokens = segmentWords(fullText).map((w) => classifyToken(w, corpus));
|
|
821
|
-
const entities = [];
|
|
822
|
-
const consumed = /* @__PURE__ */ new Set();
|
|
823
|
-
for (let i = 0; i < tokens.length; i++) {
|
|
824
|
-
if (consumed.has(i)) continue;
|
|
825
|
-
const token = tokens[i];
|
|
826
|
-
if (!token) continue;
|
|
827
|
-
if (token.type !== TOKEN_TYPE.TITLE && token.type !== TOKEN_TYPE.NAME && token.type !== TOKEN_TYPE.SURNAME && token.type !== TOKEN_TYPE.ABBREVIATION) continue;
|
|
828
|
-
const MAX_CHAIN = 5;
|
|
829
|
-
const chain = [token];
|
|
830
|
-
let j = i + 1;
|
|
831
|
-
while (j < tokens.length && chain.length < MAX_CHAIN) {
|
|
832
|
-
const next = tokens[j];
|
|
833
|
-
if (!next) break;
|
|
834
|
-
const prev = chain.at(-1);
|
|
835
|
-
if (prev) {
|
|
836
|
-
const gap = fullText.slice(prev.end, next.start);
|
|
837
|
-
const breaksOnPeriod = gap.includes(".") && !isInitialContinuationGap$1(prev.text, gap);
|
|
838
|
-
if (gap.includes("\n") || PERSON_CHAIN_BREAK_RE$1.test(gap) || breaksOnPeriod) break;
|
|
839
|
-
}
|
|
840
|
-
if (next.type === TOKEN_TYPE.NAME || next.type === TOKEN_TYPE.SURNAME || next.type === TOKEN_TYPE.TITLE || next.type === TOKEN_TYPE.ABBREVIATION || next.type === TOKEN_TYPE.CAPITALIZED) {
|
|
841
|
-
chain.push(next);
|
|
842
|
-
j++;
|
|
843
|
-
} else break;
|
|
844
|
-
}
|
|
845
|
-
const hasTitle = chain.some((t) => t.type === TOKEN_TYPE.TITLE);
|
|
846
|
-
const hasCorpusName = chain.some((t) => isCorpusMatch(t.type));
|
|
847
|
-
const hasFirstName = chain.some((t) => t.type === TOKEN_TYPE.NAME);
|
|
848
|
-
const hasAbbreviation = chain.some((t) => t.type === TOKEN_TYPE.ABBREVIATION);
|
|
849
|
-
const corpusCount = chain.filter((t) => isCorpusMatch(t.type)).length;
|
|
850
|
-
const capitalizedCount = chain.filter((t) => t.type === TOKEN_TYPE.CAPITALIZED).length;
|
|
851
|
-
let score = 0;
|
|
852
|
-
if (hasTitle && hasCorpusName) score = .95;
|
|
853
|
-
else if (corpusCount >= 2) score = .9;
|
|
854
|
-
else if (hasCorpusName && capitalizedCount > 0) score = .7;
|
|
855
|
-
else if (hasAbbreviation && hasCorpusName) score = .7;
|
|
856
|
-
else if (hasFirstName && chain.length === 1) {
|
|
857
|
-
if (isSentenceStart(fullText, token.start)) continue;
|
|
858
|
-
score = .5;
|
|
859
|
-
} else if (!hasFirstName && chain.length === 1 && chain[0]?.type === TOKEN_TYPE.SURNAME) continue;
|
|
860
|
-
else if (hasTitle && chain.length === 1) continue;
|
|
861
|
-
else {
|
|
862
|
-
if (!hasCorpusName) continue;
|
|
863
|
-
score = .5;
|
|
864
|
-
}
|
|
865
|
-
const first = chain.at(0);
|
|
866
|
-
const last = chain.at(-1);
|
|
867
|
-
if (!first || !last) continue;
|
|
868
|
-
const start = first.start;
|
|
869
|
-
const end = last.end;
|
|
870
|
-
const text = fullText.slice(start, end);
|
|
871
|
-
for (let k = i; k < i + chain.length; k++) consumed.add(k);
|
|
872
|
-
entities.push({
|
|
873
|
-
start,
|
|
874
|
-
end,
|
|
875
|
-
label: "person",
|
|
876
|
-
text,
|
|
877
|
-
score,
|
|
878
|
-
source: DETECTION_SOURCES.REGEX
|
|
879
|
-
});
|
|
880
|
-
}
|
|
881
|
-
return entities;
|
|
882
|
-
};
|
|
883
|
-
//#endregion
|
|
884
|
-
//#region src/config/titles.ts
|
|
885
|
-
/**
|
|
886
|
-
* Academic and professional title prefixes.
|
|
887
|
-
* Plain text; the detector auto-escapes for regex.
|
|
888
|
-
* Sorted longest-first at build time.
|
|
889
|
-
*/
|
|
890
|
-
const TITLE_PREFIXES = [
|
|
891
|
-
"Ing.",
|
|
892
|
-
"Mgr.",
|
|
893
|
-
"MgA.",
|
|
894
|
-
"Bc.",
|
|
895
|
-
"BcA.",
|
|
896
|
-
"JUDr.",
|
|
897
|
-
"MUDr.",
|
|
898
|
-
"MVDr.",
|
|
899
|
-
"MDDr.",
|
|
900
|
-
"PhDr.",
|
|
901
|
-
"RNDr.",
|
|
902
|
-
"PaedDr.",
|
|
903
|
-
"ThDr.",
|
|
904
|
-
"ThLic.",
|
|
905
|
-
"ICDr.",
|
|
906
|
-
"RSDr.",
|
|
907
|
-
"PharmDr.",
|
|
908
|
-
"artD.",
|
|
909
|
-
"akad.",
|
|
910
|
-
"doc.",
|
|
911
|
-
"prof.",
|
|
912
|
-
"ao. Univ.-Prof.",
|
|
913
|
-
"o. Univ.-Prof.",
|
|
914
|
-
"Univ.-Prof.",
|
|
915
|
-
"Hon.-Prof.",
|
|
916
|
-
"em. Prof.",
|
|
917
|
-
"Dr. med. dent.",
|
|
918
|
-
"Dr. med. vet.",
|
|
919
|
-
"Dr. med.",
|
|
920
|
-
"Dr. rer. nat.",
|
|
921
|
-
"Dr. rer. soc.",
|
|
922
|
-
"Dr. rer. pol.",
|
|
923
|
-
"Dr. sc. tech.",
|
|
924
|
-
"Dr. sc. nat.",
|
|
925
|
-
"Dr. sc. hum.",
|
|
926
|
-
"Dr. iur.",
|
|
927
|
-
"Dr. jur.",
|
|
928
|
-
"Dr. theol.",
|
|
929
|
-
"Dr. oec.",
|
|
930
|
-
"Dr. techn.",
|
|
931
|
-
"Dr. h. c.",
|
|
932
|
-
"Dr. phil.",
|
|
933
|
-
"Dr.-Ing.",
|
|
934
|
-
"Dr. Ing.",
|
|
935
|
-
"Dr.",
|
|
936
|
-
"Dipl.-Wirt.-Ing.",
|
|
937
|
-
"Dipl.-Betriebsw.",
|
|
938
|
-
"Dipl.-Inform.",
|
|
939
|
-
"Dipl.-Volksw.",
|
|
940
|
-
"Dipl.-Psych.",
|
|
941
|
-
"Dipl.-Phys.",
|
|
942
|
-
"Dipl.-Chem.",
|
|
943
|
-
"Dipl.-Biol.",
|
|
944
|
-
"Dipl.-Math.",
|
|
945
|
-
"Dipl.-Päd.",
|
|
946
|
-
"Dipl.-Soz.",
|
|
947
|
-
"Dipl.-Kfm.",
|
|
948
|
-
"Dipl.-Jur.",
|
|
949
|
-
"Dipl. Ing.",
|
|
950
|
-
"Dipl.-Ing.",
|
|
951
|
-
"Mag. rer. soc. oec.",
|
|
952
|
-
"Mag. rer. nat.",
|
|
953
|
-
"Mag. phil.",
|
|
954
|
-
"Mag. iur.",
|
|
955
|
-
"Mag. arch.",
|
|
956
|
-
"Mag. pharm.",
|
|
957
|
-
"Mag. (FH)",
|
|
958
|
-
"Mag.",
|
|
959
|
-
"Bakk. rer. nat.",
|
|
960
|
-
"Bakk. techn.",
|
|
961
|
-
"Bakk. phil.",
|
|
962
|
-
"Bakk.",
|
|
963
|
-
"Lic. phil.",
|
|
964
|
-
"Lic. iur.",
|
|
965
|
-
"Lic. oec.",
|
|
966
|
-
"Lic. theol.",
|
|
967
|
-
"Lic.",
|
|
968
|
-
"Priv.-Doz.",
|
|
969
|
-
"PD",
|
|
970
|
-
"RA"
|
|
971
|
-
];
|
|
972
|
-
/**
|
|
973
|
-
* Courtesy/honorific titles that precede a person's
|
|
974
|
-
* name. Sorted alphabetically. The detector escapes
|
|
975
|
-
* dots and adds \b for entries in HONORIFIC_BOUNDARY.
|
|
976
|
-
*/
|
|
977
|
-
const HONORIFICS = [
|
|
978
|
-
"Avv.",
|
|
979
|
-
"Dame",
|
|
980
|
-
"Doamna",
|
|
981
|
-
"Domnul",
|
|
982
|
-
"Don",
|
|
983
|
-
"Doña",
|
|
984
|
-
"Dott.",
|
|
985
|
-
"Judge",
|
|
986
|
-
"Justice",
|
|
987
|
-
"Lady",
|
|
988
|
-
"Lord",
|
|
989
|
-
"M.",
|
|
990
|
-
"Madame",
|
|
991
|
-
"Mademoiselle",
|
|
992
|
-
"Maître",
|
|
993
|
-
"Me",
|
|
994
|
-
"Messrs",
|
|
995
|
-
"Miss",
|
|
996
|
-
"Mlle",
|
|
997
|
-
"Mme",
|
|
998
|
-
"Monsieur",
|
|
999
|
-
"Mr",
|
|
1000
|
-
"Mrs",
|
|
1001
|
-
"Ms",
|
|
1002
|
-
"President",
|
|
1003
|
-
"Señor",
|
|
1004
|
-
"Señora",
|
|
1005
|
-
"Sig.",
|
|
1006
|
-
"Sig.ra",
|
|
1007
|
-
"Signor",
|
|
1008
|
-
"Signora",
|
|
1009
|
-
"Signorina",
|
|
1010
|
-
"Sir",
|
|
1011
|
-
"Sr.",
|
|
1012
|
-
"Sra."
|
|
1013
|
-
];
|
|
1014
|
-
/**
|
|
1015
|
-
* Honorifics that need \b word-boundary anchors
|
|
1016
|
-
* (short or common words that could match mid-word).
|
|
1017
|
-
*/
|
|
1018
|
-
const HONORIFIC_BOUNDARY = new Set([
|
|
1019
|
-
"Don",
|
|
1020
|
-
"Doña",
|
|
1021
|
-
"M.",
|
|
1022
|
-
"Me",
|
|
1023
|
-
"Señor",
|
|
1024
|
-
"Señora"
|
|
1025
|
-
]);
|
|
1026
|
-
/**
|
|
1027
|
-
* Post-nominal degrees (comma or space separated after name).
|
|
1028
|
-
* Plain text; the detector auto-escapes for regex.
|
|
1029
|
-
*/
|
|
1030
|
-
const POST_NOMINALS = [
|
|
1031
|
-
"Ph.D.",
|
|
1032
|
-
"Ph.D",
|
|
1033
|
-
"CSc.",
|
|
1034
|
-
"DrSc.",
|
|
1035
|
-
"ArtD.",
|
|
1036
|
-
"D.Phil.",
|
|
1037
|
-
"DPhil.",
|
|
1038
|
-
"MPhil.",
|
|
1039
|
-
"MBA",
|
|
1040
|
-
"MPA",
|
|
1041
|
-
"LL.M.",
|
|
1042
|
-
"LL.B.",
|
|
1043
|
-
"M.Sc.",
|
|
1044
|
-
"B.Sc.",
|
|
1045
|
-
"MSc.",
|
|
1046
|
-
"BSc.",
|
|
1047
|
-
"M.Eng.",
|
|
1048
|
-
"B.Eng.",
|
|
1049
|
-
"M.A.",
|
|
1050
|
-
"B.A.",
|
|
1051
|
-
"JCD",
|
|
1052
|
-
"JD",
|
|
1053
|
-
"DiS.",
|
|
1054
|
-
"ACCA",
|
|
1055
|
-
"FCCA",
|
|
1056
|
-
"CIPM",
|
|
1057
|
-
"CIPT",
|
|
1058
|
-
"CIPP/E",
|
|
1059
|
-
"CIPP"
|
|
1060
|
-
];
|
|
269
|
+
const LEGAL_SUFFIXES = [...[
|
|
270
|
+
"spol. s r.o.",
|
|
271
|
+
"s.r.o.",
|
|
272
|
+
"s. r. o.",
|
|
273
|
+
"a.s.",
|
|
274
|
+
"a. s.",
|
|
275
|
+
"v.o.s.",
|
|
276
|
+
"v. o. s.",
|
|
277
|
+
"k.s.",
|
|
278
|
+
"k. s.",
|
|
279
|
+
"z.s.",
|
|
280
|
+
"z. s.",
|
|
281
|
+
"z.ú.",
|
|
282
|
+
"z. ú.",
|
|
283
|
+
"o.p.s.",
|
|
284
|
+
"o. p. s.",
|
|
285
|
+
"s.p.",
|
|
286
|
+
"s. p.",
|
|
287
|
+
"GmbH",
|
|
288
|
+
"AG",
|
|
289
|
+
"SE",
|
|
290
|
+
"KG",
|
|
291
|
+
"OHG",
|
|
292
|
+
"Ltd.",
|
|
293
|
+
"Ltd",
|
|
294
|
+
"LLC",
|
|
295
|
+
"LLP",
|
|
296
|
+
"Inc.",
|
|
297
|
+
"Corp.",
|
|
298
|
+
"Corporation",
|
|
299
|
+
"Co.",
|
|
300
|
+
"LP",
|
|
301
|
+
"L.P.",
|
|
302
|
+
"PLC",
|
|
303
|
+
"plc",
|
|
304
|
+
"N.A.",
|
|
305
|
+
"N.V.",
|
|
306
|
+
"B.V.",
|
|
307
|
+
"Pty Ltd.",
|
|
308
|
+
"Pty Ltd",
|
|
309
|
+
"S.A.",
|
|
310
|
+
"SA",
|
|
311
|
+
"SAS",
|
|
312
|
+
"SARL",
|
|
313
|
+
"S.p.A.",
|
|
314
|
+
"Sp. z o.o.",
|
|
315
|
+
"Sp. k.",
|
|
316
|
+
"Sp. j.",
|
|
317
|
+
"Ltda.",
|
|
318
|
+
"LTDA.",
|
|
319
|
+
"Ltda",
|
|
320
|
+
"LTDA",
|
|
321
|
+
"S/A",
|
|
322
|
+
"EIRELI",
|
|
323
|
+
"EPP",
|
|
324
|
+
"ME",
|
|
325
|
+
"MEI"
|
|
326
|
+
]].sort((a, b) => b.length - a.length);
|
|
1061
327
|
//#endregion
|
|
1062
328
|
//#region src/data/char-groups.json
|
|
1063
329
|
var char_groups_default = {
|
|
@@ -1432,220 +698,1501 @@ var char_groups_default = {
|
|
|
1432
698
|
"code": "U+060C"
|
|
1433
699
|
},
|
|
1434
700
|
{
|
|
1435
|
-
"char": "、",
|
|
1436
|
-
"name": "ideographic comma",
|
|
1437
|
-
"code": "U+3001"
|
|
701
|
+
"char": "、",
|
|
702
|
+
"name": "ideographic comma",
|
|
703
|
+
"code": "U+3001"
|
|
704
|
+
},
|
|
705
|
+
{
|
|
706
|
+
"char": ",",
|
|
707
|
+
"name": "full-width comma",
|
|
708
|
+
"code": "U+FF0C"
|
|
709
|
+
},
|
|
710
|
+
{
|
|
711
|
+
"char": "﹐",
|
|
712
|
+
"name": "small comma",
|
|
713
|
+
"code": "U+FE50"
|
|
714
|
+
},
|
|
715
|
+
{
|
|
716
|
+
"char": "՝",
|
|
717
|
+
"name": "Armenian comma",
|
|
718
|
+
"code": "U+055D"
|
|
719
|
+
}
|
|
720
|
+
]
|
|
721
|
+
},
|
|
722
|
+
"ellipsis": {
|
|
723
|
+
"description": "Ellipsis characters",
|
|
724
|
+
"chars": [{
|
|
725
|
+
"char": "…",
|
|
726
|
+
"name": "horizontal ellipsis",
|
|
727
|
+
"code": "U+2026"
|
|
728
|
+
}]
|
|
729
|
+
},
|
|
730
|
+
"question-mark": {
|
|
731
|
+
"description": "Question mark variants",
|
|
732
|
+
"chars": [
|
|
733
|
+
{
|
|
734
|
+
"char": "?",
|
|
735
|
+
"name": "question mark",
|
|
736
|
+
"code": "U+003F"
|
|
737
|
+
},
|
|
738
|
+
{
|
|
739
|
+
"char": "?",
|
|
740
|
+
"name": "full-width question mark",
|
|
741
|
+
"code": "U+FF1F"
|
|
742
|
+
},
|
|
743
|
+
{
|
|
744
|
+
"char": "⁇",
|
|
745
|
+
"name": "double question mark",
|
|
746
|
+
"code": "U+2047"
|
|
747
|
+
},
|
|
748
|
+
{
|
|
749
|
+
"char": "⁈",
|
|
750
|
+
"name": "question exclamation mark",
|
|
751
|
+
"code": "U+2048"
|
|
752
|
+
},
|
|
753
|
+
{
|
|
754
|
+
"char": "؟",
|
|
755
|
+
"name": "Arabic question mark",
|
|
756
|
+
"code": "U+061F"
|
|
757
|
+
},
|
|
758
|
+
{
|
|
759
|
+
"char": "‽",
|
|
760
|
+
"name": "interrobang",
|
|
761
|
+
"code": "U+203D"
|
|
762
|
+
}
|
|
763
|
+
]
|
|
764
|
+
},
|
|
765
|
+
"exclamation-mark": {
|
|
766
|
+
"description": "Exclamation mark variants",
|
|
767
|
+
"chars": [
|
|
768
|
+
{
|
|
769
|
+
"char": "!",
|
|
770
|
+
"name": "exclamation mark",
|
|
771
|
+
"code": "U+0021"
|
|
772
|
+
},
|
|
773
|
+
{
|
|
774
|
+
"char": "!",
|
|
775
|
+
"name": "full-width exclamation mark",
|
|
776
|
+
"code": "U+FF01"
|
|
777
|
+
},
|
|
778
|
+
{
|
|
779
|
+
"char": "‼",
|
|
780
|
+
"name": "double exclamation mark",
|
|
781
|
+
"code": "U+203C"
|
|
782
|
+
},
|
|
783
|
+
{
|
|
784
|
+
"char": "¡",
|
|
785
|
+
"name": "inverted exclamation mark",
|
|
786
|
+
"code": "U+00A1"
|
|
787
|
+
}
|
|
788
|
+
]
|
|
789
|
+
},
|
|
790
|
+
"semicolon": {
|
|
791
|
+
"description": "Semicolon variants",
|
|
792
|
+
"chars": [
|
|
793
|
+
{
|
|
794
|
+
"char": ";",
|
|
795
|
+
"name": "semicolon",
|
|
796
|
+
"code": "U+003B"
|
|
1438
797
|
},
|
|
1439
798
|
{
|
|
1440
|
-
"char": "
|
|
1441
|
-
"name": "
|
|
1442
|
-
"code": "U+
|
|
799
|
+
"char": "؛",
|
|
800
|
+
"name": "Arabic semicolon",
|
|
801
|
+
"code": "U+061B"
|
|
1443
802
|
},
|
|
1444
803
|
{
|
|
1445
|
-
"char": "
|
|
1446
|
-
"name": "
|
|
1447
|
-
"code": "U+
|
|
804
|
+
"char": ";",
|
|
805
|
+
"name": "full-width semicolon",
|
|
806
|
+
"code": "U+FF1B"
|
|
1448
807
|
},
|
|
1449
808
|
{
|
|
1450
|
-
"char": "
|
|
1451
|
-
"name": "
|
|
1452
|
-
"code": "U+
|
|
809
|
+
"char": "፤",
|
|
810
|
+
"name": "Ethiopic semicolon",
|
|
811
|
+
"code": "U+1364"
|
|
812
|
+
}
|
|
813
|
+
]
|
|
814
|
+
},
|
|
815
|
+
"underscore": {
|
|
816
|
+
"description": "Underscore variants",
|
|
817
|
+
"chars": [{
|
|
818
|
+
"char": "_",
|
|
819
|
+
"name": "low line",
|
|
820
|
+
"code": "U+005F"
|
|
821
|
+
}, {
|
|
822
|
+
"char": "_",
|
|
823
|
+
"name": "full-width low line",
|
|
824
|
+
"code": "U+FF3F"
|
|
825
|
+
}]
|
|
826
|
+
}
|
|
827
|
+
}
|
|
828
|
+
};
|
|
829
|
+
//#endregion
|
|
830
|
+
//#region src/util/char-groups.ts
|
|
831
|
+
/**
|
|
832
|
+
* Centralized Unicode character equivalence groups.
|
|
833
|
+
*
|
|
834
|
+
* Centralized Unicode character equivalence groups.
|
|
835
|
+
*
|
|
836
|
+
* Provides helpers to build regex character classes that
|
|
837
|
+
* match all typographic variants of a character type
|
|
838
|
+
* (dashes, spaces, quotes, etc.).
|
|
839
|
+
*
|
|
840
|
+
* The JSON is statically imported so the bundler inlines
|
|
841
|
+
* it, avoiding a runtime require() that breaks browsers.
|
|
842
|
+
*/
|
|
843
|
+
/** Chars that need escaping inside a regex char class. */
|
|
844
|
+
const REGEX_CLASS_SPECIAL = /[\\\]^-]/;
|
|
845
|
+
const escapeForCharClass = (ch) => REGEX_CLASS_SPECIAL.test(ch) ? `\\${ch}` : ch;
|
|
846
|
+
const config = char_groups_default;
|
|
847
|
+
/**
|
|
848
|
+
* Get the raw characters for a named group.
|
|
849
|
+
* Throws if the group does not exist.
|
|
850
|
+
*/
|
|
851
|
+
const charSet = (group) => {
|
|
852
|
+
const g = config.groups[group];
|
|
853
|
+
if (!g) throw new Error(`Unknown char group: "${group}"`);
|
|
854
|
+
return g.chars.map((entry) => entry.char);
|
|
855
|
+
};
|
|
856
|
+
/**
|
|
857
|
+
* Build a regex character class string for a named
|
|
858
|
+
* group. E.g., charClass("dash") returns a string
|
|
859
|
+
* like "[-\u2013\u2014\u2010\u2011\u2212\u2043]".
|
|
860
|
+
*
|
|
861
|
+
* The hyphen-minus is placed first so it is treated
|
|
862
|
+
* as a literal, not a range indicator.
|
|
863
|
+
*/
|
|
864
|
+
const charClass = (group) => {
|
|
865
|
+
return `[${[...charSet(group)].sort((a, b) => {
|
|
866
|
+
if (a === "-") return -1;
|
|
867
|
+
if (b === "-") return 1;
|
|
868
|
+
return a.localeCompare(b);
|
|
869
|
+
}).map(escapeForCharClass).join("")}]`;
|
|
870
|
+
};
|
|
871
|
+
/**
|
|
872
|
+
* Return the inner content of a character class (without
|
|
873
|
+
* the surrounding brackets). Useful for embedding a group
|
|
874
|
+
* inside a larger character class, e.g.:
|
|
875
|
+
* `[\\s&,.${charClassInner("dash")}]`
|
|
876
|
+
*/
|
|
877
|
+
const charClassInner = (group) => {
|
|
878
|
+
return [...charSet(group)].sort((a, b) => {
|
|
879
|
+
if (a === "-") return -1;
|
|
880
|
+
if (b === "-") return 1;
|
|
881
|
+
return a.localeCompare(b);
|
|
882
|
+
}).map(escapeForCharClass).join("");
|
|
883
|
+
};
|
|
884
|
+
/** Regex char class matching all dash variants. */
|
|
885
|
+
const DASH = charClass("dash");
|
|
886
|
+
/**
|
|
887
|
+
* Inner content of the dash char class (no brackets).
|
|
888
|
+
* For embedding inside a larger character class:
|
|
889
|
+
* `[\\s&,.${DASH_INNER}]`
|
|
890
|
+
*/
|
|
891
|
+
const DASH_INNER = charClassInner("dash");
|
|
892
|
+
charClass("space");
|
|
893
|
+
charClass("quote-double");
|
|
894
|
+
/** Inner content of the double-quote char class (no brackets). */
|
|
895
|
+
const QUOTE_DOUBLE_INNER = charClassInner("quote-double");
|
|
896
|
+
charClass("quote-single");
|
|
897
|
+
/** Inner content of the single-quote char class (no brackets). */
|
|
898
|
+
const QUOTE_SINGLE_INNER = charClassInner("quote-single");
|
|
899
|
+
/**
|
|
900
|
+
* Inline opening-bracket characters that frequently introduce
|
|
901
|
+
* a defined-term parenthetical immediately after an address or
|
|
902
|
+
* organization span (e.g. `…GA 30326, USA (the "Premises")`).
|
|
903
|
+
* Hardcoded rather than data-driven because the set is tiny,
|
|
904
|
+
* regex-special, and not language-dependent.
|
|
905
|
+
*/
|
|
906
|
+
const OPENING_BRACKETS_INNER = "(\\[{";
|
|
907
|
+
charClass("slash");
|
|
908
|
+
charClass("dot");
|
|
909
|
+
charClass("colon");
|
|
910
|
+
//#endregion
|
|
911
|
+
//#region src/detectors/legal-forms.ts
|
|
912
|
+
const SENTENCE_VERB_INDICATORS_SEED = new Set([
|
|
913
|
+
"je",
|
|
914
|
+
"jsou",
|
|
915
|
+
"is",
|
|
916
|
+
"are",
|
|
917
|
+
"ist",
|
|
918
|
+
"sind"
|
|
919
|
+
]);
|
|
920
|
+
let sentenceVerbIndicatorsCache = null;
|
|
921
|
+
let sentenceVerbIndicatorsPromise = null;
|
|
922
|
+
const loadSentenceVerbIndicators = async () => {
|
|
923
|
+
if (sentenceVerbIndicatorsCache) return sentenceVerbIndicatorsCache;
|
|
924
|
+
if (sentenceVerbIndicatorsPromise) return sentenceVerbIndicatorsPromise;
|
|
925
|
+
sentenceVerbIndicatorsPromise = (async () => {
|
|
926
|
+
let data = {};
|
|
927
|
+
try {
|
|
928
|
+
const mod = await import("./sentence-verb-indicators.mjs");
|
|
929
|
+
data = mod.default ?? mod;
|
|
930
|
+
} catch (err) {
|
|
931
|
+
console.warn("[anonymize] legal-forms: failed to load sentence-verb-indicators.json, falling back to seed list:", err);
|
|
932
|
+
}
|
|
933
|
+
const all = new Set(SENTENCE_VERB_INDICATORS_SEED);
|
|
934
|
+
for (const [key, value] of Object.entries(data)) {
|
|
935
|
+
if (key.startsWith("_")) continue;
|
|
936
|
+
if (!Array.isArray(value)) continue;
|
|
937
|
+
for (const verb of value) {
|
|
938
|
+
if (typeof verb !== "string" || verb.length === 0) continue;
|
|
939
|
+
all.add(verb.toLowerCase());
|
|
940
|
+
}
|
|
941
|
+
}
|
|
942
|
+
sentenceVerbIndicatorsCache = all;
|
|
943
|
+
return all;
|
|
944
|
+
})();
|
|
945
|
+
return sentenceVerbIndicatorsPromise;
|
|
946
|
+
};
|
|
947
|
+
const getSentenceVerbIndicatorsSync = () => sentenceVerbIndicatorsCache ?? SENTENCE_VERB_INDICATORS_SEED;
|
|
948
|
+
const UPPER = "A-ZÁČĎÉĚÍŇÓŘŠŤÚŮÝŽÄÖÜÀÂÆÇÈÊËÎÏÔÙÛŸÑĄĆĘŁŃŚŹŻ\\u0130";
|
|
949
|
+
const LOWER = "a-záčďéěíňóřšťúůýžäöüßàâæçèêëîïôùûÿñąćęłńśźż\\u0131";
|
|
950
|
+
const CAP_WORD = `(?:[${UPPER}]{2,}|[${UPPER}][${LOWER}${UPPER}]+)`;
|
|
951
|
+
const ALLCAP_WORD = `[${UPPER}]{2,}`;
|
|
952
|
+
const ROMAN_NUMERAL_RE = /^(?=[IVXLCDM])M{0,3}(?:CM|CD|D?C{0,3})(?:XC|XL|L?X{0,3})(?:IX|IV|V?I{0,3})$/;
|
|
953
|
+
let legalRoleHeadsCache = null;
|
|
954
|
+
let legalRoleHeadsPromise = null;
|
|
955
|
+
const loadLegalRoleHeads = async () => {
|
|
956
|
+
if (legalRoleHeadsCache) return legalRoleHeadsCache;
|
|
957
|
+
if (legalRoleHeadsPromise) return legalRoleHeadsPromise;
|
|
958
|
+
legalRoleHeadsPromise = (async () => {
|
|
959
|
+
const sets = await loadLanguageConfigs("legalRoleHeads", (mod) => {
|
|
960
|
+
return mod.default ?? mod;
|
|
961
|
+
});
|
|
962
|
+
const all = /* @__PURE__ */ new Set();
|
|
963
|
+
for (const entry of sets) {
|
|
964
|
+
if (!entry || !Array.isArray(entry.words)) continue;
|
|
965
|
+
for (const word of entry.words) if (typeof word === "string" && word.length > 0) all.add(word.toLowerCase());
|
|
966
|
+
}
|
|
967
|
+
legalRoleHeadsCache = all;
|
|
968
|
+
return all;
|
|
969
|
+
})();
|
|
970
|
+
return legalRoleHeadsPromise;
|
|
971
|
+
};
|
|
972
|
+
const getLegalRoleHeadsSync = () => legalRoleHeadsCache ?? /* @__PURE__ */ new Set();
|
|
973
|
+
const warmLegalRoleHeads = async () => {
|
|
974
|
+
await Promise.all([
|
|
975
|
+
loadLegalRoleHeads(),
|
|
976
|
+
loadAllLegalSuffixes(),
|
|
977
|
+
loadSentenceVerbIndicators(),
|
|
978
|
+
loadClauseNounHeads()
|
|
979
|
+
]);
|
|
980
|
+
};
|
|
981
|
+
let allLegalSuffixesCache = null;
|
|
982
|
+
let allLegalSuffixesPromise = null;
|
|
983
|
+
const loadAllLegalSuffixes = async () => {
|
|
984
|
+
if (allLegalSuffixesCache) return allLegalSuffixesCache;
|
|
985
|
+
if (allLegalSuffixesPromise) return allLegalSuffixesPromise;
|
|
986
|
+
allLegalSuffixesPromise = (async () => {
|
|
987
|
+
let data = {};
|
|
988
|
+
try {
|
|
989
|
+
data = (await import("./legal-forms.mjs")).default;
|
|
990
|
+
} catch {
|
|
991
|
+
data = {};
|
|
992
|
+
}
|
|
993
|
+
const seen = /* @__PURE__ */ new Set();
|
|
994
|
+
const out = [];
|
|
995
|
+
for (const list of Object.values(data)) for (const form of list) {
|
|
996
|
+
if (typeof form !== "string" || form.length === 0) continue;
|
|
997
|
+
if (seen.has(form)) continue;
|
|
998
|
+
seen.add(form);
|
|
999
|
+
out.push(form);
|
|
1000
|
+
}
|
|
1001
|
+
for (const form of LEGAL_SUFFIXES) if (!seen.has(form)) {
|
|
1002
|
+
seen.add(form);
|
|
1003
|
+
out.push(form);
|
|
1004
|
+
}
|
|
1005
|
+
out.sort((a, b) => b.length - a.length);
|
|
1006
|
+
allLegalSuffixesCache = out;
|
|
1007
|
+
return out;
|
|
1008
|
+
})();
|
|
1009
|
+
return allLegalSuffixesPromise;
|
|
1010
|
+
};
|
|
1011
|
+
const getAllLegalSuffixesSync = () => allLegalSuffixesCache ?? LEGAL_SUFFIXES;
|
|
1012
|
+
/**
|
|
1013
|
+
* Sync accessor for the full legal-form vocabulary
|
|
1014
|
+
* (`data/legal-forms.json` plus `LEGAL_SUFFIXES`,
|
|
1015
|
+
* longest-first). Falls back to `LEGAL_SUFFIXES` when
|
|
1016
|
+
* `warmLegalRoleHeads()` has not run yet. Exposed so the
|
|
1017
|
+
* trailing-period strip in `sanitizeEntities` can keep
|
|
1018
|
+
* pace with the detector vocabulary rather than only the
|
|
1019
|
+
* smaller `LEGAL_SUFFIXES` propagation list.
|
|
1020
|
+
*/
|
|
1021
|
+
const getKnownLegalSuffixes = getAllLegalSuffixesSync;
|
|
1022
|
+
const CLAUSE_NOUN_HEADS_SEED = new Set(["agreement", "contract"]);
|
|
1023
|
+
let clauseNounHeadsCache = null;
|
|
1024
|
+
let clauseNounHeadsPromise = null;
|
|
1025
|
+
const loadClauseNounHeads = async () => {
|
|
1026
|
+
if (clauseNounHeadsCache) return clauseNounHeadsCache;
|
|
1027
|
+
if (clauseNounHeadsPromise) return clauseNounHeadsPromise;
|
|
1028
|
+
clauseNounHeadsPromise = (async () => {
|
|
1029
|
+
let data = {};
|
|
1030
|
+
try {
|
|
1031
|
+
const mod = await import("./clause-noun-heads.mjs");
|
|
1032
|
+
data = mod.default ?? mod;
|
|
1033
|
+
} catch (err) {
|
|
1034
|
+
console.warn("[anonymize] legal-forms: failed to load clause-noun-heads.json, falling back to seed list:", err);
|
|
1035
|
+
}
|
|
1036
|
+
const all = new Set(CLAUSE_NOUN_HEADS_SEED);
|
|
1037
|
+
for (const [key, value] of Object.entries(data)) {
|
|
1038
|
+
if (key.startsWith("_")) continue;
|
|
1039
|
+
if (!Array.isArray(value)) continue;
|
|
1040
|
+
for (const noun of value) {
|
|
1041
|
+
if (typeof noun !== "string" || noun.length === 0) continue;
|
|
1042
|
+
all.add(noun.toLowerCase());
|
|
1043
|
+
}
|
|
1044
|
+
}
|
|
1045
|
+
clauseNounHeadsCache = all;
|
|
1046
|
+
return all;
|
|
1047
|
+
})();
|
|
1048
|
+
return clauseNounHeadsPromise;
|
|
1049
|
+
};
|
|
1050
|
+
const getClauseNounHeadsSync = () => clauseNounHeadsCache ?? CLAUSE_NOUN_HEADS_SEED;
|
|
1051
|
+
const escapeForRegex = (form) => form.replace(/[.*+?^${}()|[\]\\]/g, "\\$&").replace(/\s+/g, "\\s+").replace(/\\\./g, "\\.[^\\S\\n]?");
|
|
1052
|
+
const isShortForm = (form) => form.replace(/[.\s]/g, "").length <= 3 && !form.includes(" ");
|
|
1053
|
+
const buildPatternString = (forms) => {
|
|
1054
|
+
if (forms.length === 0) return null;
|
|
1055
|
+
const alt = forms.toSorted((a, b) => b.length - a.length).map(escapeForRegex).join("|");
|
|
1056
|
+
const HSPACE = "[^\\S\\n]";
|
|
1057
|
+
const LOWER_CONNECTOR = `${HSPACE}+(?:a|and|und|et|e|y|i)${HSPACE}+(?=[${LOWER}])`;
|
|
1058
|
+
const SIMPLE_SEP = `(?:${HSPACE}|[&,.${DASH_INNER}]){1,4}`;
|
|
1059
|
+
const CAP_OR_NUM_WORD = `(?:${CAP_WORD}|\\d{1,4})`;
|
|
1060
|
+
const LOWER_WORD = `(?:(?!(?:and|und|et)(?![${UPPER}${LOWER}]))[${LOWER}][${LOWER}${UPPER}]+)`;
|
|
1061
|
+
const ANY_WORD_TAIL = `(?:(?!(?:and|und|et)(?![${UPPER}${LOWER}]))[${UPPER}${LOWER}][${LOWER}${UPPER}]+|[${UPPER}]{2,3}|\\d{1,4})`;
|
|
1062
|
+
return `${`(?:${`(?:${CAP_WORD})(?:${SIMPLE_SEP}(?:${CAP_OR_NUM_WORD})){0,10}`})(?:${`${SIMPLE_SEP}(?:${LOWER_WORD})(?:(?:${LOWER_CONNECTOR}|${SIMPLE_SEP})(?:${ANY_WORD_TAIL})){0,10}`})?`}(?:\\s+|,\\s*)(?:${alt})(?![${LOWER}])`;
|
|
1063
|
+
};
|
|
1064
|
+
/**
|
|
1065
|
+
* Build legal form regex pattern strings.
|
|
1066
|
+
* Returns an array of regex strings for the unified
|
|
1067
|
+
* TextSearch builder. Empty if data package is not
|
|
1068
|
+
* installed.
|
|
1069
|
+
*/
|
|
1070
|
+
const buildLegalFormPatterns = async () => {
|
|
1071
|
+
let data = {};
|
|
1072
|
+
try {
|
|
1073
|
+
data = (await import("./legal-forms.mjs")).default;
|
|
1074
|
+
} catch {
|
|
1075
|
+
return [];
|
|
1076
|
+
}
|
|
1077
|
+
const allForms = [];
|
|
1078
|
+
const seen = /* @__PURE__ */ new Set();
|
|
1079
|
+
for (const forms of Object.values(data)) for (const form of forms) if (!seen.has(form)) {
|
|
1080
|
+
seen.add(form);
|
|
1081
|
+
allForms.push(form);
|
|
1082
|
+
}
|
|
1083
|
+
for (const form of LEGAL_SUFFIXES) if (!seen.has(form)) {
|
|
1084
|
+
seen.add(form);
|
|
1085
|
+
allForms.push(form);
|
|
1086
|
+
}
|
|
1087
|
+
const patterns = [];
|
|
1088
|
+
const longPattern = buildPatternString(allForms.filter((f) => !isShortForm(f)));
|
|
1089
|
+
if (longPattern) patterns.push(longPattern);
|
|
1090
|
+
const shortPattern = buildPatternString(allForms.filter(isShortForm));
|
|
1091
|
+
if (shortPattern) patterns.push(shortPattern);
|
|
1092
|
+
const allcapPrefix = `(?:${ALLCAP_WORD})(?:[ \\t&,.${DASH_INNER}]{1,4}(?:${ALLCAP_WORD})){0,2}`;
|
|
1093
|
+
const allcapAlt = allForms.toSorted((a, b) => b.length - a.length).map(escapeForRegex).join("|");
|
|
1094
|
+
patterns.push(`${allcapPrefix}(?:[ \\t]+|,[ \\t]*)(?:${allcapAlt})(?![${LOWER}])`);
|
|
1095
|
+
return patterns;
|
|
1096
|
+
};
|
|
1097
|
+
const CONNECTOR_RE = /^(?:a|and|und|et|e|y|i|&)$/i;
|
|
1098
|
+
const AND_TYPE_CONNECTOR_RE = /^(?:and|und|et)$/i;
|
|
1099
|
+
const UPPER_LETTER_RE = /^\p{Lu}/u;
|
|
1100
|
+
const COMPANY_SUFFIX_WORDS_RE = /^(?:Company|Co|Bank|Brothers|Bros|Sons|Group|Holdings|Trust|Partners|Associates|Corporation|Industries|Enterprises|Solutions|Systems|Services|Foundation|Institute)$/;
|
|
1101
|
+
const IN_NAME_PREPOSITION_RE = /^(?:of|the)$/i;
|
|
1102
|
+
const ENTITY_HEAD_WORD_RE = /^[\p{L}\p{M}&]+/u;
|
|
1103
|
+
const LEADING_CLAUSE_RE = /(?:^|\s)(?:by\s+and\s+between|is\s+between)\s+/giu;
|
|
1104
|
+
/**
|
|
1105
|
+
* Find the word ending just before `pos` in `text`,
|
|
1106
|
+
* skipping any whitespace (not newlines).
|
|
1107
|
+
* Returns null if no word is found (e.g., at start
|
|
1108
|
+
* of text, or preceded by non-word chars like ".").
|
|
1109
|
+
*/
|
|
1110
|
+
const findWordBefore = (text, pos) => {
|
|
1111
|
+
let scan = pos - 1;
|
|
1112
|
+
while (scan >= 0) {
|
|
1113
|
+
const ch = text.charAt(scan);
|
|
1114
|
+
if (ch === "\n" || !/\s/.test(ch)) break;
|
|
1115
|
+
scan--;
|
|
1116
|
+
}
|
|
1117
|
+
if (scan < 0 || text.charAt(scan) === "\n") return null;
|
|
1118
|
+
const wordEnd = scan + 1;
|
|
1119
|
+
while (scan >= 0 && /[\p{L}\p{M}&]/u.test(text.charAt(scan))) scan--;
|
|
1120
|
+
const wordStart = scan + 1;
|
|
1121
|
+
const word = text.slice(wordStart, wordEnd);
|
|
1122
|
+
if (word.length === 0) return null;
|
|
1123
|
+
return {
|
|
1124
|
+
word,
|
|
1125
|
+
start: wordStart
|
|
1126
|
+
};
|
|
1127
|
+
};
|
|
1128
|
+
/**
|
|
1129
|
+
* Count consecutive uppercase-starting words immediately
|
|
1130
|
+
* before `pos`. Stops at the first non-upper word or at
|
|
1131
|
+
* text/line start. Used to disambiguate "<First> <Last>
|
|
1132
|
+
* and <ORG>" from "<Multi-word Org> and <Continuation>".
|
|
1133
|
+
*/
|
|
1134
|
+
const countUpperWordsBefore = (fullText, pos) => {
|
|
1135
|
+
let count = 0;
|
|
1136
|
+
let scan = pos;
|
|
1137
|
+
while (scan > 0) {
|
|
1138
|
+
const found = findWordBefore(fullText, scan);
|
|
1139
|
+
if (!found) break;
|
|
1140
|
+
if (!UPPER_LETTER_RE.test(found.word)) break;
|
|
1141
|
+
count++;
|
|
1142
|
+
scan = found.start;
|
|
1143
|
+
}
|
|
1144
|
+
return count;
|
|
1145
|
+
};
|
|
1146
|
+
/**
|
|
1147
|
+
* Extend a match backward through uppercase words and
|
|
1148
|
+
* lowercase connectors. Stops at start of text,
|
|
1149
|
+
* newline, or a word that doesn't qualify.
|
|
1150
|
+
*
|
|
1151
|
+
* Connectors (a, and, und, et, ...) are only consumed
|
|
1152
|
+
* when there is a valid word before them — a trailing
|
|
1153
|
+
* connector at an entity boundary is not consumed.
|
|
1154
|
+
* For multi-char "and"-type connectors we additionally
|
|
1155
|
+
* refuse to cross when exactly two uppercase words
|
|
1156
|
+
* precede them ("First Last and ORG, Inc." shape) —
|
|
1157
|
+
* unless the match itself begins with a known company-
|
|
1158
|
+
* suffix word ("…and Company, Inc."), in which case
|
|
1159
|
+
* the chain belongs to one organisation. In that
|
|
1160
|
+
* suffix-mode we also cross in-name prepositions
|
|
1161
|
+
* ("Bank of America and Trust Company, Inc.").
|
|
1162
|
+
*/
|
|
1163
|
+
const extendBackward = (fullText, matchStart, options = {}) => {
|
|
1164
|
+
const headWord = ENTITY_HEAD_WORD_RE.exec(fullText.slice(matchStart))?.[0] ?? "";
|
|
1165
|
+
const suffixMode = options.forceSuffixMode === true || COMPANY_SUFFIX_WORDS_RE.test(headWord);
|
|
1166
|
+
let pos = matchStart;
|
|
1167
|
+
while (pos > 0) {
|
|
1168
|
+
const found = findWordBefore(fullText, pos);
|
|
1169
|
+
if (!found) break;
|
|
1170
|
+
const { word, start: wordStart } = found;
|
|
1171
|
+
const isUpper = UPPER_LETTER_RE.test(word);
|
|
1172
|
+
const isConnector = CONNECTOR_RE.test(word);
|
|
1173
|
+
const isInNamePrep = suffixMode && IN_NAME_PREPOSITION_RE.test(word);
|
|
1174
|
+
if (isUpper) pos = wordStart;
|
|
1175
|
+
else if (isConnector) {
|
|
1176
|
+
if (!suffixMode && AND_TYPE_CONNECTOR_RE.test(word) && countUpperWordsBefore(fullText, wordStart) === 2) break;
|
|
1177
|
+
const prev = findWordBefore(fullText, wordStart);
|
|
1178
|
+
if (!prev) break;
|
|
1179
|
+
if (!UPPER_LETTER_RE.test(prev.word)) break;
|
|
1180
|
+
pos = prev.start;
|
|
1181
|
+
} else if (isInNamePrep) {
|
|
1182
|
+
const prev = findWordBefore(fullText, wordStart);
|
|
1183
|
+
if (!prev) break;
|
|
1184
|
+
if (!UPPER_LETTER_RE.test(prev.word)) break;
|
|
1185
|
+
pos = prev.start;
|
|
1186
|
+
} else break;
|
|
1187
|
+
}
|
|
1188
|
+
return pos;
|
|
1189
|
+
};
|
|
1190
|
+
const trimLeadingClause = (text) => {
|
|
1191
|
+
let cut = -1;
|
|
1192
|
+
for (const match of text.matchAll(LEADING_CLAUSE_RE)) cut = match.index + match[0].length;
|
|
1193
|
+
if (cut <= 0) return {
|
|
1194
|
+
offset: 0,
|
|
1195
|
+
text
|
|
1196
|
+
};
|
|
1197
|
+
const trimmed = text.slice(cut);
|
|
1198
|
+
const leadingWs = trimmed.match(/^\s*/u)?.[0].length ?? 0;
|
|
1199
|
+
return {
|
|
1200
|
+
offset: cut + leadingWs,
|
|
1201
|
+
text: trimmed.slice(leadingWs)
|
|
1202
|
+
};
|
|
1203
|
+
};
|
|
1204
|
+
/**
|
|
1205
|
+
* Process legal form matches from the unified search.
|
|
1206
|
+
* Receives all matches; filters to the legal forms
|
|
1207
|
+
* slice via sliceStart/sliceEnd.
|
|
1208
|
+
*
|
|
1209
|
+
* The role-head trimming step reads per-language data from
|
|
1210
|
+
* a cache that `runPipeline` warms via `warmLegalRoleHeads()`
|
|
1211
|
+
* before calling this. Callers that invoke
|
|
1212
|
+
* `processLegalFormMatches` directly (without going through
|
|
1213
|
+
* `runPipeline`) must `await warmLegalRoleHeads()` first;
|
|
1214
|
+
* otherwise the trim falls back to a no-op and sentence-
|
|
1215
|
+
* fragment fixes do not apply.
|
|
1216
|
+
*/
|
|
1217
|
+
const processLegalFormMatches = (allMatches, sliceStart, sliceEnd, fullText) => {
|
|
1218
|
+
const results = [];
|
|
1219
|
+
for (const match of allMatches) {
|
|
1220
|
+
const idx = match.pattern;
|
|
1221
|
+
if (idx < sliceStart || idx >= sliceEnd) continue;
|
|
1222
|
+
const text = match.text.trimEnd();
|
|
1223
|
+
if (text.length < 5) continue;
|
|
1224
|
+
const roleHeads = getLegalRoleHeadsSync();
|
|
1225
|
+
const firstWordMatch = /^[\p{L}\p{M}]+(?:-[\p{L}\p{M}]+)*/u.exec(text);
|
|
1226
|
+
let processedStart = match.start;
|
|
1227
|
+
let processedText = text;
|
|
1228
|
+
let trimmed = false;
|
|
1229
|
+
const firstWordText = firstWordMatch?.[0] ?? "";
|
|
1230
|
+
const firstWordLeading = /^[\p{L}\p{M}]+/u.exec(firstWordText)?.[0] ?? "";
|
|
1231
|
+
if (firstWordMatch !== null && (roleHeads.has(firstWordText.toLowerCase()) || firstWordLeading.length > 0 && roleHeads.has(firstWordLeading.toLowerCase()))) {
|
|
1232
|
+
let suffixOffset = -1;
|
|
1233
|
+
for (const suffix of getAllLegalSuffixesSync()) {
|
|
1234
|
+
const idx = text.lastIndexOf(suffix);
|
|
1235
|
+
if (idx !== -1 && idx + suffix.length >= text.length - 1) {
|
|
1236
|
+
suffixOffset = idx;
|
|
1237
|
+
break;
|
|
1453
1238
|
}
|
|
1454
|
-
|
|
1455
|
-
|
|
1456
|
-
|
|
1457
|
-
|
|
1458
|
-
|
|
1459
|
-
|
|
1460
|
-
|
|
1461
|
-
|
|
1462
|
-
|
|
1463
|
-
|
|
1464
|
-
|
|
1465
|
-
|
|
1466
|
-
|
|
1467
|
-
|
|
1468
|
-
"char": "?",
|
|
1469
|
-
"name": "question mark",
|
|
1470
|
-
"code": "U+003F"
|
|
1471
|
-
},
|
|
1472
|
-
{
|
|
1473
|
-
"char": "?",
|
|
1474
|
-
"name": "full-width question mark",
|
|
1475
|
-
"code": "U+FF1F"
|
|
1476
|
-
},
|
|
1477
|
-
{
|
|
1478
|
-
"char": "⁇",
|
|
1479
|
-
"name": "double question mark",
|
|
1480
|
-
"code": "U+2047"
|
|
1481
|
-
},
|
|
1482
|
-
{
|
|
1483
|
-
"char": "⁈",
|
|
1484
|
-
"name": "question exclamation mark",
|
|
1485
|
-
"code": "U+2048"
|
|
1486
|
-
},
|
|
1487
|
-
{
|
|
1488
|
-
"char": "؟",
|
|
1489
|
-
"name": "Arabic question mark",
|
|
1490
|
-
"code": "U+061F"
|
|
1491
|
-
},
|
|
1492
|
-
{
|
|
1493
|
-
"char": "‽",
|
|
1494
|
-
"name": "interrobang",
|
|
1495
|
-
"code": "U+203D"
|
|
1239
|
+
}
|
|
1240
|
+
if (suffixOffset < 0) {} else {
|
|
1241
|
+
const midStart = firstWordMatch[0].length;
|
|
1242
|
+
const midEnd = suffixOffset;
|
|
1243
|
+
const midSection = text.slice(midStart, midEnd);
|
|
1244
|
+
const verbIndicators = getSentenceVerbIndicatorsSync();
|
|
1245
|
+
let lastVerbEndInMid = -1;
|
|
1246
|
+
for (const match of midSection.matchAll(/(?<![\p{L}\p{N}])[\p{L}\p{M}]+/gu)) if (match[0] !== void 0 && match.index !== void 0 && verbIndicators.has(match[0].toLowerCase())) lastVerbEndInMid = match.index + match[0].length;
|
|
1247
|
+
const digitAfterRole = /^\s+\d+(?:\.|\b)/u.test(midSection);
|
|
1248
|
+
let appositiveRoleHead = false;
|
|
1249
|
+
if (!digitAfterRole && lastVerbEndInMid === -1 && fullText) {
|
|
1250
|
+
const before = fullText.slice(Math.max(0, match.start - 40), match.start);
|
|
1251
|
+
const prevWord = /(?<![\p{L}\p{N}])(\p{L}[\p{L}\p{M}]*)\s*$/u.exec(before);
|
|
1252
|
+
if (prevWord !== null && getSentenceVerbIndicatorsSync().has(prevWord[1].toLowerCase())) appositiveRoleHead = true;
|
|
1496
1253
|
}
|
|
1497
|
-
|
|
1498
|
-
|
|
1499
|
-
|
|
1500
|
-
|
|
1501
|
-
|
|
1502
|
-
|
|
1503
|
-
|
|
1504
|
-
|
|
1505
|
-
|
|
1506
|
-
|
|
1507
|
-
|
|
1508
|
-
|
|
1509
|
-
|
|
1510
|
-
|
|
1511
|
-
|
|
1512
|
-
|
|
1513
|
-
|
|
1514
|
-
"name": "double exclamation mark",
|
|
1515
|
-
"code": "U+203C"
|
|
1516
|
-
},
|
|
1517
|
-
{
|
|
1518
|
-
"char": "¡",
|
|
1519
|
-
"name": "inverted exclamation mark",
|
|
1520
|
-
"code": "U+00A1"
|
|
1254
|
+
if (lastVerbEndInMid !== -1 || digitAfterRole || appositiveRoleHead) {
|
|
1255
|
+
const scanStart = lastVerbEndInMid !== -1 ? midStart + lastVerbEndInMid : midStart;
|
|
1256
|
+
const capRe = /(?<![\p{L}\p{N}])\p{Lu}[\p{L}\p{M}\p{N}]*/gu;
|
|
1257
|
+
capRe.lastIndex = scanStart;
|
|
1258
|
+
const clauseNouns = getClauseNounHeadsSync();
|
|
1259
|
+
let capMatch = null;
|
|
1260
|
+
for (let next = capRe.exec(text); next !== null; next = capRe.exec(text)) {
|
|
1261
|
+
if (next.index >= suffixOffset) break;
|
|
1262
|
+
const lc = next[0].toLowerCase();
|
|
1263
|
+
if (roleHeads.has(lc) || clauseNouns.has(lc)) continue;
|
|
1264
|
+
capMatch = next;
|
|
1265
|
+
break;
|
|
1266
|
+
}
|
|
1267
|
+
if (capMatch === null) continue;
|
|
1268
|
+
processedStart = match.start + capMatch.index;
|
|
1269
|
+
processedText = text.slice(capMatch.index);
|
|
1270
|
+
trimmed = true;
|
|
1521
1271
|
}
|
|
1522
|
-
|
|
1523
|
-
}
|
|
1524
|
-
"
|
|
1525
|
-
|
|
1526
|
-
|
|
1527
|
-
|
|
1528
|
-
|
|
1529
|
-
|
|
1530
|
-
|
|
1531
|
-
|
|
1532
|
-
|
|
1533
|
-
|
|
1534
|
-
|
|
1535
|
-
|
|
1536
|
-
|
|
1537
|
-
|
|
1538
|
-
|
|
1539
|
-
|
|
1540
|
-
|
|
1541
|
-
|
|
1542
|
-
|
|
1543
|
-
|
|
1544
|
-
|
|
1545
|
-
|
|
1272
|
+
}
|
|
1273
|
+
}
|
|
1274
|
+
if (processedText.includes("\n")) continue;
|
|
1275
|
+
let entityStart = processedStart;
|
|
1276
|
+
let entityText = processedText;
|
|
1277
|
+
if (fullText && !trimmed) {
|
|
1278
|
+
const extended = extendBackward(fullText, processedStart);
|
|
1279
|
+
if (extended < processedStart) {
|
|
1280
|
+
entityStart = extended;
|
|
1281
|
+
entityText = fullText.slice(extended, processedStart + processedText.length).trimEnd();
|
|
1282
|
+
}
|
|
1283
|
+
}
|
|
1284
|
+
const clauseTrim = trimLeadingClause(entityText);
|
|
1285
|
+
if (clauseTrim.offset > 0) {
|
|
1286
|
+
entityStart += clauseTrim.offset;
|
|
1287
|
+
entityText = clauseTrim.text;
|
|
1288
|
+
}
|
|
1289
|
+
const getPrefixInfo = (value) => {
|
|
1290
|
+
const prefixEnd = value.lastIndexOf(",") !== -1 ? value.lastIndexOf(",") : value.lastIndexOf(" ");
|
|
1291
|
+
return {
|
|
1292
|
+
prefixEnd,
|
|
1293
|
+
prefixPart: prefixEnd > 0 ? value.slice(0, prefixEnd).replace(/[^a-zA-ZÀ-ž]/g, "") : value.replace(/[^a-zA-ZÀ-ž]/g, "")
|
|
1294
|
+
};
|
|
1295
|
+
};
|
|
1296
|
+
let { prefixEnd, prefixPart } = getPrefixInfo(entityText);
|
|
1297
|
+
let isAllCapsMatch = prefixPart.length > 2 && prefixPart === prefixPart.toUpperCase();
|
|
1298
|
+
if (isAllCapsMatch && fullText) {
|
|
1299
|
+
const lineStart = fullText.lastIndexOf("\n", entityStart);
|
|
1300
|
+
const lineEnd = fullText.indexOf("\n", entityStart + entityText.length);
|
|
1301
|
+
const lineLetters = fullText.slice(lineStart + 1, lineEnd === -1 ? fullText.length : lineEnd).replace(/[^a-zA-ZÀ-ž]/g, "");
|
|
1302
|
+
const upperCount = [...lineLetters].filter((c) => c === c.toUpperCase()).length;
|
|
1303
|
+
if (lineLetters.length > 5 && upperCount / lineLetters.length >= .95) continue;
|
|
1304
|
+
if ((prefixPart.length > 0 ? entityText.slice(0, prefixEnd > 0 ? prefixEnd : entityText.length).trim().split(/\s+/).length : 0) > 3) {
|
|
1305
|
+
entityStart = match.start;
|
|
1306
|
+
entityText = text;
|
|
1307
|
+
({prefixEnd, prefixPart} = getPrefixInfo(entityText));
|
|
1308
|
+
isAllCapsMatch = prefixPart.length > 2 && prefixPart === prefixPart.toUpperCase();
|
|
1309
|
+
}
|
|
1310
|
+
} else if (isAllCapsMatch) continue;
|
|
1311
|
+
const lastSpace = entityText.lastIndexOf(" ");
|
|
1312
|
+
const rawSuffix = lastSpace !== -1 ? entityText.slice(lastSpace + 1) : "";
|
|
1313
|
+
const suffixClean = rawSuffix.replace(/[.,]/g, "");
|
|
1314
|
+
if (suffixClean.length > 0 && ROMAN_NUMERAL_RE.test(suffixClean)) continue;
|
|
1315
|
+
if (suffixClean.length <= 2 && !/\./.test(rawSuffix) && /[^\x00-\x7F]/.test(entityText.slice(0, lastSpace !== -1 ? lastSpace : entityText.length))) continue;
|
|
1316
|
+
results.push({
|
|
1317
|
+
start: entityStart,
|
|
1318
|
+
end: entityStart + entityText.length,
|
|
1319
|
+
label: "organization",
|
|
1320
|
+
text: entityText,
|
|
1321
|
+
score: .95,
|
|
1322
|
+
source: DETECTION_SOURCES.LEGAL_FORM
|
|
1323
|
+
});
|
|
1324
|
+
}
|
|
1325
|
+
return results;
|
|
1326
|
+
};
|
|
1327
|
+
//#endregion
|
|
1328
|
+
//#region src/detectors/coreference.ts
|
|
1329
|
+
/**
|
|
1330
|
+
* Case-insensitive set of every known legal-form suffix
|
|
1331
|
+
* with whitespace normalized away. Used by the
|
|
1332
|
+
* coreference scan to reject parenthetical define-term
|
|
1333
|
+
* captures whose alias is only a legal-form
|
|
1334
|
+
* abbreviation (e.g. "(« SAS »)" after "ACME SAS"),
|
|
1335
|
+
* which would otherwise propagate generic suffix
|
|
1336
|
+
* mentions as document-wide org aliases.
|
|
1337
|
+
*/
|
|
1338
|
+
const isLegalFormAlias = (alias) => {
|
|
1339
|
+
const normalized = alias.replace(/\s+/g, "").toLowerCase();
|
|
1340
|
+
if (normalized.length === 0) return false;
|
|
1341
|
+
for (const suffix of getKnownLegalSuffixes()) if (suffix.replace(/\s+/g, "").toLowerCase() === normalized) return true;
|
|
1342
|
+
return false;
|
|
1343
|
+
};
|
|
1344
|
+
/**
|
|
1345
|
+
* Load coreference definition patterns from per-language
|
|
1346
|
+
* JSON configs in src/data/. Uses the
|
|
1347
|
+
* language manifest for auto-discovery.
|
|
1348
|
+
*/
|
|
1349
|
+
const loadDefinitionPatterns = async () => {
|
|
1350
|
+
const patterns = [];
|
|
1351
|
+
const allRows = await loadLanguageConfigs("coreference", (mod) => {
|
|
1352
|
+
return mod.default ?? mod;
|
|
1353
|
+
});
|
|
1354
|
+
for (const rows of allRows) {
|
|
1355
|
+
if (!Array.isArray(rows)) {
|
|
1356
|
+
console.warn("[anonymize] coreference: unexpected config shape, skipping");
|
|
1357
|
+
continue;
|
|
1358
|
+
}
|
|
1359
|
+
for (const row of rows) try {
|
|
1360
|
+
patterns.push({ pattern: new RegExp(row.pattern, row.flags) });
|
|
1361
|
+
} catch (err) {
|
|
1362
|
+
console.warn(`[anonymize] coreference: invalid regex "${row.pattern}":`, err);
|
|
1363
|
+
}
|
|
1364
|
+
}
|
|
1365
|
+
return patterns;
|
|
1366
|
+
};
|
|
1367
|
+
/**
|
|
1368
|
+
* Load generic role terms that should NOT be treated
|
|
1369
|
+
* as PII coreferences. "Prodávající" (Seller),
|
|
1370
|
+
* "Kupující" (Buyer), etc. are legal roles, not
|
|
1371
|
+
* identifying information.
|
|
1372
|
+
*/
|
|
1373
|
+
const getRoleStopSet = async (ctx) => {
|
|
1374
|
+
if (ctx.roleStopSet) return ctx.roleStopSet;
|
|
1375
|
+
if (ctx.roleStopSetPromise) return ctx.roleStopSetPromise;
|
|
1376
|
+
const promise = (async () => {
|
|
1377
|
+
let result;
|
|
1378
|
+
try {
|
|
1379
|
+
const mod = await import("./generic-roles.mjs");
|
|
1380
|
+
const data = mod.default ?? mod;
|
|
1381
|
+
result = new Set(data.roles.map((r) => r.toLowerCase()));
|
|
1382
|
+
} catch {
|
|
1383
|
+
result = /* @__PURE__ */ new Set();
|
|
1384
|
+
}
|
|
1385
|
+
ctx.roleStopSet = result;
|
|
1386
|
+
return result;
|
|
1387
|
+
})();
|
|
1388
|
+
ctx.roleStopSetPromise = promise;
|
|
1389
|
+
return promise;
|
|
1390
|
+
};
|
|
1391
|
+
const getDefinitionPatterns = async (ctx) => {
|
|
1392
|
+
if (ctx.corefPatterns) return ctx.corefPatterns;
|
|
1393
|
+
if (ctx.corefPatternsPromise) return ctx.corefPatternsPromise;
|
|
1394
|
+
ctx.corefPatternsPromise = loadDefinitionPatterns();
|
|
1395
|
+
const patterns = await ctx.corefPatternsPromise;
|
|
1396
|
+
if (patterns.length === 0) {
|
|
1397
|
+
ctx.corefPatterns = patterns;
|
|
1398
|
+
if (!ctx.corefLoadAttempted) {
|
|
1399
|
+
ctx.corefLoadAttempted = true;
|
|
1400
|
+
console.warn("[anonymize] coreference: no definition patterns loaded; coreference detection will be inactive");
|
|
1401
|
+
}
|
|
1402
|
+
return patterns;
|
|
1403
|
+
}
|
|
1404
|
+
ctx.corefPatterns = patterns;
|
|
1405
|
+
return patterns;
|
|
1406
|
+
};
|
|
1407
|
+
const SEARCH_WINDOW = 200;
|
|
1408
|
+
/**
|
|
1409
|
+
* Check whether an alias has textual similarity to
|
|
1410
|
+
* the source entity. Prevents roles and structural
|
|
1411
|
+
* terms from being treated as name aliases.
|
|
1412
|
+
*
|
|
1413
|
+
* Three checks (any passes → similar):
|
|
1414
|
+
* 1. Word overlap: a word in the alias appears in the
|
|
1415
|
+
* entity (case-insensitive, min 2 chars)
|
|
1416
|
+
* 2. Initials: alias letters match first letters of
|
|
1417
|
+
* entity words ("TB" ↔ "Tomas Bata")
|
|
1418
|
+
* 3. Substring: alias is a substring of the entity
|
|
1419
|
+
* or vice versa (min 3 chars)
|
|
1420
|
+
*/
|
|
1421
|
+
const hasEntitySimilarity = (alias, entityText) => {
|
|
1422
|
+
const aliasLower = alias.toLowerCase();
|
|
1423
|
+
const entityLower = entityText.toLowerCase();
|
|
1424
|
+
if (aliasLower.length >= 3 && entityLower.includes(aliasLower)) return true;
|
|
1425
|
+
if (entityLower.length >= 3 && aliasLower.includes(entityLower)) return true;
|
|
1426
|
+
const aliasWords = aliasLower.split(/[\s.,;:'"()/-]+/).filter((w) => w.length >= 2);
|
|
1427
|
+
const entityWords = entityLower.split(/[\s.,;:'"()/-]+/).filter((w) => w.length >= 2);
|
|
1428
|
+
const entityWordSet = new Set(entityWords);
|
|
1429
|
+
for (const word of aliasWords) if (entityWordSet.has(word)) return true;
|
|
1430
|
+
if (/^[\p{Lu}]+$/u.test(alias) && alias.length >= 2 && alias.length <= entityWords.length) {
|
|
1431
|
+
for (let start = 0; start <= entityWords.length - alias.length; start++) if (entityWords.slice(start, start + alias.length).map((w) => w.charAt(0)).join("") === aliasLower) return true;
|
|
1432
|
+
}
|
|
1433
|
+
return false;
|
|
1434
|
+
};
|
|
1435
|
+
/**
|
|
1436
|
+
* Scan for defined-term patterns near known entities.
|
|
1437
|
+
*
|
|
1438
|
+
* Legal documents universally follow:
|
|
1439
|
+
* "Dr. Heinrich Muller (hereinafter 'the Seller')..."
|
|
1440
|
+
*
|
|
1441
|
+
* After NER detects the entity, this function scans for
|
|
1442
|
+
* definitional patterns within +/-200 chars and extracts
|
|
1443
|
+
* the alias. Returns alias + label pairs that can be added
|
|
1444
|
+
* to the gazetteer for a full-text re-scan.
|
|
1445
|
+
*/
|
|
1446
|
+
/**
|
|
1447
|
+
* Labels that can be the source of a coreference alias.
|
|
1448
|
+
* Only parties (person, organization) have defined-term
|
|
1449
|
+
* aliases in legal text. Dates, addresses, IDs do not.
|
|
1450
|
+
*/
|
|
1451
|
+
const COREF_SOURCE_LABELS = new Set(["person", "organization"]);
|
|
1452
|
+
const extractDefinedTerms = async (fullText, entities, ctx = defaultContext) => {
|
|
1453
|
+
const [definitionPatterns, roleStops] = await Promise.all([getDefinitionPatterns(ctx), getRoleStopSet(ctx)]);
|
|
1454
|
+
const terms = [];
|
|
1455
|
+
const seen = /* @__PURE__ */ new Set();
|
|
1456
|
+
const sorted = [...entities].sort((a, b) => a.start - b.start);
|
|
1457
|
+
for (const { pattern } of definitionPatterns) {
|
|
1458
|
+
pattern.lastIndex = 0;
|
|
1459
|
+
for (let match = pattern.exec(fullText); match !== null; match = pattern.exec(fullText)) {
|
|
1460
|
+
const alias = match[1]?.trim();
|
|
1461
|
+
if (!alias || alias.length < 2) continue;
|
|
1462
|
+
if (roleStops.has(alias.toLowerCase())) continue;
|
|
1463
|
+
if (isLegalFormAlias(alias)) continue;
|
|
1464
|
+
const defPos = match.index;
|
|
1465
|
+
let bestEntity = null;
|
|
1466
|
+
for (let i = sorted.length - 1; i >= 0; i--) {
|
|
1467
|
+
const e = sorted[i];
|
|
1468
|
+
if (e === void 0) continue;
|
|
1469
|
+
if (e.end > defPos) continue;
|
|
1470
|
+
if (defPos - e.end > SEARCH_WINDOW) break;
|
|
1471
|
+
if (!COREF_SOURCE_LABELS.has(e.label)) continue;
|
|
1472
|
+
bestEntity = e;
|
|
1473
|
+
break;
|
|
1474
|
+
}
|
|
1475
|
+
if (bestEntity === null) continue;
|
|
1476
|
+
const gapText = fullText.slice(bestEntity.end, defPos);
|
|
1477
|
+
if (/(?:;|\.(?=\s*(?:["'„‚(]*\p{Lu}|$)))/u.test(gapText)) continue;
|
|
1478
|
+
if (!hasEntitySimilarity(alias, bestEntity.text)) continue;
|
|
1479
|
+
const key = `${alias.toLowerCase()}::${bestEntity.label}`;
|
|
1480
|
+
if (seen.has(key)) continue;
|
|
1481
|
+
seen.add(key);
|
|
1482
|
+
terms.push({
|
|
1483
|
+
alias,
|
|
1484
|
+
label: bestEntity.label,
|
|
1485
|
+
definitionStart: defPos,
|
|
1486
|
+
sourceText: bestEntity.text
|
|
1487
|
+
});
|
|
1488
|
+
}
|
|
1489
|
+
}
|
|
1490
|
+
return terms;
|
|
1491
|
+
};
|
|
1492
|
+
/**
|
|
1493
|
+
* Check if a character is a Unicode word character
|
|
1494
|
+
* (letter, digit, or combining mark). Used for word
|
|
1495
|
+
* boundary checks in coreference matching.
|
|
1496
|
+
*/
|
|
1497
|
+
const isWordChar = (ch) => {
|
|
1498
|
+
if (ch === void 0) return false;
|
|
1499
|
+
return /[\p{L}\p{M}\p{N}]/u.test(ch);
|
|
1500
|
+
};
|
|
1501
|
+
/**
|
|
1502
|
+
* Find all occurrences of defined-term aliases in the
|
|
1503
|
+
* full text. Returns Entity spans for each match.
|
|
1504
|
+
*
|
|
1505
|
+
* Respects word boundaries: "Kupující" must not match
|
|
1506
|
+
* inside "Kupujícímu". A match is valid only if the
|
|
1507
|
+
* character before the start and after the end are NOT
|
|
1508
|
+
* word characters (letter/digit).
|
|
1509
|
+
*
|
|
1510
|
+
* Populates `ctx.corefSourceMap` with entries linking
|
|
1511
|
+
* each coref entity to its source entity text, for
|
|
1512
|
+
* consistent placeholder numbering.
|
|
1513
|
+
*/
|
|
1514
|
+
const findCoreferenceSpans = (fullText, terms, ctx = defaultContext) => {
|
|
1515
|
+
const results = [];
|
|
1516
|
+
for (const term of terms) {
|
|
1517
|
+
let searchFrom = 0;
|
|
1518
|
+
while (searchFrom < fullText.length) {
|
|
1519
|
+
const idx = fullText.indexOf(term.alias, searchFrom);
|
|
1520
|
+
if (idx === -1) break;
|
|
1521
|
+
const matchEnd = idx + term.alias.length;
|
|
1522
|
+
const charBefore = idx > 0 ? fullText[idx - 1] : void 0;
|
|
1523
|
+
const charAfter = fullText[matchEnd];
|
|
1524
|
+
if (isWordChar(charBefore) || isWordChar(charAfter)) {
|
|
1525
|
+
searchFrom = idx + 1;
|
|
1526
|
+
continue;
|
|
1527
|
+
}
|
|
1528
|
+
const entity = {
|
|
1529
|
+
start: idx,
|
|
1530
|
+
end: matchEnd,
|
|
1531
|
+
label: term.label,
|
|
1532
|
+
text: term.alias,
|
|
1533
|
+
score: .95,
|
|
1534
|
+
source: DETECTION_SOURCES.COREFERENCE
|
|
1535
|
+
};
|
|
1536
|
+
ctx.corefSourceMap.set(corefKey(entity), term.sourceText);
|
|
1537
|
+
results.push(entity);
|
|
1538
|
+
searchFrom = matchEnd;
|
|
1539
|
+
}
|
|
1540
|
+
}
|
|
1541
|
+
return results;
|
|
1542
|
+
};
|
|
1543
|
+
//#endregion
|
|
1544
|
+
//#region src/detectors/gazetteer.ts
|
|
1545
|
+
const MAX_EDIT_DISTANCE = 2;
|
|
1546
|
+
const MIN_FUZZY_LENGTH = 4;
|
|
1547
|
+
const MAX_PREFIX_OVERSHOOT = 7;
|
|
1548
|
+
/**
|
|
1549
|
+
* Collect all searchable strings (canonical + variants)
|
|
1550
|
+
* from gazetteer entries, mapped to their labels and
|
|
1551
|
+
* entry IDs.
|
|
1552
|
+
*/
|
|
1553
|
+
const buildSearchTerms = (entries) => {
|
|
1554
|
+
const terms = /* @__PURE__ */ new Map();
|
|
1555
|
+
for (const entry of entries) {
|
|
1556
|
+
const meta = {
|
|
1557
|
+
label: entry.label,
|
|
1558
|
+
entryId: entry.id
|
|
1559
|
+
};
|
|
1560
|
+
terms.set(entry.canonical, meta);
|
|
1561
|
+
for (const variant of entry.variants) terms.set(variant, meta);
|
|
1562
|
+
}
|
|
1563
|
+
return terms;
|
|
1564
|
+
};
|
|
1565
|
+
/**
|
|
1566
|
+
* Build TextSearch-compatible patterns from gazetteer
|
|
1567
|
+
* entries. Returns:
|
|
1568
|
+
* - Exact literal patterns for all terms
|
|
1569
|
+
* - Fuzzy patterns (distance: 2) for terms >= 4 chars
|
|
1570
|
+
* - Parallel metadata arrays for post-processing
|
|
1571
|
+
*
|
|
1572
|
+
* Patterns are ordered: all exact first, then all
|
|
1573
|
+
* fuzzy. The isFuzzy array marks which are which.
|
|
1574
|
+
*/
|
|
1575
|
+
const buildGazetteerPatterns = (entries) => {
|
|
1576
|
+
const terms = buildSearchTerms(entries);
|
|
1577
|
+
const patterns = [];
|
|
1578
|
+
const labels = [];
|
|
1579
|
+
const isFuzzy = [];
|
|
1580
|
+
for (const [term, meta] of terms) {
|
|
1581
|
+
patterns.push({
|
|
1582
|
+
pattern: term,
|
|
1583
|
+
literal: true,
|
|
1584
|
+
wholeWords: false
|
|
1585
|
+
});
|
|
1586
|
+
labels.push(meta.label);
|
|
1587
|
+
isFuzzy.push(false);
|
|
1588
|
+
}
|
|
1589
|
+
for (const [term, meta] of terms) {
|
|
1590
|
+
if (term.length < MIN_FUZZY_LENGTH) continue;
|
|
1591
|
+
patterns.push({
|
|
1592
|
+
pattern: term,
|
|
1593
|
+
distance: MAX_EDIT_DISTANCE
|
|
1594
|
+
});
|
|
1595
|
+
labels.push(meta.label);
|
|
1596
|
+
isFuzzy.push(true);
|
|
1597
|
+
}
|
|
1598
|
+
return {
|
|
1599
|
+
patterns,
|
|
1600
|
+
data: {
|
|
1601
|
+
labels,
|
|
1602
|
+
isFuzzy
|
|
1603
|
+
}
|
|
1604
|
+
};
|
|
1605
|
+
};
|
|
1606
|
+
/**
|
|
1607
|
+
* Process gazetteer matches from the unified literal
|
|
1608
|
+
* search. Receives all matches; filters to the
|
|
1609
|
+
* gazetteer slice via sliceStart/sliceEnd.
|
|
1610
|
+
*
|
|
1611
|
+
* Exact matches get score 0.9; fuzzy matches get
|
|
1612
|
+
* 0.85. Fuzzy matches that overlap an exact match
|
|
1613
|
+
* are dropped.
|
|
1614
|
+
*
|
|
1615
|
+
* For exact matches, attempts prefix extension for
|
|
1616
|
+
* legal suffixes ("a.s.", "GmbH", "s.r.o." after
|
|
1617
|
+
* the matched term).
|
|
1618
|
+
*/
|
|
1619
|
+
const processGazetteerMatches = (allMatches, sliceStart, sliceEnd, fullText, data) => {
|
|
1620
|
+
const results = [];
|
|
1621
|
+
const exactSpans = [];
|
|
1622
|
+
for (const match of allMatches) {
|
|
1623
|
+
const idx = match.pattern;
|
|
1624
|
+
if (idx < sliceStart || idx >= sliceEnd) continue;
|
|
1625
|
+
const localIdx = idx - sliceStart;
|
|
1626
|
+
if (data.isFuzzy[localIdx]) continue;
|
|
1627
|
+
const label = data.labels[localIdx];
|
|
1628
|
+
if (!label) continue;
|
|
1629
|
+
const extended = tryPrefixExtension(fullText, match.start, match.end);
|
|
1630
|
+
const end = extended?.end ?? match.end;
|
|
1631
|
+
const text = extended?.text ?? fullText.slice(match.start, match.end);
|
|
1632
|
+
exactSpans.push({
|
|
1633
|
+
start: match.start,
|
|
1634
|
+
end
|
|
1635
|
+
});
|
|
1636
|
+
results.push({
|
|
1637
|
+
start: match.start,
|
|
1638
|
+
end,
|
|
1639
|
+
label,
|
|
1640
|
+
text,
|
|
1641
|
+
score: .9,
|
|
1642
|
+
source: DETECTION_SOURCES.GAZETTEER
|
|
1643
|
+
});
|
|
1644
|
+
}
|
|
1645
|
+
for (const match of allMatches) {
|
|
1646
|
+
const idx = match.pattern;
|
|
1647
|
+
if (idx < sliceStart || idx >= sliceEnd) continue;
|
|
1648
|
+
const localIdx = idx - sliceStart;
|
|
1649
|
+
if (!data.isFuzzy[localIdx]) continue;
|
|
1650
|
+
if (match.distance === 0) continue;
|
|
1651
|
+
const label = data.labels[localIdx];
|
|
1652
|
+
if (!label) continue;
|
|
1653
|
+
if (exactSpans.some((e) => match.start < e.end && match.end > e.start)) continue;
|
|
1654
|
+
const matchText = fullText.slice(match.start, match.end);
|
|
1655
|
+
results.push({
|
|
1656
|
+
start: match.start,
|
|
1657
|
+
end: match.end,
|
|
1658
|
+
label,
|
|
1659
|
+
text: matchText,
|
|
1660
|
+
score: .85,
|
|
1661
|
+
source: DETECTION_SOURCES.GAZETTEER
|
|
1662
|
+
});
|
|
1663
|
+
}
|
|
1664
|
+
return results;
|
|
1665
|
+
};
|
|
1666
|
+
/**
|
|
1667
|
+
* Try to extend an exact match to capture one
|
|
1668
|
+
* trailing token (max 6 chars) that may be a legal
|
|
1669
|
+
* entity suffix (e.g., "a.s.", "GmbH", "s.r.o.").
|
|
1670
|
+
*
|
|
1671
|
+
* Does not validate the token against a legal-forms
|
|
1672
|
+
* list; false extensions are filtered by mergeAndDedup
|
|
1673
|
+
* when a legal-form detector produces a competing
|
|
1674
|
+
* entity with the correct span.
|
|
1675
|
+
*/
|
|
1676
|
+
const tryPrefixExtension = (fullText, start, end) => {
|
|
1677
|
+
const maxEnd = Math.min(end + MAX_PREFIX_OVERSHOOT, fullText.length);
|
|
1678
|
+
if (maxEnd <= end + 1) return null;
|
|
1679
|
+
const after = fullText.slice(end, maxEnd);
|
|
1680
|
+
if (!after.startsWith(" ")) return null;
|
|
1681
|
+
const nextSpace = after.indexOf(" ", 1);
|
|
1682
|
+
const suffixEnd = nextSpace !== -1 ? nextSpace : after.length;
|
|
1683
|
+
if (suffixEnd <= 1) return null;
|
|
1684
|
+
const newEnd = end + suffixEnd;
|
|
1685
|
+
return {
|
|
1686
|
+
end: newEnd,
|
|
1687
|
+
text: fullText.slice(start, newEnd)
|
|
1688
|
+
};
|
|
1689
|
+
};
|
|
1690
|
+
//#endregion
|
|
1691
|
+
//#region src/util/text.ts
|
|
1692
|
+
/**
|
|
1693
|
+
* Shared text utilities for detectors.
|
|
1694
|
+
*
|
|
1695
|
+
* Extracted from names.ts and deny-list.ts to avoid
|
|
1696
|
+
* duplicating regex constants and helper functions.
|
|
1697
|
+
*/
|
|
1698
|
+
/** Matches a string that starts with an uppercase letter. */
|
|
1699
|
+
const UPPER_START_RE = /^\p{Lu}/u;
|
|
1700
|
+
/** Matches a string consisting entirely of uppercase letters. */
|
|
1701
|
+
const ALL_UPPER_RE = /^\p{Lu}+$/u;
|
|
1702
|
+
const SENTENCE_END_RE = /[.!?]/;
|
|
1703
|
+
/**
|
|
1704
|
+
* Detect whether a position is at the start of a sentence.
|
|
1705
|
+
* Looks backward past whitespace for sentence-ending
|
|
1706
|
+
* punctuation (.!?). Position 0 and positions preceded
|
|
1707
|
+
* only by whitespace are considered sentence starts.
|
|
1708
|
+
*/
|
|
1709
|
+
const isSentenceStart = (text, pos) => {
|
|
1710
|
+
if (pos === 0) return true;
|
|
1711
|
+
let i = pos - 1;
|
|
1712
|
+
while (i >= 0 && /\s/.test(text[i] ?? "")) i--;
|
|
1713
|
+
if (i < 0) return true;
|
|
1714
|
+
return SENTENCE_END_RE.test(text[i] ?? "");
|
|
1715
|
+
};
|
|
1716
|
+
//#endregion
|
|
1717
|
+
//#region src/detectors/names.ts
|
|
1718
|
+
const getCorpus = (ctx) => ctx.nameCorpus;
|
|
1719
|
+
const getNameCorpusFirstNames = (ctx = defaultContext) => ctx.nameCorpus?.firstNamesList ?? [];
|
|
1720
|
+
const getNameCorpusSurnames = (ctx = defaultContext) => ctx.nameCorpus?.surnamesList ?? [];
|
|
1721
|
+
const getNameCorpusTitles = (ctx = defaultContext) => ctx.nameCorpus?.titlesList ?? [];
|
|
1722
|
+
/**
|
|
1723
|
+
* Load name corpus data from injected dictionaries
|
|
1724
|
+
* and legacy config files. Merges all sources.
|
|
1725
|
+
*
|
|
1726
|
+
* Safe to call multiple times; only loads once per
|
|
1727
|
+
* context. Must be called before detectNameCorpus or
|
|
1728
|
+
* the getNameCorpus*() accessors are used.
|
|
1729
|
+
*
|
|
1730
|
+
* @param dictionaries Optional pre-loaded dictionaries
|
|
1731
|
+
* with per-language first names and surnames. When
|
|
1732
|
+
* omitted, only legacy config files are used.
|
|
1733
|
+
*/
|
|
1734
|
+
const initNameCorpus = (ctx = defaultContext, dictionaries) => {
|
|
1735
|
+
if (ctx.nameCorpusPromise) return ctx.nameCorpusPromise;
|
|
1736
|
+
const promise = (async () => {
|
|
1737
|
+
try {
|
|
1738
|
+
const [legacyFirstMod, legacySurnameMod, titleMod, exclusionMod] = await Promise.all([
|
|
1739
|
+
import("./names-first.mjs"),
|
|
1740
|
+
import("./names-surnames.mjs"),
|
|
1741
|
+
import("./names-title-tokens.mjs"),
|
|
1742
|
+
import("./names-exclusions.mjs")
|
|
1743
|
+
]);
|
|
1744
|
+
const firstNames = [...legacyFirstMod.default.names];
|
|
1745
|
+
if (dictionaries?.firstNames) for (const names of Object.values(dictionaries.firstNames)) for (const name of names) firstNames.push(name);
|
|
1746
|
+
const surnames = [...legacySurnameMod.default.names];
|
|
1747
|
+
if (dictionaries?.surnames) for (const names of Object.values(dictionaries.surnames)) for (const name of names) surnames.push(name);
|
|
1748
|
+
const dedup = (arr) => {
|
|
1749
|
+
const seen = /* @__PURE__ */ new Set();
|
|
1750
|
+
const result = [];
|
|
1751
|
+
for (const item of arr) {
|
|
1752
|
+
if (seen.has(item)) continue;
|
|
1753
|
+
seen.add(item);
|
|
1754
|
+
result.push(item);
|
|
1546
1755
|
}
|
|
1547
|
-
|
|
1548
|
-
|
|
1549
|
-
|
|
1550
|
-
|
|
1551
|
-
|
|
1552
|
-
|
|
1553
|
-
|
|
1554
|
-
|
|
1555
|
-
|
|
1556
|
-
|
|
1557
|
-
|
|
1558
|
-
|
|
1559
|
-
|
|
1756
|
+
return result;
|
|
1757
|
+
};
|
|
1758
|
+
const dedupFirst = dedup(firstNames);
|
|
1759
|
+
const dedupSurnames = dedup(surnames);
|
|
1760
|
+
const titles = titleMod.default.tokens;
|
|
1761
|
+
const exclusions = exclusionMod.default.words;
|
|
1762
|
+
ctx.nameCorpus = {
|
|
1763
|
+
firstNames: Object.freeze(new Set(dedupFirst)),
|
|
1764
|
+
surnames: Object.freeze(new Set(dedupSurnames)),
|
|
1765
|
+
titleTokens: Object.freeze(new Set(titles)),
|
|
1766
|
+
excludedWords: Object.freeze(new Set(exclusions)),
|
|
1767
|
+
firstNamesList: Object.freeze(dedupFirst),
|
|
1768
|
+
surnamesList: Object.freeze(dedupSurnames),
|
|
1769
|
+
titlesList: Object.freeze(titles),
|
|
1770
|
+
excludedList: Object.freeze(exclusions)
|
|
1771
|
+
};
|
|
1772
|
+
} catch (err) {
|
|
1773
|
+
ctx.nameCorpusPromise = null;
|
|
1774
|
+
console.warn("[anonymize] Failed to load name corpus JSON — name detection disabled:", err);
|
|
1775
|
+
}
|
|
1776
|
+
})();
|
|
1777
|
+
ctx.nameCorpusPromise = promise;
|
|
1778
|
+
return promise;
|
|
1779
|
+
};
|
|
1780
|
+
const INFLECTION_SUFFIXES = [
|
|
1781
|
+
"ovi",
|
|
1782
|
+
"em",
|
|
1783
|
+
"om",
|
|
1784
|
+
"ou",
|
|
1785
|
+
"é",
|
|
1786
|
+
"a",
|
|
1787
|
+
"u"
|
|
1788
|
+
];
|
|
1789
|
+
/**
|
|
1790
|
+
* Strip common Czech/Slovak case suffixes from a token.
|
|
1791
|
+
* Returns candidate base forms if stripping produces a
|
|
1792
|
+
* plausible name (capitalised, length >= 3).
|
|
1793
|
+
*
|
|
1794
|
+
* For the "-ou" instrumental feminine suffix, also yields
|
|
1795
|
+
* base + "a" (e.g., "Editou" → "Edit" and "Edita")
|
|
1796
|
+
* because Czech feminine names decline -a → -ou.
|
|
1797
|
+
*/
|
|
1798
|
+
const stripInflection = (token) => {
|
|
1799
|
+
const candidates = [];
|
|
1800
|
+
for (const suffix of INFLECTION_SUFFIXES) if (token.length > suffix.length + 2 && token.endsWith(suffix)) {
|
|
1801
|
+
const base = token.slice(0, -suffix.length);
|
|
1802
|
+
if (/^\p{Lu}/u.test(base)) {
|
|
1803
|
+
candidates.push(base);
|
|
1804
|
+
if (suffix === "ou" || suffix === "é" || suffix === "u") candidates.push(`${base}a`);
|
|
1805
|
+
}
|
|
1806
|
+
}
|
|
1807
|
+
return candidates;
|
|
1808
|
+
};
|
|
1809
|
+
const TOKEN_TYPE = {
|
|
1810
|
+
NAME: "name",
|
|
1811
|
+
SURNAME: "surname",
|
|
1812
|
+
TITLE: "title",
|
|
1813
|
+
ABBREVIATION: "abbreviation",
|
|
1814
|
+
CAPITALIZED: "capitalized",
|
|
1815
|
+
OTHER: "other"
|
|
1816
|
+
};
|
|
1817
|
+
const PERSON_CHAIN_BREAK_RE$1 = /[!?;:]/u;
|
|
1818
|
+
const isInitialContinuationGap$1 = (text, gap) => /^\p{Lu}$/u.test(text) && /^\.[^\S\n]{1,2}$/u.test(gap) || /^[^\S\n]{1,2}(?:\p{Lu}\.[^\S\n]{1,2})+$/u.test(gap);
|
|
1819
|
+
/**
|
|
1820
|
+
* Check if a token is in the first-name set, either
|
|
1821
|
+
* directly or after stripping Czech/Slovak inflection.
|
|
1822
|
+
*/
|
|
1823
|
+
const isFirstNameToken = (token, corpus) => {
|
|
1824
|
+
if (corpus.firstNames.has(token)) return true;
|
|
1825
|
+
return stripInflection(token).some((b) => corpus.firstNames.has(b));
|
|
1826
|
+
};
|
|
1827
|
+
/**
|
|
1828
|
+
* Check if a token is in the surname set, either
|
|
1829
|
+
* directly or after stripping Czech/Slovak inflection.
|
|
1830
|
+
*/
|
|
1831
|
+
const isSurnameToken = (token, corpus) => {
|
|
1832
|
+
if (corpus.surnames.has(token)) return true;
|
|
1833
|
+
return stripInflection(token).some((b) => corpus.surnames.has(b));
|
|
1834
|
+
};
|
|
1835
|
+
/**
|
|
1836
|
+
* Check if a token looks like a single-letter
|
|
1837
|
+
* abbreviation: "J.", "M.", etc.
|
|
1838
|
+
*/
|
|
1839
|
+
const isAbbreviation = (token) => token.length === 2 && /^\p{Lu}$/u.test(token[0] ?? "") && token[1] === ".";
|
|
1840
|
+
const segmenter$1 = new Intl.Segmenter(void 0, { granularity: "word" });
|
|
1841
|
+
/**
|
|
1842
|
+
* Split text into word segments using Intl.Segmenter.
|
|
1843
|
+
* Only returns segments flagged as words.
|
|
1844
|
+
*/
|
|
1845
|
+
const segmentWords = (fullText) => {
|
|
1846
|
+
const words = [];
|
|
1847
|
+
for (const seg of segmenter$1.segment(fullText)) if (seg.isWordLike) words.push({
|
|
1848
|
+
text: seg.segment,
|
|
1849
|
+
start: seg.index,
|
|
1850
|
+
end: seg.index + seg.segment.length
|
|
1851
|
+
});
|
|
1852
|
+
return words;
|
|
1853
|
+
};
|
|
1854
|
+
/** NAME or SURNAME — both represent corpus-matched tokens */
|
|
1855
|
+
const isCorpusMatch = (type) => type === TOKEN_TYPE.NAME || type === TOKEN_TYPE.SURNAME;
|
|
1856
|
+
const classifyToken = (word, corpus) => {
|
|
1857
|
+
const { text, start, end } = word;
|
|
1858
|
+
const lower = text.toLowerCase();
|
|
1859
|
+
const stripped = text.endsWith(".") ? text.slice(0, -1).toLowerCase() : lower;
|
|
1860
|
+
if (corpus.titleTokens.has(stripped)) return {
|
|
1861
|
+
text,
|
|
1862
|
+
type: TOKEN_TYPE.TITLE,
|
|
1863
|
+
start,
|
|
1864
|
+
end
|
|
1865
|
+
};
|
|
1866
|
+
if (isAbbreviation(text)) return {
|
|
1867
|
+
text,
|
|
1868
|
+
type: TOKEN_TYPE.ABBREVIATION,
|
|
1869
|
+
start,
|
|
1870
|
+
end
|
|
1871
|
+
};
|
|
1872
|
+
if (corpus.excludedWords.has(lower)) return {
|
|
1873
|
+
text,
|
|
1874
|
+
type: TOKEN_TYPE.OTHER,
|
|
1875
|
+
start,
|
|
1876
|
+
end
|
|
1877
|
+
};
|
|
1878
|
+
if (text.length < 3) return {
|
|
1879
|
+
text,
|
|
1880
|
+
type: TOKEN_TYPE.OTHER,
|
|
1881
|
+
start,
|
|
1882
|
+
end
|
|
1883
|
+
};
|
|
1884
|
+
if (text.length > 3 && ALL_UPPER_RE.test(text)) return {
|
|
1885
|
+
text,
|
|
1886
|
+
type: TOKEN_TYPE.OTHER,
|
|
1887
|
+
start,
|
|
1888
|
+
end
|
|
1889
|
+
};
|
|
1890
|
+
if (!UPPER_START_RE.test(text)) return {
|
|
1891
|
+
text,
|
|
1892
|
+
type: TOKEN_TYPE.OTHER,
|
|
1893
|
+
start,
|
|
1894
|
+
end
|
|
1895
|
+
};
|
|
1896
|
+
if (isFirstNameToken(text, corpus)) return {
|
|
1897
|
+
text,
|
|
1898
|
+
type: TOKEN_TYPE.NAME,
|
|
1899
|
+
start,
|
|
1900
|
+
end
|
|
1901
|
+
};
|
|
1902
|
+
if (isSurnameToken(text, corpus)) return {
|
|
1903
|
+
text,
|
|
1904
|
+
type: TOKEN_TYPE.SURNAME,
|
|
1905
|
+
start,
|
|
1906
|
+
end
|
|
1907
|
+
};
|
|
1908
|
+
return {
|
|
1909
|
+
text,
|
|
1910
|
+
type: TOKEN_TYPE.CAPITALIZED,
|
|
1911
|
+
start,
|
|
1912
|
+
end
|
|
1913
|
+
};
|
|
1914
|
+
};
|
|
1915
|
+
/**
|
|
1916
|
+
* Detect person names by looking up tokens against the
|
|
1917
|
+
* name corpus, then chaining adjacent name-like tokens.
|
|
1918
|
+
*
|
|
1919
|
+
* Requires initNameCorpus() to have been called first.
|
|
1920
|
+
* If not initialized, returns an empty array.
|
|
1921
|
+
*
|
|
1922
|
+
* Scoring:
|
|
1923
|
+
* TITLE + NAME/SURNAME → 0.95
|
|
1924
|
+
* NAME + NAME/SURNAME → 0.9
|
|
1925
|
+
* SURNAME + NAME/SURNAME → 0.9
|
|
1926
|
+
* NAME + CAPITALIZED → 0.7
|
|
1927
|
+
* ABBREVIATION + NAME → 0.7
|
|
1928
|
+
* Standalone NAME → 0.5 (low confidence)
|
|
1929
|
+
* Standalone SURNAME → skip (too ambiguous)
|
|
1930
|
+
*/
|
|
1931
|
+
const detectNameCorpus = (fullText, ctx = defaultContext) => {
|
|
1932
|
+
const corpus = getCorpus(ctx);
|
|
1933
|
+
if (!corpus) return [];
|
|
1934
|
+
const tokens = segmentWords(fullText).map((w) => classifyToken(w, corpus));
|
|
1935
|
+
const entities = [];
|
|
1936
|
+
const consumed = /* @__PURE__ */ new Set();
|
|
1937
|
+
for (let i = 0; i < tokens.length; i++) {
|
|
1938
|
+
if (consumed.has(i)) continue;
|
|
1939
|
+
const token = tokens[i];
|
|
1940
|
+
if (!token) continue;
|
|
1941
|
+
if (token.type !== TOKEN_TYPE.TITLE && token.type !== TOKEN_TYPE.NAME && token.type !== TOKEN_TYPE.SURNAME && token.type !== TOKEN_TYPE.ABBREVIATION) continue;
|
|
1942
|
+
const MAX_CHAIN = 5;
|
|
1943
|
+
const chain = [token];
|
|
1944
|
+
let j = i + 1;
|
|
1945
|
+
while (j < tokens.length && chain.length < MAX_CHAIN) {
|
|
1946
|
+
const next = tokens[j];
|
|
1947
|
+
if (!next) break;
|
|
1948
|
+
const prev = chain.at(-1);
|
|
1949
|
+
if (prev) {
|
|
1950
|
+
const gap = fullText.slice(prev.end, next.start);
|
|
1951
|
+
const breaksOnPeriod = gap.includes(".") && !isInitialContinuationGap$1(prev.text, gap);
|
|
1952
|
+
if (gap.includes("\n") || PERSON_CHAIN_BREAK_RE$1.test(gap) || breaksOnPeriod) break;
|
|
1953
|
+
}
|
|
1954
|
+
if (next.type === TOKEN_TYPE.NAME || next.type === TOKEN_TYPE.SURNAME || next.type === TOKEN_TYPE.TITLE || next.type === TOKEN_TYPE.ABBREVIATION || next.type === TOKEN_TYPE.CAPITALIZED) {
|
|
1955
|
+
chain.push(next);
|
|
1956
|
+
j++;
|
|
1957
|
+
} else break;
|
|
1958
|
+
}
|
|
1959
|
+
const hasTitle = chain.some((t) => t.type === TOKEN_TYPE.TITLE);
|
|
1960
|
+
const hasCorpusName = chain.some((t) => isCorpusMatch(t.type));
|
|
1961
|
+
const hasFirstName = chain.some((t) => t.type === TOKEN_TYPE.NAME);
|
|
1962
|
+
const hasAbbreviation = chain.some((t) => t.type === TOKEN_TYPE.ABBREVIATION);
|
|
1963
|
+
const corpusCount = chain.filter((t) => isCorpusMatch(t.type)).length;
|
|
1964
|
+
const capitalizedCount = chain.filter((t) => t.type === TOKEN_TYPE.CAPITALIZED).length;
|
|
1965
|
+
let score = 0;
|
|
1966
|
+
if (hasTitle && hasCorpusName) score = .95;
|
|
1967
|
+
else if (corpusCount >= 2) score = .9;
|
|
1968
|
+
else if (hasCorpusName && capitalizedCount > 0) score = .7;
|
|
1969
|
+
else if (hasAbbreviation && hasCorpusName) score = .7;
|
|
1970
|
+
else if (hasFirstName && chain.length === 1) {
|
|
1971
|
+
if (isSentenceStart(fullText, token.start)) continue;
|
|
1972
|
+
score = .5;
|
|
1973
|
+
} else if (!hasFirstName && chain.length === 1 && chain[0]?.type === TOKEN_TYPE.SURNAME) continue;
|
|
1974
|
+
else if (hasTitle && chain.length === 1) continue;
|
|
1975
|
+
else {
|
|
1976
|
+
if (!hasCorpusName) continue;
|
|
1977
|
+
score = .5;
|
|
1560
1978
|
}
|
|
1979
|
+
const first = chain.at(0);
|
|
1980
|
+
const last = chain.at(-1);
|
|
1981
|
+
if (!first || !last) continue;
|
|
1982
|
+
const start = first.start;
|
|
1983
|
+
const end = last.end;
|
|
1984
|
+
const text = fullText.slice(start, end);
|
|
1985
|
+
for (let k = i; k < i + chain.length; k++) consumed.add(k);
|
|
1986
|
+
entities.push({
|
|
1987
|
+
start,
|
|
1988
|
+
end,
|
|
1989
|
+
label: "person",
|
|
1990
|
+
text,
|
|
1991
|
+
score,
|
|
1992
|
+
source: DETECTION_SOURCES.REGEX
|
|
1993
|
+
});
|
|
1561
1994
|
}
|
|
1995
|
+
return entities;
|
|
1562
1996
|
};
|
|
1563
1997
|
//#endregion
|
|
1564
|
-
//#region src/
|
|
1565
|
-
/**
|
|
1566
|
-
* Centralized Unicode character equivalence groups.
|
|
1567
|
-
*
|
|
1568
|
-
* Centralized Unicode character equivalence groups.
|
|
1569
|
-
*
|
|
1570
|
-
* Provides helpers to build regex character classes that
|
|
1571
|
-
* match all typographic variants of a character type
|
|
1572
|
-
* (dashes, spaces, quotes, etc.).
|
|
1573
|
-
*
|
|
1574
|
-
* The JSON is statically imported so the bundler inlines
|
|
1575
|
-
* it, avoiding a runtime require() that breaks browsers.
|
|
1576
|
-
*/
|
|
1577
|
-
/** Chars that need escaping inside a regex char class. */
|
|
1578
|
-
const REGEX_CLASS_SPECIAL = /[\\\]^-]/;
|
|
1579
|
-
const escapeForCharClass = (ch) => REGEX_CLASS_SPECIAL.test(ch) ? `\\${ch}` : ch;
|
|
1580
|
-
const config = char_groups_default;
|
|
1998
|
+
//#region src/config/titles.ts
|
|
1581
1999
|
/**
|
|
1582
|
-
*
|
|
1583
|
-
*
|
|
2000
|
+
* Academic and professional title prefixes.
|
|
2001
|
+
* Plain text; the detector auto-escapes for regex.
|
|
2002
|
+
* Sorted longest-first at build time.
|
|
1584
2003
|
*/
|
|
1585
|
-
const
|
|
1586
|
-
|
|
1587
|
-
|
|
1588
|
-
|
|
1589
|
-
|
|
2004
|
+
const TITLE_PREFIXES = [
|
|
2005
|
+
"Ing.",
|
|
2006
|
+
"Mgr.",
|
|
2007
|
+
"MgA.",
|
|
2008
|
+
"Bc.",
|
|
2009
|
+
"BcA.",
|
|
2010
|
+
"JUDr.",
|
|
2011
|
+
"MUDr.",
|
|
2012
|
+
"MVDr.",
|
|
2013
|
+
"MDDr.",
|
|
2014
|
+
"PhDr.",
|
|
2015
|
+
"RNDr.",
|
|
2016
|
+
"PaedDr.",
|
|
2017
|
+
"ThDr.",
|
|
2018
|
+
"ThLic.",
|
|
2019
|
+
"ICDr.",
|
|
2020
|
+
"RSDr.",
|
|
2021
|
+
"PharmDr.",
|
|
2022
|
+
"artD.",
|
|
2023
|
+
"akad.",
|
|
2024
|
+
"doc.",
|
|
2025
|
+
"prof.",
|
|
2026
|
+
"ao. Univ.-Prof.",
|
|
2027
|
+
"o. Univ.-Prof.",
|
|
2028
|
+
"Univ.-Prof.",
|
|
2029
|
+
"Hon.-Prof.",
|
|
2030
|
+
"em. Prof.",
|
|
2031
|
+
"Dr. med. dent.",
|
|
2032
|
+
"Dr. med. vet.",
|
|
2033
|
+
"Dr. med.",
|
|
2034
|
+
"Dr. rer. nat.",
|
|
2035
|
+
"Dr. rer. soc.",
|
|
2036
|
+
"Dr. rer. pol.",
|
|
2037
|
+
"Dr. sc. tech.",
|
|
2038
|
+
"Dr. sc. nat.",
|
|
2039
|
+
"Dr. sc. hum.",
|
|
2040
|
+
"Dr. iur.",
|
|
2041
|
+
"Dr. jur.",
|
|
2042
|
+
"Dr. theol.",
|
|
2043
|
+
"Dr. oec.",
|
|
2044
|
+
"Dr. techn.",
|
|
2045
|
+
"Dr. h. c.",
|
|
2046
|
+
"Dr. phil.",
|
|
2047
|
+
"Dr.-Ing.",
|
|
2048
|
+
"Dr. Ing.",
|
|
2049
|
+
"Dr.",
|
|
2050
|
+
"Dipl.-Wirt.-Ing.",
|
|
2051
|
+
"Dipl.-Betriebsw.",
|
|
2052
|
+
"Dipl.-Inform.",
|
|
2053
|
+
"Dipl.-Volksw.",
|
|
2054
|
+
"Dipl.-Psych.",
|
|
2055
|
+
"Dipl.-Phys.",
|
|
2056
|
+
"Dipl.-Chem.",
|
|
2057
|
+
"Dipl.-Biol.",
|
|
2058
|
+
"Dipl.-Math.",
|
|
2059
|
+
"Dipl.-Päd.",
|
|
2060
|
+
"Dipl.-Soz.",
|
|
2061
|
+
"Dipl.-Kfm.",
|
|
2062
|
+
"Dipl.-Jur.",
|
|
2063
|
+
"Dipl. Ing.",
|
|
2064
|
+
"Dipl.-Ing.",
|
|
2065
|
+
"Mag. rer. soc. oec.",
|
|
2066
|
+
"Mag. rer. nat.",
|
|
2067
|
+
"Mag. phil.",
|
|
2068
|
+
"Mag. iur.",
|
|
2069
|
+
"Mag. arch.",
|
|
2070
|
+
"Mag. pharm.",
|
|
2071
|
+
"Mag. (FH)",
|
|
2072
|
+
"Mag.",
|
|
2073
|
+
"Bakk. rer. nat.",
|
|
2074
|
+
"Bakk. techn.",
|
|
2075
|
+
"Bakk. phil.",
|
|
2076
|
+
"Bakk.",
|
|
2077
|
+
"Lic. phil.",
|
|
2078
|
+
"Lic. iur.",
|
|
2079
|
+
"Lic. oec.",
|
|
2080
|
+
"Lic. theol.",
|
|
2081
|
+
"Lic.",
|
|
2082
|
+
"Priv.-Doz.",
|
|
2083
|
+
"PD",
|
|
2084
|
+
"RA"
|
|
2085
|
+
];
|
|
1590
2086
|
/**
|
|
1591
|
-
*
|
|
1592
|
-
*
|
|
1593
|
-
*
|
|
1594
|
-
*
|
|
1595
|
-
* The hyphen-minus is placed first so it is treated
|
|
1596
|
-
* as a literal, not a range indicator.
|
|
2087
|
+
* Courtesy/honorific titles that precede a person's
|
|
2088
|
+
* name. Sorted alphabetically. The detector escapes
|
|
2089
|
+
* dots and adds \b for entries in HONORIFIC_BOUNDARY.
|
|
1597
2090
|
*/
|
|
1598
|
-
const
|
|
1599
|
-
|
|
1600
|
-
|
|
1601
|
-
|
|
1602
|
-
|
|
1603
|
-
|
|
1604
|
-
|
|
2091
|
+
const HONORIFICS = [
|
|
2092
|
+
"Avv.",
|
|
2093
|
+
"Dame",
|
|
2094
|
+
"Doamna",
|
|
2095
|
+
"Domnul",
|
|
2096
|
+
"Don",
|
|
2097
|
+
"Doña",
|
|
2098
|
+
"Dott.",
|
|
2099
|
+
"Judge",
|
|
2100
|
+
"Justice",
|
|
2101
|
+
"Lady",
|
|
2102
|
+
"Lord",
|
|
2103
|
+
"M.",
|
|
2104
|
+
"Madame",
|
|
2105
|
+
"Mademoiselle",
|
|
2106
|
+
"Maître",
|
|
2107
|
+
"Me",
|
|
2108
|
+
"Messrs",
|
|
2109
|
+
"Miss",
|
|
2110
|
+
"Mlle",
|
|
2111
|
+
"Mme",
|
|
2112
|
+
"Monsieur",
|
|
2113
|
+
"Mr",
|
|
2114
|
+
"Mrs",
|
|
2115
|
+
"Ms",
|
|
2116
|
+
"Pr",
|
|
2117
|
+
"Pr.",
|
|
2118
|
+
"President",
|
|
2119
|
+
"Señor",
|
|
2120
|
+
"Señora",
|
|
2121
|
+
"Sig.",
|
|
2122
|
+
"Sig.ra",
|
|
2123
|
+
"Signor",
|
|
2124
|
+
"Signora",
|
|
2125
|
+
"Signorina",
|
|
2126
|
+
"Sir",
|
|
2127
|
+
"Sr.",
|
|
2128
|
+
"Sra."
|
|
2129
|
+
];
|
|
1605
2130
|
/**
|
|
1606
|
-
*
|
|
1607
|
-
*
|
|
1608
|
-
* inside a larger character class, e.g.:
|
|
1609
|
-
* `[\\s&,.${charClassInner("dash")}]`
|
|
2131
|
+
* Honorifics that need \b word-boundary anchors
|
|
2132
|
+
* (short or common words that could match mid-word).
|
|
1610
2133
|
*/
|
|
1611
|
-
const
|
|
1612
|
-
|
|
1613
|
-
|
|
1614
|
-
|
|
1615
|
-
|
|
1616
|
-
|
|
1617
|
-
|
|
1618
|
-
|
|
1619
|
-
|
|
2134
|
+
const HONORIFIC_BOUNDARY = new Set([
|
|
2135
|
+
"Don",
|
|
2136
|
+
"Doña",
|
|
2137
|
+
"M.",
|
|
2138
|
+
"Me",
|
|
2139
|
+
"Pr",
|
|
2140
|
+
"Pr.",
|
|
2141
|
+
"Señor",
|
|
2142
|
+
"Señora"
|
|
2143
|
+
]);
|
|
1620
2144
|
/**
|
|
1621
|
-
*
|
|
1622
|
-
*
|
|
1623
|
-
|
|
1624
|
-
|
|
1625
|
-
|
|
1626
|
-
|
|
1627
|
-
|
|
1628
|
-
|
|
1629
|
-
|
|
1630
|
-
|
|
1631
|
-
|
|
2145
|
+
* Post-nominal degrees (comma or space separated after name).
|
|
2146
|
+
* Plain text; the detector auto-escapes for regex.
|
|
2147
|
+
*/
|
|
2148
|
+
const POST_NOMINALS = [
|
|
2149
|
+
"Ph.D.",
|
|
2150
|
+
"Ph.D",
|
|
2151
|
+
"CSc.",
|
|
2152
|
+
"DrSc.",
|
|
2153
|
+
"ArtD.",
|
|
2154
|
+
"D.Phil.",
|
|
2155
|
+
"DPhil.",
|
|
2156
|
+
"MPhil.",
|
|
2157
|
+
"MBA",
|
|
2158
|
+
"MPA",
|
|
2159
|
+
"LL.M.",
|
|
2160
|
+
"LL.B.",
|
|
2161
|
+
"M.Sc.",
|
|
2162
|
+
"B.Sc.",
|
|
2163
|
+
"MSc.",
|
|
2164
|
+
"BSc.",
|
|
2165
|
+
"M.Eng.",
|
|
2166
|
+
"B.Eng.",
|
|
2167
|
+
"M.A.",
|
|
2168
|
+
"B.A.",
|
|
2169
|
+
"JCD",
|
|
2170
|
+
"JD",
|
|
2171
|
+
"DiS.",
|
|
2172
|
+
"ACCA",
|
|
2173
|
+
"FCCA",
|
|
2174
|
+
"CIPM",
|
|
2175
|
+
"CIPT",
|
|
2176
|
+
"CIPP/E",
|
|
2177
|
+
"CIPP"
|
|
2178
|
+
];
|
|
1632
2179
|
//#endregion
|
|
1633
2180
|
//#region src/detectors/regex.ts
|
|
1634
2181
|
const MIN_PHONE_LENGTH = 7;
|
|
1635
2182
|
const MIN_MONTH_NAME_LENGTH = 3;
|
|
1636
2183
|
const escapeTitle = (title) => title.replace(/[.*+?^${}()|[\]\\]/g, "\\$&").replace(/\s+/g, "\\s*");
|
|
1637
2184
|
/** Escape for use inside a regex alternation. */
|
|
1638
|
-
const escapeRegex$
|
|
2185
|
+
const escapeRegex$2 = (s) => s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
1639
2186
|
/** Escape for use inside a regex character class. */
|
|
1640
2187
|
const escapeCharClass = (s) => s.replace(/[\]\\^-]/g, "\\$&");
|
|
1641
2188
|
const TITLE_PREFIX = TITLE_PREFIXES.toSorted((a, b) => b.length - a.length).map(escapeTitle).join("|");
|
|
1642
2189
|
const POST_NOMINAL = POST_NOMINALS.toSorted((a, b) => b.length - a.length).map(escapeTitle).join("|");
|
|
1643
|
-
const NAME_WORD =
|
|
2190
|
+
const NAME_WORD = `\\p{Lu}\\p{Ll}+`;
|
|
1644
2191
|
const PARTICLE = "(?:van der|van den|de la|della|von|van|dos|ibn|ben|bin|del|zum|zur|ten|ter|da|de|di|al|el|le|la|zu|af|av)";
|
|
1645
2192
|
const SP = "[^\\S\\n\\t]";
|
|
1646
2193
|
/** Honorific alternation built from titles.ts config. */
|
|
1647
2194
|
const HONORIFIC_ALT = [...HONORIFICS].toSorted((a, b) => b.length - a.length).map((h) => {
|
|
1648
|
-
const escaped = escapeRegex$
|
|
2195
|
+
const escaped = escapeRegex$2(h);
|
|
1649
2196
|
return HONORIFIC_BOUNDARY.has(h) ? `\\b${escaped}` : escaped;
|
|
1650
2197
|
}).join("|");
|
|
1651
2198
|
const toEntry = (validator, label, score) => {
|
|
@@ -1670,8 +2217,6 @@ const toEntry = (validator, label, score) => {
|
|
|
1670
2217
|
const STDNUM_ENTRIES = [
|
|
1671
2218
|
toEntry(hu.vat, "tax identification number", .95),
|
|
1672
2219
|
toEntry(it.codiceFiscale, "national identification number", .95),
|
|
1673
|
-
toEntry(es.dni, "national identification number", .9),
|
|
1674
|
-
toEntry(es.nie, "national identification number", .95),
|
|
1675
2220
|
toEntry(se.personnummer, "national identification number", .9),
|
|
1676
2221
|
toEntry(ro.cnp, "national identification number", .95),
|
|
1677
2222
|
toEntry(fr.nir, "social security number", .9),
|
|
@@ -1697,7 +2242,6 @@ const STDNUM_ENTRIES = [
|
|
|
1697
2242
|
toEntry(fi.ytunnus, "registration number", .9),
|
|
1698
2243
|
toEntry(bg.vat, "tax identification number", .95),
|
|
1699
2244
|
toEntry(sk.dic, "tax identification number", .95),
|
|
1700
|
-
toEntry(es.cif, "registration number", .95),
|
|
1701
2245
|
toEntry(es.vat, "tax identification number", .95),
|
|
1702
2246
|
toEntry(es.nss, "social security number", .9),
|
|
1703
2247
|
toEntry(fr.tva, "tax identification number", .95),
|
|
@@ -1719,7 +2263,9 @@ const STDNUM_ENTRIES = [
|
|
|
1719
2263
|
toEntry(ee.ik, "national identification number", .9),
|
|
1720
2264
|
toEntry(cy.vat, "tax identification number", .95),
|
|
1721
2265
|
toEntry(mt.vat, "tax identification number", .95),
|
|
1722
|
-
toEntry(lu.vat, "tax identification number", .95)
|
|
2266
|
+
toEntry(lu.vat, "tax identification number", .95),
|
|
2267
|
+
toEntry(br.cpf, "tax identification number", .95),
|
|
2268
|
+
toEntry(br.cnpj, "tax identification number", .95)
|
|
1723
2269
|
].filter((e) => e !== null);
|
|
1724
2270
|
const TITLED_PERSON = {
|
|
1725
2271
|
pattern: `(?:${TITLE_PREFIX})(?:${SP}+(?:${TITLE_PREFIX}))*${SP}+(?:${NAME_WORD})(?:${SP}{1,4}(?:${PARTICLE}${SP}+)?${NAME_WORD}){1,3}(?:,?${SP}+(?:${POST_NOMINAL})(?:,?${SP}+(?:${POST_NOMINAL}))*)?`,
|
|
@@ -1768,7 +2314,7 @@ const CREDIT_CARD = {
|
|
|
1768
2314
|
};
|
|
1769
2315
|
const CZ_BIRTH_NUMBER = {
|
|
1770
2316
|
pattern: `\\b\\d{6}/\\d{3,4}\\b`,
|
|
1771
|
-
label: "
|
|
2317
|
+
label: "birth number",
|
|
1772
2318
|
score: 1,
|
|
1773
2319
|
validator: cz.rc
|
|
1774
2320
|
};
|
|
@@ -1802,6 +2348,51 @@ const CZ_POSTAL = {
|
|
|
1802
2348
|
label: "address",
|
|
1803
2349
|
score: .7
|
|
1804
2350
|
};
|
|
2351
|
+
const ES_POSTAL = {
|
|
2352
|
+
pattern: "\\b(?:C\\.?P\\.?|[Cc][óo]digo[^\\S\\n]+postal)[^\\S\\n]{0,3}:?[^\\S\\n]{0,3}\\d{5}\\b",
|
|
2353
|
+
label: "address",
|
|
2354
|
+
score: .7
|
|
2355
|
+
};
|
|
2356
|
+
const ES_DNI = {
|
|
2357
|
+
pattern: `\\b\\d{8}-?[A-Za-z]\\b`,
|
|
2358
|
+
label: "national identification number",
|
|
2359
|
+
score: .9,
|
|
2360
|
+
validator: es.dni
|
|
2361
|
+
};
|
|
2362
|
+
const ES_NIE = {
|
|
2363
|
+
pattern: `\\b[XYZxyz]-?\\d{7}-?[A-Za-z]\\b`,
|
|
2364
|
+
label: "national identification number",
|
|
2365
|
+
score: .95,
|
|
2366
|
+
validator: es.nie
|
|
2367
|
+
};
|
|
2368
|
+
const ES_CIF = {
|
|
2369
|
+
pattern: `\\b[A-HJNP-SUVWa-hjnp-suvw]-?\\d{7}-?[0-9A-Ja-j]\\b`,
|
|
2370
|
+
label: "registration number",
|
|
2371
|
+
score: .95,
|
|
2372
|
+
validator: es.cif
|
|
2373
|
+
};
|
|
2374
|
+
const BR_CPF_FORMATTED = {
|
|
2375
|
+
pattern: `\\b\\d{3}\\.\\d{3}\\.\\d{3}${DASH}\\d{2}\\b`,
|
|
2376
|
+
label: "tax identification number",
|
|
2377
|
+
score: .95,
|
|
2378
|
+
validator: br.cpf
|
|
2379
|
+
};
|
|
2380
|
+
const BR_CNPJ_FORMATTED = {
|
|
2381
|
+
pattern: `\\b\\d{2}\\.\\d{3}\\.\\d{3}/\\d{4}${DASH}\\d{2}\\b`,
|
|
2382
|
+
label: "tax identification number",
|
|
2383
|
+
score: .95,
|
|
2384
|
+
validator: br.cnpj
|
|
2385
|
+
};
|
|
2386
|
+
const BR_RG_WITH_SSP = {
|
|
2387
|
+
pattern: `\\b\\d{1,3}\\.?\\d{3}\\.?\\d{3}(?:${DASH}[0-9A-Za-z])?[^\\S\\n]+SSP(?:/[A-Z]{2})?\\b`,
|
|
2388
|
+
label: "national identification number",
|
|
2389
|
+
score: .95
|
|
2390
|
+
};
|
|
2391
|
+
const BR_OAB = {
|
|
2392
|
+
pattern: "\\bOAB/[A-Z]{2}[^\\S\\n]+(?:n[º°.][^\\S\\n]*)?(?:\\d{1,3}(?:\\.\\d{3})+|\\d{4,6})\\b",
|
|
2393
|
+
label: "registration number",
|
|
2394
|
+
score: .95
|
|
2395
|
+
};
|
|
1805
2396
|
const URL = {
|
|
1806
2397
|
pattern: "(?:https?://|https?:(?=[^\\s])|www\\.)[\\w\\-]+(?:\\.[\\w\\-]+)+(?::\\d+)?(?:[/?#][^\\s)\\]>]*[^\\s.,;:!?)\\]>])?",
|
|
1807
2398
|
label: "url",
|
|
@@ -1843,6 +2434,14 @@ const ALL_REGEX_DEFS = [
|
|
|
1843
2434
|
CZ_BANK_ACCOUNT,
|
|
1844
2435
|
HU_LANDLINE,
|
|
1845
2436
|
CZ_POSTAL,
|
|
2437
|
+
ES_POSTAL,
|
|
2438
|
+
ES_DNI,
|
|
2439
|
+
ES_NIE,
|
|
2440
|
+
ES_CIF,
|
|
2441
|
+
BR_CPF_FORMATTED,
|
|
2442
|
+
BR_CNPJ_FORMATTED,
|
|
2443
|
+
BR_RG_WITH_SSP,
|
|
2444
|
+
BR_OAB,
|
|
1846
2445
|
URL,
|
|
1847
2446
|
{
|
|
1848
2447
|
pattern: "\\b(?:[0-9a-fA-F]{1,4}:){7}[0-9a-fA-F]{1,4}\\b|\\b(?:[0-9a-fA-F]{1,4}:){1,7}:\\b|::(?:[0-9a-fA-F]{1,4}:){0,6}[0-9a-fA-F]{1,4}",
|
|
@@ -1894,7 +2493,7 @@ const buildMonthAlternation = (months) => {
|
|
|
1894
2493
|
if (clean.length >= MIN_MONTH_NAME_LENGTH) seen.add(clean);
|
|
1895
2494
|
}
|
|
1896
2495
|
}
|
|
1897
|
-
return [...seen].toSorted((a, b) => b.length - a.length).map(escapeRegex$
|
|
2496
|
+
return [...seen].toSorted((a, b) => b.length - a.length).map(escapeRegex$2).join("|");
|
|
1898
2497
|
};
|
|
1899
2498
|
/**
|
|
1900
2499
|
* Build date patterns from a month-name alternation.
|
|
@@ -1951,11 +2550,11 @@ const buildCurrencyPatterns = (data) => {
|
|
|
1951
2550
|
const isAsciiAlpha = /^[a-zA-Z\s]+$/;
|
|
1952
2551
|
const parts = data.codes.map((code) => ({
|
|
1953
2552
|
len: code.length,
|
|
1954
|
-
alt: escapeRegex$
|
|
2553
|
+
alt: escapeRegex$2(code)
|
|
1955
2554
|
}));
|
|
1956
2555
|
const MIN_CI_LENGTH = 3;
|
|
1957
2556
|
if (data.localNames) for (const name of data.localNames) {
|
|
1958
|
-
const escaped = escapeRegex$
|
|
2557
|
+
const escaped = escapeRegex$2(name);
|
|
1959
2558
|
if (isAsciiAlpha.test(name) && name.length >= MIN_CI_LENGTH) parts.push({
|
|
1960
2559
|
len: name.length,
|
|
1961
2560
|
alt: `(?i:${escaped})`
|
|
@@ -1967,7 +2566,7 @@ const buildCurrencyPatterns = (data) => {
|
|
|
1967
2566
|
}
|
|
1968
2567
|
if (symbols) for (const ch of data.symbols) parts.push({
|
|
1969
2568
|
len: ch.length,
|
|
1970
|
-
alt: escapeRegex$
|
|
2569
|
+
alt: escapeRegex$2(ch)
|
|
1971
2570
|
});
|
|
1972
2571
|
const trailingAlt = parts.toSorted((a, b) => b.len - a.len).map((p) => p.alt).join("|");
|
|
1973
2572
|
if (!symbols && !trailingAlt) return [];
|
|
@@ -2019,290 +2618,70 @@ const processRegexMatches = (allMatches, sliceStart, sliceEnd, meta_) => {
|
|
|
2019
2618
|
for (const match of allMatches) {
|
|
2020
2619
|
const idx = match.pattern;
|
|
2021
2620
|
if (idx < sliceStart || idx >= sliceEnd) continue;
|
|
2022
|
-
const meta = meta_[idx - sliceStart];
|
|
2023
|
-
if (!meta) continue;
|
|
2024
|
-
if (meta.sourceDetail !== "custom-regex" && meta.label === "phone number" && match.text.length < MIN_PHONE_LENGTH) continue;
|
|
2025
|
-
if (meta.validator) {
|
|
2026
|
-
const compacted = meta.validator.compact(match.text);
|
|
2027
|
-
if (!meta.validator.validate(compacted).valid) continue;
|
|
2028
|
-
}
|
|
2029
|
-
const entity = {
|
|
2030
|
-
start: match.start,
|
|
2031
|
-
end: match.end,
|
|
2032
|
-
label: meta.label,
|
|
2033
|
-
text: match.text,
|
|
2034
|
-
score: meta.score,
|
|
2035
|
-
source: DETECTION_SOURCES.REGEX
|
|
2036
|
-
};
|
|
2037
|
-
if (meta.sourceDetail) entity.sourceDetail = meta.sourceDetail;
|
|
2038
|
-
results.push(entity);
|
|
2039
|
-
}
|
|
2040
|
-
return results;
|
|
2041
|
-
};
|
|
2042
|
-
/**
|
|
2043
|
-
* Build signing clause place-name patterns from
|
|
2044
|
-
* signing-clauses.json. Each pattern captures the
|
|
2045
|
-
* city/place name from contract signing locations.
|
|
2046
|
-
*
|
|
2047
|
-
* The place name sub-pattern:
|
|
2048
|
-
* \p{Lu}\p{Ll}+ (capitalized word)
|
|
2049
|
-
* optionally followed by preposition + capitalized
|
|
2050
|
-
* word (for "nad Nisou", "am Main", etc.)
|
|
2051
|
-
* optionally followed by more capitalized words
|
|
2052
|
-
* (for "Hradec Králové", "New York", etc.)
|
|
2053
|
-
*/
|
|
2054
|
-
const buildSigningClausePatterns = (data) => {
|
|
2055
|
-
const patterns = [];
|
|
2056
|
-
for (const entry of data.patterns) {
|
|
2057
|
-
const prepAlt = entry.prepositions.length > 0 ? entry.prepositions.join("|") : null;
|
|
2058
|
-
const place = prepAlt ? `(\\p{Lu}\\p{Ll}+(?:\\s+(?:${prepAlt})\\s+\\p{Lu}\\p{Ll}+)*(?:\\s+\\p{Lu}\\p{Ll}+)*)` : `(\\p{Lu}\\p{Ll}+(?:[- ]\\p{Lu}\\p{Ll}+)*)`;
|
|
2059
|
-
const full = `(?:^|\\n|[^\\S\\n])` + entry.prefix + place + (entry.suffix ? `(?:${entry.suffix})` : "");
|
|
2060
|
-
patterns.push(full);
|
|
2061
|
-
}
|
|
2062
|
-
return patterns;
|
|
2063
|
-
};
|
|
2064
|
-
const SIGNING_CLAUSE_META = {
|
|
2065
|
-
label: "address",
|
|
2066
|
-
score: .9
|
|
2067
|
-
};
|
|
2068
|
-
let signingPatternPromise = null;
|
|
2069
|
-
const loadSigningPatterns = async () => {
|
|
2070
|
-
const mod = await import("./signing-clauses.mjs");
|
|
2071
|
-
return buildSigningClausePatterns(mod.default ?? mod);
|
|
2072
|
-
};
|
|
2073
|
-
const getSigningClausePatterns = () => {
|
|
2074
|
-
if (!signingPatternPromise) signingPatternPromise = loadSigningPatterns().catch((err) => {
|
|
2075
|
-
signingPatternPromise = null;
|
|
2076
|
-
throw err;
|
|
2077
|
-
});
|
|
2078
|
-
return signingPatternPromise;
|
|
2079
|
-
};
|
|
2080
|
-
//#endregion
|
|
2081
|
-
//#region src/detectors/legal-forms.ts
|
|
2082
|
-
const UPPER = "A-ZÁČĎÉĚÍŇÓŘŠŤÚŮÝŽÄÖÜÀÂÆÇÈÊËÎÏÔÙÛŸÑ\\u0130";
|
|
2083
|
-
const LOWER = "a-záčďéěíňóřšťúůýžäöüßàâæçèêëîïôùûÿñ\\u0131";
|
|
2084
|
-
const CAP_WORD = `(?:[${UPPER}]{2,}|[${UPPER}][${LOWER}${UPPER}]+)`;
|
|
2085
|
-
const ANY_WORD = `(?:[${UPPER}${LOWER}][${LOWER}${UPPER}]+|[${UPPER}]{2,3}|\\d{1,4})`;
|
|
2086
|
-
const ALLCAP_WORD = `[${UPPER}]{2,}`;
|
|
2087
|
-
const ROMAN_NUMERAL_RE = /^(?=[IVXLCDM])M{0,3}(?:CM|CD|D?C{0,3})(?:XC|XL|L?X{0,3})(?:IX|IV|V?I{0,3})$/;
|
|
2088
|
-
const escapeForRegex = (form) => form.replace(/[.*+?^${}()|[\]\\]/g, "\\$&").replace(/\s+/g, "\\s+").replace(/\\\./g, "\\.[^\\S\\n]?");
|
|
2089
|
-
const isShortForm = (form) => form.replace(/[.\s]/g, "").length <= 3 && !form.includes(" ");
|
|
2090
|
-
const buildPatternString = (forms) => {
|
|
2091
|
-
if (forms.length === 0) return null;
|
|
2092
|
-
const alt = forms.toSorted((a, b) => b.length - a.length).map(escapeForRegex).join("|");
|
|
2093
|
-
return `${`(?:${CAP_WORD})(?:${`(?:[\\s&,.${DASH_INNER}]{1,4}|${`\\s+(?:a|and|und|et|e|y|i)\\s+(?=[${LOWER}])`})`}(?:${ANY_WORD})){0,10}`}(?:\\s+|,\\s*)(?:${alt})(?![${LOWER}])`;
|
|
2094
|
-
};
|
|
2095
|
-
/**
|
|
2096
|
-
* Build legal form regex pattern strings.
|
|
2097
|
-
* Returns an array of regex strings for the unified
|
|
2098
|
-
* TextSearch builder. Empty if data package is not
|
|
2099
|
-
* installed.
|
|
2100
|
-
*/
|
|
2101
|
-
const buildLegalFormPatterns = async () => {
|
|
2102
|
-
let data = {};
|
|
2103
|
-
try {
|
|
2104
|
-
data = (await import("./legal-forms.mjs")).default;
|
|
2105
|
-
} catch {
|
|
2106
|
-
return [];
|
|
2107
|
-
}
|
|
2108
|
-
const allForms = [];
|
|
2109
|
-
const seen = /* @__PURE__ */ new Set();
|
|
2110
|
-
for (const forms of Object.values(data)) for (const form of forms) {
|
|
2111
|
-
const key = form.toLowerCase();
|
|
2112
|
-
if (!seen.has(key)) {
|
|
2113
|
-
seen.add(key);
|
|
2114
|
-
allForms.push(form);
|
|
2115
|
-
}
|
|
2116
|
-
}
|
|
2117
|
-
const patterns = [];
|
|
2118
|
-
const longPattern = buildPatternString(allForms.filter((f) => !isShortForm(f)));
|
|
2119
|
-
if (longPattern) patterns.push(longPattern);
|
|
2120
|
-
const shortPattern = buildPatternString(allForms.filter(isShortForm));
|
|
2121
|
-
if (shortPattern) patterns.push(shortPattern);
|
|
2122
|
-
const allcapPrefix = `(?:${ALLCAP_WORD})(?:[\\s&,.${DASH_INNER}]{1,4}(?:${ALLCAP_WORD})){0,2}`;
|
|
2123
|
-
const allcapAlt = allForms.toSorted((a, b) => b.length - a.length).map(escapeForRegex).join("|");
|
|
2124
|
-
patterns.push(`${allcapPrefix}(?:\\s+|,\\s*)(?:${allcapAlt})(?![${LOWER}])`);
|
|
2125
|
-
return patterns;
|
|
2126
|
-
};
|
|
2127
|
-
const CONNECTOR_RE = /^(?:a|and|und|et|e|y|i|&)$/i;
|
|
2128
|
-
const LEADING_CLAUSE_RE = /(?:^|\s)(?:by\s+and\s+between|is\s+between)\s+/giu;
|
|
2129
|
-
/**
|
|
2130
|
-
* Find the word ending just before `pos` in `text`,
|
|
2131
|
-
* skipping any whitespace (not newlines).
|
|
2132
|
-
* Returns null if no word is found (e.g., at start
|
|
2133
|
-
* of text, or preceded by non-word chars like ".").
|
|
2134
|
-
*/
|
|
2135
|
-
const findWordBefore = (text, pos) => {
|
|
2136
|
-
let scan = pos - 1;
|
|
2137
|
-
while (scan >= 0) {
|
|
2138
|
-
const ch = text.charAt(scan);
|
|
2139
|
-
if (ch === "\n" || !/\s/.test(ch)) break;
|
|
2140
|
-
scan--;
|
|
2141
|
-
}
|
|
2142
|
-
if (scan < 0 || text.charAt(scan) === "\n") return null;
|
|
2143
|
-
const wordEnd = scan + 1;
|
|
2144
|
-
while (scan >= 0 && /[\p{L}\p{M}&]/u.test(text.charAt(scan))) scan--;
|
|
2145
|
-
const wordStart = scan + 1;
|
|
2146
|
-
const word = text.slice(wordStart, wordEnd);
|
|
2147
|
-
if (word.length === 0) return null;
|
|
2148
|
-
return {
|
|
2149
|
-
word,
|
|
2150
|
-
start: wordStart
|
|
2151
|
-
};
|
|
2152
|
-
};
|
|
2153
|
-
/**
|
|
2154
|
-
* Extend a match backward through uppercase words and
|
|
2155
|
-
* lowercase connectors. Stops at start of text,
|
|
2156
|
-
* newline, or a word that doesn't qualify.
|
|
2157
|
-
*
|
|
2158
|
-
* Connectors (a, and, und, et, ...) are only consumed
|
|
2159
|
-
* when there is a valid word before them — a trailing
|
|
2160
|
-
* connector at an entity boundary is not consumed.
|
|
2161
|
-
*/
|
|
2162
|
-
const extendBackward = (fullText, matchStart) => {
|
|
2163
|
-
let pos = matchStart;
|
|
2164
|
-
while (pos > 0) {
|
|
2165
|
-
const found = findWordBefore(fullText, pos);
|
|
2166
|
-
if (!found) break;
|
|
2167
|
-
const { word, start: wordStart } = found;
|
|
2168
|
-
const isUpper = /^\p{Lu}/u.test(word);
|
|
2169
|
-
const isConnector = CONNECTOR_RE.test(word);
|
|
2170
|
-
if (isUpper) pos = wordStart;
|
|
2171
|
-
else if (isConnector) {
|
|
2172
|
-
const prev = findWordBefore(fullText, wordStart);
|
|
2173
|
-
if (!prev) break;
|
|
2174
|
-
if (!/^\p{Lu}/u.test(prev.word)) break;
|
|
2175
|
-
pos = prev.start;
|
|
2176
|
-
} else break;
|
|
2177
|
-
}
|
|
2178
|
-
return pos;
|
|
2179
|
-
};
|
|
2180
|
-
const trimLeadingClause = (text) => {
|
|
2181
|
-
let cut = -1;
|
|
2182
|
-
for (const match of text.matchAll(LEADING_CLAUSE_RE)) cut = match.index + match[0].length;
|
|
2183
|
-
if (cut <= 0) return {
|
|
2184
|
-
offset: 0,
|
|
2185
|
-
text
|
|
2186
|
-
};
|
|
2187
|
-
const trimmed = text.slice(cut);
|
|
2188
|
-
const leadingWs = trimmed.match(/^\s*/u)?.[0].length ?? 0;
|
|
2189
|
-
return {
|
|
2190
|
-
offset: cut + leadingWs,
|
|
2191
|
-
text: trimmed.slice(leadingWs)
|
|
2192
|
-
};
|
|
2193
|
-
};
|
|
2194
|
-
/**
|
|
2195
|
-
* Process legal form matches from the unified search.
|
|
2196
|
-
* Receives all matches; filters to the legal forms
|
|
2197
|
-
* slice via sliceStart/sliceEnd.
|
|
2198
|
-
*/
|
|
2199
|
-
const processLegalFormMatches = (allMatches, sliceStart, sliceEnd, fullText) => {
|
|
2200
|
-
const results = [];
|
|
2201
|
-
for (const match of allMatches) {
|
|
2202
|
-
const idx = match.pattern;
|
|
2203
|
-
if (idx < sliceStart || idx >= sliceEnd) continue;
|
|
2204
|
-
const text = match.text.trimEnd();
|
|
2205
|
-
if (text.length < 5) continue;
|
|
2206
|
-
if (text.includes("\n")) continue;
|
|
2207
|
-
let entityStart = match.start;
|
|
2208
|
-
let entityText = text;
|
|
2209
|
-
if (fullText) {
|
|
2210
|
-
const extended = extendBackward(fullText, match.start);
|
|
2211
|
-
if (extended < match.start) {
|
|
2212
|
-
entityStart = extended;
|
|
2213
|
-
entityText = fullText.slice(extended, match.start + text.length).trimEnd();
|
|
2214
|
-
}
|
|
2215
|
-
}
|
|
2216
|
-
const clauseTrim = trimLeadingClause(entityText);
|
|
2217
|
-
if (clauseTrim.offset > 0) {
|
|
2218
|
-
entityStart += clauseTrim.offset;
|
|
2219
|
-
entityText = clauseTrim.text;
|
|
2220
|
-
}
|
|
2221
|
-
const getPrefixInfo = (value) => {
|
|
2222
|
-
const prefixEnd = value.lastIndexOf(",") !== -1 ? value.lastIndexOf(",") : value.lastIndexOf(" ");
|
|
2223
|
-
return {
|
|
2224
|
-
prefixEnd,
|
|
2225
|
-
prefixPart: prefixEnd > 0 ? value.slice(0, prefixEnd).replace(/[^a-zA-ZÀ-ž]/g, "") : value.replace(/[^a-zA-ZÀ-ž]/g, "")
|
|
2226
|
-
};
|
|
2227
|
-
};
|
|
2228
|
-
let { prefixEnd, prefixPart } = getPrefixInfo(entityText);
|
|
2229
|
-
let isAllCapsMatch = prefixPart.length > 2 && prefixPart === prefixPart.toUpperCase();
|
|
2230
|
-
if (isAllCapsMatch && fullText) {
|
|
2231
|
-
const lineStart = fullText.lastIndexOf("\n", entityStart);
|
|
2232
|
-
const lineEnd = fullText.indexOf("\n", entityStart + entityText.length);
|
|
2233
|
-
const lineLetters = fullText.slice(lineStart + 1, lineEnd === -1 ? fullText.length : lineEnd).replace(/[^a-zA-ZÀ-ž]/g, "");
|
|
2234
|
-
const upperCount = [...lineLetters].filter((c) => c === c.toUpperCase()).length;
|
|
2235
|
-
if (lineLetters.length > 5 && upperCount / lineLetters.length >= .95) continue;
|
|
2236
|
-
if ((prefixPart.length > 0 ? entityText.slice(0, prefixEnd > 0 ? prefixEnd : entityText.length).trim().split(/\s+/).length : 0) > 3) {
|
|
2237
|
-
entityStart = match.start;
|
|
2238
|
-
entityText = text;
|
|
2239
|
-
({prefixEnd, prefixPart} = getPrefixInfo(entityText));
|
|
2240
|
-
isAllCapsMatch = prefixPart.length > 2 && prefixPart === prefixPart.toUpperCase();
|
|
2241
|
-
}
|
|
2242
|
-
} else if (isAllCapsMatch) continue;
|
|
2243
|
-
const lastSpace = entityText.lastIndexOf(" ");
|
|
2244
|
-
const rawSuffix = lastSpace !== -1 ? entityText.slice(lastSpace + 1) : "";
|
|
2245
|
-
const suffixClean = rawSuffix.replace(/[.,]/g, "");
|
|
2246
|
-
if (suffixClean.length > 0 && ROMAN_NUMERAL_RE.test(suffixClean)) continue;
|
|
2247
|
-
if (suffixClean.length <= 2 && !/\./.test(rawSuffix) && /[^\x00-\x7F]/.test(entityText.slice(0, lastSpace !== -1 ? lastSpace : entityText.length))) continue;
|
|
2248
|
-
results.push({
|
|
2249
|
-
start: entityStart,
|
|
2250
|
-
end: entityStart + entityText.length,
|
|
2251
|
-
label: "organization",
|
|
2252
|
-
text: entityText,
|
|
2253
|
-
score: .95,
|
|
2254
|
-
source: DETECTION_SOURCES.LEGAL_FORM
|
|
2255
|
-
});
|
|
2621
|
+
const meta = meta_[idx - sliceStart];
|
|
2622
|
+
if (!meta) continue;
|
|
2623
|
+
if (meta.sourceDetail !== "custom-regex" && meta.label === "phone number" && match.text.length < MIN_PHONE_LENGTH) continue;
|
|
2624
|
+
if (meta.validator) {
|
|
2625
|
+
const compacted = meta.validator.compact(match.text);
|
|
2626
|
+
if (!meta.validator.validate(compacted).valid) continue;
|
|
2627
|
+
}
|
|
2628
|
+
const entity = {
|
|
2629
|
+
start: match.start,
|
|
2630
|
+
end: match.end,
|
|
2631
|
+
label: meta.label,
|
|
2632
|
+
text: match.text,
|
|
2633
|
+
score: meta.score,
|
|
2634
|
+
source: DETECTION_SOURCES.REGEX
|
|
2635
|
+
};
|
|
2636
|
+
if (meta.sourceDetail) entity.sourceDetail = meta.sourceDetail;
|
|
2637
|
+
results.push(entity);
|
|
2256
2638
|
}
|
|
2257
2639
|
return results;
|
|
2258
2640
|
};
|
|
2259
|
-
//#endregion
|
|
2260
|
-
//#region src/config/legal-forms.ts
|
|
2261
2641
|
/**
|
|
2262
|
-
*
|
|
2263
|
-
*
|
|
2264
|
-
*
|
|
2265
|
-
*
|
|
2266
|
-
*
|
|
2267
|
-
*
|
|
2642
|
+
* Build signing clause place-name patterns from
|
|
2643
|
+
* signing-clauses.json. Each pattern captures the
|
|
2644
|
+
* city/place name from contract signing locations.
|
|
2645
|
+
*
|
|
2646
|
+
* The place name sub-pattern:
|
|
2647
|
+
* \p{Lu}\p{Ll}+ (capitalized word)
|
|
2648
|
+
* optionally followed by preposition + capitalized
|
|
2649
|
+
* word (for "nad Nisou", "am Main", etc.)
|
|
2650
|
+
* optionally followed by more capitalized words
|
|
2651
|
+
* (for "Hradec Králové", "New York", etc.)
|
|
2268
2652
|
*/
|
|
2269
|
-
const
|
|
2270
|
-
|
|
2271
|
-
|
|
2272
|
-
|
|
2273
|
-
|
|
2274
|
-
|
|
2275
|
-
|
|
2276
|
-
|
|
2277
|
-
|
|
2278
|
-
|
|
2279
|
-
|
|
2280
|
-
|
|
2281
|
-
|
|
2282
|
-
|
|
2283
|
-
|
|
2284
|
-
|
|
2285
|
-
"
|
|
2286
|
-
|
|
2287
|
-
|
|
2288
|
-
|
|
2289
|
-
|
|
2290
|
-
|
|
2291
|
-
|
|
2292
|
-
|
|
2293
|
-
|
|
2294
|
-
|
|
2295
|
-
"LLP",
|
|
2296
|
-
"Inc.",
|
|
2297
|
-
"S.A.",
|
|
2298
|
-
"SA",
|
|
2299
|
-
"SAS",
|
|
2300
|
-
"SARL",
|
|
2301
|
-
"Sp. z o.o.",
|
|
2302
|
-
"S.p.A."
|
|
2303
|
-
];
|
|
2653
|
+
const buildSigningClausePatterns = (data) => {
|
|
2654
|
+
const patterns = [];
|
|
2655
|
+
for (const entry of data.patterns) {
|
|
2656
|
+
const prepAlt = entry.prepositions.length > 0 ? entry.prepositions.join("|") : null;
|
|
2657
|
+
const place = prepAlt ? `(\\p{Lu}\\p{Ll}+(?:\\s+(?:${prepAlt})\\s+\\p{Lu}\\p{Ll}+)*(?:\\s+\\p{Lu}\\p{Ll}+)*)` : `(\\p{Lu}\\p{Ll}+(?:[- ]\\p{Lu}\\p{Ll}+)*)`;
|
|
2658
|
+
const full = `(?:^|\\n|[^\\S\\n])` + entry.prefix + place + (entry.suffix ? `(?:${entry.suffix})` : "");
|
|
2659
|
+
patterns.push(full);
|
|
2660
|
+
}
|
|
2661
|
+
return patterns;
|
|
2662
|
+
};
|
|
2663
|
+
const SIGNING_CLAUSE_META = {
|
|
2664
|
+
label: "address",
|
|
2665
|
+
score: .9
|
|
2666
|
+
};
|
|
2667
|
+
let signingPatternPromise = null;
|
|
2668
|
+
const loadSigningPatterns = async () => {
|
|
2669
|
+
const mod = await import("./signing-clauses.mjs");
|
|
2670
|
+
return buildSigningClausePatterns(mod.default ?? mod);
|
|
2671
|
+
};
|
|
2672
|
+
const getSigningClausePatterns = () => {
|
|
2673
|
+
if (!signingPatternPromise) signingPatternPromise = loadSigningPatterns().catch((err) => {
|
|
2674
|
+
signingPatternPromise = null;
|
|
2675
|
+
throw err;
|
|
2676
|
+
});
|
|
2677
|
+
return signingPatternPromise;
|
|
2678
|
+
};
|
|
2304
2679
|
//#endregion
|
|
2305
2680
|
//#region src/detectors/triggers.ts
|
|
2681
|
+
const VALID_ID_VALIDATORS = {
|
|
2682
|
+
"br.cpf": br.cpf,
|
|
2683
|
+
"br.cnpj": br.cnpj
|
|
2684
|
+
};
|
|
2306
2685
|
const TRIGGER_SCORE = .95;
|
|
2307
2686
|
const WHITESPACE_RE$1 = /\s+/;
|
|
2308
2687
|
const LETTER_RE = /\p{L}/u;
|
|
@@ -2317,12 +2696,23 @@ const DECIMAL_COMMA_RE = new RegExp(`^,(?:\\d|${DASH}{1,2})`);
|
|
|
2317
2696
|
* etc.), skip the comma and degree, then continue.
|
|
2318
2697
|
*/
|
|
2319
2698
|
const POST_NOMINAL_RE = new RegExp(`^,\\s*(?:${POST_NOMINALS.toSorted((a, b) => b.length - a.length).map((d) => d.replace(/[.*+?^${}()|[\]\\]/g, "\\$&").replace(/\\\./g, "\\.\\s*")).join("|")})\\.?`, "i");
|
|
2320
|
-
const
|
|
2321
|
-
const
|
|
2322
|
-
|
|
2323
|
-
|
|
2324
|
-
|
|
2325
|
-
|
|
2699
|
+
const buildLegalFormCheckRe = (forms) => {
|
|
2700
|
+
const parts = forms.map((f) => {
|
|
2701
|
+
const escaped = f.replace(/[.*+?^${}()|[\]\\]/g, "\\$&").replace(/\\\./g, "\\.\\s*");
|
|
2702
|
+
return /^[\p{L}\p{M}]+$/u.test(f) ? `(?<![\\p{L}\\p{N}])${escaped}(?![\\p{L}\\p{N}])` : escaped;
|
|
2703
|
+
});
|
|
2704
|
+
return new RegExp(parts.join("|"), "u");
|
|
2705
|
+
};
|
|
2706
|
+
let cachedLegalFormCheckRe = null;
|
|
2707
|
+
let cachedLegalFormCheckSource = null;
|
|
2708
|
+
const getLegalFormCheckRe = () => {
|
|
2709
|
+
const source = getKnownLegalSuffixes();
|
|
2710
|
+
if (cachedLegalFormCheckSource !== source || cachedLegalFormCheckRe === null) {
|
|
2711
|
+
cachedLegalFormCheckSource = source;
|
|
2712
|
+
cachedLegalFormCheckRe = buildLegalFormCheckRe(source);
|
|
2713
|
+
}
|
|
2714
|
+
return cachedLegalFormCheckRe;
|
|
2715
|
+
};
|
|
2326
2716
|
const compileValidations = (validations) => validations.map((v) => {
|
|
2327
2717
|
switch (v.type) {
|
|
2328
2718
|
case "starts-uppercase": return {
|
|
@@ -2349,6 +2739,17 @@ const compileValidations = (validations) => validations.map((v) => {
|
|
|
2349
2739
|
type: "matches-pattern",
|
|
2350
2740
|
re: new RegExp(v.pattern, (v.flags ?? "").replace(/[gy]/g, ""))
|
|
2351
2741
|
};
|
|
2742
|
+
case "valid-id": {
|
|
2743
|
+
const validator = VALID_ID_VALIDATORS[v.validator];
|
|
2744
|
+
if (!validator) throw new Error(`Unknown valid-id validator: ${JSON.stringify(v.validator)}`);
|
|
2745
|
+
return {
|
|
2746
|
+
type: "valid-id",
|
|
2747
|
+
check: (value) => {
|
|
2748
|
+
const compact = value.replace(/[\s.\-/]/g, "");
|
|
2749
|
+
return validator.validate(compact).valid;
|
|
2750
|
+
}
|
|
2751
|
+
};
|
|
2752
|
+
}
|
|
2352
2753
|
default: throw new Error(`Unknown validation type: ${JSON.stringify(v)}`);
|
|
2353
2754
|
}
|
|
2354
2755
|
});
|
|
@@ -2372,6 +2773,9 @@ const applyValidations = (text, validations) => {
|
|
|
2372
2773
|
case "matches-pattern":
|
|
2373
2774
|
if (!v.re.test(text)) return false;
|
|
2374
2775
|
break;
|
|
2776
|
+
case "valid-id":
|
|
2777
|
+
if (!v.check(text)) return false;
|
|
2778
|
+
break;
|
|
2375
2779
|
default: throw new Error(`Unknown compiled validation type: ${JSON.stringify(v)}`);
|
|
2376
2780
|
}
|
|
2377
2781
|
return true;
|
|
@@ -2466,8 +2870,10 @@ const buildTriggerPatterns = async () => {
|
|
|
2466
2870
|
strategy: rule.strategy.type
|
|
2467
2871
|
});
|
|
2468
2872
|
}
|
|
2873
|
+
const patterns = rules.map((r) => r.trigger.toLowerCase());
|
|
2874
|
+
await loadAddressStopKeywords();
|
|
2469
2875
|
return {
|
|
2470
|
-
patterns
|
|
2876
|
+
patterns,
|
|
2471
2877
|
rules
|
|
2472
2878
|
};
|
|
2473
2879
|
};
|
|
@@ -2488,14 +2894,103 @@ const stripQuotes = (value) => {
|
|
|
2488
2894
|
const COMMA_STOP_CHARS = new Set([
|
|
2489
2895
|
"\n",
|
|
2490
2896
|
"(",
|
|
2491
|
-
" "
|
|
2897
|
+
" ",
|
|
2898
|
+
";"
|
|
2492
2899
|
]);
|
|
2493
2900
|
/**
|
|
2901
|
+
* Sentence-terminator detection: a period that genuinely ends
|
|
2902
|
+
* one clause and starts another. Used by `to-next-comma` so
|
|
2903
|
+
* governing-law clauses ("…State of New York. SECTION 2…")
|
|
2904
|
+
* don't sweep across the period.
|
|
2905
|
+
*
|
|
2906
|
+
* Three positive signals (any one terminates):
|
|
2907
|
+
* 1. Long lowercase tail before the dot (>= 5 letters) —
|
|
2908
|
+
* catches "…construction. SECTION 2…", "…vykonává.".
|
|
2909
|
+
* 2. Currency/amount tail (zł, Kč, USD, €) — catches
|
|
2910
|
+
* "…w kwocie 1000 zł. Termin płatności…".
|
|
2911
|
+
* 3. Proper-noun head: a capitalized word of >= 4 letters
|
|
2912
|
+
* ending in lowercase, with no internal uppercase, AND
|
|
2913
|
+
* a substantial next clause (Capital + >= 2 lowercase).
|
|
2914
|
+
* Catches short city names that the lowercase-tail rule
|
|
2915
|
+
* misses: "…z siedzibą w Łódź. Kapitał zakładowy…",
|
|
2916
|
+
* "…seat in Brno. Section 2…".
|
|
2917
|
+
*
|
|
2918
|
+
* The rule must NOT fire on:
|
|
2919
|
+
* - title abbreviations: "Mr.", "Mrs.", "Dr.", "Hon.",
|
|
2920
|
+
* "Sr.", "Jr." (head <= 3 chars or insufficient Ll tail)
|
|
2921
|
+
* - degree abbreviations: "Ph.D.", "RNDr.", "MUDr.",
|
|
2922
|
+
* "Ing." (internal periods or internal uppercase block
|
|
2923
|
+
* the proper-noun pattern)
|
|
2924
|
+
* - street-type abbreviations: "Ste.", "Ave.", "Inc.",
|
|
2925
|
+
* "ul.", "al.", "nábř." (lowercase initials or
|
|
2926
|
+
* insufficient Ll tail)
|
|
2927
|
+
* - small-word lowercase abbreviations: "prof.", "inż.",
|
|
2928
|
+
* "hab." (no leading uppercase, so the proper-noun rule
|
|
2929
|
+
* can't fire)
|
|
2930
|
+
*/
|
|
2931
|
+
const NEXT_IS_SENTENCE_START_RE = /^\.(?:\s+\p{Lu}|\s*$)/u;
|
|
2932
|
+
const SENTENCE_TAIL_RE = /\p{Ll}{5,}$/u;
|
|
2933
|
+
/**
|
|
2934
|
+
* Proper-noun tail: capital letter + >= 3 lowercase letters,
|
|
2935
|
+
* preceded by a non-letter and non-period (so we don't slice
|
|
2936
|
+
* into the middle of an acronym or a multi-dot abbreviation).
|
|
2937
|
+
* The 4-character minimum excludes 2–3-char titles ("Mr",
|
|
2938
|
+
* "Mrs", "Dr", "Inc", "Ste"); the all-lowercase tail
|
|
2939
|
+
* excludes mixed-case degrees ("RNDr", "MUDr").
|
|
2940
|
+
*/
|
|
2941
|
+
const PROPER_NOUN_HEAD_RE = /(?:^|[^\p{L}.])\p{Lu}\p{Ll}{3,}$/u;
|
|
2942
|
+
/**
|
|
2943
|
+
* Next clause begins with a real word: capital + >= 2
|
|
2944
|
+
* lowercase letters. Filters cases where a capitalized
|
|
2945
|
+
* abbreviation (e.g., "Smith Inc.") follows a proper noun,
|
|
2946
|
+
* which would otherwise look sentence-like.
|
|
2947
|
+
*/
|
|
2948
|
+
const NEXT_IS_REAL_SENTENCE_RE = /^\.\s+\p{Lu}\p{Ll}{2,}/u;
|
|
2949
|
+
/**
|
|
2950
|
+
* Short currency-abbreviation tail (zł, Kč, gr, Ft, kr,
|
|
2951
|
+
* лв, USD, PLN, EUR, …). When a period follows one of
|
|
2952
|
+
* these and the next token starts uppercase, treat it as
|
|
2953
|
+
* a sentence boundary even though the tail is too short
|
|
2954
|
+
* to satisfy `SENTENCE_TAIL_RE`. Without this, amount
|
|
2955
|
+
* triggers using `to-next-comma` swallow the following
|
|
2956
|
+
* clause: `"w kwocie 1000 zł. Termin płatności…"`.
|
|
2957
|
+
*
|
|
2958
|
+
* Currency codes are typically uppercase (`USD`, `PLN`)
|
|
2959
|
+
* but appear lowercased in informal writing (`pln`, `eur`);
|
|
2960
|
+
* local names mix case (`zł`, `Kč`, `Ft`); symbols
|
|
2961
|
+
* (`€`, `$`, `£`) appear after the amount as well. The
|
|
2962
|
+
* negative lookbehind on a letter ensures the abbreviation
|
|
2963
|
+
* is matched only as a standalone token; symbols are
|
|
2964
|
+
* matched unconditionally since they are not letters.
|
|
2965
|
+
*/
|
|
2966
|
+
const CURRENCY_TAIL_RE = /(?:(?<![\p{L}])(?:zł|Kč|gr|Ft|kr|лв|USD|PLN|EUR|CZK|GBP|CHF|HUF|RON|SEK|NOK|DKK)|[€$£])$/iu;
|
|
2967
|
+
const NUMERIC_SENTENCE_TAIL_RE = /\d$/;
|
|
2968
|
+
const isSentenceTerminator = (text, periodIndex) => {
|
|
2969
|
+
const tail = text.slice(periodIndex);
|
|
2970
|
+
if (!NEXT_IS_SENTENCE_START_RE.test(tail)) return false;
|
|
2971
|
+
const head = text.slice(0, periodIndex);
|
|
2972
|
+
if (SENTENCE_TAIL_RE.test(head) || CURRENCY_TAIL_RE.test(head) || NUMERIC_SENTENCE_TAIL_RE.test(head)) return true;
|
|
2973
|
+
return PROPER_NOUN_HEAD_RE.test(head) && NEXT_IS_REAL_SENTENCE_RE.test(tail);
|
|
2974
|
+
};
|
|
2975
|
+
/**
|
|
2494
2976
|
* Field-label keywords that terminate address scanning.
|
|
2495
2977
|
* When a comma in the address strategy is followed by
|
|
2496
2978
|
* one of these, the address stops before the keyword.
|
|
2979
|
+
*
|
|
2980
|
+
* The list is sourced from
|
|
2981
|
+
* `data/address-stop-keywords.json` (per-language so new
|
|
2982
|
+
* languages can drop in their own labels without
|
|
2983
|
+
* touching this file). `loadAddressStopKeywords` unions
|
|
2984
|
+
* every language into a single longest-first array;
|
|
2985
|
+
* `getAddressStopKeywordsSync` returns the cached union
|
|
2986
|
+
* and falls back to a seed list so the strategy keeps
|
|
2987
|
+
* working before the warmup promise resolves.
|
|
2988
|
+
*
|
|
2989
|
+
* The address strategy doesn't know the document
|
|
2990
|
+
* language, so a flat union is intentional: any
|
|
2991
|
+
* language's labels can appear in any address.
|
|
2497
2992
|
*/
|
|
2498
|
-
const
|
|
2993
|
+
const ADDRESS_STOP_KEYWORDS_SEED = [
|
|
2499
2994
|
"číslo účtu",
|
|
2500
2995
|
"registrační",
|
|
2501
2996
|
"zastoupen",
|
|
@@ -2518,6 +3013,52 @@ const ADDRESS_STOP_KEYWORDS = [
|
|
|
2518
3013
|
"bic",
|
|
2519
3014
|
"ič"
|
|
2520
3015
|
];
|
|
3016
|
+
let addressStopKeywordsCache = null;
|
|
3017
|
+
let addressStopKeywordsPromise = null;
|
|
3018
|
+
const loadAddressStopKeywords = async () => {
|
|
3019
|
+
if (addressStopKeywordsCache) return addressStopKeywordsCache;
|
|
3020
|
+
if (addressStopKeywordsPromise) return addressStopKeywordsPromise;
|
|
3021
|
+
addressStopKeywordsPromise = (async () => {
|
|
3022
|
+
let data = {};
|
|
3023
|
+
try {
|
|
3024
|
+
const mod = await import("./address-stop-keywords.mjs");
|
|
3025
|
+
data = mod.default ?? mod;
|
|
3026
|
+
} catch (err) {
|
|
3027
|
+
console.warn("[anonymize] triggers: failed to load address-stop-keywords.json, falling back to seed list:", err);
|
|
3028
|
+
}
|
|
3029
|
+
const seen = /* @__PURE__ */ new Set();
|
|
3030
|
+
const out = [];
|
|
3031
|
+
const addAll = (list) => {
|
|
3032
|
+
for (const kw of list) {
|
|
3033
|
+
if (typeof kw !== "string" || kw.length === 0) continue;
|
|
3034
|
+
const lower = kw.toLowerCase();
|
|
3035
|
+
if (seen.has(lower)) continue;
|
|
3036
|
+
seen.add(lower);
|
|
3037
|
+
out.push(lower);
|
|
3038
|
+
}
|
|
3039
|
+
};
|
|
3040
|
+
for (const [key, value] of Object.entries(data)) {
|
|
3041
|
+
if (key.startsWith("_")) continue;
|
|
3042
|
+
if (!Array.isArray(value)) continue;
|
|
3043
|
+
addAll(value);
|
|
3044
|
+
}
|
|
3045
|
+
addAll(ADDRESS_STOP_KEYWORDS_SEED);
|
|
3046
|
+
out.sort((a, b) => b.length - a.length);
|
|
3047
|
+
addressStopKeywordsCache = out;
|
|
3048
|
+
return out;
|
|
3049
|
+
})();
|
|
3050
|
+
return addressStopKeywordsPromise;
|
|
3051
|
+
};
|
|
3052
|
+
const getAddressStopKeywordsSync = () => addressStopKeywordsCache ?? ADDRESS_STOP_KEYWORDS_SEED;
|
|
3053
|
+
/**
|
|
3054
|
+
* Warm the address-stop-keywords cache. Pipeline callers
|
|
3055
|
+
* await this before invoking trigger detection so the
|
|
3056
|
+
* synchronous `extractValue` path uses the merged list
|
|
3057
|
+
* instead of the seed fallback.
|
|
3058
|
+
*/
|
|
3059
|
+
const warmAddressStopKeywords = async () => {
|
|
3060
|
+
await loadAddressStopKeywords();
|
|
3061
|
+
};
|
|
2521
3062
|
const extractValue = (text, triggerEnd, strategy, label) => {
|
|
2522
3063
|
const remaining = text.slice(triggerEnd);
|
|
2523
3064
|
const stripped = remaining.replace(/^[\s:;]+/, "");
|
|
@@ -2526,6 +3067,8 @@ const extractValue = (text, triggerEnd, strategy, label) => {
|
|
|
2526
3067
|
if (valueText.length === 0) return null;
|
|
2527
3068
|
switch (strategy.type) {
|
|
2528
3069
|
case "to-next-comma": {
|
|
3070
|
+
const stopWords = strategy.stopWords ?? [];
|
|
3071
|
+
const stopWordRe = stopWords.length > 0 ? new RegExp(`^(?:${stopWords.map((w) => w.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")).join("|")})(?![\\p{L}\\p{N}])`, "iu") : null;
|
|
2529
3072
|
let end = 0;
|
|
2530
3073
|
let foundStop = false;
|
|
2531
3074
|
while (end < valueText.length) {
|
|
@@ -2534,6 +3077,14 @@ const extractValue = (text, triggerEnd, strategy, label) => {
|
|
|
2534
3077
|
foundStop = true;
|
|
2535
3078
|
break;
|
|
2536
3079
|
}
|
|
3080
|
+
if (ch === "." && isSentenceTerminator(valueText, end)) {
|
|
3081
|
+
foundStop = true;
|
|
3082
|
+
break;
|
|
3083
|
+
}
|
|
3084
|
+
if (stopWordRe !== null && (end === 0 || !/[\p{L}\p{N}]/u.test(valueText[end - 1] ?? "")) && stopWordRe.test(valueText.slice(end))) {
|
|
3085
|
+
foundStop = true;
|
|
3086
|
+
break;
|
|
3087
|
+
}
|
|
2537
3088
|
if (ch === ",") {
|
|
2538
3089
|
const afterComma = valueText.slice(end);
|
|
2539
3090
|
if (DECIMAL_COMMA_RE.test(afterComma)) {
|
|
@@ -2562,6 +3113,8 @@ const extractValue = (text, triggerEnd, strategy, label) => {
|
|
|
2562
3113
|
};
|
|
2563
3114
|
}
|
|
2564
3115
|
case "to-end-of-line": {
|
|
3116
|
+
const consumed = remaining.length - valueText.length;
|
|
3117
|
+
if (consumed > 0 && remaining.slice(0, consumed).includes("\n")) return null;
|
|
2565
3118
|
const LINE_STOPS = ["\n"];
|
|
2566
3119
|
let end = valueText.length;
|
|
2567
3120
|
for (const ch of LINE_STOPS) {
|
|
@@ -2582,7 +3135,8 @@ const extractValue = (text, triggerEnd, strategy, label) => {
|
|
|
2582
3135
|
const tabIdx = valueText.indexOf(" ");
|
|
2583
3136
|
const cellText = tabIdx !== -1 ? valueText.slice(0, tabIdx) : valueText;
|
|
2584
3137
|
const PUNCT_ONLY = /^[\p{P}\p{S}]+$/u;
|
|
2585
|
-
const
|
|
3138
|
+
const NUMBER_MARKER = /^(?:n[ºo°.]|№)$/i;
|
|
3139
|
+
const words = cellText.split(WHITESPACE_RE$1).filter((w) => !PUNCT_ONLY.test(w) && !NUMBER_MARKER.test(w)).slice(0, strategy.count);
|
|
2586
3140
|
if (words.length === 0) return null;
|
|
2587
3141
|
const firstWord = words[0];
|
|
2588
3142
|
if (firstWord === void 0) return null;
|
|
@@ -2605,14 +3159,22 @@ const extractValue = (text, triggerEnd, strategy, label) => {
|
|
|
2605
3159
|
}
|
|
2606
3160
|
case "company-id-value": {
|
|
2607
3161
|
const raw = text.slice(triggerEnd);
|
|
2608
|
-
const
|
|
3162
|
+
const triggerLastChar = text[triggerEnd - 1] ?? "";
|
|
3163
|
+
const sepMatch = (triggerLastChar === "°" || triggerLastChar === "º" || triggerLastChar === "№" || triggerLastChar === "#" ? /^(?:\s*:\s*|\s+|)/ : /^(?:\s*:\s*|\s+)/).exec(raw);
|
|
2609
3164
|
if (!sepMatch) return null;
|
|
2610
|
-
|
|
2611
|
-
const
|
|
3165
|
+
let afterSep = raw.slice(sepMatch[0].length);
|
|
3166
|
+
const labelMatch = /^(?:nr\.?|numer|n[ºo°.]|№|no\.?)(?:\s*:\s*|\s+)/i.exec(afterSep);
|
|
3167
|
+
let labelOffset = 0;
|
|
3168
|
+
if (labelMatch) {
|
|
3169
|
+
labelOffset = labelMatch[0].length;
|
|
3170
|
+
afterSep = afterSep.slice(labelOffset);
|
|
3171
|
+
}
|
|
3172
|
+
const idMatch = /^[A-Z]{0,6}\s?(?:[A-Z0-9]{2}\s?)?\d{2}(?:(?<=\d)[A-Z]|[\d\s.\-/]){3,}/i.exec(afterSep);
|
|
2612
3173
|
if (!idMatch) return null;
|
|
2613
|
-
const
|
|
3174
|
+
const trailingLetterMatch = /\d$/.test(idMatch[0]) ? /^[A-Z]/i.exec(afterSep.slice(idMatch[0].length)) : null;
|
|
3175
|
+
const idText = (trailingLetterMatch ? idMatch[0] + trailingLetterMatch[0] : idMatch[0]).trim();
|
|
2614
3176
|
const leadingSpaces = idMatch[0].length - idMatch[0].trimStart().length;
|
|
2615
|
-
const idStart = triggerEnd + sepMatch[0].length + leadingSpaces;
|
|
3177
|
+
const idStart = triggerEnd + sepMatch[0].length + labelOffset + leadingSpaces;
|
|
2616
3178
|
return {
|
|
2617
3179
|
start: idStart,
|
|
2618
3180
|
end: idStart + idText.length,
|
|
@@ -2622,18 +3184,36 @@ const extractValue = (text, triggerEnd, strategy, label) => {
|
|
|
2622
3184
|
case "address": {
|
|
2623
3185
|
const maxLen = strategy.maxChars ?? 120;
|
|
2624
3186
|
const UPPER_RE = /\p{Lu}/u;
|
|
3187
|
+
const stopKeywords = getAddressStopKeywordsSync();
|
|
2625
3188
|
let end = 0;
|
|
2626
3189
|
while (end < valueText.length && end < maxLen) {
|
|
2627
3190
|
const ch = valueText[end];
|
|
2628
3191
|
if (ch === "\n" || ch === "(") break;
|
|
3192
|
+
if (ch === " " || ch === " ") {
|
|
3193
|
+
const afterWs = valueText.slice(end).trimStart().toLowerCase();
|
|
3194
|
+
if (stopKeywords.some((kw) => {
|
|
3195
|
+
if (!afterWs.startsWith(kw)) return false;
|
|
3196
|
+
const next = afterWs[kw.length];
|
|
3197
|
+
return next === void 0 || /[\s:;.,!?()\d]/.test(next);
|
|
3198
|
+
})) break;
|
|
3199
|
+
}
|
|
2629
3200
|
if (ch === ".") {
|
|
2630
3201
|
const next = valueText[end + 1];
|
|
2631
3202
|
const afterNext = valueText[end + 2];
|
|
3203
|
+
const afterPeriod = valueText.slice(end + 1).replace(/^\s+/, "").toLowerCase();
|
|
3204
|
+
if (afterPeriod.length > 0) {
|
|
3205
|
+
if (getAddressStopKeywordsSync().some((kw) => {
|
|
3206
|
+
if (!afterPeriod.startsWith(kw)) return false;
|
|
3207
|
+
const afterKw = afterPeriod[kw.length];
|
|
3208
|
+
return afterKw === void 0 || /[\s:;.,!?()\d]/.test(afterKw);
|
|
3209
|
+
})) break;
|
|
3210
|
+
}
|
|
2632
3211
|
if (next !== void 0 && (/\p{L}/u.test(next) || /\d/.test(next))) {
|
|
2633
3212
|
end++;
|
|
2634
3213
|
continue;
|
|
2635
3214
|
}
|
|
2636
3215
|
if (next === " " && afterNext !== void 0 && (/\p{L}/u.test(afterNext) || /\d/.test(afterNext))) {
|
|
3216
|
+
if (isSentenceTerminator(valueText, end)) break;
|
|
2637
3217
|
end++;
|
|
2638
3218
|
continue;
|
|
2639
3219
|
}
|
|
@@ -2645,7 +3225,7 @@ const extractValue = (text, triggerEnd, strategy, label) => {
|
|
|
2645
3225
|
const peekCh = valueText[peek];
|
|
2646
3226
|
if (peekCh === void 0) break;
|
|
2647
3227
|
const afterComma = valueText.slice(end + 1).trimStart().toLowerCase();
|
|
2648
|
-
if (
|
|
3228
|
+
if (getAddressStopKeywordsSync().some((kw) => {
|
|
2649
3229
|
if (!afterComma.startsWith(kw)) return false;
|
|
2650
3230
|
const next = afterComma[kw.length];
|
|
2651
3231
|
return next === void 0 || /[\s:;.,!?()\d]/.test(next);
|
|
@@ -2672,6 +3252,28 @@ const extractValue = (text, triggerEnd, strategy, label) => {
|
|
|
2672
3252
|
text: extracted
|
|
2673
3253
|
};
|
|
2674
3254
|
}
|
|
3255
|
+
case "match-pattern": {
|
|
3256
|
+
const nlIdx = valueText.indexOf("\n");
|
|
3257
|
+
const searchText = nlIdx === -1 ? valueText : valueText.slice(0, nlIdx);
|
|
3258
|
+
if (searchText.length === 0) return null;
|
|
3259
|
+
const flags = (strategy.flags ?? "").replace(/[gy]/g, "");
|
|
3260
|
+
const anchoredSource = `^(?:${strategy.pattern})`;
|
|
3261
|
+
let re;
|
|
3262
|
+
try {
|
|
3263
|
+
re = new RegExp(anchoredSource, flags);
|
|
3264
|
+
} catch {
|
|
3265
|
+
return null;
|
|
3266
|
+
}
|
|
3267
|
+
const m = re.exec(searchText);
|
|
3268
|
+
if (!m || m[0].length === 0) return null;
|
|
3269
|
+
const matchStart = m.index;
|
|
3270
|
+
const matchEnd = matchStart + m[0].length;
|
|
3271
|
+
return {
|
|
3272
|
+
start: valueStart + matchStart,
|
|
3273
|
+
end: valueStart + matchEnd,
|
|
3274
|
+
text: m[0]
|
|
3275
|
+
};
|
|
3276
|
+
}
|
|
2675
3277
|
default: return null;
|
|
2676
3278
|
}
|
|
2677
3279
|
};
|
|
@@ -2691,7 +3293,8 @@ const processTriggerMatches = (allMatches, sliceStart, sliceEnd, fullText, rules
|
|
|
2691
3293
|
if (match.start > 0 && LETTER_RE.test(fullText[match.start - 1] ?? "")) continue;
|
|
2692
3294
|
const rule = rules[localIdx];
|
|
2693
3295
|
if (!rule) continue;
|
|
2694
|
-
|
|
3296
|
+
const triggerLastChar = rule.trigger.at(-1) ?? "";
|
|
3297
|
+
if (LETTER_RE.test(triggerLastChar) && LETTER_RE.test(fullText[match.end] ?? "")) continue;
|
|
2695
3298
|
const triggerEnd = match.end;
|
|
2696
3299
|
const rawValue = extractValue(fullText, triggerEnd, rule.strategy, rule.label);
|
|
2697
3300
|
const value = rawValue ? stripQuotes(rawValue) : null;
|
|
@@ -2700,7 +3303,7 @@ const processTriggerMatches = (allMatches, sliceStart, sliceEnd, fullText, rules
|
|
|
2700
3303
|
const entityStart = rule.includeTrigger ? match.start : value.start;
|
|
2701
3304
|
const entityEnd = value.end;
|
|
2702
3305
|
const entityText = fullText.slice(entityStart, entityEnd);
|
|
2703
|
-
const effectiveLabel = rule.label === "person" &&
|
|
3306
|
+
const effectiveLabel = rule.label === "person" && getLegalFormCheckRe().test(entityText) ? "organization" : rule.label;
|
|
2704
3307
|
results.push({
|
|
2705
3308
|
start: entityStart,
|
|
2706
3309
|
end: entityEnd,
|
|
@@ -2970,11 +3573,82 @@ const resolveCountries = (regions, countries) => {
|
|
|
2970
3573
|
return result;
|
|
2971
3574
|
};
|
|
2972
3575
|
//#endregion
|
|
3576
|
+
//#region src/util/homoglyphs.ts
|
|
3577
|
+
const CHAR_MAP$1 = new Map([
|
|
3578
|
+
[1040, 65],
|
|
3579
|
+
[1042, 66],
|
|
3580
|
+
[1045, 69],
|
|
3581
|
+
[1050, 75],
|
|
3582
|
+
[1052, 77],
|
|
3583
|
+
[1053, 72],
|
|
3584
|
+
[1054, 79],
|
|
3585
|
+
[1056, 80],
|
|
3586
|
+
[1057, 67],
|
|
3587
|
+
[1058, 84],
|
|
3588
|
+
[1061, 88],
|
|
3589
|
+
[1072, 97],
|
|
3590
|
+
[1077, 101],
|
|
3591
|
+
[1086, 111],
|
|
3592
|
+
[1088, 112],
|
|
3593
|
+
[1089, 99],
|
|
3594
|
+
[1091, 121],
|
|
3595
|
+
[1093, 120],
|
|
3596
|
+
[1110, 105],
|
|
3597
|
+
[1112, 106],
|
|
3598
|
+
[913, 65],
|
|
3599
|
+
[914, 66],
|
|
3600
|
+
[917, 69],
|
|
3601
|
+
[919, 72],
|
|
3602
|
+
[921, 73],
|
|
3603
|
+
[922, 75],
|
|
3604
|
+
[924, 77],
|
|
3605
|
+
[925, 78],
|
|
3606
|
+
[927, 79],
|
|
3607
|
+
[929, 80],
|
|
3608
|
+
[932, 84],
|
|
3609
|
+
[933, 89],
|
|
3610
|
+
[935, 88],
|
|
3611
|
+
[918, 90],
|
|
3612
|
+
[945, 97],
|
|
3613
|
+
[954, 107],
|
|
3614
|
+
[959, 111],
|
|
3615
|
+
[961, 112]
|
|
3616
|
+
]);
|
|
3617
|
+
/** Chunk size for `String.fromCharCode` to avoid hitting
|
|
3618
|
+
* the call-stack limit on very large strings. */
|
|
3619
|
+
const CHUNK_SIZE$1 = 8192;
|
|
3620
|
+
const normalizeHomoglyphs = (text) => {
|
|
3621
|
+
let hasSpecial = false;
|
|
3622
|
+
for (let i = 0; i < text.length; i++) if (CHAR_MAP$1.has(text.charCodeAt(i))) {
|
|
3623
|
+
hasSpecial = true;
|
|
3624
|
+
break;
|
|
3625
|
+
}
|
|
3626
|
+
if (!hasSpecial) return text;
|
|
3627
|
+
const len = text.length;
|
|
3628
|
+
const codes = new Uint16Array(len);
|
|
3629
|
+
for (let i = 0; i < len; i++) {
|
|
3630
|
+
const code = text.charCodeAt(i);
|
|
3631
|
+
codes[i] = CHAR_MAP$1.get(code) ?? code;
|
|
3632
|
+
}
|
|
3633
|
+
if (len <= CHUNK_SIZE$1) return String.fromCharCode(...codes);
|
|
3634
|
+
let result = "";
|
|
3635
|
+
for (let offset = 0; offset < len; offset += CHUNK_SIZE$1) {
|
|
3636
|
+
const end = Math.min(offset + CHUNK_SIZE$1, len);
|
|
3637
|
+
result += String.fromCharCode(...codes.subarray(offset, end));
|
|
3638
|
+
}
|
|
3639
|
+
return result;
|
|
3640
|
+
};
|
|
3641
|
+
//#endregion
|
|
2973
3642
|
//#region src/filters/false-positives.ts
|
|
2974
3643
|
const TEMPLATE_PLACEHOLDER_RE = /^(?:\.{3,}|_{3,}|\[[\w\s]+\]|\{[\w\s]+\})$/;
|
|
2975
3644
|
const POSTAL_CODE_RE = /\d{3}\s?\d{2}/;
|
|
2976
3645
|
const HAS_DIGIT_RE = /\d/;
|
|
2977
|
-
const
|
|
3646
|
+
const ADDRESS_COMPONENT_EXTRA_RE = /(?:^|\s)(?:č\.p\.|č\.ev\.|č\.|sídliště)(?=[\s,./]|$)/i;
|
|
3647
|
+
const BARE_COURS_PROSE_RE = /(?:^|\s)cours(?!\s+\p{Lu})(?=[\s,./]|$)/u;
|
|
3648
|
+
const isOnlyAmbiguousCours = (text) => {
|
|
3649
|
+
if (!BARE_COURS_PROSE_RE.test(text)) return false;
|
|
3650
|
+
return !hasAddressComponent(text.replace(/(?:^|\s)cours(?=[\s,./]|$)/gi, " "));
|
|
3651
|
+
};
|
|
2978
3652
|
const JURISDICTION_RE = /^(?:state|commonwealth|district|territory)\s+of\s+/i;
|
|
2979
3653
|
const MAX_ENTITY_LENGTH = {
|
|
2980
3654
|
organization: 80,
|
|
@@ -3057,6 +3731,41 @@ const loadGenericRoles = (ctx = defaultContext) => {
|
|
|
3057
3731
|
};
|
|
3058
3732
|
/** Sync accessor — returns empty set before init. */
|
|
3059
3733
|
const getGenericRoles = (ctx) => ctx.genericRoles ?? EMPTY_GENERIC_ROLES;
|
|
3734
|
+
const STREET_TYPES_SEED_RE = /(?:^|\s)(?:ul\.|ulice|nám\.|náměstí|tř\.|třída|nábř\.|nábřeží|bulvár)(?=[\s,./]|$)/i;
|
|
3735
|
+
let _streetTypesRe = STREET_TYPES_SEED_RE;
|
|
3736
|
+
let _streetTypesPromise = null;
|
|
3737
|
+
const escapeRegex$1 = (s) => s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
3738
|
+
const loadStreetTypeRegex = async () => {
|
|
3739
|
+
try {
|
|
3740
|
+
const data = (await import("./address-street-types.mjs")).default ?? {};
|
|
3741
|
+
const words = /* @__PURE__ */ new Set();
|
|
3742
|
+
for (const [key, val] of Object.entries(data)) {
|
|
3743
|
+
if (key.startsWith("_")) continue;
|
|
3744
|
+
if (!Array.isArray(val)) continue;
|
|
3745
|
+
for (const w of val) if (typeof w === "string" && w.length > 0) words.add(w.toLowerCase());
|
|
3746
|
+
}
|
|
3747
|
+
if (words.size === 0) {
|
|
3748
|
+
_streetTypesRe = STREET_TYPES_SEED_RE;
|
|
3749
|
+
return;
|
|
3750
|
+
}
|
|
3751
|
+
const alternation = [...words].sort((a, b) => b.length - a.length).map(escapeRegex$1).join("|");
|
|
3752
|
+
_streetTypesRe = new RegExp(`(?:^|\\s)(?:${alternation})(?=[\\s,./]|$)`, "iu");
|
|
3753
|
+
} catch {
|
|
3754
|
+
_streetTypesRe = STREET_TYPES_SEED_RE;
|
|
3755
|
+
}
|
|
3756
|
+
};
|
|
3757
|
+
/** Ensure street-type vocabulary is loaded. */
|
|
3758
|
+
const initAddressComponents = () => {
|
|
3759
|
+
if (!_streetTypesPromise) _streetTypesPromise = loadStreetTypeRegex();
|
|
3760
|
+
return _streetTypesPromise;
|
|
3761
|
+
};
|
|
3762
|
+
/**
|
|
3763
|
+
* True when `text` contains a recognised address
|
|
3764
|
+
* component: either a per-language street word (loaded
|
|
3765
|
+
* from JSON) or a language-agnostic anchor such as a
|
|
3766
|
+
* Czech house number form.
|
|
3767
|
+
*/
|
|
3768
|
+
const hasAddressComponent = (text) => _streetTypesRe.test(text) || ADDRESS_COMPONENT_EXTRA_RE.test(text);
|
|
3060
3769
|
const isCallerOwnedEntity$3 = (entity) => entity.sourceDetail === "custom-deny-list" || entity.sourceDetail === "custom-regex";
|
|
3061
3770
|
/**
|
|
3062
3771
|
* Filter out entities that are likely false positives:
|
|
@@ -3091,13 +3800,15 @@ const filterFalsePositives = (entities, ctx = defaultContext, fullText) => {
|
|
|
3091
3800
|
if (normalized.label === "person" && HAS_DIGIT_RE.test(trimmed)) continue;
|
|
3092
3801
|
if (normalized.label === "person") {
|
|
3093
3802
|
const tokens = trimmed.split(/\s+/u);
|
|
3094
|
-
const last = tokens.at(-1)?.replace(/[.,;:!?]+$/u, "")
|
|
3095
|
-
|
|
3803
|
+
const last = tokens.at(-1)?.replace(/[.,;:!?]+$/u, "");
|
|
3804
|
+
const lastFolded = last ? normalizeHomoglyphs(last).toLowerCase() : void 0;
|
|
3805
|
+
if (tokens.length > 1 && lastFolded && PERSON_TRAILING_NOUNS.has(lastFolded)) continue;
|
|
3096
3806
|
}
|
|
3097
|
-
if ((normalized.label === "person" || normalized.label === "organization") && roles.has(trimmed.toLowerCase())) continue;
|
|
3807
|
+
if ((normalized.label === "person" || normalized.label === "organization") && roles.has(normalizeHomoglyphs(trimmed).toLowerCase())) continue;
|
|
3098
3808
|
if (normalized.label === "organization" && normalized.source === "legal-form" && trimmed === trimmed.toUpperCase() && LEGAL_FORM_HEADING_RE.test(trimmed)) continue;
|
|
3099
|
-
if (normalized.label === "address" && trimmed.length > 40 && !POSTAL_CODE_RE.test(trimmed) && !HAS_DIGIT_RE.test(trimmed) && !
|
|
3100
|
-
if (normalized.label === "address" && normalized.source === "trigger" && !HAS_DIGIT_RE.test(trimmed) && !
|
|
3809
|
+
if (normalized.label === "address" && trimmed.length > 40 && !POSTAL_CODE_RE.test(trimmed) && !HAS_DIGIT_RE.test(trimmed) && !hasAddressComponent(trimmed) && !JURISDICTION_RE.test(trimmed)) continue;
|
|
3810
|
+
if (normalized.label === "address" && normalized.source === "trigger" && !HAS_DIGIT_RE.test(trimmed) && !hasAddressComponent(trimmed) && !JURISDICTION_RE.test(trimmed)) continue;
|
|
3811
|
+
if (normalized.label === "address" && normalized.source === "trigger" && !HAS_DIGIT_RE.test(trimmed) && isOnlyAmbiguousCours(trimmed)) continue;
|
|
3101
3812
|
if (normalized.label === "address" && SIGNING_CLAUSE_ADDRESS_RE.test(trimmed)) continue;
|
|
3102
3813
|
filtered.push(normalized);
|
|
3103
3814
|
}
|
|
@@ -3254,6 +3965,122 @@ const loadPersonStopwords = (ctx) => {
|
|
|
3254
3965
|
const EMPTY_PERSON_STOPWORDS = /* @__PURE__ */ new Set();
|
|
3255
3966
|
/** Sync accessor — returns empty set before init. */
|
|
3256
3967
|
const getPersonStopwords = (ctx) => ctx.personStopwords ?? EMPTY_PERSON_STOPWORDS;
|
|
3968
|
+
const loadAddressStopwords = (ctx) => {
|
|
3969
|
+
if (ctx.addressStopwordsPromise) return ctx.addressStopwordsPromise;
|
|
3970
|
+
ctx.addressStopwordsPromise = (async () => {
|
|
3971
|
+
try {
|
|
3972
|
+
const mod = await import("./address-stopwords.mjs");
|
|
3973
|
+
const set = new Set(mod.default?.words ?? []);
|
|
3974
|
+
ctx.addressStopwords = set;
|
|
3975
|
+
return set;
|
|
3976
|
+
} catch {
|
|
3977
|
+
const empty = /* @__PURE__ */ new Set();
|
|
3978
|
+
ctx.addressStopwords = empty;
|
|
3979
|
+
return empty;
|
|
3980
|
+
}
|
|
3981
|
+
})();
|
|
3982
|
+
return ctx.addressStopwordsPromise;
|
|
3983
|
+
};
|
|
3984
|
+
const EMPTY_ADDRESS_STOPWORDS = /* @__PURE__ */ new Set();
|
|
3985
|
+
const getAddressStopwords = (ctx) => ctx.addressStopwords ?? EMPTY_ADDRESS_STOPWORDS;
|
|
3986
|
+
/**
|
|
3987
|
+
* Word characters in unicode property notation. The check is
|
|
3988
|
+
* "single-token" — no internal whitespace — and we keep dashes
|
|
3989
|
+
* tokens like "K-12" out by requiring the surface to be a single
|
|
3990
|
+
* uninterrupted run of letters or marks.
|
|
3991
|
+
*/
|
|
3992
|
+
const SINGLE_WORD_RE = /^\p{L}+$/u;
|
|
3993
|
+
/**
|
|
3994
|
+
* Format-level address signals — structurally numeric or
|
|
3995
|
+
* 2-letter-state patterns, language-agnostic. The street-type
|
|
3996
|
+
* vocabulary is loaded from `address-street-types.json` so new
|
|
3997
|
+
* languages contribute via data, not code.
|
|
3998
|
+
* - `,\s*[A-Z]{2}\b` → US state abbreviation after a comma
|
|
3999
|
+
* - `\b\d{5}(?:-\d{4})?\b` → US ZIP / ZIP+4
|
|
4000
|
+
* - `\b\d{3}\s\d{2}\b` → Czech/Slovak postal block (140 00)
|
|
4001
|
+
* - `\b\d{2}-\d{3}\b` → Polish postal code (00-950)
|
|
4002
|
+
*/
|
|
4003
|
+
const ADDRESS_FORMAT_RE = /,\s*\p{Lu}{2}\b|\b\d{5}(?:-\d{4})?\b|\b\d{3}\s\d{2}\b|\b\d{2}-\d{3}\b/u;
|
|
4004
|
+
let cachedStreetTypeRe = null;
|
|
4005
|
+
let streetTypeReLoaded = false;
|
|
4006
|
+
const loadStreetTypeRe = async () => {
|
|
4007
|
+
if (streetTypeReLoaded) return cachedStreetTypeRe;
|
|
4008
|
+
try {
|
|
4009
|
+
const config = (await import("./address-street-types.mjs")).default ?? {};
|
|
4010
|
+
const words = [];
|
|
4011
|
+
for (const value of Object.values(config)) {
|
|
4012
|
+
if (!Array.isArray(value)) continue;
|
|
4013
|
+
for (const word of value) if (typeof word === "string" && word.length > 0) words.push(word);
|
|
4014
|
+
}
|
|
4015
|
+
if (words.length === 0) cachedStreetTypeRe = null;
|
|
4016
|
+
else {
|
|
4017
|
+
words.sort((a, b) => b.length - a.length);
|
|
4018
|
+
const isLetterDigit = (c) => /[\p{L}\p{N}]/u.test(c);
|
|
4019
|
+
const wordLikeTail = [];
|
|
4020
|
+
const punctTail = [];
|
|
4021
|
+
for (const w of words) {
|
|
4022
|
+
const escaped = w.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
4023
|
+
if (isLetterDigit(w.at(-1) ?? "")) wordLikeTail.push(escaped);
|
|
4024
|
+
else punctTail.push(escaped);
|
|
4025
|
+
}
|
|
4026
|
+
const branches = [];
|
|
4027
|
+
if (wordLikeTail.length > 0) branches.push(`(?:${wordLikeTail.join("|")})(?![\\p{L}\\p{N}])`);
|
|
4028
|
+
if (punctTail.length > 0) branches.push(`(?:${punctTail.join("|")})`);
|
|
4029
|
+
cachedStreetTypeRe = new RegExp(`(?<![\\p{L}\\p{N}])(?:${branches.join("|")})`, "iu");
|
|
4030
|
+
}
|
|
4031
|
+
} catch {
|
|
4032
|
+
cachedStreetTypeRe = null;
|
|
4033
|
+
}
|
|
4034
|
+
streetTypeReLoaded = true;
|
|
4035
|
+
return cachedStreetTypeRe;
|
|
4036
|
+
};
|
|
4037
|
+
const getStreetTypeRe = () => streetTypeReLoaded ? cachedStreetTypeRe : null;
|
|
4038
|
+
const hasAdjacentAddressEvidence = (fullText, start, end) => {
|
|
4039
|
+
const window = fullText.slice(Math.max(0, start - 40), Math.min(fullText.length, end + 40));
|
|
4040
|
+
if (ADDRESS_FORMAT_RE.test(window)) return true;
|
|
4041
|
+
const streetRe = getStreetTypeRe();
|
|
4042
|
+
return streetRe !== null && streetRe.test(window);
|
|
4043
|
+
};
|
|
4044
|
+
/**
|
|
4045
|
+
* Capitalised words that almost never start a person name. When a
|
|
4046
|
+
* single-token surname candidate is immediately followed by one of
|
|
4047
|
+
* these, the "next-word is uppercase" promotion heuristic would
|
|
4048
|
+
* otherwise turn section headings ("Purchase Price↵The Purchaser
|
|
4049
|
+
* undertakes…") into spurious person hits. Kept narrow on purpose;
|
|
4050
|
+
* the surrounding pipeline still chains real names via the deny-list
|
|
4051
|
+
* cascade when both halves are surnames.
|
|
4052
|
+
*/
|
|
4053
|
+
const SENTENCE_STARTER_WORDS = new Set([
|
|
4054
|
+
"The",
|
|
4055
|
+
"This",
|
|
4056
|
+
"These",
|
|
4057
|
+
"Those",
|
|
4058
|
+
"An",
|
|
4059
|
+
"Any",
|
|
4060
|
+
"All",
|
|
4061
|
+
"Each",
|
|
4062
|
+
"Every",
|
|
4063
|
+
"No",
|
|
4064
|
+
"Now",
|
|
4065
|
+
"Whereas",
|
|
4066
|
+
"Whereby",
|
|
4067
|
+
"Wherein",
|
|
4068
|
+
"Whereof",
|
|
4069
|
+
"Notwithstanding",
|
|
4070
|
+
"Subject",
|
|
4071
|
+
"In",
|
|
4072
|
+
"On",
|
|
4073
|
+
"At",
|
|
4074
|
+
"By",
|
|
4075
|
+
"For",
|
|
4076
|
+
"If",
|
|
4077
|
+
"Upon",
|
|
4078
|
+
"Unless",
|
|
4079
|
+
"Until",
|
|
4080
|
+
"Provided",
|
|
4081
|
+
"Pursuant",
|
|
4082
|
+
"Such"
|
|
4083
|
+
]);
|
|
3257
4084
|
const PERSON_CHAIN_BREAK_RE = /[!?;:]|,/u;
|
|
3258
4085
|
const WORD_CHAR_RE$1 = /[\p{L}\p{N}]/u;
|
|
3259
4086
|
const isInitialContinuationGap = (text, gap) => /^\p{Lu}$/u.test(text) && /^\.[^\S\n]{1,2}$/u.test(gap) || /^[^\S\n]{1,2}(?:\p{Lu}\.[^\S\n]{1,2})+$/u.test(gap);
|
|
@@ -3273,6 +4100,8 @@ const buildDenyList = async (config, ctx = defaultContext) => {
|
|
|
3273
4100
|
loadStopwords(ctx),
|
|
3274
4101
|
loadAllowList(ctx),
|
|
3275
4102
|
loadPersonStopwords(ctx),
|
|
4103
|
+
loadAddressStopwords(ctx),
|
|
4104
|
+
loadStreetTypeRe(),
|
|
3276
4105
|
loadGenericRoles(ctx)
|
|
3277
4106
|
]);
|
|
3278
4107
|
const dictionaries = config.dictionaries;
|
|
@@ -3419,6 +4248,8 @@ const ensureDenyListData = async (ctx = defaultContext, dictionaries) => {
|
|
|
3419
4248
|
loadStopwords(ctx),
|
|
3420
4249
|
loadAllowList(ctx),
|
|
3421
4250
|
loadPersonStopwords(ctx),
|
|
4251
|
+
loadAddressStopwords(ctx),
|
|
4252
|
+
loadStreetTypeRe(),
|
|
3422
4253
|
loadGenericRoles(ctx)
|
|
3423
4254
|
]);
|
|
3424
4255
|
};
|
|
@@ -3486,14 +4317,18 @@ const processDenyListMatches = (allMatches, sliceStart, sliceEnd, fullText, data
|
|
|
3486
4317
|
if (!getPersonStopwords(ctx).has(keyword)) nameHits.push(m);
|
|
3487
4318
|
}
|
|
3488
4319
|
const nonPersonLabels = m.labels.filter((l) => l !== "person");
|
|
3489
|
-
|
|
3490
|
-
|
|
3491
|
-
|
|
3492
|
-
|
|
3493
|
-
|
|
3494
|
-
|
|
3495
|
-
|
|
3496
|
-
|
|
4320
|
+
const suppressAddress = SINGLE_WORD_RE.test(m.text) && getAddressStopwords(ctx).has(m.text.toLowerCase()) && !hasAdjacentAddressEvidence(fullText, m.start, m.end);
|
|
4321
|
+
for (const label of nonPersonLabels) {
|
|
4322
|
+
if (label === "address" && suppressAddress) continue;
|
|
4323
|
+
results.push({
|
|
4324
|
+
start: m.start,
|
|
4325
|
+
end: m.end,
|
|
4326
|
+
label,
|
|
4327
|
+
text: m.text,
|
|
4328
|
+
score: .9,
|
|
4329
|
+
source: DETECTION_SOURCES.DENY_LIST
|
|
4330
|
+
});
|
|
4331
|
+
}
|
|
3497
4332
|
}
|
|
3498
4333
|
}
|
|
3499
4334
|
nameHits.sort((a, b) => a.start - b.start);
|
|
@@ -3525,6 +4360,8 @@ const processDenyListMatches = (allMatches, sliceStart, sliceEnd, fullText, data
|
|
|
3525
4360
|
const afterEnd = last.end;
|
|
3526
4361
|
const rest = fullText.slice(afterEnd).trimStart();
|
|
3527
4362
|
if (!(rest.length > 1 && /^\p{Lu}\p{Ll}/u.test(rest))) continue;
|
|
4363
|
+
const nextWord = /^\p{L}+/u.exec(rest)?.[0] ?? "";
|
|
4364
|
+
if (SENTENCE_STARTER_WORDS.has(nextWord)) continue;
|
|
3528
4365
|
}
|
|
3529
4366
|
results.push({
|
|
3530
4367
|
start: first.start,
|
|
@@ -3636,6 +4473,15 @@ const extendPersonName = (text, start, end, ctx) => {
|
|
|
3636
4473
|
};
|
|
3637
4474
|
//#endregion
|
|
3638
4475
|
//#region src/detectors/address-seeds.ts
|
|
4476
|
+
/**
|
|
4477
|
+
* Trailing chars that should never end an address span. Combines
|
|
4478
|
+
* structural separators, opening brackets, and typographic quote
|
|
4479
|
+
* variants so a span like `… GA 30326, USA (the "Premises")`
|
|
4480
|
+
* does not pull the parenthetical opener into the address.
|
|
4481
|
+
* The prime (`′`) catches measurement tails (`5′`) that the
|
|
4482
|
+
* cluster occasionally absorbs.
|
|
4483
|
+
*/
|
|
4484
|
+
const ADDRESS_TRAILING_TRIM_RE = new RegExp(`[,;:\\s${OPENING_BRACKETS_INNER}${QUOTE_DOUBLE_INNER}${QUOTE_SINGLE_INNER}′]`, "u");
|
|
3639
4485
|
let cachedBoundaryRe = null;
|
|
3640
4486
|
const loadBoundaryWords = async () => {
|
|
3641
4487
|
try {
|
|
@@ -3644,6 +4490,56 @@ const loadBoundaryWords = async () => {
|
|
|
3644
4490
|
return {};
|
|
3645
4491
|
}
|
|
3646
4492
|
};
|
|
4493
|
+
const BR_CEP_CONTEXT_WINDOW = 200;
|
|
4494
|
+
let cachedBrCepContextRe = null;
|
|
4495
|
+
let cachedBrCepContextPromise = null;
|
|
4496
|
+
const loadBrCueWords = async () => {
|
|
4497
|
+
const sources = await Promise.all([(async () => {
|
|
4498
|
+
try {
|
|
4499
|
+
return (await import("./address-street-types.mjs")).default["pt-br"];
|
|
4500
|
+
} catch {
|
|
4501
|
+
return;
|
|
4502
|
+
}
|
|
4503
|
+
})(), (async () => {
|
|
4504
|
+
try {
|
|
4505
|
+
return (await import("./address-boundaries.mjs")).default["pt-br"];
|
|
4506
|
+
} catch {
|
|
4507
|
+
return;
|
|
4508
|
+
}
|
|
4509
|
+
})()]);
|
|
4510
|
+
const out = [];
|
|
4511
|
+
const seen = /* @__PURE__ */ new Set();
|
|
4512
|
+
for (const entry of sources) {
|
|
4513
|
+
if (!Array.isArray(entry)) continue;
|
|
4514
|
+
for (const word of entry) {
|
|
4515
|
+
if (typeof word !== "string" || word.length === 0) continue;
|
|
4516
|
+
const key = word.toLowerCase();
|
|
4517
|
+
if (seen.has(key)) continue;
|
|
4518
|
+
seen.add(key);
|
|
4519
|
+
out.push(word);
|
|
4520
|
+
}
|
|
4521
|
+
}
|
|
4522
|
+
return out;
|
|
4523
|
+
};
|
|
4524
|
+
const getBrCepContextRe = async () => {
|
|
4525
|
+
if (cachedBrCepContextRe !== null) return cachedBrCepContextRe;
|
|
4526
|
+
if (cachedBrCepContextPromise) return cachedBrCepContextPromise;
|
|
4527
|
+
cachedBrCepContextPromise = (async () => {
|
|
4528
|
+
const words = await loadBrCueWords();
|
|
4529
|
+
if (words.length === 0) return null;
|
|
4530
|
+
const escaped = words.toSorted((a, b) => b.length - a.length).map((w) => w.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"));
|
|
4531
|
+
const re = new RegExp(`(?<![\\p{L}\\p{N}])(?:${escaped.join("|")})(?![\\p{L}\\p{N}])`, "iu");
|
|
4532
|
+
cachedBrCepContextRe = re;
|
|
4533
|
+
return re;
|
|
4534
|
+
})();
|
|
4535
|
+
return cachedBrCepContextPromise;
|
|
4536
|
+
};
|
|
4537
|
+
const hasBrCueNearby = (fullText, start, end, re) => {
|
|
4538
|
+
const windowStart = Math.max(0, start - BR_CEP_CONTEXT_WINDOW);
|
|
4539
|
+
const windowEnd = Math.min(fullText.length, end + BR_CEP_CONTEXT_WINDOW);
|
|
4540
|
+
const window = fullText.slice(windowStart, windowEnd);
|
|
4541
|
+
return new RegExp(re.source, re.flags.replace("g", "")).test(window);
|
|
4542
|
+
};
|
|
3647
4543
|
/**
|
|
3648
4544
|
* Build regex for boundary words. Matches any
|
|
3649
4545
|
* boundary word preceded by a word boundary.
|
|
@@ -3657,7 +4553,7 @@ const getBoundaryRe = async () => {
|
|
|
3657
4553
|
for (const word of entries) words.push(word.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"));
|
|
3658
4554
|
}
|
|
3659
4555
|
words.sort((a, b) => b.length - a.length);
|
|
3660
|
-
cachedBoundaryRe = words.length > 0 ? new RegExp(
|
|
4556
|
+
cachedBoundaryRe = words.length > 0 ? new RegExp(`(?<![\\p{L}\\p{N}])(?:${words.join("|")})(?![\\p{L}\\p{N}])`, "iu") : /(?!)/;
|
|
3661
4557
|
return cachedBoundaryRe;
|
|
3662
4558
|
};
|
|
3663
4559
|
/**
|
|
@@ -3679,7 +4575,7 @@ const buildStreetTypePatterns = async () => {
|
|
|
3679
4575
|
}
|
|
3680
4576
|
return words;
|
|
3681
4577
|
};
|
|
3682
|
-
const collectSeeds = (allMatches, sliceStart, sliceEnd, fullText, existingEntities) => {
|
|
4578
|
+
const collectSeeds = (allMatches, sliceStart, sliceEnd, fullText, existingEntities, brCepContextRe) => {
|
|
3683
4579
|
const seeds = [];
|
|
3684
4580
|
for (const match of allMatches) {
|
|
3685
4581
|
const idx = match.pattern;
|
|
@@ -3713,18 +4609,41 @@ const collectSeeds = (allMatches, sliceStart, sliceEnd, fullText, existingEntiti
|
|
|
3713
4609
|
text: e.text
|
|
3714
4610
|
});
|
|
3715
4611
|
}
|
|
3716
|
-
const postalRe = /\b(?:\d{3}\s\d{2}|\d{2}-\d{3})\b/g;
|
|
4612
|
+
const postalRe = /\b(?:\d{3}\s\d{2}|\d{2}-\d{3}|\d{5}-\d{3})\b/g;
|
|
3717
4613
|
let postalMatch;
|
|
3718
4614
|
while ((postalMatch = postalRe.exec(fullText)) !== null) {
|
|
3719
4615
|
const start = postalMatch.index;
|
|
3720
4616
|
const end = start + postalMatch[0].length;
|
|
3721
|
-
if (
|
|
4617
|
+
if (seeds.some((s) => s.start <= start && s.end >= end)) continue;
|
|
4618
|
+
if (/^\d{5}-\d{3}$/.test(postalMatch[0]) && (brCepContextRe === null || !hasBrCueNearby(fullText, start, end, brCepContextRe))) continue;
|
|
4619
|
+
seeds.push({
|
|
3722
4620
|
type: "postal-code",
|
|
3723
4621
|
start,
|
|
3724
4622
|
end,
|
|
3725
4623
|
text: postalMatch[0]
|
|
3726
4624
|
});
|
|
3727
4625
|
}
|
|
4626
|
+
const itCapRe = /\b\d{5}(?=\s+\p{Lu}\p{L}+)/gu;
|
|
4627
|
+
let itCapMatch;
|
|
4628
|
+
while ((itCapMatch = itCapRe.exec(fullText)) !== null) {
|
|
4629
|
+
const start = itCapMatch.index;
|
|
4630
|
+
const end = start + itCapMatch[0].length;
|
|
4631
|
+
if (seeds.some((s) => s.start <= start && s.end >= end)) continue;
|
|
4632
|
+
if (!seeds.some((s) => {
|
|
4633
|
+
if (Math.abs(s.start - start) > 80) return false;
|
|
4634
|
+
if (s.type === "address-trigger") return true;
|
|
4635
|
+
if (s.type === "city") return true;
|
|
4636
|
+
if (s.type === "postal-code") return true;
|
|
4637
|
+
if (s.type === "street-word") return s.text.toLowerCase() !== "via";
|
|
4638
|
+
return false;
|
|
4639
|
+
})) continue;
|
|
4640
|
+
seeds.push({
|
|
4641
|
+
type: "postal-code",
|
|
4642
|
+
start,
|
|
4643
|
+
end,
|
|
4644
|
+
text: itCapMatch[0]
|
|
4645
|
+
});
|
|
4646
|
+
}
|
|
3728
4647
|
const streetNumRe = /\b(\p{Lu}\p{Ll}{2,})\s+(\d{1,5}(?:\/\d{1,5})?)\s*[,\n]/gu;
|
|
3729
4648
|
let streetMatch;
|
|
3730
4649
|
while ((streetMatch = streetNumRe.exec(fullText)) !== null) {
|
|
@@ -3782,6 +4701,10 @@ const scoreCluster = (cluster) => {
|
|
|
3782
4701
|
const NON_ADDRESS_LABELS = new Set([
|
|
3783
4702
|
"registration number",
|
|
3784
4703
|
"tax identification number",
|
|
4704
|
+
"national identification number",
|
|
4705
|
+
"social security number",
|
|
4706
|
+
"birth number",
|
|
4707
|
+
"identity card number",
|
|
3785
4708
|
"person",
|
|
3786
4709
|
"bank account number",
|
|
3787
4710
|
"email address",
|
|
@@ -3818,7 +4741,7 @@ const expandCluster = async (fullText, cluster, existingEntities) => {
|
|
|
3818
4741
|
const doubleNewline = remaining.indexOf("\n\n");
|
|
3819
4742
|
if (doubleNewline !== -1 && doubleNewline < nearestBoundary) nearestBoundary = doubleNewline;
|
|
3820
4743
|
rightPos = end + remaining.slice(0, nearestBoundary).trimEnd().length;
|
|
3821
|
-
while (rightPos > end &&
|
|
4744
|
+
while (rightPos > end && ADDRESS_TRAILING_TRIM_RE.test(fullText[rightPos - 1] ?? "")) rightPos--;
|
|
3822
4745
|
return {
|
|
3823
4746
|
start: Math.min(leftPos, start),
|
|
3824
4747
|
end: Math.max(rightPos, end)
|
|
@@ -3835,7 +4758,7 @@ const expandCluster = async (fullText, cluster, existingEntities) => {
|
|
|
3835
4758
|
* using their output as seed sources.
|
|
3836
4759
|
*/
|
|
3837
4760
|
const processAddressSeeds = async (allMatches, sliceStart, sliceEnd, fullText, existingEntities) => {
|
|
3838
|
-
const clusters = clusterSeeds(collectSeeds(allMatches, sliceStart, sliceEnd, fullText, existingEntities), 150);
|
|
4761
|
+
const clusters = clusterSeeds(collectSeeds(allMatches, sliceStart, sliceEnd, fullText, existingEntities, await getBrCepContextRe()), 150);
|
|
3839
4762
|
const results = [];
|
|
3840
4763
|
for (const cluster of clusters) {
|
|
3841
4764
|
const score = scoreCluster(cluster);
|
|
@@ -3860,6 +4783,7 @@ const processAddressSeeds = async (allMatches, sliceStart, sliceEnd, fullText, e
|
|
|
3860
4783
|
const TRAILING_SEP = /[,\s]+$/;
|
|
3861
4784
|
const WORD_CHAR_RE = /[\p{L}\p{N}]/u;
|
|
3862
4785
|
const ORG_PROPAGATION_SCORE = .9;
|
|
4786
|
+
const ORG_DETERMINER_RE = /(?<![\p{L}\p{N}])(společnost(?:i|í|em|u)?|spolecnost(?:i|em|u)?|the\s+(?:company|corporation|firm)|die\s+(?:gesellschaft|firma)|la\s+(?:société|empresa|sociedad)|el\s+(?:empresa|sociedad))\s+$/iu;
|
|
3863
4787
|
const isCallerOwnedEntity$2 = (entity) => entity.sourceDetail === "custom-deny-list" || entity.sourceDetail === "custom-regex";
|
|
3864
4788
|
/**
|
|
3865
4789
|
* After the main detection pass, collect organization
|
|
@@ -3903,16 +4827,25 @@ const propagateOrgNames = (entities, fullText) => {
|
|
|
3903
4827
|
searchFrom = idx + 1;
|
|
3904
4828
|
continue;
|
|
3905
4829
|
}
|
|
3906
|
-
|
|
4830
|
+
let spanStart = idx;
|
|
4831
|
+
const lookbackStart = Math.max(0, idx - 40);
|
|
4832
|
+
const lookback = fullText.slice(lookbackStart, idx);
|
|
4833
|
+
const determinerMatch = ORG_DETERMINER_RE.exec(lookback);
|
|
4834
|
+
if (determinerMatch !== null) {
|
|
4835
|
+
const determiner = determinerMatch[1] ?? "";
|
|
4836
|
+
const offsetInMatch = determinerMatch[0].indexOf(determiner);
|
|
4837
|
+
spanStart = lookbackStart + determinerMatch.index + offsetInMatch;
|
|
4838
|
+
}
|
|
4839
|
+
if (!isOverlapping(spanStart, matchEnd)) {
|
|
3907
4840
|
results.push({
|
|
3908
|
-
start:
|
|
4841
|
+
start: spanStart,
|
|
3909
4842
|
end: matchEnd,
|
|
3910
4843
|
label,
|
|
3911
|
-
text:
|
|
4844
|
+
text: fullText.slice(spanStart, matchEnd),
|
|
3912
4845
|
score: ORG_PROPAGATION_SCORE,
|
|
3913
4846
|
source: DETECTION_SOURCES.COREFERENCE
|
|
3914
4847
|
});
|
|
3915
|
-
covered.push([
|
|
4848
|
+
covered.push([spanStart, matchEnd]);
|
|
3916
4849
|
}
|
|
3917
4850
|
searchFrom = matchEnd;
|
|
3918
4851
|
}
|
|
@@ -5077,13 +6010,89 @@ const shouldReplace = (a, b) => {
|
|
|
5077
6010
|
};
|
|
5078
6011
|
/** Labels where colons are structurally significant. */
|
|
5079
6012
|
const COLON_LABELS = new Set(["ip address", "mac address"]);
|
|
6013
|
+
/**
|
|
6014
|
+
* Labels whose detectors emit precise, evidence-backed spans. When
|
|
6015
|
+
* one of these fires at the exact same offsets as a fuzzier
|
|
6016
|
+
* `address` hit (city dictionary lookup, address-seed cluster), the
|
|
6017
|
+
* `address` label is almost always a dictionary collision — "March
|
|
6018
|
+
* 15" the date getting tagged as an address because "March" appears
|
|
6019
|
+
* in a city corpus, or "Brent Phillips" emitted as both `person`
|
|
6020
|
+
* and `address` because "Brent" is a UK city. The precise detector
|
|
6021
|
+
* wins.
|
|
6022
|
+
*/
|
|
6023
|
+
const PRECISE_OVER_ADDRESS = new Set([
|
|
6024
|
+
"person",
|
|
6025
|
+
"date",
|
|
6026
|
+
"date of birth",
|
|
6027
|
+
"phone number",
|
|
6028
|
+
"email address",
|
|
6029
|
+
"monetary amount",
|
|
6030
|
+
"iban",
|
|
6031
|
+
"bank account number",
|
|
6032
|
+
"tax identification number",
|
|
6033
|
+
"registration number",
|
|
6034
|
+
"identity card number",
|
|
6035
|
+
"national identification number",
|
|
6036
|
+
"passport number",
|
|
6037
|
+
"credit card number"
|
|
6038
|
+
]);
|
|
6039
|
+
/**
|
|
6040
|
+
* Labels the `person` chain wins against at identical offsets. The
|
|
6041
|
+
* chain carries adjacent-name evidence (deny-list surname plus a
|
|
6042
|
+
* capitalised follow-up) a single-token dictionary collision does
|
|
6043
|
+
* not. Kept narrow: organizations are NOT here — "Morgan Stanley"
|
|
6044
|
+
* legitimately appears in both the org and name dictionaries, and
|
|
6045
|
+
* the existing detector priority is the right tie-breaker there.
|
|
6046
|
+
*/
|
|
6047
|
+
const PERSON_PREFERRED_OVER = new Set(["address", "land parcel"]);
|
|
6048
|
+
const resolveSameSpanLabelConflicts = (entities) => {
|
|
6049
|
+
if (entities.length < 2) return entities;
|
|
6050
|
+
const byOffsets = /* @__PURE__ */ new Map();
|
|
6051
|
+
for (const entity of entities) {
|
|
6052
|
+
const key = `${entity.start}:${entity.end}`;
|
|
6053
|
+
const list = byOffsets.get(key);
|
|
6054
|
+
if (list) list.push(entity);
|
|
6055
|
+
else byOffsets.set(key, [entity]);
|
|
6056
|
+
}
|
|
6057
|
+
const dropped = /* @__PURE__ */ new Set();
|
|
6058
|
+
for (const [, group] of byOffsets) {
|
|
6059
|
+
if (group.length < 2) continue;
|
|
6060
|
+
const labels = new Set(group.map((e) => e.label));
|
|
6061
|
+
if (labels.size < 2) continue;
|
|
6062
|
+
const hasPerson = labels.has("person");
|
|
6063
|
+
const hasPreciseNonAddress = [...labels].some((l) => l !== "address" && PRECISE_OVER_ADDRESS.has(l));
|
|
6064
|
+
let maxPriority = -1;
|
|
6065
|
+
for (const e of group) {
|
|
6066
|
+
if (hasLockedBoundary(e)) continue;
|
|
6067
|
+
const pri = DETECTOR_PRIORITY[e.source] ?? 0;
|
|
6068
|
+
if (pri > maxPriority) maxPriority = pri;
|
|
6069
|
+
}
|
|
6070
|
+
for (const e of group) {
|
|
6071
|
+
if (hasLockedBoundary(e)) continue;
|
|
6072
|
+
if ((DETECTOR_PRIORITY[e.source] ?? 0) < maxPriority) {
|
|
6073
|
+
dropped.add(e);
|
|
6074
|
+
continue;
|
|
6075
|
+
}
|
|
6076
|
+
if (hasPerson && PERSON_PREFERRED_OVER.has(e.label)) {
|
|
6077
|
+
dropped.add(e);
|
|
6078
|
+
continue;
|
|
6079
|
+
}
|
|
6080
|
+
if (hasPreciseNonAddress && e.label === "address") dropped.add(e);
|
|
6081
|
+
}
|
|
6082
|
+
}
|
|
6083
|
+
if (dropped.size === 0) return entities;
|
|
6084
|
+
return entities.filter((e) => !dropped.has(e));
|
|
6085
|
+
};
|
|
5080
6086
|
/** Strip leading/trailing whitespace and punctuation. */
|
|
5081
6087
|
const sanitizeEntities = (entities) => entities.flatMap((e) => {
|
|
5082
6088
|
if (hasLockedBoundary(e)) return [e];
|
|
5083
6089
|
const strip = COLON_LABELS.has(e.label) ? /[\s,;]+/ : /[\s:,;]+/;
|
|
5084
6090
|
const leadTrimmed = e.text.replace(/^(?:\.\s)+/, "").replace(new RegExp(`^${strip.source}`, strip.flags), "");
|
|
5085
6091
|
const lead = e.text.length - leadTrimmed.length;
|
|
5086
|
-
|
|
6092
|
+
let cleaned = leadTrimmed.replace(new RegExp(`${strip.source}$`, strip.flags), "");
|
|
6093
|
+
if (e.label === "organization" && cleaned.endsWith(".") && !LITERAL_SOURCES.has(e.source)) {
|
|
6094
|
+
if (!getKnownLegalSuffixes().some((suffix) => cleaned.endsWith(suffix))) cleaned = cleaned.slice(0, -1).trimEnd();
|
|
6095
|
+
}
|
|
5087
6096
|
if (cleaned.length === 0) return [];
|
|
5088
6097
|
if (!/[\p{L}\p{N}]/u.test(cleaned)) return [];
|
|
5089
6098
|
const collapsed = cleaned.replace(/\s*\n\s*/g, " ").replace(/\s{2,}/g, " ");
|
|
@@ -5132,7 +6141,7 @@ const mergeAndDedup = (...layers) => {
|
|
|
5132
6141
|
}
|
|
5133
6142
|
if (overlaps.every((existing) => shouldReplace(entity, existing))) merged.splice(overlapStart, overlaps.length, { ...entity });
|
|
5134
6143
|
}
|
|
5135
|
-
return sanitizeEntities(merged);
|
|
6144
|
+
return resolveSameSpanLabelConflicts(sanitizeEntities(merged));
|
|
5136
6145
|
};
|
|
5137
6146
|
const escapeRegex = (s) => s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
5138
6147
|
let amountWordsRe = null;
|
|
@@ -5168,9 +6177,10 @@ const filterAllowedLabels = (entities, allowedLabels) => {
|
|
|
5168
6177
|
return entities.filter((e) => allowedLabels.has(e.label));
|
|
5169
6178
|
};
|
|
5170
6179
|
const labelIsAllowed = (label, allowedLabels) => !allowedLabels || allowedLabels.has(label);
|
|
6180
|
+
const NON_NER_LABELS = new Set(["misc"]);
|
|
5171
6181
|
const getRequestedNerLabels = (config, expandForHotwords = false) => {
|
|
5172
6182
|
const labels = config.labels.length > 0 ? config.labels : DEFAULT_ENTITY_LABELS;
|
|
5173
|
-
return expandForHotwords ? expandLabelsForHotwordRules(labels) : labels;
|
|
6183
|
+
return (expandForHotwords ? expandLabelsForHotwordRules(labels) : labels).filter((label) => !NON_NER_LABELS.has(label));
|
|
5174
6184
|
};
|
|
5175
6185
|
const checkAbort = (signal) => {
|
|
5176
6186
|
if (signal?.aborted) throw new DOMException("Pipeline aborted", "AbortError");
|
|
@@ -5251,6 +6261,8 @@ const runPipeline = async (options) => {
|
|
|
5251
6261
|
loadGenericRoles(ctx),
|
|
5252
6262
|
initPrepositions(),
|
|
5253
6263
|
initStreetAbbrevs(),
|
|
6264
|
+
initAddressComponents(),
|
|
6265
|
+
warmAddressStopKeywords(),
|
|
5254
6266
|
zoneInit,
|
|
5255
6267
|
hotwordInit
|
|
5256
6268
|
]);
|
|
@@ -5258,6 +6270,8 @@ const runPipeline = async (options) => {
|
|
|
5258
6270
|
loadGenericRoles(ctx),
|
|
5259
6271
|
initPrepositions(),
|
|
5260
6272
|
initStreetAbbrevs(),
|
|
6273
|
+
initAddressComponents(),
|
|
6274
|
+
warmAddressStopKeywords(),
|
|
5261
6275
|
hotwordInit
|
|
5262
6276
|
]);
|
|
5263
6277
|
if (cachedSearch && config.enableDenyList) await ensureDenyListData(ctx, config.dictionaries);
|
|
@@ -5277,9 +6291,11 @@ const runPipeline = async (options) => {
|
|
|
5277
6291
|
const customRegexEntities = filterAllowedLabels(config.enableRegex ? processRegexMatches(customRegexMatches, slices.customRegex.start, slices.customRegex.end, search.customRegexMeta) : [], preHotwordAllowedLabels);
|
|
5278
6292
|
const regexEntities = filterAllowedLabels([...rawRegexEntities, ...customRegexEntities], preHotwordAllowedLabels);
|
|
5279
6293
|
if (regexEntities.length > 0) log("regex", `${regexEntities.length} matches`);
|
|
6294
|
+
if (legalFormsEnabled || config.enableTriggerPhrases) await warmLegalRoleHeads();
|
|
5280
6295
|
const rawLegalFormEntities = legalFormsEnabled ? processLegalFormMatches(regexMatches, slices.legalForms.start, slices.legalForms.end, fullText) : [];
|
|
5281
6296
|
const legalFormEntities = filterAllowedLabels(rawLegalFormEntities, preHotwordAllowedLabels);
|
|
5282
6297
|
if (legalFormEntities.length > 0) log("legal-forms", `${legalFormEntities.length} matches`);
|
|
6298
|
+
if (config.enableTriggerPhrases) await warmAddressStopKeywords();
|
|
5283
6299
|
const rawTriggerEntities = config.enableTriggerPhrases ? processTriggerMatches(regexMatches, slices.triggers.start, slices.triggers.end, fullText, search.triggerRules) : [];
|
|
5284
6300
|
const triggerEntities = filterAllowedLabels(rawTriggerEntities, preHotwordAllowedLabels);
|
|
5285
6301
|
if (triggerEntities.length > 0) log("trigger-phrases", `${triggerEntities.length} matches`);
|
|
@@ -5325,10 +6341,11 @@ const runPipeline = async (options) => {
|
|
|
5325
6341
|
];
|
|
5326
6342
|
let rawNerEntities = [];
|
|
5327
6343
|
let nerEntities = [];
|
|
5328
|
-
|
|
6344
|
+
const requestedNerLabels = getRequestedNerLabels(config, hotwordsActive);
|
|
6345
|
+
if (config.enableNer && nerInference && requestedNerLabels.length > 0) {
|
|
5329
6346
|
const maskResult = maskDetectedSpans(fullText, nerMaskEntities);
|
|
5330
6347
|
log("ner", "running inference...");
|
|
5331
|
-
const rawNer = await nerInference(maskResult.maskedText, [...
|
|
6348
|
+
const rawNer = await nerInference(maskResult.maskedText, [...requestedNerLabels], config.threshold, signal);
|
|
5332
6349
|
rawNerEntities = unmaskNerEntities(rawNer, maskResult, fullText);
|
|
5333
6350
|
nerEntities = filterAllowedLabels(rawNerEntities, preHotwordAllowedLabels);
|
|
5334
6351
|
const masked = rawNer.length - rawNerEntities.length;
|
|
@@ -5389,6 +6406,7 @@ const runPipeline = async (options) => {
|
|
|
5389
6406
|
checkAbort(signal);
|
|
5390
6407
|
ctx.corefSourceMap.clear();
|
|
5391
6408
|
if (config.enableCoreference) {
|
|
6409
|
+
if (!legalFormsEnabled) await warmLegalRoleHeads();
|
|
5392
6410
|
const terms = await extractDefinedTerms(fullText, merged.filter((entity) => !isCallerOwnedEntity(entity)), ctx);
|
|
5393
6411
|
if (terms.length > 0) {
|
|
5394
6412
|
log("coreference", `${terms.length} defined terms`);
|
|
@@ -5429,7 +6447,7 @@ const resolveOperator = (config, label) => config.operators[label] ?? "replace";
|
|
|
5429
6447
|
//#region src/redact.ts
|
|
5430
6448
|
const WHITESPACE_RE = /\s+/g;
|
|
5431
6449
|
const PHONE_NOISE_RE = /[()\s-]/g;
|
|
5432
|
-
const
|
|
6450
|
+
const ID_SEPARATOR_RE = /[\s\-/.]/g;
|
|
5433
6451
|
/**
|
|
5434
6452
|
* Normalize entity text so that surface-form variations
|
|
5435
6453
|
* of the same real-world value map to a single canonical
|
|
@@ -5439,8 +6457,8 @@ const normalizeEntityText = (label, text) => {
|
|
|
5439
6457
|
const upper = label.toUpperCase().replace(WHITESPACE_RE, "_");
|
|
5440
6458
|
if (upper === "EMAIL_ADDRESS" || upper === "EMAIL") return text.toLowerCase().trim();
|
|
5441
6459
|
if (upper === "PHONE_NUMBER" || upper === "PHONE") return text.replace(PHONE_NOISE_RE, "");
|
|
5442
|
-
if (upper === "IBAN" || upper === "BANK_ACCOUNT_NUMBER" || upper === "TAX_IDENTIFICATION_NUMBER" || upper === "REGISTRATION_NUMBER") return text.replace(
|
|
5443
|
-
if (upper === "PERSON" || upper === "ORGANIZATION" || upper === "ADDRESS" || upper === "LAND_PARCEL") return text.replace(WHITESPACE_RE, " ").toLowerCase().trim();
|
|
6460
|
+
if (upper === "IBAN" || upper === "BANK_ACCOUNT_NUMBER" || upper === "TAX_IDENTIFICATION_NUMBER" || upper === "REGISTRATION_NUMBER" || upper === "NATIONAL_IDENTIFICATION_NUMBER" || upper === "SOCIAL_SECURITY_NUMBER" || upper === "BIRTH_NUMBER" || upper === "IDENTITY_CARD_NUMBER" || upper === "PASSPORT_NUMBER" || upper === "CREDIT_CARD_NUMBER") return text.replace(ID_SEPARATOR_RE, "").toUpperCase();
|
|
6461
|
+
if (upper === "PERSON" || upper === "ORGANIZATION" || upper === "ADDRESS" || upper === "LAND_PARCEL" || upper === "MISC") return text.replace(WHITESPACE_RE, " ").toLowerCase().trim();
|
|
5444
6462
|
return text.trim();
|
|
5445
6463
|
};
|
|
5446
6464
|
/**
|
|
@@ -5984,6 +7002,6 @@ const levenshtein = (rawA, rawB) => {
|
|
|
5984
7002
|
//#region src/wasm.ts
|
|
5985
7003
|
initTextSearch(TextSearch);
|
|
5986
7004
|
//#endregion
|
|
5987
|
-
export { CURRENCY_PATTERN_META, DATE_PATTERN_META, DEFAULT_ENTITY_LABELS, DEFAULT_OPERATOR_CONFIG, DETECTION_SOURCES, DETECTOR_PRIORITY, OPERATOR_REGISTRY, OPERATOR_TYPES, REGEX_META, REGEX_PATTERNS, REGIONS, ZONE_SCORE_ADJUSTMENTS, applyHotwordRules, applyZoneAdjustments, boostNearMissEntities, buildDenyList, buildGazetteerPatterns, buildLegalFormPatterns, buildPlaceholderMap, buildStreetTypePatterns, buildTriggerPatterns, buildUnifiedSearch, chunkText, classifyZones, computeChunkOffsets, corefKey, createPipelineContext, deanonymise, decodeSpans, decodeTokenSpans, detectNameCorpus, ensureDenyListData, exportRedactionKey, extractDefinedTerms, filterFalsePositives, findCoreferenceSpans, getCurrencyPatterns, getDatePatterns, initHotwordRules, initNameCorpus, initZoneClassifier, levenshtein, mergeAndDedup, mergeChunkEntities, normalizeForSearch, prepareBatch, processAddressSeeds, processDenyListMatches, processGazetteerMatches, processLegalFormMatches, processRegexMatches, processTriggerMatches, propagateOrgNames, redactText, resolveCountries, resolveOperator, runPipeline, runUnifiedSearch, sanitizeEntities, tokenizeText };
|
|
7005
|
+
export { CURRENCY_PATTERN_META, DATE_PATTERN_META, DEFAULT_ENTITY_LABELS, DEFAULT_OPERATOR_CONFIG, DETECTION_SOURCES, DETECTOR_PRIORITY, OPERATOR_REGISTRY, OPERATOR_TYPES, REGEX_META, REGEX_PATTERNS, REGIONS, ZONE_SCORE_ADJUSTMENTS, applyHotwordRules, applyZoneAdjustments, boostNearMissEntities, buildDenyList, buildGazetteerPatterns, buildLegalFormPatterns, buildPlaceholderMap, buildStreetTypePatterns, buildTriggerPatterns, buildUnifiedSearch, chunkText, classifyZones, computeChunkOffsets, corefKey, createPipelineContext, deanonymise, decodeSpans, decodeTokenSpans, detectNameCorpus, ensureDenyListData, exportRedactionKey, extractDefinedTerms, filterFalsePositives, findCoreferenceSpans, getCurrencyPatterns, getDatePatterns, initAddressComponents, initHotwordRules, initNameCorpus, initZoneClassifier, levenshtein, mergeAndDedup, mergeChunkEntities, normalizeForSearch, prepareBatch, processAddressSeeds, processDenyListMatches, processGazetteerMatches, processLegalFormMatches, processRegexMatches, processTriggerMatches, propagateOrgNames, redactText, resolveCountries, resolveOperator, runPipeline, runUnifiedSearch, sanitizeEntities, tokenizeText, warmLegalRoleHeads };
|
|
5988
7006
|
|
|
5989
7007
|
//# sourceMappingURL=wasm.mjs.map
|