@stll/anonymize-wasm 1.2.0 → 1.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/address-boundaries.mjs +113 -5
- package/dist/address-stop-keywords.mjs +137 -0
- package/dist/address-stop-keywords.mjs.map +1 -0
- package/dist/address-stopwords.mjs +81 -0
- package/dist/address-stopwords.mjs.map +1 -0
- package/dist/address-street-types.mjs +67 -4
- package/dist/clause-noun-heads.mjs +75 -0
- package/dist/clause-noun-heads.mjs.map +1 -0
- package/dist/constants.d.mts +2 -0
- package/dist/constants.mjs +85 -0
- package/dist/constants.mjs.map +1 -0
- package/dist/constants2.d.mts +65 -0
- package/dist/coreference.es.mjs +22 -0
- package/dist/coreference.es.mjs.map +1 -0
- package/dist/coreference.fr.mjs +32 -0
- package/dist/coreference.fr.mjs.map +1 -0
- package/dist/coreference.it.mjs +27 -0
- package/dist/coreference.it.mjs.map +1 -0
- package/dist/coreference.pl.mjs +27 -0
- package/dist/coreference.pl.mjs.map +1 -0
- package/dist/coreference.pt-br.mjs +14 -0
- package/dist/coreference.pt-br.mjs.map +1 -0
- package/dist/coreference.sk.mjs +22 -5
- package/dist/generic-roles.mjs +179 -1
- package/dist/hotword-rules.mjs +7 -3
- package/dist/legal-forms.mjs +3 -0
- package/dist/legal-role-heads.cs.mjs +42 -0
- package/dist/legal-role-heads.cs.mjs.map +1 -0
- package/dist/legal-role-heads.de.mjs +33 -0
- package/dist/legal-role-heads.de.mjs.map +1 -0
- package/dist/legal-role-heads.en.mjs +37 -0
- package/dist/legal-role-heads.en.mjs.map +1 -0
- package/dist/legal-role-heads.es.mjs +54 -0
- package/dist/legal-role-heads.es.mjs.map +1 -0
- package/dist/legal-role-heads.fr.mjs +72 -0
- package/dist/legal-role-heads.fr.mjs.map +1 -0
- package/dist/legal-role-heads.it.mjs +68 -0
- package/dist/legal-role-heads.it.mjs.map +1 -0
- package/dist/legal-role-heads.pl.mjs +84 -0
- package/dist/legal-role-heads.pl.mjs.map +1 -0
- package/dist/legal-role-heads.pt-br.mjs +63 -0
- package/dist/legal-role-heads.pt-br.mjs.map +1 -0
- package/dist/legal-role-heads.sk.mjs +80 -0
- package/dist/legal-role-heads.sk.mjs.map +1 -0
- package/dist/manifest.mjs +21 -8
- package/dist/names-exclusions.mjs +21 -1
- package/dist/person-stopwords.mjs +121 -1
- package/dist/section-headings.mjs +15 -3
- package/dist/sentence-verb-indicators.mjs +223 -0
- package/dist/sentence-verb-indicators.mjs.map +1 -0
- package/dist/triggers.cs.mjs +107 -3
- package/dist/triggers.en.mjs +32 -0
- package/dist/triggers.es.mjs +43 -4
- package/dist/triggers.fr.mjs +206 -7
- package/dist/triggers.it.mjs +26 -4
- package/dist/triggers.pl.mjs +191 -4
- package/dist/triggers.pt-br.mjs +193 -0
- package/dist/triggers.pt-br.mjs.map +1 -0
- package/dist/triggers.sk.mjs +493 -0
- package/dist/vite.mjs.map +1 -1
- package/dist/wasm.d.mts +56 -39
- package/dist/wasm.mjs +2382 -1408
- package/dist/wasm.mjs.map +1 -1
- package/package.json +7 -2
package/dist/wasm.mjs
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
|
+
import { DEFAULT_ENTITY_LABELS, DETECTION_SOURCES, DETECTOR_PRIORITY, OPERATOR_TYPES } from "./constants.mjs";
|
|
1
2
|
import { TextSearch } from "@stll/text-search-wasm";
|
|
2
|
-
import { at, be, bg, cy, cz, de, dk, ee, es, fi, fr, gb, gr, hr, hu, ie, it, lt, lu, lv, mt, nl, pl, pt, ro, se, si, sk } from "@stll/stdnum";
|
|
3
|
+
import { at, be, bg, br, cy, cz, de, dk, ee, es, fi, fr, gb, gr, hr, hu, ie, it, lt, lu, lv, mt, nl, pl, pt, ro, se, si, sk } from "@stll/stdnum";
|
|
3
4
|
import { toRegex } from "@stll/stdnum/patterns";
|
|
4
5
|
//#region src/search-engine.ts
|
|
5
6
|
let _TextSearch;
|
|
@@ -12,66 +13,6 @@ const getTextSearch = () => {
|
|
|
12
13
|
};
|
|
13
14
|
//#endregion
|
|
14
15
|
//#region src/types.ts
|
|
15
|
-
/**
|
|
16
|
-
* Source of a detected entity span.
|
|
17
|
-
* Ordered by detection layer in the pipeline.
|
|
18
|
-
*/
|
|
19
|
-
const DETECTION_SOURCES = {
|
|
20
|
-
TRIGGER: "trigger",
|
|
21
|
-
REGEX: "regex",
|
|
22
|
-
DENY_LIST: "deny-list",
|
|
23
|
-
LEGAL_FORM: "legal-form",
|
|
24
|
-
GAZETTEER: "gazetteer",
|
|
25
|
-
NER: "ner",
|
|
26
|
-
COREFERENCE: "coreference"
|
|
27
|
-
};
|
|
28
|
-
/**
|
|
29
|
-
* Priority levels for detection sources.
|
|
30
|
-
* Higher = more structurally reliable. Used during
|
|
31
|
-
* overlap resolution so deterministic detectors beat
|
|
32
|
-
* probabilistic ones regardless of raw score.
|
|
33
|
-
*/
|
|
34
|
-
const DETECTOR_PRIORITY = {
|
|
35
|
-
gazetteer: 5,
|
|
36
|
-
trigger: 4,
|
|
37
|
-
"legal-form": 3,
|
|
38
|
-
regex: 3,
|
|
39
|
-
"deny-list": 2,
|
|
40
|
-
coreference: 2,
|
|
41
|
-
ner: 1
|
|
42
|
-
};
|
|
43
|
-
/**
|
|
44
|
-
* Anonymisation operator types. Each operator defines
|
|
45
|
-
* how a confirmed entity is replaced in the output.
|
|
46
|
-
*/
|
|
47
|
-
const OPERATOR_TYPES = ["replace", "redact"];
|
|
48
|
-
/**
|
|
49
|
-
* Canonical entity labels used across the pipeline.
|
|
50
|
-
* NER models may use different native labels; the bench
|
|
51
|
-
* NER wrapper maps model output to these canonical names.
|
|
52
|
-
*
|
|
53
|
-
* These labels are ephemeral: entities are regenerated on
|
|
54
|
-
* every pipeline run and never persisted to the database.
|
|
55
|
-
* Renaming a label here requires no migration.
|
|
56
|
-
*/
|
|
57
|
-
const DEFAULT_ENTITY_LABELS = [
|
|
58
|
-
"person",
|
|
59
|
-
"organization",
|
|
60
|
-
"phone number",
|
|
61
|
-
"address",
|
|
62
|
-
"email address",
|
|
63
|
-
"date",
|
|
64
|
-
"date of birth",
|
|
65
|
-
"bank account number",
|
|
66
|
-
"iban",
|
|
67
|
-
"tax identification number",
|
|
68
|
-
"identity card number",
|
|
69
|
-
"registration number",
|
|
70
|
-
"credit card number",
|
|
71
|
-
"passport number",
|
|
72
|
-
"monetary amount",
|
|
73
|
-
"land parcel"
|
|
74
|
-
];
|
|
75
16
|
const isLegalFormsEnabled = (config) => config.enableLegalForms !== false;
|
|
76
17
|
//#endregion
|
|
77
18
|
//#region src/context.ts
|
|
@@ -95,6 +36,8 @@ const createPipelineContext = () => ({
|
|
|
95
36
|
allowListPromise: null,
|
|
96
37
|
personStopwords: null,
|
|
97
38
|
personStopwordsPromise: null,
|
|
39
|
+
addressStopwords: null,
|
|
40
|
+
addressStopwordsPromise: null,
|
|
98
41
|
firstNameExclusions: null,
|
|
99
42
|
firstNameExclusionCorpusLen: 0,
|
|
100
43
|
genericRoles: null,
|
|
@@ -151,6 +94,7 @@ const LOADER_REGISTRIES = {
|
|
|
151
94
|
hu: () => import("./triggers.hu.mjs"),
|
|
152
95
|
it: () => import("./triggers.it.mjs"),
|
|
153
96
|
pl: () => import("./triggers.pl.mjs"),
|
|
97
|
+
"pt-br": () => import("./triggers.pt-br.mjs"),
|
|
154
98
|
ro: () => import("./triggers.ro.mjs"),
|
|
155
99
|
sk: () => import("./triggers.sk.mjs"),
|
|
156
100
|
sv: () => import("./triggers.sv.mjs")
|
|
@@ -159,7 +103,23 @@ const LOADER_REGISTRIES = {
|
|
|
159
103
|
cs: () => import("./coreference.cs.mjs"),
|
|
160
104
|
de: () => import("./coreference.de.mjs"),
|
|
161
105
|
en: () => import("./coreference.en.mjs"),
|
|
106
|
+
es: () => import("./coreference.es.mjs"),
|
|
107
|
+
fr: () => import("./coreference.fr.mjs"),
|
|
108
|
+
it: () => import("./coreference.it.mjs"),
|
|
109
|
+
pl: () => import("./coreference.pl.mjs"),
|
|
110
|
+
"pt-br": () => import("./coreference.pt-br.mjs"),
|
|
162
111
|
sk: () => import("./coreference.sk.mjs")
|
|
112
|
+
},
|
|
113
|
+
legalRoleHeads: {
|
|
114
|
+
cs: () => import("./legal-role-heads.cs.mjs"),
|
|
115
|
+
de: () => import("./legal-role-heads.de.mjs"),
|
|
116
|
+
en: () => import("./legal-role-heads.en.mjs"),
|
|
117
|
+
es: () => import("./legal-role-heads.es.mjs"),
|
|
118
|
+
fr: () => import("./legal-role-heads.fr.mjs"),
|
|
119
|
+
it: () => import("./legal-role-heads.it.mjs"),
|
|
120
|
+
pl: () => import("./legal-role-heads.pl.mjs"),
|
|
121
|
+
"pt-br": () => import("./legal-role-heads.pt-br.mjs"),
|
|
122
|
+
sk: () => import("./legal-role-heads.sk.mjs")
|
|
163
123
|
}
|
|
164
124
|
};
|
|
165
125
|
const FALLBACK_LANGUAGES = {
|
|
@@ -172,6 +132,7 @@ const FALLBACK_LANGUAGES = {
|
|
|
172
132
|
"hu",
|
|
173
133
|
"it",
|
|
174
134
|
"pl",
|
|
135
|
+
"pt-br",
|
|
175
136
|
"ro",
|
|
176
137
|
"sk",
|
|
177
138
|
"sv"
|
|
@@ -180,6 +141,22 @@ const FALLBACK_LANGUAGES = {
|
|
|
180
141
|
"cs",
|
|
181
142
|
"de",
|
|
182
143
|
"en",
|
|
144
|
+
"es",
|
|
145
|
+
"fr",
|
|
146
|
+
"it",
|
|
147
|
+
"pl",
|
|
148
|
+
"pt-br",
|
|
149
|
+
"sk"
|
|
150
|
+
],
|
|
151
|
+
legalRoleHeads: [
|
|
152
|
+
"cs",
|
|
153
|
+
"de",
|
|
154
|
+
"en",
|
|
155
|
+
"es",
|
|
156
|
+
"fr",
|
|
157
|
+
"it",
|
|
158
|
+
"pl",
|
|
159
|
+
"pt-br",
|
|
183
160
|
"sk"
|
|
184
161
|
]
|
|
185
162
|
};
|
|
@@ -226,838 +203,64 @@ const loadLanguageConfigs = async (configType, mapFn) => {
|
|
|
226
203
|
await Promise.all(loads);
|
|
227
204
|
return results.filter((r) => r !== void 0);
|
|
228
205
|
};
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
console.warn("[anonymize] coreference: no definition patterns loaded; coreference detection will be inactive");
|
|
288
|
-
}
|
|
289
|
-
return patterns;
|
|
290
|
-
}
|
|
291
|
-
ctx.corefPatterns = patterns;
|
|
292
|
-
return patterns;
|
|
293
|
-
};
|
|
294
|
-
const SEARCH_WINDOW = 200;
|
|
295
|
-
/**
|
|
296
|
-
* Check whether an alias has textual similarity to
|
|
297
|
-
* the source entity. Prevents roles and structural
|
|
298
|
-
* terms from being treated as name aliases.
|
|
299
|
-
*
|
|
300
|
-
* Three checks (any passes → similar):
|
|
301
|
-
* 1. Word overlap: a word in the alias appears in the
|
|
302
|
-
* entity (case-insensitive, min 2 chars)
|
|
303
|
-
* 2. Initials: alias letters match first letters of
|
|
304
|
-
* entity words ("TB" ↔ "Tomas Bata")
|
|
305
|
-
* 3. Substring: alias is a substring of the entity
|
|
306
|
-
* or vice versa (min 3 chars)
|
|
307
|
-
*/
|
|
308
|
-
const hasEntitySimilarity = (alias, entityText) => {
|
|
309
|
-
const aliasLower = alias.toLowerCase();
|
|
310
|
-
const entityLower = entityText.toLowerCase();
|
|
311
|
-
if (aliasLower.length >= 3 && entityLower.includes(aliasLower)) return true;
|
|
312
|
-
if (entityLower.length >= 3 && aliasLower.includes(entityLower)) return true;
|
|
313
|
-
const aliasWords = aliasLower.split(/[\s.,;:'"()/-]+/).filter((w) => w.length >= 2);
|
|
314
|
-
const entityWords = entityLower.split(/[\s.,;:'"()/-]+/).filter((w) => w.length >= 2);
|
|
315
|
-
const entityWordSet = new Set(entityWords);
|
|
316
|
-
for (const word of aliasWords) if (entityWordSet.has(word)) return true;
|
|
317
|
-
if (/^[\p{Lu}]+$/u.test(alias) && alias.length >= 2 && alias.length <= entityWords.length) {
|
|
318
|
-
for (let start = 0; start <= entityWords.length - alias.length; start++) if (entityWords.slice(start, start + alias.length).map((w) => w.charAt(0)).join("") === aliasLower) return true;
|
|
319
|
-
}
|
|
320
|
-
return false;
|
|
321
|
-
};
|
|
322
|
-
/**
|
|
323
|
-
* Scan for defined-term patterns near known entities.
|
|
324
|
-
*
|
|
325
|
-
* Legal documents universally follow:
|
|
326
|
-
* "Dr. Heinrich Muller (hereinafter 'the Seller')..."
|
|
327
|
-
*
|
|
328
|
-
* After NER detects the entity, this function scans for
|
|
329
|
-
* definitional patterns within +/-200 chars and extracts
|
|
330
|
-
* the alias. Returns alias + label pairs that can be added
|
|
331
|
-
* to the gazetteer for a full-text re-scan.
|
|
332
|
-
*/
|
|
333
|
-
/**
|
|
334
|
-
* Labels that can be the source of a coreference alias.
|
|
335
|
-
* Only parties (person, organization) have defined-term
|
|
336
|
-
* aliases in legal text. Dates, addresses, IDs do not.
|
|
337
|
-
*/
|
|
338
|
-
const COREF_SOURCE_LABELS = new Set(["person", "organization"]);
|
|
339
|
-
const extractDefinedTerms = async (fullText, entities, ctx = defaultContext) => {
|
|
340
|
-
const [definitionPatterns, roleStops] = await Promise.all([getDefinitionPatterns(ctx), getRoleStopSet(ctx)]);
|
|
341
|
-
const terms = [];
|
|
342
|
-
const seen = /* @__PURE__ */ new Set();
|
|
343
|
-
const sorted = [...entities].sort((a, b) => a.start - b.start);
|
|
344
|
-
for (const { pattern } of definitionPatterns) {
|
|
345
|
-
pattern.lastIndex = 0;
|
|
346
|
-
for (let match = pattern.exec(fullText); match !== null; match = pattern.exec(fullText)) {
|
|
347
|
-
const alias = match[1]?.trim();
|
|
348
|
-
if (!alias || alias.length < 2) continue;
|
|
349
|
-
if (roleStops.has(alias.toLowerCase())) continue;
|
|
350
|
-
const defPos = match.index;
|
|
351
|
-
let bestEntity = null;
|
|
352
|
-
for (let i = sorted.length - 1; i >= 0; i--) {
|
|
353
|
-
const e = sorted[i];
|
|
354
|
-
if (e === void 0) continue;
|
|
355
|
-
if (e.end > defPos) continue;
|
|
356
|
-
if (defPos - e.end > SEARCH_WINDOW) break;
|
|
357
|
-
if (!COREF_SOURCE_LABELS.has(e.label)) continue;
|
|
358
|
-
bestEntity = e;
|
|
359
|
-
break;
|
|
360
|
-
}
|
|
361
|
-
if (bestEntity === null) continue;
|
|
362
|
-
const gapText = fullText.slice(bestEntity.end, defPos);
|
|
363
|
-
if (/(?:;|\.(?=\s*(?:["'„‚(]*\p{Lu}|$)))/u.test(gapText)) continue;
|
|
364
|
-
if (!hasEntitySimilarity(alias, bestEntity.text)) continue;
|
|
365
|
-
const key = `${alias.toLowerCase()}::${bestEntity.label}`;
|
|
366
|
-
if (seen.has(key)) continue;
|
|
367
|
-
seen.add(key);
|
|
368
|
-
terms.push({
|
|
369
|
-
alias,
|
|
370
|
-
label: bestEntity.label,
|
|
371
|
-
definitionStart: defPos,
|
|
372
|
-
sourceText: bestEntity.text
|
|
373
|
-
});
|
|
374
|
-
}
|
|
375
|
-
}
|
|
376
|
-
return terms;
|
|
377
|
-
};
|
|
378
|
-
/**
|
|
379
|
-
* Check if a character is a Unicode word character
|
|
380
|
-
* (letter, digit, or combining mark). Used for word
|
|
381
|
-
* boundary checks in coreference matching.
|
|
382
|
-
*/
|
|
383
|
-
const isWordChar = (ch) => {
|
|
384
|
-
if (ch === void 0) return false;
|
|
385
|
-
return /[\p{L}\p{M}\p{N}]/u.test(ch);
|
|
386
|
-
};
|
|
387
|
-
/**
|
|
388
|
-
* Find all occurrences of defined-term aliases in the
|
|
389
|
-
* full text. Returns Entity spans for each match.
|
|
390
|
-
*
|
|
391
|
-
* Respects word boundaries: "Kupující" must not match
|
|
392
|
-
* inside "Kupujícímu". A match is valid only if the
|
|
393
|
-
* character before the start and after the end are NOT
|
|
394
|
-
* word characters (letter/digit).
|
|
395
|
-
*
|
|
396
|
-
* Populates `ctx.corefSourceMap` with entries linking
|
|
397
|
-
* each coref entity to its source entity text, for
|
|
398
|
-
* consistent placeholder numbering.
|
|
399
|
-
*/
|
|
400
|
-
const findCoreferenceSpans = (fullText, terms, ctx = defaultContext) => {
|
|
401
|
-
const results = [];
|
|
402
|
-
for (const term of terms) {
|
|
403
|
-
let searchFrom = 0;
|
|
404
|
-
while (searchFrom < fullText.length) {
|
|
405
|
-
const idx = fullText.indexOf(term.alias, searchFrom);
|
|
406
|
-
if (idx === -1) break;
|
|
407
|
-
const matchEnd = idx + term.alias.length;
|
|
408
|
-
const charBefore = idx > 0 ? fullText[idx - 1] : void 0;
|
|
409
|
-
const charAfter = fullText[matchEnd];
|
|
410
|
-
if (isWordChar(charBefore) || isWordChar(charAfter)) {
|
|
411
|
-
searchFrom = idx + 1;
|
|
412
|
-
continue;
|
|
413
|
-
}
|
|
414
|
-
const entity = {
|
|
415
|
-
start: idx,
|
|
416
|
-
end: matchEnd,
|
|
417
|
-
label: term.label,
|
|
418
|
-
text: term.alias,
|
|
419
|
-
score: .95,
|
|
420
|
-
source: DETECTION_SOURCES.COREFERENCE
|
|
421
|
-
};
|
|
422
|
-
ctx.corefSourceMap.set(corefKey(entity), term.sourceText);
|
|
423
|
-
results.push(entity);
|
|
424
|
-
searchFrom = matchEnd;
|
|
425
|
-
}
|
|
426
|
-
}
|
|
427
|
-
return results;
|
|
428
|
-
};
|
|
429
|
-
//#endregion
|
|
430
|
-
//#region src/detectors/gazetteer.ts
|
|
431
|
-
const MAX_EDIT_DISTANCE = 2;
|
|
432
|
-
const MIN_FUZZY_LENGTH = 4;
|
|
433
|
-
const MAX_PREFIX_OVERSHOOT = 7;
|
|
434
|
-
/**
|
|
435
|
-
* Collect all searchable strings (canonical + variants)
|
|
436
|
-
* from gazetteer entries, mapped to their labels and
|
|
437
|
-
* entry IDs.
|
|
438
|
-
*/
|
|
439
|
-
const buildSearchTerms = (entries) => {
|
|
440
|
-
const terms = /* @__PURE__ */ new Map();
|
|
441
|
-
for (const entry of entries) {
|
|
442
|
-
const meta = {
|
|
443
|
-
label: entry.label,
|
|
444
|
-
entryId: entry.id
|
|
445
|
-
};
|
|
446
|
-
terms.set(entry.canonical, meta);
|
|
447
|
-
for (const variant of entry.variants) terms.set(variant, meta);
|
|
448
|
-
}
|
|
449
|
-
return terms;
|
|
450
|
-
};
|
|
451
|
-
/**
|
|
452
|
-
* Build TextSearch-compatible patterns from gazetteer
|
|
453
|
-
* entries. Returns:
|
|
454
|
-
* - Exact literal patterns for all terms
|
|
455
|
-
* - Fuzzy patterns (distance: 2) for terms >= 4 chars
|
|
456
|
-
* - Parallel metadata arrays for post-processing
|
|
457
|
-
*
|
|
458
|
-
* Patterns are ordered: all exact first, then all
|
|
459
|
-
* fuzzy. The isFuzzy array marks which are which.
|
|
460
|
-
*/
|
|
461
|
-
const buildGazetteerPatterns = (entries) => {
|
|
462
|
-
const terms = buildSearchTerms(entries);
|
|
463
|
-
const patterns = [];
|
|
464
|
-
const labels = [];
|
|
465
|
-
const isFuzzy = [];
|
|
466
|
-
for (const [term, meta] of terms) {
|
|
467
|
-
patterns.push({
|
|
468
|
-
pattern: term,
|
|
469
|
-
literal: true,
|
|
470
|
-
wholeWords: false
|
|
471
|
-
});
|
|
472
|
-
labels.push(meta.label);
|
|
473
|
-
isFuzzy.push(false);
|
|
474
|
-
}
|
|
475
|
-
for (const [term, meta] of terms) {
|
|
476
|
-
if (term.length < MIN_FUZZY_LENGTH) continue;
|
|
477
|
-
patterns.push({
|
|
478
|
-
pattern: term,
|
|
479
|
-
distance: MAX_EDIT_DISTANCE
|
|
480
|
-
});
|
|
481
|
-
labels.push(meta.label);
|
|
482
|
-
isFuzzy.push(true);
|
|
483
|
-
}
|
|
484
|
-
return {
|
|
485
|
-
patterns,
|
|
486
|
-
data: {
|
|
487
|
-
labels,
|
|
488
|
-
isFuzzy
|
|
489
|
-
}
|
|
490
|
-
};
|
|
491
|
-
};
|
|
492
|
-
/**
|
|
493
|
-
* Process gazetteer matches from the unified literal
|
|
494
|
-
* search. Receives all matches; filters to the
|
|
495
|
-
* gazetteer slice via sliceStart/sliceEnd.
|
|
496
|
-
*
|
|
497
|
-
* Exact matches get score 0.9; fuzzy matches get
|
|
498
|
-
* 0.85. Fuzzy matches that overlap an exact match
|
|
499
|
-
* are dropped.
|
|
500
|
-
*
|
|
501
|
-
* For exact matches, attempts prefix extension for
|
|
502
|
-
* legal suffixes ("a.s.", "GmbH", "s.r.o." after
|
|
503
|
-
* the matched term).
|
|
504
|
-
*/
|
|
505
|
-
const processGazetteerMatches = (allMatches, sliceStart, sliceEnd, fullText, data) => {
|
|
506
|
-
const results = [];
|
|
507
|
-
const exactSpans = [];
|
|
508
|
-
for (const match of allMatches) {
|
|
509
|
-
const idx = match.pattern;
|
|
510
|
-
if (idx < sliceStart || idx >= sliceEnd) continue;
|
|
511
|
-
const localIdx = idx - sliceStart;
|
|
512
|
-
if (data.isFuzzy[localIdx]) continue;
|
|
513
|
-
const label = data.labels[localIdx];
|
|
514
|
-
if (!label) continue;
|
|
515
|
-
const extended = tryPrefixExtension(fullText, match.start, match.end);
|
|
516
|
-
const end = extended?.end ?? match.end;
|
|
517
|
-
const text = extended?.text ?? fullText.slice(match.start, match.end);
|
|
518
|
-
exactSpans.push({
|
|
519
|
-
start: match.start,
|
|
520
|
-
end
|
|
521
|
-
});
|
|
522
|
-
results.push({
|
|
523
|
-
start: match.start,
|
|
524
|
-
end,
|
|
525
|
-
label,
|
|
526
|
-
text,
|
|
527
|
-
score: .9,
|
|
528
|
-
source: DETECTION_SOURCES.GAZETTEER
|
|
529
|
-
});
|
|
530
|
-
}
|
|
531
|
-
for (const match of allMatches) {
|
|
532
|
-
const idx = match.pattern;
|
|
533
|
-
if (idx < sliceStart || idx >= sliceEnd) continue;
|
|
534
|
-
const localIdx = idx - sliceStart;
|
|
535
|
-
if (!data.isFuzzy[localIdx]) continue;
|
|
536
|
-
if (match.distance === 0) continue;
|
|
537
|
-
const label = data.labels[localIdx];
|
|
538
|
-
if (!label) continue;
|
|
539
|
-
if (exactSpans.some((e) => match.start < e.end && match.end > e.start)) continue;
|
|
540
|
-
const matchText = fullText.slice(match.start, match.end);
|
|
541
|
-
results.push({
|
|
542
|
-
start: match.start,
|
|
543
|
-
end: match.end,
|
|
544
|
-
label,
|
|
545
|
-
text: matchText,
|
|
546
|
-
score: .85,
|
|
547
|
-
source: DETECTION_SOURCES.GAZETTEER
|
|
548
|
-
});
|
|
549
|
-
}
|
|
550
|
-
return results;
|
|
551
|
-
};
|
|
552
|
-
/**
|
|
553
|
-
* Try to extend an exact match to capture one
|
|
554
|
-
* trailing token (max 6 chars) that may be a legal
|
|
555
|
-
* entity suffix (e.g., "a.s.", "GmbH", "s.r.o.").
|
|
556
|
-
*
|
|
557
|
-
* Does not validate the token against a legal-forms
|
|
558
|
-
* list; false extensions are filtered by mergeAndDedup
|
|
559
|
-
* when a legal-form detector produces a competing
|
|
560
|
-
* entity with the correct span.
|
|
561
|
-
*/
|
|
562
|
-
const tryPrefixExtension = (fullText, start, end) => {
|
|
563
|
-
const maxEnd = Math.min(end + MAX_PREFIX_OVERSHOOT, fullText.length);
|
|
564
|
-
if (maxEnd <= end + 1) return null;
|
|
565
|
-
const after = fullText.slice(end, maxEnd);
|
|
566
|
-
if (!after.startsWith(" ")) return null;
|
|
567
|
-
const nextSpace = after.indexOf(" ", 1);
|
|
568
|
-
const suffixEnd = nextSpace !== -1 ? nextSpace : after.length;
|
|
569
|
-
if (suffixEnd <= 1) return null;
|
|
570
|
-
const newEnd = end + suffixEnd;
|
|
571
|
-
return {
|
|
572
|
-
end: newEnd,
|
|
573
|
-
text: fullText.slice(start, newEnd)
|
|
574
|
-
};
|
|
575
|
-
};
|
|
576
|
-
//#endregion
|
|
577
|
-
//#region src/util/text.ts
|
|
578
|
-
/**
|
|
579
|
-
* Shared text utilities for detectors.
|
|
580
|
-
*
|
|
581
|
-
* Extracted from names.ts and deny-list.ts to avoid
|
|
582
|
-
* duplicating regex constants and helper functions.
|
|
583
|
-
*/
|
|
584
|
-
/** Matches a string that starts with an uppercase letter. */
|
|
585
|
-
const UPPER_START_RE = /^\p{Lu}/u;
|
|
586
|
-
/** Matches a string consisting entirely of uppercase letters. */
|
|
587
|
-
const ALL_UPPER_RE = /^\p{Lu}+$/u;
|
|
588
|
-
const SENTENCE_END_RE = /[.!?]/;
|
|
589
|
-
/**
|
|
590
|
-
* Detect whether a position is at the start of a sentence.
|
|
591
|
-
* Looks backward past whitespace for sentence-ending
|
|
592
|
-
* punctuation (.!?). Position 0 and positions preceded
|
|
593
|
-
* only by whitespace are considered sentence starts.
|
|
594
|
-
*/
|
|
595
|
-
const isSentenceStart = (text, pos) => {
|
|
596
|
-
if (pos === 0) return true;
|
|
597
|
-
let i = pos - 1;
|
|
598
|
-
while (i >= 0 && /\s/.test(text[i] ?? "")) i--;
|
|
599
|
-
if (i < 0) return true;
|
|
600
|
-
return SENTENCE_END_RE.test(text[i] ?? "");
|
|
601
|
-
};
|
|
602
|
-
//#endregion
|
|
603
|
-
//#region src/detectors/names.ts
|
|
604
|
-
const getCorpus = (ctx) => ctx.nameCorpus;
|
|
605
|
-
const getNameCorpusFirstNames = (ctx = defaultContext) => ctx.nameCorpus?.firstNamesList ?? [];
|
|
606
|
-
const getNameCorpusSurnames = (ctx = defaultContext) => ctx.nameCorpus?.surnamesList ?? [];
|
|
607
|
-
const getNameCorpusTitles = (ctx = defaultContext) => ctx.nameCorpus?.titlesList ?? [];
|
|
608
|
-
/**
|
|
609
|
-
* Load name corpus data from injected dictionaries
|
|
610
|
-
* and legacy config files. Merges all sources.
|
|
611
|
-
*
|
|
612
|
-
* Safe to call multiple times; only loads once per
|
|
613
|
-
* context. Must be called before detectNameCorpus or
|
|
614
|
-
* the getNameCorpus*() accessors are used.
|
|
615
|
-
*
|
|
616
|
-
* @param dictionaries Optional pre-loaded dictionaries
|
|
617
|
-
* with per-language first names and surnames. When
|
|
618
|
-
* omitted, only legacy config files are used.
|
|
619
|
-
*/
|
|
620
|
-
const initNameCorpus = (ctx = defaultContext, dictionaries) => {
|
|
621
|
-
if (ctx.nameCorpusPromise) return ctx.nameCorpusPromise;
|
|
622
|
-
const promise = (async () => {
|
|
623
|
-
try {
|
|
624
|
-
const [legacyFirstMod, legacySurnameMod, titleMod, exclusionMod] = await Promise.all([
|
|
625
|
-
import("./names-first.mjs"),
|
|
626
|
-
import("./names-surnames.mjs"),
|
|
627
|
-
import("./names-title-tokens.mjs"),
|
|
628
|
-
import("./names-exclusions.mjs")
|
|
629
|
-
]);
|
|
630
|
-
const firstNames = [...legacyFirstMod.default.names];
|
|
631
|
-
if (dictionaries?.firstNames) for (const names of Object.values(dictionaries.firstNames)) for (const name of names) firstNames.push(name);
|
|
632
|
-
const surnames = [...legacySurnameMod.default.names];
|
|
633
|
-
if (dictionaries?.surnames) for (const names of Object.values(dictionaries.surnames)) for (const name of names) surnames.push(name);
|
|
634
|
-
const dedup = (arr) => {
|
|
635
|
-
const seen = /* @__PURE__ */ new Set();
|
|
636
|
-
const result = [];
|
|
637
|
-
for (const item of arr) {
|
|
638
|
-
if (seen.has(item)) continue;
|
|
639
|
-
seen.add(item);
|
|
640
|
-
result.push(item);
|
|
641
|
-
}
|
|
642
|
-
return result;
|
|
643
|
-
};
|
|
644
|
-
const dedupFirst = dedup(firstNames);
|
|
645
|
-
const dedupSurnames = dedup(surnames);
|
|
646
|
-
const titles = titleMod.default.tokens;
|
|
647
|
-
const exclusions = exclusionMod.default.words;
|
|
648
|
-
ctx.nameCorpus = {
|
|
649
|
-
firstNames: Object.freeze(new Set(dedupFirst)),
|
|
650
|
-
surnames: Object.freeze(new Set(dedupSurnames)),
|
|
651
|
-
titleTokens: Object.freeze(new Set(titles)),
|
|
652
|
-
excludedWords: Object.freeze(new Set(exclusions)),
|
|
653
|
-
firstNamesList: Object.freeze(dedupFirst),
|
|
654
|
-
surnamesList: Object.freeze(dedupSurnames),
|
|
655
|
-
titlesList: Object.freeze(titles),
|
|
656
|
-
excludedList: Object.freeze(exclusions)
|
|
657
|
-
};
|
|
658
|
-
} catch (err) {
|
|
659
|
-
ctx.nameCorpusPromise = null;
|
|
660
|
-
console.warn("[anonymize] Failed to load name corpus JSON — name detection disabled:", err);
|
|
661
|
-
}
|
|
662
|
-
})();
|
|
663
|
-
ctx.nameCorpusPromise = promise;
|
|
664
|
-
return promise;
|
|
665
|
-
};
|
|
666
|
-
const INFLECTION_SUFFIXES = [
|
|
667
|
-
"ovi",
|
|
668
|
-
"em",
|
|
669
|
-
"om",
|
|
670
|
-
"ou",
|
|
671
|
-
"é",
|
|
672
|
-
"a",
|
|
673
|
-
"u"
|
|
674
|
-
];
|
|
675
|
-
/**
|
|
676
|
-
* Strip common Czech/Slovak case suffixes from a token.
|
|
677
|
-
* Returns candidate base forms if stripping produces a
|
|
678
|
-
* plausible name (capitalised, length >= 3).
|
|
679
|
-
*
|
|
680
|
-
* For the "-ou" instrumental feminine suffix, also yields
|
|
681
|
-
* base + "a" (e.g., "Editou" → "Edit" and "Edita")
|
|
682
|
-
* because Czech feminine names decline -a → -ou.
|
|
683
|
-
*/
|
|
684
|
-
const stripInflection = (token) => {
|
|
685
|
-
const candidates = [];
|
|
686
|
-
for (const suffix of INFLECTION_SUFFIXES) if (token.length > suffix.length + 2 && token.endsWith(suffix)) {
|
|
687
|
-
const base = token.slice(0, -suffix.length);
|
|
688
|
-
if (/^\p{Lu}/u.test(base)) {
|
|
689
|
-
candidates.push(base);
|
|
690
|
-
if (suffix === "ou" || suffix === "é" || suffix === "u") candidates.push(`${base}a`);
|
|
691
|
-
}
|
|
692
|
-
}
|
|
693
|
-
return candidates;
|
|
694
|
-
};
|
|
695
|
-
const TOKEN_TYPE = {
|
|
696
|
-
NAME: "name",
|
|
697
|
-
SURNAME: "surname",
|
|
698
|
-
TITLE: "title",
|
|
699
|
-
ABBREVIATION: "abbreviation",
|
|
700
|
-
CAPITALIZED: "capitalized",
|
|
701
|
-
OTHER: "other"
|
|
702
|
-
};
|
|
703
|
-
const PERSON_CHAIN_BREAK_RE$1 = /[!?;:]/u;
|
|
704
|
-
const isInitialContinuationGap$1 = (text, gap) => /^\p{Lu}$/u.test(text) && /^\.[^\S\n]{1,2}$/u.test(gap) || /^[^\S\n]{1,2}(?:\p{Lu}\.[^\S\n]{1,2})+$/u.test(gap);
|
|
705
|
-
/**
|
|
706
|
-
* Check if a token is in the first-name set, either
|
|
707
|
-
* directly or after stripping Czech/Slovak inflection.
|
|
708
|
-
*/
|
|
709
|
-
const isFirstNameToken = (token, corpus) => {
|
|
710
|
-
if (corpus.firstNames.has(token)) return true;
|
|
711
|
-
return stripInflection(token).some((b) => corpus.firstNames.has(b));
|
|
712
|
-
};
|
|
713
|
-
/**
|
|
714
|
-
* Check if a token is in the surname set, either
|
|
715
|
-
* directly or after stripping Czech/Slovak inflection.
|
|
716
|
-
*/
|
|
717
|
-
const isSurnameToken = (token, corpus) => {
|
|
718
|
-
if (corpus.surnames.has(token)) return true;
|
|
719
|
-
return stripInflection(token).some((b) => corpus.surnames.has(b));
|
|
720
|
-
};
|
|
721
|
-
/**
|
|
722
|
-
* Check if a token looks like a single-letter
|
|
723
|
-
* abbreviation: "J.", "M.", etc.
|
|
724
|
-
*/
|
|
725
|
-
const isAbbreviation = (token) => token.length === 2 && /^\p{Lu}$/u.test(token[0] ?? "") && token[1] === ".";
|
|
726
|
-
const segmenter$1 = new Intl.Segmenter(void 0, { granularity: "word" });
|
|
727
|
-
/**
|
|
728
|
-
* Split text into word segments using Intl.Segmenter.
|
|
729
|
-
* Only returns segments flagged as words.
|
|
730
|
-
*/
|
|
731
|
-
const segmentWords = (fullText) => {
|
|
732
|
-
const words = [];
|
|
733
|
-
for (const seg of segmenter$1.segment(fullText)) if (seg.isWordLike) words.push({
|
|
734
|
-
text: seg.segment,
|
|
735
|
-
start: seg.index,
|
|
736
|
-
end: seg.index + seg.segment.length
|
|
737
|
-
});
|
|
738
|
-
return words;
|
|
739
|
-
};
|
|
740
|
-
/** NAME or SURNAME — both represent corpus-matched tokens */
|
|
741
|
-
const isCorpusMatch = (type) => type === TOKEN_TYPE.NAME || type === TOKEN_TYPE.SURNAME;
|
|
742
|
-
const classifyToken = (word, corpus) => {
|
|
743
|
-
const { text, start, end } = word;
|
|
744
|
-
const lower = text.toLowerCase();
|
|
745
|
-
const stripped = text.endsWith(".") ? text.slice(0, -1).toLowerCase() : lower;
|
|
746
|
-
if (corpus.titleTokens.has(stripped)) return {
|
|
747
|
-
text,
|
|
748
|
-
type: TOKEN_TYPE.TITLE,
|
|
749
|
-
start,
|
|
750
|
-
end
|
|
751
|
-
};
|
|
752
|
-
if (isAbbreviation(text)) return {
|
|
753
|
-
text,
|
|
754
|
-
type: TOKEN_TYPE.ABBREVIATION,
|
|
755
|
-
start,
|
|
756
|
-
end
|
|
757
|
-
};
|
|
758
|
-
if (corpus.excludedWords.has(lower)) return {
|
|
759
|
-
text,
|
|
760
|
-
type: TOKEN_TYPE.OTHER,
|
|
761
|
-
start,
|
|
762
|
-
end
|
|
763
|
-
};
|
|
764
|
-
if (text.length < 3) return {
|
|
765
|
-
text,
|
|
766
|
-
type: TOKEN_TYPE.OTHER,
|
|
767
|
-
start,
|
|
768
|
-
end
|
|
769
|
-
};
|
|
770
|
-
if (text.length > 3 && ALL_UPPER_RE.test(text)) return {
|
|
771
|
-
text,
|
|
772
|
-
type: TOKEN_TYPE.OTHER,
|
|
773
|
-
start,
|
|
774
|
-
end
|
|
775
|
-
};
|
|
776
|
-
if (!UPPER_START_RE.test(text)) return {
|
|
777
|
-
text,
|
|
778
|
-
type: TOKEN_TYPE.OTHER,
|
|
779
|
-
start,
|
|
780
|
-
end
|
|
781
|
-
};
|
|
782
|
-
if (isFirstNameToken(text, corpus)) return {
|
|
783
|
-
text,
|
|
784
|
-
type: TOKEN_TYPE.NAME,
|
|
785
|
-
start,
|
|
786
|
-
end
|
|
787
|
-
};
|
|
788
|
-
if (isSurnameToken(text, corpus)) return {
|
|
789
|
-
text,
|
|
790
|
-
type: TOKEN_TYPE.SURNAME,
|
|
791
|
-
start,
|
|
792
|
-
end
|
|
793
|
-
};
|
|
794
|
-
return {
|
|
795
|
-
text,
|
|
796
|
-
type: TOKEN_TYPE.CAPITALIZED,
|
|
797
|
-
start,
|
|
798
|
-
end
|
|
799
|
-
};
|
|
800
|
-
};
|
|
801
|
-
/**
|
|
802
|
-
* Detect person names by looking up tokens against the
|
|
803
|
-
* name corpus, then chaining adjacent name-like tokens.
|
|
804
|
-
*
|
|
805
|
-
* Requires initNameCorpus() to have been called first.
|
|
806
|
-
* If not initialized, returns an empty array.
|
|
807
|
-
*
|
|
808
|
-
* Scoring:
|
|
809
|
-
* TITLE + NAME/SURNAME → 0.95
|
|
810
|
-
* NAME + NAME/SURNAME → 0.9
|
|
811
|
-
* SURNAME + NAME/SURNAME → 0.9
|
|
812
|
-
* NAME + CAPITALIZED → 0.7
|
|
813
|
-
* ABBREVIATION + NAME → 0.7
|
|
814
|
-
* Standalone NAME → 0.5 (low confidence)
|
|
815
|
-
* Standalone SURNAME → skip (too ambiguous)
|
|
816
|
-
*/
|
|
817
|
-
const detectNameCorpus = (fullText, ctx = defaultContext) => {
|
|
818
|
-
const corpus = getCorpus(ctx);
|
|
819
|
-
if (!corpus) return [];
|
|
820
|
-
const tokens = segmentWords(fullText).map((w) => classifyToken(w, corpus));
|
|
821
|
-
const entities = [];
|
|
822
|
-
const consumed = /* @__PURE__ */ new Set();
|
|
823
|
-
for (let i = 0; i < tokens.length; i++) {
|
|
824
|
-
if (consumed.has(i)) continue;
|
|
825
|
-
const token = tokens[i];
|
|
826
|
-
if (!token) continue;
|
|
827
|
-
if (token.type !== TOKEN_TYPE.TITLE && token.type !== TOKEN_TYPE.NAME && token.type !== TOKEN_TYPE.SURNAME && token.type !== TOKEN_TYPE.ABBREVIATION) continue;
|
|
828
|
-
const MAX_CHAIN = 5;
|
|
829
|
-
const chain = [token];
|
|
830
|
-
let j = i + 1;
|
|
831
|
-
while (j < tokens.length && chain.length < MAX_CHAIN) {
|
|
832
|
-
const next = tokens[j];
|
|
833
|
-
if (!next) break;
|
|
834
|
-
const prev = chain.at(-1);
|
|
835
|
-
if (prev) {
|
|
836
|
-
const gap = fullText.slice(prev.end, next.start);
|
|
837
|
-
const breaksOnPeriod = gap.includes(".") && !isInitialContinuationGap$1(prev.text, gap);
|
|
838
|
-
if (gap.includes("\n") || PERSON_CHAIN_BREAK_RE$1.test(gap) || breaksOnPeriod) break;
|
|
839
|
-
}
|
|
840
|
-
if (next.type === TOKEN_TYPE.NAME || next.type === TOKEN_TYPE.SURNAME || next.type === TOKEN_TYPE.TITLE || next.type === TOKEN_TYPE.ABBREVIATION || next.type === TOKEN_TYPE.CAPITALIZED) {
|
|
841
|
-
chain.push(next);
|
|
842
|
-
j++;
|
|
843
|
-
} else break;
|
|
844
|
-
}
|
|
845
|
-
const hasTitle = chain.some((t) => t.type === TOKEN_TYPE.TITLE);
|
|
846
|
-
const hasCorpusName = chain.some((t) => isCorpusMatch(t.type));
|
|
847
|
-
const hasFirstName = chain.some((t) => t.type === TOKEN_TYPE.NAME);
|
|
848
|
-
const hasAbbreviation = chain.some((t) => t.type === TOKEN_TYPE.ABBREVIATION);
|
|
849
|
-
const corpusCount = chain.filter((t) => isCorpusMatch(t.type)).length;
|
|
850
|
-
const capitalizedCount = chain.filter((t) => t.type === TOKEN_TYPE.CAPITALIZED).length;
|
|
851
|
-
let score = 0;
|
|
852
|
-
if (hasTitle && hasCorpusName) score = .95;
|
|
853
|
-
else if (corpusCount >= 2) score = .9;
|
|
854
|
-
else if (hasCorpusName && capitalizedCount > 0) score = .7;
|
|
855
|
-
else if (hasAbbreviation && hasCorpusName) score = .7;
|
|
856
|
-
else if (hasFirstName && chain.length === 1) {
|
|
857
|
-
if (isSentenceStart(fullText, token.start)) continue;
|
|
858
|
-
score = .5;
|
|
859
|
-
} else if (!hasFirstName && chain.length === 1 && chain[0]?.type === TOKEN_TYPE.SURNAME) continue;
|
|
860
|
-
else if (hasTitle && chain.length === 1) continue;
|
|
861
|
-
else {
|
|
862
|
-
if (!hasCorpusName) continue;
|
|
863
|
-
score = .5;
|
|
864
|
-
}
|
|
865
|
-
const first = chain.at(0);
|
|
866
|
-
const last = chain.at(-1);
|
|
867
|
-
if (!first || !last) continue;
|
|
868
|
-
const start = first.start;
|
|
869
|
-
const end = last.end;
|
|
870
|
-
const text = fullText.slice(start, end);
|
|
871
|
-
for (let k = i; k < i + chain.length; k++) consumed.add(k);
|
|
872
|
-
entities.push({
|
|
873
|
-
start,
|
|
874
|
-
end,
|
|
875
|
-
label: "person",
|
|
876
|
-
text,
|
|
877
|
-
score,
|
|
878
|
-
source: DETECTION_SOURCES.REGEX
|
|
879
|
-
});
|
|
880
|
-
}
|
|
881
|
-
return entities;
|
|
882
|
-
};
|
|
883
|
-
//#endregion
|
|
884
|
-
//#region src/config/titles.ts
|
|
885
|
-
/**
|
|
886
|
-
* Academic and professional title prefixes.
|
|
887
|
-
* Plain text; the detector auto-escapes for regex.
|
|
888
|
-
* Sorted longest-first at build time.
|
|
889
|
-
*/
|
|
890
|
-
const TITLE_PREFIXES = [
|
|
891
|
-
"Ing.",
|
|
892
|
-
"Mgr.",
|
|
893
|
-
"MgA.",
|
|
894
|
-
"Bc.",
|
|
895
|
-
"BcA.",
|
|
896
|
-
"JUDr.",
|
|
897
|
-
"MUDr.",
|
|
898
|
-
"MVDr.",
|
|
899
|
-
"MDDr.",
|
|
900
|
-
"PhDr.",
|
|
901
|
-
"RNDr.",
|
|
902
|
-
"PaedDr.",
|
|
903
|
-
"ThDr.",
|
|
904
|
-
"ThLic.",
|
|
905
|
-
"ICDr.",
|
|
906
|
-
"RSDr.",
|
|
907
|
-
"PharmDr.",
|
|
908
|
-
"artD.",
|
|
909
|
-
"akad.",
|
|
910
|
-
"doc.",
|
|
911
|
-
"prof.",
|
|
912
|
-
"ao. Univ.-Prof.",
|
|
913
|
-
"o. Univ.-Prof.",
|
|
914
|
-
"Univ.-Prof.",
|
|
915
|
-
"Hon.-Prof.",
|
|
916
|
-
"em. Prof.",
|
|
917
|
-
"Dr. med. dent.",
|
|
918
|
-
"Dr. med. vet.",
|
|
919
|
-
"Dr. med.",
|
|
920
|
-
"Dr. rer. nat.",
|
|
921
|
-
"Dr. rer. soc.",
|
|
922
|
-
"Dr. rer. pol.",
|
|
923
|
-
"Dr. sc. tech.",
|
|
924
|
-
"Dr. sc. nat.",
|
|
925
|
-
"Dr. sc. hum.",
|
|
926
|
-
"Dr. iur.",
|
|
927
|
-
"Dr. jur.",
|
|
928
|
-
"Dr. theol.",
|
|
929
|
-
"Dr. oec.",
|
|
930
|
-
"Dr. techn.",
|
|
931
|
-
"Dr. h. c.",
|
|
932
|
-
"Dr. phil.",
|
|
933
|
-
"Dr.-Ing.",
|
|
934
|
-
"Dr. Ing.",
|
|
935
|
-
"Dr.",
|
|
936
|
-
"Dipl.-Wirt.-Ing.",
|
|
937
|
-
"Dipl.-Betriebsw.",
|
|
938
|
-
"Dipl.-Inform.",
|
|
939
|
-
"Dipl.-Volksw.",
|
|
940
|
-
"Dipl.-Psych.",
|
|
941
|
-
"Dipl.-Phys.",
|
|
942
|
-
"Dipl.-Chem.",
|
|
943
|
-
"Dipl.-Biol.",
|
|
944
|
-
"Dipl.-Math.",
|
|
945
|
-
"Dipl.-Päd.",
|
|
946
|
-
"Dipl.-Soz.",
|
|
947
|
-
"Dipl.-Kfm.",
|
|
948
|
-
"Dipl.-Jur.",
|
|
949
|
-
"Dipl. Ing.",
|
|
950
|
-
"Dipl.-Ing.",
|
|
951
|
-
"Mag. rer. soc. oec.",
|
|
952
|
-
"Mag. rer. nat.",
|
|
953
|
-
"Mag. phil.",
|
|
954
|
-
"Mag. iur.",
|
|
955
|
-
"Mag. arch.",
|
|
956
|
-
"Mag. pharm.",
|
|
957
|
-
"Mag. (FH)",
|
|
958
|
-
"Mag.",
|
|
959
|
-
"Bakk. rer. nat.",
|
|
960
|
-
"Bakk. techn.",
|
|
961
|
-
"Bakk. phil.",
|
|
962
|
-
"Bakk.",
|
|
963
|
-
"Lic. phil.",
|
|
964
|
-
"Lic. iur.",
|
|
965
|
-
"Lic. oec.",
|
|
966
|
-
"Lic. theol.",
|
|
967
|
-
"Lic.",
|
|
968
|
-
"Priv.-Doz.",
|
|
969
|
-
"PD",
|
|
970
|
-
"RA"
|
|
971
|
-
];
|
|
972
|
-
/**
|
|
973
|
-
* Courtesy/honorific titles that precede a person's
|
|
974
|
-
* name. Sorted alphabetically. The detector escapes
|
|
975
|
-
* dots and adds \b for entries in HONORIFIC_BOUNDARY.
|
|
976
|
-
*/
|
|
977
|
-
const HONORIFICS = [
|
|
978
|
-
"Avv.",
|
|
979
|
-
"Dame",
|
|
980
|
-
"Doamna",
|
|
981
|
-
"Domnul",
|
|
982
|
-
"Don",
|
|
983
|
-
"Doña",
|
|
984
|
-
"Dott.",
|
|
985
|
-
"Judge",
|
|
986
|
-
"Justice",
|
|
987
|
-
"Lady",
|
|
988
|
-
"Lord",
|
|
989
|
-
"M.",
|
|
990
|
-
"Madame",
|
|
991
|
-
"Mademoiselle",
|
|
992
|
-
"Maître",
|
|
993
|
-
"Me",
|
|
994
|
-
"Messrs",
|
|
995
|
-
"Miss",
|
|
996
|
-
"Mlle",
|
|
997
|
-
"Mme",
|
|
998
|
-
"Monsieur",
|
|
999
|
-
"Mr",
|
|
1000
|
-
"Mrs",
|
|
1001
|
-
"Ms",
|
|
1002
|
-
"President",
|
|
1003
|
-
"Señor",
|
|
1004
|
-
"Señora",
|
|
1005
|
-
"Sig.",
|
|
1006
|
-
"Sig.ra",
|
|
1007
|
-
"Signor",
|
|
1008
|
-
"Signora",
|
|
1009
|
-
"Signorina",
|
|
1010
|
-
"Sir",
|
|
1011
|
-
"Sr.",
|
|
1012
|
-
"Sra."
|
|
1013
|
-
];
|
|
1014
|
-
/**
|
|
1015
|
-
* Honorifics that need \b word-boundary anchors
|
|
1016
|
-
* (short or common words that could match mid-word).
|
|
1017
|
-
*/
|
|
1018
|
-
const HONORIFIC_BOUNDARY = new Set([
|
|
1019
|
-
"Don",
|
|
1020
|
-
"Doña",
|
|
1021
|
-
"M.",
|
|
1022
|
-
"Me",
|
|
1023
|
-
"Señor",
|
|
1024
|
-
"Señora"
|
|
1025
|
-
]);
|
|
1026
|
-
/**
|
|
1027
|
-
* Post-nominal degrees (comma or space separated after name).
|
|
1028
|
-
* Plain text; the detector auto-escapes for regex.
|
|
1029
|
-
*/
|
|
1030
|
-
const POST_NOMINALS = [
|
|
1031
|
-
"Ph.D.",
|
|
1032
|
-
"Ph.D",
|
|
1033
|
-
"CSc.",
|
|
1034
|
-
"DrSc.",
|
|
1035
|
-
"ArtD.",
|
|
1036
|
-
"D.Phil.",
|
|
1037
|
-
"DPhil.",
|
|
1038
|
-
"MPhil.",
|
|
1039
|
-
"MBA",
|
|
1040
|
-
"MPA",
|
|
1041
|
-
"LL.M.",
|
|
1042
|
-
"LL.B.",
|
|
1043
|
-
"M.Sc.",
|
|
1044
|
-
"B.Sc.",
|
|
1045
|
-
"MSc.",
|
|
1046
|
-
"BSc.",
|
|
1047
|
-
"M.Eng.",
|
|
1048
|
-
"B.Eng.",
|
|
1049
|
-
"M.A.",
|
|
1050
|
-
"B.A.",
|
|
1051
|
-
"JCD",
|
|
1052
|
-
"JD",
|
|
1053
|
-
"DiS.",
|
|
1054
|
-
"ACCA",
|
|
1055
|
-
"FCCA",
|
|
1056
|
-
"CIPM",
|
|
1057
|
-
"CIPT",
|
|
1058
|
-
"CIPP/E",
|
|
1059
|
-
"CIPP"
|
|
1060
|
-
];
|
|
206
|
+
const LEGAL_SUFFIXES = [...[
|
|
207
|
+
"spol. s r.o.",
|
|
208
|
+
"s.r.o.",
|
|
209
|
+
"s. r. o.",
|
|
210
|
+
"a.s.",
|
|
211
|
+
"a. s.",
|
|
212
|
+
"v.o.s.",
|
|
213
|
+
"v. o. s.",
|
|
214
|
+
"k.s.",
|
|
215
|
+
"k. s.",
|
|
216
|
+
"z.s.",
|
|
217
|
+
"z. s.",
|
|
218
|
+
"z.ú.",
|
|
219
|
+
"z. ú.",
|
|
220
|
+
"o.p.s.",
|
|
221
|
+
"o. p. s.",
|
|
222
|
+
"s.p.",
|
|
223
|
+
"s. p.",
|
|
224
|
+
"GmbH",
|
|
225
|
+
"AG",
|
|
226
|
+
"SE",
|
|
227
|
+
"KG",
|
|
228
|
+
"OHG",
|
|
229
|
+
"Ltd.",
|
|
230
|
+
"Ltd",
|
|
231
|
+
"LLC",
|
|
232
|
+
"LLP",
|
|
233
|
+
"Inc.",
|
|
234
|
+
"Corp.",
|
|
235
|
+
"Corporation",
|
|
236
|
+
"Co.",
|
|
237
|
+
"LP",
|
|
238
|
+
"L.P.",
|
|
239
|
+
"PLC",
|
|
240
|
+
"plc",
|
|
241
|
+
"N.A.",
|
|
242
|
+
"N.V.",
|
|
243
|
+
"B.V.",
|
|
244
|
+
"Pty Ltd.",
|
|
245
|
+
"Pty Ltd",
|
|
246
|
+
"S.A.",
|
|
247
|
+
"SA",
|
|
248
|
+
"SAS",
|
|
249
|
+
"SARL",
|
|
250
|
+
"S.p.A.",
|
|
251
|
+
"Sp. z o.o.",
|
|
252
|
+
"Sp. k.",
|
|
253
|
+
"Sp. j.",
|
|
254
|
+
"Ltda.",
|
|
255
|
+
"LTDA.",
|
|
256
|
+
"Ltda",
|
|
257
|
+
"LTDA",
|
|
258
|
+
"S/A",
|
|
259
|
+
"EIRELI",
|
|
260
|
+
"EPP",
|
|
261
|
+
"ME",
|
|
262
|
+
"MEI"
|
|
263
|
+
]].sort((a, b) => b.length - a.length);
|
|
1061
264
|
//#endregion
|
|
1062
265
|
//#region src/data/char-groups.json
|
|
1063
266
|
var char_groups_default = {
|
|
@@ -1432,220 +635,1503 @@ var char_groups_default = {
|
|
|
1432
635
|
"code": "U+060C"
|
|
1433
636
|
},
|
|
1434
637
|
{
|
|
1435
|
-
"char": "、",
|
|
1436
|
-
"name": "ideographic comma",
|
|
1437
|
-
"code": "U+3001"
|
|
638
|
+
"char": "、",
|
|
639
|
+
"name": "ideographic comma",
|
|
640
|
+
"code": "U+3001"
|
|
641
|
+
},
|
|
642
|
+
{
|
|
643
|
+
"char": ",",
|
|
644
|
+
"name": "full-width comma",
|
|
645
|
+
"code": "U+FF0C"
|
|
646
|
+
},
|
|
647
|
+
{
|
|
648
|
+
"char": "﹐",
|
|
649
|
+
"name": "small comma",
|
|
650
|
+
"code": "U+FE50"
|
|
651
|
+
},
|
|
652
|
+
{
|
|
653
|
+
"char": "՝",
|
|
654
|
+
"name": "Armenian comma",
|
|
655
|
+
"code": "U+055D"
|
|
656
|
+
}
|
|
657
|
+
]
|
|
658
|
+
},
|
|
659
|
+
"ellipsis": {
|
|
660
|
+
"description": "Ellipsis characters",
|
|
661
|
+
"chars": [{
|
|
662
|
+
"char": "…",
|
|
663
|
+
"name": "horizontal ellipsis",
|
|
664
|
+
"code": "U+2026"
|
|
665
|
+
}]
|
|
666
|
+
},
|
|
667
|
+
"question-mark": {
|
|
668
|
+
"description": "Question mark variants",
|
|
669
|
+
"chars": [
|
|
670
|
+
{
|
|
671
|
+
"char": "?",
|
|
672
|
+
"name": "question mark",
|
|
673
|
+
"code": "U+003F"
|
|
674
|
+
},
|
|
675
|
+
{
|
|
676
|
+
"char": "?",
|
|
677
|
+
"name": "full-width question mark",
|
|
678
|
+
"code": "U+FF1F"
|
|
679
|
+
},
|
|
680
|
+
{
|
|
681
|
+
"char": "⁇",
|
|
682
|
+
"name": "double question mark",
|
|
683
|
+
"code": "U+2047"
|
|
684
|
+
},
|
|
685
|
+
{
|
|
686
|
+
"char": "⁈",
|
|
687
|
+
"name": "question exclamation mark",
|
|
688
|
+
"code": "U+2048"
|
|
689
|
+
},
|
|
690
|
+
{
|
|
691
|
+
"char": "؟",
|
|
692
|
+
"name": "Arabic question mark",
|
|
693
|
+
"code": "U+061F"
|
|
694
|
+
},
|
|
695
|
+
{
|
|
696
|
+
"char": "‽",
|
|
697
|
+
"name": "interrobang",
|
|
698
|
+
"code": "U+203D"
|
|
699
|
+
}
|
|
700
|
+
]
|
|
701
|
+
},
|
|
702
|
+
"exclamation-mark": {
|
|
703
|
+
"description": "Exclamation mark variants",
|
|
704
|
+
"chars": [
|
|
705
|
+
{
|
|
706
|
+
"char": "!",
|
|
707
|
+
"name": "exclamation mark",
|
|
708
|
+
"code": "U+0021"
|
|
709
|
+
},
|
|
710
|
+
{
|
|
711
|
+
"char": "!",
|
|
712
|
+
"name": "full-width exclamation mark",
|
|
713
|
+
"code": "U+FF01"
|
|
714
|
+
},
|
|
715
|
+
{
|
|
716
|
+
"char": "‼",
|
|
717
|
+
"name": "double exclamation mark",
|
|
718
|
+
"code": "U+203C"
|
|
719
|
+
},
|
|
720
|
+
{
|
|
721
|
+
"char": "¡",
|
|
722
|
+
"name": "inverted exclamation mark",
|
|
723
|
+
"code": "U+00A1"
|
|
724
|
+
}
|
|
725
|
+
]
|
|
726
|
+
},
|
|
727
|
+
"semicolon": {
|
|
728
|
+
"description": "Semicolon variants",
|
|
729
|
+
"chars": [
|
|
730
|
+
{
|
|
731
|
+
"char": ";",
|
|
732
|
+
"name": "semicolon",
|
|
733
|
+
"code": "U+003B"
|
|
1438
734
|
},
|
|
1439
735
|
{
|
|
1440
|
-
"char": "
|
|
1441
|
-
"name": "
|
|
1442
|
-
"code": "U+
|
|
736
|
+
"char": "؛",
|
|
737
|
+
"name": "Arabic semicolon",
|
|
738
|
+
"code": "U+061B"
|
|
1443
739
|
},
|
|
1444
740
|
{
|
|
1445
|
-
"char": "
|
|
1446
|
-
"name": "
|
|
1447
|
-
"code": "U+
|
|
741
|
+
"char": ";",
|
|
742
|
+
"name": "full-width semicolon",
|
|
743
|
+
"code": "U+FF1B"
|
|
1448
744
|
},
|
|
1449
745
|
{
|
|
1450
|
-
"char": "
|
|
1451
|
-
"name": "
|
|
1452
|
-
"code": "U+
|
|
746
|
+
"char": "፤",
|
|
747
|
+
"name": "Ethiopic semicolon",
|
|
748
|
+
"code": "U+1364"
|
|
749
|
+
}
|
|
750
|
+
]
|
|
751
|
+
},
|
|
752
|
+
"underscore": {
|
|
753
|
+
"description": "Underscore variants",
|
|
754
|
+
"chars": [{
|
|
755
|
+
"char": "_",
|
|
756
|
+
"name": "low line",
|
|
757
|
+
"code": "U+005F"
|
|
758
|
+
}, {
|
|
759
|
+
"char": "_",
|
|
760
|
+
"name": "full-width low line",
|
|
761
|
+
"code": "U+FF3F"
|
|
762
|
+
}]
|
|
763
|
+
}
|
|
764
|
+
}
|
|
765
|
+
};
|
|
766
|
+
//#endregion
|
|
767
|
+
//#region src/util/char-groups.ts
|
|
768
|
+
/**
|
|
769
|
+
* Centralized Unicode character equivalence groups.
|
|
770
|
+
*
|
|
771
|
+
* Centralized Unicode character equivalence groups.
|
|
772
|
+
*
|
|
773
|
+
* Provides helpers to build regex character classes that
|
|
774
|
+
* match all typographic variants of a character type
|
|
775
|
+
* (dashes, spaces, quotes, etc.).
|
|
776
|
+
*
|
|
777
|
+
* The JSON is statically imported so the bundler inlines
|
|
778
|
+
* it, avoiding a runtime require() that breaks browsers.
|
|
779
|
+
*/
|
|
780
|
+
/** Chars that need escaping inside a regex char class. */
|
|
781
|
+
const REGEX_CLASS_SPECIAL = /[\\\]^-]/;
|
|
782
|
+
const escapeForCharClass = (ch) => REGEX_CLASS_SPECIAL.test(ch) ? `\\${ch}` : ch;
|
|
783
|
+
const config = char_groups_default;
|
|
784
|
+
/**
|
|
785
|
+
* Get the raw characters for a named group.
|
|
786
|
+
* Throws if the group does not exist.
|
|
787
|
+
*/
|
|
788
|
+
const charSet = (group) => {
|
|
789
|
+
const g = config.groups[group];
|
|
790
|
+
if (!g) throw new Error(`Unknown char group: "${group}"`);
|
|
791
|
+
return g.chars.map((entry) => entry.char);
|
|
792
|
+
};
|
|
793
|
+
/**
|
|
794
|
+
* Build a regex character class string for a named
|
|
795
|
+
* group. E.g., charClass("dash") returns a string
|
|
796
|
+
* like "[-\u2013\u2014\u2010\u2011\u2212\u2043]".
|
|
797
|
+
*
|
|
798
|
+
* The hyphen-minus is placed first so it is treated
|
|
799
|
+
* as a literal, not a range indicator.
|
|
800
|
+
*/
|
|
801
|
+
const charClass = (group) => {
|
|
802
|
+
return `[${[...charSet(group)].sort((a, b) => {
|
|
803
|
+
if (a === "-") return -1;
|
|
804
|
+
if (b === "-") return 1;
|
|
805
|
+
return a.localeCompare(b);
|
|
806
|
+
}).map(escapeForCharClass).join("")}]`;
|
|
807
|
+
};
|
|
808
|
+
/**
|
|
809
|
+
* Return the inner content of a character class (without
|
|
810
|
+
* the surrounding brackets). Useful for embedding a group
|
|
811
|
+
* inside a larger character class, e.g.:
|
|
812
|
+
* `[\\s&,.${charClassInner("dash")}]`
|
|
813
|
+
*/
|
|
814
|
+
const charClassInner = (group) => {
|
|
815
|
+
return [...charSet(group)].sort((a, b) => {
|
|
816
|
+
if (a === "-") return -1;
|
|
817
|
+
if (b === "-") return 1;
|
|
818
|
+
return a.localeCompare(b);
|
|
819
|
+
}).map(escapeForCharClass).join("");
|
|
820
|
+
};
|
|
821
|
+
/** Regex char class matching all dash variants. */
|
|
822
|
+
const DASH = charClass("dash");
|
|
823
|
+
/**
|
|
824
|
+
* Inner content of the dash char class (no brackets).
|
|
825
|
+
* For embedding inside a larger character class:
|
|
826
|
+
* `[\\s&,.${DASH_INNER}]`
|
|
827
|
+
*/
|
|
828
|
+
const DASH_INNER = charClassInner("dash");
|
|
829
|
+
charClass("space");
|
|
830
|
+
charClass("quote-double");
|
|
831
|
+
/** Inner content of the double-quote char class (no brackets). */
|
|
832
|
+
const QUOTE_DOUBLE_INNER = charClassInner("quote-double");
|
|
833
|
+
charClass("quote-single");
|
|
834
|
+
/** Inner content of the single-quote char class (no brackets). */
|
|
835
|
+
const QUOTE_SINGLE_INNER = charClassInner("quote-single");
|
|
836
|
+
/**
|
|
837
|
+
* Inline opening-bracket characters that frequently introduce
|
|
838
|
+
* a defined-term parenthetical immediately after an address or
|
|
839
|
+
* organization span (e.g. `…GA 30326, USA (the "Premises")`).
|
|
840
|
+
* Hardcoded rather than data-driven because the set is tiny,
|
|
841
|
+
* regex-special, and not language-dependent.
|
|
842
|
+
*/
|
|
843
|
+
const OPENING_BRACKETS_INNER = "(\\[{";
|
|
844
|
+
charClass("slash");
|
|
845
|
+
charClass("dot");
|
|
846
|
+
charClass("colon");
|
|
847
|
+
//#endregion
|
|
848
|
+
//#region src/detectors/legal-forms.ts
|
|
849
|
+
const SENTENCE_VERB_INDICATORS_SEED = new Set([
|
|
850
|
+
"je",
|
|
851
|
+
"jsou",
|
|
852
|
+
"is",
|
|
853
|
+
"are",
|
|
854
|
+
"ist",
|
|
855
|
+
"sind"
|
|
856
|
+
]);
|
|
857
|
+
let sentenceVerbIndicatorsCache = null;
|
|
858
|
+
let sentenceVerbIndicatorsPromise = null;
|
|
859
|
+
const loadSentenceVerbIndicators = async () => {
|
|
860
|
+
if (sentenceVerbIndicatorsCache) return sentenceVerbIndicatorsCache;
|
|
861
|
+
if (sentenceVerbIndicatorsPromise) return sentenceVerbIndicatorsPromise;
|
|
862
|
+
sentenceVerbIndicatorsPromise = (async () => {
|
|
863
|
+
let data = {};
|
|
864
|
+
try {
|
|
865
|
+
const mod = await import("./sentence-verb-indicators.mjs");
|
|
866
|
+
data = mod.default ?? mod;
|
|
867
|
+
} catch (err) {
|
|
868
|
+
console.warn("[anonymize] legal-forms: failed to load sentence-verb-indicators.json, falling back to seed list:", err);
|
|
869
|
+
}
|
|
870
|
+
const all = new Set(SENTENCE_VERB_INDICATORS_SEED);
|
|
871
|
+
for (const [key, value] of Object.entries(data)) {
|
|
872
|
+
if (key.startsWith("_")) continue;
|
|
873
|
+
if (!Array.isArray(value)) continue;
|
|
874
|
+
for (const verb of value) {
|
|
875
|
+
if (typeof verb !== "string" || verb.length === 0) continue;
|
|
876
|
+
all.add(verb.toLowerCase());
|
|
877
|
+
}
|
|
878
|
+
}
|
|
879
|
+
sentenceVerbIndicatorsCache = all;
|
|
880
|
+
return all;
|
|
881
|
+
})();
|
|
882
|
+
return sentenceVerbIndicatorsPromise;
|
|
883
|
+
};
|
|
884
|
+
const getSentenceVerbIndicatorsSync = () => sentenceVerbIndicatorsCache ?? SENTENCE_VERB_INDICATORS_SEED;
|
|
885
|
+
const UPPER = "A-ZÁČĎÉĚÍŇÓŘŠŤÚŮÝŽÄÖÜÀÂÆÇÈÊËÎÏÔÙÛŸÑĄĆĘŁŃŚŹŻ\\u0130";
|
|
886
|
+
const LOWER = "a-záčďéěíňóřšťúůýžäöüßàâæçèêëîïôùûÿñąćęłńśźż\\u0131";
|
|
887
|
+
const CAP_WORD = `(?:[${UPPER}]{2,}|[${UPPER}][${LOWER}${UPPER}]+)`;
|
|
888
|
+
const ALLCAP_WORD = `[${UPPER}]{2,}`;
|
|
889
|
+
const ROMAN_NUMERAL_RE = /^(?=[IVXLCDM])M{0,3}(?:CM|CD|D?C{0,3})(?:XC|XL|L?X{0,3})(?:IX|IV|V?I{0,3})$/;
|
|
890
|
+
let legalRoleHeadsCache = null;
|
|
891
|
+
let legalRoleHeadsPromise = null;
|
|
892
|
+
const loadLegalRoleHeads = async () => {
|
|
893
|
+
if (legalRoleHeadsCache) return legalRoleHeadsCache;
|
|
894
|
+
if (legalRoleHeadsPromise) return legalRoleHeadsPromise;
|
|
895
|
+
legalRoleHeadsPromise = (async () => {
|
|
896
|
+
const sets = await loadLanguageConfigs("legalRoleHeads", (mod) => {
|
|
897
|
+
return mod.default ?? mod;
|
|
898
|
+
});
|
|
899
|
+
const all = /* @__PURE__ */ new Set();
|
|
900
|
+
for (const entry of sets) {
|
|
901
|
+
if (!entry || !Array.isArray(entry.words)) continue;
|
|
902
|
+
for (const word of entry.words) if (typeof word === "string" && word.length > 0) all.add(word.toLowerCase());
|
|
903
|
+
}
|
|
904
|
+
legalRoleHeadsCache = all;
|
|
905
|
+
return all;
|
|
906
|
+
})();
|
|
907
|
+
return legalRoleHeadsPromise;
|
|
908
|
+
};
|
|
909
|
+
const getLegalRoleHeadsSync = () => legalRoleHeadsCache ?? /* @__PURE__ */ new Set();
|
|
910
|
+
const warmLegalRoleHeads = async () => {
|
|
911
|
+
await Promise.all([
|
|
912
|
+
loadLegalRoleHeads(),
|
|
913
|
+
loadAllLegalSuffixes(),
|
|
914
|
+
loadSentenceVerbIndicators(),
|
|
915
|
+
loadClauseNounHeads()
|
|
916
|
+
]);
|
|
917
|
+
};
|
|
918
|
+
let allLegalSuffixesCache = null;
|
|
919
|
+
let allLegalSuffixesPromise = null;
|
|
920
|
+
const loadAllLegalSuffixes = async () => {
|
|
921
|
+
if (allLegalSuffixesCache) return allLegalSuffixesCache;
|
|
922
|
+
if (allLegalSuffixesPromise) return allLegalSuffixesPromise;
|
|
923
|
+
allLegalSuffixesPromise = (async () => {
|
|
924
|
+
let data = {};
|
|
925
|
+
try {
|
|
926
|
+
data = (await import("./legal-forms.mjs")).default;
|
|
927
|
+
} catch {
|
|
928
|
+
data = {};
|
|
929
|
+
}
|
|
930
|
+
const seen = /* @__PURE__ */ new Set();
|
|
931
|
+
const out = [];
|
|
932
|
+
for (const list of Object.values(data)) for (const form of list) {
|
|
933
|
+
if (typeof form !== "string" || form.length === 0) continue;
|
|
934
|
+
if (seen.has(form)) continue;
|
|
935
|
+
seen.add(form);
|
|
936
|
+
out.push(form);
|
|
937
|
+
}
|
|
938
|
+
for (const form of LEGAL_SUFFIXES) if (!seen.has(form)) {
|
|
939
|
+
seen.add(form);
|
|
940
|
+
out.push(form);
|
|
941
|
+
}
|
|
942
|
+
out.sort((a, b) => b.length - a.length);
|
|
943
|
+
allLegalSuffixesCache = out;
|
|
944
|
+
return out;
|
|
945
|
+
})();
|
|
946
|
+
return allLegalSuffixesPromise;
|
|
947
|
+
};
|
|
948
|
+
const getAllLegalSuffixesSync = () => allLegalSuffixesCache ?? LEGAL_SUFFIXES;
|
|
949
|
+
/**
|
|
950
|
+
* Sync accessor for the full legal-form vocabulary
|
|
951
|
+
* (`data/legal-forms.json` plus `LEGAL_SUFFIXES`,
|
|
952
|
+
* longest-first). Falls back to `LEGAL_SUFFIXES` when
|
|
953
|
+
* `warmLegalRoleHeads()` has not run yet. Exposed so the
|
|
954
|
+
* trailing-period strip in `sanitizeEntities` can keep
|
|
955
|
+
* pace with the detector vocabulary rather than only the
|
|
956
|
+
* smaller `LEGAL_SUFFIXES` propagation list.
|
|
957
|
+
*/
|
|
958
|
+
const getKnownLegalSuffixes = getAllLegalSuffixesSync;
|
|
959
|
+
const CLAUSE_NOUN_HEADS_SEED = new Set(["agreement", "contract"]);
|
|
960
|
+
let clauseNounHeadsCache = null;
|
|
961
|
+
let clauseNounHeadsPromise = null;
|
|
962
|
+
const loadClauseNounHeads = async () => {
|
|
963
|
+
if (clauseNounHeadsCache) return clauseNounHeadsCache;
|
|
964
|
+
if (clauseNounHeadsPromise) return clauseNounHeadsPromise;
|
|
965
|
+
clauseNounHeadsPromise = (async () => {
|
|
966
|
+
let data = {};
|
|
967
|
+
try {
|
|
968
|
+
const mod = await import("./clause-noun-heads.mjs");
|
|
969
|
+
data = mod.default ?? mod;
|
|
970
|
+
} catch (err) {
|
|
971
|
+
console.warn("[anonymize] legal-forms: failed to load clause-noun-heads.json, falling back to seed list:", err);
|
|
972
|
+
}
|
|
973
|
+
const all = new Set(CLAUSE_NOUN_HEADS_SEED);
|
|
974
|
+
for (const [key, value] of Object.entries(data)) {
|
|
975
|
+
if (key.startsWith("_")) continue;
|
|
976
|
+
if (!Array.isArray(value)) continue;
|
|
977
|
+
for (const noun of value) {
|
|
978
|
+
if (typeof noun !== "string" || noun.length === 0) continue;
|
|
979
|
+
all.add(noun.toLowerCase());
|
|
980
|
+
}
|
|
981
|
+
}
|
|
982
|
+
clauseNounHeadsCache = all;
|
|
983
|
+
return all;
|
|
984
|
+
})();
|
|
985
|
+
return clauseNounHeadsPromise;
|
|
986
|
+
};
|
|
987
|
+
const getClauseNounHeadsSync = () => clauseNounHeadsCache ?? CLAUSE_NOUN_HEADS_SEED;
|
|
988
|
+
const escapeForRegex = (form) => form.replace(/[.*+?^${}()|[\]\\]/g, "\\$&").replace(/\s+/g, "\\s+").replace(/\\\./g, "\\.[^\\S\\n]?");
|
|
989
|
+
const isShortForm = (form) => form.replace(/[.\s]/g, "").length <= 3 && !form.includes(" ");
|
|
990
|
+
const buildPatternString = (forms) => {
|
|
991
|
+
if (forms.length === 0) return null;
|
|
992
|
+
const alt = forms.toSorted((a, b) => b.length - a.length).map(escapeForRegex).join("|");
|
|
993
|
+
const HSPACE = "[^\\S\\n]";
|
|
994
|
+
const LOWER_CONNECTOR = `${HSPACE}+(?:a|and|und|et|e|y|i)${HSPACE}+(?=[${LOWER}])`;
|
|
995
|
+
const SIMPLE_SEP = `(?:${HSPACE}|[&,.${DASH_INNER}]){1,4}`;
|
|
996
|
+
const CAP_OR_NUM_WORD = `(?:${CAP_WORD}|\\d{1,4})`;
|
|
997
|
+
const LOWER_WORD = `(?:(?!(?:and|und|et)(?![${UPPER}${LOWER}]))[${LOWER}][${LOWER}${UPPER}]+)`;
|
|
998
|
+
const ANY_WORD_TAIL = `(?:(?!(?:and|und|et)(?![${UPPER}${LOWER}]))[${UPPER}${LOWER}][${LOWER}${UPPER}]+|[${UPPER}]{2,3}|\\d{1,4})`;
|
|
999
|
+
return `${`(?:${`(?:${CAP_WORD})(?:${SIMPLE_SEP}(?:${CAP_OR_NUM_WORD})){0,10}`})(?:${`${SIMPLE_SEP}(?:${LOWER_WORD})(?:(?:${LOWER_CONNECTOR}|${SIMPLE_SEP})(?:${ANY_WORD_TAIL})){0,10}`})?`}(?:\\s+|,\\s*)(?:${alt})(?![${LOWER}])`;
|
|
1000
|
+
};
|
|
1001
|
+
/**
|
|
1002
|
+
* Build legal form regex pattern strings.
|
|
1003
|
+
* Returns an array of regex strings for the unified
|
|
1004
|
+
* TextSearch builder. Empty if data package is not
|
|
1005
|
+
* installed.
|
|
1006
|
+
*/
|
|
1007
|
+
const buildLegalFormPatterns = async () => {
|
|
1008
|
+
let data = {};
|
|
1009
|
+
try {
|
|
1010
|
+
data = (await import("./legal-forms.mjs")).default;
|
|
1011
|
+
} catch {
|
|
1012
|
+
return [];
|
|
1013
|
+
}
|
|
1014
|
+
const allForms = [];
|
|
1015
|
+
const seen = /* @__PURE__ */ new Set();
|
|
1016
|
+
for (const forms of Object.values(data)) for (const form of forms) if (!seen.has(form)) {
|
|
1017
|
+
seen.add(form);
|
|
1018
|
+
allForms.push(form);
|
|
1019
|
+
}
|
|
1020
|
+
for (const form of LEGAL_SUFFIXES) if (!seen.has(form)) {
|
|
1021
|
+
seen.add(form);
|
|
1022
|
+
allForms.push(form);
|
|
1023
|
+
}
|
|
1024
|
+
const patterns = [];
|
|
1025
|
+
const longPattern = buildPatternString(allForms.filter((f) => !isShortForm(f)));
|
|
1026
|
+
if (longPattern) patterns.push(longPattern);
|
|
1027
|
+
const shortPattern = buildPatternString(allForms.filter(isShortForm));
|
|
1028
|
+
if (shortPattern) patterns.push(shortPattern);
|
|
1029
|
+
const allcapPrefix = `(?:${ALLCAP_WORD})(?:[ \\t&,.${DASH_INNER}]{1,4}(?:${ALLCAP_WORD})){0,2}`;
|
|
1030
|
+
const allcapAlt = allForms.toSorted((a, b) => b.length - a.length).map(escapeForRegex).join("|");
|
|
1031
|
+
patterns.push(`${allcapPrefix}(?:[ \\t]+|,[ \\t]*)(?:${allcapAlt})(?![${LOWER}])`);
|
|
1032
|
+
return patterns;
|
|
1033
|
+
};
|
|
1034
|
+
const CONNECTOR_RE = /^(?:a|and|und|et|e|y|i|&)$/i;
|
|
1035
|
+
const AND_TYPE_CONNECTOR_RE = /^(?:and|und|et)$/i;
|
|
1036
|
+
const UPPER_LETTER_RE = /^\p{Lu}/u;
|
|
1037
|
+
const COMPANY_SUFFIX_WORDS_RE = /^(?:Company|Co|Bank|Brothers|Bros|Sons|Group|Holdings|Trust|Partners|Associates|Corporation|Industries|Enterprises|Solutions|Systems|Services|Foundation|Institute)$/;
|
|
1038
|
+
const IN_NAME_PREPOSITION_RE = /^(?:of|the)$/i;
|
|
1039
|
+
const ENTITY_HEAD_WORD_RE = /^[\p{L}\p{M}&]+/u;
|
|
1040
|
+
const LEADING_CLAUSE_RE = /(?:^|\s)(?:by\s+and\s+between|is\s+between)\s+/giu;
|
|
1041
|
+
/**
|
|
1042
|
+
* Find the word ending just before `pos` in `text`,
|
|
1043
|
+
* skipping any whitespace (not newlines).
|
|
1044
|
+
* Returns null if no word is found (e.g., at start
|
|
1045
|
+
* of text, or preceded by non-word chars like ".").
|
|
1046
|
+
*/
|
|
1047
|
+
const findWordBefore = (text, pos) => {
|
|
1048
|
+
let scan = pos - 1;
|
|
1049
|
+
while (scan >= 0) {
|
|
1050
|
+
const ch = text.charAt(scan);
|
|
1051
|
+
if (ch === "\n" || !/\s/.test(ch)) break;
|
|
1052
|
+
scan--;
|
|
1053
|
+
}
|
|
1054
|
+
if (scan < 0 || text.charAt(scan) === "\n") return null;
|
|
1055
|
+
const wordEnd = scan + 1;
|
|
1056
|
+
while (scan >= 0 && /[\p{L}\p{M}&]/u.test(text.charAt(scan))) scan--;
|
|
1057
|
+
const wordStart = scan + 1;
|
|
1058
|
+
const word = text.slice(wordStart, wordEnd);
|
|
1059
|
+
if (word.length === 0) return null;
|
|
1060
|
+
return {
|
|
1061
|
+
word,
|
|
1062
|
+
start: wordStart
|
|
1063
|
+
};
|
|
1064
|
+
};
|
|
1065
|
+
/**
|
|
1066
|
+
* Count consecutive uppercase-starting words immediately
|
|
1067
|
+
* before `pos`. Stops at the first non-upper word or at
|
|
1068
|
+
* text/line start. Used to disambiguate "<First> <Last>
|
|
1069
|
+
* and <ORG>" from "<Multi-word Org> and <Continuation>".
|
|
1070
|
+
*/
|
|
1071
|
+
const countUpperWordsBefore = (fullText, pos) => {
|
|
1072
|
+
let count = 0;
|
|
1073
|
+
let scan = pos;
|
|
1074
|
+
while (scan > 0) {
|
|
1075
|
+
const found = findWordBefore(fullText, scan);
|
|
1076
|
+
if (!found) break;
|
|
1077
|
+
if (!UPPER_LETTER_RE.test(found.word)) break;
|
|
1078
|
+
count++;
|
|
1079
|
+
scan = found.start;
|
|
1080
|
+
}
|
|
1081
|
+
return count;
|
|
1082
|
+
};
|
|
1083
|
+
/**
|
|
1084
|
+
* Extend a match backward through uppercase words and
|
|
1085
|
+
* lowercase connectors. Stops at start of text,
|
|
1086
|
+
* newline, or a word that doesn't qualify.
|
|
1087
|
+
*
|
|
1088
|
+
* Connectors (a, and, und, et, ...) are only consumed
|
|
1089
|
+
* when there is a valid word before them — a trailing
|
|
1090
|
+
* connector at an entity boundary is not consumed.
|
|
1091
|
+
* For multi-char "and"-type connectors we additionally
|
|
1092
|
+
* refuse to cross when exactly two uppercase words
|
|
1093
|
+
* precede them ("First Last and ORG, Inc." shape) —
|
|
1094
|
+
* unless the match itself begins with a known company-
|
|
1095
|
+
* suffix word ("…and Company, Inc."), in which case
|
|
1096
|
+
* the chain belongs to one organisation. In that
|
|
1097
|
+
* suffix-mode we also cross in-name prepositions
|
|
1098
|
+
* ("Bank of America and Trust Company, Inc.").
|
|
1099
|
+
*/
|
|
1100
|
+
const extendBackward = (fullText, matchStart, options = {}) => {
|
|
1101
|
+
const headWord = ENTITY_HEAD_WORD_RE.exec(fullText.slice(matchStart))?.[0] ?? "";
|
|
1102
|
+
const suffixMode = options.forceSuffixMode === true || COMPANY_SUFFIX_WORDS_RE.test(headWord);
|
|
1103
|
+
let pos = matchStart;
|
|
1104
|
+
while (pos > 0) {
|
|
1105
|
+
const found = findWordBefore(fullText, pos);
|
|
1106
|
+
if (!found) break;
|
|
1107
|
+
const { word, start: wordStart } = found;
|
|
1108
|
+
const isUpper = UPPER_LETTER_RE.test(word);
|
|
1109
|
+
const isConnector = CONNECTOR_RE.test(word);
|
|
1110
|
+
const isInNamePrep = suffixMode && IN_NAME_PREPOSITION_RE.test(word);
|
|
1111
|
+
if (isUpper) pos = wordStart;
|
|
1112
|
+
else if (isConnector) {
|
|
1113
|
+
if (!suffixMode && AND_TYPE_CONNECTOR_RE.test(word) && countUpperWordsBefore(fullText, wordStart) === 2) break;
|
|
1114
|
+
const prev = findWordBefore(fullText, wordStart);
|
|
1115
|
+
if (!prev) break;
|
|
1116
|
+
if (!UPPER_LETTER_RE.test(prev.word)) break;
|
|
1117
|
+
pos = prev.start;
|
|
1118
|
+
} else if (isInNamePrep) {
|
|
1119
|
+
const prev = findWordBefore(fullText, wordStart);
|
|
1120
|
+
if (!prev) break;
|
|
1121
|
+
if (!UPPER_LETTER_RE.test(prev.word)) break;
|
|
1122
|
+
pos = prev.start;
|
|
1123
|
+
} else break;
|
|
1124
|
+
}
|
|
1125
|
+
return pos;
|
|
1126
|
+
};
|
|
1127
|
+
const trimLeadingClause = (text) => {
|
|
1128
|
+
let cut = -1;
|
|
1129
|
+
for (const match of text.matchAll(LEADING_CLAUSE_RE)) cut = match.index + match[0].length;
|
|
1130
|
+
if (cut <= 0) return {
|
|
1131
|
+
offset: 0,
|
|
1132
|
+
text
|
|
1133
|
+
};
|
|
1134
|
+
const trimmed = text.slice(cut);
|
|
1135
|
+
const leadingWs = trimmed.match(/^\s*/u)?.[0].length ?? 0;
|
|
1136
|
+
return {
|
|
1137
|
+
offset: cut + leadingWs,
|
|
1138
|
+
text: trimmed.slice(leadingWs)
|
|
1139
|
+
};
|
|
1140
|
+
};
|
|
1141
|
+
/**
|
|
1142
|
+
* Process legal form matches from the unified search.
|
|
1143
|
+
* Receives all matches; filters to the legal forms
|
|
1144
|
+
* slice via sliceStart/sliceEnd.
|
|
1145
|
+
*
|
|
1146
|
+
* The role-head trimming step reads per-language data from
|
|
1147
|
+
* a cache that `runPipeline` warms via `warmLegalRoleHeads()`
|
|
1148
|
+
* before calling this. Callers that invoke
|
|
1149
|
+
* `processLegalFormMatches` directly (without going through
|
|
1150
|
+
* `runPipeline`) must `await warmLegalRoleHeads()` first;
|
|
1151
|
+
* otherwise the trim falls back to a no-op and sentence-
|
|
1152
|
+
* fragment fixes do not apply.
|
|
1153
|
+
*/
|
|
1154
|
+
const processLegalFormMatches = (allMatches, sliceStart, sliceEnd, fullText) => {
|
|
1155
|
+
const results = [];
|
|
1156
|
+
for (const match of allMatches) {
|
|
1157
|
+
const idx = match.pattern;
|
|
1158
|
+
if (idx < sliceStart || idx >= sliceEnd) continue;
|
|
1159
|
+
const text = match.text.trimEnd();
|
|
1160
|
+
if (text.length < 5) continue;
|
|
1161
|
+
const roleHeads = getLegalRoleHeadsSync();
|
|
1162
|
+
const firstWordMatch = /^[\p{L}\p{M}]+(?:-[\p{L}\p{M}]+)*/u.exec(text);
|
|
1163
|
+
let processedStart = match.start;
|
|
1164
|
+
let processedText = text;
|
|
1165
|
+
let trimmed = false;
|
|
1166
|
+
const firstWordText = firstWordMatch?.[0] ?? "";
|
|
1167
|
+
const firstWordLeading = /^[\p{L}\p{M}]+/u.exec(firstWordText)?.[0] ?? "";
|
|
1168
|
+
if (firstWordMatch !== null && (roleHeads.has(firstWordText.toLowerCase()) || firstWordLeading.length > 0 && roleHeads.has(firstWordLeading.toLowerCase()))) {
|
|
1169
|
+
let suffixOffset = -1;
|
|
1170
|
+
for (const suffix of getAllLegalSuffixesSync()) {
|
|
1171
|
+
const idx = text.lastIndexOf(suffix);
|
|
1172
|
+
if (idx !== -1 && idx + suffix.length >= text.length - 1) {
|
|
1173
|
+
suffixOffset = idx;
|
|
1174
|
+
break;
|
|
1453
1175
|
}
|
|
1454
|
-
|
|
1455
|
-
|
|
1456
|
-
|
|
1457
|
-
|
|
1458
|
-
|
|
1459
|
-
|
|
1460
|
-
|
|
1461
|
-
|
|
1462
|
-
|
|
1463
|
-
|
|
1464
|
-
|
|
1465
|
-
|
|
1466
|
-
|
|
1467
|
-
|
|
1468
|
-
"char": "?",
|
|
1469
|
-
"name": "question mark",
|
|
1470
|
-
"code": "U+003F"
|
|
1471
|
-
},
|
|
1472
|
-
{
|
|
1473
|
-
"char": "?",
|
|
1474
|
-
"name": "full-width question mark",
|
|
1475
|
-
"code": "U+FF1F"
|
|
1476
|
-
},
|
|
1477
|
-
{
|
|
1478
|
-
"char": "⁇",
|
|
1479
|
-
"name": "double question mark",
|
|
1480
|
-
"code": "U+2047"
|
|
1481
|
-
},
|
|
1482
|
-
{
|
|
1483
|
-
"char": "⁈",
|
|
1484
|
-
"name": "question exclamation mark",
|
|
1485
|
-
"code": "U+2048"
|
|
1486
|
-
},
|
|
1487
|
-
{
|
|
1488
|
-
"char": "؟",
|
|
1489
|
-
"name": "Arabic question mark",
|
|
1490
|
-
"code": "U+061F"
|
|
1491
|
-
},
|
|
1492
|
-
{
|
|
1493
|
-
"char": "‽",
|
|
1494
|
-
"name": "interrobang",
|
|
1495
|
-
"code": "U+203D"
|
|
1176
|
+
}
|
|
1177
|
+
if (suffixOffset < 0) {} else {
|
|
1178
|
+
const midStart = firstWordMatch[0].length;
|
|
1179
|
+
const midEnd = suffixOffset;
|
|
1180
|
+
const midSection = text.slice(midStart, midEnd);
|
|
1181
|
+
const verbIndicators = getSentenceVerbIndicatorsSync();
|
|
1182
|
+
let lastVerbEndInMid = -1;
|
|
1183
|
+
for (const match of midSection.matchAll(/(?<![\p{L}\p{N}])[\p{L}\p{M}]+/gu)) if (match[0] !== void 0 && match.index !== void 0 && verbIndicators.has(match[0].toLowerCase())) lastVerbEndInMid = match.index + match[0].length;
|
|
1184
|
+
const digitAfterRole = /^\s+\d+(?:\.|\b)/u.test(midSection);
|
|
1185
|
+
let appositiveRoleHead = false;
|
|
1186
|
+
if (!digitAfterRole && lastVerbEndInMid === -1 && fullText) {
|
|
1187
|
+
const before = fullText.slice(Math.max(0, match.start - 40), match.start);
|
|
1188
|
+
const prevWord = /(?<![\p{L}\p{N}])(\p{L}[\p{L}\p{M}]*)\s*$/u.exec(before);
|
|
1189
|
+
if (prevWord !== null && getSentenceVerbIndicatorsSync().has(prevWord[1].toLowerCase())) appositiveRoleHead = true;
|
|
1496
1190
|
}
|
|
1497
|
-
|
|
1498
|
-
|
|
1499
|
-
|
|
1500
|
-
|
|
1501
|
-
|
|
1502
|
-
|
|
1503
|
-
|
|
1504
|
-
|
|
1505
|
-
|
|
1506
|
-
|
|
1507
|
-
|
|
1508
|
-
|
|
1509
|
-
|
|
1510
|
-
|
|
1511
|
-
|
|
1512
|
-
|
|
1513
|
-
|
|
1514
|
-
"name": "double exclamation mark",
|
|
1515
|
-
"code": "U+203C"
|
|
1516
|
-
},
|
|
1517
|
-
{
|
|
1518
|
-
"char": "¡",
|
|
1519
|
-
"name": "inverted exclamation mark",
|
|
1520
|
-
"code": "U+00A1"
|
|
1191
|
+
if (lastVerbEndInMid !== -1 || digitAfterRole || appositiveRoleHead) {
|
|
1192
|
+
const scanStart = lastVerbEndInMid !== -1 ? midStart + lastVerbEndInMid : midStart;
|
|
1193
|
+
const capRe = /(?<![\p{L}\p{N}])\p{Lu}[\p{L}\p{M}\p{N}]*/gu;
|
|
1194
|
+
capRe.lastIndex = scanStart;
|
|
1195
|
+
const clauseNouns = getClauseNounHeadsSync();
|
|
1196
|
+
let capMatch = null;
|
|
1197
|
+
for (let next = capRe.exec(text); next !== null; next = capRe.exec(text)) {
|
|
1198
|
+
if (next.index >= suffixOffset) break;
|
|
1199
|
+
const lc = next[0].toLowerCase();
|
|
1200
|
+
if (roleHeads.has(lc) || clauseNouns.has(lc)) continue;
|
|
1201
|
+
capMatch = next;
|
|
1202
|
+
break;
|
|
1203
|
+
}
|
|
1204
|
+
if (capMatch === null) continue;
|
|
1205
|
+
processedStart = match.start + capMatch.index;
|
|
1206
|
+
processedText = text.slice(capMatch.index);
|
|
1207
|
+
trimmed = true;
|
|
1521
1208
|
}
|
|
1522
|
-
|
|
1523
|
-
}
|
|
1524
|
-
"
|
|
1525
|
-
|
|
1526
|
-
|
|
1527
|
-
|
|
1528
|
-
|
|
1529
|
-
|
|
1530
|
-
|
|
1531
|
-
|
|
1532
|
-
|
|
1533
|
-
|
|
1534
|
-
|
|
1535
|
-
|
|
1536
|
-
|
|
1537
|
-
|
|
1538
|
-
|
|
1539
|
-
|
|
1540
|
-
|
|
1541
|
-
|
|
1542
|
-
|
|
1543
|
-
|
|
1544
|
-
|
|
1545
|
-
|
|
1209
|
+
}
|
|
1210
|
+
}
|
|
1211
|
+
if (processedText.includes("\n")) continue;
|
|
1212
|
+
let entityStart = processedStart;
|
|
1213
|
+
let entityText = processedText;
|
|
1214
|
+
if (fullText && !trimmed) {
|
|
1215
|
+
const extended = extendBackward(fullText, processedStart);
|
|
1216
|
+
if (extended < processedStart) {
|
|
1217
|
+
entityStart = extended;
|
|
1218
|
+
entityText = fullText.slice(extended, processedStart + processedText.length).trimEnd();
|
|
1219
|
+
}
|
|
1220
|
+
}
|
|
1221
|
+
const clauseTrim = trimLeadingClause(entityText);
|
|
1222
|
+
if (clauseTrim.offset > 0) {
|
|
1223
|
+
entityStart += clauseTrim.offset;
|
|
1224
|
+
entityText = clauseTrim.text;
|
|
1225
|
+
}
|
|
1226
|
+
const getPrefixInfo = (value) => {
|
|
1227
|
+
const prefixEnd = value.lastIndexOf(",") !== -1 ? value.lastIndexOf(",") : value.lastIndexOf(" ");
|
|
1228
|
+
return {
|
|
1229
|
+
prefixEnd,
|
|
1230
|
+
prefixPart: prefixEnd > 0 ? value.slice(0, prefixEnd).replace(/[^a-zA-ZÀ-ž]/g, "") : value.replace(/[^a-zA-ZÀ-ž]/g, "")
|
|
1231
|
+
};
|
|
1232
|
+
};
|
|
1233
|
+
let { prefixEnd, prefixPart } = getPrefixInfo(entityText);
|
|
1234
|
+
let isAllCapsMatch = prefixPart.length > 2 && prefixPart === prefixPart.toUpperCase();
|
|
1235
|
+
if (isAllCapsMatch && fullText) {
|
|
1236
|
+
const lineStart = fullText.lastIndexOf("\n", entityStart);
|
|
1237
|
+
const lineEnd = fullText.indexOf("\n", entityStart + entityText.length);
|
|
1238
|
+
const lineLetters = fullText.slice(lineStart + 1, lineEnd === -1 ? fullText.length : lineEnd).replace(/[^a-zA-ZÀ-ž]/g, "");
|
|
1239
|
+
const upperCount = [...lineLetters].filter((c) => c === c.toUpperCase()).length;
|
|
1240
|
+
if (lineLetters.length > 5 && upperCount / lineLetters.length >= .95) continue;
|
|
1241
|
+
if ((prefixPart.length > 0 ? entityText.slice(0, prefixEnd > 0 ? prefixEnd : entityText.length).trim().split(/\s+/).length : 0) > 3) {
|
|
1242
|
+
entityStart = match.start;
|
|
1243
|
+
entityText = text;
|
|
1244
|
+
({prefixEnd, prefixPart} = getPrefixInfo(entityText));
|
|
1245
|
+
isAllCapsMatch = prefixPart.length > 2 && prefixPart === prefixPart.toUpperCase();
|
|
1246
|
+
}
|
|
1247
|
+
} else if (isAllCapsMatch) continue;
|
|
1248
|
+
const lastSpace = entityText.lastIndexOf(" ");
|
|
1249
|
+
const rawSuffix = lastSpace !== -1 ? entityText.slice(lastSpace + 1) : "";
|
|
1250
|
+
const suffixClean = rawSuffix.replace(/[.,]/g, "");
|
|
1251
|
+
if (suffixClean.length > 0 && ROMAN_NUMERAL_RE.test(suffixClean)) continue;
|
|
1252
|
+
if (suffixClean.length <= 2 && !/\./.test(rawSuffix) && /[^\x00-\x7F]/.test(entityText.slice(0, lastSpace !== -1 ? lastSpace : entityText.length))) continue;
|
|
1253
|
+
results.push({
|
|
1254
|
+
start: entityStart,
|
|
1255
|
+
end: entityStart + entityText.length,
|
|
1256
|
+
label: "organization",
|
|
1257
|
+
text: entityText,
|
|
1258
|
+
score: .95,
|
|
1259
|
+
source: DETECTION_SOURCES.LEGAL_FORM
|
|
1260
|
+
});
|
|
1261
|
+
}
|
|
1262
|
+
return results;
|
|
1263
|
+
};
|
|
1264
|
+
//#endregion
|
|
1265
|
+
//#region src/detectors/coreference.ts
|
|
1266
|
+
/**
|
|
1267
|
+
* Case-insensitive set of every known legal-form suffix
|
|
1268
|
+
* with whitespace normalized away. Used by the
|
|
1269
|
+
* coreference scan to reject parenthetical define-term
|
|
1270
|
+
* captures whose alias is only a legal-form
|
|
1271
|
+
* abbreviation (e.g. "(« SAS »)" after "ACME SAS"),
|
|
1272
|
+
* which would otherwise propagate generic suffix
|
|
1273
|
+
* mentions as document-wide org aliases.
|
|
1274
|
+
*/
|
|
1275
|
+
const isLegalFormAlias = (alias) => {
|
|
1276
|
+
const normalized = alias.replace(/\s+/g, "").toLowerCase();
|
|
1277
|
+
if (normalized.length === 0) return false;
|
|
1278
|
+
for (const suffix of getKnownLegalSuffixes()) if (suffix.replace(/\s+/g, "").toLowerCase() === normalized) return true;
|
|
1279
|
+
return false;
|
|
1280
|
+
};
|
|
1281
|
+
/**
|
|
1282
|
+
* Load coreference definition patterns from per-language
|
|
1283
|
+
* JSON configs in src/data/. Uses the
|
|
1284
|
+
* language manifest for auto-discovery.
|
|
1285
|
+
*/
|
|
1286
|
+
const loadDefinitionPatterns = async () => {
|
|
1287
|
+
const patterns = [];
|
|
1288
|
+
const allRows = await loadLanguageConfigs("coreference", (mod) => {
|
|
1289
|
+
return mod.default ?? mod;
|
|
1290
|
+
});
|
|
1291
|
+
for (const rows of allRows) {
|
|
1292
|
+
if (!Array.isArray(rows)) {
|
|
1293
|
+
console.warn("[anonymize] coreference: unexpected config shape, skipping");
|
|
1294
|
+
continue;
|
|
1295
|
+
}
|
|
1296
|
+
for (const row of rows) try {
|
|
1297
|
+
patterns.push({ pattern: new RegExp(row.pattern, row.flags) });
|
|
1298
|
+
} catch (err) {
|
|
1299
|
+
console.warn(`[anonymize] coreference: invalid regex "${row.pattern}":`, err);
|
|
1300
|
+
}
|
|
1301
|
+
}
|
|
1302
|
+
return patterns;
|
|
1303
|
+
};
|
|
1304
|
+
/**
|
|
1305
|
+
* Load generic role terms that should NOT be treated
|
|
1306
|
+
* as PII coreferences. "Prodávající" (Seller),
|
|
1307
|
+
* "Kupující" (Buyer), etc. are legal roles, not
|
|
1308
|
+
* identifying information.
|
|
1309
|
+
*/
|
|
1310
|
+
const getRoleStopSet = async (ctx) => {
|
|
1311
|
+
if (ctx.roleStopSet) return ctx.roleStopSet;
|
|
1312
|
+
if (ctx.roleStopSetPromise) return ctx.roleStopSetPromise;
|
|
1313
|
+
const promise = (async () => {
|
|
1314
|
+
let result;
|
|
1315
|
+
try {
|
|
1316
|
+
const mod = await import("./generic-roles.mjs");
|
|
1317
|
+
const data = mod.default ?? mod;
|
|
1318
|
+
result = new Set(data.roles.map((r) => r.toLowerCase()));
|
|
1319
|
+
} catch {
|
|
1320
|
+
result = /* @__PURE__ */ new Set();
|
|
1321
|
+
}
|
|
1322
|
+
ctx.roleStopSet = result;
|
|
1323
|
+
return result;
|
|
1324
|
+
})();
|
|
1325
|
+
ctx.roleStopSetPromise = promise;
|
|
1326
|
+
return promise;
|
|
1327
|
+
};
|
|
1328
|
+
const getDefinitionPatterns = async (ctx) => {
|
|
1329
|
+
if (ctx.corefPatterns) return ctx.corefPatterns;
|
|
1330
|
+
if (ctx.corefPatternsPromise) return ctx.corefPatternsPromise;
|
|
1331
|
+
ctx.corefPatternsPromise = loadDefinitionPatterns();
|
|
1332
|
+
const patterns = await ctx.corefPatternsPromise;
|
|
1333
|
+
if (patterns.length === 0) {
|
|
1334
|
+
ctx.corefPatterns = patterns;
|
|
1335
|
+
if (!ctx.corefLoadAttempted) {
|
|
1336
|
+
ctx.corefLoadAttempted = true;
|
|
1337
|
+
console.warn("[anonymize] coreference: no definition patterns loaded; coreference detection will be inactive");
|
|
1338
|
+
}
|
|
1339
|
+
return patterns;
|
|
1340
|
+
}
|
|
1341
|
+
ctx.corefPatterns = patterns;
|
|
1342
|
+
return patterns;
|
|
1343
|
+
};
|
|
1344
|
+
const SEARCH_WINDOW = 200;
|
|
1345
|
+
/**
|
|
1346
|
+
* Check whether an alias has textual similarity to
|
|
1347
|
+
* the source entity. Prevents roles and structural
|
|
1348
|
+
* terms from being treated as name aliases.
|
|
1349
|
+
*
|
|
1350
|
+
* Three checks (any passes → similar):
|
|
1351
|
+
* 1. Word overlap: a word in the alias appears in the
|
|
1352
|
+
* entity (case-insensitive, min 2 chars)
|
|
1353
|
+
* 2. Initials: alias letters match first letters of
|
|
1354
|
+
* entity words ("TB" ↔ "Tomas Bata")
|
|
1355
|
+
* 3. Substring: alias is a substring of the entity
|
|
1356
|
+
* or vice versa (min 3 chars)
|
|
1357
|
+
*/
|
|
1358
|
+
const hasEntitySimilarity = (alias, entityText) => {
|
|
1359
|
+
const aliasLower = alias.toLowerCase();
|
|
1360
|
+
const entityLower = entityText.toLowerCase();
|
|
1361
|
+
if (aliasLower.length >= 3 && entityLower.includes(aliasLower)) return true;
|
|
1362
|
+
if (entityLower.length >= 3 && aliasLower.includes(entityLower)) return true;
|
|
1363
|
+
const aliasWords = aliasLower.split(/[\s.,;:'"()/-]+/).filter((w) => w.length >= 2);
|
|
1364
|
+
const entityWords = entityLower.split(/[\s.,;:'"()/-]+/).filter((w) => w.length >= 2);
|
|
1365
|
+
const entityWordSet = new Set(entityWords);
|
|
1366
|
+
for (const word of aliasWords) if (entityWordSet.has(word)) return true;
|
|
1367
|
+
if (/^[\p{Lu}]+$/u.test(alias) && alias.length >= 2 && alias.length <= entityWords.length) {
|
|
1368
|
+
for (let start = 0; start <= entityWords.length - alias.length; start++) if (entityWords.slice(start, start + alias.length).map((w) => w.charAt(0)).join("") === aliasLower) return true;
|
|
1369
|
+
}
|
|
1370
|
+
return false;
|
|
1371
|
+
};
|
|
1372
|
+
/**
|
|
1373
|
+
* Scan for defined-term patterns near known entities.
|
|
1374
|
+
*
|
|
1375
|
+
* Legal documents universally follow:
|
|
1376
|
+
* "Dr. Heinrich Muller (hereinafter 'the Seller')..."
|
|
1377
|
+
*
|
|
1378
|
+
* After NER detects the entity, this function scans for
|
|
1379
|
+
* definitional patterns within +/-200 chars and extracts
|
|
1380
|
+
* the alias. Returns alias + label pairs that can be added
|
|
1381
|
+
* to the gazetteer for a full-text re-scan.
|
|
1382
|
+
*/
|
|
1383
|
+
/**
|
|
1384
|
+
* Labels that can be the source of a coreference alias.
|
|
1385
|
+
* Only parties (person, organization) have defined-term
|
|
1386
|
+
* aliases in legal text. Dates, addresses, IDs do not.
|
|
1387
|
+
*/
|
|
1388
|
+
const COREF_SOURCE_LABELS = new Set(["person", "organization"]);
|
|
1389
|
+
const extractDefinedTerms = async (fullText, entities, ctx = defaultContext) => {
|
|
1390
|
+
const [definitionPatterns, roleStops] = await Promise.all([getDefinitionPatterns(ctx), getRoleStopSet(ctx)]);
|
|
1391
|
+
const terms = [];
|
|
1392
|
+
const seen = /* @__PURE__ */ new Set();
|
|
1393
|
+
const sorted = [...entities].sort((a, b) => a.start - b.start);
|
|
1394
|
+
for (const { pattern } of definitionPatterns) {
|
|
1395
|
+
pattern.lastIndex = 0;
|
|
1396
|
+
for (let match = pattern.exec(fullText); match !== null; match = pattern.exec(fullText)) {
|
|
1397
|
+
const alias = match[1]?.trim();
|
|
1398
|
+
if (!alias || alias.length < 2) continue;
|
|
1399
|
+
if (roleStops.has(alias.toLowerCase())) continue;
|
|
1400
|
+
if (isLegalFormAlias(alias)) continue;
|
|
1401
|
+
const defPos = match.index;
|
|
1402
|
+
let bestEntity = null;
|
|
1403
|
+
for (let i = sorted.length - 1; i >= 0; i--) {
|
|
1404
|
+
const e = sorted[i];
|
|
1405
|
+
if (e === void 0) continue;
|
|
1406
|
+
if (e.end > defPos) continue;
|
|
1407
|
+
if (defPos - e.end > SEARCH_WINDOW) break;
|
|
1408
|
+
if (!COREF_SOURCE_LABELS.has(e.label)) continue;
|
|
1409
|
+
bestEntity = e;
|
|
1410
|
+
break;
|
|
1411
|
+
}
|
|
1412
|
+
if (bestEntity === null) continue;
|
|
1413
|
+
const gapText = fullText.slice(bestEntity.end, defPos);
|
|
1414
|
+
if (/(?:;|\.(?=\s*(?:["'„‚(]*\p{Lu}|$)))/u.test(gapText)) continue;
|
|
1415
|
+
if (!hasEntitySimilarity(alias, bestEntity.text)) continue;
|
|
1416
|
+
const key = `${alias.toLowerCase()}::${bestEntity.label}`;
|
|
1417
|
+
if (seen.has(key)) continue;
|
|
1418
|
+
seen.add(key);
|
|
1419
|
+
terms.push({
|
|
1420
|
+
alias,
|
|
1421
|
+
label: bestEntity.label,
|
|
1422
|
+
definitionStart: defPos,
|
|
1423
|
+
sourceText: bestEntity.text
|
|
1424
|
+
});
|
|
1425
|
+
}
|
|
1426
|
+
}
|
|
1427
|
+
return terms;
|
|
1428
|
+
};
|
|
1429
|
+
/**
|
|
1430
|
+
* Check if a character is a Unicode word character
|
|
1431
|
+
* (letter, digit, or combining mark). Used for word
|
|
1432
|
+
* boundary checks in coreference matching.
|
|
1433
|
+
*/
|
|
1434
|
+
const isWordChar = (ch) => {
|
|
1435
|
+
if (ch === void 0) return false;
|
|
1436
|
+
return /[\p{L}\p{M}\p{N}]/u.test(ch);
|
|
1437
|
+
};
|
|
1438
|
+
/**
|
|
1439
|
+
* Find all occurrences of defined-term aliases in the
|
|
1440
|
+
* full text. Returns Entity spans for each match.
|
|
1441
|
+
*
|
|
1442
|
+
* Respects word boundaries: "Kupující" must not match
|
|
1443
|
+
* inside "Kupujícímu". A match is valid only if the
|
|
1444
|
+
* character before the start and after the end are NOT
|
|
1445
|
+
* word characters (letter/digit).
|
|
1446
|
+
*
|
|
1447
|
+
* Populates `ctx.corefSourceMap` with entries linking
|
|
1448
|
+
* each coref entity to its source entity text, for
|
|
1449
|
+
* consistent placeholder numbering.
|
|
1450
|
+
*/
|
|
1451
|
+
const findCoreferenceSpans = (fullText, terms, ctx = defaultContext) => {
|
|
1452
|
+
const results = [];
|
|
1453
|
+
for (const term of terms) {
|
|
1454
|
+
let searchFrom = 0;
|
|
1455
|
+
while (searchFrom < fullText.length) {
|
|
1456
|
+
const idx = fullText.indexOf(term.alias, searchFrom);
|
|
1457
|
+
if (idx === -1) break;
|
|
1458
|
+
const matchEnd = idx + term.alias.length;
|
|
1459
|
+
const charBefore = idx > 0 ? fullText[idx - 1] : void 0;
|
|
1460
|
+
const charAfter = fullText[matchEnd];
|
|
1461
|
+
if (isWordChar(charBefore) || isWordChar(charAfter)) {
|
|
1462
|
+
searchFrom = idx + 1;
|
|
1463
|
+
continue;
|
|
1464
|
+
}
|
|
1465
|
+
const entity = {
|
|
1466
|
+
start: idx,
|
|
1467
|
+
end: matchEnd,
|
|
1468
|
+
label: term.label,
|
|
1469
|
+
text: term.alias,
|
|
1470
|
+
score: .95,
|
|
1471
|
+
source: DETECTION_SOURCES.COREFERENCE
|
|
1472
|
+
};
|
|
1473
|
+
ctx.corefSourceMap.set(corefKey(entity), term.sourceText);
|
|
1474
|
+
results.push(entity);
|
|
1475
|
+
searchFrom = matchEnd;
|
|
1476
|
+
}
|
|
1477
|
+
}
|
|
1478
|
+
return results;
|
|
1479
|
+
};
|
|
1480
|
+
//#endregion
|
|
1481
|
+
//#region src/detectors/gazetteer.ts
|
|
1482
|
+
const MAX_EDIT_DISTANCE = 2;
|
|
1483
|
+
const MIN_FUZZY_LENGTH = 4;
|
|
1484
|
+
const MAX_PREFIX_OVERSHOOT = 7;
|
|
1485
|
+
/**
|
|
1486
|
+
* Collect all searchable strings (canonical + variants)
|
|
1487
|
+
* from gazetteer entries, mapped to their labels and
|
|
1488
|
+
* entry IDs.
|
|
1489
|
+
*/
|
|
1490
|
+
const buildSearchTerms = (entries) => {
|
|
1491
|
+
const terms = /* @__PURE__ */ new Map();
|
|
1492
|
+
for (const entry of entries) {
|
|
1493
|
+
const meta = {
|
|
1494
|
+
label: entry.label,
|
|
1495
|
+
entryId: entry.id
|
|
1496
|
+
};
|
|
1497
|
+
terms.set(entry.canonical, meta);
|
|
1498
|
+
for (const variant of entry.variants) terms.set(variant, meta);
|
|
1499
|
+
}
|
|
1500
|
+
return terms;
|
|
1501
|
+
};
|
|
1502
|
+
/**
|
|
1503
|
+
* Build TextSearch-compatible patterns from gazetteer
|
|
1504
|
+
* entries. Returns:
|
|
1505
|
+
* - Exact literal patterns for all terms
|
|
1506
|
+
* - Fuzzy patterns (distance: 2) for terms >= 4 chars
|
|
1507
|
+
* - Parallel metadata arrays for post-processing
|
|
1508
|
+
*
|
|
1509
|
+
* Patterns are ordered: all exact first, then all
|
|
1510
|
+
* fuzzy. The isFuzzy array marks which are which.
|
|
1511
|
+
*/
|
|
1512
|
+
const buildGazetteerPatterns = (entries) => {
|
|
1513
|
+
const terms = buildSearchTerms(entries);
|
|
1514
|
+
const patterns = [];
|
|
1515
|
+
const labels = [];
|
|
1516
|
+
const isFuzzy = [];
|
|
1517
|
+
for (const [term, meta] of terms) {
|
|
1518
|
+
patterns.push({
|
|
1519
|
+
pattern: term,
|
|
1520
|
+
literal: true,
|
|
1521
|
+
wholeWords: false
|
|
1522
|
+
});
|
|
1523
|
+
labels.push(meta.label);
|
|
1524
|
+
isFuzzy.push(false);
|
|
1525
|
+
}
|
|
1526
|
+
for (const [term, meta] of terms) {
|
|
1527
|
+
if (term.length < MIN_FUZZY_LENGTH) continue;
|
|
1528
|
+
patterns.push({
|
|
1529
|
+
pattern: term,
|
|
1530
|
+
distance: MAX_EDIT_DISTANCE
|
|
1531
|
+
});
|
|
1532
|
+
labels.push(meta.label);
|
|
1533
|
+
isFuzzy.push(true);
|
|
1534
|
+
}
|
|
1535
|
+
return {
|
|
1536
|
+
patterns,
|
|
1537
|
+
data: {
|
|
1538
|
+
labels,
|
|
1539
|
+
isFuzzy
|
|
1540
|
+
}
|
|
1541
|
+
};
|
|
1542
|
+
};
|
|
1543
|
+
/**
|
|
1544
|
+
* Process gazetteer matches from the unified literal
|
|
1545
|
+
* search. Receives all matches; filters to the
|
|
1546
|
+
* gazetteer slice via sliceStart/sliceEnd.
|
|
1547
|
+
*
|
|
1548
|
+
* Exact matches get score 0.9; fuzzy matches get
|
|
1549
|
+
* 0.85. Fuzzy matches that overlap an exact match
|
|
1550
|
+
* are dropped.
|
|
1551
|
+
*
|
|
1552
|
+
* For exact matches, attempts prefix extension for
|
|
1553
|
+
* legal suffixes ("a.s.", "GmbH", "s.r.o." after
|
|
1554
|
+
* the matched term).
|
|
1555
|
+
*/
|
|
1556
|
+
const processGazetteerMatches = (allMatches, sliceStart, sliceEnd, fullText, data) => {
|
|
1557
|
+
const results = [];
|
|
1558
|
+
const exactSpans = [];
|
|
1559
|
+
for (const match of allMatches) {
|
|
1560
|
+
const idx = match.pattern;
|
|
1561
|
+
if (idx < sliceStart || idx >= sliceEnd) continue;
|
|
1562
|
+
const localIdx = idx - sliceStart;
|
|
1563
|
+
if (data.isFuzzy[localIdx]) continue;
|
|
1564
|
+
const label = data.labels[localIdx];
|
|
1565
|
+
if (!label) continue;
|
|
1566
|
+
const extended = tryPrefixExtension(fullText, match.start, match.end);
|
|
1567
|
+
const end = extended?.end ?? match.end;
|
|
1568
|
+
const text = extended?.text ?? fullText.slice(match.start, match.end);
|
|
1569
|
+
exactSpans.push({
|
|
1570
|
+
start: match.start,
|
|
1571
|
+
end
|
|
1572
|
+
});
|
|
1573
|
+
results.push({
|
|
1574
|
+
start: match.start,
|
|
1575
|
+
end,
|
|
1576
|
+
label,
|
|
1577
|
+
text,
|
|
1578
|
+
score: .9,
|
|
1579
|
+
source: DETECTION_SOURCES.GAZETTEER
|
|
1580
|
+
});
|
|
1581
|
+
}
|
|
1582
|
+
for (const match of allMatches) {
|
|
1583
|
+
const idx = match.pattern;
|
|
1584
|
+
if (idx < sliceStart || idx >= sliceEnd) continue;
|
|
1585
|
+
const localIdx = idx - sliceStart;
|
|
1586
|
+
if (!data.isFuzzy[localIdx]) continue;
|
|
1587
|
+
if (match.distance === 0) continue;
|
|
1588
|
+
const label = data.labels[localIdx];
|
|
1589
|
+
if (!label) continue;
|
|
1590
|
+
if (exactSpans.some((e) => match.start < e.end && match.end > e.start)) continue;
|
|
1591
|
+
const matchText = fullText.slice(match.start, match.end);
|
|
1592
|
+
results.push({
|
|
1593
|
+
start: match.start,
|
|
1594
|
+
end: match.end,
|
|
1595
|
+
label,
|
|
1596
|
+
text: matchText,
|
|
1597
|
+
score: .85,
|
|
1598
|
+
source: DETECTION_SOURCES.GAZETTEER
|
|
1599
|
+
});
|
|
1600
|
+
}
|
|
1601
|
+
return results;
|
|
1602
|
+
};
|
|
1603
|
+
/**
|
|
1604
|
+
* Try to extend an exact match to capture one
|
|
1605
|
+
* trailing token (max 6 chars) that may be a legal
|
|
1606
|
+
* entity suffix (e.g., "a.s.", "GmbH", "s.r.o.").
|
|
1607
|
+
*
|
|
1608
|
+
* Does not validate the token against a legal-forms
|
|
1609
|
+
* list; false extensions are filtered by mergeAndDedup
|
|
1610
|
+
* when a legal-form detector produces a competing
|
|
1611
|
+
* entity with the correct span.
|
|
1612
|
+
*/
|
|
1613
|
+
const tryPrefixExtension = (fullText, start, end) => {
|
|
1614
|
+
const maxEnd = Math.min(end + MAX_PREFIX_OVERSHOOT, fullText.length);
|
|
1615
|
+
if (maxEnd <= end + 1) return null;
|
|
1616
|
+
const after = fullText.slice(end, maxEnd);
|
|
1617
|
+
if (!after.startsWith(" ")) return null;
|
|
1618
|
+
const nextSpace = after.indexOf(" ", 1);
|
|
1619
|
+
const suffixEnd = nextSpace !== -1 ? nextSpace : after.length;
|
|
1620
|
+
if (suffixEnd <= 1) return null;
|
|
1621
|
+
const newEnd = end + suffixEnd;
|
|
1622
|
+
return {
|
|
1623
|
+
end: newEnd,
|
|
1624
|
+
text: fullText.slice(start, newEnd)
|
|
1625
|
+
};
|
|
1626
|
+
};
|
|
1627
|
+
//#endregion
|
|
1628
|
+
//#region src/util/text.ts
|
|
1629
|
+
/**
|
|
1630
|
+
* Shared text utilities for detectors.
|
|
1631
|
+
*
|
|
1632
|
+
* Extracted from names.ts and deny-list.ts to avoid
|
|
1633
|
+
* duplicating regex constants and helper functions.
|
|
1634
|
+
*/
|
|
1635
|
+
/** Matches a string that starts with an uppercase letter. */
|
|
1636
|
+
const UPPER_START_RE = /^\p{Lu}/u;
|
|
1637
|
+
/** Matches a string consisting entirely of uppercase letters. */
|
|
1638
|
+
const ALL_UPPER_RE = /^\p{Lu}+$/u;
|
|
1639
|
+
const SENTENCE_END_RE = /[.!?]/;
|
|
1640
|
+
/**
|
|
1641
|
+
* Detect whether a position is at the start of a sentence.
|
|
1642
|
+
* Looks backward past whitespace for sentence-ending
|
|
1643
|
+
* punctuation (.!?). Position 0 and positions preceded
|
|
1644
|
+
* only by whitespace are considered sentence starts.
|
|
1645
|
+
*/
|
|
1646
|
+
const isSentenceStart = (text, pos) => {
|
|
1647
|
+
if (pos === 0) return true;
|
|
1648
|
+
let i = pos - 1;
|
|
1649
|
+
while (i >= 0 && /\s/.test(text[i] ?? "")) i--;
|
|
1650
|
+
if (i < 0) return true;
|
|
1651
|
+
return SENTENCE_END_RE.test(text[i] ?? "");
|
|
1652
|
+
};
|
|
1653
|
+
//#endregion
|
|
1654
|
+
//#region src/detectors/names.ts
|
|
1655
|
+
const getCorpus = (ctx) => ctx.nameCorpus;
|
|
1656
|
+
const getNameCorpusFirstNames = (ctx = defaultContext) => ctx.nameCorpus?.firstNamesList ?? [];
|
|
1657
|
+
const getNameCorpusSurnames = (ctx = defaultContext) => ctx.nameCorpus?.surnamesList ?? [];
|
|
1658
|
+
const getNameCorpusTitles = (ctx = defaultContext) => ctx.nameCorpus?.titlesList ?? [];
|
|
1659
|
+
/**
|
|
1660
|
+
* Load name corpus data from injected dictionaries
|
|
1661
|
+
* and legacy config files. Merges all sources.
|
|
1662
|
+
*
|
|
1663
|
+
* Safe to call multiple times; only loads once per
|
|
1664
|
+
* context. Must be called before detectNameCorpus or
|
|
1665
|
+
* the getNameCorpus*() accessors are used.
|
|
1666
|
+
*
|
|
1667
|
+
* @param dictionaries Optional pre-loaded dictionaries
|
|
1668
|
+
* with per-language first names and surnames. When
|
|
1669
|
+
* omitted, only legacy config files are used.
|
|
1670
|
+
*/
|
|
1671
|
+
const initNameCorpus = (ctx = defaultContext, dictionaries) => {
|
|
1672
|
+
if (ctx.nameCorpusPromise) return ctx.nameCorpusPromise;
|
|
1673
|
+
const promise = (async () => {
|
|
1674
|
+
try {
|
|
1675
|
+
const [legacyFirstMod, legacySurnameMod, titleMod, exclusionMod] = await Promise.all([
|
|
1676
|
+
import("./names-first.mjs"),
|
|
1677
|
+
import("./names-surnames.mjs"),
|
|
1678
|
+
import("./names-title-tokens.mjs"),
|
|
1679
|
+
import("./names-exclusions.mjs")
|
|
1680
|
+
]);
|
|
1681
|
+
const firstNames = [...legacyFirstMod.default.names];
|
|
1682
|
+
if (dictionaries?.firstNames) for (const names of Object.values(dictionaries.firstNames)) for (const name of names) firstNames.push(name);
|
|
1683
|
+
const surnames = [...legacySurnameMod.default.names];
|
|
1684
|
+
if (dictionaries?.surnames) for (const names of Object.values(dictionaries.surnames)) for (const name of names) surnames.push(name);
|
|
1685
|
+
const dedup = (arr) => {
|
|
1686
|
+
const seen = /* @__PURE__ */ new Set();
|
|
1687
|
+
const result = [];
|
|
1688
|
+
for (const item of arr) {
|
|
1689
|
+
if (seen.has(item)) continue;
|
|
1690
|
+
seen.add(item);
|
|
1691
|
+
result.push(item);
|
|
1546
1692
|
}
|
|
1547
|
-
|
|
1548
|
-
|
|
1549
|
-
|
|
1550
|
-
|
|
1551
|
-
|
|
1552
|
-
|
|
1553
|
-
|
|
1554
|
-
|
|
1555
|
-
|
|
1556
|
-
|
|
1557
|
-
|
|
1558
|
-
|
|
1559
|
-
|
|
1693
|
+
return result;
|
|
1694
|
+
};
|
|
1695
|
+
const dedupFirst = dedup(firstNames);
|
|
1696
|
+
const dedupSurnames = dedup(surnames);
|
|
1697
|
+
const titles = titleMod.default.tokens;
|
|
1698
|
+
const exclusions = exclusionMod.default.words;
|
|
1699
|
+
ctx.nameCorpus = {
|
|
1700
|
+
firstNames: Object.freeze(new Set(dedupFirst)),
|
|
1701
|
+
surnames: Object.freeze(new Set(dedupSurnames)),
|
|
1702
|
+
titleTokens: Object.freeze(new Set(titles)),
|
|
1703
|
+
excludedWords: Object.freeze(new Set(exclusions)),
|
|
1704
|
+
firstNamesList: Object.freeze(dedupFirst),
|
|
1705
|
+
surnamesList: Object.freeze(dedupSurnames),
|
|
1706
|
+
titlesList: Object.freeze(titles),
|
|
1707
|
+
excludedList: Object.freeze(exclusions)
|
|
1708
|
+
};
|
|
1709
|
+
} catch (err) {
|
|
1710
|
+
ctx.nameCorpusPromise = null;
|
|
1711
|
+
console.warn("[anonymize] Failed to load name corpus JSON — name detection disabled:", err);
|
|
1712
|
+
}
|
|
1713
|
+
})();
|
|
1714
|
+
ctx.nameCorpusPromise = promise;
|
|
1715
|
+
return promise;
|
|
1716
|
+
};
|
|
1717
|
+
const INFLECTION_SUFFIXES = [
|
|
1718
|
+
"ovi",
|
|
1719
|
+
"em",
|
|
1720
|
+
"om",
|
|
1721
|
+
"ou",
|
|
1722
|
+
"é",
|
|
1723
|
+
"a",
|
|
1724
|
+
"u"
|
|
1725
|
+
];
|
|
1726
|
+
/**
|
|
1727
|
+
* Strip common Czech/Slovak case suffixes from a token.
|
|
1728
|
+
* Returns candidate base forms if stripping produces a
|
|
1729
|
+
* plausible name (capitalised, length >= 3).
|
|
1730
|
+
*
|
|
1731
|
+
* For the "-ou" instrumental feminine suffix, also yields
|
|
1732
|
+
* base + "a" (e.g., "Editou" → "Edit" and "Edita")
|
|
1733
|
+
* because Czech feminine names decline -a → -ou.
|
|
1734
|
+
*/
|
|
1735
|
+
const stripInflection = (token) => {
|
|
1736
|
+
const candidates = [];
|
|
1737
|
+
for (const suffix of INFLECTION_SUFFIXES) if (token.length > suffix.length + 2 && token.endsWith(suffix)) {
|
|
1738
|
+
const base = token.slice(0, -suffix.length);
|
|
1739
|
+
if (/^\p{Lu}/u.test(base)) {
|
|
1740
|
+
candidates.push(base);
|
|
1741
|
+
if (suffix === "ou" || suffix === "é" || suffix === "u") candidates.push(`${base}a`);
|
|
1742
|
+
}
|
|
1743
|
+
}
|
|
1744
|
+
return candidates;
|
|
1745
|
+
};
|
|
1746
|
+
const TOKEN_TYPE = {
|
|
1747
|
+
NAME: "name",
|
|
1748
|
+
SURNAME: "surname",
|
|
1749
|
+
TITLE: "title",
|
|
1750
|
+
ABBREVIATION: "abbreviation",
|
|
1751
|
+
CAPITALIZED: "capitalized",
|
|
1752
|
+
OTHER: "other"
|
|
1753
|
+
};
|
|
1754
|
+
const PERSON_CHAIN_BREAK_RE$1 = /[!?;:]/u;
|
|
1755
|
+
const isInitialContinuationGap$1 = (text, gap) => /^\p{Lu}$/u.test(text) && /^\.[^\S\n]{1,2}$/u.test(gap) || /^[^\S\n]{1,2}(?:\p{Lu}\.[^\S\n]{1,2})+$/u.test(gap);
|
|
1756
|
+
/**
|
|
1757
|
+
* Check if a token is in the first-name set, either
|
|
1758
|
+
* directly or after stripping Czech/Slovak inflection.
|
|
1759
|
+
*/
|
|
1760
|
+
const isFirstNameToken = (token, corpus) => {
|
|
1761
|
+
if (corpus.firstNames.has(token)) return true;
|
|
1762
|
+
return stripInflection(token).some((b) => corpus.firstNames.has(b));
|
|
1763
|
+
};
|
|
1764
|
+
/**
|
|
1765
|
+
* Check if a token is in the surname set, either
|
|
1766
|
+
* directly or after stripping Czech/Slovak inflection.
|
|
1767
|
+
*/
|
|
1768
|
+
const isSurnameToken = (token, corpus) => {
|
|
1769
|
+
if (corpus.surnames.has(token)) return true;
|
|
1770
|
+
return stripInflection(token).some((b) => corpus.surnames.has(b));
|
|
1771
|
+
};
|
|
1772
|
+
/**
|
|
1773
|
+
* Check if a token looks like a single-letter
|
|
1774
|
+
* abbreviation: "J.", "M.", etc.
|
|
1775
|
+
*/
|
|
1776
|
+
const isAbbreviation = (token) => token.length === 2 && /^\p{Lu}$/u.test(token[0] ?? "") && token[1] === ".";
|
|
1777
|
+
const segmenter$1 = new Intl.Segmenter(void 0, { granularity: "word" });
|
|
1778
|
+
/**
|
|
1779
|
+
* Split text into word segments using Intl.Segmenter.
|
|
1780
|
+
* Only returns segments flagged as words.
|
|
1781
|
+
*/
|
|
1782
|
+
const segmentWords = (fullText) => {
|
|
1783
|
+
const words = [];
|
|
1784
|
+
for (const seg of segmenter$1.segment(fullText)) if (seg.isWordLike) words.push({
|
|
1785
|
+
text: seg.segment,
|
|
1786
|
+
start: seg.index,
|
|
1787
|
+
end: seg.index + seg.segment.length
|
|
1788
|
+
});
|
|
1789
|
+
return words;
|
|
1790
|
+
};
|
|
1791
|
+
/** NAME or SURNAME — both represent corpus-matched tokens */
|
|
1792
|
+
const isCorpusMatch = (type) => type === TOKEN_TYPE.NAME || type === TOKEN_TYPE.SURNAME;
|
|
1793
|
+
const classifyToken = (word, corpus) => {
|
|
1794
|
+
const { text, start, end } = word;
|
|
1795
|
+
const lower = text.toLowerCase();
|
|
1796
|
+
const stripped = text.endsWith(".") ? text.slice(0, -1).toLowerCase() : lower;
|
|
1797
|
+
if (corpus.titleTokens.has(stripped)) return {
|
|
1798
|
+
text,
|
|
1799
|
+
type: TOKEN_TYPE.TITLE,
|
|
1800
|
+
start,
|
|
1801
|
+
end
|
|
1802
|
+
};
|
|
1803
|
+
if (isAbbreviation(text)) return {
|
|
1804
|
+
text,
|
|
1805
|
+
type: TOKEN_TYPE.ABBREVIATION,
|
|
1806
|
+
start,
|
|
1807
|
+
end
|
|
1808
|
+
};
|
|
1809
|
+
if (corpus.excludedWords.has(lower)) return {
|
|
1810
|
+
text,
|
|
1811
|
+
type: TOKEN_TYPE.OTHER,
|
|
1812
|
+
start,
|
|
1813
|
+
end
|
|
1814
|
+
};
|
|
1815
|
+
if (text.length < 3) return {
|
|
1816
|
+
text,
|
|
1817
|
+
type: TOKEN_TYPE.OTHER,
|
|
1818
|
+
start,
|
|
1819
|
+
end
|
|
1820
|
+
};
|
|
1821
|
+
if (text.length > 3 && ALL_UPPER_RE.test(text)) return {
|
|
1822
|
+
text,
|
|
1823
|
+
type: TOKEN_TYPE.OTHER,
|
|
1824
|
+
start,
|
|
1825
|
+
end
|
|
1826
|
+
};
|
|
1827
|
+
if (!UPPER_START_RE.test(text)) return {
|
|
1828
|
+
text,
|
|
1829
|
+
type: TOKEN_TYPE.OTHER,
|
|
1830
|
+
start,
|
|
1831
|
+
end
|
|
1832
|
+
};
|
|
1833
|
+
if (isFirstNameToken(text, corpus)) return {
|
|
1834
|
+
text,
|
|
1835
|
+
type: TOKEN_TYPE.NAME,
|
|
1836
|
+
start,
|
|
1837
|
+
end
|
|
1838
|
+
};
|
|
1839
|
+
if (isSurnameToken(text, corpus)) return {
|
|
1840
|
+
text,
|
|
1841
|
+
type: TOKEN_TYPE.SURNAME,
|
|
1842
|
+
start,
|
|
1843
|
+
end
|
|
1844
|
+
};
|
|
1845
|
+
return {
|
|
1846
|
+
text,
|
|
1847
|
+
type: TOKEN_TYPE.CAPITALIZED,
|
|
1848
|
+
start,
|
|
1849
|
+
end
|
|
1850
|
+
};
|
|
1851
|
+
};
|
|
1852
|
+
/**
|
|
1853
|
+
* Detect person names by looking up tokens against the
|
|
1854
|
+
* name corpus, then chaining adjacent name-like tokens.
|
|
1855
|
+
*
|
|
1856
|
+
* Requires initNameCorpus() to have been called first.
|
|
1857
|
+
* If not initialized, returns an empty array.
|
|
1858
|
+
*
|
|
1859
|
+
* Scoring:
|
|
1860
|
+
* TITLE + NAME/SURNAME → 0.95
|
|
1861
|
+
* NAME + NAME/SURNAME → 0.9
|
|
1862
|
+
* SURNAME + NAME/SURNAME → 0.9
|
|
1863
|
+
* NAME + CAPITALIZED → 0.7
|
|
1864
|
+
* ABBREVIATION + NAME → 0.7
|
|
1865
|
+
* Standalone NAME → 0.5 (low confidence)
|
|
1866
|
+
* Standalone SURNAME → skip (too ambiguous)
|
|
1867
|
+
*/
|
|
1868
|
+
const detectNameCorpus = (fullText, ctx = defaultContext) => {
|
|
1869
|
+
const corpus = getCorpus(ctx);
|
|
1870
|
+
if (!corpus) return [];
|
|
1871
|
+
const tokens = segmentWords(fullText).map((w) => classifyToken(w, corpus));
|
|
1872
|
+
const entities = [];
|
|
1873
|
+
const consumed = /* @__PURE__ */ new Set();
|
|
1874
|
+
for (let i = 0; i < tokens.length; i++) {
|
|
1875
|
+
if (consumed.has(i)) continue;
|
|
1876
|
+
const token = tokens[i];
|
|
1877
|
+
if (!token) continue;
|
|
1878
|
+
if (token.type !== TOKEN_TYPE.TITLE && token.type !== TOKEN_TYPE.NAME && token.type !== TOKEN_TYPE.SURNAME && token.type !== TOKEN_TYPE.ABBREVIATION) continue;
|
|
1879
|
+
const MAX_CHAIN = 5;
|
|
1880
|
+
const chain = [token];
|
|
1881
|
+
let j = i + 1;
|
|
1882
|
+
while (j < tokens.length && chain.length < MAX_CHAIN) {
|
|
1883
|
+
const next = tokens[j];
|
|
1884
|
+
if (!next) break;
|
|
1885
|
+
const prev = chain.at(-1);
|
|
1886
|
+
if (prev) {
|
|
1887
|
+
const gap = fullText.slice(prev.end, next.start);
|
|
1888
|
+
const breaksOnPeriod = gap.includes(".") && !isInitialContinuationGap$1(prev.text, gap);
|
|
1889
|
+
if (gap.includes("\n") || PERSON_CHAIN_BREAK_RE$1.test(gap) || breaksOnPeriod) break;
|
|
1890
|
+
}
|
|
1891
|
+
if (next.type === TOKEN_TYPE.NAME || next.type === TOKEN_TYPE.SURNAME || next.type === TOKEN_TYPE.TITLE || next.type === TOKEN_TYPE.ABBREVIATION || next.type === TOKEN_TYPE.CAPITALIZED) {
|
|
1892
|
+
chain.push(next);
|
|
1893
|
+
j++;
|
|
1894
|
+
} else break;
|
|
1895
|
+
}
|
|
1896
|
+
const hasTitle = chain.some((t) => t.type === TOKEN_TYPE.TITLE);
|
|
1897
|
+
const hasCorpusName = chain.some((t) => isCorpusMatch(t.type));
|
|
1898
|
+
const hasFirstName = chain.some((t) => t.type === TOKEN_TYPE.NAME);
|
|
1899
|
+
const hasAbbreviation = chain.some((t) => t.type === TOKEN_TYPE.ABBREVIATION);
|
|
1900
|
+
const corpusCount = chain.filter((t) => isCorpusMatch(t.type)).length;
|
|
1901
|
+
const capitalizedCount = chain.filter((t) => t.type === TOKEN_TYPE.CAPITALIZED).length;
|
|
1902
|
+
let score = 0;
|
|
1903
|
+
if (hasTitle && hasCorpusName) score = .95;
|
|
1904
|
+
else if (corpusCount >= 2) score = .9;
|
|
1905
|
+
else if (hasCorpusName && capitalizedCount > 0) score = .7;
|
|
1906
|
+
else if (hasAbbreviation && hasCorpusName) score = .7;
|
|
1907
|
+
else if (hasFirstName && chain.length === 1) {
|
|
1908
|
+
if (isSentenceStart(fullText, token.start)) continue;
|
|
1909
|
+
score = .5;
|
|
1910
|
+
} else if (!hasFirstName && chain.length === 1 && chain[0]?.type === TOKEN_TYPE.SURNAME) continue;
|
|
1911
|
+
else if (hasTitle && chain.length === 1) continue;
|
|
1912
|
+
else {
|
|
1913
|
+
if (!hasCorpusName) continue;
|
|
1914
|
+
score = .5;
|
|
1560
1915
|
}
|
|
1916
|
+
const first = chain.at(0);
|
|
1917
|
+
const last = chain.at(-1);
|
|
1918
|
+
if (!first || !last) continue;
|
|
1919
|
+
const start = first.start;
|
|
1920
|
+
const end = last.end;
|
|
1921
|
+
const text = fullText.slice(start, end);
|
|
1922
|
+
for (let k = i; k < i + chain.length; k++) consumed.add(k);
|
|
1923
|
+
entities.push({
|
|
1924
|
+
start,
|
|
1925
|
+
end,
|
|
1926
|
+
label: "person",
|
|
1927
|
+
text,
|
|
1928
|
+
score,
|
|
1929
|
+
source: DETECTION_SOURCES.REGEX
|
|
1930
|
+
});
|
|
1561
1931
|
}
|
|
1932
|
+
return entities;
|
|
1562
1933
|
};
|
|
1563
1934
|
//#endregion
|
|
1564
|
-
//#region src/
|
|
1565
|
-
/**
|
|
1566
|
-
* Centralized Unicode character equivalence groups.
|
|
1567
|
-
*
|
|
1568
|
-
* Centralized Unicode character equivalence groups.
|
|
1569
|
-
*
|
|
1570
|
-
* Provides helpers to build regex character classes that
|
|
1571
|
-
* match all typographic variants of a character type
|
|
1572
|
-
* (dashes, spaces, quotes, etc.).
|
|
1573
|
-
*
|
|
1574
|
-
* The JSON is statically imported so the bundler inlines
|
|
1575
|
-
* it, avoiding a runtime require() that breaks browsers.
|
|
1576
|
-
*/
|
|
1577
|
-
/** Chars that need escaping inside a regex char class. */
|
|
1578
|
-
const REGEX_CLASS_SPECIAL = /[\\\]^-]/;
|
|
1579
|
-
const escapeForCharClass = (ch) => REGEX_CLASS_SPECIAL.test(ch) ? `\\${ch}` : ch;
|
|
1580
|
-
const config = char_groups_default;
|
|
1935
|
+
//#region src/config/titles.ts
|
|
1581
1936
|
/**
|
|
1582
|
-
*
|
|
1583
|
-
*
|
|
1937
|
+
* Academic and professional title prefixes.
|
|
1938
|
+
* Plain text; the detector auto-escapes for regex.
|
|
1939
|
+
* Sorted longest-first at build time.
|
|
1584
1940
|
*/
|
|
1585
|
-
const
|
|
1586
|
-
|
|
1587
|
-
|
|
1588
|
-
|
|
1589
|
-
|
|
1941
|
+
const TITLE_PREFIXES = [
|
|
1942
|
+
"Ing.",
|
|
1943
|
+
"Mgr.",
|
|
1944
|
+
"MgA.",
|
|
1945
|
+
"Bc.",
|
|
1946
|
+
"BcA.",
|
|
1947
|
+
"JUDr.",
|
|
1948
|
+
"MUDr.",
|
|
1949
|
+
"MVDr.",
|
|
1950
|
+
"MDDr.",
|
|
1951
|
+
"PhDr.",
|
|
1952
|
+
"RNDr.",
|
|
1953
|
+
"PaedDr.",
|
|
1954
|
+
"ThDr.",
|
|
1955
|
+
"ThLic.",
|
|
1956
|
+
"ICDr.",
|
|
1957
|
+
"RSDr.",
|
|
1958
|
+
"PharmDr.",
|
|
1959
|
+
"artD.",
|
|
1960
|
+
"akad.",
|
|
1961
|
+
"doc.",
|
|
1962
|
+
"prof.",
|
|
1963
|
+
"ao. Univ.-Prof.",
|
|
1964
|
+
"o. Univ.-Prof.",
|
|
1965
|
+
"Univ.-Prof.",
|
|
1966
|
+
"Hon.-Prof.",
|
|
1967
|
+
"em. Prof.",
|
|
1968
|
+
"Dr. med. dent.",
|
|
1969
|
+
"Dr. med. vet.",
|
|
1970
|
+
"Dr. med.",
|
|
1971
|
+
"Dr. rer. nat.",
|
|
1972
|
+
"Dr. rer. soc.",
|
|
1973
|
+
"Dr. rer. pol.",
|
|
1974
|
+
"Dr. sc. tech.",
|
|
1975
|
+
"Dr. sc. nat.",
|
|
1976
|
+
"Dr. sc. hum.",
|
|
1977
|
+
"Dr. iur.",
|
|
1978
|
+
"Dr. jur.",
|
|
1979
|
+
"Dr. theol.",
|
|
1980
|
+
"Dr. oec.",
|
|
1981
|
+
"Dr. techn.",
|
|
1982
|
+
"Dr. h. c.",
|
|
1983
|
+
"Dr. phil.",
|
|
1984
|
+
"Dr.-Ing.",
|
|
1985
|
+
"Dr. Ing.",
|
|
1986
|
+
"Dr.",
|
|
1987
|
+
"Dipl.-Wirt.-Ing.",
|
|
1988
|
+
"Dipl.-Betriebsw.",
|
|
1989
|
+
"Dipl.-Inform.",
|
|
1990
|
+
"Dipl.-Volksw.",
|
|
1991
|
+
"Dipl.-Psych.",
|
|
1992
|
+
"Dipl.-Phys.",
|
|
1993
|
+
"Dipl.-Chem.",
|
|
1994
|
+
"Dipl.-Biol.",
|
|
1995
|
+
"Dipl.-Math.",
|
|
1996
|
+
"Dipl.-Päd.",
|
|
1997
|
+
"Dipl.-Soz.",
|
|
1998
|
+
"Dipl.-Kfm.",
|
|
1999
|
+
"Dipl.-Jur.",
|
|
2000
|
+
"Dipl. Ing.",
|
|
2001
|
+
"Dipl.-Ing.",
|
|
2002
|
+
"Mag. rer. soc. oec.",
|
|
2003
|
+
"Mag. rer. nat.",
|
|
2004
|
+
"Mag. phil.",
|
|
2005
|
+
"Mag. iur.",
|
|
2006
|
+
"Mag. arch.",
|
|
2007
|
+
"Mag. pharm.",
|
|
2008
|
+
"Mag. (FH)",
|
|
2009
|
+
"Mag.",
|
|
2010
|
+
"Bakk. rer. nat.",
|
|
2011
|
+
"Bakk. techn.",
|
|
2012
|
+
"Bakk. phil.",
|
|
2013
|
+
"Bakk.",
|
|
2014
|
+
"Lic. phil.",
|
|
2015
|
+
"Lic. iur.",
|
|
2016
|
+
"Lic. oec.",
|
|
2017
|
+
"Lic. theol.",
|
|
2018
|
+
"Lic.",
|
|
2019
|
+
"Priv.-Doz.",
|
|
2020
|
+
"PD",
|
|
2021
|
+
"RA"
|
|
2022
|
+
];
|
|
1590
2023
|
/**
|
|
1591
|
-
*
|
|
1592
|
-
*
|
|
1593
|
-
*
|
|
1594
|
-
*
|
|
1595
|
-
* The hyphen-minus is placed first so it is treated
|
|
1596
|
-
* as a literal, not a range indicator.
|
|
2024
|
+
* Courtesy/honorific titles that precede a person's
|
|
2025
|
+
* name. Sorted alphabetically. The detector escapes
|
|
2026
|
+
* dots and adds \b for entries in HONORIFIC_BOUNDARY.
|
|
1597
2027
|
*/
|
|
1598
|
-
const
|
|
1599
|
-
|
|
1600
|
-
|
|
1601
|
-
|
|
1602
|
-
|
|
1603
|
-
|
|
1604
|
-
|
|
2028
|
+
const HONORIFICS = [
|
|
2029
|
+
"Avv.",
|
|
2030
|
+
"Dame",
|
|
2031
|
+
"Doamna",
|
|
2032
|
+
"Domnul",
|
|
2033
|
+
"Don",
|
|
2034
|
+
"Doña",
|
|
2035
|
+
"Dott.",
|
|
2036
|
+
"Judge",
|
|
2037
|
+
"Justice",
|
|
2038
|
+
"Lady",
|
|
2039
|
+
"Lord",
|
|
2040
|
+
"M.",
|
|
2041
|
+
"Madame",
|
|
2042
|
+
"Mademoiselle",
|
|
2043
|
+
"Maître",
|
|
2044
|
+
"Me",
|
|
2045
|
+
"Messrs",
|
|
2046
|
+
"Miss",
|
|
2047
|
+
"Mlle",
|
|
2048
|
+
"Mme",
|
|
2049
|
+
"Monsieur",
|
|
2050
|
+
"Mr",
|
|
2051
|
+
"Mrs",
|
|
2052
|
+
"Ms",
|
|
2053
|
+
"Pr",
|
|
2054
|
+
"Pr.",
|
|
2055
|
+
"President",
|
|
2056
|
+
"Señor",
|
|
2057
|
+
"Señora",
|
|
2058
|
+
"Sig.",
|
|
2059
|
+
"Sig.ra",
|
|
2060
|
+
"Signor",
|
|
2061
|
+
"Signora",
|
|
2062
|
+
"Signorina",
|
|
2063
|
+
"Sir",
|
|
2064
|
+
"Sr.",
|
|
2065
|
+
"Sra."
|
|
2066
|
+
];
|
|
1605
2067
|
/**
|
|
1606
|
-
*
|
|
1607
|
-
*
|
|
1608
|
-
* inside a larger character class, e.g.:
|
|
1609
|
-
* `[\\s&,.${charClassInner("dash")}]`
|
|
2068
|
+
* Honorifics that need \b word-boundary anchors
|
|
2069
|
+
* (short or common words that could match mid-word).
|
|
1610
2070
|
*/
|
|
1611
|
-
const
|
|
1612
|
-
|
|
1613
|
-
|
|
1614
|
-
|
|
1615
|
-
|
|
1616
|
-
|
|
1617
|
-
|
|
1618
|
-
|
|
1619
|
-
|
|
2071
|
+
const HONORIFIC_BOUNDARY = new Set([
|
|
2072
|
+
"Don",
|
|
2073
|
+
"Doña",
|
|
2074
|
+
"M.",
|
|
2075
|
+
"Me",
|
|
2076
|
+
"Pr",
|
|
2077
|
+
"Pr.",
|
|
2078
|
+
"Señor",
|
|
2079
|
+
"Señora"
|
|
2080
|
+
]);
|
|
1620
2081
|
/**
|
|
1621
|
-
*
|
|
1622
|
-
*
|
|
1623
|
-
* `[\\s&,.${DASH_INNER}]`
|
|
2082
|
+
* Post-nominal degrees (comma or space separated after name).
|
|
2083
|
+
* Plain text; the detector auto-escapes for regex.
|
|
1624
2084
|
*/
|
|
1625
|
-
const
|
|
1626
|
-
|
|
1627
|
-
|
|
1628
|
-
|
|
1629
|
-
|
|
1630
|
-
|
|
1631
|
-
|
|
2085
|
+
const POST_NOMINALS = [
|
|
2086
|
+
"Ph.D.",
|
|
2087
|
+
"Ph.D",
|
|
2088
|
+
"CSc.",
|
|
2089
|
+
"DrSc.",
|
|
2090
|
+
"ArtD.",
|
|
2091
|
+
"D.Phil.",
|
|
2092
|
+
"DPhil.",
|
|
2093
|
+
"MPhil.",
|
|
2094
|
+
"MBA",
|
|
2095
|
+
"MPA",
|
|
2096
|
+
"LL.M.",
|
|
2097
|
+
"LL.B.",
|
|
2098
|
+
"M.Sc.",
|
|
2099
|
+
"B.Sc.",
|
|
2100
|
+
"MSc.",
|
|
2101
|
+
"BSc.",
|
|
2102
|
+
"M.Eng.",
|
|
2103
|
+
"B.Eng.",
|
|
2104
|
+
"M.A.",
|
|
2105
|
+
"B.A.",
|
|
2106
|
+
"JCD",
|
|
2107
|
+
"JD",
|
|
2108
|
+
"DiS.",
|
|
2109
|
+
"ACCA",
|
|
2110
|
+
"FCCA",
|
|
2111
|
+
"CIPM",
|
|
2112
|
+
"CIPT",
|
|
2113
|
+
"CIPP/E",
|
|
2114
|
+
"CIPP",
|
|
2115
|
+
"KC",
|
|
2116
|
+
"QC"
|
|
2117
|
+
];
|
|
1632
2118
|
//#endregion
|
|
1633
2119
|
//#region src/detectors/regex.ts
|
|
1634
2120
|
const MIN_PHONE_LENGTH = 7;
|
|
1635
2121
|
const MIN_MONTH_NAME_LENGTH = 3;
|
|
1636
2122
|
const escapeTitle = (title) => title.replace(/[.*+?^${}()|[\]\\]/g, "\\$&").replace(/\s+/g, "\\s*");
|
|
1637
2123
|
/** Escape for use inside a regex alternation. */
|
|
1638
|
-
const escapeRegex$
|
|
2124
|
+
const escapeRegex$2 = (s) => s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
1639
2125
|
/** Escape for use inside a regex character class. */
|
|
1640
2126
|
const escapeCharClass = (s) => s.replace(/[\]\\^-]/g, "\\$&");
|
|
1641
2127
|
const TITLE_PREFIX = TITLE_PREFIXES.toSorted((a, b) => b.length - a.length).map(escapeTitle).join("|");
|
|
1642
2128
|
const POST_NOMINAL = POST_NOMINALS.toSorted((a, b) => b.length - a.length).map(escapeTitle).join("|");
|
|
1643
|
-
const NAME_WORD =
|
|
2129
|
+
const NAME_WORD = `\\p{Lu}\\p{Ll}+`;
|
|
1644
2130
|
const PARTICLE = "(?:van der|van den|de la|della|von|van|dos|ibn|ben|bin|del|zum|zur|ten|ter|da|de|di|al|el|le|la|zu|af|av)";
|
|
1645
2131
|
const SP = "[^\\S\\n\\t]";
|
|
1646
2132
|
/** Honorific alternation built from titles.ts config. */
|
|
1647
2133
|
const HONORIFIC_ALT = [...HONORIFICS].toSorted((a, b) => b.length - a.length).map((h) => {
|
|
1648
|
-
const escaped = escapeRegex$
|
|
2134
|
+
const escaped = escapeRegex$2(h);
|
|
1649
2135
|
return HONORIFIC_BOUNDARY.has(h) ? `\\b${escaped}` : escaped;
|
|
1650
2136
|
}).join("|");
|
|
1651
2137
|
const toEntry = (validator, label, score) => {
|
|
@@ -1670,8 +2156,6 @@ const toEntry = (validator, label, score) => {
|
|
|
1670
2156
|
const STDNUM_ENTRIES = [
|
|
1671
2157
|
toEntry(hu.vat, "tax identification number", .95),
|
|
1672
2158
|
toEntry(it.codiceFiscale, "national identification number", .95),
|
|
1673
|
-
toEntry(es.dni, "national identification number", .9),
|
|
1674
|
-
toEntry(es.nie, "national identification number", .95),
|
|
1675
2159
|
toEntry(se.personnummer, "national identification number", .9),
|
|
1676
2160
|
toEntry(ro.cnp, "national identification number", .95),
|
|
1677
2161
|
toEntry(fr.nir, "social security number", .9),
|
|
@@ -1697,7 +2181,6 @@ const STDNUM_ENTRIES = [
|
|
|
1697
2181
|
toEntry(fi.ytunnus, "registration number", .9),
|
|
1698
2182
|
toEntry(bg.vat, "tax identification number", .95),
|
|
1699
2183
|
toEntry(sk.dic, "tax identification number", .95),
|
|
1700
|
-
toEntry(es.cif, "registration number", .95),
|
|
1701
2184
|
toEntry(es.vat, "tax identification number", .95),
|
|
1702
2185
|
toEntry(es.nss, "social security number", .9),
|
|
1703
2186
|
toEntry(fr.tva, "tax identification number", .95),
|
|
@@ -1719,7 +2202,9 @@ const STDNUM_ENTRIES = [
|
|
|
1719
2202
|
toEntry(ee.ik, "national identification number", .9),
|
|
1720
2203
|
toEntry(cy.vat, "tax identification number", .95),
|
|
1721
2204
|
toEntry(mt.vat, "tax identification number", .95),
|
|
1722
|
-
toEntry(lu.vat, "tax identification number", .95)
|
|
2205
|
+
toEntry(lu.vat, "tax identification number", .95),
|
|
2206
|
+
toEntry(br.cpf, "tax identification number", .95),
|
|
2207
|
+
toEntry(br.cnpj, "tax identification number", .95)
|
|
1723
2208
|
].filter((e) => e !== null);
|
|
1724
2209
|
const TITLED_PERSON = {
|
|
1725
2210
|
pattern: `(?:${TITLE_PREFIX})(?:${SP}+(?:${TITLE_PREFIX}))*${SP}+(?:${NAME_WORD})(?:${SP}{1,4}(?:${PARTICLE}${SP}+)?${NAME_WORD}){1,3}(?:,?${SP}+(?:${POST_NOMINAL})(?:,?${SP}+(?:${POST_NOMINAL}))*)?`,
|
|
@@ -1731,6 +2216,11 @@ const HONORIFIC_PERSON = {
|
|
|
1731
2216
|
label: "person",
|
|
1732
2217
|
score: .95
|
|
1733
2218
|
};
|
|
2219
|
+
const POSTNOMINAL_PERSON = {
|
|
2220
|
+
pattern: `${NAME_WORD}(?:(?:${SP}|-){1,2}(?:${PARTICLE}${SP}+)?${NAME_WORD}){1,3},?${SP}+(?:KC|QC)\\b`,
|
|
2221
|
+
label: "person",
|
|
2222
|
+
score: .95
|
|
2223
|
+
};
|
|
1734
2224
|
const IBAN = {
|
|
1735
2225
|
pattern: "\\b[A-Z]{2}\\d{2}\\s?[\\dA-Z]{4}\\s?[\\dA-Z]{4}\\s?[\\dA-Z]{4}\\s?[\\dA-Z]{4}\\s?[\\dA-Z]{0,14}\\b",
|
|
1736
2226
|
label: "iban",
|
|
@@ -1768,7 +2258,7 @@ const CREDIT_CARD = {
|
|
|
1768
2258
|
};
|
|
1769
2259
|
const CZ_BIRTH_NUMBER = {
|
|
1770
2260
|
pattern: `\\b\\d{6}/\\d{3,4}\\b`,
|
|
1771
|
-
label: "
|
|
2261
|
+
label: "birth number",
|
|
1772
2262
|
score: 1,
|
|
1773
2263
|
validator: cz.rc
|
|
1774
2264
|
};
|
|
@@ -1802,6 +2292,51 @@ const CZ_POSTAL = {
|
|
|
1802
2292
|
label: "address",
|
|
1803
2293
|
score: .7
|
|
1804
2294
|
};
|
|
2295
|
+
const ES_POSTAL = {
|
|
2296
|
+
pattern: "\\b(?:C\\.?P\\.?|[Cc][óo]digo[^\\S\\n]+postal)[^\\S\\n]{0,3}:?[^\\S\\n]{0,3}\\d{5}\\b",
|
|
2297
|
+
label: "address",
|
|
2298
|
+
score: .7
|
|
2299
|
+
};
|
|
2300
|
+
const ES_DNI = {
|
|
2301
|
+
pattern: `\\b\\d{8}-?[A-Za-z]\\b`,
|
|
2302
|
+
label: "national identification number",
|
|
2303
|
+
score: .9,
|
|
2304
|
+
validator: es.dni
|
|
2305
|
+
};
|
|
2306
|
+
const ES_NIE = {
|
|
2307
|
+
pattern: `\\b[XYZxyz]-?\\d{7}-?[A-Za-z]\\b`,
|
|
2308
|
+
label: "national identification number",
|
|
2309
|
+
score: .95,
|
|
2310
|
+
validator: es.nie
|
|
2311
|
+
};
|
|
2312
|
+
const ES_CIF = {
|
|
2313
|
+
pattern: `\\b[A-HJNP-SUVWa-hjnp-suvw]-?\\d{7}-?[0-9A-Ja-j]\\b`,
|
|
2314
|
+
label: "registration number",
|
|
2315
|
+
score: .95,
|
|
2316
|
+
validator: es.cif
|
|
2317
|
+
};
|
|
2318
|
+
const BR_CPF_FORMATTED = {
|
|
2319
|
+
pattern: `\\b\\d{3}\\.\\d{3}\\.\\d{3}${DASH}\\d{2}\\b`,
|
|
2320
|
+
label: "tax identification number",
|
|
2321
|
+
score: .95,
|
|
2322
|
+
validator: br.cpf
|
|
2323
|
+
};
|
|
2324
|
+
const BR_CNPJ_FORMATTED = {
|
|
2325
|
+
pattern: `\\b\\d{2}\\.\\d{3}\\.\\d{3}/\\d{4}${DASH}\\d{2}\\b`,
|
|
2326
|
+
label: "tax identification number",
|
|
2327
|
+
score: .95,
|
|
2328
|
+
validator: br.cnpj
|
|
2329
|
+
};
|
|
2330
|
+
const BR_RG_WITH_SSP = {
|
|
2331
|
+
pattern: `\\b\\d{1,3}\\.?\\d{3}\\.?\\d{3}(?:${DASH}[0-9A-Za-z])?[^\\S\\n]+SSP(?:/[A-Z]{2})?\\b`,
|
|
2332
|
+
label: "national identification number",
|
|
2333
|
+
score: .95
|
|
2334
|
+
};
|
|
2335
|
+
const BR_OAB = {
|
|
2336
|
+
pattern: "\\bOAB/[A-Z]{2}[^\\S\\n]+(?:n[º°.][^\\S\\n]*)?(?:\\d{1,3}(?:\\.\\d{3})+|\\d{4,6})\\b",
|
|
2337
|
+
label: "registration number",
|
|
2338
|
+
score: .95
|
|
2339
|
+
};
|
|
1805
2340
|
const URL = {
|
|
1806
2341
|
pattern: "(?:https?://|https?:(?=[^\\s])|www\\.)[\\w\\-]+(?:\\.[\\w\\-]+)+(?::\\d+)?(?:[/?#][^\\s)\\]>]*[^\\s.,;:!?)\\]>])?",
|
|
1807
2342
|
label: "url",
|
|
@@ -1830,6 +2365,7 @@ const PATH_SUFFIX = `(?:[/?#][^\\s)\\]>]*[^\\s.,;:!?)\\]>])?`;
|
|
|
1830
2365
|
const ALL_REGEX_DEFS = [
|
|
1831
2366
|
TITLED_PERSON,
|
|
1832
2367
|
HONORIFIC_PERSON,
|
|
2368
|
+
POSTNOMINAL_PERSON,
|
|
1833
2369
|
IBAN,
|
|
1834
2370
|
EMAIL,
|
|
1835
2371
|
INTL_PHONE,
|
|
@@ -1843,6 +2379,14 @@ const ALL_REGEX_DEFS = [
|
|
|
1843
2379
|
CZ_BANK_ACCOUNT,
|
|
1844
2380
|
HU_LANDLINE,
|
|
1845
2381
|
CZ_POSTAL,
|
|
2382
|
+
ES_POSTAL,
|
|
2383
|
+
ES_DNI,
|
|
2384
|
+
ES_NIE,
|
|
2385
|
+
ES_CIF,
|
|
2386
|
+
BR_CPF_FORMATTED,
|
|
2387
|
+
BR_CNPJ_FORMATTED,
|
|
2388
|
+
BR_RG_WITH_SSP,
|
|
2389
|
+
BR_OAB,
|
|
1846
2390
|
URL,
|
|
1847
2391
|
{
|
|
1848
2392
|
pattern: "\\b(?:[0-9a-fA-F]{1,4}:){7}[0-9a-fA-F]{1,4}\\b|\\b(?:[0-9a-fA-F]{1,4}:){1,7}:\\b|::(?:[0-9a-fA-F]{1,4}:){0,6}[0-9a-fA-F]{1,4}",
|
|
@@ -1859,6 +2403,17 @@ const ALL_REGEX_DEFS = [
|
|
|
1859
2403
|
label: "url",
|
|
1860
2404
|
score: .9
|
|
1861
2405
|
},
|
|
2406
|
+
{
|
|
2407
|
+
pattern: "\\b(?:GIR[^\\S\\n]?0AA|[A-PR-UWYZ](?:[A-HK-Y][0-9](?:[0-9]|[ABEHMNPRV-Y])?|[0-9](?:[0-9]|[A-HJKPS-UW])?)[^\\S\\n]?[0-9][ABD-HJLNP-UW-Z]{2})\\b",
|
|
2408
|
+
label: "address",
|
|
2409
|
+
score: .9
|
|
2410
|
+
},
|
|
2411
|
+
{
|
|
2412
|
+
pattern: "\\b[A-CEGHJ-PR-TWXYZ][A-CEGHJ-NPR-TWXYZ][^\\S\\n]?\\d{2}[^\\S\\n]?\\d{2}[^\\S\\n]?\\d{2}[^\\S\\n]?[A-D]?\\b",
|
|
2413
|
+
label: "social security number",
|
|
2414
|
+
score: .95,
|
|
2415
|
+
validator: gb.nino
|
|
2416
|
+
},
|
|
1862
2417
|
{
|
|
1863
2418
|
pattern: "\\b(?:1[0-2]|0?[1-9]):[0-5]\\d[^\\S\\n]?(?:[aApP]\\.?[mM]\\.?)(?=[\\s,;!?)]|$)",
|
|
1864
2419
|
label: "date",
|
|
@@ -1894,7 +2449,7 @@ const buildMonthAlternation = (months) => {
|
|
|
1894
2449
|
if (clean.length >= MIN_MONTH_NAME_LENGTH) seen.add(clean);
|
|
1895
2450
|
}
|
|
1896
2451
|
}
|
|
1897
|
-
return [...seen].toSorted((a, b) => b.length - a.length).map(escapeRegex$
|
|
2452
|
+
return [...seen].toSorted((a, b) => b.length - a.length).map(escapeRegex$2).join("|");
|
|
1898
2453
|
};
|
|
1899
2454
|
/**
|
|
1900
2455
|
* Build date patterns from a month-name alternation.
|
|
@@ -1951,11 +2506,11 @@ const buildCurrencyPatterns = (data) => {
|
|
|
1951
2506
|
const isAsciiAlpha = /^[a-zA-Z\s]+$/;
|
|
1952
2507
|
const parts = data.codes.map((code) => ({
|
|
1953
2508
|
len: code.length,
|
|
1954
|
-
alt: escapeRegex$
|
|
2509
|
+
alt: escapeRegex$2(code)
|
|
1955
2510
|
}));
|
|
1956
2511
|
const MIN_CI_LENGTH = 3;
|
|
1957
2512
|
if (data.localNames) for (const name of data.localNames) {
|
|
1958
|
-
const escaped = escapeRegex$
|
|
2513
|
+
const escaped = escapeRegex$2(name);
|
|
1959
2514
|
if (isAsciiAlpha.test(name) && name.length >= MIN_CI_LENGTH) parts.push({
|
|
1960
2515
|
len: name.length,
|
|
1961
2516
|
alt: `(?i:${escaped})`
|
|
@@ -1967,7 +2522,7 @@ const buildCurrencyPatterns = (data) => {
|
|
|
1967
2522
|
}
|
|
1968
2523
|
if (symbols) for (const ch of data.symbols) parts.push({
|
|
1969
2524
|
len: ch.length,
|
|
1970
|
-
alt: escapeRegex$
|
|
2525
|
+
alt: escapeRegex$2(ch)
|
|
1971
2526
|
});
|
|
1972
2527
|
const trailingAlt = parts.toSorted((a, b) => b.len - a.len).map((p) => p.alt).join("|");
|
|
1973
2528
|
if (!symbols && !trailingAlt) return [];
|
|
@@ -2019,290 +2574,70 @@ const processRegexMatches = (allMatches, sliceStart, sliceEnd, meta_) => {
|
|
|
2019
2574
|
for (const match of allMatches) {
|
|
2020
2575
|
const idx = match.pattern;
|
|
2021
2576
|
if (idx < sliceStart || idx >= sliceEnd) continue;
|
|
2022
|
-
const meta = meta_[idx - sliceStart];
|
|
2023
|
-
if (!meta) continue;
|
|
2024
|
-
if (meta.sourceDetail !== "custom-regex" && meta.label === "phone number" && match.text.length < MIN_PHONE_LENGTH) continue;
|
|
2025
|
-
if (meta.validator) {
|
|
2026
|
-
const compacted = meta.validator.compact(match.text);
|
|
2027
|
-
if (!meta.validator.validate(compacted).valid) continue;
|
|
2028
|
-
}
|
|
2029
|
-
const entity = {
|
|
2030
|
-
start: match.start,
|
|
2031
|
-
end: match.end,
|
|
2032
|
-
label: meta.label,
|
|
2033
|
-
text: match.text,
|
|
2034
|
-
score: meta.score,
|
|
2035
|
-
source: DETECTION_SOURCES.REGEX
|
|
2036
|
-
};
|
|
2037
|
-
if (meta.sourceDetail) entity.sourceDetail = meta.sourceDetail;
|
|
2038
|
-
results.push(entity);
|
|
2039
|
-
}
|
|
2040
|
-
return results;
|
|
2041
|
-
};
|
|
2042
|
-
/**
|
|
2043
|
-
* Build signing clause place-name patterns from
|
|
2044
|
-
* signing-clauses.json. Each pattern captures the
|
|
2045
|
-
* city/place name from contract signing locations.
|
|
2046
|
-
*
|
|
2047
|
-
* The place name sub-pattern:
|
|
2048
|
-
* \p{Lu}\p{Ll}+ (capitalized word)
|
|
2049
|
-
* optionally followed by preposition + capitalized
|
|
2050
|
-
* word (for "nad Nisou", "am Main", etc.)
|
|
2051
|
-
* optionally followed by more capitalized words
|
|
2052
|
-
* (for "Hradec Králové", "New York", etc.)
|
|
2053
|
-
*/
|
|
2054
|
-
const buildSigningClausePatterns = (data) => {
|
|
2055
|
-
const patterns = [];
|
|
2056
|
-
for (const entry of data.patterns) {
|
|
2057
|
-
const prepAlt = entry.prepositions.length > 0 ? entry.prepositions.join("|") : null;
|
|
2058
|
-
const place = prepAlt ? `(\\p{Lu}\\p{Ll}+(?:\\s+(?:${prepAlt})\\s+\\p{Lu}\\p{Ll}+)*(?:\\s+\\p{Lu}\\p{Ll}+)*)` : `(\\p{Lu}\\p{Ll}+(?:[- ]\\p{Lu}\\p{Ll}+)*)`;
|
|
2059
|
-
const full = `(?:^|\\n|[^\\S\\n])` + entry.prefix + place + (entry.suffix ? `(?:${entry.suffix})` : "");
|
|
2060
|
-
patterns.push(full);
|
|
2061
|
-
}
|
|
2062
|
-
return patterns;
|
|
2063
|
-
};
|
|
2064
|
-
const SIGNING_CLAUSE_META = {
|
|
2065
|
-
label: "address",
|
|
2066
|
-
score: .9
|
|
2067
|
-
};
|
|
2068
|
-
let signingPatternPromise = null;
|
|
2069
|
-
const loadSigningPatterns = async () => {
|
|
2070
|
-
const mod = await import("./signing-clauses.mjs");
|
|
2071
|
-
return buildSigningClausePatterns(mod.default ?? mod);
|
|
2072
|
-
};
|
|
2073
|
-
const getSigningClausePatterns = () => {
|
|
2074
|
-
if (!signingPatternPromise) signingPatternPromise = loadSigningPatterns().catch((err) => {
|
|
2075
|
-
signingPatternPromise = null;
|
|
2076
|
-
throw err;
|
|
2077
|
-
});
|
|
2078
|
-
return signingPatternPromise;
|
|
2079
|
-
};
|
|
2080
|
-
//#endregion
|
|
2081
|
-
//#region src/detectors/legal-forms.ts
|
|
2082
|
-
const UPPER = "A-ZÁČĎÉĚÍŇÓŘŠŤÚŮÝŽÄÖÜÀÂÆÇÈÊËÎÏÔÙÛŸÑ\\u0130";
|
|
2083
|
-
const LOWER = "a-záčďéěíňóřšťúůýžäöüßàâæçèêëîïôùûÿñ\\u0131";
|
|
2084
|
-
const CAP_WORD = `(?:[${UPPER}]{2,}|[${UPPER}][${LOWER}${UPPER}]+)`;
|
|
2085
|
-
const ANY_WORD = `(?:[${UPPER}${LOWER}][${LOWER}${UPPER}]+|[${UPPER}]{2,3}|\\d{1,4})`;
|
|
2086
|
-
const ALLCAP_WORD = `[${UPPER}]{2,}`;
|
|
2087
|
-
const ROMAN_NUMERAL_RE = /^(?=[IVXLCDM])M{0,3}(?:CM|CD|D?C{0,3})(?:XC|XL|L?X{0,3})(?:IX|IV|V?I{0,3})$/;
|
|
2088
|
-
const escapeForRegex = (form) => form.replace(/[.*+?^${}()|[\]\\]/g, "\\$&").replace(/\s+/g, "\\s+").replace(/\\\./g, "\\.[^\\S\\n]?");
|
|
2089
|
-
const isShortForm = (form) => form.replace(/[.\s]/g, "").length <= 3 && !form.includes(" ");
|
|
2090
|
-
const buildPatternString = (forms) => {
|
|
2091
|
-
if (forms.length === 0) return null;
|
|
2092
|
-
const alt = forms.toSorted((a, b) => b.length - a.length).map(escapeForRegex).join("|");
|
|
2093
|
-
return `${`(?:${CAP_WORD})(?:${`(?:[\\s&,.${DASH_INNER}]{1,4}|${`\\s+(?:a|and|und|et|e|y|i)\\s+(?=[${LOWER}])`})`}(?:${ANY_WORD})){0,10}`}(?:\\s+|,\\s*)(?:${alt})(?![${LOWER}])`;
|
|
2094
|
-
};
|
|
2095
|
-
/**
|
|
2096
|
-
* Build legal form regex pattern strings.
|
|
2097
|
-
* Returns an array of regex strings for the unified
|
|
2098
|
-
* TextSearch builder. Empty if data package is not
|
|
2099
|
-
* installed.
|
|
2100
|
-
*/
|
|
2101
|
-
const buildLegalFormPatterns = async () => {
|
|
2102
|
-
let data = {};
|
|
2103
|
-
try {
|
|
2104
|
-
data = (await import("./legal-forms.mjs")).default;
|
|
2105
|
-
} catch {
|
|
2106
|
-
return [];
|
|
2107
|
-
}
|
|
2108
|
-
const allForms = [];
|
|
2109
|
-
const seen = /* @__PURE__ */ new Set();
|
|
2110
|
-
for (const forms of Object.values(data)) for (const form of forms) {
|
|
2111
|
-
const key = form.toLowerCase();
|
|
2112
|
-
if (!seen.has(key)) {
|
|
2113
|
-
seen.add(key);
|
|
2114
|
-
allForms.push(form);
|
|
2115
|
-
}
|
|
2116
|
-
}
|
|
2117
|
-
const patterns = [];
|
|
2118
|
-
const longPattern = buildPatternString(allForms.filter((f) => !isShortForm(f)));
|
|
2119
|
-
if (longPattern) patterns.push(longPattern);
|
|
2120
|
-
const shortPattern = buildPatternString(allForms.filter(isShortForm));
|
|
2121
|
-
if (shortPattern) patterns.push(shortPattern);
|
|
2122
|
-
const allcapPrefix = `(?:${ALLCAP_WORD})(?:[\\s&,.${DASH_INNER}]{1,4}(?:${ALLCAP_WORD})){0,2}`;
|
|
2123
|
-
const allcapAlt = allForms.toSorted((a, b) => b.length - a.length).map(escapeForRegex).join("|");
|
|
2124
|
-
patterns.push(`${allcapPrefix}(?:\\s+|,\\s*)(?:${allcapAlt})(?![${LOWER}])`);
|
|
2125
|
-
return patterns;
|
|
2126
|
-
};
|
|
2127
|
-
const CONNECTOR_RE = /^(?:a|and|und|et|e|y|i|&)$/i;
|
|
2128
|
-
const LEADING_CLAUSE_RE = /(?:^|\s)(?:by\s+and\s+between|is\s+between)\s+/giu;
|
|
2129
|
-
/**
|
|
2130
|
-
* Find the word ending just before `pos` in `text`,
|
|
2131
|
-
* skipping any whitespace (not newlines).
|
|
2132
|
-
* Returns null if no word is found (e.g., at start
|
|
2133
|
-
* of text, or preceded by non-word chars like ".").
|
|
2134
|
-
*/
|
|
2135
|
-
const findWordBefore = (text, pos) => {
|
|
2136
|
-
let scan = pos - 1;
|
|
2137
|
-
while (scan >= 0) {
|
|
2138
|
-
const ch = text.charAt(scan);
|
|
2139
|
-
if (ch === "\n" || !/\s/.test(ch)) break;
|
|
2140
|
-
scan--;
|
|
2141
|
-
}
|
|
2142
|
-
if (scan < 0 || text.charAt(scan) === "\n") return null;
|
|
2143
|
-
const wordEnd = scan + 1;
|
|
2144
|
-
while (scan >= 0 && /[\p{L}\p{M}&]/u.test(text.charAt(scan))) scan--;
|
|
2145
|
-
const wordStart = scan + 1;
|
|
2146
|
-
const word = text.slice(wordStart, wordEnd);
|
|
2147
|
-
if (word.length === 0) return null;
|
|
2148
|
-
return {
|
|
2149
|
-
word,
|
|
2150
|
-
start: wordStart
|
|
2151
|
-
};
|
|
2152
|
-
};
|
|
2153
|
-
/**
|
|
2154
|
-
* Extend a match backward through uppercase words and
|
|
2155
|
-
* lowercase connectors. Stops at start of text,
|
|
2156
|
-
* newline, or a word that doesn't qualify.
|
|
2157
|
-
*
|
|
2158
|
-
* Connectors (a, and, und, et, ...) are only consumed
|
|
2159
|
-
* when there is a valid word before them — a trailing
|
|
2160
|
-
* connector at an entity boundary is not consumed.
|
|
2161
|
-
*/
|
|
2162
|
-
const extendBackward = (fullText, matchStart) => {
|
|
2163
|
-
let pos = matchStart;
|
|
2164
|
-
while (pos > 0) {
|
|
2165
|
-
const found = findWordBefore(fullText, pos);
|
|
2166
|
-
if (!found) break;
|
|
2167
|
-
const { word, start: wordStart } = found;
|
|
2168
|
-
const isUpper = /^\p{Lu}/u.test(word);
|
|
2169
|
-
const isConnector = CONNECTOR_RE.test(word);
|
|
2170
|
-
if (isUpper) pos = wordStart;
|
|
2171
|
-
else if (isConnector) {
|
|
2172
|
-
const prev = findWordBefore(fullText, wordStart);
|
|
2173
|
-
if (!prev) break;
|
|
2174
|
-
if (!/^\p{Lu}/u.test(prev.word)) break;
|
|
2175
|
-
pos = prev.start;
|
|
2176
|
-
} else break;
|
|
2177
|
-
}
|
|
2178
|
-
return pos;
|
|
2179
|
-
};
|
|
2180
|
-
const trimLeadingClause = (text) => {
|
|
2181
|
-
let cut = -1;
|
|
2182
|
-
for (const match of text.matchAll(LEADING_CLAUSE_RE)) cut = match.index + match[0].length;
|
|
2183
|
-
if (cut <= 0) return {
|
|
2184
|
-
offset: 0,
|
|
2185
|
-
text
|
|
2186
|
-
};
|
|
2187
|
-
const trimmed = text.slice(cut);
|
|
2188
|
-
const leadingWs = trimmed.match(/^\s*/u)?.[0].length ?? 0;
|
|
2189
|
-
return {
|
|
2190
|
-
offset: cut + leadingWs,
|
|
2191
|
-
text: trimmed.slice(leadingWs)
|
|
2192
|
-
};
|
|
2193
|
-
};
|
|
2194
|
-
/**
|
|
2195
|
-
* Process legal form matches from the unified search.
|
|
2196
|
-
* Receives all matches; filters to the legal forms
|
|
2197
|
-
* slice via sliceStart/sliceEnd.
|
|
2198
|
-
*/
|
|
2199
|
-
const processLegalFormMatches = (allMatches, sliceStart, sliceEnd, fullText) => {
|
|
2200
|
-
const results = [];
|
|
2201
|
-
for (const match of allMatches) {
|
|
2202
|
-
const idx = match.pattern;
|
|
2203
|
-
if (idx < sliceStart || idx >= sliceEnd) continue;
|
|
2204
|
-
const text = match.text.trimEnd();
|
|
2205
|
-
if (text.length < 5) continue;
|
|
2206
|
-
if (text.includes("\n")) continue;
|
|
2207
|
-
let entityStart = match.start;
|
|
2208
|
-
let entityText = text;
|
|
2209
|
-
if (fullText) {
|
|
2210
|
-
const extended = extendBackward(fullText, match.start);
|
|
2211
|
-
if (extended < match.start) {
|
|
2212
|
-
entityStart = extended;
|
|
2213
|
-
entityText = fullText.slice(extended, match.start + text.length).trimEnd();
|
|
2214
|
-
}
|
|
2215
|
-
}
|
|
2216
|
-
const clauseTrim = trimLeadingClause(entityText);
|
|
2217
|
-
if (clauseTrim.offset > 0) {
|
|
2218
|
-
entityStart += clauseTrim.offset;
|
|
2219
|
-
entityText = clauseTrim.text;
|
|
2220
|
-
}
|
|
2221
|
-
const getPrefixInfo = (value) => {
|
|
2222
|
-
const prefixEnd = value.lastIndexOf(",") !== -1 ? value.lastIndexOf(",") : value.lastIndexOf(" ");
|
|
2223
|
-
return {
|
|
2224
|
-
prefixEnd,
|
|
2225
|
-
prefixPart: prefixEnd > 0 ? value.slice(0, prefixEnd).replace(/[^a-zA-ZÀ-ž]/g, "") : value.replace(/[^a-zA-ZÀ-ž]/g, "")
|
|
2226
|
-
};
|
|
2227
|
-
};
|
|
2228
|
-
let { prefixEnd, prefixPart } = getPrefixInfo(entityText);
|
|
2229
|
-
let isAllCapsMatch = prefixPart.length > 2 && prefixPart === prefixPart.toUpperCase();
|
|
2230
|
-
if (isAllCapsMatch && fullText) {
|
|
2231
|
-
const lineStart = fullText.lastIndexOf("\n", entityStart);
|
|
2232
|
-
const lineEnd = fullText.indexOf("\n", entityStart + entityText.length);
|
|
2233
|
-
const lineLetters = fullText.slice(lineStart + 1, lineEnd === -1 ? fullText.length : lineEnd).replace(/[^a-zA-ZÀ-ž]/g, "");
|
|
2234
|
-
const upperCount = [...lineLetters].filter((c) => c === c.toUpperCase()).length;
|
|
2235
|
-
if (lineLetters.length > 5 && upperCount / lineLetters.length >= .95) continue;
|
|
2236
|
-
if ((prefixPart.length > 0 ? entityText.slice(0, prefixEnd > 0 ? prefixEnd : entityText.length).trim().split(/\s+/).length : 0) > 3) {
|
|
2237
|
-
entityStart = match.start;
|
|
2238
|
-
entityText = text;
|
|
2239
|
-
({prefixEnd, prefixPart} = getPrefixInfo(entityText));
|
|
2240
|
-
isAllCapsMatch = prefixPart.length > 2 && prefixPart === prefixPart.toUpperCase();
|
|
2241
|
-
}
|
|
2242
|
-
} else if (isAllCapsMatch) continue;
|
|
2243
|
-
const lastSpace = entityText.lastIndexOf(" ");
|
|
2244
|
-
const rawSuffix = lastSpace !== -1 ? entityText.slice(lastSpace + 1) : "";
|
|
2245
|
-
const suffixClean = rawSuffix.replace(/[.,]/g, "");
|
|
2246
|
-
if (suffixClean.length > 0 && ROMAN_NUMERAL_RE.test(suffixClean)) continue;
|
|
2247
|
-
if (suffixClean.length <= 2 && !/\./.test(rawSuffix) && /[^\x00-\x7F]/.test(entityText.slice(0, lastSpace !== -1 ? lastSpace : entityText.length))) continue;
|
|
2248
|
-
results.push({
|
|
2249
|
-
start: entityStart,
|
|
2250
|
-
end: entityStart + entityText.length,
|
|
2251
|
-
label: "organization",
|
|
2252
|
-
text: entityText,
|
|
2253
|
-
score: .95,
|
|
2254
|
-
source: DETECTION_SOURCES.LEGAL_FORM
|
|
2255
|
-
});
|
|
2577
|
+
const meta = meta_[idx - sliceStart];
|
|
2578
|
+
if (!meta) continue;
|
|
2579
|
+
if (meta.sourceDetail !== "custom-regex" && meta.label === "phone number" && match.text.length < MIN_PHONE_LENGTH) continue;
|
|
2580
|
+
if (meta.validator) {
|
|
2581
|
+
const compacted = meta.validator.compact(match.text);
|
|
2582
|
+
if (!meta.validator.validate(compacted).valid) continue;
|
|
2583
|
+
}
|
|
2584
|
+
const entity = {
|
|
2585
|
+
start: match.start,
|
|
2586
|
+
end: match.end,
|
|
2587
|
+
label: meta.label,
|
|
2588
|
+
text: match.text,
|
|
2589
|
+
score: meta.score,
|
|
2590
|
+
source: DETECTION_SOURCES.REGEX
|
|
2591
|
+
};
|
|
2592
|
+
if (meta.sourceDetail) entity.sourceDetail = meta.sourceDetail;
|
|
2593
|
+
results.push(entity);
|
|
2256
2594
|
}
|
|
2257
2595
|
return results;
|
|
2258
2596
|
};
|
|
2259
|
-
//#endregion
|
|
2260
|
-
//#region src/config/legal-forms.ts
|
|
2261
2597
|
/**
|
|
2262
|
-
*
|
|
2263
|
-
*
|
|
2264
|
-
*
|
|
2265
|
-
*
|
|
2266
|
-
*
|
|
2267
|
-
*
|
|
2598
|
+
* Build signing clause place-name patterns from
|
|
2599
|
+
* signing-clauses.json. Each pattern captures the
|
|
2600
|
+
* city/place name from contract signing locations.
|
|
2601
|
+
*
|
|
2602
|
+
* The place name sub-pattern:
|
|
2603
|
+
* \p{Lu}\p{Ll}+ (capitalized word)
|
|
2604
|
+
* optionally followed by preposition + capitalized
|
|
2605
|
+
* word (for "nad Nisou", "am Main", etc.)
|
|
2606
|
+
* optionally followed by more capitalized words
|
|
2607
|
+
* (for "Hradec Králové", "New York", etc.)
|
|
2268
2608
|
*/
|
|
2269
|
-
const
|
|
2270
|
-
|
|
2271
|
-
|
|
2272
|
-
|
|
2273
|
-
|
|
2274
|
-
|
|
2275
|
-
|
|
2276
|
-
|
|
2277
|
-
|
|
2278
|
-
|
|
2279
|
-
|
|
2280
|
-
|
|
2281
|
-
|
|
2282
|
-
|
|
2283
|
-
|
|
2284
|
-
|
|
2285
|
-
"
|
|
2286
|
-
|
|
2287
|
-
|
|
2288
|
-
|
|
2289
|
-
|
|
2290
|
-
|
|
2291
|
-
|
|
2292
|
-
|
|
2293
|
-
|
|
2294
|
-
|
|
2295
|
-
"LLP",
|
|
2296
|
-
"Inc.",
|
|
2297
|
-
"S.A.",
|
|
2298
|
-
"SA",
|
|
2299
|
-
"SAS",
|
|
2300
|
-
"SARL",
|
|
2301
|
-
"Sp. z o.o.",
|
|
2302
|
-
"S.p.A."
|
|
2303
|
-
];
|
|
2609
|
+
const buildSigningClausePatterns = (data) => {
|
|
2610
|
+
const patterns = [];
|
|
2611
|
+
for (const entry of data.patterns) {
|
|
2612
|
+
const prepAlt = entry.prepositions.length > 0 ? entry.prepositions.join("|") : null;
|
|
2613
|
+
const place = prepAlt ? `(\\p{Lu}\\p{Ll}+(?:\\s+(?:${prepAlt})\\s+\\p{Lu}\\p{Ll}+)*(?:\\s+\\p{Lu}\\p{Ll}+)*)` : `(\\p{Lu}\\p{Ll}+(?:[- ]\\p{Lu}\\p{Ll}+)*)`;
|
|
2614
|
+
const full = `(?:^|\\n|[^\\S\\n])` + entry.prefix + place + (entry.suffix ? `(?:${entry.suffix})` : "");
|
|
2615
|
+
patterns.push(full);
|
|
2616
|
+
}
|
|
2617
|
+
return patterns;
|
|
2618
|
+
};
|
|
2619
|
+
const SIGNING_CLAUSE_META = {
|
|
2620
|
+
label: "address",
|
|
2621
|
+
score: .9
|
|
2622
|
+
};
|
|
2623
|
+
let signingPatternPromise = null;
|
|
2624
|
+
const loadSigningPatterns = async () => {
|
|
2625
|
+
const mod = await import("./signing-clauses.mjs");
|
|
2626
|
+
return buildSigningClausePatterns(mod.default ?? mod);
|
|
2627
|
+
};
|
|
2628
|
+
const getSigningClausePatterns = () => {
|
|
2629
|
+
if (!signingPatternPromise) signingPatternPromise = loadSigningPatterns().catch((err) => {
|
|
2630
|
+
signingPatternPromise = null;
|
|
2631
|
+
throw err;
|
|
2632
|
+
});
|
|
2633
|
+
return signingPatternPromise;
|
|
2634
|
+
};
|
|
2304
2635
|
//#endregion
|
|
2305
2636
|
//#region src/detectors/triggers.ts
|
|
2637
|
+
const VALID_ID_VALIDATORS = {
|
|
2638
|
+
"br.cpf": br.cpf,
|
|
2639
|
+
"br.cnpj": br.cnpj
|
|
2640
|
+
};
|
|
2306
2641
|
const TRIGGER_SCORE = .95;
|
|
2307
2642
|
const WHITESPACE_RE$1 = /\s+/;
|
|
2308
2643
|
const LETTER_RE = /\p{L}/u;
|
|
@@ -2317,12 +2652,23 @@ const DECIMAL_COMMA_RE = new RegExp(`^,(?:\\d|${DASH}{1,2})`);
|
|
|
2317
2652
|
* etc.), skip the comma and degree, then continue.
|
|
2318
2653
|
*/
|
|
2319
2654
|
const POST_NOMINAL_RE = new RegExp(`^,\\s*(?:${POST_NOMINALS.toSorted((a, b) => b.length - a.length).map((d) => d.replace(/[.*+?^${}()|[\]\\]/g, "\\$&").replace(/\\\./g, "\\.\\s*")).join("|")})\\.?`, "i");
|
|
2320
|
-
const
|
|
2321
|
-
const
|
|
2322
|
-
|
|
2323
|
-
|
|
2324
|
-
|
|
2325
|
-
|
|
2655
|
+
const buildLegalFormCheckRe = (forms) => {
|
|
2656
|
+
const parts = forms.map((f) => {
|
|
2657
|
+
const escaped = f.replace(/[.*+?^${}()|[\]\\]/g, "\\$&").replace(/\\\./g, "\\.\\s*");
|
|
2658
|
+
return /^[\p{L}\p{M}]+$/u.test(f) ? `(?<![\\p{L}\\p{N}])${escaped}(?![\\p{L}\\p{N}])` : escaped;
|
|
2659
|
+
});
|
|
2660
|
+
return new RegExp(parts.join("|"), "u");
|
|
2661
|
+
};
|
|
2662
|
+
let cachedLegalFormCheckRe = null;
|
|
2663
|
+
let cachedLegalFormCheckSource = null;
|
|
2664
|
+
const getLegalFormCheckRe = () => {
|
|
2665
|
+
const source = getKnownLegalSuffixes();
|
|
2666
|
+
if (cachedLegalFormCheckSource !== source || cachedLegalFormCheckRe === null) {
|
|
2667
|
+
cachedLegalFormCheckSource = source;
|
|
2668
|
+
cachedLegalFormCheckRe = buildLegalFormCheckRe(source);
|
|
2669
|
+
}
|
|
2670
|
+
return cachedLegalFormCheckRe;
|
|
2671
|
+
};
|
|
2326
2672
|
const compileValidations = (validations) => validations.map((v) => {
|
|
2327
2673
|
switch (v.type) {
|
|
2328
2674
|
case "starts-uppercase": return {
|
|
@@ -2349,6 +2695,17 @@ const compileValidations = (validations) => validations.map((v) => {
|
|
|
2349
2695
|
type: "matches-pattern",
|
|
2350
2696
|
re: new RegExp(v.pattern, (v.flags ?? "").replace(/[gy]/g, ""))
|
|
2351
2697
|
};
|
|
2698
|
+
case "valid-id": {
|
|
2699
|
+
const validator = VALID_ID_VALIDATORS[v.validator];
|
|
2700
|
+
if (!validator) throw new Error(`Unknown valid-id validator: ${JSON.stringify(v.validator)}`);
|
|
2701
|
+
return {
|
|
2702
|
+
type: "valid-id",
|
|
2703
|
+
check: (value) => {
|
|
2704
|
+
const compact = value.replace(/[\s.\-/]/g, "");
|
|
2705
|
+
return validator.validate(compact).valid;
|
|
2706
|
+
}
|
|
2707
|
+
};
|
|
2708
|
+
}
|
|
2352
2709
|
default: throw new Error(`Unknown validation type: ${JSON.stringify(v)}`);
|
|
2353
2710
|
}
|
|
2354
2711
|
});
|
|
@@ -2372,6 +2729,9 @@ const applyValidations = (text, validations) => {
|
|
|
2372
2729
|
case "matches-pattern":
|
|
2373
2730
|
if (!v.re.test(text)) return false;
|
|
2374
2731
|
break;
|
|
2732
|
+
case "valid-id":
|
|
2733
|
+
if (!v.check(text)) return false;
|
|
2734
|
+
break;
|
|
2375
2735
|
default: throw new Error(`Unknown compiled validation type: ${JSON.stringify(v)}`);
|
|
2376
2736
|
}
|
|
2377
2737
|
return true;
|
|
@@ -2466,8 +2826,10 @@ const buildTriggerPatterns = async () => {
|
|
|
2466
2826
|
strategy: rule.strategy.type
|
|
2467
2827
|
});
|
|
2468
2828
|
}
|
|
2829
|
+
const patterns = rules.map((r) => r.trigger.toLowerCase());
|
|
2830
|
+
await loadAddressStopKeywords();
|
|
2469
2831
|
return {
|
|
2470
|
-
patterns
|
|
2832
|
+
patterns,
|
|
2471
2833
|
rules
|
|
2472
2834
|
};
|
|
2473
2835
|
};
|
|
@@ -2488,14 +2850,103 @@ const stripQuotes = (value) => {
|
|
|
2488
2850
|
const COMMA_STOP_CHARS = new Set([
|
|
2489
2851
|
"\n",
|
|
2490
2852
|
"(",
|
|
2491
|
-
" "
|
|
2853
|
+
" ",
|
|
2854
|
+
";"
|
|
2492
2855
|
]);
|
|
2493
2856
|
/**
|
|
2857
|
+
* Sentence-terminator detection: a period that genuinely ends
|
|
2858
|
+
* one clause and starts another. Used by `to-next-comma` so
|
|
2859
|
+
* governing-law clauses ("…State of New York. SECTION 2…")
|
|
2860
|
+
* don't sweep across the period.
|
|
2861
|
+
*
|
|
2862
|
+
* Three positive signals (any one terminates):
|
|
2863
|
+
* 1. Long lowercase tail before the dot (>= 5 letters) —
|
|
2864
|
+
* catches "…construction. SECTION 2…", "…vykonává.".
|
|
2865
|
+
* 2. Currency/amount tail (zł, Kč, USD, €) — catches
|
|
2866
|
+
* "…w kwocie 1000 zł. Termin płatności…".
|
|
2867
|
+
* 3. Proper-noun head: a capitalized word of >= 4 letters
|
|
2868
|
+
* ending in lowercase, with no internal uppercase, AND
|
|
2869
|
+
* a substantial next clause (Capital + >= 2 lowercase).
|
|
2870
|
+
* Catches short city names that the lowercase-tail rule
|
|
2871
|
+
* misses: "…z siedzibą w Łódź. Kapitał zakładowy…",
|
|
2872
|
+
* "…seat in Brno. Section 2…".
|
|
2873
|
+
*
|
|
2874
|
+
* The rule must NOT fire on:
|
|
2875
|
+
* - title abbreviations: "Mr.", "Mrs.", "Dr.", "Hon.",
|
|
2876
|
+
* "Sr.", "Jr." (head <= 3 chars or insufficient Ll tail)
|
|
2877
|
+
* - degree abbreviations: "Ph.D.", "RNDr.", "MUDr.",
|
|
2878
|
+
* "Ing." (internal periods or internal uppercase block
|
|
2879
|
+
* the proper-noun pattern)
|
|
2880
|
+
* - street-type abbreviations: "Ste.", "Ave.", "Inc.",
|
|
2881
|
+
* "ul.", "al.", "nábř." (lowercase initials or
|
|
2882
|
+
* insufficient Ll tail)
|
|
2883
|
+
* - small-word lowercase abbreviations: "prof.", "inż.",
|
|
2884
|
+
* "hab." (no leading uppercase, so the proper-noun rule
|
|
2885
|
+
* can't fire)
|
|
2886
|
+
*/
|
|
2887
|
+
const NEXT_IS_SENTENCE_START_RE = /^\.(?:\s+\p{Lu}|\s*$)/u;
|
|
2888
|
+
const SENTENCE_TAIL_RE = /\p{Ll}{5,}$/u;
|
|
2889
|
+
/**
|
|
2890
|
+
* Proper-noun tail: capital letter + >= 3 lowercase letters,
|
|
2891
|
+
* preceded by a non-letter and non-period (so we don't slice
|
|
2892
|
+
* into the middle of an acronym or a multi-dot abbreviation).
|
|
2893
|
+
* The 4-character minimum excludes 2–3-char titles ("Mr",
|
|
2894
|
+
* "Mrs", "Dr", "Inc", "Ste"); the all-lowercase tail
|
|
2895
|
+
* excludes mixed-case degrees ("RNDr", "MUDr").
|
|
2896
|
+
*/
|
|
2897
|
+
const PROPER_NOUN_HEAD_RE = /(?:^|[^\p{L}.])\p{Lu}\p{Ll}{3,}$/u;
|
|
2898
|
+
/**
|
|
2899
|
+
* Next clause begins with a real word: capital + >= 2
|
|
2900
|
+
* lowercase letters. Filters cases where a capitalized
|
|
2901
|
+
* abbreviation (e.g., "Smith Inc.") follows a proper noun,
|
|
2902
|
+
* which would otherwise look sentence-like.
|
|
2903
|
+
*/
|
|
2904
|
+
const NEXT_IS_REAL_SENTENCE_RE = /^\.\s+\p{Lu}\p{Ll}{2,}/u;
|
|
2905
|
+
/**
|
|
2906
|
+
* Short currency-abbreviation tail (zł, Kč, gr, Ft, kr,
|
|
2907
|
+
* лв, USD, PLN, EUR, …). When a period follows one of
|
|
2908
|
+
* these and the next token starts uppercase, treat it as
|
|
2909
|
+
* a sentence boundary even though the tail is too short
|
|
2910
|
+
* to satisfy `SENTENCE_TAIL_RE`. Without this, amount
|
|
2911
|
+
* triggers using `to-next-comma` swallow the following
|
|
2912
|
+
* clause: `"w kwocie 1000 zł. Termin płatności…"`.
|
|
2913
|
+
*
|
|
2914
|
+
* Currency codes are typically uppercase (`USD`, `PLN`)
|
|
2915
|
+
* but appear lowercased in informal writing (`pln`, `eur`);
|
|
2916
|
+
* local names mix case (`zł`, `Kč`, `Ft`); symbols
|
|
2917
|
+
* (`€`, `$`, `£`) appear after the amount as well. The
|
|
2918
|
+
* negative lookbehind on a letter ensures the abbreviation
|
|
2919
|
+
* is matched only as a standalone token; symbols are
|
|
2920
|
+
* matched unconditionally since they are not letters.
|
|
2921
|
+
*/
|
|
2922
|
+
const CURRENCY_TAIL_RE = /(?:(?<![\p{L}])(?:zł|Kč|gr|Ft|kr|лв|USD|PLN|EUR|CZK|GBP|CHF|HUF|RON|SEK|NOK|DKK)|[€$£])$/iu;
|
|
2923
|
+
const NUMERIC_SENTENCE_TAIL_RE = /\d$/;
|
|
2924
|
+
const isSentenceTerminator = (text, periodIndex) => {
|
|
2925
|
+
const tail = text.slice(periodIndex);
|
|
2926
|
+
if (!NEXT_IS_SENTENCE_START_RE.test(tail)) return false;
|
|
2927
|
+
const head = text.slice(0, periodIndex);
|
|
2928
|
+
if (SENTENCE_TAIL_RE.test(head) || CURRENCY_TAIL_RE.test(head) || NUMERIC_SENTENCE_TAIL_RE.test(head)) return true;
|
|
2929
|
+
return PROPER_NOUN_HEAD_RE.test(head) && NEXT_IS_REAL_SENTENCE_RE.test(tail);
|
|
2930
|
+
};
|
|
2931
|
+
/**
|
|
2494
2932
|
* Field-label keywords that terminate address scanning.
|
|
2495
2933
|
* When a comma in the address strategy is followed by
|
|
2496
2934
|
* one of these, the address stops before the keyword.
|
|
2935
|
+
*
|
|
2936
|
+
* The list is sourced from
|
|
2937
|
+
* `data/address-stop-keywords.json` (per-language so new
|
|
2938
|
+
* languages can drop in their own labels without
|
|
2939
|
+
* touching this file). `loadAddressStopKeywords` unions
|
|
2940
|
+
* every language into a single longest-first array;
|
|
2941
|
+
* `getAddressStopKeywordsSync` returns the cached union
|
|
2942
|
+
* and falls back to a seed list so the strategy keeps
|
|
2943
|
+
* working before the warmup promise resolves.
|
|
2944
|
+
*
|
|
2945
|
+
* The address strategy doesn't know the document
|
|
2946
|
+
* language, so a flat union is intentional: any
|
|
2947
|
+
* language's labels can appear in any address.
|
|
2497
2948
|
*/
|
|
2498
|
-
const
|
|
2949
|
+
const ADDRESS_STOP_KEYWORDS_SEED = [
|
|
2499
2950
|
"číslo účtu",
|
|
2500
2951
|
"registrační",
|
|
2501
2952
|
"zastoupen",
|
|
@@ -2518,6 +2969,52 @@ const ADDRESS_STOP_KEYWORDS = [
|
|
|
2518
2969
|
"bic",
|
|
2519
2970
|
"ič"
|
|
2520
2971
|
];
|
|
2972
|
+
let addressStopKeywordsCache = null;
|
|
2973
|
+
let addressStopKeywordsPromise = null;
|
|
2974
|
+
const loadAddressStopKeywords = async () => {
|
|
2975
|
+
if (addressStopKeywordsCache) return addressStopKeywordsCache;
|
|
2976
|
+
if (addressStopKeywordsPromise) return addressStopKeywordsPromise;
|
|
2977
|
+
addressStopKeywordsPromise = (async () => {
|
|
2978
|
+
let data = {};
|
|
2979
|
+
try {
|
|
2980
|
+
const mod = await import("./address-stop-keywords.mjs");
|
|
2981
|
+
data = mod.default ?? mod;
|
|
2982
|
+
} catch (err) {
|
|
2983
|
+
console.warn("[anonymize] triggers: failed to load address-stop-keywords.json, falling back to seed list:", err);
|
|
2984
|
+
}
|
|
2985
|
+
const seen = /* @__PURE__ */ new Set();
|
|
2986
|
+
const out = [];
|
|
2987
|
+
const addAll = (list) => {
|
|
2988
|
+
for (const kw of list) {
|
|
2989
|
+
if (typeof kw !== "string" || kw.length === 0) continue;
|
|
2990
|
+
const lower = kw.toLowerCase();
|
|
2991
|
+
if (seen.has(lower)) continue;
|
|
2992
|
+
seen.add(lower);
|
|
2993
|
+
out.push(lower);
|
|
2994
|
+
}
|
|
2995
|
+
};
|
|
2996
|
+
for (const [key, value] of Object.entries(data)) {
|
|
2997
|
+
if (key.startsWith("_")) continue;
|
|
2998
|
+
if (!Array.isArray(value)) continue;
|
|
2999
|
+
addAll(value);
|
|
3000
|
+
}
|
|
3001
|
+
addAll(ADDRESS_STOP_KEYWORDS_SEED);
|
|
3002
|
+
out.sort((a, b) => b.length - a.length);
|
|
3003
|
+
addressStopKeywordsCache = out;
|
|
3004
|
+
return out;
|
|
3005
|
+
})();
|
|
3006
|
+
return addressStopKeywordsPromise;
|
|
3007
|
+
};
|
|
3008
|
+
const getAddressStopKeywordsSync = () => addressStopKeywordsCache ?? ADDRESS_STOP_KEYWORDS_SEED;
|
|
3009
|
+
/**
|
|
3010
|
+
* Warm the address-stop-keywords cache. Pipeline callers
|
|
3011
|
+
* await this before invoking trigger detection so the
|
|
3012
|
+
* synchronous `extractValue` path uses the merged list
|
|
3013
|
+
* instead of the seed fallback.
|
|
3014
|
+
*/
|
|
3015
|
+
const warmAddressStopKeywords = async () => {
|
|
3016
|
+
await loadAddressStopKeywords();
|
|
3017
|
+
};
|
|
2521
3018
|
const extractValue = (text, triggerEnd, strategy, label) => {
|
|
2522
3019
|
const remaining = text.slice(triggerEnd);
|
|
2523
3020
|
const stripped = remaining.replace(/^[\s:;]+/, "");
|
|
@@ -2526,6 +3023,8 @@ const extractValue = (text, triggerEnd, strategy, label) => {
|
|
|
2526
3023
|
if (valueText.length === 0) return null;
|
|
2527
3024
|
switch (strategy.type) {
|
|
2528
3025
|
case "to-next-comma": {
|
|
3026
|
+
const stopWords = strategy.stopWords ?? [];
|
|
3027
|
+
const stopWordRe = stopWords.length > 0 ? new RegExp(`^(?:${stopWords.map((w) => w.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")).join("|")})(?![\\p{L}\\p{N}])`, "iu") : null;
|
|
2529
3028
|
let end = 0;
|
|
2530
3029
|
let foundStop = false;
|
|
2531
3030
|
while (end < valueText.length) {
|
|
@@ -2534,6 +3033,14 @@ const extractValue = (text, triggerEnd, strategy, label) => {
|
|
|
2534
3033
|
foundStop = true;
|
|
2535
3034
|
break;
|
|
2536
3035
|
}
|
|
3036
|
+
if (ch === "." && isSentenceTerminator(valueText, end)) {
|
|
3037
|
+
foundStop = true;
|
|
3038
|
+
break;
|
|
3039
|
+
}
|
|
3040
|
+
if (stopWordRe !== null && (end === 0 || !/[\p{L}\p{N}]/u.test(valueText[end - 1] ?? "")) && stopWordRe.test(valueText.slice(end))) {
|
|
3041
|
+
foundStop = true;
|
|
3042
|
+
break;
|
|
3043
|
+
}
|
|
2537
3044
|
if (ch === ",") {
|
|
2538
3045
|
const afterComma = valueText.slice(end);
|
|
2539
3046
|
if (DECIMAL_COMMA_RE.test(afterComma)) {
|
|
@@ -2562,6 +3069,8 @@ const extractValue = (text, triggerEnd, strategy, label) => {
|
|
|
2562
3069
|
};
|
|
2563
3070
|
}
|
|
2564
3071
|
case "to-end-of-line": {
|
|
3072
|
+
const consumed = remaining.length - valueText.length;
|
|
3073
|
+
if (consumed > 0 && remaining.slice(0, consumed).includes("\n")) return null;
|
|
2565
3074
|
const LINE_STOPS = ["\n"];
|
|
2566
3075
|
let end = valueText.length;
|
|
2567
3076
|
for (const ch of LINE_STOPS) {
|
|
@@ -2582,7 +3091,8 @@ const extractValue = (text, triggerEnd, strategy, label) => {
|
|
|
2582
3091
|
const tabIdx = valueText.indexOf(" ");
|
|
2583
3092
|
const cellText = tabIdx !== -1 ? valueText.slice(0, tabIdx) : valueText;
|
|
2584
3093
|
const PUNCT_ONLY = /^[\p{P}\p{S}]+$/u;
|
|
2585
|
-
const
|
|
3094
|
+
const NUMBER_MARKER = /^(?:n[ºo°.]|№)$/i;
|
|
3095
|
+
const words = cellText.split(WHITESPACE_RE$1).filter((w) => !PUNCT_ONLY.test(w) && !NUMBER_MARKER.test(w)).slice(0, strategy.count);
|
|
2586
3096
|
if (words.length === 0) return null;
|
|
2587
3097
|
const firstWord = words[0];
|
|
2588
3098
|
if (firstWord === void 0) return null;
|
|
@@ -2605,14 +3115,22 @@ const extractValue = (text, triggerEnd, strategy, label) => {
|
|
|
2605
3115
|
}
|
|
2606
3116
|
case "company-id-value": {
|
|
2607
3117
|
const raw = text.slice(triggerEnd);
|
|
2608
|
-
const
|
|
3118
|
+
const triggerLastChar = text[triggerEnd - 1] ?? "";
|
|
3119
|
+
const sepMatch = (triggerLastChar === "°" || triggerLastChar === "º" || triggerLastChar === "№" || triggerLastChar === "#" ? /^(?:\s*:\s*|\s+|)/ : /^(?:\s*:\s*|\s+)/).exec(raw);
|
|
2609
3120
|
if (!sepMatch) return null;
|
|
2610
|
-
|
|
2611
|
-
const
|
|
3121
|
+
let afterSep = raw.slice(sepMatch[0].length);
|
|
3122
|
+
const labelMatch = /^(?:nr\.?|numer|n[ºo°.]|№|no\.?)(?:\s*:\s*|\s+)/i.exec(afterSep);
|
|
3123
|
+
let labelOffset = 0;
|
|
3124
|
+
if (labelMatch) {
|
|
3125
|
+
labelOffset = labelMatch[0].length;
|
|
3126
|
+
afterSep = afterSep.slice(labelOffset);
|
|
3127
|
+
}
|
|
3128
|
+
const idMatch = /^[A-Z]{0,6}\s?(?:[A-Z0-9]{2}\s?)?\d{2}(?:(?<=\d)[A-Z]|[\d\s.\-/]){3,}/i.exec(afterSep);
|
|
2612
3129
|
if (!idMatch) return null;
|
|
2613
|
-
const
|
|
3130
|
+
const trailingLetterMatch = /\d$/.test(idMatch[0]) ? /^[A-Z]/i.exec(afterSep.slice(idMatch[0].length)) : null;
|
|
3131
|
+
const idText = (trailingLetterMatch ? idMatch[0] + trailingLetterMatch[0] : idMatch[0]).trim();
|
|
2614
3132
|
const leadingSpaces = idMatch[0].length - idMatch[0].trimStart().length;
|
|
2615
|
-
const idStart = triggerEnd + sepMatch[0].length + leadingSpaces;
|
|
3133
|
+
const idStart = triggerEnd + sepMatch[0].length + labelOffset + leadingSpaces;
|
|
2616
3134
|
return {
|
|
2617
3135
|
start: idStart,
|
|
2618
3136
|
end: idStart + idText.length,
|
|
@@ -2622,18 +3140,36 @@ const extractValue = (text, triggerEnd, strategy, label) => {
|
|
|
2622
3140
|
case "address": {
|
|
2623
3141
|
const maxLen = strategy.maxChars ?? 120;
|
|
2624
3142
|
const UPPER_RE = /\p{Lu}/u;
|
|
3143
|
+
const stopKeywords = getAddressStopKeywordsSync();
|
|
2625
3144
|
let end = 0;
|
|
2626
3145
|
while (end < valueText.length && end < maxLen) {
|
|
2627
3146
|
const ch = valueText[end];
|
|
2628
3147
|
if (ch === "\n" || ch === "(") break;
|
|
3148
|
+
if (ch === " " || ch === " ") {
|
|
3149
|
+
const afterWs = valueText.slice(end).trimStart().toLowerCase();
|
|
3150
|
+
if (stopKeywords.some((kw) => {
|
|
3151
|
+
if (!afterWs.startsWith(kw)) return false;
|
|
3152
|
+
const next = afterWs[kw.length];
|
|
3153
|
+
return next === void 0 || /[\s:;.,!?()\d]/.test(next);
|
|
3154
|
+
})) break;
|
|
3155
|
+
}
|
|
2629
3156
|
if (ch === ".") {
|
|
2630
3157
|
const next = valueText[end + 1];
|
|
2631
3158
|
const afterNext = valueText[end + 2];
|
|
3159
|
+
const afterPeriod = valueText.slice(end + 1).replace(/^\s+/, "").toLowerCase();
|
|
3160
|
+
if (afterPeriod.length > 0) {
|
|
3161
|
+
if (getAddressStopKeywordsSync().some((kw) => {
|
|
3162
|
+
if (!afterPeriod.startsWith(kw)) return false;
|
|
3163
|
+
const afterKw = afterPeriod[kw.length];
|
|
3164
|
+
return afterKw === void 0 || /[\s:;.,!?()\d]/.test(afterKw);
|
|
3165
|
+
})) break;
|
|
3166
|
+
}
|
|
2632
3167
|
if (next !== void 0 && (/\p{L}/u.test(next) || /\d/.test(next))) {
|
|
2633
3168
|
end++;
|
|
2634
3169
|
continue;
|
|
2635
3170
|
}
|
|
2636
3171
|
if (next === " " && afterNext !== void 0 && (/\p{L}/u.test(afterNext) || /\d/.test(afterNext))) {
|
|
3172
|
+
if (isSentenceTerminator(valueText, end)) break;
|
|
2637
3173
|
end++;
|
|
2638
3174
|
continue;
|
|
2639
3175
|
}
|
|
@@ -2645,7 +3181,7 @@ const extractValue = (text, triggerEnd, strategy, label) => {
|
|
|
2645
3181
|
const peekCh = valueText[peek];
|
|
2646
3182
|
if (peekCh === void 0) break;
|
|
2647
3183
|
const afterComma = valueText.slice(end + 1).trimStart().toLowerCase();
|
|
2648
|
-
if (
|
|
3184
|
+
if (getAddressStopKeywordsSync().some((kw) => {
|
|
2649
3185
|
if (!afterComma.startsWith(kw)) return false;
|
|
2650
3186
|
const next = afterComma[kw.length];
|
|
2651
3187
|
return next === void 0 || /[\s:;.,!?()\d]/.test(next);
|
|
@@ -2672,6 +3208,28 @@ const extractValue = (text, triggerEnd, strategy, label) => {
|
|
|
2672
3208
|
text: extracted
|
|
2673
3209
|
};
|
|
2674
3210
|
}
|
|
3211
|
+
case "match-pattern": {
|
|
3212
|
+
const nlIdx = valueText.indexOf("\n");
|
|
3213
|
+
const searchText = nlIdx === -1 ? valueText : valueText.slice(0, nlIdx);
|
|
3214
|
+
if (searchText.length === 0) return null;
|
|
3215
|
+
const flags = (strategy.flags ?? "").replace(/[gy]/g, "");
|
|
3216
|
+
const anchoredSource = `^(?:${strategy.pattern})`;
|
|
3217
|
+
let re;
|
|
3218
|
+
try {
|
|
3219
|
+
re = new RegExp(anchoredSource, flags);
|
|
3220
|
+
} catch {
|
|
3221
|
+
return null;
|
|
3222
|
+
}
|
|
3223
|
+
const m = re.exec(searchText);
|
|
3224
|
+
if (!m || m[0].length === 0) return null;
|
|
3225
|
+
const matchStart = m.index;
|
|
3226
|
+
const matchEnd = matchStart + m[0].length;
|
|
3227
|
+
return {
|
|
3228
|
+
start: valueStart + matchStart,
|
|
3229
|
+
end: valueStart + matchEnd,
|
|
3230
|
+
text: m[0]
|
|
3231
|
+
};
|
|
3232
|
+
}
|
|
2675
3233
|
default: return null;
|
|
2676
3234
|
}
|
|
2677
3235
|
};
|
|
@@ -2691,7 +3249,8 @@ const processTriggerMatches = (allMatches, sliceStart, sliceEnd, fullText, rules
|
|
|
2691
3249
|
if (match.start > 0 && LETTER_RE.test(fullText[match.start - 1] ?? "")) continue;
|
|
2692
3250
|
const rule = rules[localIdx];
|
|
2693
3251
|
if (!rule) continue;
|
|
2694
|
-
|
|
3252
|
+
const triggerLastChar = rule.trigger.at(-1) ?? "";
|
|
3253
|
+
if (LETTER_RE.test(triggerLastChar) && LETTER_RE.test(fullText[match.end] ?? "")) continue;
|
|
2695
3254
|
const triggerEnd = match.end;
|
|
2696
3255
|
const rawValue = extractValue(fullText, triggerEnd, rule.strategy, rule.label);
|
|
2697
3256
|
const value = rawValue ? stripQuotes(rawValue) : null;
|
|
@@ -2700,7 +3259,7 @@ const processTriggerMatches = (allMatches, sliceStart, sliceEnd, fullText, rules
|
|
|
2700
3259
|
const entityStart = rule.includeTrigger ? match.start : value.start;
|
|
2701
3260
|
const entityEnd = value.end;
|
|
2702
3261
|
const entityText = fullText.slice(entityStart, entityEnd);
|
|
2703
|
-
const effectiveLabel = rule.label === "person" &&
|
|
3262
|
+
const effectiveLabel = rule.label === "person" && getLegalFormCheckRe().test(entityText) ? "organization" : rule.label;
|
|
2704
3263
|
results.push({
|
|
2705
3264
|
start: entityStart,
|
|
2706
3265
|
end: entityEnd,
|
|
@@ -2970,11 +3529,82 @@ const resolveCountries = (regions, countries) => {
|
|
|
2970
3529
|
return result;
|
|
2971
3530
|
};
|
|
2972
3531
|
//#endregion
|
|
3532
|
+
//#region src/util/homoglyphs.ts
|
|
3533
|
+
const CHAR_MAP$1 = new Map([
|
|
3534
|
+
[1040, 65],
|
|
3535
|
+
[1042, 66],
|
|
3536
|
+
[1045, 69],
|
|
3537
|
+
[1050, 75],
|
|
3538
|
+
[1052, 77],
|
|
3539
|
+
[1053, 72],
|
|
3540
|
+
[1054, 79],
|
|
3541
|
+
[1056, 80],
|
|
3542
|
+
[1057, 67],
|
|
3543
|
+
[1058, 84],
|
|
3544
|
+
[1061, 88],
|
|
3545
|
+
[1072, 97],
|
|
3546
|
+
[1077, 101],
|
|
3547
|
+
[1086, 111],
|
|
3548
|
+
[1088, 112],
|
|
3549
|
+
[1089, 99],
|
|
3550
|
+
[1091, 121],
|
|
3551
|
+
[1093, 120],
|
|
3552
|
+
[1110, 105],
|
|
3553
|
+
[1112, 106],
|
|
3554
|
+
[913, 65],
|
|
3555
|
+
[914, 66],
|
|
3556
|
+
[917, 69],
|
|
3557
|
+
[919, 72],
|
|
3558
|
+
[921, 73],
|
|
3559
|
+
[922, 75],
|
|
3560
|
+
[924, 77],
|
|
3561
|
+
[925, 78],
|
|
3562
|
+
[927, 79],
|
|
3563
|
+
[929, 80],
|
|
3564
|
+
[932, 84],
|
|
3565
|
+
[933, 89],
|
|
3566
|
+
[935, 88],
|
|
3567
|
+
[918, 90],
|
|
3568
|
+
[945, 97],
|
|
3569
|
+
[954, 107],
|
|
3570
|
+
[959, 111],
|
|
3571
|
+
[961, 112]
|
|
3572
|
+
]);
|
|
3573
|
+
/** Chunk size for `String.fromCharCode` to avoid hitting
|
|
3574
|
+
* the call-stack limit on very large strings. */
|
|
3575
|
+
const CHUNK_SIZE$1 = 8192;
|
|
3576
|
+
const normalizeHomoglyphs = (text) => {
|
|
3577
|
+
let hasSpecial = false;
|
|
3578
|
+
for (let i = 0; i < text.length; i++) if (CHAR_MAP$1.has(text.charCodeAt(i))) {
|
|
3579
|
+
hasSpecial = true;
|
|
3580
|
+
break;
|
|
3581
|
+
}
|
|
3582
|
+
if (!hasSpecial) return text;
|
|
3583
|
+
const len = text.length;
|
|
3584
|
+
const codes = new Uint16Array(len);
|
|
3585
|
+
for (let i = 0; i < len; i++) {
|
|
3586
|
+
const code = text.charCodeAt(i);
|
|
3587
|
+
codes[i] = CHAR_MAP$1.get(code) ?? code;
|
|
3588
|
+
}
|
|
3589
|
+
if (len <= CHUNK_SIZE$1) return String.fromCharCode(...codes);
|
|
3590
|
+
let result = "";
|
|
3591
|
+
for (let offset = 0; offset < len; offset += CHUNK_SIZE$1) {
|
|
3592
|
+
const end = Math.min(offset + CHUNK_SIZE$1, len);
|
|
3593
|
+
result += String.fromCharCode(...codes.subarray(offset, end));
|
|
3594
|
+
}
|
|
3595
|
+
return result;
|
|
3596
|
+
};
|
|
3597
|
+
//#endregion
|
|
2973
3598
|
//#region src/filters/false-positives.ts
|
|
2974
3599
|
const TEMPLATE_PLACEHOLDER_RE = /^(?:\.{3,}|_{3,}|\[[\w\s]+\]|\{[\w\s]+\})$/;
|
|
2975
3600
|
const POSTAL_CODE_RE = /\d{3}\s?\d{2}/;
|
|
2976
3601
|
const HAS_DIGIT_RE = /\d/;
|
|
2977
|
-
const
|
|
3602
|
+
const ADDRESS_COMPONENT_EXTRA_RE = /(?:^|\s)(?:č\.p\.|č\.ev\.|č\.|sídliště)(?=[\s,./]|$)/i;
|
|
3603
|
+
const BARE_COURS_PROSE_RE = /(?:^|\s)cours(?!\s+\p{Lu})(?=[\s,./]|$)/u;
|
|
3604
|
+
const isOnlyAmbiguousCours = (text) => {
|
|
3605
|
+
if (!BARE_COURS_PROSE_RE.test(text)) return false;
|
|
3606
|
+
return !hasAddressComponent(text.replace(/(?:^|\s)cours(?=[\s,./]|$)/gi, " "));
|
|
3607
|
+
};
|
|
2978
3608
|
const JURISDICTION_RE = /^(?:state|commonwealth|district|territory)\s+of\s+/i;
|
|
2979
3609
|
const MAX_ENTITY_LENGTH = {
|
|
2980
3610
|
organization: 80,
|
|
@@ -3057,6 +3687,41 @@ const loadGenericRoles = (ctx = defaultContext) => {
|
|
|
3057
3687
|
};
|
|
3058
3688
|
/** Sync accessor — returns empty set before init. */
|
|
3059
3689
|
const getGenericRoles = (ctx) => ctx.genericRoles ?? EMPTY_GENERIC_ROLES;
|
|
3690
|
+
const STREET_TYPES_SEED_RE = /(?:^|\s)(?:ul\.|ulice|nám\.|náměstí|tř\.|třída|nábř\.|nábřeží|bulvár)(?=[\s,./]|$)/i;
|
|
3691
|
+
let _streetTypesRe = STREET_TYPES_SEED_RE;
|
|
3692
|
+
let _streetTypesPromise = null;
|
|
3693
|
+
const escapeRegex$1 = (s) => s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
3694
|
+
const loadStreetTypeRegex = async () => {
|
|
3695
|
+
try {
|
|
3696
|
+
const data = (await import("./address-street-types.mjs")).default ?? {};
|
|
3697
|
+
const words = /* @__PURE__ */ new Set();
|
|
3698
|
+
for (const [key, val] of Object.entries(data)) {
|
|
3699
|
+
if (key.startsWith("_")) continue;
|
|
3700
|
+
if (!Array.isArray(val)) continue;
|
|
3701
|
+
for (const w of val) if (typeof w === "string" && w.length > 0) words.add(w.toLowerCase());
|
|
3702
|
+
}
|
|
3703
|
+
if (words.size === 0) {
|
|
3704
|
+
_streetTypesRe = STREET_TYPES_SEED_RE;
|
|
3705
|
+
return;
|
|
3706
|
+
}
|
|
3707
|
+
const alternation = [...words].sort((a, b) => b.length - a.length).map(escapeRegex$1).join("|");
|
|
3708
|
+
_streetTypesRe = new RegExp(`(?:^|\\s)(?:${alternation})(?=[\\s,./]|$)`, "iu");
|
|
3709
|
+
} catch {
|
|
3710
|
+
_streetTypesRe = STREET_TYPES_SEED_RE;
|
|
3711
|
+
}
|
|
3712
|
+
};
|
|
3713
|
+
/** Ensure street-type vocabulary is loaded. */
|
|
3714
|
+
const initAddressComponents = () => {
|
|
3715
|
+
if (!_streetTypesPromise) _streetTypesPromise = loadStreetTypeRegex();
|
|
3716
|
+
return _streetTypesPromise;
|
|
3717
|
+
};
|
|
3718
|
+
/**
|
|
3719
|
+
* True when `text` contains a recognised address
|
|
3720
|
+
* component: either a per-language street word (loaded
|
|
3721
|
+
* from JSON) or a language-agnostic anchor such as a
|
|
3722
|
+
* Czech house number form.
|
|
3723
|
+
*/
|
|
3724
|
+
const hasAddressComponent = (text) => _streetTypesRe.test(text) || ADDRESS_COMPONENT_EXTRA_RE.test(text);
|
|
3060
3725
|
const isCallerOwnedEntity$3 = (entity) => entity.sourceDetail === "custom-deny-list" || entity.sourceDetail === "custom-regex";
|
|
3061
3726
|
/**
|
|
3062
3727
|
* Filter out entities that are likely false positives:
|
|
@@ -3091,13 +3756,15 @@ const filterFalsePositives = (entities, ctx = defaultContext, fullText) => {
|
|
|
3091
3756
|
if (normalized.label === "person" && HAS_DIGIT_RE.test(trimmed)) continue;
|
|
3092
3757
|
if (normalized.label === "person") {
|
|
3093
3758
|
const tokens = trimmed.split(/\s+/u);
|
|
3094
|
-
const last = tokens.at(-1)?.replace(/[.,;:!?]+$/u, "")
|
|
3095
|
-
|
|
3759
|
+
const last = tokens.at(-1)?.replace(/[.,;:!?]+$/u, "");
|
|
3760
|
+
const lastFolded = last ? normalizeHomoglyphs(last).toLowerCase() : void 0;
|
|
3761
|
+
if (tokens.length > 1 && lastFolded && PERSON_TRAILING_NOUNS.has(lastFolded)) continue;
|
|
3096
3762
|
}
|
|
3097
|
-
if ((normalized.label === "person" || normalized.label === "organization") && roles.has(trimmed.toLowerCase())) continue;
|
|
3763
|
+
if ((normalized.label === "person" || normalized.label === "organization") && roles.has(normalizeHomoglyphs(trimmed).toLowerCase())) continue;
|
|
3098
3764
|
if (normalized.label === "organization" && normalized.source === "legal-form" && trimmed === trimmed.toUpperCase() && LEGAL_FORM_HEADING_RE.test(trimmed)) continue;
|
|
3099
|
-
if (normalized.label === "address" && trimmed.length > 40 && !POSTAL_CODE_RE.test(trimmed) && !HAS_DIGIT_RE.test(trimmed) && !
|
|
3100
|
-
if (normalized.label === "address" && normalized.source === "trigger" && !HAS_DIGIT_RE.test(trimmed) && !
|
|
3765
|
+
if (normalized.label === "address" && trimmed.length > 40 && !POSTAL_CODE_RE.test(trimmed) && !HAS_DIGIT_RE.test(trimmed) && !hasAddressComponent(trimmed) && !JURISDICTION_RE.test(trimmed)) continue;
|
|
3766
|
+
if (normalized.label === "address" && normalized.source === "trigger" && !HAS_DIGIT_RE.test(trimmed) && !hasAddressComponent(trimmed) && !JURISDICTION_RE.test(trimmed)) continue;
|
|
3767
|
+
if (normalized.label === "address" && normalized.source === "trigger" && !HAS_DIGIT_RE.test(trimmed) && isOnlyAmbiguousCours(trimmed)) continue;
|
|
3101
3768
|
if (normalized.label === "address" && SIGNING_CLAUSE_ADDRESS_RE.test(trimmed)) continue;
|
|
3102
3769
|
filtered.push(normalized);
|
|
3103
3770
|
}
|
|
@@ -3254,6 +3921,122 @@ const loadPersonStopwords = (ctx) => {
|
|
|
3254
3921
|
const EMPTY_PERSON_STOPWORDS = /* @__PURE__ */ new Set();
|
|
3255
3922
|
/** Sync accessor — returns empty set before init. */
|
|
3256
3923
|
const getPersonStopwords = (ctx) => ctx.personStopwords ?? EMPTY_PERSON_STOPWORDS;
|
|
3924
|
+
const loadAddressStopwords = (ctx) => {
|
|
3925
|
+
if (ctx.addressStopwordsPromise) return ctx.addressStopwordsPromise;
|
|
3926
|
+
ctx.addressStopwordsPromise = (async () => {
|
|
3927
|
+
try {
|
|
3928
|
+
const mod = await import("./address-stopwords.mjs");
|
|
3929
|
+
const set = new Set(mod.default?.words ?? []);
|
|
3930
|
+
ctx.addressStopwords = set;
|
|
3931
|
+
return set;
|
|
3932
|
+
} catch {
|
|
3933
|
+
const empty = /* @__PURE__ */ new Set();
|
|
3934
|
+
ctx.addressStopwords = empty;
|
|
3935
|
+
return empty;
|
|
3936
|
+
}
|
|
3937
|
+
})();
|
|
3938
|
+
return ctx.addressStopwordsPromise;
|
|
3939
|
+
};
|
|
3940
|
+
const EMPTY_ADDRESS_STOPWORDS = /* @__PURE__ */ new Set();
|
|
3941
|
+
const getAddressStopwords = (ctx) => ctx.addressStopwords ?? EMPTY_ADDRESS_STOPWORDS;
|
|
3942
|
+
/**
|
|
3943
|
+
* Word characters in unicode property notation. The check is
|
|
3944
|
+
* "single-token" — no internal whitespace — and we keep dashes
|
|
3945
|
+
* tokens like "K-12" out by requiring the surface to be a single
|
|
3946
|
+
* uninterrupted run of letters or marks.
|
|
3947
|
+
*/
|
|
3948
|
+
const SINGLE_WORD_RE = /^\p{L}+$/u;
|
|
3949
|
+
/**
|
|
3950
|
+
* Format-level address signals — structurally numeric or
|
|
3951
|
+
* 2-letter-state patterns, language-agnostic. The street-type
|
|
3952
|
+
* vocabulary is loaded from `address-street-types.json` so new
|
|
3953
|
+
* languages contribute via data, not code.
|
|
3954
|
+
* - `,\s*[A-Z]{2}\b` → US state abbreviation after a comma
|
|
3955
|
+
* - `\b\d{5}(?:-\d{4})?\b` → US ZIP / ZIP+4
|
|
3956
|
+
* - `\b\d{3}\s\d{2}\b` → Czech/Slovak postal block (140 00)
|
|
3957
|
+
* - `\b\d{2}-\d{3}\b` → Polish postal code (00-950)
|
|
3958
|
+
*/
|
|
3959
|
+
const ADDRESS_FORMAT_RE = /,\s*\p{Lu}{2}\b|\b\d{5}(?:-\d{4})?\b|\b\d{3}\s\d{2}\b|\b\d{2}-\d{3}\b/u;
|
|
3960
|
+
let cachedStreetTypeRe = null;
|
|
3961
|
+
let streetTypeReLoaded = false;
|
|
3962
|
+
const loadStreetTypeRe = async () => {
|
|
3963
|
+
if (streetTypeReLoaded) return cachedStreetTypeRe;
|
|
3964
|
+
try {
|
|
3965
|
+
const config = (await import("./address-street-types.mjs")).default ?? {};
|
|
3966
|
+
const words = [];
|
|
3967
|
+
for (const value of Object.values(config)) {
|
|
3968
|
+
if (!Array.isArray(value)) continue;
|
|
3969
|
+
for (const word of value) if (typeof word === "string" && word.length > 0) words.push(word);
|
|
3970
|
+
}
|
|
3971
|
+
if (words.length === 0) cachedStreetTypeRe = null;
|
|
3972
|
+
else {
|
|
3973
|
+
words.sort((a, b) => b.length - a.length);
|
|
3974
|
+
const isLetterDigit = (c) => /[\p{L}\p{N}]/u.test(c);
|
|
3975
|
+
const wordLikeTail = [];
|
|
3976
|
+
const punctTail = [];
|
|
3977
|
+
for (const w of words) {
|
|
3978
|
+
const escaped = w.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
3979
|
+
if (isLetterDigit(w.at(-1) ?? "")) wordLikeTail.push(escaped);
|
|
3980
|
+
else punctTail.push(escaped);
|
|
3981
|
+
}
|
|
3982
|
+
const branches = [];
|
|
3983
|
+
if (wordLikeTail.length > 0) branches.push(`(?:${wordLikeTail.join("|")})(?![\\p{L}\\p{N}])`);
|
|
3984
|
+
if (punctTail.length > 0) branches.push(`(?:${punctTail.join("|")})`);
|
|
3985
|
+
cachedStreetTypeRe = new RegExp(`(?<![\\p{L}\\p{N}])(?:${branches.join("|")})`, "iu");
|
|
3986
|
+
}
|
|
3987
|
+
} catch {
|
|
3988
|
+
cachedStreetTypeRe = null;
|
|
3989
|
+
}
|
|
3990
|
+
streetTypeReLoaded = true;
|
|
3991
|
+
return cachedStreetTypeRe;
|
|
3992
|
+
};
|
|
3993
|
+
const getStreetTypeRe = () => streetTypeReLoaded ? cachedStreetTypeRe : null;
|
|
3994
|
+
const hasAdjacentAddressEvidence = (fullText, start, end) => {
|
|
3995
|
+
const window = fullText.slice(Math.max(0, start - 40), Math.min(fullText.length, end + 40));
|
|
3996
|
+
if (ADDRESS_FORMAT_RE.test(window)) return true;
|
|
3997
|
+
const streetRe = getStreetTypeRe();
|
|
3998
|
+
return streetRe !== null && streetRe.test(window);
|
|
3999
|
+
};
|
|
4000
|
+
/**
|
|
4001
|
+
* Capitalised words that almost never start a person name. When a
|
|
4002
|
+
* single-token surname candidate is immediately followed by one of
|
|
4003
|
+
* these, the "next-word is uppercase" promotion heuristic would
|
|
4004
|
+
* otherwise turn section headings ("Purchase Price↵The Purchaser
|
|
4005
|
+
* undertakes…") into spurious person hits. Kept narrow on purpose;
|
|
4006
|
+
* the surrounding pipeline still chains real names via the deny-list
|
|
4007
|
+
* cascade when both halves are surnames.
|
|
4008
|
+
*/
|
|
4009
|
+
const SENTENCE_STARTER_WORDS = new Set([
|
|
4010
|
+
"The",
|
|
4011
|
+
"This",
|
|
4012
|
+
"These",
|
|
4013
|
+
"Those",
|
|
4014
|
+
"An",
|
|
4015
|
+
"Any",
|
|
4016
|
+
"All",
|
|
4017
|
+
"Each",
|
|
4018
|
+
"Every",
|
|
4019
|
+
"No",
|
|
4020
|
+
"Now",
|
|
4021
|
+
"Whereas",
|
|
4022
|
+
"Whereby",
|
|
4023
|
+
"Wherein",
|
|
4024
|
+
"Whereof",
|
|
4025
|
+
"Notwithstanding",
|
|
4026
|
+
"Subject",
|
|
4027
|
+
"In",
|
|
4028
|
+
"On",
|
|
4029
|
+
"At",
|
|
4030
|
+
"By",
|
|
4031
|
+
"For",
|
|
4032
|
+
"If",
|
|
4033
|
+
"Upon",
|
|
4034
|
+
"Unless",
|
|
4035
|
+
"Until",
|
|
4036
|
+
"Provided",
|
|
4037
|
+
"Pursuant",
|
|
4038
|
+
"Such"
|
|
4039
|
+
]);
|
|
3257
4040
|
const PERSON_CHAIN_BREAK_RE = /[!?;:]|,/u;
|
|
3258
4041
|
const WORD_CHAR_RE$1 = /[\p{L}\p{N}]/u;
|
|
3259
4042
|
const isInitialContinuationGap = (text, gap) => /^\p{Lu}$/u.test(text) && /^\.[^\S\n]{1,2}$/u.test(gap) || /^[^\S\n]{1,2}(?:\p{Lu}\.[^\S\n]{1,2})+$/u.test(gap);
|
|
@@ -3273,6 +4056,8 @@ const buildDenyList = async (config, ctx = defaultContext) => {
|
|
|
3273
4056
|
loadStopwords(ctx),
|
|
3274
4057
|
loadAllowList(ctx),
|
|
3275
4058
|
loadPersonStopwords(ctx),
|
|
4059
|
+
loadAddressStopwords(ctx),
|
|
4060
|
+
loadStreetTypeRe(),
|
|
3276
4061
|
loadGenericRoles(ctx)
|
|
3277
4062
|
]);
|
|
3278
4063
|
const dictionaries = config.dictionaries;
|
|
@@ -3419,6 +4204,8 @@ const ensureDenyListData = async (ctx = defaultContext, dictionaries) => {
|
|
|
3419
4204
|
loadStopwords(ctx),
|
|
3420
4205
|
loadAllowList(ctx),
|
|
3421
4206
|
loadPersonStopwords(ctx),
|
|
4207
|
+
loadAddressStopwords(ctx),
|
|
4208
|
+
loadStreetTypeRe(),
|
|
3422
4209
|
loadGenericRoles(ctx)
|
|
3423
4210
|
]);
|
|
3424
4211
|
};
|
|
@@ -3486,14 +4273,18 @@ const processDenyListMatches = (allMatches, sliceStart, sliceEnd, fullText, data
|
|
|
3486
4273
|
if (!getPersonStopwords(ctx).has(keyword)) nameHits.push(m);
|
|
3487
4274
|
}
|
|
3488
4275
|
const nonPersonLabels = m.labels.filter((l) => l !== "person");
|
|
3489
|
-
|
|
3490
|
-
|
|
3491
|
-
|
|
3492
|
-
|
|
3493
|
-
|
|
3494
|
-
|
|
3495
|
-
|
|
3496
|
-
|
|
4276
|
+
const suppressAddress = SINGLE_WORD_RE.test(m.text) && getAddressStopwords(ctx).has(m.text.toLowerCase()) && !hasAdjacentAddressEvidence(fullText, m.start, m.end);
|
|
4277
|
+
for (const label of nonPersonLabels) {
|
|
4278
|
+
if (label === "address" && suppressAddress) continue;
|
|
4279
|
+
results.push({
|
|
4280
|
+
start: m.start,
|
|
4281
|
+
end: m.end,
|
|
4282
|
+
label,
|
|
4283
|
+
text: m.text,
|
|
4284
|
+
score: .9,
|
|
4285
|
+
source: DETECTION_SOURCES.DENY_LIST
|
|
4286
|
+
});
|
|
4287
|
+
}
|
|
3497
4288
|
}
|
|
3498
4289
|
}
|
|
3499
4290
|
nameHits.sort((a, b) => a.start - b.start);
|
|
@@ -3525,6 +4316,8 @@ const processDenyListMatches = (allMatches, sliceStart, sliceEnd, fullText, data
|
|
|
3525
4316
|
const afterEnd = last.end;
|
|
3526
4317
|
const rest = fullText.slice(afterEnd).trimStart();
|
|
3527
4318
|
if (!(rest.length > 1 && /^\p{Lu}\p{Ll}/u.test(rest))) continue;
|
|
4319
|
+
const nextWord = /^\p{L}+/u.exec(rest)?.[0] ?? "";
|
|
4320
|
+
if (SENTENCE_STARTER_WORDS.has(nextWord)) continue;
|
|
3528
4321
|
}
|
|
3529
4322
|
results.push({
|
|
3530
4323
|
start: first.start,
|
|
@@ -3636,6 +4429,15 @@ const extendPersonName = (text, start, end, ctx) => {
|
|
|
3636
4429
|
};
|
|
3637
4430
|
//#endregion
|
|
3638
4431
|
//#region src/detectors/address-seeds.ts
|
|
4432
|
+
/**
|
|
4433
|
+
* Trailing chars that should never end an address span. Combines
|
|
4434
|
+
* structural separators, opening brackets, and typographic quote
|
|
4435
|
+
* variants so a span like `… GA 30326, USA (the "Premises")`
|
|
4436
|
+
* does not pull the parenthetical opener into the address.
|
|
4437
|
+
* The prime (`′`) catches measurement tails (`5′`) that the
|
|
4438
|
+
* cluster occasionally absorbs.
|
|
4439
|
+
*/
|
|
4440
|
+
const ADDRESS_TRAILING_TRIM_RE = new RegExp(`[,;:\\s${OPENING_BRACKETS_INNER}${QUOTE_DOUBLE_INNER}${QUOTE_SINGLE_INNER}′]`, "u");
|
|
3639
4441
|
let cachedBoundaryRe = null;
|
|
3640
4442
|
const loadBoundaryWords = async () => {
|
|
3641
4443
|
try {
|
|
@@ -3644,6 +4446,56 @@ const loadBoundaryWords = async () => {
|
|
|
3644
4446
|
return {};
|
|
3645
4447
|
}
|
|
3646
4448
|
};
|
|
4449
|
+
const BR_CEP_CONTEXT_WINDOW = 200;
|
|
4450
|
+
let cachedBrCepContextRe = null;
|
|
4451
|
+
let cachedBrCepContextPromise = null;
|
|
4452
|
+
const loadBrCueWords = async () => {
|
|
4453
|
+
const sources = await Promise.all([(async () => {
|
|
4454
|
+
try {
|
|
4455
|
+
return (await import("./address-street-types.mjs")).default["pt-br"];
|
|
4456
|
+
} catch {
|
|
4457
|
+
return;
|
|
4458
|
+
}
|
|
4459
|
+
})(), (async () => {
|
|
4460
|
+
try {
|
|
4461
|
+
return (await import("./address-boundaries.mjs")).default["pt-br"];
|
|
4462
|
+
} catch {
|
|
4463
|
+
return;
|
|
4464
|
+
}
|
|
4465
|
+
})()]);
|
|
4466
|
+
const out = [];
|
|
4467
|
+
const seen = /* @__PURE__ */ new Set();
|
|
4468
|
+
for (const entry of sources) {
|
|
4469
|
+
if (!Array.isArray(entry)) continue;
|
|
4470
|
+
for (const word of entry) {
|
|
4471
|
+
if (typeof word !== "string" || word.length === 0) continue;
|
|
4472
|
+
const key = word.toLowerCase();
|
|
4473
|
+
if (seen.has(key)) continue;
|
|
4474
|
+
seen.add(key);
|
|
4475
|
+
out.push(word);
|
|
4476
|
+
}
|
|
4477
|
+
}
|
|
4478
|
+
return out;
|
|
4479
|
+
};
|
|
4480
|
+
const getBrCepContextRe = async () => {
|
|
4481
|
+
if (cachedBrCepContextRe !== null) return cachedBrCepContextRe;
|
|
4482
|
+
if (cachedBrCepContextPromise) return cachedBrCepContextPromise;
|
|
4483
|
+
cachedBrCepContextPromise = (async () => {
|
|
4484
|
+
const words = await loadBrCueWords();
|
|
4485
|
+
if (words.length === 0) return null;
|
|
4486
|
+
const escaped = words.toSorted((a, b) => b.length - a.length).map((w) => w.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"));
|
|
4487
|
+
const re = new RegExp(`(?<![\\p{L}\\p{N}])(?:${escaped.join("|")})(?![\\p{L}\\p{N}])`, "iu");
|
|
4488
|
+
cachedBrCepContextRe = re;
|
|
4489
|
+
return re;
|
|
4490
|
+
})();
|
|
4491
|
+
return cachedBrCepContextPromise;
|
|
4492
|
+
};
|
|
4493
|
+
const hasBrCueNearby = (fullText, start, end, re) => {
|
|
4494
|
+
const windowStart = Math.max(0, start - BR_CEP_CONTEXT_WINDOW);
|
|
4495
|
+
const windowEnd = Math.min(fullText.length, end + BR_CEP_CONTEXT_WINDOW);
|
|
4496
|
+
const window = fullText.slice(windowStart, windowEnd);
|
|
4497
|
+
return new RegExp(re.source, re.flags.replace("g", "")).test(window);
|
|
4498
|
+
};
|
|
3647
4499
|
/**
|
|
3648
4500
|
* Build regex for boundary words. Matches any
|
|
3649
4501
|
* boundary word preceded by a word boundary.
|
|
@@ -3657,7 +4509,7 @@ const getBoundaryRe = async () => {
|
|
|
3657
4509
|
for (const word of entries) words.push(word.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"));
|
|
3658
4510
|
}
|
|
3659
4511
|
words.sort((a, b) => b.length - a.length);
|
|
3660
|
-
cachedBoundaryRe = words.length > 0 ? new RegExp(
|
|
4512
|
+
cachedBoundaryRe = words.length > 0 ? new RegExp(`(?<![\\p{L}\\p{N}])(?:${words.join("|")})(?![\\p{L}\\p{N}])`, "iu") : /(?!)/;
|
|
3661
4513
|
return cachedBoundaryRe;
|
|
3662
4514
|
};
|
|
3663
4515
|
/**
|
|
@@ -3679,7 +4531,7 @@ const buildStreetTypePatterns = async () => {
|
|
|
3679
4531
|
}
|
|
3680
4532
|
return words;
|
|
3681
4533
|
};
|
|
3682
|
-
const collectSeeds = (allMatches, sliceStart, sliceEnd, fullText, existingEntities) => {
|
|
4534
|
+
const collectSeeds = (allMatches, sliceStart, sliceEnd, fullText, existingEntities, brCepContextRe) => {
|
|
3683
4535
|
const seeds = [];
|
|
3684
4536
|
for (const match of allMatches) {
|
|
3685
4537
|
const idx = match.pattern;
|
|
@@ -3713,18 +4565,41 @@ const collectSeeds = (allMatches, sliceStart, sliceEnd, fullText, existingEntiti
|
|
|
3713
4565
|
text: e.text
|
|
3714
4566
|
});
|
|
3715
4567
|
}
|
|
3716
|
-
const postalRe = /\b(?:\d{3}\s\d{2}|\d{2}-\d{3})\b/g;
|
|
4568
|
+
const postalRe = /\b(?:\d{3}\s\d{2}|\d{2}-\d{3}|\d{5}-\d{3})\b/g;
|
|
3717
4569
|
let postalMatch;
|
|
3718
4570
|
while ((postalMatch = postalRe.exec(fullText)) !== null) {
|
|
3719
4571
|
const start = postalMatch.index;
|
|
3720
4572
|
const end = start + postalMatch[0].length;
|
|
3721
|
-
if (
|
|
4573
|
+
if (seeds.some((s) => s.start <= start && s.end >= end)) continue;
|
|
4574
|
+
if (/^\d{5}-\d{3}$/.test(postalMatch[0]) && (brCepContextRe === null || !hasBrCueNearby(fullText, start, end, brCepContextRe))) continue;
|
|
4575
|
+
seeds.push({
|
|
3722
4576
|
type: "postal-code",
|
|
3723
4577
|
start,
|
|
3724
4578
|
end,
|
|
3725
4579
|
text: postalMatch[0]
|
|
3726
4580
|
});
|
|
3727
4581
|
}
|
|
4582
|
+
const itCapRe = /\b\d{5}(?=\s+\p{Lu}\p{L}+)/gu;
|
|
4583
|
+
let itCapMatch;
|
|
4584
|
+
while ((itCapMatch = itCapRe.exec(fullText)) !== null) {
|
|
4585
|
+
const start = itCapMatch.index;
|
|
4586
|
+
const end = start + itCapMatch[0].length;
|
|
4587
|
+
if (seeds.some((s) => s.start <= start && s.end >= end)) continue;
|
|
4588
|
+
if (!seeds.some((s) => {
|
|
4589
|
+
if (Math.abs(s.start - start) > 80) return false;
|
|
4590
|
+
if (s.type === "address-trigger") return true;
|
|
4591
|
+
if (s.type === "city") return true;
|
|
4592
|
+
if (s.type === "postal-code") return true;
|
|
4593
|
+
if (s.type === "street-word") return s.text.toLowerCase() !== "via";
|
|
4594
|
+
return false;
|
|
4595
|
+
})) continue;
|
|
4596
|
+
seeds.push({
|
|
4597
|
+
type: "postal-code",
|
|
4598
|
+
start,
|
|
4599
|
+
end,
|
|
4600
|
+
text: itCapMatch[0]
|
|
4601
|
+
});
|
|
4602
|
+
}
|
|
3728
4603
|
const streetNumRe = /\b(\p{Lu}\p{Ll}{2,})\s+(\d{1,5}(?:\/\d{1,5})?)\s*[,\n]/gu;
|
|
3729
4604
|
let streetMatch;
|
|
3730
4605
|
while ((streetMatch = streetNumRe.exec(fullText)) !== null) {
|
|
@@ -3782,6 +4657,10 @@ const scoreCluster = (cluster) => {
|
|
|
3782
4657
|
const NON_ADDRESS_LABELS = new Set([
|
|
3783
4658
|
"registration number",
|
|
3784
4659
|
"tax identification number",
|
|
4660
|
+
"national identification number",
|
|
4661
|
+
"social security number",
|
|
4662
|
+
"birth number",
|
|
4663
|
+
"identity card number",
|
|
3785
4664
|
"person",
|
|
3786
4665
|
"bank account number",
|
|
3787
4666
|
"email address",
|
|
@@ -3818,7 +4697,7 @@ const expandCluster = async (fullText, cluster, existingEntities) => {
|
|
|
3818
4697
|
const doubleNewline = remaining.indexOf("\n\n");
|
|
3819
4698
|
if (doubleNewline !== -1 && doubleNewline < nearestBoundary) nearestBoundary = doubleNewline;
|
|
3820
4699
|
rightPos = end + remaining.slice(0, nearestBoundary).trimEnd().length;
|
|
3821
|
-
while (rightPos > end &&
|
|
4700
|
+
while (rightPos > end && ADDRESS_TRAILING_TRIM_RE.test(fullText[rightPos - 1] ?? "")) rightPos--;
|
|
3822
4701
|
return {
|
|
3823
4702
|
start: Math.min(leftPos, start),
|
|
3824
4703
|
end: Math.max(rightPos, end)
|
|
@@ -3835,7 +4714,7 @@ const expandCluster = async (fullText, cluster, existingEntities) => {
|
|
|
3835
4714
|
* using their output as seed sources.
|
|
3836
4715
|
*/
|
|
3837
4716
|
const processAddressSeeds = async (allMatches, sliceStart, sliceEnd, fullText, existingEntities) => {
|
|
3838
|
-
const clusters = clusterSeeds(collectSeeds(allMatches, sliceStart, sliceEnd, fullText, existingEntities), 150);
|
|
4717
|
+
const clusters = clusterSeeds(collectSeeds(allMatches, sliceStart, sliceEnd, fullText, existingEntities, await getBrCepContextRe()), 150);
|
|
3839
4718
|
const results = [];
|
|
3840
4719
|
for (const cluster of clusters) {
|
|
3841
4720
|
const score = scoreCluster(cluster);
|
|
@@ -3860,6 +4739,7 @@ const processAddressSeeds = async (allMatches, sliceStart, sliceEnd, fullText, e
|
|
|
3860
4739
|
const TRAILING_SEP = /[,\s]+$/;
|
|
3861
4740
|
const WORD_CHAR_RE = /[\p{L}\p{N}]/u;
|
|
3862
4741
|
const ORG_PROPAGATION_SCORE = .9;
|
|
4742
|
+
const ORG_DETERMINER_RE = /(?<![\p{L}\p{N}])(společnost(?:i|í|em|u)?|spolecnost(?:i|em|u)?|the\s+(?:company|corporation|firm)|die\s+(?:gesellschaft|firma)|la\s+(?:société|empresa|sociedad)|el\s+(?:empresa|sociedad))\s+$/iu;
|
|
3863
4743
|
const isCallerOwnedEntity$2 = (entity) => entity.sourceDetail === "custom-deny-list" || entity.sourceDetail === "custom-regex";
|
|
3864
4744
|
/**
|
|
3865
4745
|
* After the main detection pass, collect organization
|
|
@@ -3903,16 +4783,25 @@ const propagateOrgNames = (entities, fullText) => {
|
|
|
3903
4783
|
searchFrom = idx + 1;
|
|
3904
4784
|
continue;
|
|
3905
4785
|
}
|
|
3906
|
-
|
|
4786
|
+
let spanStart = idx;
|
|
4787
|
+
const lookbackStart = Math.max(0, idx - 40);
|
|
4788
|
+
const lookback = fullText.slice(lookbackStart, idx);
|
|
4789
|
+
const determinerMatch = ORG_DETERMINER_RE.exec(lookback);
|
|
4790
|
+
if (determinerMatch !== null) {
|
|
4791
|
+
const determiner = determinerMatch[1] ?? "";
|
|
4792
|
+
const offsetInMatch = determinerMatch[0].indexOf(determiner);
|
|
4793
|
+
spanStart = lookbackStart + determinerMatch.index + offsetInMatch;
|
|
4794
|
+
}
|
|
4795
|
+
if (!isOverlapping(spanStart, matchEnd)) {
|
|
3907
4796
|
results.push({
|
|
3908
|
-
start:
|
|
4797
|
+
start: spanStart,
|
|
3909
4798
|
end: matchEnd,
|
|
3910
4799
|
label,
|
|
3911
|
-
text:
|
|
4800
|
+
text: fullText.slice(spanStart, matchEnd),
|
|
3912
4801
|
score: ORG_PROPAGATION_SCORE,
|
|
3913
4802
|
source: DETECTION_SOURCES.COREFERENCE
|
|
3914
4803
|
});
|
|
3915
|
-
covered.push([
|
|
4804
|
+
covered.push([spanStart, matchEnd]);
|
|
3916
4805
|
}
|
|
3917
4806
|
searchFrom = matchEnd;
|
|
3918
4807
|
}
|
|
@@ -5077,13 +5966,89 @@ const shouldReplace = (a, b) => {
|
|
|
5077
5966
|
};
|
|
5078
5967
|
/** Labels where colons are structurally significant. */
|
|
5079
5968
|
const COLON_LABELS = new Set(["ip address", "mac address"]);
|
|
5969
|
+
/**
|
|
5970
|
+
* Labels whose detectors emit precise, evidence-backed spans. When
|
|
5971
|
+
* one of these fires at the exact same offsets as a fuzzier
|
|
5972
|
+
* `address` hit (city dictionary lookup, address-seed cluster), the
|
|
5973
|
+
* `address` label is almost always a dictionary collision — "March
|
|
5974
|
+
* 15" the date getting tagged as an address because "March" appears
|
|
5975
|
+
* in a city corpus, or "Brent Phillips" emitted as both `person`
|
|
5976
|
+
* and `address` because "Brent" is a UK city. The precise detector
|
|
5977
|
+
* wins.
|
|
5978
|
+
*/
|
|
5979
|
+
const PRECISE_OVER_ADDRESS = new Set([
|
|
5980
|
+
"person",
|
|
5981
|
+
"date",
|
|
5982
|
+
"date of birth",
|
|
5983
|
+
"phone number",
|
|
5984
|
+
"email address",
|
|
5985
|
+
"monetary amount",
|
|
5986
|
+
"iban",
|
|
5987
|
+
"bank account number",
|
|
5988
|
+
"tax identification number",
|
|
5989
|
+
"registration number",
|
|
5990
|
+
"identity card number",
|
|
5991
|
+
"national identification number",
|
|
5992
|
+
"passport number",
|
|
5993
|
+
"credit card number"
|
|
5994
|
+
]);
|
|
5995
|
+
/**
|
|
5996
|
+
* Labels the `person` chain wins against at identical offsets. The
|
|
5997
|
+
* chain carries adjacent-name evidence (deny-list surname plus a
|
|
5998
|
+
* capitalised follow-up) a single-token dictionary collision does
|
|
5999
|
+
* not. Kept narrow: organizations are NOT here — "Morgan Stanley"
|
|
6000
|
+
* legitimately appears in both the org and name dictionaries, and
|
|
6001
|
+
* the existing detector priority is the right tie-breaker there.
|
|
6002
|
+
*/
|
|
6003
|
+
const PERSON_PREFERRED_OVER = new Set(["address", "land parcel"]);
|
|
6004
|
+
const resolveSameSpanLabelConflicts = (entities) => {
|
|
6005
|
+
if (entities.length < 2) return entities;
|
|
6006
|
+
const byOffsets = /* @__PURE__ */ new Map();
|
|
6007
|
+
for (const entity of entities) {
|
|
6008
|
+
const key = `${entity.start}:${entity.end}`;
|
|
6009
|
+
const list = byOffsets.get(key);
|
|
6010
|
+
if (list) list.push(entity);
|
|
6011
|
+
else byOffsets.set(key, [entity]);
|
|
6012
|
+
}
|
|
6013
|
+
const dropped = /* @__PURE__ */ new Set();
|
|
6014
|
+
for (const [, group] of byOffsets) {
|
|
6015
|
+
if (group.length < 2) continue;
|
|
6016
|
+
const labels = new Set(group.map((e) => e.label));
|
|
6017
|
+
if (labels.size < 2) continue;
|
|
6018
|
+
const hasPerson = labels.has("person");
|
|
6019
|
+
const hasPreciseNonAddress = [...labels].some((l) => l !== "address" && PRECISE_OVER_ADDRESS.has(l));
|
|
6020
|
+
let maxPriority = -1;
|
|
6021
|
+
for (const e of group) {
|
|
6022
|
+
if (hasLockedBoundary(e)) continue;
|
|
6023
|
+
const pri = DETECTOR_PRIORITY[e.source] ?? 0;
|
|
6024
|
+
if (pri > maxPriority) maxPriority = pri;
|
|
6025
|
+
}
|
|
6026
|
+
for (const e of group) {
|
|
6027
|
+
if (hasLockedBoundary(e)) continue;
|
|
6028
|
+
if ((DETECTOR_PRIORITY[e.source] ?? 0) < maxPriority) {
|
|
6029
|
+
dropped.add(e);
|
|
6030
|
+
continue;
|
|
6031
|
+
}
|
|
6032
|
+
if (hasPerson && PERSON_PREFERRED_OVER.has(e.label)) {
|
|
6033
|
+
dropped.add(e);
|
|
6034
|
+
continue;
|
|
6035
|
+
}
|
|
6036
|
+
if (hasPreciseNonAddress && e.label === "address") dropped.add(e);
|
|
6037
|
+
}
|
|
6038
|
+
}
|
|
6039
|
+
if (dropped.size === 0) return entities;
|
|
6040
|
+
return entities.filter((e) => !dropped.has(e));
|
|
6041
|
+
};
|
|
5080
6042
|
/** Strip leading/trailing whitespace and punctuation. */
|
|
5081
6043
|
const sanitizeEntities = (entities) => entities.flatMap((e) => {
|
|
5082
6044
|
if (hasLockedBoundary(e)) return [e];
|
|
5083
6045
|
const strip = COLON_LABELS.has(e.label) ? /[\s,;]+/ : /[\s:,;]+/;
|
|
5084
6046
|
const leadTrimmed = e.text.replace(/^(?:\.\s)+/, "").replace(new RegExp(`^${strip.source}`, strip.flags), "");
|
|
5085
6047
|
const lead = e.text.length - leadTrimmed.length;
|
|
5086
|
-
|
|
6048
|
+
let cleaned = leadTrimmed.replace(new RegExp(`${strip.source}$`, strip.flags), "");
|
|
6049
|
+
if (e.label === "organization" && cleaned.endsWith(".") && !LITERAL_SOURCES.has(e.source)) {
|
|
6050
|
+
if (!getKnownLegalSuffixes().some((suffix) => cleaned.endsWith(suffix))) cleaned = cleaned.slice(0, -1).trimEnd();
|
|
6051
|
+
}
|
|
5087
6052
|
if (cleaned.length === 0) return [];
|
|
5088
6053
|
if (!/[\p{L}\p{N}]/u.test(cleaned)) return [];
|
|
5089
6054
|
const collapsed = cleaned.replace(/\s*\n\s*/g, " ").replace(/\s{2,}/g, " ");
|
|
@@ -5132,7 +6097,7 @@ const mergeAndDedup = (...layers) => {
|
|
|
5132
6097
|
}
|
|
5133
6098
|
if (overlaps.every((existing) => shouldReplace(entity, existing))) merged.splice(overlapStart, overlaps.length, { ...entity });
|
|
5134
6099
|
}
|
|
5135
|
-
return sanitizeEntities(merged);
|
|
6100
|
+
return resolveSameSpanLabelConflicts(sanitizeEntities(merged));
|
|
5136
6101
|
};
|
|
5137
6102
|
const escapeRegex = (s) => s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
5138
6103
|
let amountWordsRe = null;
|
|
@@ -5168,9 +6133,10 @@ const filterAllowedLabels = (entities, allowedLabels) => {
|
|
|
5168
6133
|
return entities.filter((e) => allowedLabels.has(e.label));
|
|
5169
6134
|
};
|
|
5170
6135
|
const labelIsAllowed = (label, allowedLabels) => !allowedLabels || allowedLabels.has(label);
|
|
6136
|
+
const NON_NER_LABELS = new Set(["misc"]);
|
|
5171
6137
|
const getRequestedNerLabels = (config, expandForHotwords = false) => {
|
|
5172
6138
|
const labels = config.labels.length > 0 ? config.labels : DEFAULT_ENTITY_LABELS;
|
|
5173
|
-
return expandForHotwords ? expandLabelsForHotwordRules(labels) : labels;
|
|
6139
|
+
return (expandForHotwords ? expandLabelsForHotwordRules(labels) : labels).filter((label) => !NON_NER_LABELS.has(label));
|
|
5174
6140
|
};
|
|
5175
6141
|
const checkAbort = (signal) => {
|
|
5176
6142
|
if (signal?.aborted) throw new DOMException("Pipeline aborted", "AbortError");
|
|
@@ -5251,6 +6217,8 @@ const runPipeline = async (options) => {
|
|
|
5251
6217
|
loadGenericRoles(ctx),
|
|
5252
6218
|
initPrepositions(),
|
|
5253
6219
|
initStreetAbbrevs(),
|
|
6220
|
+
initAddressComponents(),
|
|
6221
|
+
warmAddressStopKeywords(),
|
|
5254
6222
|
zoneInit,
|
|
5255
6223
|
hotwordInit
|
|
5256
6224
|
]);
|
|
@@ -5258,6 +6226,8 @@ const runPipeline = async (options) => {
|
|
|
5258
6226
|
loadGenericRoles(ctx),
|
|
5259
6227
|
initPrepositions(),
|
|
5260
6228
|
initStreetAbbrevs(),
|
|
6229
|
+
initAddressComponents(),
|
|
6230
|
+
warmAddressStopKeywords(),
|
|
5261
6231
|
hotwordInit
|
|
5262
6232
|
]);
|
|
5263
6233
|
if (cachedSearch && config.enableDenyList) await ensureDenyListData(ctx, config.dictionaries);
|
|
@@ -5277,9 +6247,11 @@ const runPipeline = async (options) => {
|
|
|
5277
6247
|
const customRegexEntities = filterAllowedLabels(config.enableRegex ? processRegexMatches(customRegexMatches, slices.customRegex.start, slices.customRegex.end, search.customRegexMeta) : [], preHotwordAllowedLabels);
|
|
5278
6248
|
const regexEntities = filterAllowedLabels([...rawRegexEntities, ...customRegexEntities], preHotwordAllowedLabels);
|
|
5279
6249
|
if (regexEntities.length > 0) log("regex", `${regexEntities.length} matches`);
|
|
6250
|
+
if (legalFormsEnabled || config.enableTriggerPhrases) await warmLegalRoleHeads();
|
|
5280
6251
|
const rawLegalFormEntities = legalFormsEnabled ? processLegalFormMatches(regexMatches, slices.legalForms.start, slices.legalForms.end, fullText) : [];
|
|
5281
6252
|
const legalFormEntities = filterAllowedLabels(rawLegalFormEntities, preHotwordAllowedLabels);
|
|
5282
6253
|
if (legalFormEntities.length > 0) log("legal-forms", `${legalFormEntities.length} matches`);
|
|
6254
|
+
if (config.enableTriggerPhrases) await warmAddressStopKeywords();
|
|
5283
6255
|
const rawTriggerEntities = config.enableTriggerPhrases ? processTriggerMatches(regexMatches, slices.triggers.start, slices.triggers.end, fullText, search.triggerRules) : [];
|
|
5284
6256
|
const triggerEntities = filterAllowedLabels(rawTriggerEntities, preHotwordAllowedLabels);
|
|
5285
6257
|
if (triggerEntities.length > 0) log("trigger-phrases", `${triggerEntities.length} matches`);
|
|
@@ -5325,10 +6297,11 @@ const runPipeline = async (options) => {
|
|
|
5325
6297
|
];
|
|
5326
6298
|
let rawNerEntities = [];
|
|
5327
6299
|
let nerEntities = [];
|
|
5328
|
-
|
|
6300
|
+
const requestedNerLabels = getRequestedNerLabels(config, hotwordsActive);
|
|
6301
|
+
if (config.enableNer && nerInference && requestedNerLabels.length > 0) {
|
|
5329
6302
|
const maskResult = maskDetectedSpans(fullText, nerMaskEntities);
|
|
5330
6303
|
log("ner", "running inference...");
|
|
5331
|
-
const rawNer = await nerInference(maskResult.maskedText, [...
|
|
6304
|
+
const rawNer = await nerInference(maskResult.maskedText, [...requestedNerLabels], config.threshold, signal);
|
|
5332
6305
|
rawNerEntities = unmaskNerEntities(rawNer, maskResult, fullText);
|
|
5333
6306
|
nerEntities = filterAllowedLabels(rawNerEntities, preHotwordAllowedLabels);
|
|
5334
6307
|
const masked = rawNer.length - rawNerEntities.length;
|
|
@@ -5389,6 +6362,7 @@ const runPipeline = async (options) => {
|
|
|
5389
6362
|
checkAbort(signal);
|
|
5390
6363
|
ctx.corefSourceMap.clear();
|
|
5391
6364
|
if (config.enableCoreference) {
|
|
6365
|
+
if (!legalFormsEnabled) await warmLegalRoleHeads();
|
|
5392
6366
|
const terms = await extractDefinedTerms(fullText, merged.filter((entity) => !isCallerOwnedEntity(entity)), ctx);
|
|
5393
6367
|
if (terms.length > 0) {
|
|
5394
6368
|
log("coreference", `${terms.length} defined terms`);
|
|
@@ -5429,7 +6403,7 @@ const resolveOperator = (config, label) => config.operators[label] ?? "replace";
|
|
|
5429
6403
|
//#region src/redact.ts
|
|
5430
6404
|
const WHITESPACE_RE = /\s+/g;
|
|
5431
6405
|
const PHONE_NOISE_RE = /[()\s-]/g;
|
|
5432
|
-
const
|
|
6406
|
+
const ID_SEPARATOR_RE = /[\s\-/.]/g;
|
|
5433
6407
|
/**
|
|
5434
6408
|
* Normalize entity text so that surface-form variations
|
|
5435
6409
|
* of the same real-world value map to a single canonical
|
|
@@ -5439,8 +6413,8 @@ const normalizeEntityText = (label, text) => {
|
|
|
5439
6413
|
const upper = label.toUpperCase().replace(WHITESPACE_RE, "_");
|
|
5440
6414
|
if (upper === "EMAIL_ADDRESS" || upper === "EMAIL") return text.toLowerCase().trim();
|
|
5441
6415
|
if (upper === "PHONE_NUMBER" || upper === "PHONE") return text.replace(PHONE_NOISE_RE, "");
|
|
5442
|
-
if (upper === "IBAN" || upper === "BANK_ACCOUNT_NUMBER" || upper === "TAX_IDENTIFICATION_NUMBER" || upper === "REGISTRATION_NUMBER") return text.replace(
|
|
5443
|
-
if (upper === "PERSON" || upper === "ORGANIZATION" || upper === "ADDRESS" || upper === "LAND_PARCEL") return text.replace(WHITESPACE_RE, " ").toLowerCase().trim();
|
|
6416
|
+
if (upper === "IBAN" || upper === "BANK_ACCOUNT_NUMBER" || upper === "TAX_IDENTIFICATION_NUMBER" || upper === "REGISTRATION_NUMBER" || upper === "NATIONAL_IDENTIFICATION_NUMBER" || upper === "SOCIAL_SECURITY_NUMBER" || upper === "BIRTH_NUMBER" || upper === "IDENTITY_CARD_NUMBER" || upper === "PASSPORT_NUMBER" || upper === "CREDIT_CARD_NUMBER") return text.replace(ID_SEPARATOR_RE, "").toUpperCase();
|
|
6417
|
+
if (upper === "PERSON" || upper === "ORGANIZATION" || upper === "ADDRESS" || upper === "LAND_PARCEL" || upper === "MISC") return text.replace(WHITESPACE_RE, " ").toLowerCase().trim();
|
|
5444
6418
|
return text.trim();
|
|
5445
6419
|
};
|
|
5446
6420
|
/**
|
|
@@ -5984,6 +6958,6 @@ const levenshtein = (rawA, rawB) => {
|
|
|
5984
6958
|
//#region src/wasm.ts
|
|
5985
6959
|
initTextSearch(TextSearch);
|
|
5986
6960
|
//#endregion
|
|
5987
|
-
export { CURRENCY_PATTERN_META, DATE_PATTERN_META, DEFAULT_ENTITY_LABELS, DEFAULT_OPERATOR_CONFIG, DETECTION_SOURCES, DETECTOR_PRIORITY, OPERATOR_REGISTRY, OPERATOR_TYPES, REGEX_META, REGEX_PATTERNS, REGIONS, ZONE_SCORE_ADJUSTMENTS, applyHotwordRules, applyZoneAdjustments, boostNearMissEntities, buildDenyList, buildGazetteerPatterns, buildLegalFormPatterns, buildPlaceholderMap, buildStreetTypePatterns, buildTriggerPatterns, buildUnifiedSearch, chunkText, classifyZones, computeChunkOffsets, corefKey, createPipelineContext, deanonymise, decodeSpans, decodeTokenSpans, detectNameCorpus, ensureDenyListData, exportRedactionKey, extractDefinedTerms, filterFalsePositives, findCoreferenceSpans, getCurrencyPatterns, getDatePatterns, initHotwordRules, initNameCorpus, initZoneClassifier, levenshtein, mergeAndDedup, mergeChunkEntities, normalizeForSearch, prepareBatch, processAddressSeeds, processDenyListMatches, processGazetteerMatches, processLegalFormMatches, processRegexMatches, processTriggerMatches, propagateOrgNames, redactText, resolveCountries, resolveOperator, runPipeline, runUnifiedSearch, sanitizeEntities, tokenizeText };
|
|
6961
|
+
export { CURRENCY_PATTERN_META, DATE_PATTERN_META, DEFAULT_ENTITY_LABELS, DEFAULT_OPERATOR_CONFIG, DETECTION_SOURCES, DETECTOR_PRIORITY, OPERATOR_REGISTRY, OPERATOR_TYPES, REGEX_META, REGEX_PATTERNS, REGIONS, ZONE_SCORE_ADJUSTMENTS, applyHotwordRules, applyZoneAdjustments, boostNearMissEntities, buildDenyList, buildGazetteerPatterns, buildLegalFormPatterns, buildPlaceholderMap, buildStreetTypePatterns, buildTriggerPatterns, buildUnifiedSearch, chunkText, classifyZones, computeChunkOffsets, corefKey, createPipelineContext, deanonymise, decodeSpans, decodeTokenSpans, detectNameCorpus, ensureDenyListData, exportRedactionKey, extractDefinedTerms, filterFalsePositives, findCoreferenceSpans, getCurrencyPatterns, getDatePatterns, initAddressComponents, initHotwordRules, initNameCorpus, initZoneClassifier, levenshtein, mergeAndDedup, mergeChunkEntities, normalizeForSearch, prepareBatch, processAddressSeeds, processDenyListMatches, processGazetteerMatches, processLegalFormMatches, processRegexMatches, processTriggerMatches, propagateOrgNames, redactText, resolveCountries, resolveOperator, runPipeline, runUnifiedSearch, sanitizeEntities, tokenizeText, warmLegalRoleHeads };
|
|
5988
6962
|
|
|
5989
6963
|
//# sourceMappingURL=wasm.mjs.map
|