@stll/anonymize-wasm 1.1.1 → 1.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (59) hide show
  1. package/dist/address-boundaries.mjs +113 -5
  2. package/dist/address-stop-keywords.mjs +137 -0
  3. package/dist/address-stop-keywords.mjs.map +1 -0
  4. package/dist/address-stopwords.mjs +81 -0
  5. package/dist/address-stopwords.mjs.map +1 -0
  6. package/dist/address-street-types.mjs +67 -4
  7. package/dist/clause-noun-heads.mjs +75 -0
  8. package/dist/clause-noun-heads.mjs.map +1 -0
  9. package/dist/coreference.es.mjs +22 -0
  10. package/dist/coreference.es.mjs.map +1 -0
  11. package/dist/coreference.fr.mjs +32 -0
  12. package/dist/coreference.fr.mjs.map +1 -0
  13. package/dist/coreference.it.mjs +27 -0
  14. package/dist/coreference.it.mjs.map +1 -0
  15. package/dist/coreference.pl.mjs +27 -0
  16. package/dist/coreference.pl.mjs.map +1 -0
  17. package/dist/coreference.pt-br.mjs +14 -0
  18. package/dist/coreference.pt-br.mjs.map +1 -0
  19. package/dist/coreference.sk.mjs +22 -5
  20. package/dist/generic-roles.mjs +179 -1
  21. package/dist/hotword-rules.mjs +7 -3
  22. package/dist/legal-forms.mjs +3 -0
  23. package/dist/legal-role-heads.cs.mjs +42 -0
  24. package/dist/legal-role-heads.cs.mjs.map +1 -0
  25. package/dist/legal-role-heads.de.mjs +33 -0
  26. package/dist/legal-role-heads.de.mjs.map +1 -0
  27. package/dist/legal-role-heads.en.mjs +37 -0
  28. package/dist/legal-role-heads.en.mjs.map +1 -0
  29. package/dist/legal-role-heads.es.mjs +54 -0
  30. package/dist/legal-role-heads.es.mjs.map +1 -0
  31. package/dist/legal-role-heads.fr.mjs +72 -0
  32. package/dist/legal-role-heads.fr.mjs.map +1 -0
  33. package/dist/legal-role-heads.it.mjs +68 -0
  34. package/dist/legal-role-heads.it.mjs.map +1 -0
  35. package/dist/legal-role-heads.pl.mjs +84 -0
  36. package/dist/legal-role-heads.pl.mjs.map +1 -0
  37. package/dist/legal-role-heads.pt-br.mjs +63 -0
  38. package/dist/legal-role-heads.pt-br.mjs.map +1 -0
  39. package/dist/legal-role-heads.sk.mjs +80 -0
  40. package/dist/legal-role-heads.sk.mjs.map +1 -0
  41. package/dist/manifest.mjs +21 -8
  42. package/dist/names-exclusions.mjs +21 -1
  43. package/dist/person-stopwords.mjs +121 -1
  44. package/dist/section-headings.mjs +15 -3
  45. package/dist/sentence-verb-indicators.mjs +223 -0
  46. package/dist/sentence-verb-indicators.mjs.map +1 -0
  47. package/dist/triggers.cs.mjs +107 -3
  48. package/dist/triggers.es.mjs +43 -4
  49. package/dist/triggers.fr.mjs +206 -7
  50. package/dist/triggers.it.mjs +26 -4
  51. package/dist/triggers.pl.mjs +191 -4
  52. package/dist/triggers.pt-br.mjs +193 -0
  53. package/dist/triggers.pt-br.mjs.map +1 -0
  54. package/dist/triggers.sk.mjs +493 -0
  55. package/dist/vite.mjs.map +1 -1
  56. package/dist/wasm.d.mts +100 -8
  57. package/dist/wasm.mjs +2520 -1360
  58. package/dist/wasm.mjs.map +1 -1
  59. package/package.json +2 -2
package/dist/wasm.mjs CHANGED
@@ -1,5 +1,5 @@
1
1
  import { TextSearch } from "@stll/text-search-wasm";
2
- import { at, be, bg, cy, cz, de, dk, ee, es, fi, fr, gb, gr, hr, hu, ie, it, lt, lu, lv, mt, nl, pl, pt, ro, se, si, sk } from "@stll/stdnum";
2
+ import { at, be, bg, br, cy, cz, de, dk, ee, es, fi, fr, gb, gr, hr, hu, ie, it, lt, lu, lv, mt, nl, pl, pt, ro, se, si, sk } from "@stll/stdnum";
3
3
  import { toRegex } from "@stll/stdnum/patterns";
4
4
  //#region src/search-engine.ts
5
5
  let _TextSearch;
@@ -66,11 +66,15 @@ const DEFAULT_ENTITY_LABELS = [
66
66
  "iban",
67
67
  "tax identification number",
68
68
  "identity card number",
69
+ "birth number",
70
+ "national identification number",
71
+ "social security number",
69
72
  "registration number",
70
73
  "credit card number",
71
74
  "passport number",
72
75
  "monetary amount",
73
- "land parcel"
76
+ "land parcel",
77
+ "misc"
74
78
  ];
75
79
  const isLegalFormsEnabled = (config) => config.enableLegalForms !== false;
76
80
  //#endregion
@@ -95,6 +99,8 @@ const createPipelineContext = () => ({
95
99
  allowListPromise: null,
96
100
  personStopwords: null,
97
101
  personStopwordsPromise: null,
102
+ addressStopwords: null,
103
+ addressStopwordsPromise: null,
98
104
  firstNameExclusions: null,
99
105
  firstNameExclusionCorpusLen: 0,
100
106
  genericRoles: null,
@@ -151,6 +157,7 @@ const LOADER_REGISTRIES = {
151
157
  hu: () => import("./triggers.hu.mjs"),
152
158
  it: () => import("./triggers.it.mjs"),
153
159
  pl: () => import("./triggers.pl.mjs"),
160
+ "pt-br": () => import("./triggers.pt-br.mjs"),
154
161
  ro: () => import("./triggers.ro.mjs"),
155
162
  sk: () => import("./triggers.sk.mjs"),
156
163
  sv: () => import("./triggers.sv.mjs")
@@ -159,7 +166,23 @@ const LOADER_REGISTRIES = {
159
166
  cs: () => import("./coreference.cs.mjs"),
160
167
  de: () => import("./coreference.de.mjs"),
161
168
  en: () => import("./coreference.en.mjs"),
169
+ es: () => import("./coreference.es.mjs"),
170
+ fr: () => import("./coreference.fr.mjs"),
171
+ it: () => import("./coreference.it.mjs"),
172
+ pl: () => import("./coreference.pl.mjs"),
173
+ "pt-br": () => import("./coreference.pt-br.mjs"),
162
174
  sk: () => import("./coreference.sk.mjs")
175
+ },
176
+ legalRoleHeads: {
177
+ cs: () => import("./legal-role-heads.cs.mjs"),
178
+ de: () => import("./legal-role-heads.de.mjs"),
179
+ en: () => import("./legal-role-heads.en.mjs"),
180
+ es: () => import("./legal-role-heads.es.mjs"),
181
+ fr: () => import("./legal-role-heads.fr.mjs"),
182
+ it: () => import("./legal-role-heads.it.mjs"),
183
+ pl: () => import("./legal-role-heads.pl.mjs"),
184
+ "pt-br": () => import("./legal-role-heads.pt-br.mjs"),
185
+ sk: () => import("./legal-role-heads.sk.mjs")
163
186
  }
164
187
  };
165
188
  const FALLBACK_LANGUAGES = {
@@ -172,6 +195,7 @@ const FALLBACK_LANGUAGES = {
172
195
  "hu",
173
196
  "it",
174
197
  "pl",
198
+ "pt-br",
175
199
  "ro",
176
200
  "sk",
177
201
  "sv"
@@ -180,6 +204,22 @@ const FALLBACK_LANGUAGES = {
180
204
  "cs",
181
205
  "de",
182
206
  "en",
207
+ "es",
208
+ "fr",
209
+ "it",
210
+ "pl",
211
+ "pt-br",
212
+ "sk"
213
+ ],
214
+ legalRoleHeads: [
215
+ "cs",
216
+ "de",
217
+ "en",
218
+ "es",
219
+ "fr",
220
+ "it",
221
+ "pl",
222
+ "pt-br",
183
223
  "sk"
184
224
  ]
185
225
  };
@@ -226,838 +266,64 @@ const loadLanguageConfigs = async (configType, mapFn) => {
226
266
  await Promise.all(loads);
227
267
  return results.filter((r) => r !== void 0);
228
268
  };
229
- //#endregion
230
- //#region src/detectors/coreference.ts
231
- /**
232
- * Load coreference definition patterns from per-language
233
- * JSON configs in src/data/. Uses the
234
- * language manifest for auto-discovery.
235
- */
236
- const loadDefinitionPatterns = async () => {
237
- const patterns = [];
238
- const allRows = await loadLanguageConfigs("coreference", (mod) => {
239
- return mod.default ?? mod;
240
- });
241
- for (const rows of allRows) {
242
- if (!Array.isArray(rows)) {
243
- console.warn("[anonymize] coreference: unexpected config shape, skipping");
244
- continue;
245
- }
246
- for (const row of rows) try {
247
- patterns.push({ pattern: new RegExp(row.pattern, row.flags) });
248
- } catch (err) {
249
- console.warn(`[anonymize] coreference: invalid regex "${row.pattern}":`, err);
250
- }
251
- }
252
- return patterns;
253
- };
254
- /**
255
- * Load generic role terms that should NOT be treated
256
- * as PII coreferences. "Prodávající" (Seller),
257
- * "Kupující" (Buyer), etc. are legal roles, not
258
- * identifying information.
259
- */
260
- const getRoleStopSet = async (ctx) => {
261
- if (ctx.roleStopSet) return ctx.roleStopSet;
262
- if (ctx.roleStopSetPromise) return ctx.roleStopSetPromise;
263
- const promise = (async () => {
264
- let result;
265
- try {
266
- const mod = await import("./generic-roles.mjs");
267
- const data = mod.default ?? mod;
268
- result = new Set(data.roles.map((r) => r.toLowerCase()));
269
- } catch {
270
- result = /* @__PURE__ */ new Set();
271
- }
272
- ctx.roleStopSet = result;
273
- return result;
274
- })();
275
- ctx.roleStopSetPromise = promise;
276
- return promise;
277
- };
278
- const getDefinitionPatterns = async (ctx) => {
279
- if (ctx.corefPatterns) return ctx.corefPatterns;
280
- if (ctx.corefPatternsPromise) return ctx.corefPatternsPromise;
281
- ctx.corefPatternsPromise = loadDefinitionPatterns();
282
- const patterns = await ctx.corefPatternsPromise;
283
- if (patterns.length === 0) {
284
- ctx.corefPatterns = patterns;
285
- if (!ctx.corefLoadAttempted) {
286
- ctx.corefLoadAttempted = true;
287
- console.warn("[anonymize] coreference: no definition patterns loaded; coreference detection will be inactive");
288
- }
289
- return patterns;
290
- }
291
- ctx.corefPatterns = patterns;
292
- return patterns;
293
- };
294
- const SEARCH_WINDOW = 200;
295
- /**
296
- * Check whether an alias has textual similarity to
297
- * the source entity. Prevents roles and structural
298
- * terms from being treated as name aliases.
299
- *
300
- * Three checks (any passes → similar):
301
- * 1. Word overlap: a word in the alias appears in the
302
- * entity (case-insensitive, min 2 chars)
303
- * 2. Initials: alias letters match first letters of
304
- * entity words ("TB" ↔ "Tomas Bata")
305
- * 3. Substring: alias is a substring of the entity
306
- * or vice versa (min 3 chars)
307
- */
308
- const hasEntitySimilarity = (alias, entityText) => {
309
- const aliasLower = alias.toLowerCase();
310
- const entityLower = entityText.toLowerCase();
311
- if (aliasLower.length >= 3 && entityLower.includes(aliasLower)) return true;
312
- if (entityLower.length >= 3 && aliasLower.includes(entityLower)) return true;
313
- const aliasWords = aliasLower.split(/[\s.,;:'"()/-]+/).filter((w) => w.length >= 2);
314
- const entityWords = entityLower.split(/[\s.,;:'"()/-]+/).filter((w) => w.length >= 2);
315
- const entityWordSet = new Set(entityWords);
316
- for (const word of aliasWords) if (entityWordSet.has(word)) return true;
317
- if (/^[\p{Lu}]+$/u.test(alias) && alias.length >= 2 && alias.length <= entityWords.length) {
318
- for (let start = 0; start <= entityWords.length - alias.length; start++) if (entityWords.slice(start, start + alias.length).map((w) => w.charAt(0)).join("") === aliasLower) return true;
319
- }
320
- return false;
321
- };
322
- /**
323
- * Scan for defined-term patterns near known entities.
324
- *
325
- * Legal documents universally follow:
326
- * "Dr. Heinrich Muller (hereinafter 'the Seller')..."
327
- *
328
- * After NER detects the entity, this function scans for
329
- * definitional patterns within +/-200 chars and extracts
330
- * the alias. Returns alias + label pairs that can be added
331
- * to the gazetteer for a full-text re-scan.
332
- */
333
- /**
334
- * Labels that can be the source of a coreference alias.
335
- * Only parties (person, organization) have defined-term
336
- * aliases in legal text. Dates, addresses, IDs do not.
337
- */
338
- const COREF_SOURCE_LABELS = new Set(["person", "organization"]);
339
- const extractDefinedTerms = async (fullText, entities, ctx = defaultContext) => {
340
- const [definitionPatterns, roleStops] = await Promise.all([getDefinitionPatterns(ctx), getRoleStopSet(ctx)]);
341
- const terms = [];
342
- const seen = /* @__PURE__ */ new Set();
343
- const sorted = [...entities].sort((a, b) => a.start - b.start);
344
- for (const { pattern } of definitionPatterns) {
345
- pattern.lastIndex = 0;
346
- for (let match = pattern.exec(fullText); match !== null; match = pattern.exec(fullText)) {
347
- const alias = match[1]?.trim();
348
- if (!alias || alias.length < 2) continue;
349
- if (roleStops.has(alias.toLowerCase())) continue;
350
- const defPos = match.index;
351
- let bestEntity = null;
352
- for (let i = sorted.length - 1; i >= 0; i--) {
353
- const e = sorted[i];
354
- if (e === void 0) continue;
355
- if (e.end > defPos) continue;
356
- if (defPos - e.end > SEARCH_WINDOW) break;
357
- if (!COREF_SOURCE_LABELS.has(e.label)) continue;
358
- bestEntity = e;
359
- break;
360
- }
361
- if (bestEntity === null) continue;
362
- const gapText = fullText.slice(bestEntity.end, defPos);
363
- if (/(?:;|\.(?=\s*(?:["'„‚(]*\p{Lu}|$)))/u.test(gapText)) continue;
364
- if (!hasEntitySimilarity(alias, bestEntity.text)) continue;
365
- const key = `${alias.toLowerCase()}::${bestEntity.label}`;
366
- if (seen.has(key)) continue;
367
- seen.add(key);
368
- terms.push({
369
- alias,
370
- label: bestEntity.label,
371
- definitionStart: defPos,
372
- sourceText: bestEntity.text
373
- });
374
- }
375
- }
376
- return terms;
377
- };
378
- /**
379
- * Check if a character is a Unicode word character
380
- * (letter, digit, or combining mark). Used for word
381
- * boundary checks in coreference matching.
382
- */
383
- const isWordChar = (ch) => {
384
- if (ch === void 0) return false;
385
- return /[\p{L}\p{M}\p{N}]/u.test(ch);
386
- };
387
- /**
388
- * Find all occurrences of defined-term aliases in the
389
- * full text. Returns Entity spans for each match.
390
- *
391
- * Respects word boundaries: "Kupující" must not match
392
- * inside "Kupujícímu". A match is valid only if the
393
- * character before the start and after the end are NOT
394
- * word characters (letter/digit).
395
- *
396
- * Populates `ctx.corefSourceMap` with entries linking
397
- * each coref entity to its source entity text, for
398
- * consistent placeholder numbering.
399
- */
400
- const findCoreferenceSpans = (fullText, terms, ctx = defaultContext) => {
401
- const results = [];
402
- for (const term of terms) {
403
- let searchFrom = 0;
404
- while (searchFrom < fullText.length) {
405
- const idx = fullText.indexOf(term.alias, searchFrom);
406
- if (idx === -1) break;
407
- const matchEnd = idx + term.alias.length;
408
- const charBefore = idx > 0 ? fullText[idx - 1] : void 0;
409
- const charAfter = fullText[matchEnd];
410
- if (isWordChar(charBefore) || isWordChar(charAfter)) {
411
- searchFrom = idx + 1;
412
- continue;
413
- }
414
- const entity = {
415
- start: idx,
416
- end: matchEnd,
417
- label: term.label,
418
- text: term.alias,
419
- score: .95,
420
- source: DETECTION_SOURCES.COREFERENCE
421
- };
422
- ctx.corefSourceMap.set(corefKey(entity), term.sourceText);
423
- results.push(entity);
424
- searchFrom = matchEnd;
425
- }
426
- }
427
- return results;
428
- };
429
- //#endregion
430
- //#region src/detectors/gazetteer.ts
431
- const MAX_EDIT_DISTANCE = 2;
432
- const MIN_FUZZY_LENGTH = 4;
433
- const MAX_PREFIX_OVERSHOOT = 7;
434
- /**
435
- * Collect all searchable strings (canonical + variants)
436
- * from gazetteer entries, mapped to their labels and
437
- * entry IDs.
438
- */
439
- const buildSearchTerms = (entries) => {
440
- const terms = /* @__PURE__ */ new Map();
441
- for (const entry of entries) {
442
- const meta = {
443
- label: entry.label,
444
- entryId: entry.id
445
- };
446
- terms.set(entry.canonical, meta);
447
- for (const variant of entry.variants) terms.set(variant, meta);
448
- }
449
- return terms;
450
- };
451
- /**
452
- * Build TextSearch-compatible patterns from gazetteer
453
- * entries. Returns:
454
- * - Exact literal patterns for all terms
455
- * - Fuzzy patterns (distance: 2) for terms >= 4 chars
456
- * - Parallel metadata arrays for post-processing
457
- *
458
- * Patterns are ordered: all exact first, then all
459
- * fuzzy. The isFuzzy array marks which are which.
460
- */
461
- const buildGazetteerPatterns = (entries) => {
462
- const terms = buildSearchTerms(entries);
463
- const patterns = [];
464
- const labels = [];
465
- const isFuzzy = [];
466
- for (const [term, meta] of terms) {
467
- patterns.push({
468
- pattern: term,
469
- literal: true,
470
- wholeWords: false
471
- });
472
- labels.push(meta.label);
473
- isFuzzy.push(false);
474
- }
475
- for (const [term, meta] of terms) {
476
- if (term.length < MIN_FUZZY_LENGTH) continue;
477
- patterns.push({
478
- pattern: term,
479
- distance: MAX_EDIT_DISTANCE
480
- });
481
- labels.push(meta.label);
482
- isFuzzy.push(true);
483
- }
484
- return {
485
- patterns,
486
- data: {
487
- labels,
488
- isFuzzy
489
- }
490
- };
491
- };
492
- /**
493
- * Process gazetteer matches from the unified literal
494
- * search. Receives all matches; filters to the
495
- * gazetteer slice via sliceStart/sliceEnd.
496
- *
497
- * Exact matches get score 0.9; fuzzy matches get
498
- * 0.85. Fuzzy matches that overlap an exact match
499
- * are dropped.
500
- *
501
- * For exact matches, attempts prefix extension for
502
- * legal suffixes ("a.s.", "GmbH", "s.r.o." after
503
- * the matched term).
504
- */
505
- const processGazetteerMatches = (allMatches, sliceStart, sliceEnd, fullText, data) => {
506
- const results = [];
507
- const exactSpans = [];
508
- for (const match of allMatches) {
509
- const idx = match.pattern;
510
- if (idx < sliceStart || idx >= sliceEnd) continue;
511
- const localIdx = idx - sliceStart;
512
- if (data.isFuzzy[localIdx]) continue;
513
- const label = data.labels[localIdx];
514
- if (!label) continue;
515
- const extended = tryPrefixExtension(fullText, match.start, match.end);
516
- const end = extended?.end ?? match.end;
517
- const text = extended?.text ?? fullText.slice(match.start, match.end);
518
- exactSpans.push({
519
- start: match.start,
520
- end
521
- });
522
- results.push({
523
- start: match.start,
524
- end,
525
- label,
526
- text,
527
- score: .9,
528
- source: DETECTION_SOURCES.GAZETTEER
529
- });
530
- }
531
- for (const match of allMatches) {
532
- const idx = match.pattern;
533
- if (idx < sliceStart || idx >= sliceEnd) continue;
534
- const localIdx = idx - sliceStart;
535
- if (!data.isFuzzy[localIdx]) continue;
536
- if (match.distance === 0) continue;
537
- const label = data.labels[localIdx];
538
- if (!label) continue;
539
- if (exactSpans.some((e) => match.start < e.end && match.end > e.start)) continue;
540
- const matchText = fullText.slice(match.start, match.end);
541
- results.push({
542
- start: match.start,
543
- end: match.end,
544
- label,
545
- text: matchText,
546
- score: .85,
547
- source: DETECTION_SOURCES.GAZETTEER
548
- });
549
- }
550
- return results;
551
- };
552
- /**
553
- * Try to extend an exact match to capture one
554
- * trailing token (max 6 chars) that may be a legal
555
- * entity suffix (e.g., "a.s.", "GmbH", "s.r.o.").
556
- *
557
- * Does not validate the token against a legal-forms
558
- * list; false extensions are filtered by mergeAndDedup
559
- * when a legal-form detector produces a competing
560
- * entity with the correct span.
561
- */
562
- const tryPrefixExtension = (fullText, start, end) => {
563
- const maxEnd = Math.min(end + MAX_PREFIX_OVERSHOOT, fullText.length);
564
- if (maxEnd <= end + 1) return null;
565
- const after = fullText.slice(end, maxEnd);
566
- if (!after.startsWith(" ")) return null;
567
- const nextSpace = after.indexOf(" ", 1);
568
- const suffixEnd = nextSpace !== -1 ? nextSpace : after.length;
569
- if (suffixEnd <= 1) return null;
570
- const newEnd = end + suffixEnd;
571
- return {
572
- end: newEnd,
573
- text: fullText.slice(start, newEnd)
574
- };
575
- };
576
- //#endregion
577
- //#region src/util/text.ts
578
- /**
579
- * Shared text utilities for detectors.
580
- *
581
- * Extracted from names.ts and deny-list.ts to avoid
582
- * duplicating regex constants and helper functions.
583
- */
584
- /** Matches a string that starts with an uppercase letter. */
585
- const UPPER_START_RE = /^\p{Lu}/u;
586
- /** Matches a string consisting entirely of uppercase letters. */
587
- const ALL_UPPER_RE = /^\p{Lu}+$/u;
588
- const SENTENCE_END_RE = /[.!?]/;
589
- /**
590
- * Detect whether a position is at the start of a sentence.
591
- * Looks backward past whitespace for sentence-ending
592
- * punctuation (.!?). Position 0 and positions preceded
593
- * only by whitespace are considered sentence starts.
594
- */
595
- const isSentenceStart = (text, pos) => {
596
- if (pos === 0) return true;
597
- let i = pos - 1;
598
- while (i >= 0 && /\s/.test(text[i] ?? "")) i--;
599
- if (i < 0) return true;
600
- return SENTENCE_END_RE.test(text[i] ?? "");
601
- };
602
- //#endregion
603
- //#region src/detectors/names.ts
604
- const getCorpus = (ctx) => ctx.nameCorpus;
605
- const getNameCorpusFirstNames = (ctx = defaultContext) => ctx.nameCorpus?.firstNamesList ?? [];
606
- const getNameCorpusSurnames = (ctx = defaultContext) => ctx.nameCorpus?.surnamesList ?? [];
607
- const getNameCorpusTitles = (ctx = defaultContext) => ctx.nameCorpus?.titlesList ?? [];
608
- /**
609
- * Load name corpus data from injected dictionaries
610
- * and legacy config files. Merges all sources.
611
- *
612
- * Safe to call multiple times; only loads once per
613
- * context. Must be called before detectNameCorpus or
614
- * the getNameCorpus*() accessors are used.
615
- *
616
- * @param dictionaries Optional pre-loaded dictionaries
617
- * with per-language first names and surnames. When
618
- * omitted, only legacy config files are used.
619
- */
620
- const initNameCorpus = (ctx = defaultContext, dictionaries) => {
621
- if (ctx.nameCorpusPromise) return ctx.nameCorpusPromise;
622
- const promise = (async () => {
623
- try {
624
- const [legacyFirstMod, legacySurnameMod, titleMod, exclusionMod] = await Promise.all([
625
- import("./names-first.mjs"),
626
- import("./names-surnames.mjs"),
627
- import("./names-title-tokens.mjs"),
628
- import("./names-exclusions.mjs")
629
- ]);
630
- const firstNames = [...legacyFirstMod.default.names];
631
- if (dictionaries?.firstNames) for (const names of Object.values(dictionaries.firstNames)) for (const name of names) firstNames.push(name);
632
- const surnames = [...legacySurnameMod.default.names];
633
- if (dictionaries?.surnames) for (const names of Object.values(dictionaries.surnames)) for (const name of names) surnames.push(name);
634
- const dedup = (arr) => {
635
- const seen = /* @__PURE__ */ new Set();
636
- const result = [];
637
- for (const item of arr) {
638
- if (seen.has(item)) continue;
639
- seen.add(item);
640
- result.push(item);
641
- }
642
- return result;
643
- };
644
- const dedupFirst = dedup(firstNames);
645
- const dedupSurnames = dedup(surnames);
646
- const titles = titleMod.default.tokens;
647
- const exclusions = exclusionMod.default.words;
648
- ctx.nameCorpus = {
649
- firstNames: Object.freeze(new Set(dedupFirst)),
650
- surnames: Object.freeze(new Set(dedupSurnames)),
651
- titleTokens: Object.freeze(new Set(titles)),
652
- excludedWords: Object.freeze(new Set(exclusions)),
653
- firstNamesList: Object.freeze(dedupFirst),
654
- surnamesList: Object.freeze(dedupSurnames),
655
- titlesList: Object.freeze(titles),
656
- excludedList: Object.freeze(exclusions)
657
- };
658
- } catch (err) {
659
- ctx.nameCorpusPromise = null;
660
- console.warn("[anonymize] Failed to load name corpus JSON — name detection disabled:", err);
661
- }
662
- })();
663
- ctx.nameCorpusPromise = promise;
664
- return promise;
665
- };
666
- const INFLECTION_SUFFIXES = [
667
- "ovi",
668
- "em",
669
- "om",
670
- "ou",
671
- "é",
672
- "a",
673
- "u"
674
- ];
675
- /**
676
- * Strip common Czech/Slovak case suffixes from a token.
677
- * Returns candidate base forms if stripping produces a
678
- * plausible name (capitalised, length >= 3).
679
- *
680
- * For the "-ou" instrumental feminine suffix, also yields
681
- * base + "a" (e.g., "Editou" → "Edit" and "Edita")
682
- * because Czech feminine names decline -a → -ou.
683
- */
684
- const stripInflection = (token) => {
685
- const candidates = [];
686
- for (const suffix of INFLECTION_SUFFIXES) if (token.length > suffix.length + 2 && token.endsWith(suffix)) {
687
- const base = token.slice(0, -suffix.length);
688
- if (/^\p{Lu}/u.test(base)) {
689
- candidates.push(base);
690
- if (suffix === "ou" || suffix === "é" || suffix === "u") candidates.push(`${base}a`);
691
- }
692
- }
693
- return candidates;
694
- };
695
- const TOKEN_TYPE = {
696
- NAME: "name",
697
- SURNAME: "surname",
698
- TITLE: "title",
699
- ABBREVIATION: "abbreviation",
700
- CAPITALIZED: "capitalized",
701
- OTHER: "other"
702
- };
703
- const PERSON_CHAIN_BREAK_RE$1 = /[!?;:]/u;
704
- const isInitialContinuationGap$1 = (text, gap) => /^\p{Lu}$/u.test(text) && /^\.[^\S\n]{1,2}$/u.test(gap) || /^[^\S\n]{1,2}(?:\p{Lu}\.[^\S\n]{1,2})+$/u.test(gap);
705
- /**
706
- * Check if a token is in the first-name set, either
707
- * directly or after stripping Czech/Slovak inflection.
708
- */
709
- const isFirstNameToken = (token, corpus) => {
710
- if (corpus.firstNames.has(token)) return true;
711
- return stripInflection(token).some((b) => corpus.firstNames.has(b));
712
- };
713
- /**
714
- * Check if a token is in the surname set, either
715
- * directly or after stripping Czech/Slovak inflection.
716
- */
717
- const isSurnameToken = (token, corpus) => {
718
- if (corpus.surnames.has(token)) return true;
719
- return stripInflection(token).some((b) => corpus.surnames.has(b));
720
- };
721
- /**
722
- * Check if a token looks like a single-letter
723
- * abbreviation: "J.", "M.", etc.
724
- */
725
- const isAbbreviation = (token) => token.length === 2 && /^\p{Lu}$/u.test(token[0] ?? "") && token[1] === ".";
726
- const segmenter$1 = new Intl.Segmenter(void 0, { granularity: "word" });
727
- /**
728
- * Split text into word segments using Intl.Segmenter.
729
- * Only returns segments flagged as words.
730
- */
731
- const segmentWords = (fullText) => {
732
- const words = [];
733
- for (const seg of segmenter$1.segment(fullText)) if (seg.isWordLike) words.push({
734
- text: seg.segment,
735
- start: seg.index,
736
- end: seg.index + seg.segment.length
737
- });
738
- return words;
739
- };
740
- /** NAME or SURNAME — both represent corpus-matched tokens */
741
- const isCorpusMatch = (type) => type === TOKEN_TYPE.NAME || type === TOKEN_TYPE.SURNAME;
742
- const classifyToken = (word, corpus) => {
743
- const { text, start, end } = word;
744
- const lower = text.toLowerCase();
745
- const stripped = text.endsWith(".") ? text.slice(0, -1).toLowerCase() : lower;
746
- if (corpus.titleTokens.has(stripped)) return {
747
- text,
748
- type: TOKEN_TYPE.TITLE,
749
- start,
750
- end
751
- };
752
- if (isAbbreviation(text)) return {
753
- text,
754
- type: TOKEN_TYPE.ABBREVIATION,
755
- start,
756
- end
757
- };
758
- if (corpus.excludedWords.has(lower)) return {
759
- text,
760
- type: TOKEN_TYPE.OTHER,
761
- start,
762
- end
763
- };
764
- if (text.length < 3) return {
765
- text,
766
- type: TOKEN_TYPE.OTHER,
767
- start,
768
- end
769
- };
770
- if (text.length > 3 && ALL_UPPER_RE.test(text)) return {
771
- text,
772
- type: TOKEN_TYPE.OTHER,
773
- start,
774
- end
775
- };
776
- if (!UPPER_START_RE.test(text)) return {
777
- text,
778
- type: TOKEN_TYPE.OTHER,
779
- start,
780
- end
781
- };
782
- if (isFirstNameToken(text, corpus)) return {
783
- text,
784
- type: TOKEN_TYPE.NAME,
785
- start,
786
- end
787
- };
788
- if (isSurnameToken(text, corpus)) return {
789
- text,
790
- type: TOKEN_TYPE.SURNAME,
791
- start,
792
- end
793
- };
794
- return {
795
- text,
796
- type: TOKEN_TYPE.CAPITALIZED,
797
- start,
798
- end
799
- };
800
- };
801
- /**
802
- * Detect person names by looking up tokens against the
803
- * name corpus, then chaining adjacent name-like tokens.
804
- *
805
- * Requires initNameCorpus() to have been called first.
806
- * If not initialized, returns an empty array.
807
- *
808
- * Scoring:
809
- * TITLE + NAME/SURNAME → 0.95
810
- * NAME + NAME/SURNAME → 0.9
811
- * SURNAME + NAME/SURNAME → 0.9
812
- * NAME + CAPITALIZED → 0.7
813
- * ABBREVIATION + NAME → 0.7
814
- * Standalone NAME → 0.5 (low confidence)
815
- * Standalone SURNAME → skip (too ambiguous)
816
- */
817
- const detectNameCorpus = (fullText, ctx = defaultContext) => {
818
- const corpus = getCorpus(ctx);
819
- if (!corpus) return [];
820
- const tokens = segmentWords(fullText).map((w) => classifyToken(w, corpus));
821
- const entities = [];
822
- const consumed = /* @__PURE__ */ new Set();
823
- for (let i = 0; i < tokens.length; i++) {
824
- if (consumed.has(i)) continue;
825
- const token = tokens[i];
826
- if (!token) continue;
827
- if (token.type !== TOKEN_TYPE.TITLE && token.type !== TOKEN_TYPE.NAME && token.type !== TOKEN_TYPE.SURNAME && token.type !== TOKEN_TYPE.ABBREVIATION) continue;
828
- const MAX_CHAIN = 5;
829
- const chain = [token];
830
- let j = i + 1;
831
- while (j < tokens.length && chain.length < MAX_CHAIN) {
832
- const next = tokens[j];
833
- if (!next) break;
834
- const prev = chain.at(-1);
835
- if (prev) {
836
- const gap = fullText.slice(prev.end, next.start);
837
- const breaksOnPeriod = gap.includes(".") && !isInitialContinuationGap$1(prev.text, gap);
838
- if (gap.includes("\n") || PERSON_CHAIN_BREAK_RE$1.test(gap) || breaksOnPeriod) break;
839
- }
840
- if (next.type === TOKEN_TYPE.NAME || next.type === TOKEN_TYPE.SURNAME || next.type === TOKEN_TYPE.TITLE || next.type === TOKEN_TYPE.ABBREVIATION || next.type === TOKEN_TYPE.CAPITALIZED) {
841
- chain.push(next);
842
- j++;
843
- } else break;
844
- }
845
- const hasTitle = chain.some((t) => t.type === TOKEN_TYPE.TITLE);
846
- const hasCorpusName = chain.some((t) => isCorpusMatch(t.type));
847
- const hasFirstName = chain.some((t) => t.type === TOKEN_TYPE.NAME);
848
- const hasAbbreviation = chain.some((t) => t.type === TOKEN_TYPE.ABBREVIATION);
849
- const corpusCount = chain.filter((t) => isCorpusMatch(t.type)).length;
850
- const capitalizedCount = chain.filter((t) => t.type === TOKEN_TYPE.CAPITALIZED).length;
851
- let score = 0;
852
- if (hasTitle && hasCorpusName) score = .95;
853
- else if (corpusCount >= 2) score = .9;
854
- else if (hasCorpusName && capitalizedCount > 0) score = .7;
855
- else if (hasAbbreviation && hasCorpusName) score = .7;
856
- else if (hasFirstName && chain.length === 1) {
857
- if (isSentenceStart(fullText, token.start)) continue;
858
- score = .5;
859
- } else if (!hasFirstName && chain.length === 1 && chain[0]?.type === TOKEN_TYPE.SURNAME) continue;
860
- else if (hasTitle && chain.length === 1) continue;
861
- else {
862
- if (!hasCorpusName) continue;
863
- score = .5;
864
- }
865
- const first = chain.at(0);
866
- const last = chain.at(-1);
867
- if (!first || !last) continue;
868
- const start = first.start;
869
- const end = last.end;
870
- const text = fullText.slice(start, end);
871
- for (let k = i; k < i + chain.length; k++) consumed.add(k);
872
- entities.push({
873
- start,
874
- end,
875
- label: "person",
876
- text,
877
- score,
878
- source: DETECTION_SOURCES.REGEX
879
- });
880
- }
881
- return entities;
882
- };
883
- //#endregion
884
- //#region src/config/titles.ts
885
- /**
886
- * Academic and professional title prefixes.
887
- * Plain text; the detector auto-escapes for regex.
888
- * Sorted longest-first at build time.
889
- */
890
- const TITLE_PREFIXES = [
891
- "Ing.",
892
- "Mgr.",
893
- "MgA.",
894
- "Bc.",
895
- "BcA.",
896
- "JUDr.",
897
- "MUDr.",
898
- "MVDr.",
899
- "MDDr.",
900
- "PhDr.",
901
- "RNDr.",
902
- "PaedDr.",
903
- "ThDr.",
904
- "ThLic.",
905
- "ICDr.",
906
- "RSDr.",
907
- "PharmDr.",
908
- "artD.",
909
- "akad.",
910
- "doc.",
911
- "prof.",
912
- "ao. Univ.-Prof.",
913
- "o. Univ.-Prof.",
914
- "Univ.-Prof.",
915
- "Hon.-Prof.",
916
- "em. Prof.",
917
- "Dr. med. dent.",
918
- "Dr. med. vet.",
919
- "Dr. med.",
920
- "Dr. rer. nat.",
921
- "Dr. rer. soc.",
922
- "Dr. rer. pol.",
923
- "Dr. sc. tech.",
924
- "Dr. sc. nat.",
925
- "Dr. sc. hum.",
926
- "Dr. iur.",
927
- "Dr. jur.",
928
- "Dr. theol.",
929
- "Dr. oec.",
930
- "Dr. techn.",
931
- "Dr. h. c.",
932
- "Dr. phil.",
933
- "Dr.-Ing.",
934
- "Dr. Ing.",
935
- "Dr.",
936
- "Dipl.-Wirt.-Ing.",
937
- "Dipl.-Betriebsw.",
938
- "Dipl.-Inform.",
939
- "Dipl.-Volksw.",
940
- "Dipl.-Psych.",
941
- "Dipl.-Phys.",
942
- "Dipl.-Chem.",
943
- "Dipl.-Biol.",
944
- "Dipl.-Math.",
945
- "Dipl.-Päd.",
946
- "Dipl.-Soz.",
947
- "Dipl.-Kfm.",
948
- "Dipl.-Jur.",
949
- "Dipl. Ing.",
950
- "Dipl.-Ing.",
951
- "Mag. rer. soc. oec.",
952
- "Mag. rer. nat.",
953
- "Mag. phil.",
954
- "Mag. iur.",
955
- "Mag. arch.",
956
- "Mag. pharm.",
957
- "Mag. (FH)",
958
- "Mag.",
959
- "Bakk. rer. nat.",
960
- "Bakk. techn.",
961
- "Bakk. phil.",
962
- "Bakk.",
963
- "Lic. phil.",
964
- "Lic. iur.",
965
- "Lic. oec.",
966
- "Lic. theol.",
967
- "Lic.",
968
- "Priv.-Doz.",
969
- "PD",
970
- "RA"
971
- ];
972
- /**
973
- * Courtesy/honorific titles that precede a person's
974
- * name. Sorted alphabetically. The detector escapes
975
- * dots and adds \b for entries in HONORIFIC_BOUNDARY.
976
- */
977
- const HONORIFICS = [
978
- "Avv.",
979
- "Dame",
980
- "Doamna",
981
- "Domnul",
982
- "Don",
983
- "Doña",
984
- "Dott.",
985
- "Judge",
986
- "Justice",
987
- "Lady",
988
- "Lord",
989
- "M.",
990
- "Madame",
991
- "Mademoiselle",
992
- "Maître",
993
- "Me",
994
- "Messrs",
995
- "Miss",
996
- "Mlle",
997
- "Mme",
998
- "Monsieur",
999
- "Mr",
1000
- "Mrs",
1001
- "Ms",
1002
- "President",
1003
- "Señor",
1004
- "Señora",
1005
- "Sig.",
1006
- "Sig.ra",
1007
- "Signor",
1008
- "Signora",
1009
- "Signorina",
1010
- "Sir",
1011
- "Sr.",
1012
- "Sra."
1013
- ];
1014
- /**
1015
- * Honorifics that need \b word-boundary anchors
1016
- * (short or common words that could match mid-word).
1017
- */
1018
- const HONORIFIC_BOUNDARY = new Set([
1019
- "Don",
1020
- "Doña",
1021
- "M.",
1022
- "Me",
1023
- "Señor",
1024
- "Señora"
1025
- ]);
1026
- /**
1027
- * Post-nominal degrees (comma or space separated after name).
1028
- * Plain text; the detector auto-escapes for regex.
1029
- */
1030
- const POST_NOMINALS = [
1031
- "Ph.D.",
1032
- "Ph.D",
1033
- "CSc.",
1034
- "DrSc.",
1035
- "ArtD.",
1036
- "D.Phil.",
1037
- "DPhil.",
1038
- "MPhil.",
1039
- "MBA",
1040
- "MPA",
1041
- "LL.M.",
1042
- "LL.B.",
1043
- "M.Sc.",
1044
- "B.Sc.",
1045
- "MSc.",
1046
- "BSc.",
1047
- "M.Eng.",
1048
- "B.Eng.",
1049
- "M.A.",
1050
- "B.A.",
1051
- "JCD",
1052
- "JD",
1053
- "DiS.",
1054
- "ACCA",
1055
- "FCCA",
1056
- "CIPM",
1057
- "CIPT",
1058
- "CIPP/E",
1059
- "CIPP"
1060
- ];
269
+ const LEGAL_SUFFIXES = [...[
270
+ "spol. s r.o.",
271
+ "s.r.o.",
272
+ "s. r. o.",
273
+ "a.s.",
274
+ "a. s.",
275
+ "v.o.s.",
276
+ "v. o. s.",
277
+ "k.s.",
278
+ "k. s.",
279
+ "z.s.",
280
+ "z. s.",
281
+ "z.ú.",
282
+ "z. ú.",
283
+ "o.p.s.",
284
+ "o. p. s.",
285
+ "s.p.",
286
+ "s. p.",
287
+ "GmbH",
288
+ "AG",
289
+ "SE",
290
+ "KG",
291
+ "OHG",
292
+ "Ltd.",
293
+ "Ltd",
294
+ "LLC",
295
+ "LLP",
296
+ "Inc.",
297
+ "Corp.",
298
+ "Corporation",
299
+ "Co.",
300
+ "LP",
301
+ "L.P.",
302
+ "PLC",
303
+ "plc",
304
+ "N.A.",
305
+ "N.V.",
306
+ "B.V.",
307
+ "Pty Ltd.",
308
+ "Pty Ltd",
309
+ "S.A.",
310
+ "SA",
311
+ "SAS",
312
+ "SARL",
313
+ "S.p.A.",
314
+ "Sp. z o.o.",
315
+ "Sp. k.",
316
+ "Sp. j.",
317
+ "Ltda.",
318
+ "LTDA.",
319
+ "Ltda",
320
+ "LTDA",
321
+ "S/A",
322
+ "EIRELI",
323
+ "EPP",
324
+ "ME",
325
+ "MEI"
326
+ ]].sort((a, b) => b.length - a.length);
1061
327
  //#endregion
1062
328
  //#region src/data/char-groups.json
1063
329
  var char_groups_default = {
@@ -1397,255 +663,1536 @@ var char_groups_default = {
1397
663
  "code": "U+A789"
1398
664
  },
1399
665
  {
1400
- "char": "∶",
1401
- "name": "ratio",
1402
- "code": "U+2236"
666
+ "char": "∶",
667
+ "name": "ratio",
668
+ "code": "U+2236"
669
+ },
670
+ {
671
+ "char": ":",
672
+ "name": "full-width colon",
673
+ "code": "U+FF1A"
674
+ },
675
+ {
676
+ "char": "፥",
677
+ "name": "Ethiopic colon",
678
+ "code": "U+1365"
679
+ }
680
+ ]
681
+ },
682
+ "comma": {
683
+ "description": "Comma-like characters",
684
+ "chars": [
685
+ {
686
+ "char": ",",
687
+ "name": "comma",
688
+ "code": "U+002C"
689
+ },
690
+ {
691
+ "char": "‚",
692
+ "name": "single low-9 quotation mark (used as comma)",
693
+ "code": "U+201A"
694
+ },
695
+ {
696
+ "char": "،",
697
+ "name": "Arabic comma",
698
+ "code": "U+060C"
699
+ },
700
+ {
701
+ "char": "、",
702
+ "name": "ideographic comma",
703
+ "code": "U+3001"
704
+ },
705
+ {
706
+ "char": ",",
707
+ "name": "full-width comma",
708
+ "code": "U+FF0C"
709
+ },
710
+ {
711
+ "char": "﹐",
712
+ "name": "small comma",
713
+ "code": "U+FE50"
714
+ },
715
+ {
716
+ "char": "՝",
717
+ "name": "Armenian comma",
718
+ "code": "U+055D"
719
+ }
720
+ ]
721
+ },
722
+ "ellipsis": {
723
+ "description": "Ellipsis characters",
724
+ "chars": [{
725
+ "char": "…",
726
+ "name": "horizontal ellipsis",
727
+ "code": "U+2026"
728
+ }]
729
+ },
730
+ "question-mark": {
731
+ "description": "Question mark variants",
732
+ "chars": [
733
+ {
734
+ "char": "?",
735
+ "name": "question mark",
736
+ "code": "U+003F"
737
+ },
738
+ {
739
+ "char": "?",
740
+ "name": "full-width question mark",
741
+ "code": "U+FF1F"
742
+ },
743
+ {
744
+ "char": "⁇",
745
+ "name": "double question mark",
746
+ "code": "U+2047"
1403
747
  },
1404
748
  {
1405
- "char": "",
1406
- "name": "full-width colon",
1407
- "code": "U+FF1A"
749
+ "char": "",
750
+ "name": "question exclamation mark",
751
+ "code": "U+2048"
1408
752
  },
1409
753
  {
1410
- "char": "",
1411
- "name": "Ethiopic colon",
1412
- "code": "U+1365"
754
+ "char": "؟",
755
+ "name": "Arabic question mark",
756
+ "code": "U+061F"
757
+ },
758
+ {
759
+ "char": "‽",
760
+ "name": "interrobang",
761
+ "code": "U+203D"
1413
762
  }
1414
763
  ]
1415
764
  },
1416
- "comma": {
1417
- "description": "Comma-like characters",
765
+ "exclamation-mark": {
766
+ "description": "Exclamation mark variants",
1418
767
  "chars": [
1419
768
  {
1420
- "char": ",",
1421
- "name": "comma",
1422
- "code": "U+002C"
769
+ "char": "!",
770
+ "name": "exclamation mark",
771
+ "code": "U+0021"
1423
772
  },
1424
773
  {
1425
- "char": "",
1426
- "name": "single low-9 quotation mark (used as comma)",
1427
- "code": "U+201A"
774
+ "char": "",
775
+ "name": "full-width exclamation mark",
776
+ "code": "U+FF01"
1428
777
  },
1429
778
  {
1430
- "char": "،",
1431
- "name": "Arabic comma",
1432
- "code": "U+060C"
779
+ "char": "",
780
+ "name": "double exclamation mark",
781
+ "code": "U+203C"
1433
782
  },
1434
783
  {
1435
- "char": "",
1436
- "name": "ideographic comma",
1437
- "code": "U+3001"
784
+ "char": "¡",
785
+ "name": "inverted exclamation mark",
786
+ "code": "U+00A1"
787
+ }
788
+ ]
789
+ },
790
+ "semicolon": {
791
+ "description": "Semicolon variants",
792
+ "chars": [
793
+ {
794
+ "char": ";",
795
+ "name": "semicolon",
796
+ "code": "U+003B"
1438
797
  },
1439
798
  {
1440
- "char": "",
1441
- "name": "full-width comma",
1442
- "code": "U+FF0C"
799
+ "char": "؛",
800
+ "name": "Arabic semicolon",
801
+ "code": "U+061B"
1443
802
  },
1444
803
  {
1445
- "char": "",
1446
- "name": "small comma",
1447
- "code": "U+FE50"
804
+ "char": "",
805
+ "name": "full-width semicolon",
806
+ "code": "U+FF1B"
1448
807
  },
1449
808
  {
1450
- "char": "՝",
1451
- "name": "Armenian comma",
1452
- "code": "U+055D"
809
+ "char": "",
810
+ "name": "Ethiopic semicolon",
811
+ "code": "U+1364"
812
+ }
813
+ ]
814
+ },
815
+ "underscore": {
816
+ "description": "Underscore variants",
817
+ "chars": [{
818
+ "char": "_",
819
+ "name": "low line",
820
+ "code": "U+005F"
821
+ }, {
822
+ "char": "_",
823
+ "name": "full-width low line",
824
+ "code": "U+FF3F"
825
+ }]
826
+ }
827
+ }
828
+ };
829
+ //#endregion
830
+ //#region src/util/char-groups.ts
831
+ /**
832
+ * Centralized Unicode character equivalence groups.
833
+ *
834
+ * Centralized Unicode character equivalence groups.
835
+ *
836
+ * Provides helpers to build regex character classes that
837
+ * match all typographic variants of a character type
838
+ * (dashes, spaces, quotes, etc.).
839
+ *
840
+ * The JSON is statically imported so the bundler inlines
841
+ * it, avoiding a runtime require() that breaks browsers.
842
+ */
843
+ /** Chars that need escaping inside a regex char class. */
844
+ const REGEX_CLASS_SPECIAL = /[\\\]^-]/;
845
+ const escapeForCharClass = (ch) => REGEX_CLASS_SPECIAL.test(ch) ? `\\${ch}` : ch;
846
+ const config = char_groups_default;
847
+ /**
848
+ * Get the raw characters for a named group.
849
+ * Throws if the group does not exist.
850
+ */
851
+ const charSet = (group) => {
852
+ const g = config.groups[group];
853
+ if (!g) throw new Error(`Unknown char group: "${group}"`);
854
+ return g.chars.map((entry) => entry.char);
855
+ };
856
+ /**
857
+ * Build a regex character class string for a named
858
+ * group. E.g., charClass("dash") returns a string
859
+ * like "[-\u2013\u2014\u2010\u2011\u2212\u2043]".
860
+ *
861
+ * The hyphen-minus is placed first so it is treated
862
+ * as a literal, not a range indicator.
863
+ */
864
+ const charClass = (group) => {
865
+ return `[${[...charSet(group)].sort((a, b) => {
866
+ if (a === "-") return -1;
867
+ if (b === "-") return 1;
868
+ return a.localeCompare(b);
869
+ }).map(escapeForCharClass).join("")}]`;
870
+ };
871
+ /**
872
+ * Return the inner content of a character class (without
873
+ * the surrounding brackets). Useful for embedding a group
874
+ * inside a larger character class, e.g.:
875
+ * `[\\s&,.${charClassInner("dash")}]`
876
+ */
877
+ const charClassInner = (group) => {
878
+ return [...charSet(group)].sort((a, b) => {
879
+ if (a === "-") return -1;
880
+ if (b === "-") return 1;
881
+ return a.localeCompare(b);
882
+ }).map(escapeForCharClass).join("");
883
+ };
884
+ /** Regex char class matching all dash variants. */
885
+ const DASH = charClass("dash");
886
+ /**
887
+ * Inner content of the dash char class (no brackets).
888
+ * For embedding inside a larger character class:
889
+ * `[\\s&,.${DASH_INNER}]`
890
+ */
891
+ const DASH_INNER = charClassInner("dash");
892
+ charClass("space");
893
+ charClass("quote-double");
894
+ /** Inner content of the double-quote char class (no brackets). */
895
+ const QUOTE_DOUBLE_INNER = charClassInner("quote-double");
896
+ charClass("quote-single");
897
+ /** Inner content of the single-quote char class (no brackets). */
898
+ const QUOTE_SINGLE_INNER = charClassInner("quote-single");
899
+ /**
900
+ * Inline opening-bracket characters that frequently introduce
901
+ * a defined-term parenthetical immediately after an address or
902
+ * organization span (e.g. `…GA 30326, USA (the "Premises")`).
903
+ * Hardcoded rather than data-driven because the set is tiny,
904
+ * regex-special, and not language-dependent.
905
+ */
906
+ const OPENING_BRACKETS_INNER = "(\\[{";
907
+ charClass("slash");
908
+ charClass("dot");
909
+ charClass("colon");
910
+ //#endregion
911
+ //#region src/detectors/legal-forms.ts
912
+ const SENTENCE_VERB_INDICATORS_SEED = new Set([
913
+ "je",
914
+ "jsou",
915
+ "is",
916
+ "are",
917
+ "ist",
918
+ "sind"
919
+ ]);
920
+ let sentenceVerbIndicatorsCache = null;
921
+ let sentenceVerbIndicatorsPromise = null;
922
+ const loadSentenceVerbIndicators = async () => {
923
+ if (sentenceVerbIndicatorsCache) return sentenceVerbIndicatorsCache;
924
+ if (sentenceVerbIndicatorsPromise) return sentenceVerbIndicatorsPromise;
925
+ sentenceVerbIndicatorsPromise = (async () => {
926
+ let data = {};
927
+ try {
928
+ const mod = await import("./sentence-verb-indicators.mjs");
929
+ data = mod.default ?? mod;
930
+ } catch (err) {
931
+ console.warn("[anonymize] legal-forms: failed to load sentence-verb-indicators.json, falling back to seed list:", err);
932
+ }
933
+ const all = new Set(SENTENCE_VERB_INDICATORS_SEED);
934
+ for (const [key, value] of Object.entries(data)) {
935
+ if (key.startsWith("_")) continue;
936
+ if (!Array.isArray(value)) continue;
937
+ for (const verb of value) {
938
+ if (typeof verb !== "string" || verb.length === 0) continue;
939
+ all.add(verb.toLowerCase());
940
+ }
941
+ }
942
+ sentenceVerbIndicatorsCache = all;
943
+ return all;
944
+ })();
945
+ return sentenceVerbIndicatorsPromise;
946
+ };
947
+ const getSentenceVerbIndicatorsSync = () => sentenceVerbIndicatorsCache ?? SENTENCE_VERB_INDICATORS_SEED;
948
+ const UPPER = "A-ZÁČĎÉĚÍŇÓŘŠŤÚŮÝŽÄÖÜÀÂÆÇÈÊËÎÏÔÙÛŸÑĄĆĘŁŃŚŹŻ\\u0130";
949
+ const LOWER = "a-záčďéěíňóřšťúůýžäöüßàâæçèêëîïôùûÿñąćęłńśźż\\u0131";
950
+ const CAP_WORD = `(?:[${UPPER}]{2,}|[${UPPER}][${LOWER}${UPPER}]+)`;
951
+ const ALLCAP_WORD = `[${UPPER}]{2,}`;
952
+ const ROMAN_NUMERAL_RE = /^(?=[IVXLCDM])M{0,3}(?:CM|CD|D?C{0,3})(?:XC|XL|L?X{0,3})(?:IX|IV|V?I{0,3})$/;
953
+ let legalRoleHeadsCache = null;
954
+ let legalRoleHeadsPromise = null;
955
+ const loadLegalRoleHeads = async () => {
956
+ if (legalRoleHeadsCache) return legalRoleHeadsCache;
957
+ if (legalRoleHeadsPromise) return legalRoleHeadsPromise;
958
+ legalRoleHeadsPromise = (async () => {
959
+ const sets = await loadLanguageConfigs("legalRoleHeads", (mod) => {
960
+ return mod.default ?? mod;
961
+ });
962
+ const all = /* @__PURE__ */ new Set();
963
+ for (const entry of sets) {
964
+ if (!entry || !Array.isArray(entry.words)) continue;
965
+ for (const word of entry.words) if (typeof word === "string" && word.length > 0) all.add(word.toLowerCase());
966
+ }
967
+ legalRoleHeadsCache = all;
968
+ return all;
969
+ })();
970
+ return legalRoleHeadsPromise;
971
+ };
972
+ const getLegalRoleHeadsSync = () => legalRoleHeadsCache ?? /* @__PURE__ */ new Set();
973
+ const warmLegalRoleHeads = async () => {
974
+ await Promise.all([
975
+ loadLegalRoleHeads(),
976
+ loadAllLegalSuffixes(),
977
+ loadSentenceVerbIndicators(),
978
+ loadClauseNounHeads()
979
+ ]);
980
+ };
981
+ let allLegalSuffixesCache = null;
982
+ let allLegalSuffixesPromise = null;
983
+ const loadAllLegalSuffixes = async () => {
984
+ if (allLegalSuffixesCache) return allLegalSuffixesCache;
985
+ if (allLegalSuffixesPromise) return allLegalSuffixesPromise;
986
+ allLegalSuffixesPromise = (async () => {
987
+ let data = {};
988
+ try {
989
+ data = (await import("./legal-forms.mjs")).default;
990
+ } catch {
991
+ data = {};
992
+ }
993
+ const seen = /* @__PURE__ */ new Set();
994
+ const out = [];
995
+ for (const list of Object.values(data)) for (const form of list) {
996
+ if (typeof form !== "string" || form.length === 0) continue;
997
+ if (seen.has(form)) continue;
998
+ seen.add(form);
999
+ out.push(form);
1000
+ }
1001
+ for (const form of LEGAL_SUFFIXES) if (!seen.has(form)) {
1002
+ seen.add(form);
1003
+ out.push(form);
1004
+ }
1005
+ out.sort((a, b) => b.length - a.length);
1006
+ allLegalSuffixesCache = out;
1007
+ return out;
1008
+ })();
1009
+ return allLegalSuffixesPromise;
1010
+ };
1011
+ const getAllLegalSuffixesSync = () => allLegalSuffixesCache ?? LEGAL_SUFFIXES;
1012
+ /**
1013
+ * Sync accessor for the full legal-form vocabulary
1014
+ * (`data/legal-forms.json` plus `LEGAL_SUFFIXES`,
1015
+ * longest-first). Falls back to `LEGAL_SUFFIXES` when
1016
+ * `warmLegalRoleHeads()` has not run yet. Exposed so the
1017
+ * trailing-period strip in `sanitizeEntities` can keep
1018
+ * pace with the detector vocabulary rather than only the
1019
+ * smaller `LEGAL_SUFFIXES` propagation list.
1020
+ */
1021
+ const getKnownLegalSuffixes = getAllLegalSuffixesSync;
1022
+ const CLAUSE_NOUN_HEADS_SEED = new Set(["agreement", "contract"]);
1023
+ let clauseNounHeadsCache = null;
1024
+ let clauseNounHeadsPromise = null;
1025
+ const loadClauseNounHeads = async () => {
1026
+ if (clauseNounHeadsCache) return clauseNounHeadsCache;
1027
+ if (clauseNounHeadsPromise) return clauseNounHeadsPromise;
1028
+ clauseNounHeadsPromise = (async () => {
1029
+ let data = {};
1030
+ try {
1031
+ const mod = await import("./clause-noun-heads.mjs");
1032
+ data = mod.default ?? mod;
1033
+ } catch (err) {
1034
+ console.warn("[anonymize] legal-forms: failed to load clause-noun-heads.json, falling back to seed list:", err);
1035
+ }
1036
+ const all = new Set(CLAUSE_NOUN_HEADS_SEED);
1037
+ for (const [key, value] of Object.entries(data)) {
1038
+ if (key.startsWith("_")) continue;
1039
+ if (!Array.isArray(value)) continue;
1040
+ for (const noun of value) {
1041
+ if (typeof noun !== "string" || noun.length === 0) continue;
1042
+ all.add(noun.toLowerCase());
1043
+ }
1044
+ }
1045
+ clauseNounHeadsCache = all;
1046
+ return all;
1047
+ })();
1048
+ return clauseNounHeadsPromise;
1049
+ };
1050
+ const getClauseNounHeadsSync = () => clauseNounHeadsCache ?? CLAUSE_NOUN_HEADS_SEED;
1051
+ const escapeForRegex = (form) => form.replace(/[.*+?^${}()|[\]\\]/g, "\\$&").replace(/\s+/g, "\\s+").replace(/\\\./g, "\\.[^\\S\\n]?");
1052
+ const isShortForm = (form) => form.replace(/[.\s]/g, "").length <= 3 && !form.includes(" ");
1053
+ const buildPatternString = (forms) => {
1054
+ if (forms.length === 0) return null;
1055
+ const alt = forms.toSorted((a, b) => b.length - a.length).map(escapeForRegex).join("|");
1056
+ const HSPACE = "[^\\S\\n]";
1057
+ const LOWER_CONNECTOR = `${HSPACE}+(?:a|and|und|et|e|y|i)${HSPACE}+(?=[${LOWER}])`;
1058
+ const SIMPLE_SEP = `(?:${HSPACE}|[&,.${DASH_INNER}]){1,4}`;
1059
+ const CAP_OR_NUM_WORD = `(?:${CAP_WORD}|\\d{1,4})`;
1060
+ const LOWER_WORD = `(?:(?!(?:and|und|et)(?![${UPPER}${LOWER}]))[${LOWER}][${LOWER}${UPPER}]+)`;
1061
+ const ANY_WORD_TAIL = `(?:(?!(?:and|und|et)(?![${UPPER}${LOWER}]))[${UPPER}${LOWER}][${LOWER}${UPPER}]+|[${UPPER}]{2,3}|\\d{1,4})`;
1062
+ return `${`(?:${`(?:${CAP_WORD})(?:${SIMPLE_SEP}(?:${CAP_OR_NUM_WORD})){0,10}`})(?:${`${SIMPLE_SEP}(?:${LOWER_WORD})(?:(?:${LOWER_CONNECTOR}|${SIMPLE_SEP})(?:${ANY_WORD_TAIL})){0,10}`})?`}(?:\\s+|,\\s*)(?:${alt})(?![${LOWER}])`;
1063
+ };
1064
+ /**
1065
+ * Build legal form regex pattern strings.
1066
+ * Returns an array of regex strings for the unified
1067
+ * TextSearch builder. Empty if data package is not
1068
+ * installed.
1069
+ */
1070
+ const buildLegalFormPatterns = async () => {
1071
+ let data = {};
1072
+ try {
1073
+ data = (await import("./legal-forms.mjs")).default;
1074
+ } catch {
1075
+ return [];
1076
+ }
1077
+ const allForms = [];
1078
+ const seen = /* @__PURE__ */ new Set();
1079
+ for (const forms of Object.values(data)) for (const form of forms) if (!seen.has(form)) {
1080
+ seen.add(form);
1081
+ allForms.push(form);
1082
+ }
1083
+ for (const form of LEGAL_SUFFIXES) if (!seen.has(form)) {
1084
+ seen.add(form);
1085
+ allForms.push(form);
1086
+ }
1087
+ const patterns = [];
1088
+ const longPattern = buildPatternString(allForms.filter((f) => !isShortForm(f)));
1089
+ if (longPattern) patterns.push(longPattern);
1090
+ const shortPattern = buildPatternString(allForms.filter(isShortForm));
1091
+ if (shortPattern) patterns.push(shortPattern);
1092
+ const allcapPrefix = `(?:${ALLCAP_WORD})(?:[ \\t&,.${DASH_INNER}]{1,4}(?:${ALLCAP_WORD})){0,2}`;
1093
+ const allcapAlt = allForms.toSorted((a, b) => b.length - a.length).map(escapeForRegex).join("|");
1094
+ patterns.push(`${allcapPrefix}(?:[ \\t]+|,[ \\t]*)(?:${allcapAlt})(?![${LOWER}])`);
1095
+ return patterns;
1096
+ };
1097
+ const CONNECTOR_RE = /^(?:a|and|und|et|e|y|i|&)$/i;
1098
+ const AND_TYPE_CONNECTOR_RE = /^(?:and|und|et)$/i;
1099
+ const UPPER_LETTER_RE = /^\p{Lu}/u;
1100
+ const COMPANY_SUFFIX_WORDS_RE = /^(?:Company|Co|Bank|Brothers|Bros|Sons|Group|Holdings|Trust|Partners|Associates|Corporation|Industries|Enterprises|Solutions|Systems|Services|Foundation|Institute)$/;
1101
+ const IN_NAME_PREPOSITION_RE = /^(?:of|the)$/i;
1102
+ const ENTITY_HEAD_WORD_RE = /^[\p{L}\p{M}&]+/u;
1103
+ const LEADING_CLAUSE_RE = /(?:^|\s)(?:by\s+and\s+between|is\s+between)\s+/giu;
1104
+ /**
1105
+ * Find the word ending just before `pos` in `text`,
1106
+ * skipping any whitespace (not newlines).
1107
+ * Returns null if no word is found (e.g., at start
1108
+ * of text, or preceded by non-word chars like ".").
1109
+ */
1110
+ const findWordBefore = (text, pos) => {
1111
+ let scan = pos - 1;
1112
+ while (scan >= 0) {
1113
+ const ch = text.charAt(scan);
1114
+ if (ch === "\n" || !/\s/.test(ch)) break;
1115
+ scan--;
1116
+ }
1117
+ if (scan < 0 || text.charAt(scan) === "\n") return null;
1118
+ const wordEnd = scan + 1;
1119
+ while (scan >= 0 && /[\p{L}\p{M}&]/u.test(text.charAt(scan))) scan--;
1120
+ const wordStart = scan + 1;
1121
+ const word = text.slice(wordStart, wordEnd);
1122
+ if (word.length === 0) return null;
1123
+ return {
1124
+ word,
1125
+ start: wordStart
1126
+ };
1127
+ };
1128
+ /**
1129
+ * Count consecutive uppercase-starting words immediately
1130
+ * before `pos`. Stops at the first non-upper word or at
1131
+ * text/line start. Used to disambiguate "<First> <Last>
1132
+ * and <ORG>" from "<Multi-word Org> and <Continuation>".
1133
+ */
1134
+ const countUpperWordsBefore = (fullText, pos) => {
1135
+ let count = 0;
1136
+ let scan = pos;
1137
+ while (scan > 0) {
1138
+ const found = findWordBefore(fullText, scan);
1139
+ if (!found) break;
1140
+ if (!UPPER_LETTER_RE.test(found.word)) break;
1141
+ count++;
1142
+ scan = found.start;
1143
+ }
1144
+ return count;
1145
+ };
1146
+ /**
1147
+ * Extend a match backward through uppercase words and
1148
+ * lowercase connectors. Stops at start of text,
1149
+ * newline, or a word that doesn't qualify.
1150
+ *
1151
+ * Connectors (a, and, und, et, ...) are only consumed
1152
+ * when there is a valid word before them — a trailing
1153
+ * connector at an entity boundary is not consumed.
1154
+ * For multi-char "and"-type connectors we additionally
1155
+ * refuse to cross when exactly two uppercase words
1156
+ * precede them ("First Last and ORG, Inc." shape) —
1157
+ * unless the match itself begins with a known company-
1158
+ * suffix word ("…and Company, Inc."), in which case
1159
+ * the chain belongs to one organisation. In that
1160
+ * suffix-mode we also cross in-name prepositions
1161
+ * ("Bank of America and Trust Company, Inc.").
1162
+ */
1163
+ const extendBackward = (fullText, matchStart, options = {}) => {
1164
+ const headWord = ENTITY_HEAD_WORD_RE.exec(fullText.slice(matchStart))?.[0] ?? "";
1165
+ const suffixMode = options.forceSuffixMode === true || COMPANY_SUFFIX_WORDS_RE.test(headWord);
1166
+ let pos = matchStart;
1167
+ while (pos > 0) {
1168
+ const found = findWordBefore(fullText, pos);
1169
+ if (!found) break;
1170
+ const { word, start: wordStart } = found;
1171
+ const isUpper = UPPER_LETTER_RE.test(word);
1172
+ const isConnector = CONNECTOR_RE.test(word);
1173
+ const isInNamePrep = suffixMode && IN_NAME_PREPOSITION_RE.test(word);
1174
+ if (isUpper) pos = wordStart;
1175
+ else if (isConnector) {
1176
+ if (!suffixMode && AND_TYPE_CONNECTOR_RE.test(word) && countUpperWordsBefore(fullText, wordStart) === 2) break;
1177
+ const prev = findWordBefore(fullText, wordStart);
1178
+ if (!prev) break;
1179
+ if (!UPPER_LETTER_RE.test(prev.word)) break;
1180
+ pos = prev.start;
1181
+ } else if (isInNamePrep) {
1182
+ const prev = findWordBefore(fullText, wordStart);
1183
+ if (!prev) break;
1184
+ if (!UPPER_LETTER_RE.test(prev.word)) break;
1185
+ pos = prev.start;
1186
+ } else break;
1187
+ }
1188
+ return pos;
1189
+ };
1190
+ const trimLeadingClause = (text) => {
1191
+ let cut = -1;
1192
+ for (const match of text.matchAll(LEADING_CLAUSE_RE)) cut = match.index + match[0].length;
1193
+ if (cut <= 0) return {
1194
+ offset: 0,
1195
+ text
1196
+ };
1197
+ const trimmed = text.slice(cut);
1198
+ const leadingWs = trimmed.match(/^\s*/u)?.[0].length ?? 0;
1199
+ return {
1200
+ offset: cut + leadingWs,
1201
+ text: trimmed.slice(leadingWs)
1202
+ };
1203
+ };
1204
+ /**
1205
+ * Process legal form matches from the unified search.
1206
+ * Receives all matches; filters to the legal forms
1207
+ * slice via sliceStart/sliceEnd.
1208
+ *
1209
+ * The role-head trimming step reads per-language data from
1210
+ * a cache that `runPipeline` warms via `warmLegalRoleHeads()`
1211
+ * before calling this. Callers that invoke
1212
+ * `processLegalFormMatches` directly (without going through
1213
+ * `runPipeline`) must `await warmLegalRoleHeads()` first;
1214
+ * otherwise the trim falls back to a no-op and sentence-
1215
+ * fragment fixes do not apply.
1216
+ */
1217
+ const processLegalFormMatches = (allMatches, sliceStart, sliceEnd, fullText) => {
1218
+ const results = [];
1219
+ for (const match of allMatches) {
1220
+ const idx = match.pattern;
1221
+ if (idx < sliceStart || idx >= sliceEnd) continue;
1222
+ const text = match.text.trimEnd();
1223
+ if (text.length < 5) continue;
1224
+ const roleHeads = getLegalRoleHeadsSync();
1225
+ const firstWordMatch = /^[\p{L}\p{M}]+(?:-[\p{L}\p{M}]+)*/u.exec(text);
1226
+ let processedStart = match.start;
1227
+ let processedText = text;
1228
+ let trimmed = false;
1229
+ const firstWordText = firstWordMatch?.[0] ?? "";
1230
+ const firstWordLeading = /^[\p{L}\p{M}]+/u.exec(firstWordText)?.[0] ?? "";
1231
+ if (firstWordMatch !== null && (roleHeads.has(firstWordText.toLowerCase()) || firstWordLeading.length > 0 && roleHeads.has(firstWordLeading.toLowerCase()))) {
1232
+ let suffixOffset = -1;
1233
+ for (const suffix of getAllLegalSuffixesSync()) {
1234
+ const idx = text.lastIndexOf(suffix);
1235
+ if (idx !== -1 && idx + suffix.length >= text.length - 1) {
1236
+ suffixOffset = idx;
1237
+ break;
1453
1238
  }
1454
- ]
1455
- },
1456
- "ellipsis": {
1457
- "description": "Ellipsis characters",
1458
- "chars": [{
1459
- "char": "…",
1460
- "name": "horizontal ellipsis",
1461
- "code": "U+2026"
1462
- }]
1463
- },
1464
- "question-mark": {
1465
- "description": "Question mark variants",
1466
- "chars": [
1467
- {
1468
- "char": "?",
1469
- "name": "question mark",
1470
- "code": "U+003F"
1471
- },
1472
- {
1473
- "char": "?",
1474
- "name": "full-width question mark",
1475
- "code": "U+FF1F"
1476
- },
1477
- {
1478
- "char": "⁇",
1479
- "name": "double question mark",
1480
- "code": "U+2047"
1481
- },
1482
- {
1483
- "char": "⁈",
1484
- "name": "question exclamation mark",
1485
- "code": "U+2048"
1486
- },
1487
- {
1488
- "char": "؟",
1489
- "name": "Arabic question mark",
1490
- "code": "U+061F"
1491
- },
1492
- {
1493
- "char": "‽",
1494
- "name": "interrobang",
1495
- "code": "U+203D"
1239
+ }
1240
+ if (suffixOffset < 0) {} else {
1241
+ const midStart = firstWordMatch[0].length;
1242
+ const midEnd = suffixOffset;
1243
+ const midSection = text.slice(midStart, midEnd);
1244
+ const verbIndicators = getSentenceVerbIndicatorsSync();
1245
+ let lastVerbEndInMid = -1;
1246
+ for (const match of midSection.matchAll(/(?<![\p{L}\p{N}])[\p{L}\p{M}]+/gu)) if (match[0] !== void 0 && match.index !== void 0 && verbIndicators.has(match[0].toLowerCase())) lastVerbEndInMid = match.index + match[0].length;
1247
+ const digitAfterRole = /^\s+\d+(?:\.|\b)/u.test(midSection);
1248
+ let appositiveRoleHead = false;
1249
+ if (!digitAfterRole && lastVerbEndInMid === -1 && fullText) {
1250
+ const before = fullText.slice(Math.max(0, match.start - 40), match.start);
1251
+ const prevWord = /(?<![\p{L}\p{N}])(\p{L}[\p{L}\p{M}]*)\s*$/u.exec(before);
1252
+ if (prevWord !== null && getSentenceVerbIndicatorsSync().has(prevWord[1].toLowerCase())) appositiveRoleHead = true;
1496
1253
  }
1497
- ]
1498
- },
1499
- "exclamation-mark": {
1500
- "description": "Exclamation mark variants",
1501
- "chars": [
1502
- {
1503
- "char": "!",
1504
- "name": "exclamation mark",
1505
- "code": "U+0021"
1506
- },
1507
- {
1508
- "char": "!",
1509
- "name": "full-width exclamation mark",
1510
- "code": "U+FF01"
1511
- },
1512
- {
1513
- "char": "‼",
1514
- "name": "double exclamation mark",
1515
- "code": "U+203C"
1516
- },
1517
- {
1518
- "char": "¡",
1519
- "name": "inverted exclamation mark",
1520
- "code": "U+00A1"
1254
+ if (lastVerbEndInMid !== -1 || digitAfterRole || appositiveRoleHead) {
1255
+ const scanStart = lastVerbEndInMid !== -1 ? midStart + lastVerbEndInMid : midStart;
1256
+ const capRe = /(?<![\p{L}\p{N}])\p{Lu}[\p{L}\p{M}\p{N}]*/gu;
1257
+ capRe.lastIndex = scanStart;
1258
+ const clauseNouns = getClauseNounHeadsSync();
1259
+ let capMatch = null;
1260
+ for (let next = capRe.exec(text); next !== null; next = capRe.exec(text)) {
1261
+ if (next.index >= suffixOffset) break;
1262
+ const lc = next[0].toLowerCase();
1263
+ if (roleHeads.has(lc) || clauseNouns.has(lc)) continue;
1264
+ capMatch = next;
1265
+ break;
1266
+ }
1267
+ if (capMatch === null) continue;
1268
+ processedStart = match.start + capMatch.index;
1269
+ processedText = text.slice(capMatch.index);
1270
+ trimmed = true;
1521
1271
  }
1522
- ]
1523
- },
1524
- "semicolon": {
1525
- "description": "Semicolon variants",
1526
- "chars": [
1527
- {
1528
- "char": ";",
1529
- "name": "semicolon",
1530
- "code": "U+003B"
1531
- },
1532
- {
1533
- "char": "؛",
1534
- "name": "Arabic semicolon",
1535
- "code": "U+061B"
1536
- },
1537
- {
1538
- "char": ";",
1539
- "name": "full-width semicolon",
1540
- "code": "U+FF1B"
1541
- },
1542
- {
1543
- "char": "",
1544
- "name": "Ethiopic semicolon",
1545
- "code": "U+1364"
1272
+ }
1273
+ }
1274
+ if (processedText.includes("\n")) continue;
1275
+ let entityStart = processedStart;
1276
+ let entityText = processedText;
1277
+ if (fullText && !trimmed) {
1278
+ const extended = extendBackward(fullText, processedStart);
1279
+ if (extended < processedStart) {
1280
+ entityStart = extended;
1281
+ entityText = fullText.slice(extended, processedStart + processedText.length).trimEnd();
1282
+ }
1283
+ }
1284
+ const clauseTrim = trimLeadingClause(entityText);
1285
+ if (clauseTrim.offset > 0) {
1286
+ entityStart += clauseTrim.offset;
1287
+ entityText = clauseTrim.text;
1288
+ }
1289
+ const getPrefixInfo = (value) => {
1290
+ const prefixEnd = value.lastIndexOf(",") !== -1 ? value.lastIndexOf(",") : value.lastIndexOf(" ");
1291
+ return {
1292
+ prefixEnd,
1293
+ prefixPart: prefixEnd > 0 ? value.slice(0, prefixEnd).replace(/[^a-zA-ZÀ-ž]/g, "") : value.replace(/[^a-zA-ZÀ-ž]/g, "")
1294
+ };
1295
+ };
1296
+ let { prefixEnd, prefixPart } = getPrefixInfo(entityText);
1297
+ let isAllCapsMatch = prefixPart.length > 2 && prefixPart === prefixPart.toUpperCase();
1298
+ if (isAllCapsMatch && fullText) {
1299
+ const lineStart = fullText.lastIndexOf("\n", entityStart);
1300
+ const lineEnd = fullText.indexOf("\n", entityStart + entityText.length);
1301
+ const lineLetters = fullText.slice(lineStart + 1, lineEnd === -1 ? fullText.length : lineEnd).replace(/[^a-zA-ZÀ-ž]/g, "");
1302
+ const upperCount = [...lineLetters].filter((c) => c === c.toUpperCase()).length;
1303
+ if (lineLetters.length > 5 && upperCount / lineLetters.length >= .95) continue;
1304
+ if ((prefixPart.length > 0 ? entityText.slice(0, prefixEnd > 0 ? prefixEnd : entityText.length).trim().split(/\s+/).length : 0) > 3) {
1305
+ entityStart = match.start;
1306
+ entityText = text;
1307
+ ({prefixEnd, prefixPart} = getPrefixInfo(entityText));
1308
+ isAllCapsMatch = prefixPart.length > 2 && prefixPart === prefixPart.toUpperCase();
1309
+ }
1310
+ } else if (isAllCapsMatch) continue;
1311
+ const lastSpace = entityText.lastIndexOf(" ");
1312
+ const rawSuffix = lastSpace !== -1 ? entityText.slice(lastSpace + 1) : "";
1313
+ const suffixClean = rawSuffix.replace(/[.,]/g, "");
1314
+ if (suffixClean.length > 0 && ROMAN_NUMERAL_RE.test(suffixClean)) continue;
1315
+ if (suffixClean.length <= 2 && !/\./.test(rawSuffix) && /[^\x00-\x7F]/.test(entityText.slice(0, lastSpace !== -1 ? lastSpace : entityText.length))) continue;
1316
+ results.push({
1317
+ start: entityStart,
1318
+ end: entityStart + entityText.length,
1319
+ label: "organization",
1320
+ text: entityText,
1321
+ score: .95,
1322
+ source: DETECTION_SOURCES.LEGAL_FORM
1323
+ });
1324
+ }
1325
+ return results;
1326
+ };
1327
+ //#endregion
1328
+ //#region src/detectors/coreference.ts
1329
+ /**
1330
+ * Case-insensitive set of every known legal-form suffix
1331
+ * with whitespace normalized away. Used by the
1332
+ * coreference scan to reject parenthetical define-term
1333
+ * captures whose alias is only a legal-form
1334
+ * abbreviation (e.g. "(« SAS »)" after "ACME SAS"),
1335
+ * which would otherwise propagate generic suffix
1336
+ * mentions as document-wide org aliases.
1337
+ */
1338
+ const isLegalFormAlias = (alias) => {
1339
+ const normalized = alias.replace(/\s+/g, "").toLowerCase();
1340
+ if (normalized.length === 0) return false;
1341
+ for (const suffix of getKnownLegalSuffixes()) if (suffix.replace(/\s+/g, "").toLowerCase() === normalized) return true;
1342
+ return false;
1343
+ };
1344
+ /**
1345
+ * Load coreference definition patterns from per-language
1346
+ * JSON configs in src/data/. Uses the
1347
+ * language manifest for auto-discovery.
1348
+ */
1349
+ const loadDefinitionPatterns = async () => {
1350
+ const patterns = [];
1351
+ const allRows = await loadLanguageConfigs("coreference", (mod) => {
1352
+ return mod.default ?? mod;
1353
+ });
1354
+ for (const rows of allRows) {
1355
+ if (!Array.isArray(rows)) {
1356
+ console.warn("[anonymize] coreference: unexpected config shape, skipping");
1357
+ continue;
1358
+ }
1359
+ for (const row of rows) try {
1360
+ patterns.push({ pattern: new RegExp(row.pattern, row.flags) });
1361
+ } catch (err) {
1362
+ console.warn(`[anonymize] coreference: invalid regex "${row.pattern}":`, err);
1363
+ }
1364
+ }
1365
+ return patterns;
1366
+ };
1367
+ /**
1368
+ * Load generic role terms that should NOT be treated
1369
+ * as PII coreferences. "Prodávající" (Seller),
1370
+ * "Kupující" (Buyer), etc. are legal roles, not
1371
+ * identifying information.
1372
+ */
1373
+ const getRoleStopSet = async (ctx) => {
1374
+ if (ctx.roleStopSet) return ctx.roleStopSet;
1375
+ if (ctx.roleStopSetPromise) return ctx.roleStopSetPromise;
1376
+ const promise = (async () => {
1377
+ let result;
1378
+ try {
1379
+ const mod = await import("./generic-roles.mjs");
1380
+ const data = mod.default ?? mod;
1381
+ result = new Set(data.roles.map((r) => r.toLowerCase()));
1382
+ } catch {
1383
+ result = /* @__PURE__ */ new Set();
1384
+ }
1385
+ ctx.roleStopSet = result;
1386
+ return result;
1387
+ })();
1388
+ ctx.roleStopSetPromise = promise;
1389
+ return promise;
1390
+ };
1391
+ const getDefinitionPatterns = async (ctx) => {
1392
+ if (ctx.corefPatterns) return ctx.corefPatterns;
1393
+ if (ctx.corefPatternsPromise) return ctx.corefPatternsPromise;
1394
+ ctx.corefPatternsPromise = loadDefinitionPatterns();
1395
+ const patterns = await ctx.corefPatternsPromise;
1396
+ if (patterns.length === 0) {
1397
+ ctx.corefPatterns = patterns;
1398
+ if (!ctx.corefLoadAttempted) {
1399
+ ctx.corefLoadAttempted = true;
1400
+ console.warn("[anonymize] coreference: no definition patterns loaded; coreference detection will be inactive");
1401
+ }
1402
+ return patterns;
1403
+ }
1404
+ ctx.corefPatterns = patterns;
1405
+ return patterns;
1406
+ };
1407
+ const SEARCH_WINDOW = 200;
1408
+ /**
1409
+ * Check whether an alias has textual similarity to
1410
+ * the source entity. Prevents roles and structural
1411
+ * terms from being treated as name aliases.
1412
+ *
1413
+ * Three checks (any passes → similar):
1414
+ * 1. Word overlap: a word in the alias appears in the
1415
+ * entity (case-insensitive, min 2 chars)
1416
+ * 2. Initials: alias letters match first letters of
1417
+ * entity words ("TB" ↔ "Tomas Bata")
1418
+ * 3. Substring: alias is a substring of the entity
1419
+ * or vice versa (min 3 chars)
1420
+ */
1421
+ const hasEntitySimilarity = (alias, entityText) => {
1422
+ const aliasLower = alias.toLowerCase();
1423
+ const entityLower = entityText.toLowerCase();
1424
+ if (aliasLower.length >= 3 && entityLower.includes(aliasLower)) return true;
1425
+ if (entityLower.length >= 3 && aliasLower.includes(entityLower)) return true;
1426
+ const aliasWords = aliasLower.split(/[\s.,;:'"()/-]+/).filter((w) => w.length >= 2);
1427
+ const entityWords = entityLower.split(/[\s.,;:'"()/-]+/).filter((w) => w.length >= 2);
1428
+ const entityWordSet = new Set(entityWords);
1429
+ for (const word of aliasWords) if (entityWordSet.has(word)) return true;
1430
+ if (/^[\p{Lu}]+$/u.test(alias) && alias.length >= 2 && alias.length <= entityWords.length) {
1431
+ for (let start = 0; start <= entityWords.length - alias.length; start++) if (entityWords.slice(start, start + alias.length).map((w) => w.charAt(0)).join("") === aliasLower) return true;
1432
+ }
1433
+ return false;
1434
+ };
1435
+ /**
1436
+ * Scan for defined-term patterns near known entities.
1437
+ *
1438
+ * Legal documents universally follow:
1439
+ * "Dr. Heinrich Muller (hereinafter 'the Seller')..."
1440
+ *
1441
+ * After NER detects the entity, this function scans for
1442
+ * definitional patterns within +/-200 chars and extracts
1443
+ * the alias. Returns alias + label pairs that can be added
1444
+ * to the gazetteer for a full-text re-scan.
1445
+ */
1446
+ /**
1447
+ * Labels that can be the source of a coreference alias.
1448
+ * Only parties (person, organization) have defined-term
1449
+ * aliases in legal text. Dates, addresses, IDs do not.
1450
+ */
1451
+ const COREF_SOURCE_LABELS = new Set(["person", "organization"]);
1452
+ const extractDefinedTerms = async (fullText, entities, ctx = defaultContext) => {
1453
+ const [definitionPatterns, roleStops] = await Promise.all([getDefinitionPatterns(ctx), getRoleStopSet(ctx)]);
1454
+ const terms = [];
1455
+ const seen = /* @__PURE__ */ new Set();
1456
+ const sorted = [...entities].sort((a, b) => a.start - b.start);
1457
+ for (const { pattern } of definitionPatterns) {
1458
+ pattern.lastIndex = 0;
1459
+ for (let match = pattern.exec(fullText); match !== null; match = pattern.exec(fullText)) {
1460
+ const alias = match[1]?.trim();
1461
+ if (!alias || alias.length < 2) continue;
1462
+ if (roleStops.has(alias.toLowerCase())) continue;
1463
+ if (isLegalFormAlias(alias)) continue;
1464
+ const defPos = match.index;
1465
+ let bestEntity = null;
1466
+ for (let i = sorted.length - 1; i >= 0; i--) {
1467
+ const e = sorted[i];
1468
+ if (e === void 0) continue;
1469
+ if (e.end > defPos) continue;
1470
+ if (defPos - e.end > SEARCH_WINDOW) break;
1471
+ if (!COREF_SOURCE_LABELS.has(e.label)) continue;
1472
+ bestEntity = e;
1473
+ break;
1474
+ }
1475
+ if (bestEntity === null) continue;
1476
+ const gapText = fullText.slice(bestEntity.end, defPos);
1477
+ if (/(?:;|\.(?=\s*(?:["'„‚(]*\p{Lu}|$)))/u.test(gapText)) continue;
1478
+ if (!hasEntitySimilarity(alias, bestEntity.text)) continue;
1479
+ const key = `${alias.toLowerCase()}::${bestEntity.label}`;
1480
+ if (seen.has(key)) continue;
1481
+ seen.add(key);
1482
+ terms.push({
1483
+ alias,
1484
+ label: bestEntity.label,
1485
+ definitionStart: defPos,
1486
+ sourceText: bestEntity.text
1487
+ });
1488
+ }
1489
+ }
1490
+ return terms;
1491
+ };
1492
+ /**
1493
+ * Check if a character is a Unicode word character
1494
+ * (letter, digit, or combining mark). Used for word
1495
+ * boundary checks in coreference matching.
1496
+ */
1497
+ const isWordChar = (ch) => {
1498
+ if (ch === void 0) return false;
1499
+ return /[\p{L}\p{M}\p{N}]/u.test(ch);
1500
+ };
1501
+ /**
1502
+ * Find all occurrences of defined-term aliases in the
1503
+ * full text. Returns Entity spans for each match.
1504
+ *
1505
+ * Respects word boundaries: "Kupující" must not match
1506
+ * inside "Kupujícímu". A match is valid only if the
1507
+ * character before the start and after the end are NOT
1508
+ * word characters (letter/digit).
1509
+ *
1510
+ * Populates `ctx.corefSourceMap` with entries linking
1511
+ * each coref entity to its source entity text, for
1512
+ * consistent placeholder numbering.
1513
+ */
1514
+ const findCoreferenceSpans = (fullText, terms, ctx = defaultContext) => {
1515
+ const results = [];
1516
+ for (const term of terms) {
1517
+ let searchFrom = 0;
1518
+ while (searchFrom < fullText.length) {
1519
+ const idx = fullText.indexOf(term.alias, searchFrom);
1520
+ if (idx === -1) break;
1521
+ const matchEnd = idx + term.alias.length;
1522
+ const charBefore = idx > 0 ? fullText[idx - 1] : void 0;
1523
+ const charAfter = fullText[matchEnd];
1524
+ if (isWordChar(charBefore) || isWordChar(charAfter)) {
1525
+ searchFrom = idx + 1;
1526
+ continue;
1527
+ }
1528
+ const entity = {
1529
+ start: idx,
1530
+ end: matchEnd,
1531
+ label: term.label,
1532
+ text: term.alias,
1533
+ score: .95,
1534
+ source: DETECTION_SOURCES.COREFERENCE
1535
+ };
1536
+ ctx.corefSourceMap.set(corefKey(entity), term.sourceText);
1537
+ results.push(entity);
1538
+ searchFrom = matchEnd;
1539
+ }
1540
+ }
1541
+ return results;
1542
+ };
1543
+ //#endregion
1544
+ //#region src/detectors/gazetteer.ts
1545
+ const MAX_EDIT_DISTANCE = 2;
1546
+ const MIN_FUZZY_LENGTH = 4;
1547
+ const MAX_PREFIX_OVERSHOOT = 7;
1548
+ /**
1549
+ * Collect all searchable strings (canonical + variants)
1550
+ * from gazetteer entries, mapped to their labels and
1551
+ * entry IDs.
1552
+ */
1553
+ const buildSearchTerms = (entries) => {
1554
+ const terms = /* @__PURE__ */ new Map();
1555
+ for (const entry of entries) {
1556
+ const meta = {
1557
+ label: entry.label,
1558
+ entryId: entry.id
1559
+ };
1560
+ terms.set(entry.canonical, meta);
1561
+ for (const variant of entry.variants) terms.set(variant, meta);
1562
+ }
1563
+ return terms;
1564
+ };
1565
+ /**
1566
+ * Build TextSearch-compatible patterns from gazetteer
1567
+ * entries. Returns:
1568
+ * - Exact literal patterns for all terms
1569
+ * - Fuzzy patterns (distance: 2) for terms >= 4 chars
1570
+ * - Parallel metadata arrays for post-processing
1571
+ *
1572
+ * Patterns are ordered: all exact first, then all
1573
+ * fuzzy. The isFuzzy array marks which are which.
1574
+ */
1575
+ const buildGazetteerPatterns = (entries) => {
1576
+ const terms = buildSearchTerms(entries);
1577
+ const patterns = [];
1578
+ const labels = [];
1579
+ const isFuzzy = [];
1580
+ for (const [term, meta] of terms) {
1581
+ patterns.push({
1582
+ pattern: term,
1583
+ literal: true,
1584
+ wholeWords: false
1585
+ });
1586
+ labels.push(meta.label);
1587
+ isFuzzy.push(false);
1588
+ }
1589
+ for (const [term, meta] of terms) {
1590
+ if (term.length < MIN_FUZZY_LENGTH) continue;
1591
+ patterns.push({
1592
+ pattern: term,
1593
+ distance: MAX_EDIT_DISTANCE
1594
+ });
1595
+ labels.push(meta.label);
1596
+ isFuzzy.push(true);
1597
+ }
1598
+ return {
1599
+ patterns,
1600
+ data: {
1601
+ labels,
1602
+ isFuzzy
1603
+ }
1604
+ };
1605
+ };
1606
+ /**
1607
+ * Process gazetteer matches from the unified literal
1608
+ * search. Receives all matches; filters to the
1609
+ * gazetteer slice via sliceStart/sliceEnd.
1610
+ *
1611
+ * Exact matches get score 0.9; fuzzy matches get
1612
+ * 0.85. Fuzzy matches that overlap an exact match
1613
+ * are dropped.
1614
+ *
1615
+ * For exact matches, attempts prefix extension for
1616
+ * legal suffixes ("a.s.", "GmbH", "s.r.o." after
1617
+ * the matched term).
1618
+ */
1619
+ const processGazetteerMatches = (allMatches, sliceStart, sliceEnd, fullText, data) => {
1620
+ const results = [];
1621
+ const exactSpans = [];
1622
+ for (const match of allMatches) {
1623
+ const idx = match.pattern;
1624
+ if (idx < sliceStart || idx >= sliceEnd) continue;
1625
+ const localIdx = idx - sliceStart;
1626
+ if (data.isFuzzy[localIdx]) continue;
1627
+ const label = data.labels[localIdx];
1628
+ if (!label) continue;
1629
+ const extended = tryPrefixExtension(fullText, match.start, match.end);
1630
+ const end = extended?.end ?? match.end;
1631
+ const text = extended?.text ?? fullText.slice(match.start, match.end);
1632
+ exactSpans.push({
1633
+ start: match.start,
1634
+ end
1635
+ });
1636
+ results.push({
1637
+ start: match.start,
1638
+ end,
1639
+ label,
1640
+ text,
1641
+ score: .9,
1642
+ source: DETECTION_SOURCES.GAZETTEER
1643
+ });
1644
+ }
1645
+ for (const match of allMatches) {
1646
+ const idx = match.pattern;
1647
+ if (idx < sliceStart || idx >= sliceEnd) continue;
1648
+ const localIdx = idx - sliceStart;
1649
+ if (!data.isFuzzy[localIdx]) continue;
1650
+ if (match.distance === 0) continue;
1651
+ const label = data.labels[localIdx];
1652
+ if (!label) continue;
1653
+ if (exactSpans.some((e) => match.start < e.end && match.end > e.start)) continue;
1654
+ const matchText = fullText.slice(match.start, match.end);
1655
+ results.push({
1656
+ start: match.start,
1657
+ end: match.end,
1658
+ label,
1659
+ text: matchText,
1660
+ score: .85,
1661
+ source: DETECTION_SOURCES.GAZETTEER
1662
+ });
1663
+ }
1664
+ return results;
1665
+ };
1666
+ /**
1667
+ * Try to extend an exact match to capture one
1668
+ * trailing token (max 6 chars) that may be a legal
1669
+ * entity suffix (e.g., "a.s.", "GmbH", "s.r.o.").
1670
+ *
1671
+ * Does not validate the token against a legal-forms
1672
+ * list; false extensions are filtered by mergeAndDedup
1673
+ * when a legal-form detector produces a competing
1674
+ * entity with the correct span.
1675
+ */
1676
+ const tryPrefixExtension = (fullText, start, end) => {
1677
+ const maxEnd = Math.min(end + MAX_PREFIX_OVERSHOOT, fullText.length);
1678
+ if (maxEnd <= end + 1) return null;
1679
+ const after = fullText.slice(end, maxEnd);
1680
+ if (!after.startsWith(" ")) return null;
1681
+ const nextSpace = after.indexOf(" ", 1);
1682
+ const suffixEnd = nextSpace !== -1 ? nextSpace : after.length;
1683
+ if (suffixEnd <= 1) return null;
1684
+ const newEnd = end + suffixEnd;
1685
+ return {
1686
+ end: newEnd,
1687
+ text: fullText.slice(start, newEnd)
1688
+ };
1689
+ };
1690
+ //#endregion
1691
+ //#region src/util/text.ts
1692
+ /**
1693
+ * Shared text utilities for detectors.
1694
+ *
1695
+ * Extracted from names.ts and deny-list.ts to avoid
1696
+ * duplicating regex constants and helper functions.
1697
+ */
1698
+ /** Matches a string that starts with an uppercase letter. */
1699
+ const UPPER_START_RE = /^\p{Lu}/u;
1700
+ /** Matches a string consisting entirely of uppercase letters. */
1701
+ const ALL_UPPER_RE = /^\p{Lu}+$/u;
1702
+ const SENTENCE_END_RE = /[.!?]/;
1703
+ /**
1704
+ * Detect whether a position is at the start of a sentence.
1705
+ * Looks backward past whitespace for sentence-ending
1706
+ * punctuation (.!?). Position 0 and positions preceded
1707
+ * only by whitespace are considered sentence starts.
1708
+ */
1709
+ const isSentenceStart = (text, pos) => {
1710
+ if (pos === 0) return true;
1711
+ let i = pos - 1;
1712
+ while (i >= 0 && /\s/.test(text[i] ?? "")) i--;
1713
+ if (i < 0) return true;
1714
+ return SENTENCE_END_RE.test(text[i] ?? "");
1715
+ };
1716
+ //#endregion
1717
+ //#region src/detectors/names.ts
1718
+ const getCorpus = (ctx) => ctx.nameCorpus;
1719
+ const getNameCorpusFirstNames = (ctx = defaultContext) => ctx.nameCorpus?.firstNamesList ?? [];
1720
+ const getNameCorpusSurnames = (ctx = defaultContext) => ctx.nameCorpus?.surnamesList ?? [];
1721
+ const getNameCorpusTitles = (ctx = defaultContext) => ctx.nameCorpus?.titlesList ?? [];
1722
+ /**
1723
+ * Load name corpus data from injected dictionaries
1724
+ * and legacy config files. Merges all sources.
1725
+ *
1726
+ * Safe to call multiple times; only loads once per
1727
+ * context. Must be called before detectNameCorpus or
1728
+ * the getNameCorpus*() accessors are used.
1729
+ *
1730
+ * @param dictionaries Optional pre-loaded dictionaries
1731
+ * with per-language first names and surnames. When
1732
+ * omitted, only legacy config files are used.
1733
+ */
1734
+ const initNameCorpus = (ctx = defaultContext, dictionaries) => {
1735
+ if (ctx.nameCorpusPromise) return ctx.nameCorpusPromise;
1736
+ const promise = (async () => {
1737
+ try {
1738
+ const [legacyFirstMod, legacySurnameMod, titleMod, exclusionMod] = await Promise.all([
1739
+ import("./names-first.mjs"),
1740
+ import("./names-surnames.mjs"),
1741
+ import("./names-title-tokens.mjs"),
1742
+ import("./names-exclusions.mjs")
1743
+ ]);
1744
+ const firstNames = [...legacyFirstMod.default.names];
1745
+ if (dictionaries?.firstNames) for (const names of Object.values(dictionaries.firstNames)) for (const name of names) firstNames.push(name);
1746
+ const surnames = [...legacySurnameMod.default.names];
1747
+ if (dictionaries?.surnames) for (const names of Object.values(dictionaries.surnames)) for (const name of names) surnames.push(name);
1748
+ const dedup = (arr) => {
1749
+ const seen = /* @__PURE__ */ new Set();
1750
+ const result = [];
1751
+ for (const item of arr) {
1752
+ if (seen.has(item)) continue;
1753
+ seen.add(item);
1754
+ result.push(item);
1546
1755
  }
1547
- ]
1548
- },
1549
- "underscore": {
1550
- "description": "Underscore variants",
1551
- "chars": [{
1552
- "char": "_",
1553
- "name": "low line",
1554
- "code": "U+005F"
1555
- }, {
1556
- "char": "_",
1557
- "name": "full-width low line",
1558
- "code": "U+FF3F"
1559
- }]
1756
+ return result;
1757
+ };
1758
+ const dedupFirst = dedup(firstNames);
1759
+ const dedupSurnames = dedup(surnames);
1760
+ const titles = titleMod.default.tokens;
1761
+ const exclusions = exclusionMod.default.words;
1762
+ ctx.nameCorpus = {
1763
+ firstNames: Object.freeze(new Set(dedupFirst)),
1764
+ surnames: Object.freeze(new Set(dedupSurnames)),
1765
+ titleTokens: Object.freeze(new Set(titles)),
1766
+ excludedWords: Object.freeze(new Set(exclusions)),
1767
+ firstNamesList: Object.freeze(dedupFirst),
1768
+ surnamesList: Object.freeze(dedupSurnames),
1769
+ titlesList: Object.freeze(titles),
1770
+ excludedList: Object.freeze(exclusions)
1771
+ };
1772
+ } catch (err) {
1773
+ ctx.nameCorpusPromise = null;
1774
+ console.warn("[anonymize] Failed to load name corpus JSON — name detection disabled:", err);
1775
+ }
1776
+ })();
1777
+ ctx.nameCorpusPromise = promise;
1778
+ return promise;
1779
+ };
1780
+ const INFLECTION_SUFFIXES = [
1781
+ "ovi",
1782
+ "em",
1783
+ "om",
1784
+ "ou",
1785
+ "é",
1786
+ "a",
1787
+ "u"
1788
+ ];
1789
+ /**
1790
+ * Strip common Czech/Slovak case suffixes from a token.
1791
+ * Returns candidate base forms if stripping produces a
1792
+ * plausible name (capitalised, length >= 3).
1793
+ *
1794
+ * For the "-ou" instrumental feminine suffix, also yields
1795
+ * base + "a" (e.g., "Editou" → "Edit" and "Edita")
1796
+ * because Czech feminine names decline -a → -ou.
1797
+ */
1798
+ const stripInflection = (token) => {
1799
+ const candidates = [];
1800
+ for (const suffix of INFLECTION_SUFFIXES) if (token.length > suffix.length + 2 && token.endsWith(suffix)) {
1801
+ const base = token.slice(0, -suffix.length);
1802
+ if (/^\p{Lu}/u.test(base)) {
1803
+ candidates.push(base);
1804
+ if (suffix === "ou" || suffix === "é" || suffix === "u") candidates.push(`${base}a`);
1560
1805
  }
1561
1806
  }
1807
+ return candidates;
1808
+ };
1809
+ const TOKEN_TYPE = {
1810
+ NAME: "name",
1811
+ SURNAME: "surname",
1812
+ TITLE: "title",
1813
+ ABBREVIATION: "abbreviation",
1814
+ CAPITALIZED: "capitalized",
1815
+ OTHER: "other"
1816
+ };
1817
+ const PERSON_CHAIN_BREAK_RE$1 = /[!?;:]/u;
1818
+ const isInitialContinuationGap$1 = (text, gap) => /^\p{Lu}$/u.test(text) && /^\.[^\S\n]{1,2}$/u.test(gap) || /^[^\S\n]{1,2}(?:\p{Lu}\.[^\S\n]{1,2})+$/u.test(gap);
1819
+ /**
1820
+ * Check if a token is in the first-name set, either
1821
+ * directly or after stripping Czech/Slovak inflection.
1822
+ */
1823
+ const isFirstNameToken = (token, corpus) => {
1824
+ if (corpus.firstNames.has(token)) return true;
1825
+ return stripInflection(token).some((b) => corpus.firstNames.has(b));
1826
+ };
1827
+ /**
1828
+ * Check if a token is in the surname set, either
1829
+ * directly or after stripping Czech/Slovak inflection.
1830
+ */
1831
+ const isSurnameToken = (token, corpus) => {
1832
+ if (corpus.surnames.has(token)) return true;
1833
+ return stripInflection(token).some((b) => corpus.surnames.has(b));
1834
+ };
1835
+ /**
1836
+ * Check if a token looks like a single-letter
1837
+ * abbreviation: "J.", "M.", etc.
1838
+ */
1839
+ const isAbbreviation = (token) => token.length === 2 && /^\p{Lu}$/u.test(token[0] ?? "") && token[1] === ".";
1840
+ const segmenter$1 = new Intl.Segmenter(void 0, { granularity: "word" });
1841
+ /**
1842
+ * Split text into word segments using Intl.Segmenter.
1843
+ * Only returns segments flagged as words.
1844
+ */
1845
+ const segmentWords = (fullText) => {
1846
+ const words = [];
1847
+ for (const seg of segmenter$1.segment(fullText)) if (seg.isWordLike) words.push({
1848
+ text: seg.segment,
1849
+ start: seg.index,
1850
+ end: seg.index + seg.segment.length
1851
+ });
1852
+ return words;
1853
+ };
1854
+ /** NAME or SURNAME — both represent corpus-matched tokens */
1855
+ const isCorpusMatch = (type) => type === TOKEN_TYPE.NAME || type === TOKEN_TYPE.SURNAME;
1856
+ const classifyToken = (word, corpus) => {
1857
+ const { text, start, end } = word;
1858
+ const lower = text.toLowerCase();
1859
+ const stripped = text.endsWith(".") ? text.slice(0, -1).toLowerCase() : lower;
1860
+ if (corpus.titleTokens.has(stripped)) return {
1861
+ text,
1862
+ type: TOKEN_TYPE.TITLE,
1863
+ start,
1864
+ end
1865
+ };
1866
+ if (isAbbreviation(text)) return {
1867
+ text,
1868
+ type: TOKEN_TYPE.ABBREVIATION,
1869
+ start,
1870
+ end
1871
+ };
1872
+ if (corpus.excludedWords.has(lower)) return {
1873
+ text,
1874
+ type: TOKEN_TYPE.OTHER,
1875
+ start,
1876
+ end
1877
+ };
1878
+ if (text.length < 3) return {
1879
+ text,
1880
+ type: TOKEN_TYPE.OTHER,
1881
+ start,
1882
+ end
1883
+ };
1884
+ if (text.length > 3 && ALL_UPPER_RE.test(text)) return {
1885
+ text,
1886
+ type: TOKEN_TYPE.OTHER,
1887
+ start,
1888
+ end
1889
+ };
1890
+ if (!UPPER_START_RE.test(text)) return {
1891
+ text,
1892
+ type: TOKEN_TYPE.OTHER,
1893
+ start,
1894
+ end
1895
+ };
1896
+ if (isFirstNameToken(text, corpus)) return {
1897
+ text,
1898
+ type: TOKEN_TYPE.NAME,
1899
+ start,
1900
+ end
1901
+ };
1902
+ if (isSurnameToken(text, corpus)) return {
1903
+ text,
1904
+ type: TOKEN_TYPE.SURNAME,
1905
+ start,
1906
+ end
1907
+ };
1908
+ return {
1909
+ text,
1910
+ type: TOKEN_TYPE.CAPITALIZED,
1911
+ start,
1912
+ end
1913
+ };
1562
1914
  };
1563
- //#endregion
1564
- //#region src/util/char-groups.ts
1565
1915
  /**
1566
- * Centralized Unicode character equivalence groups.
1567
- *
1568
- * Centralized Unicode character equivalence groups.
1916
+ * Detect person names by looking up tokens against the
1917
+ * name corpus, then chaining adjacent name-like tokens.
1569
1918
  *
1570
- * Provides helpers to build regex character classes that
1571
- * match all typographic variants of a character type
1572
- * (dashes, spaces, quotes, etc.).
1919
+ * Requires initNameCorpus() to have been called first.
1920
+ * If not initialized, returns an empty array.
1573
1921
  *
1574
- * The JSON is statically imported so the bundler inlines
1575
- * it, avoiding a runtime require() that breaks browsers.
1922
+ * Scoring:
1923
+ * TITLE + NAME/SURNAME → 0.95
1924
+ * NAME + NAME/SURNAME → 0.9
1925
+ * SURNAME + NAME/SURNAME → 0.9
1926
+ * NAME + CAPITALIZED → 0.7
1927
+ * ABBREVIATION + NAME → 0.7
1928
+ * Standalone NAME → 0.5 (low confidence)
1929
+ * Standalone SURNAME → skip (too ambiguous)
1576
1930
  */
1577
- /** Chars that need escaping inside a regex char class. */
1578
- const REGEX_CLASS_SPECIAL = /[\\\]^-]/;
1579
- const escapeForCharClass = (ch) => REGEX_CLASS_SPECIAL.test(ch) ? `\\${ch}` : ch;
1580
- const config = char_groups_default;
1931
+ const detectNameCorpus = (fullText, ctx = defaultContext) => {
1932
+ const corpus = getCorpus(ctx);
1933
+ if (!corpus) return [];
1934
+ const tokens = segmentWords(fullText).map((w) => classifyToken(w, corpus));
1935
+ const entities = [];
1936
+ const consumed = /* @__PURE__ */ new Set();
1937
+ for (let i = 0; i < tokens.length; i++) {
1938
+ if (consumed.has(i)) continue;
1939
+ const token = tokens[i];
1940
+ if (!token) continue;
1941
+ if (token.type !== TOKEN_TYPE.TITLE && token.type !== TOKEN_TYPE.NAME && token.type !== TOKEN_TYPE.SURNAME && token.type !== TOKEN_TYPE.ABBREVIATION) continue;
1942
+ const MAX_CHAIN = 5;
1943
+ const chain = [token];
1944
+ let j = i + 1;
1945
+ while (j < tokens.length && chain.length < MAX_CHAIN) {
1946
+ const next = tokens[j];
1947
+ if (!next) break;
1948
+ const prev = chain.at(-1);
1949
+ if (prev) {
1950
+ const gap = fullText.slice(prev.end, next.start);
1951
+ const breaksOnPeriod = gap.includes(".") && !isInitialContinuationGap$1(prev.text, gap);
1952
+ if (gap.includes("\n") || PERSON_CHAIN_BREAK_RE$1.test(gap) || breaksOnPeriod) break;
1953
+ }
1954
+ if (next.type === TOKEN_TYPE.NAME || next.type === TOKEN_TYPE.SURNAME || next.type === TOKEN_TYPE.TITLE || next.type === TOKEN_TYPE.ABBREVIATION || next.type === TOKEN_TYPE.CAPITALIZED) {
1955
+ chain.push(next);
1956
+ j++;
1957
+ } else break;
1958
+ }
1959
+ const hasTitle = chain.some((t) => t.type === TOKEN_TYPE.TITLE);
1960
+ const hasCorpusName = chain.some((t) => isCorpusMatch(t.type));
1961
+ const hasFirstName = chain.some((t) => t.type === TOKEN_TYPE.NAME);
1962
+ const hasAbbreviation = chain.some((t) => t.type === TOKEN_TYPE.ABBREVIATION);
1963
+ const corpusCount = chain.filter((t) => isCorpusMatch(t.type)).length;
1964
+ const capitalizedCount = chain.filter((t) => t.type === TOKEN_TYPE.CAPITALIZED).length;
1965
+ let score = 0;
1966
+ if (hasTitle && hasCorpusName) score = .95;
1967
+ else if (corpusCount >= 2) score = .9;
1968
+ else if (hasCorpusName && capitalizedCount > 0) score = .7;
1969
+ else if (hasAbbreviation && hasCorpusName) score = .7;
1970
+ else if (hasFirstName && chain.length === 1) {
1971
+ if (isSentenceStart(fullText, token.start)) continue;
1972
+ score = .5;
1973
+ } else if (!hasFirstName && chain.length === 1 && chain[0]?.type === TOKEN_TYPE.SURNAME) continue;
1974
+ else if (hasTitle && chain.length === 1) continue;
1975
+ else {
1976
+ if (!hasCorpusName) continue;
1977
+ score = .5;
1978
+ }
1979
+ const first = chain.at(0);
1980
+ const last = chain.at(-1);
1981
+ if (!first || !last) continue;
1982
+ const start = first.start;
1983
+ const end = last.end;
1984
+ const text = fullText.slice(start, end);
1985
+ for (let k = i; k < i + chain.length; k++) consumed.add(k);
1986
+ entities.push({
1987
+ start,
1988
+ end,
1989
+ label: "person",
1990
+ text,
1991
+ score,
1992
+ source: DETECTION_SOURCES.REGEX
1993
+ });
1994
+ }
1995
+ return entities;
1996
+ };
1997
+ //#endregion
1998
+ //#region src/config/titles.ts
1581
1999
  /**
1582
- * Get the raw characters for a named group.
1583
- * Throws if the group does not exist.
2000
+ * Academic and professional title prefixes.
2001
+ * Plain text; the detector auto-escapes for regex.
2002
+ * Sorted longest-first at build time.
1584
2003
  */
1585
- const charSet = (group) => {
1586
- const g = config.groups[group];
1587
- if (!g) throw new Error(`Unknown char group: "${group}"`);
1588
- return g.chars.map((entry) => entry.char);
1589
- };
2004
+ const TITLE_PREFIXES = [
2005
+ "Ing.",
2006
+ "Mgr.",
2007
+ "MgA.",
2008
+ "Bc.",
2009
+ "BcA.",
2010
+ "JUDr.",
2011
+ "MUDr.",
2012
+ "MVDr.",
2013
+ "MDDr.",
2014
+ "PhDr.",
2015
+ "RNDr.",
2016
+ "PaedDr.",
2017
+ "ThDr.",
2018
+ "ThLic.",
2019
+ "ICDr.",
2020
+ "RSDr.",
2021
+ "PharmDr.",
2022
+ "artD.",
2023
+ "akad.",
2024
+ "doc.",
2025
+ "prof.",
2026
+ "ao. Univ.-Prof.",
2027
+ "o. Univ.-Prof.",
2028
+ "Univ.-Prof.",
2029
+ "Hon.-Prof.",
2030
+ "em. Prof.",
2031
+ "Dr. med. dent.",
2032
+ "Dr. med. vet.",
2033
+ "Dr. med.",
2034
+ "Dr. rer. nat.",
2035
+ "Dr. rer. soc.",
2036
+ "Dr. rer. pol.",
2037
+ "Dr. sc. tech.",
2038
+ "Dr. sc. nat.",
2039
+ "Dr. sc. hum.",
2040
+ "Dr. iur.",
2041
+ "Dr. jur.",
2042
+ "Dr. theol.",
2043
+ "Dr. oec.",
2044
+ "Dr. techn.",
2045
+ "Dr. h. c.",
2046
+ "Dr. phil.",
2047
+ "Dr.-Ing.",
2048
+ "Dr. Ing.",
2049
+ "Dr.",
2050
+ "Dipl.-Wirt.-Ing.",
2051
+ "Dipl.-Betriebsw.",
2052
+ "Dipl.-Inform.",
2053
+ "Dipl.-Volksw.",
2054
+ "Dipl.-Psych.",
2055
+ "Dipl.-Phys.",
2056
+ "Dipl.-Chem.",
2057
+ "Dipl.-Biol.",
2058
+ "Dipl.-Math.",
2059
+ "Dipl.-Päd.",
2060
+ "Dipl.-Soz.",
2061
+ "Dipl.-Kfm.",
2062
+ "Dipl.-Jur.",
2063
+ "Dipl. Ing.",
2064
+ "Dipl.-Ing.",
2065
+ "Mag. rer. soc. oec.",
2066
+ "Mag. rer. nat.",
2067
+ "Mag. phil.",
2068
+ "Mag. iur.",
2069
+ "Mag. arch.",
2070
+ "Mag. pharm.",
2071
+ "Mag. (FH)",
2072
+ "Mag.",
2073
+ "Bakk. rer. nat.",
2074
+ "Bakk. techn.",
2075
+ "Bakk. phil.",
2076
+ "Bakk.",
2077
+ "Lic. phil.",
2078
+ "Lic. iur.",
2079
+ "Lic. oec.",
2080
+ "Lic. theol.",
2081
+ "Lic.",
2082
+ "Priv.-Doz.",
2083
+ "PD",
2084
+ "RA"
2085
+ ];
1590
2086
  /**
1591
- * Build a regex character class string for a named
1592
- * group. E.g., charClass("dash") returns a string
1593
- * like "[-\u2013\u2014\u2010\u2011\u2212\u2043]".
1594
- *
1595
- * The hyphen-minus is placed first so it is treated
1596
- * as a literal, not a range indicator.
2087
+ * Courtesy/honorific titles that precede a person's
2088
+ * name. Sorted alphabetically. The detector escapes
2089
+ * dots and adds \b for entries in HONORIFIC_BOUNDARY.
1597
2090
  */
1598
- const charClass = (group) => {
1599
- return `[${[...charSet(group)].sort((a, b) => {
1600
- if (a === "-") return -1;
1601
- if (b === "-") return 1;
1602
- return a.localeCompare(b);
1603
- }).map(escapeForCharClass).join("")}]`;
1604
- };
2091
+ const HONORIFICS = [
2092
+ "Avv.",
2093
+ "Dame",
2094
+ "Doamna",
2095
+ "Domnul",
2096
+ "Don",
2097
+ "Doña",
2098
+ "Dott.",
2099
+ "Judge",
2100
+ "Justice",
2101
+ "Lady",
2102
+ "Lord",
2103
+ "M.",
2104
+ "Madame",
2105
+ "Mademoiselle",
2106
+ "Maître",
2107
+ "Me",
2108
+ "Messrs",
2109
+ "Miss",
2110
+ "Mlle",
2111
+ "Mme",
2112
+ "Monsieur",
2113
+ "Mr",
2114
+ "Mrs",
2115
+ "Ms",
2116
+ "Pr",
2117
+ "Pr.",
2118
+ "President",
2119
+ "Señor",
2120
+ "Señora",
2121
+ "Sig.",
2122
+ "Sig.ra",
2123
+ "Signor",
2124
+ "Signora",
2125
+ "Signorina",
2126
+ "Sir",
2127
+ "Sr.",
2128
+ "Sra."
2129
+ ];
1605
2130
  /**
1606
- * Return the inner content of a character class (without
1607
- * the surrounding brackets). Useful for embedding a group
1608
- * inside a larger character class, e.g.:
1609
- * `[\\s&,.${charClassInner("dash")}]`
2131
+ * Honorifics that need \b word-boundary anchors
2132
+ * (short or common words that could match mid-word).
1610
2133
  */
1611
- const charClassInner = (group) => {
1612
- return [...charSet(group)].sort((a, b) => {
1613
- if (a === "-") return -1;
1614
- if (b === "-") return 1;
1615
- return a.localeCompare(b);
1616
- }).map(escapeForCharClass).join("");
1617
- };
1618
- /** Regex char class matching all dash variants. */
1619
- const DASH = charClass("dash");
2134
+ const HONORIFIC_BOUNDARY = new Set([
2135
+ "Don",
2136
+ "Doña",
2137
+ "M.",
2138
+ "Me",
2139
+ "Pr",
2140
+ "Pr.",
2141
+ "Señor",
2142
+ "Señora"
2143
+ ]);
1620
2144
  /**
1621
- * Inner content of the dash char class (no brackets).
1622
- * For embedding inside a larger character class:
1623
- * `[\\s&,.${DASH_INNER}]`
2145
+ * Post-nominal degrees (comma or space separated after name).
2146
+ * Plain text; the detector auto-escapes for regex.
1624
2147
  */
1625
- const DASH_INNER = charClassInner("dash");
1626
- charClass("space");
1627
- charClass("quote-double");
1628
- charClass("quote-single");
1629
- charClass("slash");
1630
- charClass("dot");
1631
- charClass("colon");
2148
+ const POST_NOMINALS = [
2149
+ "Ph.D.",
2150
+ "Ph.D",
2151
+ "CSc.",
2152
+ "DrSc.",
2153
+ "ArtD.",
2154
+ "D.Phil.",
2155
+ "DPhil.",
2156
+ "MPhil.",
2157
+ "MBA",
2158
+ "MPA",
2159
+ "LL.M.",
2160
+ "LL.B.",
2161
+ "M.Sc.",
2162
+ "B.Sc.",
2163
+ "MSc.",
2164
+ "BSc.",
2165
+ "M.Eng.",
2166
+ "B.Eng.",
2167
+ "M.A.",
2168
+ "B.A.",
2169
+ "JCD",
2170
+ "JD",
2171
+ "DiS.",
2172
+ "ACCA",
2173
+ "FCCA",
2174
+ "CIPM",
2175
+ "CIPT",
2176
+ "CIPP/E",
2177
+ "CIPP"
2178
+ ];
1632
2179
  //#endregion
1633
2180
  //#region src/detectors/regex.ts
1634
2181
  const MIN_PHONE_LENGTH = 7;
1635
2182
  const MIN_MONTH_NAME_LENGTH = 3;
1636
2183
  const escapeTitle = (title) => title.replace(/[.*+?^${}()|[\]\\]/g, "\\$&").replace(/\s+/g, "\\s*");
1637
2184
  /** Escape for use inside a regex alternation. */
1638
- const escapeRegex$1 = (s) => s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
2185
+ const escapeRegex$2 = (s) => s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
1639
2186
  /** Escape for use inside a regex character class. */
1640
2187
  const escapeCharClass = (s) => s.replace(/[\]\\^-]/g, "\\$&");
1641
2188
  const TITLE_PREFIX = TITLE_PREFIXES.toSorted((a, b) => b.length - a.length).map(escapeTitle).join("|");
1642
2189
  const POST_NOMINAL = POST_NOMINALS.toSorted((a, b) => b.length - a.length).map(escapeTitle).join("|");
1643
- const NAME_WORD = `[A-ZÁČĎÉĚÍŇÓŘŠŤÚŮÝŽ][a-záčďéěíňóřšťúůýžäöüß]+`;
2190
+ const NAME_WORD = `\\p{Lu}\\p{Ll}+`;
1644
2191
  const PARTICLE = "(?:van der|van den|de la|della|von|van|dos|ibn|ben|bin|del|zum|zur|ten|ter|da|de|di|al|el|le|la|zu|af|av)";
1645
2192
  const SP = "[^\\S\\n\\t]";
1646
2193
  /** Honorific alternation built from titles.ts config. */
1647
2194
  const HONORIFIC_ALT = [...HONORIFICS].toSorted((a, b) => b.length - a.length).map((h) => {
1648
- const escaped = escapeRegex$1(h);
2195
+ const escaped = escapeRegex$2(h);
1649
2196
  return HONORIFIC_BOUNDARY.has(h) ? `\\b${escaped}` : escaped;
1650
2197
  }).join("|");
1651
2198
  const toEntry = (validator, label, score) => {
@@ -1670,8 +2217,6 @@ const toEntry = (validator, label, score) => {
1670
2217
  const STDNUM_ENTRIES = [
1671
2218
  toEntry(hu.vat, "tax identification number", .95),
1672
2219
  toEntry(it.codiceFiscale, "national identification number", .95),
1673
- toEntry(es.dni, "national identification number", .9),
1674
- toEntry(es.nie, "national identification number", .95),
1675
2220
  toEntry(se.personnummer, "national identification number", .9),
1676
2221
  toEntry(ro.cnp, "national identification number", .95),
1677
2222
  toEntry(fr.nir, "social security number", .9),
@@ -1697,7 +2242,6 @@ const STDNUM_ENTRIES = [
1697
2242
  toEntry(fi.ytunnus, "registration number", .9),
1698
2243
  toEntry(bg.vat, "tax identification number", .95),
1699
2244
  toEntry(sk.dic, "tax identification number", .95),
1700
- toEntry(es.cif, "registration number", .95),
1701
2245
  toEntry(es.vat, "tax identification number", .95),
1702
2246
  toEntry(es.nss, "social security number", .9),
1703
2247
  toEntry(fr.tva, "tax identification number", .95),
@@ -1719,7 +2263,9 @@ const STDNUM_ENTRIES = [
1719
2263
  toEntry(ee.ik, "national identification number", .9),
1720
2264
  toEntry(cy.vat, "tax identification number", .95),
1721
2265
  toEntry(mt.vat, "tax identification number", .95),
1722
- toEntry(lu.vat, "tax identification number", .95)
2266
+ toEntry(lu.vat, "tax identification number", .95),
2267
+ toEntry(br.cpf, "tax identification number", .95),
2268
+ toEntry(br.cnpj, "tax identification number", .95)
1723
2269
  ].filter((e) => e !== null);
1724
2270
  const TITLED_PERSON = {
1725
2271
  pattern: `(?:${TITLE_PREFIX})(?:${SP}+(?:${TITLE_PREFIX}))*${SP}+(?:${NAME_WORD})(?:${SP}{1,4}(?:${PARTICLE}${SP}+)?${NAME_WORD}){1,3}(?:,?${SP}+(?:${POST_NOMINAL})(?:,?${SP}+(?:${POST_NOMINAL}))*)?`,
@@ -1768,7 +2314,7 @@ const CREDIT_CARD = {
1768
2314
  };
1769
2315
  const CZ_BIRTH_NUMBER = {
1770
2316
  pattern: `\\b\\d{6}/\\d{3,4}\\b`,
1771
- label: "czech birth number",
2317
+ label: "birth number",
1772
2318
  score: 1,
1773
2319
  validator: cz.rc
1774
2320
  };
@@ -1802,6 +2348,51 @@ const CZ_POSTAL = {
1802
2348
  label: "address",
1803
2349
  score: .7
1804
2350
  };
2351
+ const ES_POSTAL = {
2352
+ pattern: "\\b(?:C\\.?P\\.?|[Cc][óo]digo[^\\S\\n]+postal)[^\\S\\n]{0,3}:?[^\\S\\n]{0,3}\\d{5}\\b",
2353
+ label: "address",
2354
+ score: .7
2355
+ };
2356
+ const ES_DNI = {
2357
+ pattern: `\\b\\d{8}-?[A-Za-z]\\b`,
2358
+ label: "national identification number",
2359
+ score: .9,
2360
+ validator: es.dni
2361
+ };
2362
+ const ES_NIE = {
2363
+ pattern: `\\b[XYZxyz]-?\\d{7}-?[A-Za-z]\\b`,
2364
+ label: "national identification number",
2365
+ score: .95,
2366
+ validator: es.nie
2367
+ };
2368
+ const ES_CIF = {
2369
+ pattern: `\\b[A-HJNP-SUVWa-hjnp-suvw]-?\\d{7}-?[0-9A-Ja-j]\\b`,
2370
+ label: "registration number",
2371
+ score: .95,
2372
+ validator: es.cif
2373
+ };
2374
+ const BR_CPF_FORMATTED = {
2375
+ pattern: `\\b\\d{3}\\.\\d{3}\\.\\d{3}${DASH}\\d{2}\\b`,
2376
+ label: "tax identification number",
2377
+ score: .95,
2378
+ validator: br.cpf
2379
+ };
2380
+ const BR_CNPJ_FORMATTED = {
2381
+ pattern: `\\b\\d{2}\\.\\d{3}\\.\\d{3}/\\d{4}${DASH}\\d{2}\\b`,
2382
+ label: "tax identification number",
2383
+ score: .95,
2384
+ validator: br.cnpj
2385
+ };
2386
+ const BR_RG_WITH_SSP = {
2387
+ pattern: `\\b\\d{1,3}\\.?\\d{3}\\.?\\d{3}(?:${DASH}[0-9A-Za-z])?[^\\S\\n]+SSP(?:/[A-Z]{2})?\\b`,
2388
+ label: "national identification number",
2389
+ score: .95
2390
+ };
2391
+ const BR_OAB = {
2392
+ pattern: "\\bOAB/[A-Z]{2}[^\\S\\n]+(?:n[º°.][^\\S\\n]*)?(?:\\d{1,3}(?:\\.\\d{3})+|\\d{4,6})\\b",
2393
+ label: "registration number",
2394
+ score: .95
2395
+ };
1805
2396
  const URL = {
1806
2397
  pattern: "(?:https?://|https?:(?=[^\\s])|www\\.)[\\w\\-]+(?:\\.[\\w\\-]+)+(?::\\d+)?(?:[/?#][^\\s)\\]>]*[^\\s.,;:!?)\\]>])?",
1807
2398
  label: "url",
@@ -1843,6 +2434,14 @@ const ALL_REGEX_DEFS = [
1843
2434
  CZ_BANK_ACCOUNT,
1844
2435
  HU_LANDLINE,
1845
2436
  CZ_POSTAL,
2437
+ ES_POSTAL,
2438
+ ES_DNI,
2439
+ ES_NIE,
2440
+ ES_CIF,
2441
+ BR_CPF_FORMATTED,
2442
+ BR_CNPJ_FORMATTED,
2443
+ BR_RG_WITH_SSP,
2444
+ BR_OAB,
1846
2445
  URL,
1847
2446
  {
1848
2447
  pattern: "\\b(?:[0-9a-fA-F]{1,4}:){7}[0-9a-fA-F]{1,4}\\b|\\b(?:[0-9a-fA-F]{1,4}:){1,7}:\\b|::(?:[0-9a-fA-F]{1,4}:){0,6}[0-9a-fA-F]{1,4}",
@@ -1894,7 +2493,7 @@ const buildMonthAlternation = (months) => {
1894
2493
  if (clean.length >= MIN_MONTH_NAME_LENGTH) seen.add(clean);
1895
2494
  }
1896
2495
  }
1897
- return [...seen].toSorted((a, b) => b.length - a.length).map(escapeRegex$1).join("|");
2496
+ return [...seen].toSorted((a, b) => b.length - a.length).map(escapeRegex$2).join("|");
1898
2497
  };
1899
2498
  /**
1900
2499
  * Build date patterns from a month-name alternation.
@@ -1951,11 +2550,11 @@ const buildCurrencyPatterns = (data) => {
1951
2550
  const isAsciiAlpha = /^[a-zA-Z\s]+$/;
1952
2551
  const parts = data.codes.map((code) => ({
1953
2552
  len: code.length,
1954
- alt: escapeRegex$1(code)
2553
+ alt: escapeRegex$2(code)
1955
2554
  }));
1956
2555
  const MIN_CI_LENGTH = 3;
1957
2556
  if (data.localNames) for (const name of data.localNames) {
1958
- const escaped = escapeRegex$1(name);
2557
+ const escaped = escapeRegex$2(name);
1959
2558
  if (isAsciiAlpha.test(name) && name.length >= MIN_CI_LENGTH) parts.push({
1960
2559
  len: name.length,
1961
2560
  alt: `(?i:${escaped})`
@@ -1967,7 +2566,7 @@ const buildCurrencyPatterns = (data) => {
1967
2566
  }
1968
2567
  if (symbols) for (const ch of data.symbols) parts.push({
1969
2568
  len: ch.length,
1970
- alt: escapeRegex$1(ch)
2569
+ alt: escapeRegex$2(ch)
1971
2570
  });
1972
2571
  const trailingAlt = parts.toSorted((a, b) => b.len - a.len).map((p) => p.alt).join("|");
1973
2572
  if (!symbols && !trailingAlt) return [];
@@ -2021,19 +2620,21 @@ const processRegexMatches = (allMatches, sliceStart, sliceEnd, meta_) => {
2021
2620
  if (idx < sliceStart || idx >= sliceEnd) continue;
2022
2621
  const meta = meta_[idx - sliceStart];
2023
2622
  if (!meta) continue;
2024
- if (meta.label === "phone number" && match.text.length < MIN_PHONE_LENGTH) continue;
2623
+ if (meta.sourceDetail !== "custom-regex" && meta.label === "phone number" && match.text.length < MIN_PHONE_LENGTH) continue;
2025
2624
  if (meta.validator) {
2026
2625
  const compacted = meta.validator.compact(match.text);
2027
2626
  if (!meta.validator.validate(compacted).valid) continue;
2028
2627
  }
2029
- results.push({
2628
+ const entity = {
2030
2629
  start: match.start,
2031
2630
  end: match.end,
2032
2631
  label: meta.label,
2033
2632
  text: match.text,
2034
2633
  score: meta.score,
2035
2634
  source: DETECTION_SOURCES.REGEX
2036
- });
2635
+ };
2636
+ if (meta.sourceDetail) entity.sourceDetail = meta.sourceDetail;
2637
+ results.push(entity);
2037
2638
  }
2038
2639
  return results;
2039
2640
  };
@@ -2063,244 +2664,24 @@ const SIGNING_CLAUSE_META = {
2063
2664
  label: "address",
2064
2665
  score: .9
2065
2666
  };
2066
- let signingPatternPromise = null;
2067
- const loadSigningPatterns = async () => {
2068
- const mod = await import("./signing-clauses.mjs");
2069
- return buildSigningClausePatterns(mod.default ?? mod);
2070
- };
2071
- const getSigningClausePatterns = () => {
2072
- if (!signingPatternPromise) signingPatternPromise = loadSigningPatterns().catch((err) => {
2073
- signingPatternPromise = null;
2074
- throw err;
2075
- });
2076
- return signingPatternPromise;
2077
- };
2078
- //#endregion
2079
- //#region src/detectors/legal-forms.ts
2080
- const UPPER = "A-ZÁČĎÉĚÍŇÓŘŠŤÚŮÝŽÄÖÜÀÂÆÇÈÊËÎÏÔÙÛŸÑ\\u0130";
2081
- const LOWER = "a-záčďéěíňóřšťúůýžäöüßàâæçèêëîïôùûÿñ\\u0131";
2082
- const CAP_WORD = `(?:[${UPPER}]{2,}|[${UPPER}][${LOWER}${UPPER}]+)`;
2083
- const ANY_WORD = `(?:[${UPPER}${LOWER}][${LOWER}${UPPER}]+|[${UPPER}]{2,3}|\\d{1,4})`;
2084
- const ALLCAP_WORD = `[${UPPER}]{2,}`;
2085
- const ROMAN_NUMERAL_RE = /^(?=[IVXLCDM])M{0,3}(?:CM|CD|D?C{0,3})(?:XC|XL|L?X{0,3})(?:IX|IV|V?I{0,3})$/;
2086
- const escapeForRegex = (form) => form.replace(/[.*+?^${}()|[\]\\]/g, "\\$&").replace(/\s+/g, "\\s+").replace(/\\\./g, "\\.[^\\S\\n]?");
2087
- const isShortForm = (form) => form.replace(/[.\s]/g, "").length <= 3 && !form.includes(" ");
2088
- const buildPatternString = (forms) => {
2089
- if (forms.length === 0) return null;
2090
- const alt = forms.toSorted((a, b) => b.length - a.length).map(escapeForRegex).join("|");
2091
- return `${`(?:${CAP_WORD})(?:${`(?:[\\s&,.${DASH_INNER}]{1,4}|${`\\s+(?:a|and|und|et|e|y|i)\\s+(?=[${LOWER}])`})`}(?:${ANY_WORD})){0,10}`}(?:\\s+|,\\s*)(?:${alt})(?![${LOWER}])`;
2092
- };
2093
- /**
2094
- * Build legal form regex pattern strings.
2095
- * Returns an array of regex strings for the unified
2096
- * TextSearch builder. Empty if data package is not
2097
- * installed.
2098
- */
2099
- const buildLegalFormPatterns = async () => {
2100
- let data = {};
2101
- try {
2102
- data = (await import("./legal-forms.mjs")).default;
2103
- } catch {
2104
- return [];
2105
- }
2106
- const allForms = [];
2107
- const seen = /* @__PURE__ */ new Set();
2108
- for (const forms of Object.values(data)) for (const form of forms) {
2109
- const key = form.toLowerCase();
2110
- if (!seen.has(key)) {
2111
- seen.add(key);
2112
- allForms.push(form);
2113
- }
2114
- }
2115
- const patterns = [];
2116
- const longPattern = buildPatternString(allForms.filter((f) => !isShortForm(f)));
2117
- if (longPattern) patterns.push(longPattern);
2118
- const shortPattern = buildPatternString(allForms.filter(isShortForm));
2119
- if (shortPattern) patterns.push(shortPattern);
2120
- const allcapPrefix = `(?:${ALLCAP_WORD})(?:[\\s&,.${DASH_INNER}]{1,4}(?:${ALLCAP_WORD})){0,2}`;
2121
- const allcapAlt = allForms.toSorted((a, b) => b.length - a.length).map(escapeForRegex).join("|");
2122
- patterns.push(`${allcapPrefix}(?:\\s+|,\\s*)(?:${allcapAlt})(?![${LOWER}])`);
2123
- return patterns;
2124
- };
2125
- const CONNECTOR_RE = /^(?:a|and|und|et|e|y|i|&)$/i;
2126
- const LEADING_CLAUSE_RE = /(?:^|\s)(?:by\s+and\s+between|is\s+between)\s+/giu;
2127
- /**
2128
- * Find the word ending just before `pos` in `text`,
2129
- * skipping any whitespace (not newlines).
2130
- * Returns null if no word is found (e.g., at start
2131
- * of text, or preceded by non-word chars like ".").
2132
- */
2133
- const findWordBefore = (text, pos) => {
2134
- let scan = pos - 1;
2135
- while (scan >= 0) {
2136
- const ch = text.charAt(scan);
2137
- if (ch === "\n" || !/\s/.test(ch)) break;
2138
- scan--;
2139
- }
2140
- if (scan < 0 || text.charAt(scan) === "\n") return null;
2141
- const wordEnd = scan + 1;
2142
- while (scan >= 0 && /[\p{L}\p{M}&]/u.test(text.charAt(scan))) scan--;
2143
- const wordStart = scan + 1;
2144
- const word = text.slice(wordStart, wordEnd);
2145
- if (word.length === 0) return null;
2146
- return {
2147
- word,
2148
- start: wordStart
2149
- };
2150
- };
2151
- /**
2152
- * Extend a match backward through uppercase words and
2153
- * lowercase connectors. Stops at start of text,
2154
- * newline, or a word that doesn't qualify.
2155
- *
2156
- * Connectors (a, and, und, et, ...) are only consumed
2157
- * when there is a valid word before them — a trailing
2158
- * connector at an entity boundary is not consumed.
2159
- */
2160
- const extendBackward = (fullText, matchStart) => {
2161
- let pos = matchStart;
2162
- while (pos > 0) {
2163
- const found = findWordBefore(fullText, pos);
2164
- if (!found) break;
2165
- const { word, start: wordStart } = found;
2166
- const isUpper = /^\p{Lu}/u.test(word);
2167
- const isConnector = CONNECTOR_RE.test(word);
2168
- if (isUpper) pos = wordStart;
2169
- else if (isConnector) {
2170
- const prev = findWordBefore(fullText, wordStart);
2171
- if (!prev) break;
2172
- if (!/^\p{Lu}/u.test(prev.word)) break;
2173
- pos = prev.start;
2174
- } else break;
2175
- }
2176
- return pos;
2177
- };
2178
- const trimLeadingClause = (text) => {
2179
- let cut = -1;
2180
- for (const match of text.matchAll(LEADING_CLAUSE_RE)) cut = match.index + match[0].length;
2181
- if (cut <= 0) return {
2182
- offset: 0,
2183
- text
2184
- };
2185
- const trimmed = text.slice(cut);
2186
- const leadingWs = trimmed.match(/^\s*/u)?.[0].length ?? 0;
2187
- return {
2188
- offset: cut + leadingWs,
2189
- text: trimmed.slice(leadingWs)
2190
- };
2191
- };
2192
- /**
2193
- * Process legal form matches from the unified search.
2194
- * Receives all matches; filters to the legal forms
2195
- * slice via sliceStart/sliceEnd.
2196
- */
2197
- const processLegalFormMatches = (allMatches, sliceStart, sliceEnd, fullText) => {
2198
- const results = [];
2199
- for (const match of allMatches) {
2200
- const idx = match.pattern;
2201
- if (idx < sliceStart || idx >= sliceEnd) continue;
2202
- const text = match.text.trimEnd();
2203
- if (text.length < 5) continue;
2204
- if (text.includes("\n")) continue;
2205
- let entityStart = match.start;
2206
- let entityText = text;
2207
- if (fullText) {
2208
- const extended = extendBackward(fullText, match.start);
2209
- if (extended < match.start) {
2210
- entityStart = extended;
2211
- entityText = fullText.slice(extended, match.start + text.length).trimEnd();
2212
- }
2213
- }
2214
- const clauseTrim = trimLeadingClause(entityText);
2215
- if (clauseTrim.offset > 0) {
2216
- entityStart += clauseTrim.offset;
2217
- entityText = clauseTrim.text;
2218
- }
2219
- const getPrefixInfo = (value) => {
2220
- const prefixEnd = value.lastIndexOf(",") !== -1 ? value.lastIndexOf(",") : value.lastIndexOf(" ");
2221
- return {
2222
- prefixEnd,
2223
- prefixPart: prefixEnd > 0 ? value.slice(0, prefixEnd).replace(/[^a-zA-ZÀ-ž]/g, "") : value.replace(/[^a-zA-ZÀ-ž]/g, "")
2224
- };
2225
- };
2226
- let { prefixEnd, prefixPart } = getPrefixInfo(entityText);
2227
- let isAllCapsMatch = prefixPart.length > 2 && prefixPart === prefixPart.toUpperCase();
2228
- if (isAllCapsMatch && fullText) {
2229
- const lineStart = fullText.lastIndexOf("\n", entityStart);
2230
- const lineEnd = fullText.indexOf("\n", entityStart + entityText.length);
2231
- const lineLetters = fullText.slice(lineStart + 1, lineEnd === -1 ? fullText.length : lineEnd).replace(/[^a-zA-ZÀ-ž]/g, "");
2232
- const upperCount = [...lineLetters].filter((c) => c === c.toUpperCase()).length;
2233
- if (lineLetters.length > 5 && upperCount / lineLetters.length >= .95) continue;
2234
- if ((prefixPart.length > 0 ? entityText.slice(0, prefixEnd > 0 ? prefixEnd : entityText.length).trim().split(/\s+/).length : 0) > 3) {
2235
- entityStart = match.start;
2236
- entityText = text;
2237
- ({prefixEnd, prefixPart} = getPrefixInfo(entityText));
2238
- isAllCapsMatch = prefixPart.length > 2 && prefixPart === prefixPart.toUpperCase();
2239
- }
2240
- } else if (isAllCapsMatch) continue;
2241
- const lastSpace = entityText.lastIndexOf(" ");
2242
- const rawSuffix = lastSpace !== -1 ? entityText.slice(lastSpace + 1) : "";
2243
- const suffixClean = rawSuffix.replace(/[.,]/g, "");
2244
- if (suffixClean.length > 0 && ROMAN_NUMERAL_RE.test(suffixClean)) continue;
2245
- if (suffixClean.length <= 2 && !/\./.test(rawSuffix) && /[^\x00-\x7F]/.test(entityText.slice(0, lastSpace !== -1 ? lastSpace : entityText.length))) continue;
2246
- results.push({
2247
- start: entityStart,
2248
- end: entityStart + entityText.length,
2249
- label: "organization",
2250
- text: entityText,
2251
- score: .95,
2252
- source: DETECTION_SOURCES.LEGAL_FORM
2253
- });
2254
- }
2255
- return results;
2256
- };
2257
- //#endregion
2258
- //#region src/config/legal-forms.ts
2259
- /**
2260
- * Known legal form suffixes. Shared between trigger
2261
- * detection (reclassification) and org-propagation
2262
- * (suffix stripping). Within each family of related
2263
- * forms, longer variants come first so that
2264
- * "spol. s r.o." matches before "s.r.o." and
2265
- * "s. r. o." matches before "s.r.o.".
2266
- */
2267
- const LEGAL_SUFFIXES = [
2268
- "spol. s r.o.",
2269
- "s.r.o.",
2270
- "s. r. o.",
2271
- "a.s.",
2272
- "a. s.",
2273
- "v.o.s.",
2274
- "v. o. s.",
2275
- "k.s.",
2276
- "k. s.",
2277
- "z.s.",
2278
- "z. s.",
2279
- "z.ú.",
2280
- "z. ú.",
2281
- "o.p.s.",
2282
- "o. p. s.",
2283
- "s.p.",
2284
- "s. p.",
2285
- "GmbH",
2286
- "AG",
2287
- "SE",
2288
- "KG",
2289
- "OHG",
2290
- "Ltd.",
2291
- "Ltd",
2292
- "LLC",
2293
- "LLP",
2294
- "Inc.",
2295
- "S.A.",
2296
- "SA",
2297
- "SAS",
2298
- "SARL",
2299
- "Sp. z o.o.",
2300
- "S.p.A."
2301
- ];
2667
+ let signingPatternPromise = null;
2668
+ const loadSigningPatterns = async () => {
2669
+ const mod = await import("./signing-clauses.mjs");
2670
+ return buildSigningClausePatterns(mod.default ?? mod);
2671
+ };
2672
+ const getSigningClausePatterns = () => {
2673
+ if (!signingPatternPromise) signingPatternPromise = loadSigningPatterns().catch((err) => {
2674
+ signingPatternPromise = null;
2675
+ throw err;
2676
+ });
2677
+ return signingPatternPromise;
2678
+ };
2302
2679
  //#endregion
2303
2680
  //#region src/detectors/triggers.ts
2681
+ const VALID_ID_VALIDATORS = {
2682
+ "br.cpf": br.cpf,
2683
+ "br.cnpj": br.cnpj
2684
+ };
2304
2685
  const TRIGGER_SCORE = .95;
2305
2686
  const WHITESPACE_RE$1 = /\s+/;
2306
2687
  const LETTER_RE = /\p{L}/u;
@@ -2315,12 +2696,23 @@ const DECIMAL_COMMA_RE = new RegExp(`^,(?:\\d|${DASH}{1,2})`);
2315
2696
  * etc.), skip the comma and degree, then continue.
2316
2697
  */
2317
2698
  const POST_NOMINAL_RE = new RegExp(`^,\\s*(?:${POST_NOMINALS.toSorted((a, b) => b.length - a.length).map((d) => d.replace(/[.*+?^${}()|[\]\\]/g, "\\$&").replace(/\\\./g, "\\.\\s*")).join("|")})\\.?`, "i");
2318
- const LEGAL_FORM_CHECK_RE = new RegExp(LEGAL_SUFFIXES.map((f) => {
2319
- const escaped = f.replace(/[.*+?^${}()|[\]\\]/g, "\\$&").replace(/\\\./g, "\\.\\s*");
2320
- const isDotFree = !f.includes(".");
2321
- const isShort = f.length <= 4;
2322
- return isDotFree && isShort ? `\\b${escaped}\\b` : escaped;
2323
- }).join("|"));
2699
+ const buildLegalFormCheckRe = (forms) => {
2700
+ const parts = forms.map((f) => {
2701
+ const escaped = f.replace(/[.*+?^${}()|[\]\\]/g, "\\$&").replace(/\\\./g, "\\.\\s*");
2702
+ return /^[\p{L}\p{M}]+$/u.test(f) ? `(?<![\\p{L}\\p{N}])${escaped}(?![\\p{L}\\p{N}])` : escaped;
2703
+ });
2704
+ return new RegExp(parts.join("|"), "u");
2705
+ };
2706
+ let cachedLegalFormCheckRe = null;
2707
+ let cachedLegalFormCheckSource = null;
2708
+ const getLegalFormCheckRe = () => {
2709
+ const source = getKnownLegalSuffixes();
2710
+ if (cachedLegalFormCheckSource !== source || cachedLegalFormCheckRe === null) {
2711
+ cachedLegalFormCheckSource = source;
2712
+ cachedLegalFormCheckRe = buildLegalFormCheckRe(source);
2713
+ }
2714
+ return cachedLegalFormCheckRe;
2715
+ };
2324
2716
  const compileValidations = (validations) => validations.map((v) => {
2325
2717
  switch (v.type) {
2326
2718
  case "starts-uppercase": return {
@@ -2347,6 +2739,17 @@ const compileValidations = (validations) => validations.map((v) => {
2347
2739
  type: "matches-pattern",
2348
2740
  re: new RegExp(v.pattern, (v.flags ?? "").replace(/[gy]/g, ""))
2349
2741
  };
2742
+ case "valid-id": {
2743
+ const validator = VALID_ID_VALIDATORS[v.validator];
2744
+ if (!validator) throw new Error(`Unknown valid-id validator: ${JSON.stringify(v.validator)}`);
2745
+ return {
2746
+ type: "valid-id",
2747
+ check: (value) => {
2748
+ const compact = value.replace(/[\s.\-/]/g, "");
2749
+ return validator.validate(compact).valid;
2750
+ }
2751
+ };
2752
+ }
2350
2753
  default: throw new Error(`Unknown validation type: ${JSON.stringify(v)}`);
2351
2754
  }
2352
2755
  });
@@ -2370,6 +2773,9 @@ const applyValidations = (text, validations) => {
2370
2773
  case "matches-pattern":
2371
2774
  if (!v.re.test(text)) return false;
2372
2775
  break;
2776
+ case "valid-id":
2777
+ if (!v.check(text)) return false;
2778
+ break;
2373
2779
  default: throw new Error(`Unknown compiled validation type: ${JSON.stringify(v)}`);
2374
2780
  }
2375
2781
  return true;
@@ -2464,8 +2870,10 @@ const buildTriggerPatterns = async () => {
2464
2870
  strategy: rule.strategy.type
2465
2871
  });
2466
2872
  }
2873
+ const patterns = rules.map((r) => r.trigger.toLowerCase());
2874
+ await loadAddressStopKeywords();
2467
2875
  return {
2468
- patterns: rules.map((r) => r.trigger.toLowerCase()),
2876
+ patterns,
2469
2877
  rules
2470
2878
  };
2471
2879
  };
@@ -2486,14 +2894,103 @@ const stripQuotes = (value) => {
2486
2894
  const COMMA_STOP_CHARS = new Set([
2487
2895
  "\n",
2488
2896
  "(",
2489
- " "
2897
+ " ",
2898
+ ";"
2490
2899
  ]);
2491
2900
  /**
2901
+ * Sentence-terminator detection: a period that genuinely ends
2902
+ * one clause and starts another. Used by `to-next-comma` so
2903
+ * governing-law clauses ("…State of New York. SECTION 2…")
2904
+ * don't sweep across the period.
2905
+ *
2906
+ * Three positive signals (any one terminates):
2907
+ * 1. Long lowercase tail before the dot (>= 5 letters) —
2908
+ * catches "…construction. SECTION 2…", "…vykonává.".
2909
+ * 2. Currency/amount tail (zł, Kč, USD, €) — catches
2910
+ * "…w kwocie 1000 zł. Termin płatności…".
2911
+ * 3. Proper-noun head: a capitalized word of >= 4 letters
2912
+ * ending in lowercase, with no internal uppercase, AND
2913
+ * a substantial next clause (Capital + >= 2 lowercase).
2914
+ * Catches short city names that the lowercase-tail rule
2915
+ * misses: "…z siedzibą w Łódź. Kapitał zakładowy…",
2916
+ * "…seat in Brno. Section 2…".
2917
+ *
2918
+ * The rule must NOT fire on:
2919
+ * - title abbreviations: "Mr.", "Mrs.", "Dr.", "Hon.",
2920
+ * "Sr.", "Jr." (head <= 3 chars or insufficient Ll tail)
2921
+ * - degree abbreviations: "Ph.D.", "RNDr.", "MUDr.",
2922
+ * "Ing." (internal periods or internal uppercase block
2923
+ * the proper-noun pattern)
2924
+ * - street-type abbreviations: "Ste.", "Ave.", "Inc.",
2925
+ * "ul.", "al.", "nábř." (lowercase initials or
2926
+ * insufficient Ll tail)
2927
+ * - small-word lowercase abbreviations: "prof.", "inż.",
2928
+ * "hab." (no leading uppercase, so the proper-noun rule
2929
+ * can't fire)
2930
+ */
2931
+ const NEXT_IS_SENTENCE_START_RE = /^\.(?:\s+\p{Lu}|\s*$)/u;
2932
+ const SENTENCE_TAIL_RE = /\p{Ll}{5,}$/u;
2933
+ /**
2934
+ * Proper-noun tail: capital letter + >= 3 lowercase letters,
2935
+ * preceded by a non-letter and non-period (so we don't slice
2936
+ * into the middle of an acronym or a multi-dot abbreviation).
2937
+ * The 4-character minimum excludes 2–3-char titles ("Mr",
2938
+ * "Mrs", "Dr", "Inc", "Ste"); the all-lowercase tail
2939
+ * excludes mixed-case degrees ("RNDr", "MUDr").
2940
+ */
2941
+ const PROPER_NOUN_HEAD_RE = /(?:^|[^\p{L}.])\p{Lu}\p{Ll}{3,}$/u;
2942
+ /**
2943
+ * Next clause begins with a real word: capital + >= 2
2944
+ * lowercase letters. Filters cases where a capitalized
2945
+ * abbreviation (e.g., "Smith Inc.") follows a proper noun,
2946
+ * which would otherwise look sentence-like.
2947
+ */
2948
+ const NEXT_IS_REAL_SENTENCE_RE = /^\.\s+\p{Lu}\p{Ll}{2,}/u;
2949
+ /**
2950
+ * Short currency-abbreviation tail (zł, Kč, gr, Ft, kr,
2951
+ * лв, USD, PLN, EUR, …). When a period follows one of
2952
+ * these and the next token starts uppercase, treat it as
2953
+ * a sentence boundary even though the tail is too short
2954
+ * to satisfy `SENTENCE_TAIL_RE`. Without this, amount
2955
+ * triggers using `to-next-comma` swallow the following
2956
+ * clause: `"w kwocie 1000 zł. Termin płatności…"`.
2957
+ *
2958
+ * Currency codes are typically uppercase (`USD`, `PLN`)
2959
+ * but appear lowercased in informal writing (`pln`, `eur`);
2960
+ * local names mix case (`zł`, `Kč`, `Ft`); symbols
2961
+ * (`€`, `$`, `£`) appear after the amount as well. The
2962
+ * negative lookbehind on a letter ensures the abbreviation
2963
+ * is matched only as a standalone token; symbols are
2964
+ * matched unconditionally since they are not letters.
2965
+ */
2966
+ const CURRENCY_TAIL_RE = /(?:(?<![\p{L}])(?:zł|Kč|gr|Ft|kr|лв|USD|PLN|EUR|CZK|GBP|CHF|HUF|RON|SEK|NOK|DKK)|[€$£])$/iu;
2967
+ const NUMERIC_SENTENCE_TAIL_RE = /\d$/;
2968
+ const isSentenceTerminator = (text, periodIndex) => {
2969
+ const tail = text.slice(periodIndex);
2970
+ if (!NEXT_IS_SENTENCE_START_RE.test(tail)) return false;
2971
+ const head = text.slice(0, periodIndex);
2972
+ if (SENTENCE_TAIL_RE.test(head) || CURRENCY_TAIL_RE.test(head) || NUMERIC_SENTENCE_TAIL_RE.test(head)) return true;
2973
+ return PROPER_NOUN_HEAD_RE.test(head) && NEXT_IS_REAL_SENTENCE_RE.test(tail);
2974
+ };
2975
+ /**
2492
2976
  * Field-label keywords that terminate address scanning.
2493
2977
  * When a comma in the address strategy is followed by
2494
2978
  * one of these, the address stops before the keyword.
2979
+ *
2980
+ * The list is sourced from
2981
+ * `data/address-stop-keywords.json` (per-language so new
2982
+ * languages can drop in their own labels without
2983
+ * touching this file). `loadAddressStopKeywords` unions
2984
+ * every language into a single longest-first array;
2985
+ * `getAddressStopKeywordsSync` returns the cached union
2986
+ * and falls back to a seed list so the strategy keeps
2987
+ * working before the warmup promise resolves.
2988
+ *
2989
+ * The address strategy doesn't know the document
2990
+ * language, so a flat union is intentional: any
2991
+ * language's labels can appear in any address.
2495
2992
  */
2496
- const ADDRESS_STOP_KEYWORDS = [
2993
+ const ADDRESS_STOP_KEYWORDS_SEED = [
2497
2994
  "číslo účtu",
2498
2995
  "registrační",
2499
2996
  "zastoupen",
@@ -2516,6 +3013,52 @@ const ADDRESS_STOP_KEYWORDS = [
2516
3013
  "bic",
2517
3014
  "ič"
2518
3015
  ];
3016
+ let addressStopKeywordsCache = null;
3017
+ let addressStopKeywordsPromise = null;
3018
+ const loadAddressStopKeywords = async () => {
3019
+ if (addressStopKeywordsCache) return addressStopKeywordsCache;
3020
+ if (addressStopKeywordsPromise) return addressStopKeywordsPromise;
3021
+ addressStopKeywordsPromise = (async () => {
3022
+ let data = {};
3023
+ try {
3024
+ const mod = await import("./address-stop-keywords.mjs");
3025
+ data = mod.default ?? mod;
3026
+ } catch (err) {
3027
+ console.warn("[anonymize] triggers: failed to load address-stop-keywords.json, falling back to seed list:", err);
3028
+ }
3029
+ const seen = /* @__PURE__ */ new Set();
3030
+ const out = [];
3031
+ const addAll = (list) => {
3032
+ for (const kw of list) {
3033
+ if (typeof kw !== "string" || kw.length === 0) continue;
3034
+ const lower = kw.toLowerCase();
3035
+ if (seen.has(lower)) continue;
3036
+ seen.add(lower);
3037
+ out.push(lower);
3038
+ }
3039
+ };
3040
+ for (const [key, value] of Object.entries(data)) {
3041
+ if (key.startsWith("_")) continue;
3042
+ if (!Array.isArray(value)) continue;
3043
+ addAll(value);
3044
+ }
3045
+ addAll(ADDRESS_STOP_KEYWORDS_SEED);
3046
+ out.sort((a, b) => b.length - a.length);
3047
+ addressStopKeywordsCache = out;
3048
+ return out;
3049
+ })();
3050
+ return addressStopKeywordsPromise;
3051
+ };
3052
+ const getAddressStopKeywordsSync = () => addressStopKeywordsCache ?? ADDRESS_STOP_KEYWORDS_SEED;
3053
+ /**
3054
+ * Warm the address-stop-keywords cache. Pipeline callers
3055
+ * await this before invoking trigger detection so the
3056
+ * synchronous `extractValue` path uses the merged list
3057
+ * instead of the seed fallback.
3058
+ */
3059
+ const warmAddressStopKeywords = async () => {
3060
+ await loadAddressStopKeywords();
3061
+ };
2519
3062
  const extractValue = (text, triggerEnd, strategy, label) => {
2520
3063
  const remaining = text.slice(triggerEnd);
2521
3064
  const stripped = remaining.replace(/^[\s:;]+/, "");
@@ -2524,6 +3067,8 @@ const extractValue = (text, triggerEnd, strategy, label) => {
2524
3067
  if (valueText.length === 0) return null;
2525
3068
  switch (strategy.type) {
2526
3069
  case "to-next-comma": {
3070
+ const stopWords = strategy.stopWords ?? [];
3071
+ const stopWordRe = stopWords.length > 0 ? new RegExp(`^(?:${stopWords.map((w) => w.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")).join("|")})(?![\\p{L}\\p{N}])`, "iu") : null;
2527
3072
  let end = 0;
2528
3073
  let foundStop = false;
2529
3074
  while (end < valueText.length) {
@@ -2532,6 +3077,14 @@ const extractValue = (text, triggerEnd, strategy, label) => {
2532
3077
  foundStop = true;
2533
3078
  break;
2534
3079
  }
3080
+ if (ch === "." && isSentenceTerminator(valueText, end)) {
3081
+ foundStop = true;
3082
+ break;
3083
+ }
3084
+ if (stopWordRe !== null && (end === 0 || !/[\p{L}\p{N}]/u.test(valueText[end - 1] ?? "")) && stopWordRe.test(valueText.slice(end))) {
3085
+ foundStop = true;
3086
+ break;
3087
+ }
2535
3088
  if (ch === ",") {
2536
3089
  const afterComma = valueText.slice(end);
2537
3090
  if (DECIMAL_COMMA_RE.test(afterComma)) {
@@ -2560,6 +3113,8 @@ const extractValue = (text, triggerEnd, strategy, label) => {
2560
3113
  };
2561
3114
  }
2562
3115
  case "to-end-of-line": {
3116
+ const consumed = remaining.length - valueText.length;
3117
+ if (consumed > 0 && remaining.slice(0, consumed).includes("\n")) return null;
2563
3118
  const LINE_STOPS = ["\n"];
2564
3119
  let end = valueText.length;
2565
3120
  for (const ch of LINE_STOPS) {
@@ -2580,7 +3135,8 @@ const extractValue = (text, triggerEnd, strategy, label) => {
2580
3135
  const tabIdx = valueText.indexOf(" ");
2581
3136
  const cellText = tabIdx !== -1 ? valueText.slice(0, tabIdx) : valueText;
2582
3137
  const PUNCT_ONLY = /^[\p{P}\p{S}]+$/u;
2583
- const words = cellText.split(WHITESPACE_RE$1).filter((w) => !PUNCT_ONLY.test(w)).slice(0, strategy.count);
3138
+ const NUMBER_MARKER = /^(?:n[ºo°.]|№)$/i;
3139
+ const words = cellText.split(WHITESPACE_RE$1).filter((w) => !PUNCT_ONLY.test(w) && !NUMBER_MARKER.test(w)).slice(0, strategy.count);
2584
3140
  if (words.length === 0) return null;
2585
3141
  const firstWord = words[0];
2586
3142
  if (firstWord === void 0) return null;
@@ -2603,14 +3159,22 @@ const extractValue = (text, triggerEnd, strategy, label) => {
2603
3159
  }
2604
3160
  case "company-id-value": {
2605
3161
  const raw = text.slice(triggerEnd);
2606
- const sepMatch = /^(?:\s*:\s*|\s+)/.exec(raw);
3162
+ const triggerLastChar = text[triggerEnd - 1] ?? "";
3163
+ const sepMatch = (triggerLastChar === "°" || triggerLastChar === "º" || triggerLastChar === "№" || triggerLastChar === "#" ? /^(?:\s*:\s*|\s+|)/ : /^(?:\s*:\s*|\s+)/).exec(raw);
2607
3164
  if (!sepMatch) return null;
2608
- const afterSep = raw.slice(sepMatch[0].length);
2609
- const idMatch = /^[A-Z]{0,6}\s?\d[\d\s\-/]{4,}/i.exec(afterSep);
3165
+ let afterSep = raw.slice(sepMatch[0].length);
3166
+ const labelMatch = /^(?:nr\.?|numer|n[ºo°.]|№|no\.?)(?:\s*:\s*|\s+)/i.exec(afterSep);
3167
+ let labelOffset = 0;
3168
+ if (labelMatch) {
3169
+ labelOffset = labelMatch[0].length;
3170
+ afterSep = afterSep.slice(labelOffset);
3171
+ }
3172
+ const idMatch = /^[A-Z]{0,6}\s?(?:[A-Z0-9]{2}\s?)?\d{2}(?:(?<=\d)[A-Z]|[\d\s.\-/]){3,}/i.exec(afterSep);
2610
3173
  if (!idMatch) return null;
2611
- const idText = idMatch[0].trim();
3174
+ const trailingLetterMatch = /\d$/.test(idMatch[0]) ? /^[A-Z]/i.exec(afterSep.slice(idMatch[0].length)) : null;
3175
+ const idText = (trailingLetterMatch ? idMatch[0] + trailingLetterMatch[0] : idMatch[0]).trim();
2612
3176
  const leadingSpaces = idMatch[0].length - idMatch[0].trimStart().length;
2613
- const idStart = triggerEnd + sepMatch[0].length + leadingSpaces;
3177
+ const idStart = triggerEnd + sepMatch[0].length + labelOffset + leadingSpaces;
2614
3178
  return {
2615
3179
  start: idStart,
2616
3180
  end: idStart + idText.length,
@@ -2620,18 +3184,36 @@ const extractValue = (text, triggerEnd, strategy, label) => {
2620
3184
  case "address": {
2621
3185
  const maxLen = strategy.maxChars ?? 120;
2622
3186
  const UPPER_RE = /\p{Lu}/u;
3187
+ const stopKeywords = getAddressStopKeywordsSync();
2623
3188
  let end = 0;
2624
3189
  while (end < valueText.length && end < maxLen) {
2625
3190
  const ch = valueText[end];
2626
3191
  if (ch === "\n" || ch === "(") break;
3192
+ if (ch === " " || ch === " ") {
3193
+ const afterWs = valueText.slice(end).trimStart().toLowerCase();
3194
+ if (stopKeywords.some((kw) => {
3195
+ if (!afterWs.startsWith(kw)) return false;
3196
+ const next = afterWs[kw.length];
3197
+ return next === void 0 || /[\s:;.,!?()\d]/.test(next);
3198
+ })) break;
3199
+ }
2627
3200
  if (ch === ".") {
2628
3201
  const next = valueText[end + 1];
2629
3202
  const afterNext = valueText[end + 2];
3203
+ const afterPeriod = valueText.slice(end + 1).replace(/^\s+/, "").toLowerCase();
3204
+ if (afterPeriod.length > 0) {
3205
+ if (getAddressStopKeywordsSync().some((kw) => {
3206
+ if (!afterPeriod.startsWith(kw)) return false;
3207
+ const afterKw = afterPeriod[kw.length];
3208
+ return afterKw === void 0 || /[\s:;.,!?()\d]/.test(afterKw);
3209
+ })) break;
3210
+ }
2630
3211
  if (next !== void 0 && (/\p{L}/u.test(next) || /\d/.test(next))) {
2631
3212
  end++;
2632
3213
  continue;
2633
3214
  }
2634
3215
  if (next === " " && afterNext !== void 0 && (/\p{L}/u.test(afterNext) || /\d/.test(afterNext))) {
3216
+ if (isSentenceTerminator(valueText, end)) break;
2635
3217
  end++;
2636
3218
  continue;
2637
3219
  }
@@ -2643,7 +3225,7 @@ const extractValue = (text, triggerEnd, strategy, label) => {
2643
3225
  const peekCh = valueText[peek];
2644
3226
  if (peekCh === void 0) break;
2645
3227
  const afterComma = valueText.slice(end + 1).trimStart().toLowerCase();
2646
- if (ADDRESS_STOP_KEYWORDS.some((kw) => {
3228
+ if (getAddressStopKeywordsSync().some((kw) => {
2647
3229
  if (!afterComma.startsWith(kw)) return false;
2648
3230
  const next = afterComma[kw.length];
2649
3231
  return next === void 0 || /[\s:;.,!?()\d]/.test(next);
@@ -2670,6 +3252,28 @@ const extractValue = (text, triggerEnd, strategy, label) => {
2670
3252
  text: extracted
2671
3253
  };
2672
3254
  }
3255
+ case "match-pattern": {
3256
+ const nlIdx = valueText.indexOf("\n");
3257
+ const searchText = nlIdx === -1 ? valueText : valueText.slice(0, nlIdx);
3258
+ if (searchText.length === 0) return null;
3259
+ const flags = (strategy.flags ?? "").replace(/[gy]/g, "");
3260
+ const anchoredSource = `^(?:${strategy.pattern})`;
3261
+ let re;
3262
+ try {
3263
+ re = new RegExp(anchoredSource, flags);
3264
+ } catch {
3265
+ return null;
3266
+ }
3267
+ const m = re.exec(searchText);
3268
+ if (!m || m[0].length === 0) return null;
3269
+ const matchStart = m.index;
3270
+ const matchEnd = matchStart + m[0].length;
3271
+ return {
3272
+ start: valueStart + matchStart,
3273
+ end: valueStart + matchEnd,
3274
+ text: m[0]
3275
+ };
3276
+ }
2673
3277
  default: return null;
2674
3278
  }
2675
3279
  };
@@ -2689,7 +3293,8 @@ const processTriggerMatches = (allMatches, sliceStart, sliceEnd, fullText, rules
2689
3293
  if (match.start > 0 && LETTER_RE.test(fullText[match.start - 1] ?? "")) continue;
2690
3294
  const rule = rules[localIdx];
2691
3295
  if (!rule) continue;
2692
- if (!rule.trigger.endsWith(" ") && LETTER_RE.test(fullText[match.end] ?? "")) continue;
3296
+ const triggerLastChar = rule.trigger.at(-1) ?? "";
3297
+ if (LETTER_RE.test(triggerLastChar) && LETTER_RE.test(fullText[match.end] ?? "")) continue;
2693
3298
  const triggerEnd = match.end;
2694
3299
  const rawValue = extractValue(fullText, triggerEnd, rule.strategy, rule.label);
2695
3300
  const value = rawValue ? stripQuotes(rawValue) : null;
@@ -2698,7 +3303,7 @@ const processTriggerMatches = (allMatches, sliceStart, sliceEnd, fullText, rules
2698
3303
  const entityStart = rule.includeTrigger ? match.start : value.start;
2699
3304
  const entityEnd = value.end;
2700
3305
  const entityText = fullText.slice(entityStart, entityEnd);
2701
- const effectiveLabel = rule.label === "person" && LEGAL_FORM_CHECK_RE.test(entityText) ? "organization" : rule.label;
3306
+ const effectiveLabel = rule.label === "person" && getLegalFormCheckRe().test(entityText) ? "organization" : rule.label;
2702
3307
  results.push({
2703
3308
  start: entityStart,
2704
3309
  end: entityEnd,
@@ -2968,11 +3573,82 @@ const resolveCountries = (regions, countries) => {
2968
3573
  return result;
2969
3574
  };
2970
3575
  //#endregion
3576
+ //#region src/util/homoglyphs.ts
3577
+ const CHAR_MAP$1 = new Map([
3578
+ [1040, 65],
3579
+ [1042, 66],
3580
+ [1045, 69],
3581
+ [1050, 75],
3582
+ [1052, 77],
3583
+ [1053, 72],
3584
+ [1054, 79],
3585
+ [1056, 80],
3586
+ [1057, 67],
3587
+ [1058, 84],
3588
+ [1061, 88],
3589
+ [1072, 97],
3590
+ [1077, 101],
3591
+ [1086, 111],
3592
+ [1088, 112],
3593
+ [1089, 99],
3594
+ [1091, 121],
3595
+ [1093, 120],
3596
+ [1110, 105],
3597
+ [1112, 106],
3598
+ [913, 65],
3599
+ [914, 66],
3600
+ [917, 69],
3601
+ [919, 72],
3602
+ [921, 73],
3603
+ [922, 75],
3604
+ [924, 77],
3605
+ [925, 78],
3606
+ [927, 79],
3607
+ [929, 80],
3608
+ [932, 84],
3609
+ [933, 89],
3610
+ [935, 88],
3611
+ [918, 90],
3612
+ [945, 97],
3613
+ [954, 107],
3614
+ [959, 111],
3615
+ [961, 112]
3616
+ ]);
3617
+ /** Chunk size for `String.fromCharCode` to avoid hitting
3618
+ * the call-stack limit on very large strings. */
3619
+ const CHUNK_SIZE$1 = 8192;
3620
+ const normalizeHomoglyphs = (text) => {
3621
+ let hasSpecial = false;
3622
+ for (let i = 0; i < text.length; i++) if (CHAR_MAP$1.has(text.charCodeAt(i))) {
3623
+ hasSpecial = true;
3624
+ break;
3625
+ }
3626
+ if (!hasSpecial) return text;
3627
+ const len = text.length;
3628
+ const codes = new Uint16Array(len);
3629
+ for (let i = 0; i < len; i++) {
3630
+ const code = text.charCodeAt(i);
3631
+ codes[i] = CHAR_MAP$1.get(code) ?? code;
3632
+ }
3633
+ if (len <= CHUNK_SIZE$1) return String.fromCharCode(...codes);
3634
+ let result = "";
3635
+ for (let offset = 0; offset < len; offset += CHUNK_SIZE$1) {
3636
+ const end = Math.min(offset + CHUNK_SIZE$1, len);
3637
+ result += String.fromCharCode(...codes.subarray(offset, end));
3638
+ }
3639
+ return result;
3640
+ };
3641
+ //#endregion
2971
3642
  //#region src/filters/false-positives.ts
2972
3643
  const TEMPLATE_PLACEHOLDER_RE = /^(?:\.{3,}|_{3,}|\[[\w\s]+\]|\{[\w\s]+\})$/;
2973
3644
  const POSTAL_CODE_RE = /\d{3}\s?\d{2}/;
2974
3645
  const HAS_DIGIT_RE = /\d/;
2975
- const ADDRESS_COMPONENTS_RE = /(?:^|\s)(?:ul\.|ulice|nám\.|náměstí|tř\.|třída|nábř\.|nábřeží|č\.p\.|č\.ev\.|č\.|sídliště|bulvár)(?=[\s,./]|$)/i;
3646
+ const ADDRESS_COMPONENT_EXTRA_RE = /(?:^|\s)(?:č\.p\.|č\.ev\.|č\.|sídliště)(?=[\s,./]|$)/i;
3647
+ const BARE_COURS_PROSE_RE = /(?:^|\s)cours(?!\s+\p{Lu})(?=[\s,./]|$)/u;
3648
+ const isOnlyAmbiguousCours = (text) => {
3649
+ if (!BARE_COURS_PROSE_RE.test(text)) return false;
3650
+ return !hasAddressComponent(text.replace(/(?:^|\s)cours(?=[\s,./]|$)/gi, " "));
3651
+ };
2976
3652
  const JURISDICTION_RE = /^(?:state|commonwealth|district|territory)\s+of\s+/i;
2977
3653
  const MAX_ENTITY_LENGTH = {
2978
3654
  organization: 80,
@@ -3055,6 +3731,42 @@ const loadGenericRoles = (ctx = defaultContext) => {
3055
3731
  };
3056
3732
  /** Sync accessor — returns empty set before init. */
3057
3733
  const getGenericRoles = (ctx) => ctx.genericRoles ?? EMPTY_GENERIC_ROLES;
3734
+ const STREET_TYPES_SEED_RE = /(?:^|\s)(?:ul\.|ulice|nám\.|náměstí|tř\.|třída|nábř\.|nábřeží|bulvár)(?=[\s,./]|$)/i;
3735
+ let _streetTypesRe = STREET_TYPES_SEED_RE;
3736
+ let _streetTypesPromise = null;
3737
+ const escapeRegex$1 = (s) => s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
3738
+ const loadStreetTypeRegex = async () => {
3739
+ try {
3740
+ const data = (await import("./address-street-types.mjs")).default ?? {};
3741
+ const words = /* @__PURE__ */ new Set();
3742
+ for (const [key, val] of Object.entries(data)) {
3743
+ if (key.startsWith("_")) continue;
3744
+ if (!Array.isArray(val)) continue;
3745
+ for (const w of val) if (typeof w === "string" && w.length > 0) words.add(w.toLowerCase());
3746
+ }
3747
+ if (words.size === 0) {
3748
+ _streetTypesRe = STREET_TYPES_SEED_RE;
3749
+ return;
3750
+ }
3751
+ const alternation = [...words].sort((a, b) => b.length - a.length).map(escapeRegex$1).join("|");
3752
+ _streetTypesRe = new RegExp(`(?:^|\\s)(?:${alternation})(?=[\\s,./]|$)`, "iu");
3753
+ } catch {
3754
+ _streetTypesRe = STREET_TYPES_SEED_RE;
3755
+ }
3756
+ };
3757
+ /** Ensure street-type vocabulary is loaded. */
3758
+ const initAddressComponents = () => {
3759
+ if (!_streetTypesPromise) _streetTypesPromise = loadStreetTypeRegex();
3760
+ return _streetTypesPromise;
3761
+ };
3762
+ /**
3763
+ * True when `text` contains a recognised address
3764
+ * component: either a per-language street word (loaded
3765
+ * from JSON) or a language-agnostic anchor such as a
3766
+ * Czech house number form.
3767
+ */
3768
+ const hasAddressComponent = (text) => _streetTypesRe.test(text) || ADDRESS_COMPONENT_EXTRA_RE.test(text);
3769
+ const isCallerOwnedEntity$3 = (entity) => entity.sourceDetail === "custom-deny-list" || entity.sourceDetail === "custom-regex";
3058
3770
  /**
3059
3771
  * Filter out entities that are likely false positives:
3060
3772
  * template placeholders, clause/section numbers,
@@ -3067,9 +3779,17 @@ const filterFalsePositives = (entities, ctx = defaultContext, fullText) => {
3067
3779
  const filtered = [];
3068
3780
  const roles = getGenericRoles(ctx);
3069
3781
  for (const entity of entities) {
3782
+ if (isCallerOwnedEntity$3(entity)) {
3783
+ filtered.push(entity);
3784
+ continue;
3785
+ }
3070
3786
  const normalized = normalizeEntity(entity);
3071
3787
  if (!normalized) continue;
3072
3788
  const trimmed = normalized.text;
3789
+ if (isCallerOwnedEntity$3(normalized)) {
3790
+ filtered.push(normalized);
3791
+ continue;
3792
+ }
3073
3793
  if (TEMPLATE_PLACEHOLDER_RE.test(trimmed)) continue;
3074
3794
  const maxLen = MAX_ENTITY_LENGTH[normalized.label];
3075
3795
  if (maxLen && trimmed.length > maxLen && normalized.source !== "legal-form") continue;
@@ -3080,13 +3800,15 @@ const filterFalsePositives = (entities, ctx = defaultContext, fullText) => {
3080
3800
  if (normalized.label === "person" && HAS_DIGIT_RE.test(trimmed)) continue;
3081
3801
  if (normalized.label === "person") {
3082
3802
  const tokens = trimmed.split(/\s+/u);
3083
- const last = tokens.at(-1)?.replace(/[.,;:!?]+$/u, "").toLowerCase();
3084
- if (tokens.length > 1 && last && PERSON_TRAILING_NOUNS.has(last)) continue;
3803
+ const last = tokens.at(-1)?.replace(/[.,;:!?]+$/u, "");
3804
+ const lastFolded = last ? normalizeHomoglyphs(last).toLowerCase() : void 0;
3805
+ if (tokens.length > 1 && lastFolded && PERSON_TRAILING_NOUNS.has(lastFolded)) continue;
3085
3806
  }
3086
- if ((normalized.label === "person" || normalized.label === "organization") && roles.has(trimmed.toLowerCase())) continue;
3807
+ if ((normalized.label === "person" || normalized.label === "organization") && roles.has(normalizeHomoglyphs(trimmed).toLowerCase())) continue;
3087
3808
  if (normalized.label === "organization" && normalized.source === "legal-form" && trimmed === trimmed.toUpperCase() && LEGAL_FORM_HEADING_RE.test(trimmed)) continue;
3088
- if (normalized.label === "address" && trimmed.length > 40 && !POSTAL_CODE_RE.test(trimmed) && !HAS_DIGIT_RE.test(trimmed) && !ADDRESS_COMPONENTS_RE.test(trimmed) && !JURISDICTION_RE.test(trimmed)) continue;
3089
- if (normalized.label === "address" && normalized.source === "trigger" && !HAS_DIGIT_RE.test(trimmed) && !ADDRESS_COMPONENTS_RE.test(trimmed) && !JURISDICTION_RE.test(trimmed)) continue;
3809
+ if (normalized.label === "address" && trimmed.length > 40 && !POSTAL_CODE_RE.test(trimmed) && !HAS_DIGIT_RE.test(trimmed) && !hasAddressComponent(trimmed) && !JURISDICTION_RE.test(trimmed)) continue;
3810
+ if (normalized.label === "address" && normalized.source === "trigger" && !HAS_DIGIT_RE.test(trimmed) && !hasAddressComponent(trimmed) && !JURISDICTION_RE.test(trimmed)) continue;
3811
+ if (normalized.label === "address" && normalized.source === "trigger" && !HAS_DIGIT_RE.test(trimmed) && isOnlyAmbiguousCours(trimmed)) continue;
3090
3812
  if (normalized.label === "address" && SIGNING_CLAUSE_ADDRESS_RE.test(trimmed)) continue;
3091
3813
  filtered.push(normalized);
3092
3814
  }
@@ -3243,7 +3965,124 @@ const loadPersonStopwords = (ctx) => {
3243
3965
  const EMPTY_PERSON_STOPWORDS = /* @__PURE__ */ new Set();
3244
3966
  /** Sync accessor — returns empty set before init. */
3245
3967
  const getPersonStopwords = (ctx) => ctx.personStopwords ?? EMPTY_PERSON_STOPWORDS;
3968
+ const loadAddressStopwords = (ctx) => {
3969
+ if (ctx.addressStopwordsPromise) return ctx.addressStopwordsPromise;
3970
+ ctx.addressStopwordsPromise = (async () => {
3971
+ try {
3972
+ const mod = await import("./address-stopwords.mjs");
3973
+ const set = new Set(mod.default?.words ?? []);
3974
+ ctx.addressStopwords = set;
3975
+ return set;
3976
+ } catch {
3977
+ const empty = /* @__PURE__ */ new Set();
3978
+ ctx.addressStopwords = empty;
3979
+ return empty;
3980
+ }
3981
+ })();
3982
+ return ctx.addressStopwordsPromise;
3983
+ };
3984
+ const EMPTY_ADDRESS_STOPWORDS = /* @__PURE__ */ new Set();
3985
+ const getAddressStopwords = (ctx) => ctx.addressStopwords ?? EMPTY_ADDRESS_STOPWORDS;
3986
+ /**
3987
+ * Word characters in unicode property notation. The check is
3988
+ * "single-token" — no internal whitespace — and we keep dashes
3989
+ * tokens like "K-12" out by requiring the surface to be a single
3990
+ * uninterrupted run of letters or marks.
3991
+ */
3992
+ const SINGLE_WORD_RE = /^\p{L}+$/u;
3993
+ /**
3994
+ * Format-level address signals — structurally numeric or
3995
+ * 2-letter-state patterns, language-agnostic. The street-type
3996
+ * vocabulary is loaded from `address-street-types.json` so new
3997
+ * languages contribute via data, not code.
3998
+ * - `,\s*[A-Z]{2}\b` → US state abbreviation after a comma
3999
+ * - `\b\d{5}(?:-\d{4})?\b` → US ZIP / ZIP+4
4000
+ * - `\b\d{3}\s\d{2}\b` → Czech/Slovak postal block (140 00)
4001
+ * - `\b\d{2}-\d{3}\b` → Polish postal code (00-950)
4002
+ */
4003
+ const ADDRESS_FORMAT_RE = /,\s*\p{Lu}{2}\b|\b\d{5}(?:-\d{4})?\b|\b\d{3}\s\d{2}\b|\b\d{2}-\d{3}\b/u;
4004
+ let cachedStreetTypeRe = null;
4005
+ let streetTypeReLoaded = false;
4006
+ const loadStreetTypeRe = async () => {
4007
+ if (streetTypeReLoaded) return cachedStreetTypeRe;
4008
+ try {
4009
+ const config = (await import("./address-street-types.mjs")).default ?? {};
4010
+ const words = [];
4011
+ for (const value of Object.values(config)) {
4012
+ if (!Array.isArray(value)) continue;
4013
+ for (const word of value) if (typeof word === "string" && word.length > 0) words.push(word);
4014
+ }
4015
+ if (words.length === 0) cachedStreetTypeRe = null;
4016
+ else {
4017
+ words.sort((a, b) => b.length - a.length);
4018
+ const isLetterDigit = (c) => /[\p{L}\p{N}]/u.test(c);
4019
+ const wordLikeTail = [];
4020
+ const punctTail = [];
4021
+ for (const w of words) {
4022
+ const escaped = w.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
4023
+ if (isLetterDigit(w.at(-1) ?? "")) wordLikeTail.push(escaped);
4024
+ else punctTail.push(escaped);
4025
+ }
4026
+ const branches = [];
4027
+ if (wordLikeTail.length > 0) branches.push(`(?:${wordLikeTail.join("|")})(?![\\p{L}\\p{N}])`);
4028
+ if (punctTail.length > 0) branches.push(`(?:${punctTail.join("|")})`);
4029
+ cachedStreetTypeRe = new RegExp(`(?<![\\p{L}\\p{N}])(?:${branches.join("|")})`, "iu");
4030
+ }
4031
+ } catch {
4032
+ cachedStreetTypeRe = null;
4033
+ }
4034
+ streetTypeReLoaded = true;
4035
+ return cachedStreetTypeRe;
4036
+ };
4037
+ const getStreetTypeRe = () => streetTypeReLoaded ? cachedStreetTypeRe : null;
4038
+ const hasAdjacentAddressEvidence = (fullText, start, end) => {
4039
+ const window = fullText.slice(Math.max(0, start - 40), Math.min(fullText.length, end + 40));
4040
+ if (ADDRESS_FORMAT_RE.test(window)) return true;
4041
+ const streetRe = getStreetTypeRe();
4042
+ return streetRe !== null && streetRe.test(window);
4043
+ };
4044
+ /**
4045
+ * Capitalised words that almost never start a person name. When a
4046
+ * single-token surname candidate is immediately followed by one of
4047
+ * these, the "next-word is uppercase" promotion heuristic would
4048
+ * otherwise turn section headings ("Purchase Price↵The Purchaser
4049
+ * undertakes…") into spurious person hits. Kept narrow on purpose;
4050
+ * the surrounding pipeline still chains real names via the deny-list
4051
+ * cascade when both halves are surnames.
4052
+ */
4053
+ const SENTENCE_STARTER_WORDS = new Set([
4054
+ "The",
4055
+ "This",
4056
+ "These",
4057
+ "Those",
4058
+ "An",
4059
+ "Any",
4060
+ "All",
4061
+ "Each",
4062
+ "Every",
4063
+ "No",
4064
+ "Now",
4065
+ "Whereas",
4066
+ "Whereby",
4067
+ "Wherein",
4068
+ "Whereof",
4069
+ "Notwithstanding",
4070
+ "Subject",
4071
+ "In",
4072
+ "On",
4073
+ "At",
4074
+ "By",
4075
+ "For",
4076
+ "If",
4077
+ "Upon",
4078
+ "Unless",
4079
+ "Until",
4080
+ "Provided",
4081
+ "Pursuant",
4082
+ "Such"
4083
+ ]);
3246
4084
  const PERSON_CHAIN_BREAK_RE = /[!?;:]|,/u;
4085
+ const WORD_CHAR_RE$1 = /[\p{L}\p{N}]/u;
3247
4086
  const isInitialContinuationGap = (text, gap) => /^\p{Lu}$/u.test(text) && /^\.[^\S\n]{1,2}$/u.test(gap) || /^[^\S\n]{1,2}(?:\p{Lu}\.[^\S\n]{1,2})+$/u.test(gap);
3248
4087
  /**
3249
4088
  * Resolve which dictionaries to load based on country
@@ -3261,32 +4100,38 @@ const buildDenyList = async (config, ctx = defaultContext) => {
3261
4100
  loadStopwords(ctx),
3262
4101
  loadAllowList(ctx),
3263
4102
  loadPersonStopwords(ctx),
4103
+ loadAddressStopwords(ctx),
4104
+ loadStreetTypeRe(),
3264
4105
  loadGenericRoles(ctx)
3265
4106
  ]);
3266
4107
  const dictionaries = config.dictionaries;
3267
4108
  const hasDenyList = dictionaries?.denyList && dictionaries?.denyListMeta;
3268
4109
  const hasCities = dictionaries?.cities && dictionaries.cities.length > 0;
3269
- if (!hasDenyList && !hasCities) return buildNameCorpusOnly(config, ctx);
4110
+ const hasCustomDenyList = config.customDenyList !== void 0 && config.customDenyList.length > 0;
4111
+ if (!hasDenyList && !hasCities && !hasCustomDenyList) return buildNameCorpusOnly(config, ctx);
3270
4112
  const allowedCountries = resolveCountries(config.denyListRegions, config.denyListCountries);
3271
4113
  const excluded = config.denyListExcludeCategories;
3272
4114
  const excludeCategories = excluded ? new Set(excluded) : /* @__PURE__ */ new Set();
3273
4115
  const patternList = [];
3274
4116
  const labelList = [];
4117
+ const customLabelList = [];
3275
4118
  const sourceList = [];
3276
4119
  const patternIndex = /* @__PURE__ */ new Map();
3277
- const addDenyListEntry = (entry, label) => {
3278
- const normalized = normalizeForSearch(entry).replace(/[|\\]/g, "");
4120
+ const addDenyListEntry = (entry, label, source = "deny-list") => {
4121
+ const normalized = source === "custom-deny-list" ? normalizeForSearch(entry) : normalizeForSearch(entry).replace(/[|\\]/g, "");
3279
4122
  if (normalized.length === 0) return;
3280
4123
  const lower = normalized.toLowerCase();
3281
4124
  const existing = patternIndex.get(lower);
3282
4125
  if (existing !== void 0) {
3283
4126
  if (!labelList[existing].includes(label)) labelList[existing].push(label);
3284
- if (!sourceList[existing].includes("deny-list")) sourceList[existing].push("deny-list");
4127
+ if (!sourceList[existing].includes(source)) sourceList[existing].push(source);
4128
+ if (source === "custom-deny-list" && !customLabelList[existing].includes(label)) customLabelList[existing].push(label);
3285
4129
  } else {
3286
4130
  patternIndex.set(lower, patternList.length);
3287
4131
  patternList.push(normalized);
3288
4132
  labelList.push([label]);
3289
- sourceList.push(["deny-list"]);
4133
+ customLabelList.push(source === "custom-deny-list" ? [label] : []);
4134
+ sourceList.push([source]);
3290
4135
  }
3291
4136
  };
3292
4137
  if (hasDenyList) {
@@ -3304,10 +4149,15 @@ const buildDenyList = async (config, ctx = defaultContext) => {
3304
4149
  }
3305
4150
  }
3306
4151
  if (hasCities && !excludeCategories.has("Places")) for (const entry of dictionaries.cities) addDenyListEntry(entry, "address");
4152
+ if (hasCustomDenyList) for (const entry of config.customDenyList) {
4153
+ addDenyListEntry(entry.value, entry.label, "custom-deny-list");
4154
+ for (const variant of entry.variants ?? []) addDenyListEntry(variant, entry.label, "custom-deny-list");
4155
+ }
3307
4156
  appendNameCorpusEntries(config, ctx, patternList, labelList, sourceList, patternIndex);
3308
4157
  if (patternList.length === 0) return null;
3309
4158
  return {
3310
4159
  labels: labelList,
4160
+ customLabels: customLabelList,
3311
4161
  originals: patternList,
3312
4162
  sources: sourceList
3313
4163
  };
@@ -3324,11 +4174,13 @@ const buildNameCorpusOnly = (config, ctx) => {
3324
4174
  if ((excluded ? new Set(excluded) : /* @__PURE__ */ new Set()).has("Names")) return null;
3325
4175
  const patternList = [];
3326
4176
  const labelList = [];
4177
+ const customLabelList = [];
3327
4178
  const sourceList = [];
3328
4179
  appendNameCorpusEntries(config, ctx, patternList, labelList, sourceList, /* @__PURE__ */ new Map());
3329
4180
  if (patternList.length === 0) return null;
3330
4181
  return {
3331
4182
  labels: labelList,
4183
+ customLabels: customLabelList,
3332
4184
  originals: patternList,
3333
4185
  sources: sourceList
3334
4186
  };
@@ -3374,6 +4226,14 @@ const appendNameCorpusEntries = (config, ctx, patternList, labelList, sourceList
3374
4226
  }
3375
4227
  }
3376
4228
  };
4229
+ const customMatchHasValidEdges = (fullText, start, end, pattern) => {
4230
+ if (!WORD_CHAR_RE$1.test(pattern)) return true;
4231
+ const prev = fullText[start - 1] ?? "";
4232
+ const next = fullText[end] ?? "";
4233
+ if (WORD_CHAR_RE$1.test(prev)) return false;
4234
+ if (WORD_CHAR_RE$1.test(next)) return false;
4235
+ return true;
4236
+ };
3377
4237
  /**
3378
4238
  * Ensure all deny-list support data (stopwords, allow
3379
4239
  * list, person stopwords, generic roles) is loaded on
@@ -3388,6 +4248,8 @@ const ensureDenyListData = async (ctx = defaultContext, dictionaries) => {
3388
4248
  loadStopwords(ctx),
3389
4249
  loadAllowList(ctx),
3390
4250
  loadPersonStopwords(ctx),
4251
+ loadAddressStopwords(ctx),
4252
+ loadStreetTypeRe(),
3391
4253
  loadGenericRoles(ctx)
3392
4254
  ]);
3393
4255
  };
@@ -3411,18 +4273,24 @@ const processDenyListMatches = (allMatches, sliceStart, sliceEnd, fullText, data
3411
4273
  const idx = match.pattern;
3412
4274
  if (idx < sliceStart || idx >= sliceEnd) continue;
3413
4275
  const localIdx = idx - sliceStart;
3414
- const sourceChar = fullText[match.start] ?? "";
3415
- if (!UPPER_START_RE.test(sourceChar)) continue;
4276
+ const sources = data.sources[localIdx] ?? [];
3416
4277
  const matchText = fullText.slice(match.start, match.end);
4278
+ const sourceChar = fullText[match.start] ?? "";
3417
4279
  const keyword = matchText.toLowerCase();
3418
- if (getStopwords(ctx).has(keyword) || getAllowList(ctx).has(keyword)) continue;
3419
- if (ALL_UPPER_RE.test(matchText)) continue;
3420
4280
  const labels = data.labels[localIdx];
3421
- if (!labels || labels.length === 0) continue;
4281
+ const pattern = data.originals[localIdx] ?? "";
4282
+ const customPatternLabels = data.customLabels[localIdx] ?? [];
4283
+ const customEdgesAreValid = customMatchHasValidEdges(fullText, match.start, match.end, pattern);
4284
+ const customLabels = customEdgesAreValid ? customPatternLabels : [];
4285
+ if ((!labels || labels.length === 0) && customLabels.length === 0) continue;
4286
+ const curatedLabels = UPPER_START_RE.test(sourceChar) && !getStopwords(ctx).has(keyword) && !getAllowList(ctx).has(keyword) && !ALL_UPPER_RE.test(matchText) ? (labels ?? []).filter((label) => !customPatternLabels.includes(label) && customEdgesAreValid) : [];
4287
+ if (curatedLabels.length === 0 && customLabels.length === 0) continue;
3422
4288
  const entry = {
3423
4289
  start: match.start,
3424
4290
  end: match.end,
3425
- labels,
4291
+ labels: curatedLabels,
4292
+ customLabels,
4293
+ sources,
3426
4294
  text: matchText,
3427
4295
  patternIdx: localIdx
3428
4296
  };
@@ -3433,22 +4301,35 @@ const processDenyListMatches = (allMatches, sliceStart, sliceEnd, fullText, data
3433
4301
  const results = [];
3434
4302
  const nameHits = [];
3435
4303
  for (const [, matches] of Array.from(matchesByPattern)) {
3436
- const first = matches[0];
3437
- if (!first) continue;
3438
- const hasPerson = first.labels.includes("person");
3439
- const nonPersonLabels = first.labels.filter((l) => l !== "person");
3440
- if (hasPerson) {
3441
- const keyword = first.text.toLowerCase();
3442
- if (!getPersonStopwords(ctx).has(keyword)) for (const m of matches) nameHits.push(m);
3443
- }
3444
- for (const m of matches) for (const label of nonPersonLabels) results.push({
4304
+ if (!matches[0]) continue;
4305
+ for (const m of matches) for (const label of m.customLabels) results.push({
3445
4306
  start: m.start,
3446
4307
  end: m.end,
3447
4308
  label,
3448
4309
  text: m.text,
3449
4310
  score: .9,
3450
- source: DETECTION_SOURCES.DENY_LIST
4311
+ source: DETECTION_SOURCES.DENY_LIST,
4312
+ sourceDetail: "custom-deny-list"
3451
4313
  });
4314
+ for (const m of matches) {
4315
+ if (m.labels.includes("person")) {
4316
+ const keyword = m.text.toLowerCase();
4317
+ if (!getPersonStopwords(ctx).has(keyword)) nameHits.push(m);
4318
+ }
4319
+ const nonPersonLabels = m.labels.filter((l) => l !== "person");
4320
+ const suppressAddress = SINGLE_WORD_RE.test(m.text) && getAddressStopwords(ctx).has(m.text.toLowerCase()) && !hasAdjacentAddressEvidence(fullText, m.start, m.end);
4321
+ for (const label of nonPersonLabels) {
4322
+ if (label === "address" && suppressAddress) continue;
4323
+ results.push({
4324
+ start: m.start,
4325
+ end: m.end,
4326
+ label,
4327
+ text: m.text,
4328
+ score: .9,
4329
+ source: DETECTION_SOURCES.DENY_LIST
4330
+ });
4331
+ }
4332
+ }
3452
4333
  }
3453
4334
  nameHits.sort((a, b) => a.start - b.start);
3454
4335
  const nameConsumed = /* @__PURE__ */ new Set();
@@ -3479,6 +4360,8 @@ const processDenyListMatches = (allMatches, sliceStart, sliceEnd, fullText, data
3479
4360
  const afterEnd = last.end;
3480
4361
  const rest = fullText.slice(afterEnd).trimStart();
3481
4362
  if (!(rest.length > 1 && /^\p{Lu}\p{Ll}/u.test(rest))) continue;
4363
+ const nextWord = /^\p{L}+/u.exec(rest)?.[0] ?? "";
4364
+ if (SENTENCE_STARTER_WORDS.has(nextWord)) continue;
3482
4365
  }
3483
4366
  results.push({
3484
4367
  start: first.start,
@@ -3528,6 +4411,7 @@ const TRAILING_WORD_EXCLUSIONS = new Set([
3528
4411
  const extendCityDistricts = (entities, fullText) => {
3529
4412
  for (const entity of entities) {
3530
4413
  if (entity.label !== "address") continue;
4414
+ if (entity.sourceDetail === "custom-deny-list") continue;
3531
4415
  const afterMatch = fullText.slice(entity.end);
3532
4416
  const suffixM = DISTRICT_SUFFIX_RE.exec(afterMatch);
3533
4417
  if (suffixM) {
@@ -3589,6 +4473,15 @@ const extendPersonName = (text, start, end, ctx) => {
3589
4473
  };
3590
4474
  //#endregion
3591
4475
  //#region src/detectors/address-seeds.ts
4476
+ /**
4477
+ * Trailing chars that should never end an address span. Combines
4478
+ * structural separators, opening brackets, and typographic quote
4479
+ * variants so a span like `… GA 30326, USA (the "Premises")`
4480
+ * does not pull the parenthetical opener into the address.
4481
+ * The prime (`′`) catches measurement tails (`5′`) that the
4482
+ * cluster occasionally absorbs.
4483
+ */
4484
+ const ADDRESS_TRAILING_TRIM_RE = new RegExp(`[,;:\\s${OPENING_BRACKETS_INNER}${QUOTE_DOUBLE_INNER}${QUOTE_SINGLE_INNER}′]`, "u");
3592
4485
  let cachedBoundaryRe = null;
3593
4486
  const loadBoundaryWords = async () => {
3594
4487
  try {
@@ -3597,6 +4490,56 @@ const loadBoundaryWords = async () => {
3597
4490
  return {};
3598
4491
  }
3599
4492
  };
4493
+ const BR_CEP_CONTEXT_WINDOW = 200;
4494
+ let cachedBrCepContextRe = null;
4495
+ let cachedBrCepContextPromise = null;
4496
+ const loadBrCueWords = async () => {
4497
+ const sources = await Promise.all([(async () => {
4498
+ try {
4499
+ return (await import("./address-street-types.mjs")).default["pt-br"];
4500
+ } catch {
4501
+ return;
4502
+ }
4503
+ })(), (async () => {
4504
+ try {
4505
+ return (await import("./address-boundaries.mjs")).default["pt-br"];
4506
+ } catch {
4507
+ return;
4508
+ }
4509
+ })()]);
4510
+ const out = [];
4511
+ const seen = /* @__PURE__ */ new Set();
4512
+ for (const entry of sources) {
4513
+ if (!Array.isArray(entry)) continue;
4514
+ for (const word of entry) {
4515
+ if (typeof word !== "string" || word.length === 0) continue;
4516
+ const key = word.toLowerCase();
4517
+ if (seen.has(key)) continue;
4518
+ seen.add(key);
4519
+ out.push(word);
4520
+ }
4521
+ }
4522
+ return out;
4523
+ };
4524
+ const getBrCepContextRe = async () => {
4525
+ if (cachedBrCepContextRe !== null) return cachedBrCepContextRe;
4526
+ if (cachedBrCepContextPromise) return cachedBrCepContextPromise;
4527
+ cachedBrCepContextPromise = (async () => {
4528
+ const words = await loadBrCueWords();
4529
+ if (words.length === 0) return null;
4530
+ const escaped = words.toSorted((a, b) => b.length - a.length).map((w) => w.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"));
4531
+ const re = new RegExp(`(?<![\\p{L}\\p{N}])(?:${escaped.join("|")})(?![\\p{L}\\p{N}])`, "iu");
4532
+ cachedBrCepContextRe = re;
4533
+ return re;
4534
+ })();
4535
+ return cachedBrCepContextPromise;
4536
+ };
4537
+ const hasBrCueNearby = (fullText, start, end, re) => {
4538
+ const windowStart = Math.max(0, start - BR_CEP_CONTEXT_WINDOW);
4539
+ const windowEnd = Math.min(fullText.length, end + BR_CEP_CONTEXT_WINDOW);
4540
+ const window = fullText.slice(windowStart, windowEnd);
4541
+ return new RegExp(re.source, re.flags.replace("g", "")).test(window);
4542
+ };
3600
4543
  /**
3601
4544
  * Build regex for boundary words. Matches any
3602
4545
  * boundary word preceded by a word boundary.
@@ -3610,7 +4553,7 @@ const getBoundaryRe = async () => {
3610
4553
  for (const word of entries) words.push(word.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"));
3611
4554
  }
3612
4555
  words.sort((a, b) => b.length - a.length);
3613
- cachedBoundaryRe = words.length > 0 ? new RegExp(`\\b(?:${words.join("|")})\\b`, "i") : /(?!)/;
4556
+ cachedBoundaryRe = words.length > 0 ? new RegExp(`(?<![\\p{L}\\p{N}])(?:${words.join("|")})(?![\\p{L}\\p{N}])`, "iu") : /(?!)/;
3614
4557
  return cachedBoundaryRe;
3615
4558
  };
3616
4559
  /**
@@ -3632,7 +4575,7 @@ const buildStreetTypePatterns = async () => {
3632
4575
  }
3633
4576
  return words;
3634
4577
  };
3635
- const collectSeeds = (allMatches, sliceStart, sliceEnd, fullText, existingEntities) => {
4578
+ const collectSeeds = (allMatches, sliceStart, sliceEnd, fullText, existingEntities, brCepContextRe) => {
3636
4579
  const seeds = [];
3637
4580
  for (const match of allMatches) {
3638
4581
  const idx = match.pattern;
@@ -3646,6 +4589,7 @@ const collectSeeds = (allMatches, sliceStart, sliceEnd, fullText, existingEntiti
3646
4589
  }
3647
4590
  for (const e of existingEntities) {
3648
4591
  if (e.label !== "address") continue;
4592
+ if (e.sourceDetail === "custom-deny-list") continue;
3649
4593
  if (e.source === "deny-list") seeds.push({
3650
4594
  type: "city",
3651
4595
  start: e.start,
@@ -3665,18 +4609,41 @@ const collectSeeds = (allMatches, sliceStart, sliceEnd, fullText, existingEntiti
3665
4609
  text: e.text
3666
4610
  });
3667
4611
  }
3668
- const postalRe = /\b(?:\d{3}\s\d{2}|\d{2}-\d{3})\b/g;
4612
+ const postalRe = /\b(?:\d{3}\s\d{2}|\d{2}-\d{3}|\d{5}-\d{3})\b/g;
3669
4613
  let postalMatch;
3670
4614
  while ((postalMatch = postalRe.exec(fullText)) !== null) {
3671
4615
  const start = postalMatch.index;
3672
4616
  const end = start + postalMatch[0].length;
3673
- if (!seeds.some((s) => s.start <= start && s.end >= end)) seeds.push({
4617
+ if (seeds.some((s) => s.start <= start && s.end >= end)) continue;
4618
+ if (/^\d{5}-\d{3}$/.test(postalMatch[0]) && (brCepContextRe === null || !hasBrCueNearby(fullText, start, end, brCepContextRe))) continue;
4619
+ seeds.push({
3674
4620
  type: "postal-code",
3675
4621
  start,
3676
4622
  end,
3677
4623
  text: postalMatch[0]
3678
4624
  });
3679
4625
  }
4626
+ const itCapRe = /\b\d{5}(?=\s+\p{Lu}\p{L}+)/gu;
4627
+ let itCapMatch;
4628
+ while ((itCapMatch = itCapRe.exec(fullText)) !== null) {
4629
+ const start = itCapMatch.index;
4630
+ const end = start + itCapMatch[0].length;
4631
+ if (seeds.some((s) => s.start <= start && s.end >= end)) continue;
4632
+ if (!seeds.some((s) => {
4633
+ if (Math.abs(s.start - start) > 80) return false;
4634
+ if (s.type === "address-trigger") return true;
4635
+ if (s.type === "city") return true;
4636
+ if (s.type === "postal-code") return true;
4637
+ if (s.type === "street-word") return s.text.toLowerCase() !== "via";
4638
+ return false;
4639
+ })) continue;
4640
+ seeds.push({
4641
+ type: "postal-code",
4642
+ start,
4643
+ end,
4644
+ text: itCapMatch[0]
4645
+ });
4646
+ }
3680
4647
  const streetNumRe = /\b(\p{Lu}\p{Ll}{2,})\s+(\d{1,5}(?:\/\d{1,5})?)\s*[,\n]/gu;
3681
4648
  let streetMatch;
3682
4649
  while ((streetMatch = streetNumRe.exec(fullText)) !== null) {
@@ -3734,6 +4701,10 @@ const scoreCluster = (cluster) => {
3734
4701
  const NON_ADDRESS_LABELS = new Set([
3735
4702
  "registration number",
3736
4703
  "tax identification number",
4704
+ "national identification number",
4705
+ "social security number",
4706
+ "birth number",
4707
+ "identity card number",
3737
4708
  "person",
3738
4709
  "bank account number",
3739
4710
  "email address",
@@ -3770,7 +4741,7 @@ const expandCluster = async (fullText, cluster, existingEntities) => {
3770
4741
  const doubleNewline = remaining.indexOf("\n\n");
3771
4742
  if (doubleNewline !== -1 && doubleNewline < nearestBoundary) nearestBoundary = doubleNewline;
3772
4743
  rightPos = end + remaining.slice(0, nearestBoundary).trimEnd().length;
3773
- while (rightPos > end && /[,;:\s]/.test(fullText[rightPos - 1] ?? "")) rightPos--;
4744
+ while (rightPos > end && ADDRESS_TRAILING_TRIM_RE.test(fullText[rightPos - 1] ?? "")) rightPos--;
3774
4745
  return {
3775
4746
  start: Math.min(leftPos, start),
3776
4747
  end: Math.max(rightPos, end)
@@ -3787,7 +4758,7 @@ const expandCluster = async (fullText, cluster, existingEntities) => {
3787
4758
  * using their output as seed sources.
3788
4759
  */
3789
4760
  const processAddressSeeds = async (allMatches, sliceStart, sliceEnd, fullText, existingEntities) => {
3790
- const clusters = clusterSeeds(collectSeeds(allMatches, sliceStart, sliceEnd, fullText, existingEntities), 150);
4761
+ const clusters = clusterSeeds(collectSeeds(allMatches, sliceStart, sliceEnd, fullText, existingEntities, await getBrCepContextRe()), 150);
3791
4762
  const results = [];
3792
4763
  for (const cluster of clusters) {
3793
4764
  const score = scoreCluster(cluster);
@@ -3812,6 +4783,8 @@ const processAddressSeeds = async (allMatches, sliceStart, sliceEnd, fullText, e
3812
4783
  const TRAILING_SEP = /[,\s]+$/;
3813
4784
  const WORD_CHAR_RE = /[\p{L}\p{N}]/u;
3814
4785
  const ORG_PROPAGATION_SCORE = .9;
4786
+ const ORG_DETERMINER_RE = /(?<![\p{L}\p{N}])(společnost(?:i|í|em|u)?|spolecnost(?:i|em|u)?|the\s+(?:company|corporation|firm)|die\s+(?:gesellschaft|firma)|la\s+(?:société|empresa|sociedad)|el\s+(?:empresa|sociedad))\s+$/iu;
4787
+ const isCallerOwnedEntity$2 = (entity) => entity.sourceDetail === "custom-deny-list" || entity.sourceDetail === "custom-regex";
3815
4788
  /**
3816
4789
  * After the main detection pass, collect organization
3817
4790
  * entities with a legal form suffix, strip the suffix
@@ -3824,6 +4797,7 @@ const propagateOrgNames = (entities, fullText) => {
3824
4797
  const seenBases = /* @__PURE__ */ new Set();
3825
4798
  for (const e of entities) {
3826
4799
  if (e.label !== "organization") continue;
4800
+ if (isCallerOwnedEntity$2(e)) continue;
3827
4801
  for (const suffix of LEGAL_SUFFIXES) if (e.text.endsWith(suffix)) {
3828
4802
  const base = e.text.slice(0, -suffix.length).replace(TRAILING_SEP, "").trim();
3829
4803
  if (base.length >= 3 && !seenBases.has(base)) {
@@ -3853,16 +4827,25 @@ const propagateOrgNames = (entities, fullText) => {
3853
4827
  searchFrom = idx + 1;
3854
4828
  continue;
3855
4829
  }
3856
- if (!isOverlapping(idx, matchEnd)) {
4830
+ let spanStart = idx;
4831
+ const lookbackStart = Math.max(0, idx - 40);
4832
+ const lookback = fullText.slice(lookbackStart, idx);
4833
+ const determinerMatch = ORG_DETERMINER_RE.exec(lookback);
4834
+ if (determinerMatch !== null) {
4835
+ const determiner = determinerMatch[1] ?? "";
4836
+ const offsetInMatch = determinerMatch[0].indexOf(determiner);
4837
+ spanStart = lookbackStart + determinerMatch.index + offsetInMatch;
4838
+ }
4839
+ if (!isOverlapping(spanStart, matchEnd)) {
3857
4840
  results.push({
3858
- start: idx,
4841
+ start: spanStart,
3859
4842
  end: matchEnd,
3860
4843
  label,
3861
- text: baseName,
4844
+ text: fullText.slice(spanStart, matchEnd),
3862
4845
  score: ORG_PROPAGATION_SCORE,
3863
4846
  source: DETECTION_SOURCES.COREFERENCE
3864
4847
  });
3865
- covered.push([idx, matchEnd]);
4848
+ covered.push([spanStart, matchEnd]);
3866
4849
  }
3867
4850
  searchFrom = matchEnd;
3868
4851
  }
@@ -3903,6 +4886,7 @@ const NEAR_MISS_BAND = .15;
3903
4886
  const BOOST_PER_NEIGHBOUR = .05;
3904
4887
  const CONTEXT_WINDOW_CHARS = 150;
3905
4888
  const HIGH_CONFIDENCE_FLOOR = .9;
4889
+ const isCallerOwnedEntity$1 = (entity) => entity.sourceDetail === "custom-deny-list" || entity.sourceDetail === "custom-regex";
3906
4890
  /**
3907
4891
  * Boost confidence of near-miss NER entities that appear
3908
4892
  * near high-confidence detections (regex, trigger phrase).
@@ -4011,7 +4995,7 @@ const getStreetAbbrevs = () => _streetAbbrevs ?? /* @__PURE__ */ new Set();
4011
4995
  */
4012
4996
  const detectStreetPatternsNearAddresses = (fullText, existingEntities) => {
4013
4997
  const results = [];
4014
- const addressEntities = existingEntities.filter((e) => e.label === "address");
4998
+ const addressEntities = existingEntities.filter((entity) => !(entity.label === "address" && isCallerOwnedEntity$1(entity))).filter((e) => e.label === "address");
4015
4999
  const headerEnd = Math.floor(fullText.length * HEADER_ZONE_FRACTION);
4016
5000
  const houseNumRe = /\b(?:\d{1,4}\/\d+[a-zA-Z]\b|\d{3,4}\/\d+\b|(?:1[3-9]|[2-9]\d)\/\d{3,}\b)/g;
4017
5001
  houseNumRe.lastIndex = 0;
@@ -4107,6 +5091,7 @@ const ORPHAN_STREET_RE = /^\s*(\p{Lu}[\p{Ll}\p{Lu}]+(?:\s+[\p{Lu}\p{Ll}][\p{Ll}]
4107
5091
  */
4108
5092
  const detectOrphanStreetLines = (fullText, existingEntities) => {
4109
5093
  const headerEnd = Math.floor(fullText.length * HEADER_ZONE_FRACTION);
5094
+ const contextEntities = existingEntities.filter((entity) => !(entity.label === "address" && isCallerOwnedEntity$1(entity)));
4110
5095
  const results = [];
4111
5096
  ORPHAN_STREET_RE.lastIndex = 0;
4112
5097
  for (let m = ORPHAN_STREET_RE.exec(fullText); m !== null; m = ORPHAN_STREET_RE.exec(fullText)) {
@@ -4116,7 +5101,7 @@ const detectOrphanStreetLines = (fullText, existingEntities) => {
4116
5101
  const end = start + captured.length;
4117
5102
  if (start >= headerEnd) continue;
4118
5103
  if (existingEntities.some((e) => e.start <= start && e.end >= end)) continue;
4119
- if (!existingEntities.some((e) => Math.abs(e.start - end) < 200 || Math.abs(e.end - start) < 200)) continue;
5104
+ if (!contextEntities.some((e) => Math.abs(e.start - end) < 200 || Math.abs(e.end - start) < 200)) continue;
4120
5105
  results.push({
4121
5106
  start,
4122
5107
  end,
@@ -4464,6 +5449,7 @@ const applyHotwordRules = (entities, fullText) => {
4464
5449
  //#region src/filters/boundary-consistency.ts
4465
5450
  /** Max gap (in chars) between entities to merge. */
4466
5451
  const MAX_GAP = 3;
5452
+ const hasLockedBoundary$1 = (entity) => entity.sourceDetail === "custom-deny-list" || entity.sourceDetail === "custom-regex";
4467
5453
  /**
4468
5454
  * Characters allowed in the gap between two adjacent
4469
5455
  * same-label entities that should be merged: spaces,
@@ -4579,6 +5565,11 @@ const mergeAdjacent = (entities, fullText) => {
4579
5565
  const result = [];
4580
5566
  const lastByLabel = /* @__PURE__ */ new Map();
4581
5567
  for (const entity of sorted) {
5568
+ if (hasLockedBoundary$1(entity)) {
5569
+ const copy = { ...entity };
5570
+ result.push(copy);
5571
+ continue;
5572
+ }
4582
5573
  const prev = lastByLabel.get(entity.label);
4583
5574
  if (!prev) {
4584
5575
  const copy = { ...entity };
@@ -4586,7 +5577,7 @@ const mergeAdjacent = (entities, fullText) => {
4586
5577
  lastByLabel.set(entity.label, copy);
4587
5578
  continue;
4588
5579
  }
4589
- if (entity.start < prev.end) {
5580
+ if (!hasLockedBoundary$1(prev) && entity.start < prev.end) {
4590
5581
  prev.end = Math.max(prev.end, entity.end);
4591
5582
  prev.text = fullText.slice(prev.start, prev.end);
4592
5583
  prev.score = Math.max(prev.score, entity.score);
@@ -4605,7 +5596,7 @@ const mergeAdjacent = (entities, fullText) => {
4605
5596
  break;
4606
5597
  }
4607
5598
  }
4608
- if (!gapOccupied && gap.length <= MAX_GAP && GAP_PATTERN.test(gap)) {
5599
+ if (!hasLockedBoundary$1(prev) && !gapOccupied && gap.length <= MAX_GAP && GAP_PATTERN.test(gap)) {
4609
5600
  prev.end = entity.end;
4610
5601
  prev.text = fullText.slice(prev.start, prev.end);
4611
5602
  prev.score = Math.max(prev.score, entity.score);
@@ -4637,6 +5628,7 @@ const fixPartialWords = (entities, fullText) => {
4637
5628
  })).sort((a, b) => a.entity.end - b.entity.end);
4638
5629
  const endPositions = byEnd.map((x) => x.entity.end);
4639
5630
  return sorted.map((e, eIdx) => {
5631
+ if (hasLockedBoundary$1(e)) return e;
4640
5632
  let newStart = wordStartAt(e.start, boundaries, fullText);
4641
5633
  let newEnd = wordEndAt(e.end, boundaries, fullText);
4642
5634
  let lo = 0;
@@ -4731,7 +5723,8 @@ const resolveCrossLabelOverlaps = (entities, fullText) => {
4731
5723
  if (aContainsB || bContainsA) continue;
4732
5724
  const aLen = a.end - a.start;
4733
5725
  const bLen = b.end - b.start;
4734
- if (a.score > b.score || a.score === b.score && aLen >= bLen) {
5726
+ const aLocked = hasLockedBoundary$1(a);
5727
+ if (aLocked !== hasLockedBoundary$1(b) ? aLocked : a.score > b.score || a.score === b.score && aLen >= bLen) {
4735
5728
  b.start = a.end;
4736
5729
  b.text = fullText.slice(b.start, b.end);
4737
5730
  } else {
@@ -4759,8 +5752,11 @@ const enforceBoundaryConsistency = (entities, fullText) => {
4759
5752
  };
4760
5753
  //#endregion
4761
5754
  //#region src/build-unified-search.ts
5755
+ const DEFAULT_CUSTOM_REGEX_SCORE$1 = .9;
5756
+ const ALNUM_RE = /[\p{L}\p{N}]/u;
4762
5757
  const buildUnifiedSearch = async (config, gazetteerEntries = [], ctx = defaultContext) => {
4763
5758
  const legalFormsEnabled = isLegalFormsEnabled(config);
5759
+ const customRegexes = config.enableRegex ? config.customRegexes ?? [] : [];
4764
5760
  const [legalForms, triggers, denyListData, streetTypes, currencyPatterns, datePatterns, signingPatterns] = await Promise.all([
4765
5761
  legalFormsEnabled ? buildLegalFormPatterns() : Promise.resolve([]),
4766
5762
  config.enableTriggerPhrases ? buildTriggerPatterns() : Promise.resolve({
@@ -4785,12 +5781,21 @@ const buildUnifiedSearch = async (config, gazetteerEntries = [], ctx = defaultCo
4785
5781
  ...datePatterns.map(() => DATE_PATTERN_META),
4786
5782
  ...signingPatterns.map(() => SIGNING_CLAUSE_META)
4787
5783
  ];
5784
+ const customRegexMeta = customRegexes.map((entry) => ({
5785
+ label: entry.label,
5786
+ score: entry.score ?? DEFAULT_CUSTOM_REGEX_SCORE$1,
5787
+ sourceDetail: "custom-regex"
5788
+ }));
4788
5789
  let offset = 0;
4789
5790
  const regexSlice = {
4790
5791
  start: offset,
4791
5792
  end: offset + allRegex.length
4792
5793
  };
4793
5794
  offset = regexSlice.end;
5795
+ const customRegexSlice = {
5796
+ start: 0,
5797
+ end: customRegexes.length
5798
+ };
4794
5799
  const legalFormsSlice = {
4795
5800
  start: offset,
4796
5801
  end: offset + legalForms.length
@@ -4811,6 +5816,7 @@ const buildUnifiedSearch = async (config, gazetteerEntries = [], ctx = defaultCo
4811
5816
  ...triggerEntries
4812
5817
  ];
4813
5818
  const tsRegex = new (getTextSearch())(regexAllPatterns);
5819
+ const tsCustomRegex = new (getTextSearch())(customRegexes.map((entry) => entry.pattern), { overlapStrategy: "all" });
4814
5820
  offset = 0;
4815
5821
  const denyListOriginals = denyListData?.originals ?? [];
4816
5822
  const denyListSlice = {
@@ -4829,24 +5835,31 @@ const buildUnifiedSearch = async (config, gazetteerEntries = [], ctx = defaultCo
4829
5835
  end: offset + (gazResult?.patterns.length ?? 0)
4830
5836
  };
4831
5837
  offset = gazetteerSlice.end;
4832
- const wrapWholeWord = (s) => ({
5838
+ const wrapWholeWord = (s, wholeWords) => ({
4833
5839
  pattern: s,
4834
5840
  literal: true,
4835
- wholeWords: true
5841
+ wholeWords
4836
5842
  });
5843
+ const customDenyListNeedsWholeWords = (pattern) => {
5844
+ const first = pattern.at(0) ?? "";
5845
+ const last = pattern.at(-1) ?? "";
5846
+ return ALNUM_RE.test(first) && ALNUM_RE.test(last);
5847
+ };
4837
5848
  const literalAllPatterns = [
4838
- ...denyListOriginals.map(wrapWholeWord),
4839
- ...streetTypes.map(wrapWholeWord),
5849
+ ...denyListOriginals.map((pattern, index) => wrapWholeWord(pattern, (denyListData?.sources[index] ?? []).includes("custom-deny-list") ? customDenyListNeedsWholeWords(pattern) : true)),
5850
+ ...streetTypes.map((pattern) => wrapWholeWord(pattern, true)),
4840
5851
  ...gazResult?.patterns ?? []
4841
5852
  ];
4842
5853
  return {
4843
5854
  tsRegex,
5855
+ tsCustomRegex,
4844
5856
  tsLiterals: literalAllPatterns.length > 0 ? new (getTextSearch())(literalAllPatterns, {
4845
5857
  caseInsensitive: true,
4846
5858
  overlapStrategy: "all"
4847
5859
  }) : new (getTextSearch())([]),
4848
5860
  slices: {
4849
5861
  regex: regexSlice,
5862
+ customRegex: customRegexSlice,
4850
5863
  legalForms: legalFormsSlice,
4851
5864
  triggers: triggersSlice,
4852
5865
  denyList: denyListSlice,
@@ -4854,6 +5867,7 @@ const buildUnifiedSearch = async (config, gazetteerEntries = [], ctx = defaultCo
4854
5867
  gazetteer: gazetteerSlice
4855
5868
  },
4856
5869
  regexMeta,
5870
+ customRegexMeta,
4857
5871
  triggerRules: triggers.rules,
4858
5872
  denyListData,
4859
5873
  gazetteerData: gazResult?.data ?? null
@@ -4863,9 +5877,11 @@ const buildUnifiedSearch = async (config, gazetteerEntries = [], ctx = defaultCo
4863
5877
  //#region src/unified-search.ts
4864
5878
  const runUnifiedSearch = (instance, fullText) => {
4865
5879
  const regexMatches = instance.tsRegex.findIter(fullText);
5880
+ const customRegexMatches = instance.tsCustomRegex.findIter(fullText);
4866
5881
  const normalized = normalizeForSearch(fullText);
4867
5882
  return {
4868
5883
  regexMatches,
5884
+ customRegexMatches,
4869
5885
  literalMatches: instance.tsLiterals.findIter(normalized)
4870
5886
  };
4871
5887
  };
@@ -4978,9 +5994,13 @@ const unmaskNerEntities = (nerEntities, maskResult, fullText) => {
4978
5994
  * so the containment rule trusts their length.
4979
5995
  */
4980
5996
  const LITERAL_SOURCES = new Set(["deny-list", "gazetteer"]);
5997
+ const isCallerOwnedEntity = (entity) => entity.sourceDetail === "custom-deny-list" || entity.sourceDetail === "custom-regex";
5998
+ const hasLockedBoundary = (entity) => isCallerOwnedEntity(entity);
4981
5999
  const shouldReplace = (a, b) => {
4982
6000
  const aLen = a.end - a.start;
4983
6001
  const bLen = b.end - b.start;
6002
+ const aCallerOwned = isCallerOwnedEntity(a);
6003
+ if (aCallerOwned !== isCallerOwnedEntity(b)) return aCallerOwned;
4984
6004
  if (a.label === b.label && LITERAL_SOURCES.has(a.source) && a.start <= b.start && a.end >= b.end && aLen > bLen) return true;
4985
6005
  if (a.label === b.label && LITERAL_SOURCES.has(b.source) && b.start <= a.start && b.end >= a.end && bLen > aLen) return false;
4986
6006
  const aPri = DETECTOR_PRIORITY[a.source] ?? 0;
@@ -4990,12 +6010,89 @@ const shouldReplace = (a, b) => {
4990
6010
  };
4991
6011
  /** Labels where colons are structurally significant. */
4992
6012
  const COLON_LABELS = new Set(["ip address", "mac address"]);
6013
+ /**
6014
+ * Labels whose detectors emit precise, evidence-backed spans. When
6015
+ * one of these fires at the exact same offsets as a fuzzier
6016
+ * `address` hit (city dictionary lookup, address-seed cluster), the
6017
+ * `address` label is almost always a dictionary collision — "March
6018
+ * 15" the date getting tagged as an address because "March" appears
6019
+ * in a city corpus, or "Brent Phillips" emitted as both `person`
6020
+ * and `address` because "Brent" is a UK city. The precise detector
6021
+ * wins.
6022
+ */
6023
+ const PRECISE_OVER_ADDRESS = new Set([
6024
+ "person",
6025
+ "date",
6026
+ "date of birth",
6027
+ "phone number",
6028
+ "email address",
6029
+ "monetary amount",
6030
+ "iban",
6031
+ "bank account number",
6032
+ "tax identification number",
6033
+ "registration number",
6034
+ "identity card number",
6035
+ "national identification number",
6036
+ "passport number",
6037
+ "credit card number"
6038
+ ]);
6039
+ /**
6040
+ * Labels the `person` chain wins against at identical offsets. The
6041
+ * chain carries adjacent-name evidence (deny-list surname plus a
6042
+ * capitalised follow-up) a single-token dictionary collision does
6043
+ * not. Kept narrow: organizations are NOT here — "Morgan Stanley"
6044
+ * legitimately appears in both the org and name dictionaries, and
6045
+ * the existing detector priority is the right tie-breaker there.
6046
+ */
6047
+ const PERSON_PREFERRED_OVER = new Set(["address", "land parcel"]);
6048
+ const resolveSameSpanLabelConflicts = (entities) => {
6049
+ if (entities.length < 2) return entities;
6050
+ const byOffsets = /* @__PURE__ */ new Map();
6051
+ for (const entity of entities) {
6052
+ const key = `${entity.start}:${entity.end}`;
6053
+ const list = byOffsets.get(key);
6054
+ if (list) list.push(entity);
6055
+ else byOffsets.set(key, [entity]);
6056
+ }
6057
+ const dropped = /* @__PURE__ */ new Set();
6058
+ for (const [, group] of byOffsets) {
6059
+ if (group.length < 2) continue;
6060
+ const labels = new Set(group.map((e) => e.label));
6061
+ if (labels.size < 2) continue;
6062
+ const hasPerson = labels.has("person");
6063
+ const hasPreciseNonAddress = [...labels].some((l) => l !== "address" && PRECISE_OVER_ADDRESS.has(l));
6064
+ let maxPriority = -1;
6065
+ for (const e of group) {
6066
+ if (hasLockedBoundary(e)) continue;
6067
+ const pri = DETECTOR_PRIORITY[e.source] ?? 0;
6068
+ if (pri > maxPriority) maxPriority = pri;
6069
+ }
6070
+ for (const e of group) {
6071
+ if (hasLockedBoundary(e)) continue;
6072
+ if ((DETECTOR_PRIORITY[e.source] ?? 0) < maxPriority) {
6073
+ dropped.add(e);
6074
+ continue;
6075
+ }
6076
+ if (hasPerson && PERSON_PREFERRED_OVER.has(e.label)) {
6077
+ dropped.add(e);
6078
+ continue;
6079
+ }
6080
+ if (hasPreciseNonAddress && e.label === "address") dropped.add(e);
6081
+ }
6082
+ }
6083
+ if (dropped.size === 0) return entities;
6084
+ return entities.filter((e) => !dropped.has(e));
6085
+ };
4993
6086
  /** Strip leading/trailing whitespace and punctuation. */
4994
6087
  const sanitizeEntities = (entities) => entities.flatMap((e) => {
6088
+ if (hasLockedBoundary(e)) return [e];
4995
6089
  const strip = COLON_LABELS.has(e.label) ? /[\s,;]+/ : /[\s:,;]+/;
4996
6090
  const leadTrimmed = e.text.replace(/^(?:\.\s)+/, "").replace(new RegExp(`^${strip.source}`, strip.flags), "");
4997
6091
  const lead = e.text.length - leadTrimmed.length;
4998
- const cleaned = leadTrimmed.replace(new RegExp(`${strip.source}$`, strip.flags), "");
6092
+ let cleaned = leadTrimmed.replace(new RegExp(`${strip.source}$`, strip.flags), "");
6093
+ if (e.label === "organization" && cleaned.endsWith(".") && !LITERAL_SOURCES.has(e.source)) {
6094
+ if (!getKnownLegalSuffixes().some((suffix) => cleaned.endsWith(suffix))) cleaned = cleaned.slice(0, -1).trimEnd();
6095
+ }
4999
6096
  if (cleaned.length === 0) return [];
5000
6097
  if (!/[\p{L}\p{N}]/u.test(cleaned)) return [];
5001
6098
  const collapsed = cleaned.replace(/\s*\n\s*/g, " ").replace(/\s{2,}/g, " ");
@@ -5019,10 +6116,32 @@ const mergeAndDedup = (...layers) => {
5019
6116
  const entity = sorted[i];
5020
6117
  const last = merged[merged.length - 1];
5021
6118
  if (!entity || !last) continue;
5022
- if (last.end <= entity.start) merged.push({ ...entity });
5023
- else if (shouldReplace(entity, last)) merged[merged.length - 1] = { ...entity };
6119
+ if (last.end <= entity.start) {
6120
+ merged.push({ ...entity });
6121
+ continue;
6122
+ }
6123
+ const overlapStart = (() => {
6124
+ for (let j = merged.length - 1; j >= 0; j--) {
6125
+ const existing = merged[j];
6126
+ if (!existing || existing.end <= entity.start) return j + 1;
6127
+ }
6128
+ return 0;
6129
+ })();
6130
+ const overlaps = merged.slice(overlapStart);
6131
+ if (!overlaps.some((existing) => existing.start !== entity.start || existing.end !== entity.end)) {
6132
+ const sameLabelIndex = overlaps.findIndex((existing) => existing.label === entity.label);
6133
+ if (sameLabelIndex === -1) {
6134
+ merged.push({ ...entity });
6135
+ continue;
6136
+ }
6137
+ const actualIndex = overlapStart + sameLabelIndex;
6138
+ const sameLabel = merged[actualIndex];
6139
+ if (sameLabel && shouldReplace(entity, sameLabel)) merged[actualIndex] = { ...entity };
6140
+ continue;
6141
+ }
6142
+ if (overlaps.every((existing) => shouldReplace(entity, existing))) merged.splice(overlapStart, overlaps.length, { ...entity });
5024
6143
  }
5025
- return sanitizeEntities(merged);
6144
+ return resolveSameSpanLabelConflicts(sanitizeEntities(merged));
5026
6145
  };
5027
6146
  const escapeRegex = (s) => s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
5028
6147
  let amountWordsRe = null;
@@ -5039,7 +6158,7 @@ const getAmountWordsRe = async () => {
5039
6158
  return amountWordsRe;
5040
6159
  };
5041
6160
  const extendMonetaryAmountWords = (entities, fullText, re) => entities.map((e) => {
5042
- if (e.label !== "monetary amount") return e;
6161
+ if (e.label !== "monetary amount" || isCallerOwnedEntity(e)) return e;
5043
6162
  const after = fullText.slice(e.end);
5044
6163
  const m = re.exec(after);
5045
6164
  if (!m) return e;
@@ -5052,22 +6171,34 @@ const extendMonetaryAmountWords = (entities, fullText, re) => entities.map((e) =
5052
6171
  });
5053
6172
  const createAllowedLabelSetFromLabels = (labels) => labels.length > 0 ? new Set(labels) : null;
5054
6173
  const createAllowedLabelSet = (config) => createAllowedLabelSetFromLabels(config.labels);
6174
+ const DEFAULT_CUSTOM_REGEX_SCORE = .9;
5055
6175
  const filterAllowedLabels = (entities, allowedLabels) => {
5056
6176
  if (!allowedLabels) return entities;
5057
6177
  return entities.filter((e) => allowedLabels.has(e.label));
5058
6178
  };
5059
6179
  const labelIsAllowed = (label, allowedLabels) => !allowedLabels || allowedLabels.has(label);
6180
+ const NON_NER_LABELS = new Set(["misc"]);
5060
6181
  const getRequestedNerLabels = (config, expandForHotwords = false) => {
5061
6182
  const labels = config.labels.length > 0 ? config.labels : DEFAULT_ENTITY_LABELS;
5062
- return expandForHotwords ? expandLabelsForHotwordRules(labels) : labels;
6183
+ return (expandForHotwords ? expandLabelsForHotwordRules(labels) : labels).filter((label) => !NON_NER_LABELS.has(label));
5063
6184
  };
5064
6185
  const checkAbort = (signal) => {
5065
6186
  if (signal?.aborted) throw new DOMException("Pipeline aborted", "AbortError");
5066
6187
  };
5067
6188
  const configKey = (config, gazetteerEntries) => {
5068
6189
  const legalFormsEnabled = isLegalFormsEnabled(config);
6190
+ const customDenyFingerprint = config.enableDenyList && config.customDenyList ? config.customDenyList.map((entry) => JSON.stringify({
6191
+ label: entry.label,
6192
+ value: entry.value,
6193
+ variants: [...entry.variants ?? []].sort()
6194
+ })).sort().join("\n") : "";
6195
+ const customRegexFingerprint = config.enableRegex && config.customRegexes ? config.customRegexes.map((entry) => JSON.stringify({
6196
+ label: entry.label,
6197
+ pattern: entry.pattern,
6198
+ score: entry.score ?? DEFAULT_CUSTOM_REGEX_SCORE
6199
+ })).sort().join("\n") : "";
5069
6200
  const gazFingerprint = config.enableGazetteer && gazetteerEntries.length > 0 ? gazetteerEntries.map((e) => `${e.id}:${e.canonical}:${e.label}:${[...e.variants].sort().join(",")}`).toSorted().join(";") : "";
5070
- return `${config.enableDenyList}:${config.enableTriggerPhrases}:${legalFormsEnabled}:${config.enableNameCorpus}:${config.denyListCountries?.toSorted().join(",") ?? ""}:${config.denyListRegions?.toSorted().join(",") ?? ""}:${config.denyListExcludeCategories?.toSorted().join(",") ?? ""}:${config.enableGazetteer}:${gazFingerprint}`;
6201
+ return `${config.enableDenyList}:${config.enableTriggerPhrases}:${legalFormsEnabled}:${config.enableNameCorpus}:${config.enableRegex}:${config.denyListCountries?.toSorted().join(",") ?? ""}:${config.denyListRegions?.toSorted().join(",") ?? ""}:${config.denyListExcludeCategories?.toSorted().join(",") ?? ""}:${customDenyFingerprint}:${customRegexFingerprint}:${config.enableGazetteer}:${gazFingerprint}`;
5071
6202
  };
5072
6203
  /**
5073
6204
  * Get or build a cached search instance. Cache state
@@ -5130,6 +6261,8 @@ const runPipeline = async (options) => {
5130
6261
  loadGenericRoles(ctx),
5131
6262
  initPrepositions(),
5132
6263
  initStreetAbbrevs(),
6264
+ initAddressComponents(),
6265
+ warmAddressStopKeywords(),
5133
6266
  zoneInit,
5134
6267
  hotwordInit
5135
6268
  ]);
@@ -5137,6 +6270,8 @@ const runPipeline = async (options) => {
5137
6270
  loadGenericRoles(ctx),
5138
6271
  initPrepositions(),
5139
6272
  initStreetAbbrevs(),
6273
+ initAddressComponents(),
6274
+ warmAddressStopKeywords(),
5140
6275
  hotwordInit
5141
6276
  ]);
5142
6277
  if (cachedSearch && config.enableDenyList) await ensureDenyListData(ctx, config.dictionaries);
@@ -5150,14 +6285,17 @@ const runPipeline = async (options) => {
5150
6285
  const preHotwordAllowedLabels = hotwordsActive ? createAllowedLabelSetFromLabels(expandLabelsForHotwordRules(config.labels)) : allowedLabels;
5151
6286
  const search = cachedSearch ?? await getCachedSearch(config, gazetteerEntries, ctx);
5152
6287
  checkAbort(signal);
5153
- const { regexMatches, literalMatches } = runUnifiedSearch(search, fullText);
6288
+ const { regexMatches, customRegexMatches, literalMatches } = runUnifiedSearch(search, fullText);
5154
6289
  const { slices } = search;
5155
6290
  const rawRegexEntities = config.enableRegex ? processRegexMatches(regexMatches, slices.regex.start, slices.regex.end, search.regexMeta) : [];
5156
- const regexEntities = filterAllowedLabels(rawRegexEntities, preHotwordAllowedLabels);
6291
+ const customRegexEntities = filterAllowedLabels(config.enableRegex ? processRegexMatches(customRegexMatches, slices.customRegex.start, slices.customRegex.end, search.customRegexMeta) : [], preHotwordAllowedLabels);
6292
+ const regexEntities = filterAllowedLabels([...rawRegexEntities, ...customRegexEntities], preHotwordAllowedLabels);
5157
6293
  if (regexEntities.length > 0) log("regex", `${regexEntities.length} matches`);
6294
+ if (legalFormsEnabled || config.enableTriggerPhrases) await warmLegalRoleHeads();
5158
6295
  const rawLegalFormEntities = legalFormsEnabled ? processLegalFormMatches(regexMatches, slices.legalForms.start, slices.legalForms.end, fullText) : [];
5159
6296
  const legalFormEntities = filterAllowedLabels(rawLegalFormEntities, preHotwordAllowedLabels);
5160
6297
  if (legalFormEntities.length > 0) log("legal-forms", `${legalFormEntities.length} matches`);
6298
+ if (config.enableTriggerPhrases) await warmAddressStopKeywords();
5161
6299
  const rawTriggerEntities = config.enableTriggerPhrases ? processTriggerMatches(regexMatches, slices.triggers.start, slices.triggers.end, fullText, search.triggerRules) : [];
5162
6300
  const triggerEntities = filterAllowedLabels(rawTriggerEntities, preHotwordAllowedLabels);
5163
6301
  if (triggerEntities.length > 0) log("trigger-phrases", `${triggerEntities.length} matches`);
@@ -5178,20 +6316,36 @@ const runPipeline = async (options) => {
5178
6316
  const gazetteerEntities = filterAllowedLabels(rawGazetteerEntities, preHotwordAllowedLabels);
5179
6317
  if (gazetteerEntities.length > 0) log("gazetteer", `${gazetteerEntities.length} matches`);
5180
6318
  checkAbort(signal);
6319
+ const rawCustomDenyListEntities = rawDenyListEntities.filter((entity) => isCallerOwnedEntity(entity));
6320
+ const rawCuratedDenyListEntities = rawDenyListEntities.filter((entity) => !isCallerOwnedEntity(entity));
6321
+ const customDenyListEntities = filterAllowedLabels(rawCustomDenyListEntities, preHotwordAllowedLabels);
5181
6322
  const ruleContextEntities = [
5182
6323
  ...rawTriggerEntities,
5183
6324
  ...rawRegexEntities,
6325
+ ...customRegexEntities,
6326
+ ...rawLegalFormEntities,
6327
+ ...rawNameCorpusEntities,
6328
+ ...rawCuratedDenyListEntities,
6329
+ ...customDenyListEntities,
6330
+ ...rawGazetteerEntities
6331
+ ];
6332
+ const nerMaskEntities = [
6333
+ ...rawTriggerEntities,
6334
+ ...rawRegexEntities,
6335
+ ...customRegexEntities,
5184
6336
  ...rawLegalFormEntities,
5185
6337
  ...rawNameCorpusEntities,
5186
- ...rawDenyListEntities,
6338
+ ...rawCuratedDenyListEntities,
6339
+ ...customDenyListEntities,
5187
6340
  ...rawGazetteerEntities
5188
6341
  ];
5189
6342
  let rawNerEntities = [];
5190
6343
  let nerEntities = [];
5191
- if (config.enableNer && nerInference) {
5192
- const maskResult = maskDetectedSpans(fullText, ruleContextEntities);
6344
+ const requestedNerLabels = getRequestedNerLabels(config, hotwordsActive);
6345
+ if (config.enableNer && nerInference && requestedNerLabels.length > 0) {
6346
+ const maskResult = maskDetectedSpans(fullText, nerMaskEntities);
5193
6347
  log("ner", "running inference...");
5194
- const rawNer = await nerInference(maskResult.maskedText, [...getRequestedNerLabels(config, hotwordsActive)], config.threshold, signal);
6348
+ const rawNer = await nerInference(maskResult.maskedText, [...requestedNerLabels], config.threshold, signal);
5195
6349
  rawNerEntities = unmaskNerEntities(rawNer, maskResult, fullText);
5196
6350
  nerEntities = filterAllowedLabels(rawNerEntities, preHotwordAllowedLabels);
5197
6351
  const masked = rawNer.length - rawNerEntities.length;
@@ -5212,7 +6366,12 @@ const runPipeline = async (options) => {
5212
6366
  if (addressSeedEntities.length > 0) log("address-seeds", `${addressSeedEntities.length} expanded`);
5213
6367
  checkAbort(signal);
5214
6368
  const zoneAdjusted = applyZoneAdjustments([...preAddressEntities, ...addressSeedEntities], zones);
5215
- const preBoostEntities = hotwordsActive ? filterAllowedLabels(applyHotwordRules(zoneAdjusted, fullText), allowedLabels) : zoneAdjusted;
6369
+ const preBoostEntities = (() => {
6370
+ if (!hotwordsActive) return zoneAdjusted;
6371
+ const hotwordCandidates = zoneAdjusted.filter((entity) => !isCallerOwnedEntity(entity));
6372
+ const callerOwnedEntities = zoneAdjusted.filter(isCallerOwnedEntity);
6373
+ return [...filterAllowedLabels(applyHotwordRules(hotwordCandidates, fullText), allowedLabels), ...callerOwnedEntities];
6374
+ })();
5216
6375
  let allEntities;
5217
6376
  if (config.enableConfidenceBoost) {
5218
6377
  allEntities = boostNearMissEntities(preBoostEntities, config.threshold);
@@ -5247,7 +6406,8 @@ const runPipeline = async (options) => {
5247
6406
  checkAbort(signal);
5248
6407
  ctx.corefSourceMap.clear();
5249
6408
  if (config.enableCoreference) {
5250
- const terms = await extractDefinedTerms(fullText, merged, ctx);
6409
+ if (!legalFormsEnabled) await warmLegalRoleHeads();
6410
+ const terms = await extractDefinedTerms(fullText, merged.filter((entity) => !isCallerOwnedEntity(entity)), ctx);
5251
6411
  if (terms.length > 0) {
5252
6412
  log("coreference", `${terms.length} defined terms`);
5253
6413
  const corefSpans = findCoreferenceSpans(fullText, terms, ctx);
@@ -5287,7 +6447,7 @@ const resolveOperator = (config, label) => config.operators[label] ?? "replace";
5287
6447
  //#region src/redact.ts
5288
6448
  const WHITESPACE_RE = /\s+/g;
5289
6449
  const PHONE_NOISE_RE = /[()\s-]/g;
5290
- const SPACE_DASH_RE = /[\s-]/g;
6450
+ const ID_SEPARATOR_RE = /[\s\-/.]/g;
5291
6451
  /**
5292
6452
  * Normalize entity text so that surface-form variations
5293
6453
  * of the same real-world value map to a single canonical
@@ -5297,8 +6457,8 @@ const normalizeEntityText = (label, text) => {
5297
6457
  const upper = label.toUpperCase().replace(WHITESPACE_RE, "_");
5298
6458
  if (upper === "EMAIL_ADDRESS" || upper === "EMAIL") return text.toLowerCase().trim();
5299
6459
  if (upper === "PHONE_NUMBER" || upper === "PHONE") return text.replace(PHONE_NOISE_RE, "");
5300
- if (upper === "IBAN" || upper === "BANK_ACCOUNT_NUMBER" || upper === "TAX_IDENTIFICATION_NUMBER" || upper === "REGISTRATION_NUMBER") return text.replace(SPACE_DASH_RE, "").toUpperCase();
5301
- if (upper === "PERSON" || upper === "ORGANIZATION" || upper === "ADDRESS" || upper === "LAND_PARCEL") return text.replace(WHITESPACE_RE, " ").toLowerCase().trim();
6460
+ if (upper === "IBAN" || upper === "BANK_ACCOUNT_NUMBER" || upper === "TAX_IDENTIFICATION_NUMBER" || upper === "REGISTRATION_NUMBER" || upper === "NATIONAL_IDENTIFICATION_NUMBER" || upper === "SOCIAL_SECURITY_NUMBER" || upper === "BIRTH_NUMBER" || upper === "IDENTITY_CARD_NUMBER" || upper === "PASSPORT_NUMBER" || upper === "CREDIT_CARD_NUMBER") return text.replace(ID_SEPARATOR_RE, "").toUpperCase();
6461
+ if (upper === "PERSON" || upper === "ORGANIZATION" || upper === "ADDRESS" || upper === "LAND_PARCEL" || upper === "MISC") return text.replace(WHITESPACE_RE, " ").toLowerCase().trim();
5302
6462
  return text.trim();
5303
6463
  };
5304
6464
  /**
@@ -5842,6 +7002,6 @@ const levenshtein = (rawA, rawB) => {
5842
7002
  //#region src/wasm.ts
5843
7003
  initTextSearch(TextSearch);
5844
7004
  //#endregion
5845
- export { CURRENCY_PATTERN_META, DATE_PATTERN_META, DEFAULT_ENTITY_LABELS, DEFAULT_OPERATOR_CONFIG, DETECTION_SOURCES, DETECTOR_PRIORITY, OPERATOR_REGISTRY, OPERATOR_TYPES, REGEX_META, REGEX_PATTERNS, REGIONS, ZONE_SCORE_ADJUSTMENTS, applyHotwordRules, applyZoneAdjustments, boostNearMissEntities, buildDenyList, buildGazetteerPatterns, buildLegalFormPatterns, buildPlaceholderMap, buildStreetTypePatterns, buildTriggerPatterns, buildUnifiedSearch, chunkText, classifyZones, computeChunkOffsets, corefKey, createPipelineContext, deanonymise, decodeSpans, decodeTokenSpans, detectNameCorpus, ensureDenyListData, exportRedactionKey, extractDefinedTerms, filterFalsePositives, findCoreferenceSpans, getCurrencyPatterns, getDatePatterns, initHotwordRules, initNameCorpus, initZoneClassifier, levenshtein, mergeAndDedup, mergeChunkEntities, normalizeForSearch, prepareBatch, processAddressSeeds, processDenyListMatches, processGazetteerMatches, processLegalFormMatches, processRegexMatches, processTriggerMatches, propagateOrgNames, redactText, resolveCountries, resolveOperator, runPipeline, runUnifiedSearch, sanitizeEntities, tokenizeText };
7005
+ export { CURRENCY_PATTERN_META, DATE_PATTERN_META, DEFAULT_ENTITY_LABELS, DEFAULT_OPERATOR_CONFIG, DETECTION_SOURCES, DETECTOR_PRIORITY, OPERATOR_REGISTRY, OPERATOR_TYPES, REGEX_META, REGEX_PATTERNS, REGIONS, ZONE_SCORE_ADJUSTMENTS, applyHotwordRules, applyZoneAdjustments, boostNearMissEntities, buildDenyList, buildGazetteerPatterns, buildLegalFormPatterns, buildPlaceholderMap, buildStreetTypePatterns, buildTriggerPatterns, buildUnifiedSearch, chunkText, classifyZones, computeChunkOffsets, corefKey, createPipelineContext, deanonymise, decodeSpans, decodeTokenSpans, detectNameCorpus, ensureDenyListData, exportRedactionKey, extractDefinedTerms, filterFalsePositives, findCoreferenceSpans, getCurrencyPatterns, getDatePatterns, initAddressComponents, initHotwordRules, initNameCorpus, initZoneClassifier, levenshtein, mergeAndDedup, mergeChunkEntities, normalizeForSearch, prepareBatch, processAddressSeeds, processDenyListMatches, processGazetteerMatches, processLegalFormMatches, processRegexMatches, processTriggerMatches, propagateOrgNames, redactText, resolveCountries, resolveOperator, runPipeline, runUnifiedSearch, sanitizeEntities, tokenizeText, warmLegalRoleHeads };
5846
7006
 
5847
7007
  //# sourceMappingURL=wasm.mjs.map