@stll/anonymize-wasm 1.2.0 → 1.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (59) hide show
  1. package/dist/address-boundaries.mjs +113 -5
  2. package/dist/address-stop-keywords.mjs +137 -0
  3. package/dist/address-stop-keywords.mjs.map +1 -0
  4. package/dist/address-stopwords.mjs +81 -0
  5. package/dist/address-stopwords.mjs.map +1 -0
  6. package/dist/address-street-types.mjs +67 -4
  7. package/dist/clause-noun-heads.mjs +75 -0
  8. package/dist/clause-noun-heads.mjs.map +1 -0
  9. package/dist/coreference.es.mjs +22 -0
  10. package/dist/coreference.es.mjs.map +1 -0
  11. package/dist/coreference.fr.mjs +32 -0
  12. package/dist/coreference.fr.mjs.map +1 -0
  13. package/dist/coreference.it.mjs +27 -0
  14. package/dist/coreference.it.mjs.map +1 -0
  15. package/dist/coreference.pl.mjs +27 -0
  16. package/dist/coreference.pl.mjs.map +1 -0
  17. package/dist/coreference.pt-br.mjs +14 -0
  18. package/dist/coreference.pt-br.mjs.map +1 -0
  19. package/dist/coreference.sk.mjs +22 -5
  20. package/dist/generic-roles.mjs +179 -1
  21. package/dist/hotword-rules.mjs +7 -3
  22. package/dist/legal-forms.mjs +3 -0
  23. package/dist/legal-role-heads.cs.mjs +42 -0
  24. package/dist/legal-role-heads.cs.mjs.map +1 -0
  25. package/dist/legal-role-heads.de.mjs +33 -0
  26. package/dist/legal-role-heads.de.mjs.map +1 -0
  27. package/dist/legal-role-heads.en.mjs +37 -0
  28. package/dist/legal-role-heads.en.mjs.map +1 -0
  29. package/dist/legal-role-heads.es.mjs +54 -0
  30. package/dist/legal-role-heads.es.mjs.map +1 -0
  31. package/dist/legal-role-heads.fr.mjs +72 -0
  32. package/dist/legal-role-heads.fr.mjs.map +1 -0
  33. package/dist/legal-role-heads.it.mjs +68 -0
  34. package/dist/legal-role-heads.it.mjs.map +1 -0
  35. package/dist/legal-role-heads.pl.mjs +84 -0
  36. package/dist/legal-role-heads.pl.mjs.map +1 -0
  37. package/dist/legal-role-heads.pt-br.mjs +63 -0
  38. package/dist/legal-role-heads.pt-br.mjs.map +1 -0
  39. package/dist/legal-role-heads.sk.mjs +80 -0
  40. package/dist/legal-role-heads.sk.mjs.map +1 -0
  41. package/dist/manifest.mjs +21 -8
  42. package/dist/names-exclusions.mjs +21 -1
  43. package/dist/person-stopwords.mjs +121 -1
  44. package/dist/section-headings.mjs +15 -3
  45. package/dist/sentence-verb-indicators.mjs +223 -0
  46. package/dist/sentence-verb-indicators.mjs.map +1 -0
  47. package/dist/triggers.cs.mjs +107 -3
  48. package/dist/triggers.es.mjs +43 -4
  49. package/dist/triggers.fr.mjs +206 -7
  50. package/dist/triggers.it.mjs +26 -4
  51. package/dist/triggers.pl.mjs +191 -4
  52. package/dist/triggers.pt-br.mjs +193 -0
  53. package/dist/triggers.pt-br.mjs.map +1 -0
  54. package/dist/triggers.sk.mjs +493 -0
  55. package/dist/vite.mjs.map +1 -1
  56. package/dist/wasm.d.mts +56 -3
  57. package/dist/wasm.mjs +2368 -1350
  58. package/dist/wasm.mjs.map +1 -1
  59. package/package.json +2 -2
package/dist/wasm.mjs CHANGED
@@ -1,5 +1,5 @@
1
1
  import { TextSearch } from "@stll/text-search-wasm";
2
- import { at, be, bg, cy, cz, de, dk, ee, es, fi, fr, gb, gr, hr, hu, ie, it, lt, lu, lv, mt, nl, pl, pt, ro, se, si, sk } from "@stll/stdnum";
2
+ import { at, be, bg, br, cy, cz, de, dk, ee, es, fi, fr, gb, gr, hr, hu, ie, it, lt, lu, lv, mt, nl, pl, pt, ro, se, si, sk } from "@stll/stdnum";
3
3
  import { toRegex } from "@stll/stdnum/patterns";
4
4
  //#region src/search-engine.ts
5
5
  let _TextSearch;
@@ -66,11 +66,15 @@ const DEFAULT_ENTITY_LABELS = [
66
66
  "iban",
67
67
  "tax identification number",
68
68
  "identity card number",
69
+ "birth number",
70
+ "national identification number",
71
+ "social security number",
69
72
  "registration number",
70
73
  "credit card number",
71
74
  "passport number",
72
75
  "monetary amount",
73
- "land parcel"
76
+ "land parcel",
77
+ "misc"
74
78
  ];
75
79
  const isLegalFormsEnabled = (config) => config.enableLegalForms !== false;
76
80
  //#endregion
@@ -95,6 +99,8 @@ const createPipelineContext = () => ({
95
99
  allowListPromise: null,
96
100
  personStopwords: null,
97
101
  personStopwordsPromise: null,
102
+ addressStopwords: null,
103
+ addressStopwordsPromise: null,
98
104
  firstNameExclusions: null,
99
105
  firstNameExclusionCorpusLen: 0,
100
106
  genericRoles: null,
@@ -151,6 +157,7 @@ const LOADER_REGISTRIES = {
151
157
  hu: () => import("./triggers.hu.mjs"),
152
158
  it: () => import("./triggers.it.mjs"),
153
159
  pl: () => import("./triggers.pl.mjs"),
160
+ "pt-br": () => import("./triggers.pt-br.mjs"),
154
161
  ro: () => import("./triggers.ro.mjs"),
155
162
  sk: () => import("./triggers.sk.mjs"),
156
163
  sv: () => import("./triggers.sv.mjs")
@@ -159,7 +166,23 @@ const LOADER_REGISTRIES = {
159
166
  cs: () => import("./coreference.cs.mjs"),
160
167
  de: () => import("./coreference.de.mjs"),
161
168
  en: () => import("./coreference.en.mjs"),
169
+ es: () => import("./coreference.es.mjs"),
170
+ fr: () => import("./coreference.fr.mjs"),
171
+ it: () => import("./coreference.it.mjs"),
172
+ pl: () => import("./coreference.pl.mjs"),
173
+ "pt-br": () => import("./coreference.pt-br.mjs"),
162
174
  sk: () => import("./coreference.sk.mjs")
175
+ },
176
+ legalRoleHeads: {
177
+ cs: () => import("./legal-role-heads.cs.mjs"),
178
+ de: () => import("./legal-role-heads.de.mjs"),
179
+ en: () => import("./legal-role-heads.en.mjs"),
180
+ es: () => import("./legal-role-heads.es.mjs"),
181
+ fr: () => import("./legal-role-heads.fr.mjs"),
182
+ it: () => import("./legal-role-heads.it.mjs"),
183
+ pl: () => import("./legal-role-heads.pl.mjs"),
184
+ "pt-br": () => import("./legal-role-heads.pt-br.mjs"),
185
+ sk: () => import("./legal-role-heads.sk.mjs")
163
186
  }
164
187
  };
165
188
  const FALLBACK_LANGUAGES = {
@@ -172,6 +195,7 @@ const FALLBACK_LANGUAGES = {
172
195
  "hu",
173
196
  "it",
174
197
  "pl",
198
+ "pt-br",
175
199
  "ro",
176
200
  "sk",
177
201
  "sv"
@@ -180,6 +204,22 @@ const FALLBACK_LANGUAGES = {
180
204
  "cs",
181
205
  "de",
182
206
  "en",
207
+ "es",
208
+ "fr",
209
+ "it",
210
+ "pl",
211
+ "pt-br",
212
+ "sk"
213
+ ],
214
+ legalRoleHeads: [
215
+ "cs",
216
+ "de",
217
+ "en",
218
+ "es",
219
+ "fr",
220
+ "it",
221
+ "pl",
222
+ "pt-br",
183
223
  "sk"
184
224
  ]
185
225
  };
@@ -226,838 +266,64 @@ const loadLanguageConfigs = async (configType, mapFn) => {
226
266
  await Promise.all(loads);
227
267
  return results.filter((r) => r !== void 0);
228
268
  };
229
- //#endregion
230
- //#region src/detectors/coreference.ts
231
- /**
232
- * Load coreference definition patterns from per-language
233
- * JSON configs in src/data/. Uses the
234
- * language manifest for auto-discovery.
235
- */
236
- const loadDefinitionPatterns = async () => {
237
- const patterns = [];
238
- const allRows = await loadLanguageConfigs("coreference", (mod) => {
239
- return mod.default ?? mod;
240
- });
241
- for (const rows of allRows) {
242
- if (!Array.isArray(rows)) {
243
- console.warn("[anonymize] coreference: unexpected config shape, skipping");
244
- continue;
245
- }
246
- for (const row of rows) try {
247
- patterns.push({ pattern: new RegExp(row.pattern, row.flags) });
248
- } catch (err) {
249
- console.warn(`[anonymize] coreference: invalid regex "${row.pattern}":`, err);
250
- }
251
- }
252
- return patterns;
253
- };
254
- /**
255
- * Load generic role terms that should NOT be treated
256
- * as PII coreferences. "Prodávající" (Seller),
257
- * "Kupující" (Buyer), etc. are legal roles, not
258
- * identifying information.
259
- */
260
- const getRoleStopSet = async (ctx) => {
261
- if (ctx.roleStopSet) return ctx.roleStopSet;
262
- if (ctx.roleStopSetPromise) return ctx.roleStopSetPromise;
263
- const promise = (async () => {
264
- let result;
265
- try {
266
- const mod = await import("./generic-roles.mjs");
267
- const data = mod.default ?? mod;
268
- result = new Set(data.roles.map((r) => r.toLowerCase()));
269
- } catch {
270
- result = /* @__PURE__ */ new Set();
271
- }
272
- ctx.roleStopSet = result;
273
- return result;
274
- })();
275
- ctx.roleStopSetPromise = promise;
276
- return promise;
277
- };
278
- const getDefinitionPatterns = async (ctx) => {
279
- if (ctx.corefPatterns) return ctx.corefPatterns;
280
- if (ctx.corefPatternsPromise) return ctx.corefPatternsPromise;
281
- ctx.corefPatternsPromise = loadDefinitionPatterns();
282
- const patterns = await ctx.corefPatternsPromise;
283
- if (patterns.length === 0) {
284
- ctx.corefPatterns = patterns;
285
- if (!ctx.corefLoadAttempted) {
286
- ctx.corefLoadAttempted = true;
287
- console.warn("[anonymize] coreference: no definition patterns loaded; coreference detection will be inactive");
288
- }
289
- return patterns;
290
- }
291
- ctx.corefPatterns = patterns;
292
- return patterns;
293
- };
294
- const SEARCH_WINDOW = 200;
295
- /**
296
- * Check whether an alias has textual similarity to
297
- * the source entity. Prevents roles and structural
298
- * terms from being treated as name aliases.
299
- *
300
- * Three checks (any passes → similar):
301
- * 1. Word overlap: a word in the alias appears in the
302
- * entity (case-insensitive, min 2 chars)
303
- * 2. Initials: alias letters match first letters of
304
- * entity words ("TB" ↔ "Tomas Bata")
305
- * 3. Substring: alias is a substring of the entity
306
- * or vice versa (min 3 chars)
307
- */
308
- const hasEntitySimilarity = (alias, entityText) => {
309
- const aliasLower = alias.toLowerCase();
310
- const entityLower = entityText.toLowerCase();
311
- if (aliasLower.length >= 3 && entityLower.includes(aliasLower)) return true;
312
- if (entityLower.length >= 3 && aliasLower.includes(entityLower)) return true;
313
- const aliasWords = aliasLower.split(/[\s.,;:'"()/-]+/).filter((w) => w.length >= 2);
314
- const entityWords = entityLower.split(/[\s.,;:'"()/-]+/).filter((w) => w.length >= 2);
315
- const entityWordSet = new Set(entityWords);
316
- for (const word of aliasWords) if (entityWordSet.has(word)) return true;
317
- if (/^[\p{Lu}]+$/u.test(alias) && alias.length >= 2 && alias.length <= entityWords.length) {
318
- for (let start = 0; start <= entityWords.length - alias.length; start++) if (entityWords.slice(start, start + alias.length).map((w) => w.charAt(0)).join("") === aliasLower) return true;
319
- }
320
- return false;
321
- };
322
- /**
323
- * Scan for defined-term patterns near known entities.
324
- *
325
- * Legal documents universally follow:
326
- * "Dr. Heinrich Muller (hereinafter 'the Seller')..."
327
- *
328
- * After NER detects the entity, this function scans for
329
- * definitional patterns within +/-200 chars and extracts
330
- * the alias. Returns alias + label pairs that can be added
331
- * to the gazetteer for a full-text re-scan.
332
- */
333
- /**
334
- * Labels that can be the source of a coreference alias.
335
- * Only parties (person, organization) have defined-term
336
- * aliases in legal text. Dates, addresses, IDs do not.
337
- */
338
- const COREF_SOURCE_LABELS = new Set(["person", "organization"]);
339
- const extractDefinedTerms = async (fullText, entities, ctx = defaultContext) => {
340
- const [definitionPatterns, roleStops] = await Promise.all([getDefinitionPatterns(ctx), getRoleStopSet(ctx)]);
341
- const terms = [];
342
- const seen = /* @__PURE__ */ new Set();
343
- const sorted = [...entities].sort((a, b) => a.start - b.start);
344
- for (const { pattern } of definitionPatterns) {
345
- pattern.lastIndex = 0;
346
- for (let match = pattern.exec(fullText); match !== null; match = pattern.exec(fullText)) {
347
- const alias = match[1]?.trim();
348
- if (!alias || alias.length < 2) continue;
349
- if (roleStops.has(alias.toLowerCase())) continue;
350
- const defPos = match.index;
351
- let bestEntity = null;
352
- for (let i = sorted.length - 1; i >= 0; i--) {
353
- const e = sorted[i];
354
- if (e === void 0) continue;
355
- if (e.end > defPos) continue;
356
- if (defPos - e.end > SEARCH_WINDOW) break;
357
- if (!COREF_SOURCE_LABELS.has(e.label)) continue;
358
- bestEntity = e;
359
- break;
360
- }
361
- if (bestEntity === null) continue;
362
- const gapText = fullText.slice(bestEntity.end, defPos);
363
- if (/(?:;|\.(?=\s*(?:["'„‚(]*\p{Lu}|$)))/u.test(gapText)) continue;
364
- if (!hasEntitySimilarity(alias, bestEntity.text)) continue;
365
- const key = `${alias.toLowerCase()}::${bestEntity.label}`;
366
- if (seen.has(key)) continue;
367
- seen.add(key);
368
- terms.push({
369
- alias,
370
- label: bestEntity.label,
371
- definitionStart: defPos,
372
- sourceText: bestEntity.text
373
- });
374
- }
375
- }
376
- return terms;
377
- };
378
- /**
379
- * Check if a character is a Unicode word character
380
- * (letter, digit, or combining mark). Used for word
381
- * boundary checks in coreference matching.
382
- */
383
- const isWordChar = (ch) => {
384
- if (ch === void 0) return false;
385
- return /[\p{L}\p{M}\p{N}]/u.test(ch);
386
- };
387
- /**
388
- * Find all occurrences of defined-term aliases in the
389
- * full text. Returns Entity spans for each match.
390
- *
391
- * Respects word boundaries: "Kupující" must not match
392
- * inside "Kupujícímu". A match is valid only if the
393
- * character before the start and after the end are NOT
394
- * word characters (letter/digit).
395
- *
396
- * Populates `ctx.corefSourceMap` with entries linking
397
- * each coref entity to its source entity text, for
398
- * consistent placeholder numbering.
399
- */
400
- const findCoreferenceSpans = (fullText, terms, ctx = defaultContext) => {
401
- const results = [];
402
- for (const term of terms) {
403
- let searchFrom = 0;
404
- while (searchFrom < fullText.length) {
405
- const idx = fullText.indexOf(term.alias, searchFrom);
406
- if (idx === -1) break;
407
- const matchEnd = idx + term.alias.length;
408
- const charBefore = idx > 0 ? fullText[idx - 1] : void 0;
409
- const charAfter = fullText[matchEnd];
410
- if (isWordChar(charBefore) || isWordChar(charAfter)) {
411
- searchFrom = idx + 1;
412
- continue;
413
- }
414
- const entity = {
415
- start: idx,
416
- end: matchEnd,
417
- label: term.label,
418
- text: term.alias,
419
- score: .95,
420
- source: DETECTION_SOURCES.COREFERENCE
421
- };
422
- ctx.corefSourceMap.set(corefKey(entity), term.sourceText);
423
- results.push(entity);
424
- searchFrom = matchEnd;
425
- }
426
- }
427
- return results;
428
- };
429
- //#endregion
430
- //#region src/detectors/gazetteer.ts
431
- const MAX_EDIT_DISTANCE = 2;
432
- const MIN_FUZZY_LENGTH = 4;
433
- const MAX_PREFIX_OVERSHOOT = 7;
434
- /**
435
- * Collect all searchable strings (canonical + variants)
436
- * from gazetteer entries, mapped to their labels and
437
- * entry IDs.
438
- */
439
- const buildSearchTerms = (entries) => {
440
- const terms = /* @__PURE__ */ new Map();
441
- for (const entry of entries) {
442
- const meta = {
443
- label: entry.label,
444
- entryId: entry.id
445
- };
446
- terms.set(entry.canonical, meta);
447
- for (const variant of entry.variants) terms.set(variant, meta);
448
- }
449
- return terms;
450
- };
451
- /**
452
- * Build TextSearch-compatible patterns from gazetteer
453
- * entries. Returns:
454
- * - Exact literal patterns for all terms
455
- * - Fuzzy patterns (distance: 2) for terms >= 4 chars
456
- * - Parallel metadata arrays for post-processing
457
- *
458
- * Patterns are ordered: all exact first, then all
459
- * fuzzy. The isFuzzy array marks which are which.
460
- */
461
- const buildGazetteerPatterns = (entries) => {
462
- const terms = buildSearchTerms(entries);
463
- const patterns = [];
464
- const labels = [];
465
- const isFuzzy = [];
466
- for (const [term, meta] of terms) {
467
- patterns.push({
468
- pattern: term,
469
- literal: true,
470
- wholeWords: false
471
- });
472
- labels.push(meta.label);
473
- isFuzzy.push(false);
474
- }
475
- for (const [term, meta] of terms) {
476
- if (term.length < MIN_FUZZY_LENGTH) continue;
477
- patterns.push({
478
- pattern: term,
479
- distance: MAX_EDIT_DISTANCE
480
- });
481
- labels.push(meta.label);
482
- isFuzzy.push(true);
483
- }
484
- return {
485
- patterns,
486
- data: {
487
- labels,
488
- isFuzzy
489
- }
490
- };
491
- };
492
- /**
493
- * Process gazetteer matches from the unified literal
494
- * search. Receives all matches; filters to the
495
- * gazetteer slice via sliceStart/sliceEnd.
496
- *
497
- * Exact matches get score 0.9; fuzzy matches get
498
- * 0.85. Fuzzy matches that overlap an exact match
499
- * are dropped.
500
- *
501
- * For exact matches, attempts prefix extension for
502
- * legal suffixes ("a.s.", "GmbH", "s.r.o." after
503
- * the matched term).
504
- */
505
- const processGazetteerMatches = (allMatches, sliceStart, sliceEnd, fullText, data) => {
506
- const results = [];
507
- const exactSpans = [];
508
- for (const match of allMatches) {
509
- const idx = match.pattern;
510
- if (idx < sliceStart || idx >= sliceEnd) continue;
511
- const localIdx = idx - sliceStart;
512
- if (data.isFuzzy[localIdx]) continue;
513
- const label = data.labels[localIdx];
514
- if (!label) continue;
515
- const extended = tryPrefixExtension(fullText, match.start, match.end);
516
- const end = extended?.end ?? match.end;
517
- const text = extended?.text ?? fullText.slice(match.start, match.end);
518
- exactSpans.push({
519
- start: match.start,
520
- end
521
- });
522
- results.push({
523
- start: match.start,
524
- end,
525
- label,
526
- text,
527
- score: .9,
528
- source: DETECTION_SOURCES.GAZETTEER
529
- });
530
- }
531
- for (const match of allMatches) {
532
- const idx = match.pattern;
533
- if (idx < sliceStart || idx >= sliceEnd) continue;
534
- const localIdx = idx - sliceStart;
535
- if (!data.isFuzzy[localIdx]) continue;
536
- if (match.distance === 0) continue;
537
- const label = data.labels[localIdx];
538
- if (!label) continue;
539
- if (exactSpans.some((e) => match.start < e.end && match.end > e.start)) continue;
540
- const matchText = fullText.slice(match.start, match.end);
541
- results.push({
542
- start: match.start,
543
- end: match.end,
544
- label,
545
- text: matchText,
546
- score: .85,
547
- source: DETECTION_SOURCES.GAZETTEER
548
- });
549
- }
550
- return results;
551
- };
552
- /**
553
- * Try to extend an exact match to capture one
554
- * trailing token (max 6 chars) that may be a legal
555
- * entity suffix (e.g., "a.s.", "GmbH", "s.r.o.").
556
- *
557
- * Does not validate the token against a legal-forms
558
- * list; false extensions are filtered by mergeAndDedup
559
- * when a legal-form detector produces a competing
560
- * entity with the correct span.
561
- */
562
- const tryPrefixExtension = (fullText, start, end) => {
563
- const maxEnd = Math.min(end + MAX_PREFIX_OVERSHOOT, fullText.length);
564
- if (maxEnd <= end + 1) return null;
565
- const after = fullText.slice(end, maxEnd);
566
- if (!after.startsWith(" ")) return null;
567
- const nextSpace = after.indexOf(" ", 1);
568
- const suffixEnd = nextSpace !== -1 ? nextSpace : after.length;
569
- if (suffixEnd <= 1) return null;
570
- const newEnd = end + suffixEnd;
571
- return {
572
- end: newEnd,
573
- text: fullText.slice(start, newEnd)
574
- };
575
- };
576
- //#endregion
577
- //#region src/util/text.ts
578
- /**
579
- * Shared text utilities for detectors.
580
- *
581
- * Extracted from names.ts and deny-list.ts to avoid
582
- * duplicating regex constants and helper functions.
583
- */
584
- /** Matches a string that starts with an uppercase letter. */
585
- const UPPER_START_RE = /^\p{Lu}/u;
586
- /** Matches a string consisting entirely of uppercase letters. */
587
- const ALL_UPPER_RE = /^\p{Lu}+$/u;
588
- const SENTENCE_END_RE = /[.!?]/;
589
- /**
590
- * Detect whether a position is at the start of a sentence.
591
- * Looks backward past whitespace for sentence-ending
592
- * punctuation (.!?). Position 0 and positions preceded
593
- * only by whitespace are considered sentence starts.
594
- */
595
- const isSentenceStart = (text, pos) => {
596
- if (pos === 0) return true;
597
- let i = pos - 1;
598
- while (i >= 0 && /\s/.test(text[i] ?? "")) i--;
599
- if (i < 0) return true;
600
- return SENTENCE_END_RE.test(text[i] ?? "");
601
- };
602
- //#endregion
603
- //#region src/detectors/names.ts
604
- const getCorpus = (ctx) => ctx.nameCorpus;
605
- const getNameCorpusFirstNames = (ctx = defaultContext) => ctx.nameCorpus?.firstNamesList ?? [];
606
- const getNameCorpusSurnames = (ctx = defaultContext) => ctx.nameCorpus?.surnamesList ?? [];
607
- const getNameCorpusTitles = (ctx = defaultContext) => ctx.nameCorpus?.titlesList ?? [];
608
- /**
609
- * Load name corpus data from injected dictionaries
610
- * and legacy config files. Merges all sources.
611
- *
612
- * Safe to call multiple times; only loads once per
613
- * context. Must be called before detectNameCorpus or
614
- * the getNameCorpus*() accessors are used.
615
- *
616
- * @param dictionaries Optional pre-loaded dictionaries
617
- * with per-language first names and surnames. When
618
- * omitted, only legacy config files are used.
619
- */
620
- const initNameCorpus = (ctx = defaultContext, dictionaries) => {
621
- if (ctx.nameCorpusPromise) return ctx.nameCorpusPromise;
622
- const promise = (async () => {
623
- try {
624
- const [legacyFirstMod, legacySurnameMod, titleMod, exclusionMod] = await Promise.all([
625
- import("./names-first.mjs"),
626
- import("./names-surnames.mjs"),
627
- import("./names-title-tokens.mjs"),
628
- import("./names-exclusions.mjs")
629
- ]);
630
- const firstNames = [...legacyFirstMod.default.names];
631
- if (dictionaries?.firstNames) for (const names of Object.values(dictionaries.firstNames)) for (const name of names) firstNames.push(name);
632
- const surnames = [...legacySurnameMod.default.names];
633
- if (dictionaries?.surnames) for (const names of Object.values(dictionaries.surnames)) for (const name of names) surnames.push(name);
634
- const dedup = (arr) => {
635
- const seen = /* @__PURE__ */ new Set();
636
- const result = [];
637
- for (const item of arr) {
638
- if (seen.has(item)) continue;
639
- seen.add(item);
640
- result.push(item);
641
- }
642
- return result;
643
- };
644
- const dedupFirst = dedup(firstNames);
645
- const dedupSurnames = dedup(surnames);
646
- const titles = titleMod.default.tokens;
647
- const exclusions = exclusionMod.default.words;
648
- ctx.nameCorpus = {
649
- firstNames: Object.freeze(new Set(dedupFirst)),
650
- surnames: Object.freeze(new Set(dedupSurnames)),
651
- titleTokens: Object.freeze(new Set(titles)),
652
- excludedWords: Object.freeze(new Set(exclusions)),
653
- firstNamesList: Object.freeze(dedupFirst),
654
- surnamesList: Object.freeze(dedupSurnames),
655
- titlesList: Object.freeze(titles),
656
- excludedList: Object.freeze(exclusions)
657
- };
658
- } catch (err) {
659
- ctx.nameCorpusPromise = null;
660
- console.warn("[anonymize] Failed to load name corpus JSON — name detection disabled:", err);
661
- }
662
- })();
663
- ctx.nameCorpusPromise = promise;
664
- return promise;
665
- };
666
- const INFLECTION_SUFFIXES = [
667
- "ovi",
668
- "em",
669
- "om",
670
- "ou",
671
- "é",
672
- "a",
673
- "u"
674
- ];
675
- /**
676
- * Strip common Czech/Slovak case suffixes from a token.
677
- * Returns candidate base forms if stripping produces a
678
- * plausible name (capitalised, length >= 3).
679
- *
680
- * For the "-ou" instrumental feminine suffix, also yields
681
- * base + "a" (e.g., "Editou" → "Edit" and "Edita")
682
- * because Czech feminine names decline -a → -ou.
683
- */
684
- const stripInflection = (token) => {
685
- const candidates = [];
686
- for (const suffix of INFLECTION_SUFFIXES) if (token.length > suffix.length + 2 && token.endsWith(suffix)) {
687
- const base = token.slice(0, -suffix.length);
688
- if (/^\p{Lu}/u.test(base)) {
689
- candidates.push(base);
690
- if (suffix === "ou" || suffix === "é" || suffix === "u") candidates.push(`${base}a`);
691
- }
692
- }
693
- return candidates;
694
- };
695
- const TOKEN_TYPE = {
696
- NAME: "name",
697
- SURNAME: "surname",
698
- TITLE: "title",
699
- ABBREVIATION: "abbreviation",
700
- CAPITALIZED: "capitalized",
701
- OTHER: "other"
702
- };
703
- const PERSON_CHAIN_BREAK_RE$1 = /[!?;:]/u;
704
- const isInitialContinuationGap$1 = (text, gap) => /^\p{Lu}$/u.test(text) && /^\.[^\S\n]{1,2}$/u.test(gap) || /^[^\S\n]{1,2}(?:\p{Lu}\.[^\S\n]{1,2})+$/u.test(gap);
705
- /**
706
- * Check if a token is in the first-name set, either
707
- * directly or after stripping Czech/Slovak inflection.
708
- */
709
- const isFirstNameToken = (token, corpus) => {
710
- if (corpus.firstNames.has(token)) return true;
711
- return stripInflection(token).some((b) => corpus.firstNames.has(b));
712
- };
713
- /**
714
- * Check if a token is in the surname set, either
715
- * directly or after stripping Czech/Slovak inflection.
716
- */
717
- const isSurnameToken = (token, corpus) => {
718
- if (corpus.surnames.has(token)) return true;
719
- return stripInflection(token).some((b) => corpus.surnames.has(b));
720
- };
721
- /**
722
- * Check if a token looks like a single-letter
723
- * abbreviation: "J.", "M.", etc.
724
- */
725
- const isAbbreviation = (token) => token.length === 2 && /^\p{Lu}$/u.test(token[0] ?? "") && token[1] === ".";
726
- const segmenter$1 = new Intl.Segmenter(void 0, { granularity: "word" });
727
- /**
728
- * Split text into word segments using Intl.Segmenter.
729
- * Only returns segments flagged as words.
730
- */
731
- const segmentWords = (fullText) => {
732
- const words = [];
733
- for (const seg of segmenter$1.segment(fullText)) if (seg.isWordLike) words.push({
734
- text: seg.segment,
735
- start: seg.index,
736
- end: seg.index + seg.segment.length
737
- });
738
- return words;
739
- };
740
- /** NAME or SURNAME — both represent corpus-matched tokens */
741
- const isCorpusMatch = (type) => type === TOKEN_TYPE.NAME || type === TOKEN_TYPE.SURNAME;
742
- const classifyToken = (word, corpus) => {
743
- const { text, start, end } = word;
744
- const lower = text.toLowerCase();
745
- const stripped = text.endsWith(".") ? text.slice(0, -1).toLowerCase() : lower;
746
- if (corpus.titleTokens.has(stripped)) return {
747
- text,
748
- type: TOKEN_TYPE.TITLE,
749
- start,
750
- end
751
- };
752
- if (isAbbreviation(text)) return {
753
- text,
754
- type: TOKEN_TYPE.ABBREVIATION,
755
- start,
756
- end
757
- };
758
- if (corpus.excludedWords.has(lower)) return {
759
- text,
760
- type: TOKEN_TYPE.OTHER,
761
- start,
762
- end
763
- };
764
- if (text.length < 3) return {
765
- text,
766
- type: TOKEN_TYPE.OTHER,
767
- start,
768
- end
769
- };
770
- if (text.length > 3 && ALL_UPPER_RE.test(text)) return {
771
- text,
772
- type: TOKEN_TYPE.OTHER,
773
- start,
774
- end
775
- };
776
- if (!UPPER_START_RE.test(text)) return {
777
- text,
778
- type: TOKEN_TYPE.OTHER,
779
- start,
780
- end
781
- };
782
- if (isFirstNameToken(text, corpus)) return {
783
- text,
784
- type: TOKEN_TYPE.NAME,
785
- start,
786
- end
787
- };
788
- if (isSurnameToken(text, corpus)) return {
789
- text,
790
- type: TOKEN_TYPE.SURNAME,
791
- start,
792
- end
793
- };
794
- return {
795
- text,
796
- type: TOKEN_TYPE.CAPITALIZED,
797
- start,
798
- end
799
- };
800
- };
801
- /**
802
- * Detect person names by looking up tokens against the
803
- * name corpus, then chaining adjacent name-like tokens.
804
- *
805
- * Requires initNameCorpus() to have been called first.
806
- * If not initialized, returns an empty array.
807
- *
808
- * Scoring:
809
- * TITLE + NAME/SURNAME → 0.95
810
- * NAME + NAME/SURNAME → 0.9
811
- * SURNAME + NAME/SURNAME → 0.9
812
- * NAME + CAPITALIZED → 0.7
813
- * ABBREVIATION + NAME → 0.7
814
- * Standalone NAME → 0.5 (low confidence)
815
- * Standalone SURNAME → skip (too ambiguous)
816
- */
817
- const detectNameCorpus = (fullText, ctx = defaultContext) => {
818
- const corpus = getCorpus(ctx);
819
- if (!corpus) return [];
820
- const tokens = segmentWords(fullText).map((w) => classifyToken(w, corpus));
821
- const entities = [];
822
- const consumed = /* @__PURE__ */ new Set();
823
- for (let i = 0; i < tokens.length; i++) {
824
- if (consumed.has(i)) continue;
825
- const token = tokens[i];
826
- if (!token) continue;
827
- if (token.type !== TOKEN_TYPE.TITLE && token.type !== TOKEN_TYPE.NAME && token.type !== TOKEN_TYPE.SURNAME && token.type !== TOKEN_TYPE.ABBREVIATION) continue;
828
- const MAX_CHAIN = 5;
829
- const chain = [token];
830
- let j = i + 1;
831
- while (j < tokens.length && chain.length < MAX_CHAIN) {
832
- const next = tokens[j];
833
- if (!next) break;
834
- const prev = chain.at(-1);
835
- if (prev) {
836
- const gap = fullText.slice(prev.end, next.start);
837
- const breaksOnPeriod = gap.includes(".") && !isInitialContinuationGap$1(prev.text, gap);
838
- if (gap.includes("\n") || PERSON_CHAIN_BREAK_RE$1.test(gap) || breaksOnPeriod) break;
839
- }
840
- if (next.type === TOKEN_TYPE.NAME || next.type === TOKEN_TYPE.SURNAME || next.type === TOKEN_TYPE.TITLE || next.type === TOKEN_TYPE.ABBREVIATION || next.type === TOKEN_TYPE.CAPITALIZED) {
841
- chain.push(next);
842
- j++;
843
- } else break;
844
- }
845
- const hasTitle = chain.some((t) => t.type === TOKEN_TYPE.TITLE);
846
- const hasCorpusName = chain.some((t) => isCorpusMatch(t.type));
847
- const hasFirstName = chain.some((t) => t.type === TOKEN_TYPE.NAME);
848
- const hasAbbreviation = chain.some((t) => t.type === TOKEN_TYPE.ABBREVIATION);
849
- const corpusCount = chain.filter((t) => isCorpusMatch(t.type)).length;
850
- const capitalizedCount = chain.filter((t) => t.type === TOKEN_TYPE.CAPITALIZED).length;
851
- let score = 0;
852
- if (hasTitle && hasCorpusName) score = .95;
853
- else if (corpusCount >= 2) score = .9;
854
- else if (hasCorpusName && capitalizedCount > 0) score = .7;
855
- else if (hasAbbreviation && hasCorpusName) score = .7;
856
- else if (hasFirstName && chain.length === 1) {
857
- if (isSentenceStart(fullText, token.start)) continue;
858
- score = .5;
859
- } else if (!hasFirstName && chain.length === 1 && chain[0]?.type === TOKEN_TYPE.SURNAME) continue;
860
- else if (hasTitle && chain.length === 1) continue;
861
- else {
862
- if (!hasCorpusName) continue;
863
- score = .5;
864
- }
865
- const first = chain.at(0);
866
- const last = chain.at(-1);
867
- if (!first || !last) continue;
868
- const start = first.start;
869
- const end = last.end;
870
- const text = fullText.slice(start, end);
871
- for (let k = i; k < i + chain.length; k++) consumed.add(k);
872
- entities.push({
873
- start,
874
- end,
875
- label: "person",
876
- text,
877
- score,
878
- source: DETECTION_SOURCES.REGEX
879
- });
880
- }
881
- return entities;
882
- };
883
- //#endregion
884
- //#region src/config/titles.ts
885
- /**
886
- * Academic and professional title prefixes.
887
- * Plain text; the detector auto-escapes for regex.
888
- * Sorted longest-first at build time.
889
- */
890
- const TITLE_PREFIXES = [
891
- "Ing.",
892
- "Mgr.",
893
- "MgA.",
894
- "Bc.",
895
- "BcA.",
896
- "JUDr.",
897
- "MUDr.",
898
- "MVDr.",
899
- "MDDr.",
900
- "PhDr.",
901
- "RNDr.",
902
- "PaedDr.",
903
- "ThDr.",
904
- "ThLic.",
905
- "ICDr.",
906
- "RSDr.",
907
- "PharmDr.",
908
- "artD.",
909
- "akad.",
910
- "doc.",
911
- "prof.",
912
- "ao. Univ.-Prof.",
913
- "o. Univ.-Prof.",
914
- "Univ.-Prof.",
915
- "Hon.-Prof.",
916
- "em. Prof.",
917
- "Dr. med. dent.",
918
- "Dr. med. vet.",
919
- "Dr. med.",
920
- "Dr. rer. nat.",
921
- "Dr. rer. soc.",
922
- "Dr. rer. pol.",
923
- "Dr. sc. tech.",
924
- "Dr. sc. nat.",
925
- "Dr. sc. hum.",
926
- "Dr. iur.",
927
- "Dr. jur.",
928
- "Dr. theol.",
929
- "Dr. oec.",
930
- "Dr. techn.",
931
- "Dr. h. c.",
932
- "Dr. phil.",
933
- "Dr.-Ing.",
934
- "Dr. Ing.",
935
- "Dr.",
936
- "Dipl.-Wirt.-Ing.",
937
- "Dipl.-Betriebsw.",
938
- "Dipl.-Inform.",
939
- "Dipl.-Volksw.",
940
- "Dipl.-Psych.",
941
- "Dipl.-Phys.",
942
- "Dipl.-Chem.",
943
- "Dipl.-Biol.",
944
- "Dipl.-Math.",
945
- "Dipl.-Päd.",
946
- "Dipl.-Soz.",
947
- "Dipl.-Kfm.",
948
- "Dipl.-Jur.",
949
- "Dipl. Ing.",
950
- "Dipl.-Ing.",
951
- "Mag. rer. soc. oec.",
952
- "Mag. rer. nat.",
953
- "Mag. phil.",
954
- "Mag. iur.",
955
- "Mag. arch.",
956
- "Mag. pharm.",
957
- "Mag. (FH)",
958
- "Mag.",
959
- "Bakk. rer. nat.",
960
- "Bakk. techn.",
961
- "Bakk. phil.",
962
- "Bakk.",
963
- "Lic. phil.",
964
- "Lic. iur.",
965
- "Lic. oec.",
966
- "Lic. theol.",
967
- "Lic.",
968
- "Priv.-Doz.",
969
- "PD",
970
- "RA"
971
- ];
972
- /**
973
- * Courtesy/honorific titles that precede a person's
974
- * name. Sorted alphabetically. The detector escapes
975
- * dots and adds \b for entries in HONORIFIC_BOUNDARY.
976
- */
977
- const HONORIFICS = [
978
- "Avv.",
979
- "Dame",
980
- "Doamna",
981
- "Domnul",
982
- "Don",
983
- "Doña",
984
- "Dott.",
985
- "Judge",
986
- "Justice",
987
- "Lady",
988
- "Lord",
989
- "M.",
990
- "Madame",
991
- "Mademoiselle",
992
- "Maître",
993
- "Me",
994
- "Messrs",
995
- "Miss",
996
- "Mlle",
997
- "Mme",
998
- "Monsieur",
999
- "Mr",
1000
- "Mrs",
1001
- "Ms",
1002
- "President",
1003
- "Señor",
1004
- "Señora",
1005
- "Sig.",
1006
- "Sig.ra",
1007
- "Signor",
1008
- "Signora",
1009
- "Signorina",
1010
- "Sir",
1011
- "Sr.",
1012
- "Sra."
1013
- ];
1014
- /**
1015
- * Honorifics that need \b word-boundary anchors
1016
- * (short or common words that could match mid-word).
1017
- */
1018
- const HONORIFIC_BOUNDARY = new Set([
1019
- "Don",
1020
- "Doña",
1021
- "M.",
1022
- "Me",
1023
- "Señor",
1024
- "Señora"
1025
- ]);
1026
- /**
1027
- * Post-nominal degrees (comma or space separated after name).
1028
- * Plain text; the detector auto-escapes for regex.
1029
- */
1030
- const POST_NOMINALS = [
1031
- "Ph.D.",
1032
- "Ph.D",
1033
- "CSc.",
1034
- "DrSc.",
1035
- "ArtD.",
1036
- "D.Phil.",
1037
- "DPhil.",
1038
- "MPhil.",
1039
- "MBA",
1040
- "MPA",
1041
- "LL.M.",
1042
- "LL.B.",
1043
- "M.Sc.",
1044
- "B.Sc.",
1045
- "MSc.",
1046
- "BSc.",
1047
- "M.Eng.",
1048
- "B.Eng.",
1049
- "M.A.",
1050
- "B.A.",
1051
- "JCD",
1052
- "JD",
1053
- "DiS.",
1054
- "ACCA",
1055
- "FCCA",
1056
- "CIPM",
1057
- "CIPT",
1058
- "CIPP/E",
1059
- "CIPP"
1060
- ];
269
+ const LEGAL_SUFFIXES = [...[
270
+ "spol. s r.o.",
271
+ "s.r.o.",
272
+ "s. r. o.",
273
+ "a.s.",
274
+ "a. s.",
275
+ "v.o.s.",
276
+ "v. o. s.",
277
+ "k.s.",
278
+ "k. s.",
279
+ "z.s.",
280
+ "z. s.",
281
+ "z.ú.",
282
+ "z. ú.",
283
+ "o.p.s.",
284
+ "o. p. s.",
285
+ "s.p.",
286
+ "s. p.",
287
+ "GmbH",
288
+ "AG",
289
+ "SE",
290
+ "KG",
291
+ "OHG",
292
+ "Ltd.",
293
+ "Ltd",
294
+ "LLC",
295
+ "LLP",
296
+ "Inc.",
297
+ "Corp.",
298
+ "Corporation",
299
+ "Co.",
300
+ "LP",
301
+ "L.P.",
302
+ "PLC",
303
+ "plc",
304
+ "N.A.",
305
+ "N.V.",
306
+ "B.V.",
307
+ "Pty Ltd.",
308
+ "Pty Ltd",
309
+ "S.A.",
310
+ "SA",
311
+ "SAS",
312
+ "SARL",
313
+ "S.p.A.",
314
+ "Sp. z o.o.",
315
+ "Sp. k.",
316
+ "Sp. j.",
317
+ "Ltda.",
318
+ "LTDA.",
319
+ "Ltda",
320
+ "LTDA",
321
+ "S/A",
322
+ "EIRELI",
323
+ "EPP",
324
+ "ME",
325
+ "MEI"
326
+ ]].sort((a, b) => b.length - a.length);
1061
327
  //#endregion
1062
328
  //#region src/data/char-groups.json
1063
329
  var char_groups_default = {
@@ -1432,220 +698,1501 @@ var char_groups_default = {
1432
698
  "code": "U+060C"
1433
699
  },
1434
700
  {
1435
- "char": "、",
1436
- "name": "ideographic comma",
1437
- "code": "U+3001"
701
+ "char": "、",
702
+ "name": "ideographic comma",
703
+ "code": "U+3001"
704
+ },
705
+ {
706
+ "char": ",",
707
+ "name": "full-width comma",
708
+ "code": "U+FF0C"
709
+ },
710
+ {
711
+ "char": "﹐",
712
+ "name": "small comma",
713
+ "code": "U+FE50"
714
+ },
715
+ {
716
+ "char": "՝",
717
+ "name": "Armenian comma",
718
+ "code": "U+055D"
719
+ }
720
+ ]
721
+ },
722
+ "ellipsis": {
723
+ "description": "Ellipsis characters",
724
+ "chars": [{
725
+ "char": "…",
726
+ "name": "horizontal ellipsis",
727
+ "code": "U+2026"
728
+ }]
729
+ },
730
+ "question-mark": {
731
+ "description": "Question mark variants",
732
+ "chars": [
733
+ {
734
+ "char": "?",
735
+ "name": "question mark",
736
+ "code": "U+003F"
737
+ },
738
+ {
739
+ "char": "?",
740
+ "name": "full-width question mark",
741
+ "code": "U+FF1F"
742
+ },
743
+ {
744
+ "char": "⁇",
745
+ "name": "double question mark",
746
+ "code": "U+2047"
747
+ },
748
+ {
749
+ "char": "⁈",
750
+ "name": "question exclamation mark",
751
+ "code": "U+2048"
752
+ },
753
+ {
754
+ "char": "؟",
755
+ "name": "Arabic question mark",
756
+ "code": "U+061F"
757
+ },
758
+ {
759
+ "char": "‽",
760
+ "name": "interrobang",
761
+ "code": "U+203D"
762
+ }
763
+ ]
764
+ },
765
+ "exclamation-mark": {
766
+ "description": "Exclamation mark variants",
767
+ "chars": [
768
+ {
769
+ "char": "!",
770
+ "name": "exclamation mark",
771
+ "code": "U+0021"
772
+ },
773
+ {
774
+ "char": "!",
775
+ "name": "full-width exclamation mark",
776
+ "code": "U+FF01"
777
+ },
778
+ {
779
+ "char": "‼",
780
+ "name": "double exclamation mark",
781
+ "code": "U+203C"
782
+ },
783
+ {
784
+ "char": "¡",
785
+ "name": "inverted exclamation mark",
786
+ "code": "U+00A1"
787
+ }
788
+ ]
789
+ },
790
+ "semicolon": {
791
+ "description": "Semicolon variants",
792
+ "chars": [
793
+ {
794
+ "char": ";",
795
+ "name": "semicolon",
796
+ "code": "U+003B"
1438
797
  },
1439
798
  {
1440
- "char": "",
1441
- "name": "full-width comma",
1442
- "code": "U+FF0C"
799
+ "char": "؛",
800
+ "name": "Arabic semicolon",
801
+ "code": "U+061B"
1443
802
  },
1444
803
  {
1445
- "char": "",
1446
- "name": "small comma",
1447
- "code": "U+FE50"
804
+ "char": "",
805
+ "name": "full-width semicolon",
806
+ "code": "U+FF1B"
1448
807
  },
1449
808
  {
1450
- "char": "՝",
1451
- "name": "Armenian comma",
1452
- "code": "U+055D"
809
+ "char": "",
810
+ "name": "Ethiopic semicolon",
811
+ "code": "U+1364"
812
+ }
813
+ ]
814
+ },
815
+ "underscore": {
816
+ "description": "Underscore variants",
817
+ "chars": [{
818
+ "char": "_",
819
+ "name": "low line",
820
+ "code": "U+005F"
821
+ }, {
822
+ "char": "_",
823
+ "name": "full-width low line",
824
+ "code": "U+FF3F"
825
+ }]
826
+ }
827
+ }
828
+ };
829
+ //#endregion
830
+ //#region src/util/char-groups.ts
831
+ /**
832
+ * Centralized Unicode character equivalence groups.
833
+ *
834
+ * Centralized Unicode character equivalence groups.
835
+ *
836
+ * Provides helpers to build regex character classes that
837
+ * match all typographic variants of a character type
838
+ * (dashes, spaces, quotes, etc.).
839
+ *
840
+ * The JSON is statically imported so the bundler inlines
841
+ * it, avoiding a runtime require() that breaks browsers.
842
+ */
843
+ /** Chars that need escaping inside a regex char class. */
844
+ const REGEX_CLASS_SPECIAL = /[\\\]^-]/;
845
+ const escapeForCharClass = (ch) => REGEX_CLASS_SPECIAL.test(ch) ? `\\${ch}` : ch;
846
+ const config = char_groups_default;
847
+ /**
848
+ * Get the raw characters for a named group.
849
+ * Throws if the group does not exist.
850
+ */
851
+ const charSet = (group) => {
852
+ const g = config.groups[group];
853
+ if (!g) throw new Error(`Unknown char group: "${group}"`);
854
+ return g.chars.map((entry) => entry.char);
855
+ };
856
+ /**
857
+ * Build a regex character class string for a named
858
+ * group. E.g., charClass("dash") returns a string
859
+ * like "[-\u2013\u2014\u2010\u2011\u2212\u2043]".
860
+ *
861
+ * The hyphen-minus is placed first so it is treated
862
+ * as a literal, not a range indicator.
863
+ */
864
+ const charClass = (group) => {
865
+ return `[${[...charSet(group)].sort((a, b) => {
866
+ if (a === "-") return -1;
867
+ if (b === "-") return 1;
868
+ return a.localeCompare(b);
869
+ }).map(escapeForCharClass).join("")}]`;
870
+ };
871
+ /**
872
+ * Return the inner content of a character class (without
873
+ * the surrounding brackets). Useful for embedding a group
874
+ * inside a larger character class, e.g.:
875
+ * `[\\s&,.${charClassInner("dash")}]`
876
+ */
877
+ const charClassInner = (group) => {
878
+ return [...charSet(group)].sort((a, b) => {
879
+ if (a === "-") return -1;
880
+ if (b === "-") return 1;
881
+ return a.localeCompare(b);
882
+ }).map(escapeForCharClass).join("");
883
+ };
884
+ /** Regex char class matching all dash variants. */
885
+ const DASH = charClass("dash");
886
+ /**
887
+ * Inner content of the dash char class (no brackets).
888
+ * For embedding inside a larger character class:
889
+ * `[\\s&,.${DASH_INNER}]`
890
+ */
891
+ const DASH_INNER = charClassInner("dash");
892
+ charClass("space");
893
+ charClass("quote-double");
894
+ /** Inner content of the double-quote char class (no brackets). */
895
+ const QUOTE_DOUBLE_INNER = charClassInner("quote-double");
896
+ charClass("quote-single");
897
+ /** Inner content of the single-quote char class (no brackets). */
898
+ const QUOTE_SINGLE_INNER = charClassInner("quote-single");
899
+ /**
900
+ * Inline opening-bracket characters that frequently introduce
901
+ * a defined-term parenthetical immediately after an address or
902
+ * organization span (e.g. `…GA 30326, USA (the "Premises")`).
903
+ * Hardcoded rather than data-driven because the set is tiny,
904
+ * regex-special, and not language-dependent.
905
+ */
906
+ const OPENING_BRACKETS_INNER = "(\\[{";
907
+ charClass("slash");
908
+ charClass("dot");
909
+ charClass("colon");
910
+ //#endregion
911
+ //#region src/detectors/legal-forms.ts
912
+ const SENTENCE_VERB_INDICATORS_SEED = new Set([
913
+ "je",
914
+ "jsou",
915
+ "is",
916
+ "are",
917
+ "ist",
918
+ "sind"
919
+ ]);
920
+ let sentenceVerbIndicatorsCache = null;
921
+ let sentenceVerbIndicatorsPromise = null;
922
+ const loadSentenceVerbIndicators = async () => {
923
+ if (sentenceVerbIndicatorsCache) return sentenceVerbIndicatorsCache;
924
+ if (sentenceVerbIndicatorsPromise) return sentenceVerbIndicatorsPromise;
925
+ sentenceVerbIndicatorsPromise = (async () => {
926
+ let data = {};
927
+ try {
928
+ const mod = await import("./sentence-verb-indicators.mjs");
929
+ data = mod.default ?? mod;
930
+ } catch (err) {
931
+ console.warn("[anonymize] legal-forms: failed to load sentence-verb-indicators.json, falling back to seed list:", err);
932
+ }
933
+ const all = new Set(SENTENCE_VERB_INDICATORS_SEED);
934
+ for (const [key, value] of Object.entries(data)) {
935
+ if (key.startsWith("_")) continue;
936
+ if (!Array.isArray(value)) continue;
937
+ for (const verb of value) {
938
+ if (typeof verb !== "string" || verb.length === 0) continue;
939
+ all.add(verb.toLowerCase());
940
+ }
941
+ }
942
+ sentenceVerbIndicatorsCache = all;
943
+ return all;
944
+ })();
945
+ return sentenceVerbIndicatorsPromise;
946
+ };
947
+ const getSentenceVerbIndicatorsSync = () => sentenceVerbIndicatorsCache ?? SENTENCE_VERB_INDICATORS_SEED;
948
+ const UPPER = "A-ZÁČĎÉĚÍŇÓŘŠŤÚŮÝŽÄÖÜÀÂÆÇÈÊËÎÏÔÙÛŸÑĄĆĘŁŃŚŹŻ\\u0130";
949
+ const LOWER = "a-záčďéěíňóřšťúůýžäöüßàâæçèêëîïôùûÿñąćęłńśźż\\u0131";
950
+ const CAP_WORD = `(?:[${UPPER}]{2,}|[${UPPER}][${LOWER}${UPPER}]+)`;
951
+ const ALLCAP_WORD = `[${UPPER}]{2,}`;
952
+ const ROMAN_NUMERAL_RE = /^(?=[IVXLCDM])M{0,3}(?:CM|CD|D?C{0,3})(?:XC|XL|L?X{0,3})(?:IX|IV|V?I{0,3})$/;
953
+ let legalRoleHeadsCache = null;
954
+ let legalRoleHeadsPromise = null;
955
+ const loadLegalRoleHeads = async () => {
956
+ if (legalRoleHeadsCache) return legalRoleHeadsCache;
957
+ if (legalRoleHeadsPromise) return legalRoleHeadsPromise;
958
+ legalRoleHeadsPromise = (async () => {
959
+ const sets = await loadLanguageConfigs("legalRoleHeads", (mod) => {
960
+ return mod.default ?? mod;
961
+ });
962
+ const all = /* @__PURE__ */ new Set();
963
+ for (const entry of sets) {
964
+ if (!entry || !Array.isArray(entry.words)) continue;
965
+ for (const word of entry.words) if (typeof word === "string" && word.length > 0) all.add(word.toLowerCase());
966
+ }
967
+ legalRoleHeadsCache = all;
968
+ return all;
969
+ })();
970
+ return legalRoleHeadsPromise;
971
+ };
972
+ const getLegalRoleHeadsSync = () => legalRoleHeadsCache ?? /* @__PURE__ */ new Set();
973
+ const warmLegalRoleHeads = async () => {
974
+ await Promise.all([
975
+ loadLegalRoleHeads(),
976
+ loadAllLegalSuffixes(),
977
+ loadSentenceVerbIndicators(),
978
+ loadClauseNounHeads()
979
+ ]);
980
+ };
981
+ let allLegalSuffixesCache = null;
982
+ let allLegalSuffixesPromise = null;
983
+ const loadAllLegalSuffixes = async () => {
984
+ if (allLegalSuffixesCache) return allLegalSuffixesCache;
985
+ if (allLegalSuffixesPromise) return allLegalSuffixesPromise;
986
+ allLegalSuffixesPromise = (async () => {
987
+ let data = {};
988
+ try {
989
+ data = (await import("./legal-forms.mjs")).default;
990
+ } catch {
991
+ data = {};
992
+ }
993
+ const seen = /* @__PURE__ */ new Set();
994
+ const out = [];
995
+ for (const list of Object.values(data)) for (const form of list) {
996
+ if (typeof form !== "string" || form.length === 0) continue;
997
+ if (seen.has(form)) continue;
998
+ seen.add(form);
999
+ out.push(form);
1000
+ }
1001
+ for (const form of LEGAL_SUFFIXES) if (!seen.has(form)) {
1002
+ seen.add(form);
1003
+ out.push(form);
1004
+ }
1005
+ out.sort((a, b) => b.length - a.length);
1006
+ allLegalSuffixesCache = out;
1007
+ return out;
1008
+ })();
1009
+ return allLegalSuffixesPromise;
1010
+ };
1011
+ const getAllLegalSuffixesSync = () => allLegalSuffixesCache ?? LEGAL_SUFFIXES;
1012
+ /**
1013
+ * Sync accessor for the full legal-form vocabulary
1014
+ * (`data/legal-forms.json` plus `LEGAL_SUFFIXES`,
1015
+ * longest-first). Falls back to `LEGAL_SUFFIXES` when
1016
+ * `warmLegalRoleHeads()` has not run yet. Exposed so the
1017
+ * trailing-period strip in `sanitizeEntities` can keep
1018
+ * pace with the detector vocabulary rather than only the
1019
+ * smaller `LEGAL_SUFFIXES` propagation list.
1020
+ */
1021
+ const getKnownLegalSuffixes = getAllLegalSuffixesSync;
1022
+ const CLAUSE_NOUN_HEADS_SEED = new Set(["agreement", "contract"]);
1023
+ let clauseNounHeadsCache = null;
1024
+ let clauseNounHeadsPromise = null;
1025
+ const loadClauseNounHeads = async () => {
1026
+ if (clauseNounHeadsCache) return clauseNounHeadsCache;
1027
+ if (clauseNounHeadsPromise) return clauseNounHeadsPromise;
1028
+ clauseNounHeadsPromise = (async () => {
1029
+ let data = {};
1030
+ try {
1031
+ const mod = await import("./clause-noun-heads.mjs");
1032
+ data = mod.default ?? mod;
1033
+ } catch (err) {
1034
+ console.warn("[anonymize] legal-forms: failed to load clause-noun-heads.json, falling back to seed list:", err);
1035
+ }
1036
+ const all = new Set(CLAUSE_NOUN_HEADS_SEED);
1037
+ for (const [key, value] of Object.entries(data)) {
1038
+ if (key.startsWith("_")) continue;
1039
+ if (!Array.isArray(value)) continue;
1040
+ for (const noun of value) {
1041
+ if (typeof noun !== "string" || noun.length === 0) continue;
1042
+ all.add(noun.toLowerCase());
1043
+ }
1044
+ }
1045
+ clauseNounHeadsCache = all;
1046
+ return all;
1047
+ })();
1048
+ return clauseNounHeadsPromise;
1049
+ };
1050
+ const getClauseNounHeadsSync = () => clauseNounHeadsCache ?? CLAUSE_NOUN_HEADS_SEED;
1051
+ const escapeForRegex = (form) => form.replace(/[.*+?^${}()|[\]\\]/g, "\\$&").replace(/\s+/g, "\\s+").replace(/\\\./g, "\\.[^\\S\\n]?");
1052
+ const isShortForm = (form) => form.replace(/[.\s]/g, "").length <= 3 && !form.includes(" ");
1053
+ const buildPatternString = (forms) => {
1054
+ if (forms.length === 0) return null;
1055
+ const alt = forms.toSorted((a, b) => b.length - a.length).map(escapeForRegex).join("|");
1056
+ const HSPACE = "[^\\S\\n]";
1057
+ const LOWER_CONNECTOR = `${HSPACE}+(?:a|and|und|et|e|y|i)${HSPACE}+(?=[${LOWER}])`;
1058
+ const SIMPLE_SEP = `(?:${HSPACE}|[&,.${DASH_INNER}]){1,4}`;
1059
+ const CAP_OR_NUM_WORD = `(?:${CAP_WORD}|\\d{1,4})`;
1060
+ const LOWER_WORD = `(?:(?!(?:and|und|et)(?![${UPPER}${LOWER}]))[${LOWER}][${LOWER}${UPPER}]+)`;
1061
+ const ANY_WORD_TAIL = `(?:(?!(?:and|und|et)(?![${UPPER}${LOWER}]))[${UPPER}${LOWER}][${LOWER}${UPPER}]+|[${UPPER}]{2,3}|\\d{1,4})`;
1062
+ return `${`(?:${`(?:${CAP_WORD})(?:${SIMPLE_SEP}(?:${CAP_OR_NUM_WORD})){0,10}`})(?:${`${SIMPLE_SEP}(?:${LOWER_WORD})(?:(?:${LOWER_CONNECTOR}|${SIMPLE_SEP})(?:${ANY_WORD_TAIL})){0,10}`})?`}(?:\\s+|,\\s*)(?:${alt})(?![${LOWER}])`;
1063
+ };
1064
+ /**
1065
+ * Build legal form regex pattern strings.
1066
+ * Returns an array of regex strings for the unified
1067
+ * TextSearch builder. Empty if data package is not
1068
+ * installed.
1069
+ */
1070
+ const buildLegalFormPatterns = async () => {
1071
+ let data = {};
1072
+ try {
1073
+ data = (await import("./legal-forms.mjs")).default;
1074
+ } catch {
1075
+ return [];
1076
+ }
1077
+ const allForms = [];
1078
+ const seen = /* @__PURE__ */ new Set();
1079
+ for (const forms of Object.values(data)) for (const form of forms) if (!seen.has(form)) {
1080
+ seen.add(form);
1081
+ allForms.push(form);
1082
+ }
1083
+ for (const form of LEGAL_SUFFIXES) if (!seen.has(form)) {
1084
+ seen.add(form);
1085
+ allForms.push(form);
1086
+ }
1087
+ const patterns = [];
1088
+ const longPattern = buildPatternString(allForms.filter((f) => !isShortForm(f)));
1089
+ if (longPattern) patterns.push(longPattern);
1090
+ const shortPattern = buildPatternString(allForms.filter(isShortForm));
1091
+ if (shortPattern) patterns.push(shortPattern);
1092
+ const allcapPrefix = `(?:${ALLCAP_WORD})(?:[ \\t&,.${DASH_INNER}]{1,4}(?:${ALLCAP_WORD})){0,2}`;
1093
+ const allcapAlt = allForms.toSorted((a, b) => b.length - a.length).map(escapeForRegex).join("|");
1094
+ patterns.push(`${allcapPrefix}(?:[ \\t]+|,[ \\t]*)(?:${allcapAlt})(?![${LOWER}])`);
1095
+ return patterns;
1096
+ };
1097
+ const CONNECTOR_RE = /^(?:a|and|und|et|e|y|i|&)$/i;
1098
+ const AND_TYPE_CONNECTOR_RE = /^(?:and|und|et)$/i;
1099
+ const UPPER_LETTER_RE = /^\p{Lu}/u;
1100
+ const COMPANY_SUFFIX_WORDS_RE = /^(?:Company|Co|Bank|Brothers|Bros|Sons|Group|Holdings|Trust|Partners|Associates|Corporation|Industries|Enterprises|Solutions|Systems|Services|Foundation|Institute)$/;
1101
+ const IN_NAME_PREPOSITION_RE = /^(?:of|the)$/i;
1102
+ const ENTITY_HEAD_WORD_RE = /^[\p{L}\p{M}&]+/u;
1103
+ const LEADING_CLAUSE_RE = /(?:^|\s)(?:by\s+and\s+between|is\s+between)\s+/giu;
1104
+ /**
1105
+ * Find the word ending just before `pos` in `text`,
1106
+ * skipping any whitespace (not newlines).
1107
+ * Returns null if no word is found (e.g., at start
1108
+ * of text, or preceded by non-word chars like ".").
1109
+ */
1110
+ const findWordBefore = (text, pos) => {
1111
+ let scan = pos - 1;
1112
+ while (scan >= 0) {
1113
+ const ch = text.charAt(scan);
1114
+ if (ch === "\n" || !/\s/.test(ch)) break;
1115
+ scan--;
1116
+ }
1117
+ if (scan < 0 || text.charAt(scan) === "\n") return null;
1118
+ const wordEnd = scan + 1;
1119
+ while (scan >= 0 && /[\p{L}\p{M}&]/u.test(text.charAt(scan))) scan--;
1120
+ const wordStart = scan + 1;
1121
+ const word = text.slice(wordStart, wordEnd);
1122
+ if (word.length === 0) return null;
1123
+ return {
1124
+ word,
1125
+ start: wordStart
1126
+ };
1127
+ };
1128
+ /**
1129
+ * Count consecutive uppercase-starting words immediately
1130
+ * before `pos`. Stops at the first non-upper word or at
1131
+ * text/line start. Used to disambiguate "<First> <Last>
1132
+ * and <ORG>" from "<Multi-word Org> and <Continuation>".
1133
+ */
1134
+ const countUpperWordsBefore = (fullText, pos) => {
1135
+ let count = 0;
1136
+ let scan = pos;
1137
+ while (scan > 0) {
1138
+ const found = findWordBefore(fullText, scan);
1139
+ if (!found) break;
1140
+ if (!UPPER_LETTER_RE.test(found.word)) break;
1141
+ count++;
1142
+ scan = found.start;
1143
+ }
1144
+ return count;
1145
+ };
1146
+ /**
1147
+ * Extend a match backward through uppercase words and
1148
+ * lowercase connectors. Stops at start of text,
1149
+ * newline, or a word that doesn't qualify.
1150
+ *
1151
+ * Connectors (a, and, und, et, ...) are only consumed
1152
+ * when there is a valid word before them — a trailing
1153
+ * connector at an entity boundary is not consumed.
1154
+ * For multi-char "and"-type connectors we additionally
1155
+ * refuse to cross when exactly two uppercase words
1156
+ * precede them ("First Last and ORG, Inc." shape) —
1157
+ * unless the match itself begins with a known company-
1158
+ * suffix word ("…and Company, Inc."), in which case
1159
+ * the chain belongs to one organisation. In that
1160
+ * suffix-mode we also cross in-name prepositions
1161
+ * ("Bank of America and Trust Company, Inc.").
1162
+ */
1163
+ const extendBackward = (fullText, matchStart, options = {}) => {
1164
+ const headWord = ENTITY_HEAD_WORD_RE.exec(fullText.slice(matchStart))?.[0] ?? "";
1165
+ const suffixMode = options.forceSuffixMode === true || COMPANY_SUFFIX_WORDS_RE.test(headWord);
1166
+ let pos = matchStart;
1167
+ while (pos > 0) {
1168
+ const found = findWordBefore(fullText, pos);
1169
+ if (!found) break;
1170
+ const { word, start: wordStart } = found;
1171
+ const isUpper = UPPER_LETTER_RE.test(word);
1172
+ const isConnector = CONNECTOR_RE.test(word);
1173
+ const isInNamePrep = suffixMode && IN_NAME_PREPOSITION_RE.test(word);
1174
+ if (isUpper) pos = wordStart;
1175
+ else if (isConnector) {
1176
+ if (!suffixMode && AND_TYPE_CONNECTOR_RE.test(word) && countUpperWordsBefore(fullText, wordStart) === 2) break;
1177
+ const prev = findWordBefore(fullText, wordStart);
1178
+ if (!prev) break;
1179
+ if (!UPPER_LETTER_RE.test(prev.word)) break;
1180
+ pos = prev.start;
1181
+ } else if (isInNamePrep) {
1182
+ const prev = findWordBefore(fullText, wordStart);
1183
+ if (!prev) break;
1184
+ if (!UPPER_LETTER_RE.test(prev.word)) break;
1185
+ pos = prev.start;
1186
+ } else break;
1187
+ }
1188
+ return pos;
1189
+ };
1190
+ const trimLeadingClause = (text) => {
1191
+ let cut = -1;
1192
+ for (const match of text.matchAll(LEADING_CLAUSE_RE)) cut = match.index + match[0].length;
1193
+ if (cut <= 0) return {
1194
+ offset: 0,
1195
+ text
1196
+ };
1197
+ const trimmed = text.slice(cut);
1198
+ const leadingWs = trimmed.match(/^\s*/u)?.[0].length ?? 0;
1199
+ return {
1200
+ offset: cut + leadingWs,
1201
+ text: trimmed.slice(leadingWs)
1202
+ };
1203
+ };
1204
+ /**
1205
+ * Process legal form matches from the unified search.
1206
+ * Receives all matches; filters to the legal forms
1207
+ * slice via sliceStart/sliceEnd.
1208
+ *
1209
+ * The role-head trimming step reads per-language data from
1210
+ * a cache that `runPipeline` warms via `warmLegalRoleHeads()`
1211
+ * before calling this. Callers that invoke
1212
+ * `processLegalFormMatches` directly (without going through
1213
+ * `runPipeline`) must `await warmLegalRoleHeads()` first;
1214
+ * otherwise the trim falls back to a no-op and sentence-
1215
+ * fragment fixes do not apply.
1216
+ */
1217
+ const processLegalFormMatches = (allMatches, sliceStart, sliceEnd, fullText) => {
1218
+ const results = [];
1219
+ for (const match of allMatches) {
1220
+ const idx = match.pattern;
1221
+ if (idx < sliceStart || idx >= sliceEnd) continue;
1222
+ const text = match.text.trimEnd();
1223
+ if (text.length < 5) continue;
1224
+ const roleHeads = getLegalRoleHeadsSync();
1225
+ const firstWordMatch = /^[\p{L}\p{M}]+(?:-[\p{L}\p{M}]+)*/u.exec(text);
1226
+ let processedStart = match.start;
1227
+ let processedText = text;
1228
+ let trimmed = false;
1229
+ const firstWordText = firstWordMatch?.[0] ?? "";
1230
+ const firstWordLeading = /^[\p{L}\p{M}]+/u.exec(firstWordText)?.[0] ?? "";
1231
+ if (firstWordMatch !== null && (roleHeads.has(firstWordText.toLowerCase()) || firstWordLeading.length > 0 && roleHeads.has(firstWordLeading.toLowerCase()))) {
1232
+ let suffixOffset = -1;
1233
+ for (const suffix of getAllLegalSuffixesSync()) {
1234
+ const idx = text.lastIndexOf(suffix);
1235
+ if (idx !== -1 && idx + suffix.length >= text.length - 1) {
1236
+ suffixOffset = idx;
1237
+ break;
1453
1238
  }
1454
- ]
1455
- },
1456
- "ellipsis": {
1457
- "description": "Ellipsis characters",
1458
- "chars": [{
1459
- "char": "…",
1460
- "name": "horizontal ellipsis",
1461
- "code": "U+2026"
1462
- }]
1463
- },
1464
- "question-mark": {
1465
- "description": "Question mark variants",
1466
- "chars": [
1467
- {
1468
- "char": "?",
1469
- "name": "question mark",
1470
- "code": "U+003F"
1471
- },
1472
- {
1473
- "char": "?",
1474
- "name": "full-width question mark",
1475
- "code": "U+FF1F"
1476
- },
1477
- {
1478
- "char": "⁇",
1479
- "name": "double question mark",
1480
- "code": "U+2047"
1481
- },
1482
- {
1483
- "char": "⁈",
1484
- "name": "question exclamation mark",
1485
- "code": "U+2048"
1486
- },
1487
- {
1488
- "char": "؟",
1489
- "name": "Arabic question mark",
1490
- "code": "U+061F"
1491
- },
1492
- {
1493
- "char": "‽",
1494
- "name": "interrobang",
1495
- "code": "U+203D"
1239
+ }
1240
+ if (suffixOffset < 0) {} else {
1241
+ const midStart = firstWordMatch[0].length;
1242
+ const midEnd = suffixOffset;
1243
+ const midSection = text.slice(midStart, midEnd);
1244
+ const verbIndicators = getSentenceVerbIndicatorsSync();
1245
+ let lastVerbEndInMid = -1;
1246
+ for (const match of midSection.matchAll(/(?<![\p{L}\p{N}])[\p{L}\p{M}]+/gu)) if (match[0] !== void 0 && match.index !== void 0 && verbIndicators.has(match[0].toLowerCase())) lastVerbEndInMid = match.index + match[0].length;
1247
+ const digitAfterRole = /^\s+\d+(?:\.|\b)/u.test(midSection);
1248
+ let appositiveRoleHead = false;
1249
+ if (!digitAfterRole && lastVerbEndInMid === -1 && fullText) {
1250
+ const before = fullText.slice(Math.max(0, match.start - 40), match.start);
1251
+ const prevWord = /(?<![\p{L}\p{N}])(\p{L}[\p{L}\p{M}]*)\s*$/u.exec(before);
1252
+ if (prevWord !== null && getSentenceVerbIndicatorsSync().has(prevWord[1].toLowerCase())) appositiveRoleHead = true;
1496
1253
  }
1497
- ]
1498
- },
1499
- "exclamation-mark": {
1500
- "description": "Exclamation mark variants",
1501
- "chars": [
1502
- {
1503
- "char": "!",
1504
- "name": "exclamation mark",
1505
- "code": "U+0021"
1506
- },
1507
- {
1508
- "char": "!",
1509
- "name": "full-width exclamation mark",
1510
- "code": "U+FF01"
1511
- },
1512
- {
1513
- "char": "‼",
1514
- "name": "double exclamation mark",
1515
- "code": "U+203C"
1516
- },
1517
- {
1518
- "char": "¡",
1519
- "name": "inverted exclamation mark",
1520
- "code": "U+00A1"
1254
+ if (lastVerbEndInMid !== -1 || digitAfterRole || appositiveRoleHead) {
1255
+ const scanStart = lastVerbEndInMid !== -1 ? midStart + lastVerbEndInMid : midStart;
1256
+ const capRe = /(?<![\p{L}\p{N}])\p{Lu}[\p{L}\p{M}\p{N}]*/gu;
1257
+ capRe.lastIndex = scanStart;
1258
+ const clauseNouns = getClauseNounHeadsSync();
1259
+ let capMatch = null;
1260
+ for (let next = capRe.exec(text); next !== null; next = capRe.exec(text)) {
1261
+ if (next.index >= suffixOffset) break;
1262
+ const lc = next[0].toLowerCase();
1263
+ if (roleHeads.has(lc) || clauseNouns.has(lc)) continue;
1264
+ capMatch = next;
1265
+ break;
1266
+ }
1267
+ if (capMatch === null) continue;
1268
+ processedStart = match.start + capMatch.index;
1269
+ processedText = text.slice(capMatch.index);
1270
+ trimmed = true;
1521
1271
  }
1522
- ]
1523
- },
1524
- "semicolon": {
1525
- "description": "Semicolon variants",
1526
- "chars": [
1527
- {
1528
- "char": ";",
1529
- "name": "semicolon",
1530
- "code": "U+003B"
1531
- },
1532
- {
1533
- "char": "؛",
1534
- "name": "Arabic semicolon",
1535
- "code": "U+061B"
1536
- },
1537
- {
1538
- "char": ";",
1539
- "name": "full-width semicolon",
1540
- "code": "U+FF1B"
1541
- },
1542
- {
1543
- "char": "",
1544
- "name": "Ethiopic semicolon",
1545
- "code": "U+1364"
1272
+ }
1273
+ }
1274
+ if (processedText.includes("\n")) continue;
1275
+ let entityStart = processedStart;
1276
+ let entityText = processedText;
1277
+ if (fullText && !trimmed) {
1278
+ const extended = extendBackward(fullText, processedStart);
1279
+ if (extended < processedStart) {
1280
+ entityStart = extended;
1281
+ entityText = fullText.slice(extended, processedStart + processedText.length).trimEnd();
1282
+ }
1283
+ }
1284
+ const clauseTrim = trimLeadingClause(entityText);
1285
+ if (clauseTrim.offset > 0) {
1286
+ entityStart += clauseTrim.offset;
1287
+ entityText = clauseTrim.text;
1288
+ }
1289
+ const getPrefixInfo = (value) => {
1290
+ const prefixEnd = value.lastIndexOf(",") !== -1 ? value.lastIndexOf(",") : value.lastIndexOf(" ");
1291
+ return {
1292
+ prefixEnd,
1293
+ prefixPart: prefixEnd > 0 ? value.slice(0, prefixEnd).replace(/[^a-zA-ZÀ-ž]/g, "") : value.replace(/[^a-zA-ZÀ-ž]/g, "")
1294
+ };
1295
+ };
1296
+ let { prefixEnd, prefixPart } = getPrefixInfo(entityText);
1297
+ let isAllCapsMatch = prefixPart.length > 2 && prefixPart === prefixPart.toUpperCase();
1298
+ if (isAllCapsMatch && fullText) {
1299
+ const lineStart = fullText.lastIndexOf("\n", entityStart);
1300
+ const lineEnd = fullText.indexOf("\n", entityStart + entityText.length);
1301
+ const lineLetters = fullText.slice(lineStart + 1, lineEnd === -1 ? fullText.length : lineEnd).replace(/[^a-zA-ZÀ-ž]/g, "");
1302
+ const upperCount = [...lineLetters].filter((c) => c === c.toUpperCase()).length;
1303
+ if (lineLetters.length > 5 && upperCount / lineLetters.length >= .95) continue;
1304
+ if ((prefixPart.length > 0 ? entityText.slice(0, prefixEnd > 0 ? prefixEnd : entityText.length).trim().split(/\s+/).length : 0) > 3) {
1305
+ entityStart = match.start;
1306
+ entityText = text;
1307
+ ({prefixEnd, prefixPart} = getPrefixInfo(entityText));
1308
+ isAllCapsMatch = prefixPart.length > 2 && prefixPart === prefixPart.toUpperCase();
1309
+ }
1310
+ } else if (isAllCapsMatch) continue;
1311
+ const lastSpace = entityText.lastIndexOf(" ");
1312
+ const rawSuffix = lastSpace !== -1 ? entityText.slice(lastSpace + 1) : "";
1313
+ const suffixClean = rawSuffix.replace(/[.,]/g, "");
1314
+ if (suffixClean.length > 0 && ROMAN_NUMERAL_RE.test(suffixClean)) continue;
1315
+ if (suffixClean.length <= 2 && !/\./.test(rawSuffix) && /[^\x00-\x7F]/.test(entityText.slice(0, lastSpace !== -1 ? lastSpace : entityText.length))) continue;
1316
+ results.push({
1317
+ start: entityStart,
1318
+ end: entityStart + entityText.length,
1319
+ label: "organization",
1320
+ text: entityText,
1321
+ score: .95,
1322
+ source: DETECTION_SOURCES.LEGAL_FORM
1323
+ });
1324
+ }
1325
+ return results;
1326
+ };
1327
+ //#endregion
1328
+ //#region src/detectors/coreference.ts
1329
+ /**
1330
+ * Case-insensitive set of every known legal-form suffix
1331
+ * with whitespace normalized away. Used by the
1332
+ * coreference scan to reject parenthetical define-term
1333
+ * captures whose alias is only a legal-form
1334
+ * abbreviation (e.g. "(« SAS »)" after "ACME SAS"),
1335
+ * which would otherwise propagate generic suffix
1336
+ * mentions as document-wide org aliases.
1337
+ */
1338
+ const isLegalFormAlias = (alias) => {
1339
+ const normalized = alias.replace(/\s+/g, "").toLowerCase();
1340
+ if (normalized.length === 0) return false;
1341
+ for (const suffix of getKnownLegalSuffixes()) if (suffix.replace(/\s+/g, "").toLowerCase() === normalized) return true;
1342
+ return false;
1343
+ };
1344
+ /**
1345
+ * Load coreference definition patterns from per-language
1346
+ * JSON configs in src/data/. Uses the
1347
+ * language manifest for auto-discovery.
1348
+ */
1349
+ const loadDefinitionPatterns = async () => {
1350
+ const patterns = [];
1351
+ const allRows = await loadLanguageConfigs("coreference", (mod) => {
1352
+ return mod.default ?? mod;
1353
+ });
1354
+ for (const rows of allRows) {
1355
+ if (!Array.isArray(rows)) {
1356
+ console.warn("[anonymize] coreference: unexpected config shape, skipping");
1357
+ continue;
1358
+ }
1359
+ for (const row of rows) try {
1360
+ patterns.push({ pattern: new RegExp(row.pattern, row.flags) });
1361
+ } catch (err) {
1362
+ console.warn(`[anonymize] coreference: invalid regex "${row.pattern}":`, err);
1363
+ }
1364
+ }
1365
+ return patterns;
1366
+ };
1367
+ /**
1368
+ * Load generic role terms that should NOT be treated
1369
+ * as PII coreferences. "Prodávající" (Seller),
1370
+ * "Kupující" (Buyer), etc. are legal roles, not
1371
+ * identifying information.
1372
+ */
1373
+ const getRoleStopSet = async (ctx) => {
1374
+ if (ctx.roleStopSet) return ctx.roleStopSet;
1375
+ if (ctx.roleStopSetPromise) return ctx.roleStopSetPromise;
1376
+ const promise = (async () => {
1377
+ let result;
1378
+ try {
1379
+ const mod = await import("./generic-roles.mjs");
1380
+ const data = mod.default ?? mod;
1381
+ result = new Set(data.roles.map((r) => r.toLowerCase()));
1382
+ } catch {
1383
+ result = /* @__PURE__ */ new Set();
1384
+ }
1385
+ ctx.roleStopSet = result;
1386
+ return result;
1387
+ })();
1388
+ ctx.roleStopSetPromise = promise;
1389
+ return promise;
1390
+ };
1391
+ const getDefinitionPatterns = async (ctx) => {
1392
+ if (ctx.corefPatterns) return ctx.corefPatterns;
1393
+ if (ctx.corefPatternsPromise) return ctx.corefPatternsPromise;
1394
+ ctx.corefPatternsPromise = loadDefinitionPatterns();
1395
+ const patterns = await ctx.corefPatternsPromise;
1396
+ if (patterns.length === 0) {
1397
+ ctx.corefPatterns = patterns;
1398
+ if (!ctx.corefLoadAttempted) {
1399
+ ctx.corefLoadAttempted = true;
1400
+ console.warn("[anonymize] coreference: no definition patterns loaded; coreference detection will be inactive");
1401
+ }
1402
+ return patterns;
1403
+ }
1404
+ ctx.corefPatterns = patterns;
1405
+ return patterns;
1406
+ };
1407
+ const SEARCH_WINDOW = 200;
1408
+ /**
1409
+ * Check whether an alias has textual similarity to
1410
+ * the source entity. Prevents roles and structural
1411
+ * terms from being treated as name aliases.
1412
+ *
1413
+ * Three checks (any passes → similar):
1414
+ * 1. Word overlap: a word in the alias appears in the
1415
+ * entity (case-insensitive, min 2 chars)
1416
+ * 2. Initials: alias letters match first letters of
1417
+ * entity words ("TB" ↔ "Tomas Bata")
1418
+ * 3. Substring: alias is a substring of the entity
1419
+ * or vice versa (min 3 chars)
1420
+ */
1421
+ const hasEntitySimilarity = (alias, entityText) => {
1422
+ const aliasLower = alias.toLowerCase();
1423
+ const entityLower = entityText.toLowerCase();
1424
+ if (aliasLower.length >= 3 && entityLower.includes(aliasLower)) return true;
1425
+ if (entityLower.length >= 3 && aliasLower.includes(entityLower)) return true;
1426
+ const aliasWords = aliasLower.split(/[\s.,;:'"()/-]+/).filter((w) => w.length >= 2);
1427
+ const entityWords = entityLower.split(/[\s.,;:'"()/-]+/).filter((w) => w.length >= 2);
1428
+ const entityWordSet = new Set(entityWords);
1429
+ for (const word of aliasWords) if (entityWordSet.has(word)) return true;
1430
+ if (/^[\p{Lu}]+$/u.test(alias) && alias.length >= 2 && alias.length <= entityWords.length) {
1431
+ for (let start = 0; start <= entityWords.length - alias.length; start++) if (entityWords.slice(start, start + alias.length).map((w) => w.charAt(0)).join("") === aliasLower) return true;
1432
+ }
1433
+ return false;
1434
+ };
1435
+ /**
1436
+ * Scan for defined-term patterns near known entities.
1437
+ *
1438
+ * Legal documents universally follow:
1439
+ * "Dr. Heinrich Muller (hereinafter 'the Seller')..."
1440
+ *
1441
+ * After NER detects the entity, this function scans for
1442
+ * definitional patterns within +/-200 chars and extracts
1443
+ * the alias. Returns alias + label pairs that can be added
1444
+ * to the gazetteer for a full-text re-scan.
1445
+ */
1446
+ /**
1447
+ * Labels that can be the source of a coreference alias.
1448
+ * Only parties (person, organization) have defined-term
1449
+ * aliases in legal text. Dates, addresses, IDs do not.
1450
+ */
1451
+ const COREF_SOURCE_LABELS = new Set(["person", "organization"]);
1452
+ const extractDefinedTerms = async (fullText, entities, ctx = defaultContext) => {
1453
+ const [definitionPatterns, roleStops] = await Promise.all([getDefinitionPatterns(ctx), getRoleStopSet(ctx)]);
1454
+ const terms = [];
1455
+ const seen = /* @__PURE__ */ new Set();
1456
+ const sorted = [...entities].sort((a, b) => a.start - b.start);
1457
+ for (const { pattern } of definitionPatterns) {
1458
+ pattern.lastIndex = 0;
1459
+ for (let match = pattern.exec(fullText); match !== null; match = pattern.exec(fullText)) {
1460
+ const alias = match[1]?.trim();
1461
+ if (!alias || alias.length < 2) continue;
1462
+ if (roleStops.has(alias.toLowerCase())) continue;
1463
+ if (isLegalFormAlias(alias)) continue;
1464
+ const defPos = match.index;
1465
+ let bestEntity = null;
1466
+ for (let i = sorted.length - 1; i >= 0; i--) {
1467
+ const e = sorted[i];
1468
+ if (e === void 0) continue;
1469
+ if (e.end > defPos) continue;
1470
+ if (defPos - e.end > SEARCH_WINDOW) break;
1471
+ if (!COREF_SOURCE_LABELS.has(e.label)) continue;
1472
+ bestEntity = e;
1473
+ break;
1474
+ }
1475
+ if (bestEntity === null) continue;
1476
+ const gapText = fullText.slice(bestEntity.end, defPos);
1477
+ if (/(?:;|\.(?=\s*(?:["'„‚(]*\p{Lu}|$)))/u.test(gapText)) continue;
1478
+ if (!hasEntitySimilarity(alias, bestEntity.text)) continue;
1479
+ const key = `${alias.toLowerCase()}::${bestEntity.label}`;
1480
+ if (seen.has(key)) continue;
1481
+ seen.add(key);
1482
+ terms.push({
1483
+ alias,
1484
+ label: bestEntity.label,
1485
+ definitionStart: defPos,
1486
+ sourceText: bestEntity.text
1487
+ });
1488
+ }
1489
+ }
1490
+ return terms;
1491
+ };
1492
+ /**
1493
+ * Check if a character is a Unicode word character
1494
+ * (letter, digit, or combining mark). Used for word
1495
+ * boundary checks in coreference matching.
1496
+ */
1497
+ const isWordChar = (ch) => {
1498
+ if (ch === void 0) return false;
1499
+ return /[\p{L}\p{M}\p{N}]/u.test(ch);
1500
+ };
1501
+ /**
1502
+ * Find all occurrences of defined-term aliases in the
1503
+ * full text. Returns Entity spans for each match.
1504
+ *
1505
+ * Respects word boundaries: "Kupující" must not match
1506
+ * inside "Kupujícímu". A match is valid only if the
1507
+ * character before the start and after the end are NOT
1508
+ * word characters (letter/digit).
1509
+ *
1510
+ * Populates `ctx.corefSourceMap` with entries linking
1511
+ * each coref entity to its source entity text, for
1512
+ * consistent placeholder numbering.
1513
+ */
1514
+ const findCoreferenceSpans = (fullText, terms, ctx = defaultContext) => {
1515
+ const results = [];
1516
+ for (const term of terms) {
1517
+ let searchFrom = 0;
1518
+ while (searchFrom < fullText.length) {
1519
+ const idx = fullText.indexOf(term.alias, searchFrom);
1520
+ if (idx === -1) break;
1521
+ const matchEnd = idx + term.alias.length;
1522
+ const charBefore = idx > 0 ? fullText[idx - 1] : void 0;
1523
+ const charAfter = fullText[matchEnd];
1524
+ if (isWordChar(charBefore) || isWordChar(charAfter)) {
1525
+ searchFrom = idx + 1;
1526
+ continue;
1527
+ }
1528
+ const entity = {
1529
+ start: idx,
1530
+ end: matchEnd,
1531
+ label: term.label,
1532
+ text: term.alias,
1533
+ score: .95,
1534
+ source: DETECTION_SOURCES.COREFERENCE
1535
+ };
1536
+ ctx.corefSourceMap.set(corefKey(entity), term.sourceText);
1537
+ results.push(entity);
1538
+ searchFrom = matchEnd;
1539
+ }
1540
+ }
1541
+ return results;
1542
+ };
1543
+ //#endregion
1544
+ //#region src/detectors/gazetteer.ts
1545
+ const MAX_EDIT_DISTANCE = 2;
1546
+ const MIN_FUZZY_LENGTH = 4;
1547
+ const MAX_PREFIX_OVERSHOOT = 7;
1548
+ /**
1549
+ * Collect all searchable strings (canonical + variants)
1550
+ * from gazetteer entries, mapped to their labels and
1551
+ * entry IDs.
1552
+ */
1553
+ const buildSearchTerms = (entries) => {
1554
+ const terms = /* @__PURE__ */ new Map();
1555
+ for (const entry of entries) {
1556
+ const meta = {
1557
+ label: entry.label,
1558
+ entryId: entry.id
1559
+ };
1560
+ terms.set(entry.canonical, meta);
1561
+ for (const variant of entry.variants) terms.set(variant, meta);
1562
+ }
1563
+ return terms;
1564
+ };
1565
+ /**
1566
+ * Build TextSearch-compatible patterns from gazetteer
1567
+ * entries. Returns:
1568
+ * - Exact literal patterns for all terms
1569
+ * - Fuzzy patterns (distance: 2) for terms >= 4 chars
1570
+ * - Parallel metadata arrays for post-processing
1571
+ *
1572
+ * Patterns are ordered: all exact first, then all
1573
+ * fuzzy. The isFuzzy array marks which are which.
1574
+ */
1575
+ const buildGazetteerPatterns = (entries) => {
1576
+ const terms = buildSearchTerms(entries);
1577
+ const patterns = [];
1578
+ const labels = [];
1579
+ const isFuzzy = [];
1580
+ for (const [term, meta] of terms) {
1581
+ patterns.push({
1582
+ pattern: term,
1583
+ literal: true,
1584
+ wholeWords: false
1585
+ });
1586
+ labels.push(meta.label);
1587
+ isFuzzy.push(false);
1588
+ }
1589
+ for (const [term, meta] of terms) {
1590
+ if (term.length < MIN_FUZZY_LENGTH) continue;
1591
+ patterns.push({
1592
+ pattern: term,
1593
+ distance: MAX_EDIT_DISTANCE
1594
+ });
1595
+ labels.push(meta.label);
1596
+ isFuzzy.push(true);
1597
+ }
1598
+ return {
1599
+ patterns,
1600
+ data: {
1601
+ labels,
1602
+ isFuzzy
1603
+ }
1604
+ };
1605
+ };
1606
+ /**
1607
+ * Process gazetteer matches from the unified literal
1608
+ * search. Receives all matches; filters to the
1609
+ * gazetteer slice via sliceStart/sliceEnd.
1610
+ *
1611
+ * Exact matches get score 0.9; fuzzy matches get
1612
+ * 0.85. Fuzzy matches that overlap an exact match
1613
+ * are dropped.
1614
+ *
1615
+ * For exact matches, attempts prefix extension for
1616
+ * legal suffixes ("a.s.", "GmbH", "s.r.o." after
1617
+ * the matched term).
1618
+ */
1619
+ const processGazetteerMatches = (allMatches, sliceStart, sliceEnd, fullText, data) => {
1620
+ const results = [];
1621
+ const exactSpans = [];
1622
+ for (const match of allMatches) {
1623
+ const idx = match.pattern;
1624
+ if (idx < sliceStart || idx >= sliceEnd) continue;
1625
+ const localIdx = idx - sliceStart;
1626
+ if (data.isFuzzy[localIdx]) continue;
1627
+ const label = data.labels[localIdx];
1628
+ if (!label) continue;
1629
+ const extended = tryPrefixExtension(fullText, match.start, match.end);
1630
+ const end = extended?.end ?? match.end;
1631
+ const text = extended?.text ?? fullText.slice(match.start, match.end);
1632
+ exactSpans.push({
1633
+ start: match.start,
1634
+ end
1635
+ });
1636
+ results.push({
1637
+ start: match.start,
1638
+ end,
1639
+ label,
1640
+ text,
1641
+ score: .9,
1642
+ source: DETECTION_SOURCES.GAZETTEER
1643
+ });
1644
+ }
1645
+ for (const match of allMatches) {
1646
+ const idx = match.pattern;
1647
+ if (idx < sliceStart || idx >= sliceEnd) continue;
1648
+ const localIdx = idx - sliceStart;
1649
+ if (!data.isFuzzy[localIdx]) continue;
1650
+ if (match.distance === 0) continue;
1651
+ const label = data.labels[localIdx];
1652
+ if (!label) continue;
1653
+ if (exactSpans.some((e) => match.start < e.end && match.end > e.start)) continue;
1654
+ const matchText = fullText.slice(match.start, match.end);
1655
+ results.push({
1656
+ start: match.start,
1657
+ end: match.end,
1658
+ label,
1659
+ text: matchText,
1660
+ score: .85,
1661
+ source: DETECTION_SOURCES.GAZETTEER
1662
+ });
1663
+ }
1664
+ return results;
1665
+ };
1666
+ /**
1667
+ * Try to extend an exact match to capture one
1668
+ * trailing token (max 6 chars) that may be a legal
1669
+ * entity suffix (e.g., "a.s.", "GmbH", "s.r.o.").
1670
+ *
1671
+ * Does not validate the token against a legal-forms
1672
+ * list; false extensions are filtered by mergeAndDedup
1673
+ * when a legal-form detector produces a competing
1674
+ * entity with the correct span.
1675
+ */
1676
+ const tryPrefixExtension = (fullText, start, end) => {
1677
+ const maxEnd = Math.min(end + MAX_PREFIX_OVERSHOOT, fullText.length);
1678
+ if (maxEnd <= end + 1) return null;
1679
+ const after = fullText.slice(end, maxEnd);
1680
+ if (!after.startsWith(" ")) return null;
1681
+ const nextSpace = after.indexOf(" ", 1);
1682
+ const suffixEnd = nextSpace !== -1 ? nextSpace : after.length;
1683
+ if (suffixEnd <= 1) return null;
1684
+ const newEnd = end + suffixEnd;
1685
+ return {
1686
+ end: newEnd,
1687
+ text: fullText.slice(start, newEnd)
1688
+ };
1689
+ };
1690
+ //#endregion
1691
+ //#region src/util/text.ts
1692
+ /**
1693
+ * Shared text utilities for detectors.
1694
+ *
1695
+ * Extracted from names.ts and deny-list.ts to avoid
1696
+ * duplicating regex constants and helper functions.
1697
+ */
1698
+ /** Matches a string that starts with an uppercase letter. */
1699
+ const UPPER_START_RE = /^\p{Lu}/u;
1700
+ /** Matches a string consisting entirely of uppercase letters. */
1701
+ const ALL_UPPER_RE = /^\p{Lu}+$/u;
1702
+ const SENTENCE_END_RE = /[.!?]/;
1703
+ /**
1704
+ * Detect whether a position is at the start of a sentence.
1705
+ * Looks backward past whitespace for sentence-ending
1706
+ * punctuation (.!?). Position 0 and positions preceded
1707
+ * only by whitespace are considered sentence starts.
1708
+ */
1709
+ const isSentenceStart = (text, pos) => {
1710
+ if (pos === 0) return true;
1711
+ let i = pos - 1;
1712
+ while (i >= 0 && /\s/.test(text[i] ?? "")) i--;
1713
+ if (i < 0) return true;
1714
+ return SENTENCE_END_RE.test(text[i] ?? "");
1715
+ };
1716
+ //#endregion
1717
+ //#region src/detectors/names.ts
1718
+ const getCorpus = (ctx) => ctx.nameCorpus;
1719
+ const getNameCorpusFirstNames = (ctx = defaultContext) => ctx.nameCorpus?.firstNamesList ?? [];
1720
+ const getNameCorpusSurnames = (ctx = defaultContext) => ctx.nameCorpus?.surnamesList ?? [];
1721
+ const getNameCorpusTitles = (ctx = defaultContext) => ctx.nameCorpus?.titlesList ?? [];
1722
+ /**
1723
+ * Load name corpus data from injected dictionaries
1724
+ * and legacy config files. Merges all sources.
1725
+ *
1726
+ * Safe to call multiple times; only loads once per
1727
+ * context. Must be called before detectNameCorpus or
1728
+ * the getNameCorpus*() accessors are used.
1729
+ *
1730
+ * @param dictionaries Optional pre-loaded dictionaries
1731
+ * with per-language first names and surnames. When
1732
+ * omitted, only legacy config files are used.
1733
+ */
1734
+ const initNameCorpus = (ctx = defaultContext, dictionaries) => {
1735
+ if (ctx.nameCorpusPromise) return ctx.nameCorpusPromise;
1736
+ const promise = (async () => {
1737
+ try {
1738
+ const [legacyFirstMod, legacySurnameMod, titleMod, exclusionMod] = await Promise.all([
1739
+ import("./names-first.mjs"),
1740
+ import("./names-surnames.mjs"),
1741
+ import("./names-title-tokens.mjs"),
1742
+ import("./names-exclusions.mjs")
1743
+ ]);
1744
+ const firstNames = [...legacyFirstMod.default.names];
1745
+ if (dictionaries?.firstNames) for (const names of Object.values(dictionaries.firstNames)) for (const name of names) firstNames.push(name);
1746
+ const surnames = [...legacySurnameMod.default.names];
1747
+ if (dictionaries?.surnames) for (const names of Object.values(dictionaries.surnames)) for (const name of names) surnames.push(name);
1748
+ const dedup = (arr) => {
1749
+ const seen = /* @__PURE__ */ new Set();
1750
+ const result = [];
1751
+ for (const item of arr) {
1752
+ if (seen.has(item)) continue;
1753
+ seen.add(item);
1754
+ result.push(item);
1546
1755
  }
1547
- ]
1548
- },
1549
- "underscore": {
1550
- "description": "Underscore variants",
1551
- "chars": [{
1552
- "char": "_",
1553
- "name": "low line",
1554
- "code": "U+005F"
1555
- }, {
1556
- "char": "_",
1557
- "name": "full-width low line",
1558
- "code": "U+FF3F"
1559
- }]
1756
+ return result;
1757
+ };
1758
+ const dedupFirst = dedup(firstNames);
1759
+ const dedupSurnames = dedup(surnames);
1760
+ const titles = titleMod.default.tokens;
1761
+ const exclusions = exclusionMod.default.words;
1762
+ ctx.nameCorpus = {
1763
+ firstNames: Object.freeze(new Set(dedupFirst)),
1764
+ surnames: Object.freeze(new Set(dedupSurnames)),
1765
+ titleTokens: Object.freeze(new Set(titles)),
1766
+ excludedWords: Object.freeze(new Set(exclusions)),
1767
+ firstNamesList: Object.freeze(dedupFirst),
1768
+ surnamesList: Object.freeze(dedupSurnames),
1769
+ titlesList: Object.freeze(titles),
1770
+ excludedList: Object.freeze(exclusions)
1771
+ };
1772
+ } catch (err) {
1773
+ ctx.nameCorpusPromise = null;
1774
+ console.warn("[anonymize] Failed to load name corpus JSON — name detection disabled:", err);
1775
+ }
1776
+ })();
1777
+ ctx.nameCorpusPromise = promise;
1778
+ return promise;
1779
+ };
1780
+ const INFLECTION_SUFFIXES = [
1781
+ "ovi",
1782
+ "em",
1783
+ "om",
1784
+ "ou",
1785
+ "é",
1786
+ "a",
1787
+ "u"
1788
+ ];
1789
+ /**
1790
+ * Strip common Czech/Slovak case suffixes from a token.
1791
+ * Returns candidate base forms if stripping produces a
1792
+ * plausible name (capitalised, length >= 3).
1793
+ *
1794
+ * For the "-ou" instrumental feminine suffix, also yields
1795
+ * base + "a" (e.g., "Editou" → "Edit" and "Edita")
1796
+ * because Czech feminine names decline -a → -ou.
1797
+ */
1798
+ const stripInflection = (token) => {
1799
+ const candidates = [];
1800
+ for (const suffix of INFLECTION_SUFFIXES) if (token.length > suffix.length + 2 && token.endsWith(suffix)) {
1801
+ const base = token.slice(0, -suffix.length);
1802
+ if (/^\p{Lu}/u.test(base)) {
1803
+ candidates.push(base);
1804
+ if (suffix === "ou" || suffix === "é" || suffix === "u") candidates.push(`${base}a`);
1805
+ }
1806
+ }
1807
+ return candidates;
1808
+ };
1809
+ const TOKEN_TYPE = {
1810
+ NAME: "name",
1811
+ SURNAME: "surname",
1812
+ TITLE: "title",
1813
+ ABBREVIATION: "abbreviation",
1814
+ CAPITALIZED: "capitalized",
1815
+ OTHER: "other"
1816
+ };
1817
+ const PERSON_CHAIN_BREAK_RE$1 = /[!?;:]/u;
1818
+ const isInitialContinuationGap$1 = (text, gap) => /^\p{Lu}$/u.test(text) && /^\.[^\S\n]{1,2}$/u.test(gap) || /^[^\S\n]{1,2}(?:\p{Lu}\.[^\S\n]{1,2})+$/u.test(gap);
1819
+ /**
1820
+ * Check if a token is in the first-name set, either
1821
+ * directly or after stripping Czech/Slovak inflection.
1822
+ */
1823
+ const isFirstNameToken = (token, corpus) => {
1824
+ if (corpus.firstNames.has(token)) return true;
1825
+ return stripInflection(token).some((b) => corpus.firstNames.has(b));
1826
+ };
1827
+ /**
1828
+ * Check if a token is in the surname set, either
1829
+ * directly or after stripping Czech/Slovak inflection.
1830
+ */
1831
+ const isSurnameToken = (token, corpus) => {
1832
+ if (corpus.surnames.has(token)) return true;
1833
+ return stripInflection(token).some((b) => corpus.surnames.has(b));
1834
+ };
1835
+ /**
1836
+ * Check if a token looks like a single-letter
1837
+ * abbreviation: "J.", "M.", etc.
1838
+ */
1839
+ const isAbbreviation = (token) => token.length === 2 && /^\p{Lu}$/u.test(token[0] ?? "") && token[1] === ".";
1840
+ const segmenter$1 = new Intl.Segmenter(void 0, { granularity: "word" });
1841
+ /**
1842
+ * Split text into word segments using Intl.Segmenter.
1843
+ * Only returns segments flagged as words.
1844
+ */
1845
+ const segmentWords = (fullText) => {
1846
+ const words = [];
1847
+ for (const seg of segmenter$1.segment(fullText)) if (seg.isWordLike) words.push({
1848
+ text: seg.segment,
1849
+ start: seg.index,
1850
+ end: seg.index + seg.segment.length
1851
+ });
1852
+ return words;
1853
+ };
1854
+ /** NAME or SURNAME — both represent corpus-matched tokens */
1855
+ const isCorpusMatch = (type) => type === TOKEN_TYPE.NAME || type === TOKEN_TYPE.SURNAME;
1856
+ const classifyToken = (word, corpus) => {
1857
+ const { text, start, end } = word;
1858
+ const lower = text.toLowerCase();
1859
+ const stripped = text.endsWith(".") ? text.slice(0, -1).toLowerCase() : lower;
1860
+ if (corpus.titleTokens.has(stripped)) return {
1861
+ text,
1862
+ type: TOKEN_TYPE.TITLE,
1863
+ start,
1864
+ end
1865
+ };
1866
+ if (isAbbreviation(text)) return {
1867
+ text,
1868
+ type: TOKEN_TYPE.ABBREVIATION,
1869
+ start,
1870
+ end
1871
+ };
1872
+ if (corpus.excludedWords.has(lower)) return {
1873
+ text,
1874
+ type: TOKEN_TYPE.OTHER,
1875
+ start,
1876
+ end
1877
+ };
1878
+ if (text.length < 3) return {
1879
+ text,
1880
+ type: TOKEN_TYPE.OTHER,
1881
+ start,
1882
+ end
1883
+ };
1884
+ if (text.length > 3 && ALL_UPPER_RE.test(text)) return {
1885
+ text,
1886
+ type: TOKEN_TYPE.OTHER,
1887
+ start,
1888
+ end
1889
+ };
1890
+ if (!UPPER_START_RE.test(text)) return {
1891
+ text,
1892
+ type: TOKEN_TYPE.OTHER,
1893
+ start,
1894
+ end
1895
+ };
1896
+ if (isFirstNameToken(text, corpus)) return {
1897
+ text,
1898
+ type: TOKEN_TYPE.NAME,
1899
+ start,
1900
+ end
1901
+ };
1902
+ if (isSurnameToken(text, corpus)) return {
1903
+ text,
1904
+ type: TOKEN_TYPE.SURNAME,
1905
+ start,
1906
+ end
1907
+ };
1908
+ return {
1909
+ text,
1910
+ type: TOKEN_TYPE.CAPITALIZED,
1911
+ start,
1912
+ end
1913
+ };
1914
+ };
1915
+ /**
1916
+ * Detect person names by looking up tokens against the
1917
+ * name corpus, then chaining adjacent name-like tokens.
1918
+ *
1919
+ * Requires initNameCorpus() to have been called first.
1920
+ * If not initialized, returns an empty array.
1921
+ *
1922
+ * Scoring:
1923
+ * TITLE + NAME/SURNAME → 0.95
1924
+ * NAME + NAME/SURNAME → 0.9
1925
+ * SURNAME + NAME/SURNAME → 0.9
1926
+ * NAME + CAPITALIZED → 0.7
1927
+ * ABBREVIATION + NAME → 0.7
1928
+ * Standalone NAME → 0.5 (low confidence)
1929
+ * Standalone SURNAME → skip (too ambiguous)
1930
+ */
1931
+ const detectNameCorpus = (fullText, ctx = defaultContext) => {
1932
+ const corpus = getCorpus(ctx);
1933
+ if (!corpus) return [];
1934
+ const tokens = segmentWords(fullText).map((w) => classifyToken(w, corpus));
1935
+ const entities = [];
1936
+ const consumed = /* @__PURE__ */ new Set();
1937
+ for (let i = 0; i < tokens.length; i++) {
1938
+ if (consumed.has(i)) continue;
1939
+ const token = tokens[i];
1940
+ if (!token) continue;
1941
+ if (token.type !== TOKEN_TYPE.TITLE && token.type !== TOKEN_TYPE.NAME && token.type !== TOKEN_TYPE.SURNAME && token.type !== TOKEN_TYPE.ABBREVIATION) continue;
1942
+ const MAX_CHAIN = 5;
1943
+ const chain = [token];
1944
+ let j = i + 1;
1945
+ while (j < tokens.length && chain.length < MAX_CHAIN) {
1946
+ const next = tokens[j];
1947
+ if (!next) break;
1948
+ const prev = chain.at(-1);
1949
+ if (prev) {
1950
+ const gap = fullText.slice(prev.end, next.start);
1951
+ const breaksOnPeriod = gap.includes(".") && !isInitialContinuationGap$1(prev.text, gap);
1952
+ if (gap.includes("\n") || PERSON_CHAIN_BREAK_RE$1.test(gap) || breaksOnPeriod) break;
1953
+ }
1954
+ if (next.type === TOKEN_TYPE.NAME || next.type === TOKEN_TYPE.SURNAME || next.type === TOKEN_TYPE.TITLE || next.type === TOKEN_TYPE.ABBREVIATION || next.type === TOKEN_TYPE.CAPITALIZED) {
1955
+ chain.push(next);
1956
+ j++;
1957
+ } else break;
1958
+ }
1959
+ const hasTitle = chain.some((t) => t.type === TOKEN_TYPE.TITLE);
1960
+ const hasCorpusName = chain.some((t) => isCorpusMatch(t.type));
1961
+ const hasFirstName = chain.some((t) => t.type === TOKEN_TYPE.NAME);
1962
+ const hasAbbreviation = chain.some((t) => t.type === TOKEN_TYPE.ABBREVIATION);
1963
+ const corpusCount = chain.filter((t) => isCorpusMatch(t.type)).length;
1964
+ const capitalizedCount = chain.filter((t) => t.type === TOKEN_TYPE.CAPITALIZED).length;
1965
+ let score = 0;
1966
+ if (hasTitle && hasCorpusName) score = .95;
1967
+ else if (corpusCount >= 2) score = .9;
1968
+ else if (hasCorpusName && capitalizedCount > 0) score = .7;
1969
+ else if (hasAbbreviation && hasCorpusName) score = .7;
1970
+ else if (hasFirstName && chain.length === 1) {
1971
+ if (isSentenceStart(fullText, token.start)) continue;
1972
+ score = .5;
1973
+ } else if (!hasFirstName && chain.length === 1 && chain[0]?.type === TOKEN_TYPE.SURNAME) continue;
1974
+ else if (hasTitle && chain.length === 1) continue;
1975
+ else {
1976
+ if (!hasCorpusName) continue;
1977
+ score = .5;
1560
1978
  }
1979
+ const first = chain.at(0);
1980
+ const last = chain.at(-1);
1981
+ if (!first || !last) continue;
1982
+ const start = first.start;
1983
+ const end = last.end;
1984
+ const text = fullText.slice(start, end);
1985
+ for (let k = i; k < i + chain.length; k++) consumed.add(k);
1986
+ entities.push({
1987
+ start,
1988
+ end,
1989
+ label: "person",
1990
+ text,
1991
+ score,
1992
+ source: DETECTION_SOURCES.REGEX
1993
+ });
1561
1994
  }
1995
+ return entities;
1562
1996
  };
1563
1997
  //#endregion
1564
- //#region src/util/char-groups.ts
1565
- /**
1566
- * Centralized Unicode character equivalence groups.
1567
- *
1568
- * Centralized Unicode character equivalence groups.
1569
- *
1570
- * Provides helpers to build regex character classes that
1571
- * match all typographic variants of a character type
1572
- * (dashes, spaces, quotes, etc.).
1573
- *
1574
- * The JSON is statically imported so the bundler inlines
1575
- * it, avoiding a runtime require() that breaks browsers.
1576
- */
1577
- /** Chars that need escaping inside a regex char class. */
1578
- const REGEX_CLASS_SPECIAL = /[\\\]^-]/;
1579
- const escapeForCharClass = (ch) => REGEX_CLASS_SPECIAL.test(ch) ? `\\${ch}` : ch;
1580
- const config = char_groups_default;
1998
+ //#region src/config/titles.ts
1581
1999
  /**
1582
- * Get the raw characters for a named group.
1583
- * Throws if the group does not exist.
2000
+ * Academic and professional title prefixes.
2001
+ * Plain text; the detector auto-escapes for regex.
2002
+ * Sorted longest-first at build time.
1584
2003
  */
1585
- const charSet = (group) => {
1586
- const g = config.groups[group];
1587
- if (!g) throw new Error(`Unknown char group: "${group}"`);
1588
- return g.chars.map((entry) => entry.char);
1589
- };
2004
+ const TITLE_PREFIXES = [
2005
+ "Ing.",
2006
+ "Mgr.",
2007
+ "MgA.",
2008
+ "Bc.",
2009
+ "BcA.",
2010
+ "JUDr.",
2011
+ "MUDr.",
2012
+ "MVDr.",
2013
+ "MDDr.",
2014
+ "PhDr.",
2015
+ "RNDr.",
2016
+ "PaedDr.",
2017
+ "ThDr.",
2018
+ "ThLic.",
2019
+ "ICDr.",
2020
+ "RSDr.",
2021
+ "PharmDr.",
2022
+ "artD.",
2023
+ "akad.",
2024
+ "doc.",
2025
+ "prof.",
2026
+ "ao. Univ.-Prof.",
2027
+ "o. Univ.-Prof.",
2028
+ "Univ.-Prof.",
2029
+ "Hon.-Prof.",
2030
+ "em. Prof.",
2031
+ "Dr. med. dent.",
2032
+ "Dr. med. vet.",
2033
+ "Dr. med.",
2034
+ "Dr. rer. nat.",
2035
+ "Dr. rer. soc.",
2036
+ "Dr. rer. pol.",
2037
+ "Dr. sc. tech.",
2038
+ "Dr. sc. nat.",
2039
+ "Dr. sc. hum.",
2040
+ "Dr. iur.",
2041
+ "Dr. jur.",
2042
+ "Dr. theol.",
2043
+ "Dr. oec.",
2044
+ "Dr. techn.",
2045
+ "Dr. h. c.",
2046
+ "Dr. phil.",
2047
+ "Dr.-Ing.",
2048
+ "Dr. Ing.",
2049
+ "Dr.",
2050
+ "Dipl.-Wirt.-Ing.",
2051
+ "Dipl.-Betriebsw.",
2052
+ "Dipl.-Inform.",
2053
+ "Dipl.-Volksw.",
2054
+ "Dipl.-Psych.",
2055
+ "Dipl.-Phys.",
2056
+ "Dipl.-Chem.",
2057
+ "Dipl.-Biol.",
2058
+ "Dipl.-Math.",
2059
+ "Dipl.-Päd.",
2060
+ "Dipl.-Soz.",
2061
+ "Dipl.-Kfm.",
2062
+ "Dipl.-Jur.",
2063
+ "Dipl. Ing.",
2064
+ "Dipl.-Ing.",
2065
+ "Mag. rer. soc. oec.",
2066
+ "Mag. rer. nat.",
2067
+ "Mag. phil.",
2068
+ "Mag. iur.",
2069
+ "Mag. arch.",
2070
+ "Mag. pharm.",
2071
+ "Mag. (FH)",
2072
+ "Mag.",
2073
+ "Bakk. rer. nat.",
2074
+ "Bakk. techn.",
2075
+ "Bakk. phil.",
2076
+ "Bakk.",
2077
+ "Lic. phil.",
2078
+ "Lic. iur.",
2079
+ "Lic. oec.",
2080
+ "Lic. theol.",
2081
+ "Lic.",
2082
+ "Priv.-Doz.",
2083
+ "PD",
2084
+ "RA"
2085
+ ];
1590
2086
  /**
1591
- * Build a regex character class string for a named
1592
- * group. E.g., charClass("dash") returns a string
1593
- * like "[-\u2013\u2014\u2010\u2011\u2212\u2043]".
1594
- *
1595
- * The hyphen-minus is placed first so it is treated
1596
- * as a literal, not a range indicator.
2087
+ * Courtesy/honorific titles that precede a person's
2088
+ * name. Sorted alphabetically. The detector escapes
2089
+ * dots and adds \b for entries in HONORIFIC_BOUNDARY.
1597
2090
  */
1598
- const charClass = (group) => {
1599
- return `[${[...charSet(group)].sort((a, b) => {
1600
- if (a === "-") return -1;
1601
- if (b === "-") return 1;
1602
- return a.localeCompare(b);
1603
- }).map(escapeForCharClass).join("")}]`;
1604
- };
2091
+ const HONORIFICS = [
2092
+ "Avv.",
2093
+ "Dame",
2094
+ "Doamna",
2095
+ "Domnul",
2096
+ "Don",
2097
+ "Doña",
2098
+ "Dott.",
2099
+ "Judge",
2100
+ "Justice",
2101
+ "Lady",
2102
+ "Lord",
2103
+ "M.",
2104
+ "Madame",
2105
+ "Mademoiselle",
2106
+ "Maître",
2107
+ "Me",
2108
+ "Messrs",
2109
+ "Miss",
2110
+ "Mlle",
2111
+ "Mme",
2112
+ "Monsieur",
2113
+ "Mr",
2114
+ "Mrs",
2115
+ "Ms",
2116
+ "Pr",
2117
+ "Pr.",
2118
+ "President",
2119
+ "Señor",
2120
+ "Señora",
2121
+ "Sig.",
2122
+ "Sig.ra",
2123
+ "Signor",
2124
+ "Signora",
2125
+ "Signorina",
2126
+ "Sir",
2127
+ "Sr.",
2128
+ "Sra."
2129
+ ];
1605
2130
  /**
1606
- * Return the inner content of a character class (without
1607
- * the surrounding brackets). Useful for embedding a group
1608
- * inside a larger character class, e.g.:
1609
- * `[\\s&,.${charClassInner("dash")}]`
2131
+ * Honorifics that need \b word-boundary anchors
2132
+ * (short or common words that could match mid-word).
1610
2133
  */
1611
- const charClassInner = (group) => {
1612
- return [...charSet(group)].sort((a, b) => {
1613
- if (a === "-") return -1;
1614
- if (b === "-") return 1;
1615
- return a.localeCompare(b);
1616
- }).map(escapeForCharClass).join("");
1617
- };
1618
- /** Regex char class matching all dash variants. */
1619
- const DASH = charClass("dash");
2134
+ const HONORIFIC_BOUNDARY = new Set([
2135
+ "Don",
2136
+ "Doña",
2137
+ "M.",
2138
+ "Me",
2139
+ "Pr",
2140
+ "Pr.",
2141
+ "Señor",
2142
+ "Señora"
2143
+ ]);
1620
2144
  /**
1621
- * Inner content of the dash char class (no brackets).
1622
- * For embedding inside a larger character class:
1623
- * `[\\s&,.${DASH_INNER}]`
1624
- */
1625
- const DASH_INNER = charClassInner("dash");
1626
- charClass("space");
1627
- charClass("quote-double");
1628
- charClass("quote-single");
1629
- charClass("slash");
1630
- charClass("dot");
1631
- charClass("colon");
2145
+ * Post-nominal degrees (comma or space separated after name).
2146
+ * Plain text; the detector auto-escapes for regex.
2147
+ */
2148
+ const POST_NOMINALS = [
2149
+ "Ph.D.",
2150
+ "Ph.D",
2151
+ "CSc.",
2152
+ "DrSc.",
2153
+ "ArtD.",
2154
+ "D.Phil.",
2155
+ "DPhil.",
2156
+ "MPhil.",
2157
+ "MBA",
2158
+ "MPA",
2159
+ "LL.M.",
2160
+ "LL.B.",
2161
+ "M.Sc.",
2162
+ "B.Sc.",
2163
+ "MSc.",
2164
+ "BSc.",
2165
+ "M.Eng.",
2166
+ "B.Eng.",
2167
+ "M.A.",
2168
+ "B.A.",
2169
+ "JCD",
2170
+ "JD",
2171
+ "DiS.",
2172
+ "ACCA",
2173
+ "FCCA",
2174
+ "CIPM",
2175
+ "CIPT",
2176
+ "CIPP/E",
2177
+ "CIPP"
2178
+ ];
1632
2179
  //#endregion
1633
2180
  //#region src/detectors/regex.ts
1634
2181
  const MIN_PHONE_LENGTH = 7;
1635
2182
  const MIN_MONTH_NAME_LENGTH = 3;
1636
2183
  const escapeTitle = (title) => title.replace(/[.*+?^${}()|[\]\\]/g, "\\$&").replace(/\s+/g, "\\s*");
1637
2184
  /** Escape for use inside a regex alternation. */
1638
- const escapeRegex$1 = (s) => s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
2185
+ const escapeRegex$2 = (s) => s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
1639
2186
  /** Escape for use inside a regex character class. */
1640
2187
  const escapeCharClass = (s) => s.replace(/[\]\\^-]/g, "\\$&");
1641
2188
  const TITLE_PREFIX = TITLE_PREFIXES.toSorted((a, b) => b.length - a.length).map(escapeTitle).join("|");
1642
2189
  const POST_NOMINAL = POST_NOMINALS.toSorted((a, b) => b.length - a.length).map(escapeTitle).join("|");
1643
- const NAME_WORD = `[A-ZÁČĎÉĚÍŇÓŘŠŤÚŮÝŽ][a-záčďéěíňóřšťúůýžäöüß]+`;
2190
+ const NAME_WORD = `\\p{Lu}\\p{Ll}+`;
1644
2191
  const PARTICLE = "(?:van der|van den|de la|della|von|van|dos|ibn|ben|bin|del|zum|zur|ten|ter|da|de|di|al|el|le|la|zu|af|av)";
1645
2192
  const SP = "[^\\S\\n\\t]";
1646
2193
  /** Honorific alternation built from titles.ts config. */
1647
2194
  const HONORIFIC_ALT = [...HONORIFICS].toSorted((a, b) => b.length - a.length).map((h) => {
1648
- const escaped = escapeRegex$1(h);
2195
+ const escaped = escapeRegex$2(h);
1649
2196
  return HONORIFIC_BOUNDARY.has(h) ? `\\b${escaped}` : escaped;
1650
2197
  }).join("|");
1651
2198
  const toEntry = (validator, label, score) => {
@@ -1670,8 +2217,6 @@ const toEntry = (validator, label, score) => {
1670
2217
  const STDNUM_ENTRIES = [
1671
2218
  toEntry(hu.vat, "tax identification number", .95),
1672
2219
  toEntry(it.codiceFiscale, "national identification number", .95),
1673
- toEntry(es.dni, "national identification number", .9),
1674
- toEntry(es.nie, "national identification number", .95),
1675
2220
  toEntry(se.personnummer, "national identification number", .9),
1676
2221
  toEntry(ro.cnp, "national identification number", .95),
1677
2222
  toEntry(fr.nir, "social security number", .9),
@@ -1697,7 +2242,6 @@ const STDNUM_ENTRIES = [
1697
2242
  toEntry(fi.ytunnus, "registration number", .9),
1698
2243
  toEntry(bg.vat, "tax identification number", .95),
1699
2244
  toEntry(sk.dic, "tax identification number", .95),
1700
- toEntry(es.cif, "registration number", .95),
1701
2245
  toEntry(es.vat, "tax identification number", .95),
1702
2246
  toEntry(es.nss, "social security number", .9),
1703
2247
  toEntry(fr.tva, "tax identification number", .95),
@@ -1719,7 +2263,9 @@ const STDNUM_ENTRIES = [
1719
2263
  toEntry(ee.ik, "national identification number", .9),
1720
2264
  toEntry(cy.vat, "tax identification number", .95),
1721
2265
  toEntry(mt.vat, "tax identification number", .95),
1722
- toEntry(lu.vat, "tax identification number", .95)
2266
+ toEntry(lu.vat, "tax identification number", .95),
2267
+ toEntry(br.cpf, "tax identification number", .95),
2268
+ toEntry(br.cnpj, "tax identification number", .95)
1723
2269
  ].filter((e) => e !== null);
1724
2270
  const TITLED_PERSON = {
1725
2271
  pattern: `(?:${TITLE_PREFIX})(?:${SP}+(?:${TITLE_PREFIX}))*${SP}+(?:${NAME_WORD})(?:${SP}{1,4}(?:${PARTICLE}${SP}+)?${NAME_WORD}){1,3}(?:,?${SP}+(?:${POST_NOMINAL})(?:,?${SP}+(?:${POST_NOMINAL}))*)?`,
@@ -1768,7 +2314,7 @@ const CREDIT_CARD = {
1768
2314
  };
1769
2315
  const CZ_BIRTH_NUMBER = {
1770
2316
  pattern: `\\b\\d{6}/\\d{3,4}\\b`,
1771
- label: "czech birth number",
2317
+ label: "birth number",
1772
2318
  score: 1,
1773
2319
  validator: cz.rc
1774
2320
  };
@@ -1802,6 +2348,51 @@ const CZ_POSTAL = {
1802
2348
  label: "address",
1803
2349
  score: .7
1804
2350
  };
2351
+ const ES_POSTAL = {
2352
+ pattern: "\\b(?:C\\.?P\\.?|[Cc][óo]digo[^\\S\\n]+postal)[^\\S\\n]{0,3}:?[^\\S\\n]{0,3}\\d{5}\\b",
2353
+ label: "address",
2354
+ score: .7
2355
+ };
2356
+ const ES_DNI = {
2357
+ pattern: `\\b\\d{8}-?[A-Za-z]\\b`,
2358
+ label: "national identification number",
2359
+ score: .9,
2360
+ validator: es.dni
2361
+ };
2362
+ const ES_NIE = {
2363
+ pattern: `\\b[XYZxyz]-?\\d{7}-?[A-Za-z]\\b`,
2364
+ label: "national identification number",
2365
+ score: .95,
2366
+ validator: es.nie
2367
+ };
2368
+ const ES_CIF = {
2369
+ pattern: `\\b[A-HJNP-SUVWa-hjnp-suvw]-?\\d{7}-?[0-9A-Ja-j]\\b`,
2370
+ label: "registration number",
2371
+ score: .95,
2372
+ validator: es.cif
2373
+ };
2374
+ const BR_CPF_FORMATTED = {
2375
+ pattern: `\\b\\d{3}\\.\\d{3}\\.\\d{3}${DASH}\\d{2}\\b`,
2376
+ label: "tax identification number",
2377
+ score: .95,
2378
+ validator: br.cpf
2379
+ };
2380
+ const BR_CNPJ_FORMATTED = {
2381
+ pattern: `\\b\\d{2}\\.\\d{3}\\.\\d{3}/\\d{4}${DASH}\\d{2}\\b`,
2382
+ label: "tax identification number",
2383
+ score: .95,
2384
+ validator: br.cnpj
2385
+ };
2386
+ const BR_RG_WITH_SSP = {
2387
+ pattern: `\\b\\d{1,3}\\.?\\d{3}\\.?\\d{3}(?:${DASH}[0-9A-Za-z])?[^\\S\\n]+SSP(?:/[A-Z]{2})?\\b`,
2388
+ label: "national identification number",
2389
+ score: .95
2390
+ };
2391
+ const BR_OAB = {
2392
+ pattern: "\\bOAB/[A-Z]{2}[^\\S\\n]+(?:n[º°.][^\\S\\n]*)?(?:\\d{1,3}(?:\\.\\d{3})+|\\d{4,6})\\b",
2393
+ label: "registration number",
2394
+ score: .95
2395
+ };
1805
2396
  const URL = {
1806
2397
  pattern: "(?:https?://|https?:(?=[^\\s])|www\\.)[\\w\\-]+(?:\\.[\\w\\-]+)+(?::\\d+)?(?:[/?#][^\\s)\\]>]*[^\\s.,;:!?)\\]>])?",
1807
2398
  label: "url",
@@ -1843,6 +2434,14 @@ const ALL_REGEX_DEFS = [
1843
2434
  CZ_BANK_ACCOUNT,
1844
2435
  HU_LANDLINE,
1845
2436
  CZ_POSTAL,
2437
+ ES_POSTAL,
2438
+ ES_DNI,
2439
+ ES_NIE,
2440
+ ES_CIF,
2441
+ BR_CPF_FORMATTED,
2442
+ BR_CNPJ_FORMATTED,
2443
+ BR_RG_WITH_SSP,
2444
+ BR_OAB,
1846
2445
  URL,
1847
2446
  {
1848
2447
  pattern: "\\b(?:[0-9a-fA-F]{1,4}:){7}[0-9a-fA-F]{1,4}\\b|\\b(?:[0-9a-fA-F]{1,4}:){1,7}:\\b|::(?:[0-9a-fA-F]{1,4}:){0,6}[0-9a-fA-F]{1,4}",
@@ -1894,7 +2493,7 @@ const buildMonthAlternation = (months) => {
1894
2493
  if (clean.length >= MIN_MONTH_NAME_LENGTH) seen.add(clean);
1895
2494
  }
1896
2495
  }
1897
- return [...seen].toSorted((a, b) => b.length - a.length).map(escapeRegex$1).join("|");
2496
+ return [...seen].toSorted((a, b) => b.length - a.length).map(escapeRegex$2).join("|");
1898
2497
  };
1899
2498
  /**
1900
2499
  * Build date patterns from a month-name alternation.
@@ -1951,11 +2550,11 @@ const buildCurrencyPatterns = (data) => {
1951
2550
  const isAsciiAlpha = /^[a-zA-Z\s]+$/;
1952
2551
  const parts = data.codes.map((code) => ({
1953
2552
  len: code.length,
1954
- alt: escapeRegex$1(code)
2553
+ alt: escapeRegex$2(code)
1955
2554
  }));
1956
2555
  const MIN_CI_LENGTH = 3;
1957
2556
  if (data.localNames) for (const name of data.localNames) {
1958
- const escaped = escapeRegex$1(name);
2557
+ const escaped = escapeRegex$2(name);
1959
2558
  if (isAsciiAlpha.test(name) && name.length >= MIN_CI_LENGTH) parts.push({
1960
2559
  len: name.length,
1961
2560
  alt: `(?i:${escaped})`
@@ -1967,7 +2566,7 @@ const buildCurrencyPatterns = (data) => {
1967
2566
  }
1968
2567
  if (symbols) for (const ch of data.symbols) parts.push({
1969
2568
  len: ch.length,
1970
- alt: escapeRegex$1(ch)
2569
+ alt: escapeRegex$2(ch)
1971
2570
  });
1972
2571
  const trailingAlt = parts.toSorted((a, b) => b.len - a.len).map((p) => p.alt).join("|");
1973
2572
  if (!symbols && !trailingAlt) return [];
@@ -2019,290 +2618,70 @@ const processRegexMatches = (allMatches, sliceStart, sliceEnd, meta_) => {
2019
2618
  for (const match of allMatches) {
2020
2619
  const idx = match.pattern;
2021
2620
  if (idx < sliceStart || idx >= sliceEnd) continue;
2022
- const meta = meta_[idx - sliceStart];
2023
- if (!meta) continue;
2024
- if (meta.sourceDetail !== "custom-regex" && meta.label === "phone number" && match.text.length < MIN_PHONE_LENGTH) continue;
2025
- if (meta.validator) {
2026
- const compacted = meta.validator.compact(match.text);
2027
- if (!meta.validator.validate(compacted).valid) continue;
2028
- }
2029
- const entity = {
2030
- start: match.start,
2031
- end: match.end,
2032
- label: meta.label,
2033
- text: match.text,
2034
- score: meta.score,
2035
- source: DETECTION_SOURCES.REGEX
2036
- };
2037
- if (meta.sourceDetail) entity.sourceDetail = meta.sourceDetail;
2038
- results.push(entity);
2039
- }
2040
- return results;
2041
- };
2042
- /**
2043
- * Build signing clause place-name patterns from
2044
- * signing-clauses.json. Each pattern captures the
2045
- * city/place name from contract signing locations.
2046
- *
2047
- * The place name sub-pattern:
2048
- * \p{Lu}\p{Ll}+ (capitalized word)
2049
- * optionally followed by preposition + capitalized
2050
- * word (for "nad Nisou", "am Main", etc.)
2051
- * optionally followed by more capitalized words
2052
- * (for "Hradec Králové", "New York", etc.)
2053
- */
2054
- const buildSigningClausePatterns = (data) => {
2055
- const patterns = [];
2056
- for (const entry of data.patterns) {
2057
- const prepAlt = entry.prepositions.length > 0 ? entry.prepositions.join("|") : null;
2058
- const place = prepAlt ? `(\\p{Lu}\\p{Ll}+(?:\\s+(?:${prepAlt})\\s+\\p{Lu}\\p{Ll}+)*(?:\\s+\\p{Lu}\\p{Ll}+)*)` : `(\\p{Lu}\\p{Ll}+(?:[- ]\\p{Lu}\\p{Ll}+)*)`;
2059
- const full = `(?:^|\\n|[^\\S\\n])` + entry.prefix + place + (entry.suffix ? `(?:${entry.suffix})` : "");
2060
- patterns.push(full);
2061
- }
2062
- return patterns;
2063
- };
2064
- const SIGNING_CLAUSE_META = {
2065
- label: "address",
2066
- score: .9
2067
- };
2068
- let signingPatternPromise = null;
2069
- const loadSigningPatterns = async () => {
2070
- const mod = await import("./signing-clauses.mjs");
2071
- return buildSigningClausePatterns(mod.default ?? mod);
2072
- };
2073
- const getSigningClausePatterns = () => {
2074
- if (!signingPatternPromise) signingPatternPromise = loadSigningPatterns().catch((err) => {
2075
- signingPatternPromise = null;
2076
- throw err;
2077
- });
2078
- return signingPatternPromise;
2079
- };
2080
- //#endregion
2081
- //#region src/detectors/legal-forms.ts
2082
- const UPPER = "A-ZÁČĎÉĚÍŇÓŘŠŤÚŮÝŽÄÖÜÀÂÆÇÈÊËÎÏÔÙÛŸÑ\\u0130";
2083
- const LOWER = "a-záčďéěíňóřšťúůýžäöüßàâæçèêëîïôùûÿñ\\u0131";
2084
- const CAP_WORD = `(?:[${UPPER}]{2,}|[${UPPER}][${LOWER}${UPPER}]+)`;
2085
- const ANY_WORD = `(?:[${UPPER}${LOWER}][${LOWER}${UPPER}]+|[${UPPER}]{2,3}|\\d{1,4})`;
2086
- const ALLCAP_WORD = `[${UPPER}]{2,}`;
2087
- const ROMAN_NUMERAL_RE = /^(?=[IVXLCDM])M{0,3}(?:CM|CD|D?C{0,3})(?:XC|XL|L?X{0,3})(?:IX|IV|V?I{0,3})$/;
2088
- const escapeForRegex = (form) => form.replace(/[.*+?^${}()|[\]\\]/g, "\\$&").replace(/\s+/g, "\\s+").replace(/\\\./g, "\\.[^\\S\\n]?");
2089
- const isShortForm = (form) => form.replace(/[.\s]/g, "").length <= 3 && !form.includes(" ");
2090
- const buildPatternString = (forms) => {
2091
- if (forms.length === 0) return null;
2092
- const alt = forms.toSorted((a, b) => b.length - a.length).map(escapeForRegex).join("|");
2093
- return `${`(?:${CAP_WORD})(?:${`(?:[\\s&,.${DASH_INNER}]{1,4}|${`\\s+(?:a|and|und|et|e|y|i)\\s+(?=[${LOWER}])`})`}(?:${ANY_WORD})){0,10}`}(?:\\s+|,\\s*)(?:${alt})(?![${LOWER}])`;
2094
- };
2095
- /**
2096
- * Build legal form regex pattern strings.
2097
- * Returns an array of regex strings for the unified
2098
- * TextSearch builder. Empty if data package is not
2099
- * installed.
2100
- */
2101
- const buildLegalFormPatterns = async () => {
2102
- let data = {};
2103
- try {
2104
- data = (await import("./legal-forms.mjs")).default;
2105
- } catch {
2106
- return [];
2107
- }
2108
- const allForms = [];
2109
- const seen = /* @__PURE__ */ new Set();
2110
- for (const forms of Object.values(data)) for (const form of forms) {
2111
- const key = form.toLowerCase();
2112
- if (!seen.has(key)) {
2113
- seen.add(key);
2114
- allForms.push(form);
2115
- }
2116
- }
2117
- const patterns = [];
2118
- const longPattern = buildPatternString(allForms.filter((f) => !isShortForm(f)));
2119
- if (longPattern) patterns.push(longPattern);
2120
- const shortPattern = buildPatternString(allForms.filter(isShortForm));
2121
- if (shortPattern) patterns.push(shortPattern);
2122
- const allcapPrefix = `(?:${ALLCAP_WORD})(?:[\\s&,.${DASH_INNER}]{1,4}(?:${ALLCAP_WORD})){0,2}`;
2123
- const allcapAlt = allForms.toSorted((a, b) => b.length - a.length).map(escapeForRegex).join("|");
2124
- patterns.push(`${allcapPrefix}(?:\\s+|,\\s*)(?:${allcapAlt})(?![${LOWER}])`);
2125
- return patterns;
2126
- };
2127
- const CONNECTOR_RE = /^(?:a|and|und|et|e|y|i|&)$/i;
2128
- const LEADING_CLAUSE_RE = /(?:^|\s)(?:by\s+and\s+between|is\s+between)\s+/giu;
2129
- /**
2130
- * Find the word ending just before `pos` in `text`,
2131
- * skipping any whitespace (not newlines).
2132
- * Returns null if no word is found (e.g., at start
2133
- * of text, or preceded by non-word chars like ".").
2134
- */
2135
- const findWordBefore = (text, pos) => {
2136
- let scan = pos - 1;
2137
- while (scan >= 0) {
2138
- const ch = text.charAt(scan);
2139
- if (ch === "\n" || !/\s/.test(ch)) break;
2140
- scan--;
2141
- }
2142
- if (scan < 0 || text.charAt(scan) === "\n") return null;
2143
- const wordEnd = scan + 1;
2144
- while (scan >= 0 && /[\p{L}\p{M}&]/u.test(text.charAt(scan))) scan--;
2145
- const wordStart = scan + 1;
2146
- const word = text.slice(wordStart, wordEnd);
2147
- if (word.length === 0) return null;
2148
- return {
2149
- word,
2150
- start: wordStart
2151
- };
2152
- };
2153
- /**
2154
- * Extend a match backward through uppercase words and
2155
- * lowercase connectors. Stops at start of text,
2156
- * newline, or a word that doesn't qualify.
2157
- *
2158
- * Connectors (a, and, und, et, ...) are only consumed
2159
- * when there is a valid word before them — a trailing
2160
- * connector at an entity boundary is not consumed.
2161
- */
2162
- const extendBackward = (fullText, matchStart) => {
2163
- let pos = matchStart;
2164
- while (pos > 0) {
2165
- const found = findWordBefore(fullText, pos);
2166
- if (!found) break;
2167
- const { word, start: wordStart } = found;
2168
- const isUpper = /^\p{Lu}/u.test(word);
2169
- const isConnector = CONNECTOR_RE.test(word);
2170
- if (isUpper) pos = wordStart;
2171
- else if (isConnector) {
2172
- const prev = findWordBefore(fullText, wordStart);
2173
- if (!prev) break;
2174
- if (!/^\p{Lu}/u.test(prev.word)) break;
2175
- pos = prev.start;
2176
- } else break;
2177
- }
2178
- return pos;
2179
- };
2180
- const trimLeadingClause = (text) => {
2181
- let cut = -1;
2182
- for (const match of text.matchAll(LEADING_CLAUSE_RE)) cut = match.index + match[0].length;
2183
- if (cut <= 0) return {
2184
- offset: 0,
2185
- text
2186
- };
2187
- const trimmed = text.slice(cut);
2188
- const leadingWs = trimmed.match(/^\s*/u)?.[0].length ?? 0;
2189
- return {
2190
- offset: cut + leadingWs,
2191
- text: trimmed.slice(leadingWs)
2192
- };
2193
- };
2194
- /**
2195
- * Process legal form matches from the unified search.
2196
- * Receives all matches; filters to the legal forms
2197
- * slice via sliceStart/sliceEnd.
2198
- */
2199
- const processLegalFormMatches = (allMatches, sliceStart, sliceEnd, fullText) => {
2200
- const results = [];
2201
- for (const match of allMatches) {
2202
- const idx = match.pattern;
2203
- if (idx < sliceStart || idx >= sliceEnd) continue;
2204
- const text = match.text.trimEnd();
2205
- if (text.length < 5) continue;
2206
- if (text.includes("\n")) continue;
2207
- let entityStart = match.start;
2208
- let entityText = text;
2209
- if (fullText) {
2210
- const extended = extendBackward(fullText, match.start);
2211
- if (extended < match.start) {
2212
- entityStart = extended;
2213
- entityText = fullText.slice(extended, match.start + text.length).trimEnd();
2214
- }
2215
- }
2216
- const clauseTrim = trimLeadingClause(entityText);
2217
- if (clauseTrim.offset > 0) {
2218
- entityStart += clauseTrim.offset;
2219
- entityText = clauseTrim.text;
2220
- }
2221
- const getPrefixInfo = (value) => {
2222
- const prefixEnd = value.lastIndexOf(",") !== -1 ? value.lastIndexOf(",") : value.lastIndexOf(" ");
2223
- return {
2224
- prefixEnd,
2225
- prefixPart: prefixEnd > 0 ? value.slice(0, prefixEnd).replace(/[^a-zA-ZÀ-ž]/g, "") : value.replace(/[^a-zA-ZÀ-ž]/g, "")
2226
- };
2227
- };
2228
- let { prefixEnd, prefixPart } = getPrefixInfo(entityText);
2229
- let isAllCapsMatch = prefixPart.length > 2 && prefixPart === prefixPart.toUpperCase();
2230
- if (isAllCapsMatch && fullText) {
2231
- const lineStart = fullText.lastIndexOf("\n", entityStart);
2232
- const lineEnd = fullText.indexOf("\n", entityStart + entityText.length);
2233
- const lineLetters = fullText.slice(lineStart + 1, lineEnd === -1 ? fullText.length : lineEnd).replace(/[^a-zA-ZÀ-ž]/g, "");
2234
- const upperCount = [...lineLetters].filter((c) => c === c.toUpperCase()).length;
2235
- if (lineLetters.length > 5 && upperCount / lineLetters.length >= .95) continue;
2236
- if ((prefixPart.length > 0 ? entityText.slice(0, prefixEnd > 0 ? prefixEnd : entityText.length).trim().split(/\s+/).length : 0) > 3) {
2237
- entityStart = match.start;
2238
- entityText = text;
2239
- ({prefixEnd, prefixPart} = getPrefixInfo(entityText));
2240
- isAllCapsMatch = prefixPart.length > 2 && prefixPart === prefixPart.toUpperCase();
2241
- }
2242
- } else if (isAllCapsMatch) continue;
2243
- const lastSpace = entityText.lastIndexOf(" ");
2244
- const rawSuffix = lastSpace !== -1 ? entityText.slice(lastSpace + 1) : "";
2245
- const suffixClean = rawSuffix.replace(/[.,]/g, "");
2246
- if (suffixClean.length > 0 && ROMAN_NUMERAL_RE.test(suffixClean)) continue;
2247
- if (suffixClean.length <= 2 && !/\./.test(rawSuffix) && /[^\x00-\x7F]/.test(entityText.slice(0, lastSpace !== -1 ? lastSpace : entityText.length))) continue;
2248
- results.push({
2249
- start: entityStart,
2250
- end: entityStart + entityText.length,
2251
- label: "organization",
2252
- text: entityText,
2253
- score: .95,
2254
- source: DETECTION_SOURCES.LEGAL_FORM
2255
- });
2621
+ const meta = meta_[idx - sliceStart];
2622
+ if (!meta) continue;
2623
+ if (meta.sourceDetail !== "custom-regex" && meta.label === "phone number" && match.text.length < MIN_PHONE_LENGTH) continue;
2624
+ if (meta.validator) {
2625
+ const compacted = meta.validator.compact(match.text);
2626
+ if (!meta.validator.validate(compacted).valid) continue;
2627
+ }
2628
+ const entity = {
2629
+ start: match.start,
2630
+ end: match.end,
2631
+ label: meta.label,
2632
+ text: match.text,
2633
+ score: meta.score,
2634
+ source: DETECTION_SOURCES.REGEX
2635
+ };
2636
+ if (meta.sourceDetail) entity.sourceDetail = meta.sourceDetail;
2637
+ results.push(entity);
2256
2638
  }
2257
2639
  return results;
2258
2640
  };
2259
- //#endregion
2260
- //#region src/config/legal-forms.ts
2261
2641
  /**
2262
- * Known legal form suffixes. Shared between trigger
2263
- * detection (reclassification) and org-propagation
2264
- * (suffix stripping). Within each family of related
2265
- * forms, longer variants come first so that
2266
- * "spol. s r.o." matches before "s.r.o." and
2267
- * "s. r. o." matches before "s.r.o.".
2642
+ * Build signing clause place-name patterns from
2643
+ * signing-clauses.json. Each pattern captures the
2644
+ * city/place name from contract signing locations.
2645
+ *
2646
+ * The place name sub-pattern:
2647
+ * \p{Lu}\p{Ll}+ (capitalized word)
2648
+ * optionally followed by preposition + capitalized
2649
+ * word (for "nad Nisou", "am Main", etc.)
2650
+ * optionally followed by more capitalized words
2651
+ * (for "Hradec Králové", "New York", etc.)
2268
2652
  */
2269
- const LEGAL_SUFFIXES = [
2270
- "spol. s r.o.",
2271
- "s.r.o.",
2272
- "s. r. o.",
2273
- "a.s.",
2274
- "a. s.",
2275
- "v.o.s.",
2276
- "v. o. s.",
2277
- "k.s.",
2278
- "k. s.",
2279
- "z.s.",
2280
- "z. s.",
2281
- "z.ú.",
2282
- "z. ú.",
2283
- "o.p.s.",
2284
- "o. p. s.",
2285
- "s.p.",
2286
- "s. p.",
2287
- "GmbH",
2288
- "AG",
2289
- "SE",
2290
- "KG",
2291
- "OHG",
2292
- "Ltd.",
2293
- "Ltd",
2294
- "LLC",
2295
- "LLP",
2296
- "Inc.",
2297
- "S.A.",
2298
- "SA",
2299
- "SAS",
2300
- "SARL",
2301
- "Sp. z o.o.",
2302
- "S.p.A."
2303
- ];
2653
+ const buildSigningClausePatterns = (data) => {
2654
+ const patterns = [];
2655
+ for (const entry of data.patterns) {
2656
+ const prepAlt = entry.prepositions.length > 0 ? entry.prepositions.join("|") : null;
2657
+ const place = prepAlt ? `(\\p{Lu}\\p{Ll}+(?:\\s+(?:${prepAlt})\\s+\\p{Lu}\\p{Ll}+)*(?:\\s+\\p{Lu}\\p{Ll}+)*)` : `(\\p{Lu}\\p{Ll}+(?:[- ]\\p{Lu}\\p{Ll}+)*)`;
2658
+ const full = `(?:^|\\n|[^\\S\\n])` + entry.prefix + place + (entry.suffix ? `(?:${entry.suffix})` : "");
2659
+ patterns.push(full);
2660
+ }
2661
+ return patterns;
2662
+ };
2663
+ const SIGNING_CLAUSE_META = {
2664
+ label: "address",
2665
+ score: .9
2666
+ };
2667
+ let signingPatternPromise = null;
2668
+ const loadSigningPatterns = async () => {
2669
+ const mod = await import("./signing-clauses.mjs");
2670
+ return buildSigningClausePatterns(mod.default ?? mod);
2671
+ };
2672
+ const getSigningClausePatterns = () => {
2673
+ if (!signingPatternPromise) signingPatternPromise = loadSigningPatterns().catch((err) => {
2674
+ signingPatternPromise = null;
2675
+ throw err;
2676
+ });
2677
+ return signingPatternPromise;
2678
+ };
2304
2679
  //#endregion
2305
2680
  //#region src/detectors/triggers.ts
2681
+ const VALID_ID_VALIDATORS = {
2682
+ "br.cpf": br.cpf,
2683
+ "br.cnpj": br.cnpj
2684
+ };
2306
2685
  const TRIGGER_SCORE = .95;
2307
2686
  const WHITESPACE_RE$1 = /\s+/;
2308
2687
  const LETTER_RE = /\p{L}/u;
@@ -2317,12 +2696,23 @@ const DECIMAL_COMMA_RE = new RegExp(`^,(?:\\d|${DASH}{1,2})`);
2317
2696
  * etc.), skip the comma and degree, then continue.
2318
2697
  */
2319
2698
  const POST_NOMINAL_RE = new RegExp(`^,\\s*(?:${POST_NOMINALS.toSorted((a, b) => b.length - a.length).map((d) => d.replace(/[.*+?^${}()|[\]\\]/g, "\\$&").replace(/\\\./g, "\\.\\s*")).join("|")})\\.?`, "i");
2320
- const LEGAL_FORM_CHECK_RE = new RegExp(LEGAL_SUFFIXES.map((f) => {
2321
- const escaped = f.replace(/[.*+?^${}()|[\]\\]/g, "\\$&").replace(/\\\./g, "\\.\\s*");
2322
- const isDotFree = !f.includes(".");
2323
- const isShort = f.length <= 4;
2324
- return isDotFree && isShort ? `\\b${escaped}\\b` : escaped;
2325
- }).join("|"));
2699
+ const buildLegalFormCheckRe = (forms) => {
2700
+ const parts = forms.map((f) => {
2701
+ const escaped = f.replace(/[.*+?^${}()|[\]\\]/g, "\\$&").replace(/\\\./g, "\\.\\s*");
2702
+ return /^[\p{L}\p{M}]+$/u.test(f) ? `(?<![\\p{L}\\p{N}])${escaped}(?![\\p{L}\\p{N}])` : escaped;
2703
+ });
2704
+ return new RegExp(parts.join("|"), "u");
2705
+ };
2706
+ let cachedLegalFormCheckRe = null;
2707
+ let cachedLegalFormCheckSource = null;
2708
+ const getLegalFormCheckRe = () => {
2709
+ const source = getKnownLegalSuffixes();
2710
+ if (cachedLegalFormCheckSource !== source || cachedLegalFormCheckRe === null) {
2711
+ cachedLegalFormCheckSource = source;
2712
+ cachedLegalFormCheckRe = buildLegalFormCheckRe(source);
2713
+ }
2714
+ return cachedLegalFormCheckRe;
2715
+ };
2326
2716
  const compileValidations = (validations) => validations.map((v) => {
2327
2717
  switch (v.type) {
2328
2718
  case "starts-uppercase": return {
@@ -2349,6 +2739,17 @@ const compileValidations = (validations) => validations.map((v) => {
2349
2739
  type: "matches-pattern",
2350
2740
  re: new RegExp(v.pattern, (v.flags ?? "").replace(/[gy]/g, ""))
2351
2741
  };
2742
+ case "valid-id": {
2743
+ const validator = VALID_ID_VALIDATORS[v.validator];
2744
+ if (!validator) throw new Error(`Unknown valid-id validator: ${JSON.stringify(v.validator)}`);
2745
+ return {
2746
+ type: "valid-id",
2747
+ check: (value) => {
2748
+ const compact = value.replace(/[\s.\-/]/g, "");
2749
+ return validator.validate(compact).valid;
2750
+ }
2751
+ };
2752
+ }
2352
2753
  default: throw new Error(`Unknown validation type: ${JSON.stringify(v)}`);
2353
2754
  }
2354
2755
  });
@@ -2372,6 +2773,9 @@ const applyValidations = (text, validations) => {
2372
2773
  case "matches-pattern":
2373
2774
  if (!v.re.test(text)) return false;
2374
2775
  break;
2776
+ case "valid-id":
2777
+ if (!v.check(text)) return false;
2778
+ break;
2375
2779
  default: throw new Error(`Unknown compiled validation type: ${JSON.stringify(v)}`);
2376
2780
  }
2377
2781
  return true;
@@ -2466,8 +2870,10 @@ const buildTriggerPatterns = async () => {
2466
2870
  strategy: rule.strategy.type
2467
2871
  });
2468
2872
  }
2873
+ const patterns = rules.map((r) => r.trigger.toLowerCase());
2874
+ await loadAddressStopKeywords();
2469
2875
  return {
2470
- patterns: rules.map((r) => r.trigger.toLowerCase()),
2876
+ patterns,
2471
2877
  rules
2472
2878
  };
2473
2879
  };
@@ -2488,14 +2894,103 @@ const stripQuotes = (value) => {
2488
2894
  const COMMA_STOP_CHARS = new Set([
2489
2895
  "\n",
2490
2896
  "(",
2491
- " "
2897
+ " ",
2898
+ ";"
2492
2899
  ]);
2493
2900
  /**
2901
+ * Sentence-terminator detection: a period that genuinely ends
2902
+ * one clause and starts another. Used by `to-next-comma` so
2903
+ * governing-law clauses ("…State of New York. SECTION 2…")
2904
+ * don't sweep across the period.
2905
+ *
2906
+ * Three positive signals (any one terminates):
2907
+ * 1. Long lowercase tail before the dot (>= 5 letters) —
2908
+ * catches "…construction. SECTION 2…", "…vykonává.".
2909
+ * 2. Currency/amount tail (zł, Kč, USD, €) — catches
2910
+ * "…w kwocie 1000 zł. Termin płatności…".
2911
+ * 3. Proper-noun head: a capitalized word of >= 4 letters
2912
+ * ending in lowercase, with no internal uppercase, AND
2913
+ * a substantial next clause (Capital + >= 2 lowercase).
2914
+ * Catches short city names that the lowercase-tail rule
2915
+ * misses: "…z siedzibą w Łódź. Kapitał zakładowy…",
2916
+ * "…seat in Brno. Section 2…".
2917
+ *
2918
+ * The rule must NOT fire on:
2919
+ * - title abbreviations: "Mr.", "Mrs.", "Dr.", "Hon.",
2920
+ * "Sr.", "Jr." (head <= 3 chars or insufficient Ll tail)
2921
+ * - degree abbreviations: "Ph.D.", "RNDr.", "MUDr.",
2922
+ * "Ing." (internal periods or internal uppercase block
2923
+ * the proper-noun pattern)
2924
+ * - street-type abbreviations: "Ste.", "Ave.", "Inc.",
2925
+ * "ul.", "al.", "nábř." (lowercase initials or
2926
+ * insufficient Ll tail)
2927
+ * - small-word lowercase abbreviations: "prof.", "inż.",
2928
+ * "hab." (no leading uppercase, so the proper-noun rule
2929
+ * can't fire)
2930
+ */
2931
+ const NEXT_IS_SENTENCE_START_RE = /^\.(?:\s+\p{Lu}|\s*$)/u;
2932
+ const SENTENCE_TAIL_RE = /\p{Ll}{5,}$/u;
2933
+ /**
2934
+ * Proper-noun tail: capital letter + >= 3 lowercase letters,
2935
+ * preceded by a non-letter and non-period (so we don't slice
2936
+ * into the middle of an acronym or a multi-dot abbreviation).
2937
+ * The 4-character minimum excludes 2–3-char titles ("Mr",
2938
+ * "Mrs", "Dr", "Inc", "Ste"); the all-lowercase tail
2939
+ * excludes mixed-case degrees ("RNDr", "MUDr").
2940
+ */
2941
+ const PROPER_NOUN_HEAD_RE = /(?:^|[^\p{L}.])\p{Lu}\p{Ll}{3,}$/u;
2942
+ /**
2943
+ * Next clause begins with a real word: capital + >= 2
2944
+ * lowercase letters. Filters cases where a capitalized
2945
+ * abbreviation (e.g., "Smith Inc.") follows a proper noun,
2946
+ * which would otherwise look sentence-like.
2947
+ */
2948
+ const NEXT_IS_REAL_SENTENCE_RE = /^\.\s+\p{Lu}\p{Ll}{2,}/u;
2949
+ /**
2950
+ * Short currency-abbreviation tail (zł, Kč, gr, Ft, kr,
2951
+ * лв, USD, PLN, EUR, …). When a period follows one of
2952
+ * these and the next token starts uppercase, treat it as
2953
+ * a sentence boundary even though the tail is too short
2954
+ * to satisfy `SENTENCE_TAIL_RE`. Without this, amount
2955
+ * triggers using `to-next-comma` swallow the following
2956
+ * clause: `"w kwocie 1000 zł. Termin płatności…"`.
2957
+ *
2958
+ * Currency codes are typically uppercase (`USD`, `PLN`)
2959
+ * but appear lowercased in informal writing (`pln`, `eur`);
2960
+ * local names mix case (`zł`, `Kč`, `Ft`); symbols
2961
+ * (`€`, `$`, `£`) appear after the amount as well. The
2962
+ * negative lookbehind on a letter ensures the abbreviation
2963
+ * is matched only as a standalone token; symbols are
2964
+ * matched unconditionally since they are not letters.
2965
+ */
2966
+ const CURRENCY_TAIL_RE = /(?:(?<![\p{L}])(?:zł|Kč|gr|Ft|kr|лв|USD|PLN|EUR|CZK|GBP|CHF|HUF|RON|SEK|NOK|DKK)|[€$£])$/iu;
2967
+ const NUMERIC_SENTENCE_TAIL_RE = /\d$/;
2968
+ const isSentenceTerminator = (text, periodIndex) => {
2969
+ const tail = text.slice(periodIndex);
2970
+ if (!NEXT_IS_SENTENCE_START_RE.test(tail)) return false;
2971
+ const head = text.slice(0, periodIndex);
2972
+ if (SENTENCE_TAIL_RE.test(head) || CURRENCY_TAIL_RE.test(head) || NUMERIC_SENTENCE_TAIL_RE.test(head)) return true;
2973
+ return PROPER_NOUN_HEAD_RE.test(head) && NEXT_IS_REAL_SENTENCE_RE.test(tail);
2974
+ };
2975
+ /**
2494
2976
  * Field-label keywords that terminate address scanning.
2495
2977
  * When a comma in the address strategy is followed by
2496
2978
  * one of these, the address stops before the keyword.
2979
+ *
2980
+ * The list is sourced from
2981
+ * `data/address-stop-keywords.json` (per-language so new
2982
+ * languages can drop in their own labels without
2983
+ * touching this file). `loadAddressStopKeywords` unions
2984
+ * every language into a single longest-first array;
2985
+ * `getAddressStopKeywordsSync` returns the cached union
2986
+ * and falls back to a seed list so the strategy keeps
2987
+ * working before the warmup promise resolves.
2988
+ *
2989
+ * The address strategy doesn't know the document
2990
+ * language, so a flat union is intentional: any
2991
+ * language's labels can appear in any address.
2497
2992
  */
2498
- const ADDRESS_STOP_KEYWORDS = [
2993
+ const ADDRESS_STOP_KEYWORDS_SEED = [
2499
2994
  "číslo účtu",
2500
2995
  "registrační",
2501
2996
  "zastoupen",
@@ -2518,6 +3013,52 @@ const ADDRESS_STOP_KEYWORDS = [
2518
3013
  "bic",
2519
3014
  "ič"
2520
3015
  ];
3016
+ let addressStopKeywordsCache = null;
3017
+ let addressStopKeywordsPromise = null;
3018
+ const loadAddressStopKeywords = async () => {
3019
+ if (addressStopKeywordsCache) return addressStopKeywordsCache;
3020
+ if (addressStopKeywordsPromise) return addressStopKeywordsPromise;
3021
+ addressStopKeywordsPromise = (async () => {
3022
+ let data = {};
3023
+ try {
3024
+ const mod = await import("./address-stop-keywords.mjs");
3025
+ data = mod.default ?? mod;
3026
+ } catch (err) {
3027
+ console.warn("[anonymize] triggers: failed to load address-stop-keywords.json, falling back to seed list:", err);
3028
+ }
3029
+ const seen = /* @__PURE__ */ new Set();
3030
+ const out = [];
3031
+ const addAll = (list) => {
3032
+ for (const kw of list) {
3033
+ if (typeof kw !== "string" || kw.length === 0) continue;
3034
+ const lower = kw.toLowerCase();
3035
+ if (seen.has(lower)) continue;
3036
+ seen.add(lower);
3037
+ out.push(lower);
3038
+ }
3039
+ };
3040
+ for (const [key, value] of Object.entries(data)) {
3041
+ if (key.startsWith("_")) continue;
3042
+ if (!Array.isArray(value)) continue;
3043
+ addAll(value);
3044
+ }
3045
+ addAll(ADDRESS_STOP_KEYWORDS_SEED);
3046
+ out.sort((a, b) => b.length - a.length);
3047
+ addressStopKeywordsCache = out;
3048
+ return out;
3049
+ })();
3050
+ return addressStopKeywordsPromise;
3051
+ };
3052
+ const getAddressStopKeywordsSync = () => addressStopKeywordsCache ?? ADDRESS_STOP_KEYWORDS_SEED;
3053
+ /**
3054
+ * Warm the address-stop-keywords cache. Pipeline callers
3055
+ * await this before invoking trigger detection so the
3056
+ * synchronous `extractValue` path uses the merged list
3057
+ * instead of the seed fallback.
3058
+ */
3059
+ const warmAddressStopKeywords = async () => {
3060
+ await loadAddressStopKeywords();
3061
+ };
2521
3062
  const extractValue = (text, triggerEnd, strategy, label) => {
2522
3063
  const remaining = text.slice(triggerEnd);
2523
3064
  const stripped = remaining.replace(/^[\s:;]+/, "");
@@ -2526,6 +3067,8 @@ const extractValue = (text, triggerEnd, strategy, label) => {
2526
3067
  if (valueText.length === 0) return null;
2527
3068
  switch (strategy.type) {
2528
3069
  case "to-next-comma": {
3070
+ const stopWords = strategy.stopWords ?? [];
3071
+ const stopWordRe = stopWords.length > 0 ? new RegExp(`^(?:${stopWords.map((w) => w.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")).join("|")})(?![\\p{L}\\p{N}])`, "iu") : null;
2529
3072
  let end = 0;
2530
3073
  let foundStop = false;
2531
3074
  while (end < valueText.length) {
@@ -2534,6 +3077,14 @@ const extractValue = (text, triggerEnd, strategy, label) => {
2534
3077
  foundStop = true;
2535
3078
  break;
2536
3079
  }
3080
+ if (ch === "." && isSentenceTerminator(valueText, end)) {
3081
+ foundStop = true;
3082
+ break;
3083
+ }
3084
+ if (stopWordRe !== null && (end === 0 || !/[\p{L}\p{N}]/u.test(valueText[end - 1] ?? "")) && stopWordRe.test(valueText.slice(end))) {
3085
+ foundStop = true;
3086
+ break;
3087
+ }
2537
3088
  if (ch === ",") {
2538
3089
  const afterComma = valueText.slice(end);
2539
3090
  if (DECIMAL_COMMA_RE.test(afterComma)) {
@@ -2562,6 +3113,8 @@ const extractValue = (text, triggerEnd, strategy, label) => {
2562
3113
  };
2563
3114
  }
2564
3115
  case "to-end-of-line": {
3116
+ const consumed = remaining.length - valueText.length;
3117
+ if (consumed > 0 && remaining.slice(0, consumed).includes("\n")) return null;
2565
3118
  const LINE_STOPS = ["\n"];
2566
3119
  let end = valueText.length;
2567
3120
  for (const ch of LINE_STOPS) {
@@ -2582,7 +3135,8 @@ const extractValue = (text, triggerEnd, strategy, label) => {
2582
3135
  const tabIdx = valueText.indexOf(" ");
2583
3136
  const cellText = tabIdx !== -1 ? valueText.slice(0, tabIdx) : valueText;
2584
3137
  const PUNCT_ONLY = /^[\p{P}\p{S}]+$/u;
2585
- const words = cellText.split(WHITESPACE_RE$1).filter((w) => !PUNCT_ONLY.test(w)).slice(0, strategy.count);
3138
+ const NUMBER_MARKER = /^(?:n[ºo°.]|№)$/i;
3139
+ const words = cellText.split(WHITESPACE_RE$1).filter((w) => !PUNCT_ONLY.test(w) && !NUMBER_MARKER.test(w)).slice(0, strategy.count);
2586
3140
  if (words.length === 0) return null;
2587
3141
  const firstWord = words[0];
2588
3142
  if (firstWord === void 0) return null;
@@ -2605,14 +3159,22 @@ const extractValue = (text, triggerEnd, strategy, label) => {
2605
3159
  }
2606
3160
  case "company-id-value": {
2607
3161
  const raw = text.slice(triggerEnd);
2608
- const sepMatch = /^(?:\s*:\s*|\s+)/.exec(raw);
3162
+ const triggerLastChar = text[triggerEnd - 1] ?? "";
3163
+ const sepMatch = (triggerLastChar === "°" || triggerLastChar === "º" || triggerLastChar === "№" || triggerLastChar === "#" ? /^(?:\s*:\s*|\s+|)/ : /^(?:\s*:\s*|\s+)/).exec(raw);
2609
3164
  if (!sepMatch) return null;
2610
- const afterSep = raw.slice(sepMatch[0].length);
2611
- const idMatch = /^[A-Z]{0,6}\s?\d[\d\s\-/]{4,}/i.exec(afterSep);
3165
+ let afterSep = raw.slice(sepMatch[0].length);
3166
+ const labelMatch = /^(?:nr\.?|numer|n[ºo°.]|№|no\.?)(?:\s*:\s*|\s+)/i.exec(afterSep);
3167
+ let labelOffset = 0;
3168
+ if (labelMatch) {
3169
+ labelOffset = labelMatch[0].length;
3170
+ afterSep = afterSep.slice(labelOffset);
3171
+ }
3172
+ const idMatch = /^[A-Z]{0,6}\s?(?:[A-Z0-9]{2}\s?)?\d{2}(?:(?<=\d)[A-Z]|[\d\s.\-/]){3,}/i.exec(afterSep);
2612
3173
  if (!idMatch) return null;
2613
- const idText = idMatch[0].trim();
3174
+ const trailingLetterMatch = /\d$/.test(idMatch[0]) ? /^[A-Z]/i.exec(afterSep.slice(idMatch[0].length)) : null;
3175
+ const idText = (trailingLetterMatch ? idMatch[0] + trailingLetterMatch[0] : idMatch[0]).trim();
2614
3176
  const leadingSpaces = idMatch[0].length - idMatch[0].trimStart().length;
2615
- const idStart = triggerEnd + sepMatch[0].length + leadingSpaces;
3177
+ const idStart = triggerEnd + sepMatch[0].length + labelOffset + leadingSpaces;
2616
3178
  return {
2617
3179
  start: idStart,
2618
3180
  end: idStart + idText.length,
@@ -2622,18 +3184,36 @@ const extractValue = (text, triggerEnd, strategy, label) => {
2622
3184
  case "address": {
2623
3185
  const maxLen = strategy.maxChars ?? 120;
2624
3186
  const UPPER_RE = /\p{Lu}/u;
3187
+ const stopKeywords = getAddressStopKeywordsSync();
2625
3188
  let end = 0;
2626
3189
  while (end < valueText.length && end < maxLen) {
2627
3190
  const ch = valueText[end];
2628
3191
  if (ch === "\n" || ch === "(") break;
3192
+ if (ch === " " || ch === " ") {
3193
+ const afterWs = valueText.slice(end).trimStart().toLowerCase();
3194
+ if (stopKeywords.some((kw) => {
3195
+ if (!afterWs.startsWith(kw)) return false;
3196
+ const next = afterWs[kw.length];
3197
+ return next === void 0 || /[\s:;.,!?()\d]/.test(next);
3198
+ })) break;
3199
+ }
2629
3200
  if (ch === ".") {
2630
3201
  const next = valueText[end + 1];
2631
3202
  const afterNext = valueText[end + 2];
3203
+ const afterPeriod = valueText.slice(end + 1).replace(/^\s+/, "").toLowerCase();
3204
+ if (afterPeriod.length > 0) {
3205
+ if (getAddressStopKeywordsSync().some((kw) => {
3206
+ if (!afterPeriod.startsWith(kw)) return false;
3207
+ const afterKw = afterPeriod[kw.length];
3208
+ return afterKw === void 0 || /[\s:;.,!?()\d]/.test(afterKw);
3209
+ })) break;
3210
+ }
2632
3211
  if (next !== void 0 && (/\p{L}/u.test(next) || /\d/.test(next))) {
2633
3212
  end++;
2634
3213
  continue;
2635
3214
  }
2636
3215
  if (next === " " && afterNext !== void 0 && (/\p{L}/u.test(afterNext) || /\d/.test(afterNext))) {
3216
+ if (isSentenceTerminator(valueText, end)) break;
2637
3217
  end++;
2638
3218
  continue;
2639
3219
  }
@@ -2645,7 +3225,7 @@ const extractValue = (text, triggerEnd, strategy, label) => {
2645
3225
  const peekCh = valueText[peek];
2646
3226
  if (peekCh === void 0) break;
2647
3227
  const afterComma = valueText.slice(end + 1).trimStart().toLowerCase();
2648
- if (ADDRESS_STOP_KEYWORDS.some((kw) => {
3228
+ if (getAddressStopKeywordsSync().some((kw) => {
2649
3229
  if (!afterComma.startsWith(kw)) return false;
2650
3230
  const next = afterComma[kw.length];
2651
3231
  return next === void 0 || /[\s:;.,!?()\d]/.test(next);
@@ -2672,6 +3252,28 @@ const extractValue = (text, triggerEnd, strategy, label) => {
2672
3252
  text: extracted
2673
3253
  };
2674
3254
  }
3255
+ case "match-pattern": {
3256
+ const nlIdx = valueText.indexOf("\n");
3257
+ const searchText = nlIdx === -1 ? valueText : valueText.slice(0, nlIdx);
3258
+ if (searchText.length === 0) return null;
3259
+ const flags = (strategy.flags ?? "").replace(/[gy]/g, "");
3260
+ const anchoredSource = `^(?:${strategy.pattern})`;
3261
+ let re;
3262
+ try {
3263
+ re = new RegExp(anchoredSource, flags);
3264
+ } catch {
3265
+ return null;
3266
+ }
3267
+ const m = re.exec(searchText);
3268
+ if (!m || m[0].length === 0) return null;
3269
+ const matchStart = m.index;
3270
+ const matchEnd = matchStart + m[0].length;
3271
+ return {
3272
+ start: valueStart + matchStart,
3273
+ end: valueStart + matchEnd,
3274
+ text: m[0]
3275
+ };
3276
+ }
2675
3277
  default: return null;
2676
3278
  }
2677
3279
  };
@@ -2691,7 +3293,8 @@ const processTriggerMatches = (allMatches, sliceStart, sliceEnd, fullText, rules
2691
3293
  if (match.start > 0 && LETTER_RE.test(fullText[match.start - 1] ?? "")) continue;
2692
3294
  const rule = rules[localIdx];
2693
3295
  if (!rule) continue;
2694
- if (!rule.trigger.endsWith(" ") && LETTER_RE.test(fullText[match.end] ?? "")) continue;
3296
+ const triggerLastChar = rule.trigger.at(-1) ?? "";
3297
+ if (LETTER_RE.test(triggerLastChar) && LETTER_RE.test(fullText[match.end] ?? "")) continue;
2695
3298
  const triggerEnd = match.end;
2696
3299
  const rawValue = extractValue(fullText, triggerEnd, rule.strategy, rule.label);
2697
3300
  const value = rawValue ? stripQuotes(rawValue) : null;
@@ -2700,7 +3303,7 @@ const processTriggerMatches = (allMatches, sliceStart, sliceEnd, fullText, rules
2700
3303
  const entityStart = rule.includeTrigger ? match.start : value.start;
2701
3304
  const entityEnd = value.end;
2702
3305
  const entityText = fullText.slice(entityStart, entityEnd);
2703
- const effectiveLabel = rule.label === "person" && LEGAL_FORM_CHECK_RE.test(entityText) ? "organization" : rule.label;
3306
+ const effectiveLabel = rule.label === "person" && getLegalFormCheckRe().test(entityText) ? "organization" : rule.label;
2704
3307
  results.push({
2705
3308
  start: entityStart,
2706
3309
  end: entityEnd,
@@ -2970,11 +3573,82 @@ const resolveCountries = (regions, countries) => {
2970
3573
  return result;
2971
3574
  };
2972
3575
  //#endregion
3576
+ //#region src/util/homoglyphs.ts
3577
+ const CHAR_MAP$1 = new Map([
3578
+ [1040, 65],
3579
+ [1042, 66],
3580
+ [1045, 69],
3581
+ [1050, 75],
3582
+ [1052, 77],
3583
+ [1053, 72],
3584
+ [1054, 79],
3585
+ [1056, 80],
3586
+ [1057, 67],
3587
+ [1058, 84],
3588
+ [1061, 88],
3589
+ [1072, 97],
3590
+ [1077, 101],
3591
+ [1086, 111],
3592
+ [1088, 112],
3593
+ [1089, 99],
3594
+ [1091, 121],
3595
+ [1093, 120],
3596
+ [1110, 105],
3597
+ [1112, 106],
3598
+ [913, 65],
3599
+ [914, 66],
3600
+ [917, 69],
3601
+ [919, 72],
3602
+ [921, 73],
3603
+ [922, 75],
3604
+ [924, 77],
3605
+ [925, 78],
3606
+ [927, 79],
3607
+ [929, 80],
3608
+ [932, 84],
3609
+ [933, 89],
3610
+ [935, 88],
3611
+ [918, 90],
3612
+ [945, 97],
3613
+ [954, 107],
3614
+ [959, 111],
3615
+ [961, 112]
3616
+ ]);
3617
+ /** Chunk size for `String.fromCharCode` to avoid hitting
3618
+ * the call-stack limit on very large strings. */
3619
+ const CHUNK_SIZE$1 = 8192;
3620
+ const normalizeHomoglyphs = (text) => {
3621
+ let hasSpecial = false;
3622
+ for (let i = 0; i < text.length; i++) if (CHAR_MAP$1.has(text.charCodeAt(i))) {
3623
+ hasSpecial = true;
3624
+ break;
3625
+ }
3626
+ if (!hasSpecial) return text;
3627
+ const len = text.length;
3628
+ const codes = new Uint16Array(len);
3629
+ for (let i = 0; i < len; i++) {
3630
+ const code = text.charCodeAt(i);
3631
+ codes[i] = CHAR_MAP$1.get(code) ?? code;
3632
+ }
3633
+ if (len <= CHUNK_SIZE$1) return String.fromCharCode(...codes);
3634
+ let result = "";
3635
+ for (let offset = 0; offset < len; offset += CHUNK_SIZE$1) {
3636
+ const end = Math.min(offset + CHUNK_SIZE$1, len);
3637
+ result += String.fromCharCode(...codes.subarray(offset, end));
3638
+ }
3639
+ return result;
3640
+ };
3641
+ //#endregion
2973
3642
  //#region src/filters/false-positives.ts
2974
3643
  const TEMPLATE_PLACEHOLDER_RE = /^(?:\.{3,}|_{3,}|\[[\w\s]+\]|\{[\w\s]+\})$/;
2975
3644
  const POSTAL_CODE_RE = /\d{3}\s?\d{2}/;
2976
3645
  const HAS_DIGIT_RE = /\d/;
2977
- const ADDRESS_COMPONENTS_RE = /(?:^|\s)(?:ul\.|ulice|nám\.|náměstí|tř\.|třída|nábř\.|nábřeží|č\.p\.|č\.ev\.|č\.|sídliště|bulvár)(?=[\s,./]|$)/i;
3646
+ const ADDRESS_COMPONENT_EXTRA_RE = /(?:^|\s)(?:č\.p\.|č\.ev\.|č\.|sídliště)(?=[\s,./]|$)/i;
3647
+ const BARE_COURS_PROSE_RE = /(?:^|\s)cours(?!\s+\p{Lu})(?=[\s,./]|$)/u;
3648
+ const isOnlyAmbiguousCours = (text) => {
3649
+ if (!BARE_COURS_PROSE_RE.test(text)) return false;
3650
+ return !hasAddressComponent(text.replace(/(?:^|\s)cours(?=[\s,./]|$)/gi, " "));
3651
+ };
2978
3652
  const JURISDICTION_RE = /^(?:state|commonwealth|district|territory)\s+of\s+/i;
2979
3653
  const MAX_ENTITY_LENGTH = {
2980
3654
  organization: 80,
@@ -3057,6 +3731,41 @@ const loadGenericRoles = (ctx = defaultContext) => {
3057
3731
  };
3058
3732
  /** Sync accessor — returns empty set before init. */
3059
3733
  const getGenericRoles = (ctx) => ctx.genericRoles ?? EMPTY_GENERIC_ROLES;
3734
+ const STREET_TYPES_SEED_RE = /(?:^|\s)(?:ul\.|ulice|nám\.|náměstí|tř\.|třída|nábř\.|nábřeží|bulvár)(?=[\s,./]|$)/i;
3735
+ let _streetTypesRe = STREET_TYPES_SEED_RE;
3736
+ let _streetTypesPromise = null;
3737
+ const escapeRegex$1 = (s) => s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
3738
+ const loadStreetTypeRegex = async () => {
3739
+ try {
3740
+ const data = (await import("./address-street-types.mjs")).default ?? {};
3741
+ const words = /* @__PURE__ */ new Set();
3742
+ for (const [key, val] of Object.entries(data)) {
3743
+ if (key.startsWith("_")) continue;
3744
+ if (!Array.isArray(val)) continue;
3745
+ for (const w of val) if (typeof w === "string" && w.length > 0) words.add(w.toLowerCase());
3746
+ }
3747
+ if (words.size === 0) {
3748
+ _streetTypesRe = STREET_TYPES_SEED_RE;
3749
+ return;
3750
+ }
3751
+ const alternation = [...words].sort((a, b) => b.length - a.length).map(escapeRegex$1).join("|");
3752
+ _streetTypesRe = new RegExp(`(?:^|\\s)(?:${alternation})(?=[\\s,./]|$)`, "iu");
3753
+ } catch {
3754
+ _streetTypesRe = STREET_TYPES_SEED_RE;
3755
+ }
3756
+ };
3757
+ /** Ensure street-type vocabulary is loaded. */
3758
+ const initAddressComponents = () => {
3759
+ if (!_streetTypesPromise) _streetTypesPromise = loadStreetTypeRegex();
3760
+ return _streetTypesPromise;
3761
+ };
3762
+ /**
3763
+ * True when `text` contains a recognised address
3764
+ * component: either a per-language street word (loaded
3765
+ * from JSON) or a language-agnostic anchor such as a
3766
+ * Czech house number form.
3767
+ */
3768
+ const hasAddressComponent = (text) => _streetTypesRe.test(text) || ADDRESS_COMPONENT_EXTRA_RE.test(text);
3060
3769
  const isCallerOwnedEntity$3 = (entity) => entity.sourceDetail === "custom-deny-list" || entity.sourceDetail === "custom-regex";
3061
3770
  /**
3062
3771
  * Filter out entities that are likely false positives:
@@ -3091,13 +3800,15 @@ const filterFalsePositives = (entities, ctx = defaultContext, fullText) => {
3091
3800
  if (normalized.label === "person" && HAS_DIGIT_RE.test(trimmed)) continue;
3092
3801
  if (normalized.label === "person") {
3093
3802
  const tokens = trimmed.split(/\s+/u);
3094
- const last = tokens.at(-1)?.replace(/[.,;:!?]+$/u, "").toLowerCase();
3095
- if (tokens.length > 1 && last && PERSON_TRAILING_NOUNS.has(last)) continue;
3803
+ const last = tokens.at(-1)?.replace(/[.,;:!?]+$/u, "");
3804
+ const lastFolded = last ? normalizeHomoglyphs(last).toLowerCase() : void 0;
3805
+ if (tokens.length > 1 && lastFolded && PERSON_TRAILING_NOUNS.has(lastFolded)) continue;
3096
3806
  }
3097
- if ((normalized.label === "person" || normalized.label === "organization") && roles.has(trimmed.toLowerCase())) continue;
3807
+ if ((normalized.label === "person" || normalized.label === "organization") && roles.has(normalizeHomoglyphs(trimmed).toLowerCase())) continue;
3098
3808
  if (normalized.label === "organization" && normalized.source === "legal-form" && trimmed === trimmed.toUpperCase() && LEGAL_FORM_HEADING_RE.test(trimmed)) continue;
3099
- if (normalized.label === "address" && trimmed.length > 40 && !POSTAL_CODE_RE.test(trimmed) && !HAS_DIGIT_RE.test(trimmed) && !ADDRESS_COMPONENTS_RE.test(trimmed) && !JURISDICTION_RE.test(trimmed)) continue;
3100
- if (normalized.label === "address" && normalized.source === "trigger" && !HAS_DIGIT_RE.test(trimmed) && !ADDRESS_COMPONENTS_RE.test(trimmed) && !JURISDICTION_RE.test(trimmed)) continue;
3809
+ if (normalized.label === "address" && trimmed.length > 40 && !POSTAL_CODE_RE.test(trimmed) && !HAS_DIGIT_RE.test(trimmed) && !hasAddressComponent(trimmed) && !JURISDICTION_RE.test(trimmed)) continue;
3810
+ if (normalized.label === "address" && normalized.source === "trigger" && !HAS_DIGIT_RE.test(trimmed) && !hasAddressComponent(trimmed) && !JURISDICTION_RE.test(trimmed)) continue;
3811
+ if (normalized.label === "address" && normalized.source === "trigger" && !HAS_DIGIT_RE.test(trimmed) && isOnlyAmbiguousCours(trimmed)) continue;
3101
3812
  if (normalized.label === "address" && SIGNING_CLAUSE_ADDRESS_RE.test(trimmed)) continue;
3102
3813
  filtered.push(normalized);
3103
3814
  }
@@ -3254,6 +3965,122 @@ const loadPersonStopwords = (ctx) => {
3254
3965
  const EMPTY_PERSON_STOPWORDS = /* @__PURE__ */ new Set();
3255
3966
  /** Sync accessor — returns empty set before init. */
3256
3967
  const getPersonStopwords = (ctx) => ctx.personStopwords ?? EMPTY_PERSON_STOPWORDS;
3968
+ const loadAddressStopwords = (ctx) => {
3969
+ if (ctx.addressStopwordsPromise) return ctx.addressStopwordsPromise;
3970
+ ctx.addressStopwordsPromise = (async () => {
3971
+ try {
3972
+ const mod = await import("./address-stopwords.mjs");
3973
+ const set = new Set(mod.default?.words ?? []);
3974
+ ctx.addressStopwords = set;
3975
+ return set;
3976
+ } catch {
3977
+ const empty = /* @__PURE__ */ new Set();
3978
+ ctx.addressStopwords = empty;
3979
+ return empty;
3980
+ }
3981
+ })();
3982
+ return ctx.addressStopwordsPromise;
3983
+ };
3984
+ const EMPTY_ADDRESS_STOPWORDS = /* @__PURE__ */ new Set();
3985
+ const getAddressStopwords = (ctx) => ctx.addressStopwords ?? EMPTY_ADDRESS_STOPWORDS;
3986
+ /**
3987
+ * Word characters in unicode property notation. The check is
3988
+ * "single-token" — no internal whitespace — and we keep dashes
3989
+ * tokens like "K-12" out by requiring the surface to be a single
3990
+ * uninterrupted run of letters or marks.
3991
+ */
3992
+ const SINGLE_WORD_RE = /^\p{L}+$/u;
3993
+ /**
3994
+ * Format-level address signals — structurally numeric or
3995
+ * 2-letter-state patterns, language-agnostic. The street-type
3996
+ * vocabulary is loaded from `address-street-types.json` so new
3997
+ * languages contribute via data, not code.
3998
+ * - `,\s*[A-Z]{2}\b` → US state abbreviation after a comma
3999
+ * - `\b\d{5}(?:-\d{4})?\b` → US ZIP / ZIP+4
4000
+ * - `\b\d{3}\s\d{2}\b` → Czech/Slovak postal block (140 00)
4001
+ * - `\b\d{2}-\d{3}\b` → Polish postal code (00-950)
4002
+ */
4003
+ const ADDRESS_FORMAT_RE = /,\s*\p{Lu}{2}\b|\b\d{5}(?:-\d{4})?\b|\b\d{3}\s\d{2}\b|\b\d{2}-\d{3}\b/u;
4004
+ let cachedStreetTypeRe = null;
4005
+ let streetTypeReLoaded = false;
4006
+ const loadStreetTypeRe = async () => {
4007
+ if (streetTypeReLoaded) return cachedStreetTypeRe;
4008
+ try {
4009
+ const config = (await import("./address-street-types.mjs")).default ?? {};
4010
+ const words = [];
4011
+ for (const value of Object.values(config)) {
4012
+ if (!Array.isArray(value)) continue;
4013
+ for (const word of value) if (typeof word === "string" && word.length > 0) words.push(word);
4014
+ }
4015
+ if (words.length === 0) cachedStreetTypeRe = null;
4016
+ else {
4017
+ words.sort((a, b) => b.length - a.length);
4018
+ const isLetterDigit = (c) => /[\p{L}\p{N}]/u.test(c);
4019
+ const wordLikeTail = [];
4020
+ const punctTail = [];
4021
+ for (const w of words) {
4022
+ const escaped = w.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
4023
+ if (isLetterDigit(w.at(-1) ?? "")) wordLikeTail.push(escaped);
4024
+ else punctTail.push(escaped);
4025
+ }
4026
+ const branches = [];
4027
+ if (wordLikeTail.length > 0) branches.push(`(?:${wordLikeTail.join("|")})(?![\\p{L}\\p{N}])`);
4028
+ if (punctTail.length > 0) branches.push(`(?:${punctTail.join("|")})`);
4029
+ cachedStreetTypeRe = new RegExp(`(?<![\\p{L}\\p{N}])(?:${branches.join("|")})`, "iu");
4030
+ }
4031
+ } catch {
4032
+ cachedStreetTypeRe = null;
4033
+ }
4034
+ streetTypeReLoaded = true;
4035
+ return cachedStreetTypeRe;
4036
+ };
4037
+ const getStreetTypeRe = () => streetTypeReLoaded ? cachedStreetTypeRe : null;
4038
+ const hasAdjacentAddressEvidence = (fullText, start, end) => {
4039
+ const window = fullText.slice(Math.max(0, start - 40), Math.min(fullText.length, end + 40));
4040
+ if (ADDRESS_FORMAT_RE.test(window)) return true;
4041
+ const streetRe = getStreetTypeRe();
4042
+ return streetRe !== null && streetRe.test(window);
4043
+ };
4044
+ /**
4045
+ * Capitalised words that almost never start a person name. When a
4046
+ * single-token surname candidate is immediately followed by one of
4047
+ * these, the "next-word is uppercase" promotion heuristic would
4048
+ * otherwise turn section headings ("Purchase Price↵The Purchaser
4049
+ * undertakes…") into spurious person hits. Kept narrow on purpose;
4050
+ * the surrounding pipeline still chains real names via the deny-list
4051
+ * cascade when both halves are surnames.
4052
+ */
4053
+ const SENTENCE_STARTER_WORDS = new Set([
4054
+ "The",
4055
+ "This",
4056
+ "These",
4057
+ "Those",
4058
+ "An",
4059
+ "Any",
4060
+ "All",
4061
+ "Each",
4062
+ "Every",
4063
+ "No",
4064
+ "Now",
4065
+ "Whereas",
4066
+ "Whereby",
4067
+ "Wherein",
4068
+ "Whereof",
4069
+ "Notwithstanding",
4070
+ "Subject",
4071
+ "In",
4072
+ "On",
4073
+ "At",
4074
+ "By",
4075
+ "For",
4076
+ "If",
4077
+ "Upon",
4078
+ "Unless",
4079
+ "Until",
4080
+ "Provided",
4081
+ "Pursuant",
4082
+ "Such"
4083
+ ]);
3257
4084
  const PERSON_CHAIN_BREAK_RE = /[!?;:]|,/u;
3258
4085
  const WORD_CHAR_RE$1 = /[\p{L}\p{N}]/u;
3259
4086
  const isInitialContinuationGap = (text, gap) => /^\p{Lu}$/u.test(text) && /^\.[^\S\n]{1,2}$/u.test(gap) || /^[^\S\n]{1,2}(?:\p{Lu}\.[^\S\n]{1,2})+$/u.test(gap);
@@ -3273,6 +4100,8 @@ const buildDenyList = async (config, ctx = defaultContext) => {
3273
4100
  loadStopwords(ctx),
3274
4101
  loadAllowList(ctx),
3275
4102
  loadPersonStopwords(ctx),
4103
+ loadAddressStopwords(ctx),
4104
+ loadStreetTypeRe(),
3276
4105
  loadGenericRoles(ctx)
3277
4106
  ]);
3278
4107
  const dictionaries = config.dictionaries;
@@ -3419,6 +4248,8 @@ const ensureDenyListData = async (ctx = defaultContext, dictionaries) => {
3419
4248
  loadStopwords(ctx),
3420
4249
  loadAllowList(ctx),
3421
4250
  loadPersonStopwords(ctx),
4251
+ loadAddressStopwords(ctx),
4252
+ loadStreetTypeRe(),
3422
4253
  loadGenericRoles(ctx)
3423
4254
  ]);
3424
4255
  };
@@ -3486,14 +4317,18 @@ const processDenyListMatches = (allMatches, sliceStart, sliceEnd, fullText, data
3486
4317
  if (!getPersonStopwords(ctx).has(keyword)) nameHits.push(m);
3487
4318
  }
3488
4319
  const nonPersonLabels = m.labels.filter((l) => l !== "person");
3489
- for (const label of nonPersonLabels) results.push({
3490
- start: m.start,
3491
- end: m.end,
3492
- label,
3493
- text: m.text,
3494
- score: .9,
3495
- source: DETECTION_SOURCES.DENY_LIST
3496
- });
4320
+ const suppressAddress = SINGLE_WORD_RE.test(m.text) && getAddressStopwords(ctx).has(m.text.toLowerCase()) && !hasAdjacentAddressEvidence(fullText, m.start, m.end);
4321
+ for (const label of nonPersonLabels) {
4322
+ if (label === "address" && suppressAddress) continue;
4323
+ results.push({
4324
+ start: m.start,
4325
+ end: m.end,
4326
+ label,
4327
+ text: m.text,
4328
+ score: .9,
4329
+ source: DETECTION_SOURCES.DENY_LIST
4330
+ });
4331
+ }
3497
4332
  }
3498
4333
  }
3499
4334
  nameHits.sort((a, b) => a.start - b.start);
@@ -3525,6 +4360,8 @@ const processDenyListMatches = (allMatches, sliceStart, sliceEnd, fullText, data
3525
4360
  const afterEnd = last.end;
3526
4361
  const rest = fullText.slice(afterEnd).trimStart();
3527
4362
  if (!(rest.length > 1 && /^\p{Lu}\p{Ll}/u.test(rest))) continue;
4363
+ const nextWord = /^\p{L}+/u.exec(rest)?.[0] ?? "";
4364
+ if (SENTENCE_STARTER_WORDS.has(nextWord)) continue;
3528
4365
  }
3529
4366
  results.push({
3530
4367
  start: first.start,
@@ -3636,6 +4473,15 @@ const extendPersonName = (text, start, end, ctx) => {
3636
4473
  };
3637
4474
  //#endregion
3638
4475
  //#region src/detectors/address-seeds.ts
4476
+ /**
4477
+ * Trailing chars that should never end an address span. Combines
4478
+ * structural separators, opening brackets, and typographic quote
4479
+ * variants so a span like `… GA 30326, USA (the "Premises")`
4480
+ * does not pull the parenthetical opener into the address.
4481
+ * The prime (`′`) catches measurement tails (`5′`) that the
4482
+ * cluster occasionally absorbs.
4483
+ */
4484
+ const ADDRESS_TRAILING_TRIM_RE = new RegExp(`[,;:\\s${OPENING_BRACKETS_INNER}${QUOTE_DOUBLE_INNER}${QUOTE_SINGLE_INNER}′]`, "u");
3639
4485
  let cachedBoundaryRe = null;
3640
4486
  const loadBoundaryWords = async () => {
3641
4487
  try {
@@ -3644,6 +4490,56 @@ const loadBoundaryWords = async () => {
3644
4490
  return {};
3645
4491
  }
3646
4492
  };
4493
+ const BR_CEP_CONTEXT_WINDOW = 200;
4494
+ let cachedBrCepContextRe = null;
4495
+ let cachedBrCepContextPromise = null;
4496
+ const loadBrCueWords = async () => {
4497
+ const sources = await Promise.all([(async () => {
4498
+ try {
4499
+ return (await import("./address-street-types.mjs")).default["pt-br"];
4500
+ } catch {
4501
+ return;
4502
+ }
4503
+ })(), (async () => {
4504
+ try {
4505
+ return (await import("./address-boundaries.mjs")).default["pt-br"];
4506
+ } catch {
4507
+ return;
4508
+ }
4509
+ })()]);
4510
+ const out = [];
4511
+ const seen = /* @__PURE__ */ new Set();
4512
+ for (const entry of sources) {
4513
+ if (!Array.isArray(entry)) continue;
4514
+ for (const word of entry) {
4515
+ if (typeof word !== "string" || word.length === 0) continue;
4516
+ const key = word.toLowerCase();
4517
+ if (seen.has(key)) continue;
4518
+ seen.add(key);
4519
+ out.push(word);
4520
+ }
4521
+ }
4522
+ return out;
4523
+ };
4524
+ const getBrCepContextRe = async () => {
4525
+ if (cachedBrCepContextRe !== null) return cachedBrCepContextRe;
4526
+ if (cachedBrCepContextPromise) return cachedBrCepContextPromise;
4527
+ cachedBrCepContextPromise = (async () => {
4528
+ const words = await loadBrCueWords();
4529
+ if (words.length === 0) return null;
4530
+ const escaped = words.toSorted((a, b) => b.length - a.length).map((w) => w.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"));
4531
+ const re = new RegExp(`(?<![\\p{L}\\p{N}])(?:${escaped.join("|")})(?![\\p{L}\\p{N}])`, "iu");
4532
+ cachedBrCepContextRe = re;
4533
+ return re;
4534
+ })();
4535
+ return cachedBrCepContextPromise;
4536
+ };
4537
+ const hasBrCueNearby = (fullText, start, end, re) => {
4538
+ const windowStart = Math.max(0, start - BR_CEP_CONTEXT_WINDOW);
4539
+ const windowEnd = Math.min(fullText.length, end + BR_CEP_CONTEXT_WINDOW);
4540
+ const window = fullText.slice(windowStart, windowEnd);
4541
+ return new RegExp(re.source, re.flags.replace("g", "")).test(window);
4542
+ };
3647
4543
  /**
3648
4544
  * Build regex for boundary words. Matches any
3649
4545
  * boundary word preceded by a word boundary.
@@ -3657,7 +4553,7 @@ const getBoundaryRe = async () => {
3657
4553
  for (const word of entries) words.push(word.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"));
3658
4554
  }
3659
4555
  words.sort((a, b) => b.length - a.length);
3660
- cachedBoundaryRe = words.length > 0 ? new RegExp(`\\b(?:${words.join("|")})\\b`, "i") : /(?!)/;
4556
+ cachedBoundaryRe = words.length > 0 ? new RegExp(`(?<![\\p{L}\\p{N}])(?:${words.join("|")})(?![\\p{L}\\p{N}])`, "iu") : /(?!)/;
3661
4557
  return cachedBoundaryRe;
3662
4558
  };
3663
4559
  /**
@@ -3679,7 +4575,7 @@ const buildStreetTypePatterns = async () => {
3679
4575
  }
3680
4576
  return words;
3681
4577
  };
3682
- const collectSeeds = (allMatches, sliceStart, sliceEnd, fullText, existingEntities) => {
4578
+ const collectSeeds = (allMatches, sliceStart, sliceEnd, fullText, existingEntities, brCepContextRe) => {
3683
4579
  const seeds = [];
3684
4580
  for (const match of allMatches) {
3685
4581
  const idx = match.pattern;
@@ -3713,18 +4609,41 @@ const collectSeeds = (allMatches, sliceStart, sliceEnd, fullText, existingEntiti
3713
4609
  text: e.text
3714
4610
  });
3715
4611
  }
3716
- const postalRe = /\b(?:\d{3}\s\d{2}|\d{2}-\d{3})\b/g;
4612
+ const postalRe = /\b(?:\d{3}\s\d{2}|\d{2}-\d{3}|\d{5}-\d{3})\b/g;
3717
4613
  let postalMatch;
3718
4614
  while ((postalMatch = postalRe.exec(fullText)) !== null) {
3719
4615
  const start = postalMatch.index;
3720
4616
  const end = start + postalMatch[0].length;
3721
- if (!seeds.some((s) => s.start <= start && s.end >= end)) seeds.push({
4617
+ if (seeds.some((s) => s.start <= start && s.end >= end)) continue;
4618
+ if (/^\d{5}-\d{3}$/.test(postalMatch[0]) && (brCepContextRe === null || !hasBrCueNearby(fullText, start, end, brCepContextRe))) continue;
4619
+ seeds.push({
3722
4620
  type: "postal-code",
3723
4621
  start,
3724
4622
  end,
3725
4623
  text: postalMatch[0]
3726
4624
  });
3727
4625
  }
4626
+ const itCapRe = /\b\d{5}(?=\s+\p{Lu}\p{L}+)/gu;
4627
+ let itCapMatch;
4628
+ while ((itCapMatch = itCapRe.exec(fullText)) !== null) {
4629
+ const start = itCapMatch.index;
4630
+ const end = start + itCapMatch[0].length;
4631
+ if (seeds.some((s) => s.start <= start && s.end >= end)) continue;
4632
+ if (!seeds.some((s) => {
4633
+ if (Math.abs(s.start - start) > 80) return false;
4634
+ if (s.type === "address-trigger") return true;
4635
+ if (s.type === "city") return true;
4636
+ if (s.type === "postal-code") return true;
4637
+ if (s.type === "street-word") return s.text.toLowerCase() !== "via";
4638
+ return false;
4639
+ })) continue;
4640
+ seeds.push({
4641
+ type: "postal-code",
4642
+ start,
4643
+ end,
4644
+ text: itCapMatch[0]
4645
+ });
4646
+ }
3728
4647
  const streetNumRe = /\b(\p{Lu}\p{Ll}{2,})\s+(\d{1,5}(?:\/\d{1,5})?)\s*[,\n]/gu;
3729
4648
  let streetMatch;
3730
4649
  while ((streetMatch = streetNumRe.exec(fullText)) !== null) {
@@ -3782,6 +4701,10 @@ const scoreCluster = (cluster) => {
3782
4701
  const NON_ADDRESS_LABELS = new Set([
3783
4702
  "registration number",
3784
4703
  "tax identification number",
4704
+ "national identification number",
4705
+ "social security number",
4706
+ "birth number",
4707
+ "identity card number",
3785
4708
  "person",
3786
4709
  "bank account number",
3787
4710
  "email address",
@@ -3818,7 +4741,7 @@ const expandCluster = async (fullText, cluster, existingEntities) => {
3818
4741
  const doubleNewline = remaining.indexOf("\n\n");
3819
4742
  if (doubleNewline !== -1 && doubleNewline < nearestBoundary) nearestBoundary = doubleNewline;
3820
4743
  rightPos = end + remaining.slice(0, nearestBoundary).trimEnd().length;
3821
- while (rightPos > end && /[,;:\s]/.test(fullText[rightPos - 1] ?? "")) rightPos--;
4744
+ while (rightPos > end && ADDRESS_TRAILING_TRIM_RE.test(fullText[rightPos - 1] ?? "")) rightPos--;
3822
4745
  return {
3823
4746
  start: Math.min(leftPos, start),
3824
4747
  end: Math.max(rightPos, end)
@@ -3835,7 +4758,7 @@ const expandCluster = async (fullText, cluster, existingEntities) => {
3835
4758
  * using their output as seed sources.
3836
4759
  */
3837
4760
  const processAddressSeeds = async (allMatches, sliceStart, sliceEnd, fullText, existingEntities) => {
3838
- const clusters = clusterSeeds(collectSeeds(allMatches, sliceStart, sliceEnd, fullText, existingEntities), 150);
4761
+ const clusters = clusterSeeds(collectSeeds(allMatches, sliceStart, sliceEnd, fullText, existingEntities, await getBrCepContextRe()), 150);
3839
4762
  const results = [];
3840
4763
  for (const cluster of clusters) {
3841
4764
  const score = scoreCluster(cluster);
@@ -3860,6 +4783,7 @@ const processAddressSeeds = async (allMatches, sliceStart, sliceEnd, fullText, e
3860
4783
  const TRAILING_SEP = /[,\s]+$/;
3861
4784
  const WORD_CHAR_RE = /[\p{L}\p{N}]/u;
3862
4785
  const ORG_PROPAGATION_SCORE = .9;
4786
+ const ORG_DETERMINER_RE = /(?<![\p{L}\p{N}])(společnost(?:i|í|em|u)?|spolecnost(?:i|em|u)?|the\s+(?:company|corporation|firm)|die\s+(?:gesellschaft|firma)|la\s+(?:société|empresa|sociedad)|el\s+(?:empresa|sociedad))\s+$/iu;
3863
4787
  const isCallerOwnedEntity$2 = (entity) => entity.sourceDetail === "custom-deny-list" || entity.sourceDetail === "custom-regex";
3864
4788
  /**
3865
4789
  * After the main detection pass, collect organization
@@ -3903,16 +4827,25 @@ const propagateOrgNames = (entities, fullText) => {
3903
4827
  searchFrom = idx + 1;
3904
4828
  continue;
3905
4829
  }
3906
- if (!isOverlapping(idx, matchEnd)) {
4830
+ let spanStart = idx;
4831
+ const lookbackStart = Math.max(0, idx - 40);
4832
+ const lookback = fullText.slice(lookbackStart, idx);
4833
+ const determinerMatch = ORG_DETERMINER_RE.exec(lookback);
4834
+ if (determinerMatch !== null) {
4835
+ const determiner = determinerMatch[1] ?? "";
4836
+ const offsetInMatch = determinerMatch[0].indexOf(determiner);
4837
+ spanStart = lookbackStart + determinerMatch.index + offsetInMatch;
4838
+ }
4839
+ if (!isOverlapping(spanStart, matchEnd)) {
3907
4840
  results.push({
3908
- start: idx,
4841
+ start: spanStart,
3909
4842
  end: matchEnd,
3910
4843
  label,
3911
- text: baseName,
4844
+ text: fullText.slice(spanStart, matchEnd),
3912
4845
  score: ORG_PROPAGATION_SCORE,
3913
4846
  source: DETECTION_SOURCES.COREFERENCE
3914
4847
  });
3915
- covered.push([idx, matchEnd]);
4848
+ covered.push([spanStart, matchEnd]);
3916
4849
  }
3917
4850
  searchFrom = matchEnd;
3918
4851
  }
@@ -5077,13 +6010,89 @@ const shouldReplace = (a, b) => {
5077
6010
  };
5078
6011
  /** Labels where colons are structurally significant. */
5079
6012
  const COLON_LABELS = new Set(["ip address", "mac address"]);
6013
+ /**
6014
+ * Labels whose detectors emit precise, evidence-backed spans. When
6015
+ * one of these fires at the exact same offsets as a fuzzier
6016
+ * `address` hit (city dictionary lookup, address-seed cluster), the
6017
+ * `address` label is almost always a dictionary collision — "March
6018
+ * 15" the date getting tagged as an address because "March" appears
6019
+ * in a city corpus, or "Brent Phillips" emitted as both `person`
6020
+ * and `address` because "Brent" is a UK city. The precise detector
6021
+ * wins.
6022
+ */
6023
+ const PRECISE_OVER_ADDRESS = new Set([
6024
+ "person",
6025
+ "date",
6026
+ "date of birth",
6027
+ "phone number",
6028
+ "email address",
6029
+ "monetary amount",
6030
+ "iban",
6031
+ "bank account number",
6032
+ "tax identification number",
6033
+ "registration number",
6034
+ "identity card number",
6035
+ "national identification number",
6036
+ "passport number",
6037
+ "credit card number"
6038
+ ]);
6039
+ /**
6040
+ * Labels the `person` chain wins against at identical offsets. The
6041
+ * chain carries adjacent-name evidence (deny-list surname plus a
6042
+ * capitalised follow-up) a single-token dictionary collision does
6043
+ * not. Kept narrow: organizations are NOT here — "Morgan Stanley"
6044
+ * legitimately appears in both the org and name dictionaries, and
6045
+ * the existing detector priority is the right tie-breaker there.
6046
+ */
6047
+ const PERSON_PREFERRED_OVER = new Set(["address", "land parcel"]);
6048
+ const resolveSameSpanLabelConflicts = (entities) => {
6049
+ if (entities.length < 2) return entities;
6050
+ const byOffsets = /* @__PURE__ */ new Map();
6051
+ for (const entity of entities) {
6052
+ const key = `${entity.start}:${entity.end}`;
6053
+ const list = byOffsets.get(key);
6054
+ if (list) list.push(entity);
6055
+ else byOffsets.set(key, [entity]);
6056
+ }
6057
+ const dropped = /* @__PURE__ */ new Set();
6058
+ for (const [, group] of byOffsets) {
6059
+ if (group.length < 2) continue;
6060
+ const labels = new Set(group.map((e) => e.label));
6061
+ if (labels.size < 2) continue;
6062
+ const hasPerson = labels.has("person");
6063
+ const hasPreciseNonAddress = [...labels].some((l) => l !== "address" && PRECISE_OVER_ADDRESS.has(l));
6064
+ let maxPriority = -1;
6065
+ for (const e of group) {
6066
+ if (hasLockedBoundary(e)) continue;
6067
+ const pri = DETECTOR_PRIORITY[e.source] ?? 0;
6068
+ if (pri > maxPriority) maxPriority = pri;
6069
+ }
6070
+ for (const e of group) {
6071
+ if (hasLockedBoundary(e)) continue;
6072
+ if ((DETECTOR_PRIORITY[e.source] ?? 0) < maxPriority) {
6073
+ dropped.add(e);
6074
+ continue;
6075
+ }
6076
+ if (hasPerson && PERSON_PREFERRED_OVER.has(e.label)) {
6077
+ dropped.add(e);
6078
+ continue;
6079
+ }
6080
+ if (hasPreciseNonAddress && e.label === "address") dropped.add(e);
6081
+ }
6082
+ }
6083
+ if (dropped.size === 0) return entities;
6084
+ return entities.filter((e) => !dropped.has(e));
6085
+ };
5080
6086
  /** Strip leading/trailing whitespace and punctuation. */
5081
6087
  const sanitizeEntities = (entities) => entities.flatMap((e) => {
5082
6088
  if (hasLockedBoundary(e)) return [e];
5083
6089
  const strip = COLON_LABELS.has(e.label) ? /[\s,;]+/ : /[\s:,;]+/;
5084
6090
  const leadTrimmed = e.text.replace(/^(?:\.\s)+/, "").replace(new RegExp(`^${strip.source}`, strip.flags), "");
5085
6091
  const lead = e.text.length - leadTrimmed.length;
5086
- const cleaned = leadTrimmed.replace(new RegExp(`${strip.source}$`, strip.flags), "");
6092
+ let cleaned = leadTrimmed.replace(new RegExp(`${strip.source}$`, strip.flags), "");
6093
+ if (e.label === "organization" && cleaned.endsWith(".") && !LITERAL_SOURCES.has(e.source)) {
6094
+ if (!getKnownLegalSuffixes().some((suffix) => cleaned.endsWith(suffix))) cleaned = cleaned.slice(0, -1).trimEnd();
6095
+ }
5087
6096
  if (cleaned.length === 0) return [];
5088
6097
  if (!/[\p{L}\p{N}]/u.test(cleaned)) return [];
5089
6098
  const collapsed = cleaned.replace(/\s*\n\s*/g, " ").replace(/\s{2,}/g, " ");
@@ -5132,7 +6141,7 @@ const mergeAndDedup = (...layers) => {
5132
6141
  }
5133
6142
  if (overlaps.every((existing) => shouldReplace(entity, existing))) merged.splice(overlapStart, overlaps.length, { ...entity });
5134
6143
  }
5135
- return sanitizeEntities(merged);
6144
+ return resolveSameSpanLabelConflicts(sanitizeEntities(merged));
5136
6145
  };
5137
6146
  const escapeRegex = (s) => s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
5138
6147
  let amountWordsRe = null;
@@ -5168,9 +6177,10 @@ const filterAllowedLabels = (entities, allowedLabels) => {
5168
6177
  return entities.filter((e) => allowedLabels.has(e.label));
5169
6178
  };
5170
6179
  const labelIsAllowed = (label, allowedLabels) => !allowedLabels || allowedLabels.has(label);
6180
+ const NON_NER_LABELS = new Set(["misc"]);
5171
6181
  const getRequestedNerLabels = (config, expandForHotwords = false) => {
5172
6182
  const labels = config.labels.length > 0 ? config.labels : DEFAULT_ENTITY_LABELS;
5173
- return expandForHotwords ? expandLabelsForHotwordRules(labels) : labels;
6183
+ return (expandForHotwords ? expandLabelsForHotwordRules(labels) : labels).filter((label) => !NON_NER_LABELS.has(label));
5174
6184
  };
5175
6185
  const checkAbort = (signal) => {
5176
6186
  if (signal?.aborted) throw new DOMException("Pipeline aborted", "AbortError");
@@ -5251,6 +6261,8 @@ const runPipeline = async (options) => {
5251
6261
  loadGenericRoles(ctx),
5252
6262
  initPrepositions(),
5253
6263
  initStreetAbbrevs(),
6264
+ initAddressComponents(),
6265
+ warmAddressStopKeywords(),
5254
6266
  zoneInit,
5255
6267
  hotwordInit
5256
6268
  ]);
@@ -5258,6 +6270,8 @@ const runPipeline = async (options) => {
5258
6270
  loadGenericRoles(ctx),
5259
6271
  initPrepositions(),
5260
6272
  initStreetAbbrevs(),
6273
+ initAddressComponents(),
6274
+ warmAddressStopKeywords(),
5261
6275
  hotwordInit
5262
6276
  ]);
5263
6277
  if (cachedSearch && config.enableDenyList) await ensureDenyListData(ctx, config.dictionaries);
@@ -5277,9 +6291,11 @@ const runPipeline = async (options) => {
5277
6291
  const customRegexEntities = filterAllowedLabels(config.enableRegex ? processRegexMatches(customRegexMatches, slices.customRegex.start, slices.customRegex.end, search.customRegexMeta) : [], preHotwordAllowedLabels);
5278
6292
  const regexEntities = filterAllowedLabels([...rawRegexEntities, ...customRegexEntities], preHotwordAllowedLabels);
5279
6293
  if (regexEntities.length > 0) log("regex", `${regexEntities.length} matches`);
6294
+ if (legalFormsEnabled || config.enableTriggerPhrases) await warmLegalRoleHeads();
5280
6295
  const rawLegalFormEntities = legalFormsEnabled ? processLegalFormMatches(regexMatches, slices.legalForms.start, slices.legalForms.end, fullText) : [];
5281
6296
  const legalFormEntities = filterAllowedLabels(rawLegalFormEntities, preHotwordAllowedLabels);
5282
6297
  if (legalFormEntities.length > 0) log("legal-forms", `${legalFormEntities.length} matches`);
6298
+ if (config.enableTriggerPhrases) await warmAddressStopKeywords();
5283
6299
  const rawTriggerEntities = config.enableTriggerPhrases ? processTriggerMatches(regexMatches, slices.triggers.start, slices.triggers.end, fullText, search.triggerRules) : [];
5284
6300
  const triggerEntities = filterAllowedLabels(rawTriggerEntities, preHotwordAllowedLabels);
5285
6301
  if (triggerEntities.length > 0) log("trigger-phrases", `${triggerEntities.length} matches`);
@@ -5325,10 +6341,11 @@ const runPipeline = async (options) => {
5325
6341
  ];
5326
6342
  let rawNerEntities = [];
5327
6343
  let nerEntities = [];
5328
- if (config.enableNer && nerInference) {
6344
+ const requestedNerLabels = getRequestedNerLabels(config, hotwordsActive);
6345
+ if (config.enableNer && nerInference && requestedNerLabels.length > 0) {
5329
6346
  const maskResult = maskDetectedSpans(fullText, nerMaskEntities);
5330
6347
  log("ner", "running inference...");
5331
- const rawNer = await nerInference(maskResult.maskedText, [...getRequestedNerLabels(config, hotwordsActive)], config.threshold, signal);
6348
+ const rawNer = await nerInference(maskResult.maskedText, [...requestedNerLabels], config.threshold, signal);
5332
6349
  rawNerEntities = unmaskNerEntities(rawNer, maskResult, fullText);
5333
6350
  nerEntities = filterAllowedLabels(rawNerEntities, preHotwordAllowedLabels);
5334
6351
  const masked = rawNer.length - rawNerEntities.length;
@@ -5389,6 +6406,7 @@ const runPipeline = async (options) => {
5389
6406
  checkAbort(signal);
5390
6407
  ctx.corefSourceMap.clear();
5391
6408
  if (config.enableCoreference) {
6409
+ if (!legalFormsEnabled) await warmLegalRoleHeads();
5392
6410
  const terms = await extractDefinedTerms(fullText, merged.filter((entity) => !isCallerOwnedEntity(entity)), ctx);
5393
6411
  if (terms.length > 0) {
5394
6412
  log("coreference", `${terms.length} defined terms`);
@@ -5429,7 +6447,7 @@ const resolveOperator = (config, label) => config.operators[label] ?? "replace";
5429
6447
  //#region src/redact.ts
5430
6448
  const WHITESPACE_RE = /\s+/g;
5431
6449
  const PHONE_NOISE_RE = /[()\s-]/g;
5432
- const SPACE_DASH_RE = /[\s-]/g;
6450
+ const ID_SEPARATOR_RE = /[\s\-/.]/g;
5433
6451
  /**
5434
6452
  * Normalize entity text so that surface-form variations
5435
6453
  * of the same real-world value map to a single canonical
@@ -5439,8 +6457,8 @@ const normalizeEntityText = (label, text) => {
5439
6457
  const upper = label.toUpperCase().replace(WHITESPACE_RE, "_");
5440
6458
  if (upper === "EMAIL_ADDRESS" || upper === "EMAIL") return text.toLowerCase().trim();
5441
6459
  if (upper === "PHONE_NUMBER" || upper === "PHONE") return text.replace(PHONE_NOISE_RE, "");
5442
- if (upper === "IBAN" || upper === "BANK_ACCOUNT_NUMBER" || upper === "TAX_IDENTIFICATION_NUMBER" || upper === "REGISTRATION_NUMBER") return text.replace(SPACE_DASH_RE, "").toUpperCase();
5443
- if (upper === "PERSON" || upper === "ORGANIZATION" || upper === "ADDRESS" || upper === "LAND_PARCEL") return text.replace(WHITESPACE_RE, " ").toLowerCase().trim();
6460
+ if (upper === "IBAN" || upper === "BANK_ACCOUNT_NUMBER" || upper === "TAX_IDENTIFICATION_NUMBER" || upper === "REGISTRATION_NUMBER" || upper === "NATIONAL_IDENTIFICATION_NUMBER" || upper === "SOCIAL_SECURITY_NUMBER" || upper === "BIRTH_NUMBER" || upper === "IDENTITY_CARD_NUMBER" || upper === "PASSPORT_NUMBER" || upper === "CREDIT_CARD_NUMBER") return text.replace(ID_SEPARATOR_RE, "").toUpperCase();
6461
+ if (upper === "PERSON" || upper === "ORGANIZATION" || upper === "ADDRESS" || upper === "LAND_PARCEL" || upper === "MISC") return text.replace(WHITESPACE_RE, " ").toLowerCase().trim();
5444
6462
  return text.trim();
5445
6463
  };
5446
6464
  /**
@@ -5984,6 +7002,6 @@ const levenshtein = (rawA, rawB) => {
5984
7002
  //#region src/wasm.ts
5985
7003
  initTextSearch(TextSearch);
5986
7004
  //#endregion
5987
- export { CURRENCY_PATTERN_META, DATE_PATTERN_META, DEFAULT_ENTITY_LABELS, DEFAULT_OPERATOR_CONFIG, DETECTION_SOURCES, DETECTOR_PRIORITY, OPERATOR_REGISTRY, OPERATOR_TYPES, REGEX_META, REGEX_PATTERNS, REGIONS, ZONE_SCORE_ADJUSTMENTS, applyHotwordRules, applyZoneAdjustments, boostNearMissEntities, buildDenyList, buildGazetteerPatterns, buildLegalFormPatterns, buildPlaceholderMap, buildStreetTypePatterns, buildTriggerPatterns, buildUnifiedSearch, chunkText, classifyZones, computeChunkOffsets, corefKey, createPipelineContext, deanonymise, decodeSpans, decodeTokenSpans, detectNameCorpus, ensureDenyListData, exportRedactionKey, extractDefinedTerms, filterFalsePositives, findCoreferenceSpans, getCurrencyPatterns, getDatePatterns, initHotwordRules, initNameCorpus, initZoneClassifier, levenshtein, mergeAndDedup, mergeChunkEntities, normalizeForSearch, prepareBatch, processAddressSeeds, processDenyListMatches, processGazetteerMatches, processLegalFormMatches, processRegexMatches, processTriggerMatches, propagateOrgNames, redactText, resolveCountries, resolveOperator, runPipeline, runUnifiedSearch, sanitizeEntities, tokenizeText };
7005
+ export { CURRENCY_PATTERN_META, DATE_PATTERN_META, DEFAULT_ENTITY_LABELS, DEFAULT_OPERATOR_CONFIG, DETECTION_SOURCES, DETECTOR_PRIORITY, OPERATOR_REGISTRY, OPERATOR_TYPES, REGEX_META, REGEX_PATTERNS, REGIONS, ZONE_SCORE_ADJUSTMENTS, applyHotwordRules, applyZoneAdjustments, boostNearMissEntities, buildDenyList, buildGazetteerPatterns, buildLegalFormPatterns, buildPlaceholderMap, buildStreetTypePatterns, buildTriggerPatterns, buildUnifiedSearch, chunkText, classifyZones, computeChunkOffsets, corefKey, createPipelineContext, deanonymise, decodeSpans, decodeTokenSpans, detectNameCorpus, ensureDenyListData, exportRedactionKey, extractDefinedTerms, filterFalsePositives, findCoreferenceSpans, getCurrencyPatterns, getDatePatterns, initAddressComponents, initHotwordRules, initNameCorpus, initZoneClassifier, levenshtein, mergeAndDedup, mergeChunkEntities, normalizeForSearch, prepareBatch, processAddressSeeds, processDenyListMatches, processGazetteerMatches, processLegalFormMatches, processRegexMatches, processTriggerMatches, propagateOrgNames, redactText, resolveCountries, resolveOperator, runPipeline, runUnifiedSearch, sanitizeEntities, tokenizeText, warmLegalRoleHeads };
5988
7006
 
5989
7007
  //# sourceMappingURL=wasm.mjs.map