@stll/anonymize-wasm 1.2.0 → 1.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (64) hide show
  1. package/dist/address-boundaries.mjs +113 -5
  2. package/dist/address-stop-keywords.mjs +137 -0
  3. package/dist/address-stop-keywords.mjs.map +1 -0
  4. package/dist/address-stopwords.mjs +81 -0
  5. package/dist/address-stopwords.mjs.map +1 -0
  6. package/dist/address-street-types.mjs +67 -4
  7. package/dist/clause-noun-heads.mjs +75 -0
  8. package/dist/clause-noun-heads.mjs.map +1 -0
  9. package/dist/constants.d.mts +2 -0
  10. package/dist/constants.mjs +85 -0
  11. package/dist/constants.mjs.map +1 -0
  12. package/dist/constants2.d.mts +65 -0
  13. package/dist/coreference.es.mjs +22 -0
  14. package/dist/coreference.es.mjs.map +1 -0
  15. package/dist/coreference.fr.mjs +32 -0
  16. package/dist/coreference.fr.mjs.map +1 -0
  17. package/dist/coreference.it.mjs +27 -0
  18. package/dist/coreference.it.mjs.map +1 -0
  19. package/dist/coreference.pl.mjs +27 -0
  20. package/dist/coreference.pl.mjs.map +1 -0
  21. package/dist/coreference.pt-br.mjs +14 -0
  22. package/dist/coreference.pt-br.mjs.map +1 -0
  23. package/dist/coreference.sk.mjs +22 -5
  24. package/dist/generic-roles.mjs +179 -1
  25. package/dist/hotword-rules.mjs +7 -3
  26. package/dist/legal-forms.mjs +3 -0
  27. package/dist/legal-role-heads.cs.mjs +42 -0
  28. package/dist/legal-role-heads.cs.mjs.map +1 -0
  29. package/dist/legal-role-heads.de.mjs +33 -0
  30. package/dist/legal-role-heads.de.mjs.map +1 -0
  31. package/dist/legal-role-heads.en.mjs +37 -0
  32. package/dist/legal-role-heads.en.mjs.map +1 -0
  33. package/dist/legal-role-heads.es.mjs +54 -0
  34. package/dist/legal-role-heads.es.mjs.map +1 -0
  35. package/dist/legal-role-heads.fr.mjs +72 -0
  36. package/dist/legal-role-heads.fr.mjs.map +1 -0
  37. package/dist/legal-role-heads.it.mjs +68 -0
  38. package/dist/legal-role-heads.it.mjs.map +1 -0
  39. package/dist/legal-role-heads.pl.mjs +84 -0
  40. package/dist/legal-role-heads.pl.mjs.map +1 -0
  41. package/dist/legal-role-heads.pt-br.mjs +63 -0
  42. package/dist/legal-role-heads.pt-br.mjs.map +1 -0
  43. package/dist/legal-role-heads.sk.mjs +80 -0
  44. package/dist/legal-role-heads.sk.mjs.map +1 -0
  45. package/dist/manifest.mjs +21 -8
  46. package/dist/names-exclusions.mjs +21 -1
  47. package/dist/person-stopwords.mjs +121 -1
  48. package/dist/section-headings.mjs +15 -3
  49. package/dist/sentence-verb-indicators.mjs +223 -0
  50. package/dist/sentence-verb-indicators.mjs.map +1 -0
  51. package/dist/triggers.cs.mjs +107 -3
  52. package/dist/triggers.en.mjs +32 -0
  53. package/dist/triggers.es.mjs +43 -4
  54. package/dist/triggers.fr.mjs +206 -7
  55. package/dist/triggers.it.mjs +26 -4
  56. package/dist/triggers.pl.mjs +191 -4
  57. package/dist/triggers.pt-br.mjs +193 -0
  58. package/dist/triggers.pt-br.mjs.map +1 -0
  59. package/dist/triggers.sk.mjs +493 -0
  60. package/dist/vite.mjs.map +1 -1
  61. package/dist/wasm.d.mts +56 -39
  62. package/dist/wasm.mjs +2382 -1408
  63. package/dist/wasm.mjs.map +1 -1
  64. package/package.json +7 -2
package/dist/wasm.mjs CHANGED
@@ -1,5 +1,6 @@
1
+ import { DEFAULT_ENTITY_LABELS, DETECTION_SOURCES, DETECTOR_PRIORITY, OPERATOR_TYPES } from "./constants.mjs";
1
2
  import { TextSearch } from "@stll/text-search-wasm";
2
- import { at, be, bg, cy, cz, de, dk, ee, es, fi, fr, gb, gr, hr, hu, ie, it, lt, lu, lv, mt, nl, pl, pt, ro, se, si, sk } from "@stll/stdnum";
3
+ import { at, be, bg, br, cy, cz, de, dk, ee, es, fi, fr, gb, gr, hr, hu, ie, it, lt, lu, lv, mt, nl, pl, pt, ro, se, si, sk } from "@stll/stdnum";
3
4
  import { toRegex } from "@stll/stdnum/patterns";
4
5
  //#region src/search-engine.ts
5
6
  let _TextSearch;
@@ -12,66 +13,6 @@ const getTextSearch = () => {
12
13
  };
13
14
  //#endregion
14
15
  //#region src/types.ts
15
- /**
16
- * Source of a detected entity span.
17
- * Ordered by detection layer in the pipeline.
18
- */
19
- const DETECTION_SOURCES = {
20
- TRIGGER: "trigger",
21
- REGEX: "regex",
22
- DENY_LIST: "deny-list",
23
- LEGAL_FORM: "legal-form",
24
- GAZETTEER: "gazetteer",
25
- NER: "ner",
26
- COREFERENCE: "coreference"
27
- };
28
- /**
29
- * Priority levels for detection sources.
30
- * Higher = more structurally reliable. Used during
31
- * overlap resolution so deterministic detectors beat
32
- * probabilistic ones regardless of raw score.
33
- */
34
- const DETECTOR_PRIORITY = {
35
- gazetteer: 5,
36
- trigger: 4,
37
- "legal-form": 3,
38
- regex: 3,
39
- "deny-list": 2,
40
- coreference: 2,
41
- ner: 1
42
- };
43
- /**
44
- * Anonymisation operator types. Each operator defines
45
- * how a confirmed entity is replaced in the output.
46
- */
47
- const OPERATOR_TYPES = ["replace", "redact"];
48
- /**
49
- * Canonical entity labels used across the pipeline.
50
- * NER models may use different native labels; the bench
51
- * NER wrapper maps model output to these canonical names.
52
- *
53
- * These labels are ephemeral: entities are regenerated on
54
- * every pipeline run and never persisted to the database.
55
- * Renaming a label here requires no migration.
56
- */
57
- const DEFAULT_ENTITY_LABELS = [
58
- "person",
59
- "organization",
60
- "phone number",
61
- "address",
62
- "email address",
63
- "date",
64
- "date of birth",
65
- "bank account number",
66
- "iban",
67
- "tax identification number",
68
- "identity card number",
69
- "registration number",
70
- "credit card number",
71
- "passport number",
72
- "monetary amount",
73
- "land parcel"
74
- ];
75
16
  const isLegalFormsEnabled = (config) => config.enableLegalForms !== false;
76
17
  //#endregion
77
18
  //#region src/context.ts
@@ -95,6 +36,8 @@ const createPipelineContext = () => ({
95
36
  allowListPromise: null,
96
37
  personStopwords: null,
97
38
  personStopwordsPromise: null,
39
+ addressStopwords: null,
40
+ addressStopwordsPromise: null,
98
41
  firstNameExclusions: null,
99
42
  firstNameExclusionCorpusLen: 0,
100
43
  genericRoles: null,
@@ -151,6 +94,7 @@ const LOADER_REGISTRIES = {
151
94
  hu: () => import("./triggers.hu.mjs"),
152
95
  it: () => import("./triggers.it.mjs"),
153
96
  pl: () => import("./triggers.pl.mjs"),
97
+ "pt-br": () => import("./triggers.pt-br.mjs"),
154
98
  ro: () => import("./triggers.ro.mjs"),
155
99
  sk: () => import("./triggers.sk.mjs"),
156
100
  sv: () => import("./triggers.sv.mjs")
@@ -159,7 +103,23 @@ const LOADER_REGISTRIES = {
159
103
  cs: () => import("./coreference.cs.mjs"),
160
104
  de: () => import("./coreference.de.mjs"),
161
105
  en: () => import("./coreference.en.mjs"),
106
+ es: () => import("./coreference.es.mjs"),
107
+ fr: () => import("./coreference.fr.mjs"),
108
+ it: () => import("./coreference.it.mjs"),
109
+ pl: () => import("./coreference.pl.mjs"),
110
+ "pt-br": () => import("./coreference.pt-br.mjs"),
162
111
  sk: () => import("./coreference.sk.mjs")
112
+ },
113
+ legalRoleHeads: {
114
+ cs: () => import("./legal-role-heads.cs.mjs"),
115
+ de: () => import("./legal-role-heads.de.mjs"),
116
+ en: () => import("./legal-role-heads.en.mjs"),
117
+ es: () => import("./legal-role-heads.es.mjs"),
118
+ fr: () => import("./legal-role-heads.fr.mjs"),
119
+ it: () => import("./legal-role-heads.it.mjs"),
120
+ pl: () => import("./legal-role-heads.pl.mjs"),
121
+ "pt-br": () => import("./legal-role-heads.pt-br.mjs"),
122
+ sk: () => import("./legal-role-heads.sk.mjs")
163
123
  }
164
124
  };
165
125
  const FALLBACK_LANGUAGES = {
@@ -172,6 +132,7 @@ const FALLBACK_LANGUAGES = {
172
132
  "hu",
173
133
  "it",
174
134
  "pl",
135
+ "pt-br",
175
136
  "ro",
176
137
  "sk",
177
138
  "sv"
@@ -180,6 +141,22 @@ const FALLBACK_LANGUAGES = {
180
141
  "cs",
181
142
  "de",
182
143
  "en",
144
+ "es",
145
+ "fr",
146
+ "it",
147
+ "pl",
148
+ "pt-br",
149
+ "sk"
150
+ ],
151
+ legalRoleHeads: [
152
+ "cs",
153
+ "de",
154
+ "en",
155
+ "es",
156
+ "fr",
157
+ "it",
158
+ "pl",
159
+ "pt-br",
183
160
  "sk"
184
161
  ]
185
162
  };
@@ -226,838 +203,64 @@ const loadLanguageConfigs = async (configType, mapFn) => {
226
203
  await Promise.all(loads);
227
204
  return results.filter((r) => r !== void 0);
228
205
  };
229
- //#endregion
230
- //#region src/detectors/coreference.ts
231
- /**
232
- * Load coreference definition patterns from per-language
233
- * JSON configs in src/data/. Uses the
234
- * language manifest for auto-discovery.
235
- */
236
- const loadDefinitionPatterns = async () => {
237
- const patterns = [];
238
- const allRows = await loadLanguageConfigs("coreference", (mod) => {
239
- return mod.default ?? mod;
240
- });
241
- for (const rows of allRows) {
242
- if (!Array.isArray(rows)) {
243
- console.warn("[anonymize] coreference: unexpected config shape, skipping");
244
- continue;
245
- }
246
- for (const row of rows) try {
247
- patterns.push({ pattern: new RegExp(row.pattern, row.flags) });
248
- } catch (err) {
249
- console.warn(`[anonymize] coreference: invalid regex "${row.pattern}":`, err);
250
- }
251
- }
252
- return patterns;
253
- };
254
- /**
255
- * Load generic role terms that should NOT be treated
256
- * as PII coreferences. "Prodávající" (Seller),
257
- * "Kupující" (Buyer), etc. are legal roles, not
258
- * identifying information.
259
- */
260
- const getRoleStopSet = async (ctx) => {
261
- if (ctx.roleStopSet) return ctx.roleStopSet;
262
- if (ctx.roleStopSetPromise) return ctx.roleStopSetPromise;
263
- const promise = (async () => {
264
- let result;
265
- try {
266
- const mod = await import("./generic-roles.mjs");
267
- const data = mod.default ?? mod;
268
- result = new Set(data.roles.map((r) => r.toLowerCase()));
269
- } catch {
270
- result = /* @__PURE__ */ new Set();
271
- }
272
- ctx.roleStopSet = result;
273
- return result;
274
- })();
275
- ctx.roleStopSetPromise = promise;
276
- return promise;
277
- };
278
- const getDefinitionPatterns = async (ctx) => {
279
- if (ctx.corefPatterns) return ctx.corefPatterns;
280
- if (ctx.corefPatternsPromise) return ctx.corefPatternsPromise;
281
- ctx.corefPatternsPromise = loadDefinitionPatterns();
282
- const patterns = await ctx.corefPatternsPromise;
283
- if (patterns.length === 0) {
284
- ctx.corefPatterns = patterns;
285
- if (!ctx.corefLoadAttempted) {
286
- ctx.corefLoadAttempted = true;
287
- console.warn("[anonymize] coreference: no definition patterns loaded; coreference detection will be inactive");
288
- }
289
- return patterns;
290
- }
291
- ctx.corefPatterns = patterns;
292
- return patterns;
293
- };
294
- const SEARCH_WINDOW = 200;
295
- /**
296
- * Check whether an alias has textual similarity to
297
- * the source entity. Prevents roles and structural
298
- * terms from being treated as name aliases.
299
- *
300
- * Three checks (any passes → similar):
301
- * 1. Word overlap: a word in the alias appears in the
302
- * entity (case-insensitive, min 2 chars)
303
- * 2. Initials: alias letters match first letters of
304
- * entity words ("TB" ↔ "Tomas Bata")
305
- * 3. Substring: alias is a substring of the entity
306
- * or vice versa (min 3 chars)
307
- */
308
- const hasEntitySimilarity = (alias, entityText) => {
309
- const aliasLower = alias.toLowerCase();
310
- const entityLower = entityText.toLowerCase();
311
- if (aliasLower.length >= 3 && entityLower.includes(aliasLower)) return true;
312
- if (entityLower.length >= 3 && aliasLower.includes(entityLower)) return true;
313
- const aliasWords = aliasLower.split(/[\s.,;:'"()/-]+/).filter((w) => w.length >= 2);
314
- const entityWords = entityLower.split(/[\s.,;:'"()/-]+/).filter((w) => w.length >= 2);
315
- const entityWordSet = new Set(entityWords);
316
- for (const word of aliasWords) if (entityWordSet.has(word)) return true;
317
- if (/^[\p{Lu}]+$/u.test(alias) && alias.length >= 2 && alias.length <= entityWords.length) {
318
- for (let start = 0; start <= entityWords.length - alias.length; start++) if (entityWords.slice(start, start + alias.length).map((w) => w.charAt(0)).join("") === aliasLower) return true;
319
- }
320
- return false;
321
- };
322
- /**
323
- * Scan for defined-term patterns near known entities.
324
- *
325
- * Legal documents universally follow:
326
- * "Dr. Heinrich Muller (hereinafter 'the Seller')..."
327
- *
328
- * After NER detects the entity, this function scans for
329
- * definitional patterns within +/-200 chars and extracts
330
- * the alias. Returns alias + label pairs that can be added
331
- * to the gazetteer for a full-text re-scan.
332
- */
333
- /**
334
- * Labels that can be the source of a coreference alias.
335
- * Only parties (person, organization) have defined-term
336
- * aliases in legal text. Dates, addresses, IDs do not.
337
- */
338
- const COREF_SOURCE_LABELS = new Set(["person", "organization"]);
339
- const extractDefinedTerms = async (fullText, entities, ctx = defaultContext) => {
340
- const [definitionPatterns, roleStops] = await Promise.all([getDefinitionPatterns(ctx), getRoleStopSet(ctx)]);
341
- const terms = [];
342
- const seen = /* @__PURE__ */ new Set();
343
- const sorted = [...entities].sort((a, b) => a.start - b.start);
344
- for (const { pattern } of definitionPatterns) {
345
- pattern.lastIndex = 0;
346
- for (let match = pattern.exec(fullText); match !== null; match = pattern.exec(fullText)) {
347
- const alias = match[1]?.trim();
348
- if (!alias || alias.length < 2) continue;
349
- if (roleStops.has(alias.toLowerCase())) continue;
350
- const defPos = match.index;
351
- let bestEntity = null;
352
- for (let i = sorted.length - 1; i >= 0; i--) {
353
- const e = sorted[i];
354
- if (e === void 0) continue;
355
- if (e.end > defPos) continue;
356
- if (defPos - e.end > SEARCH_WINDOW) break;
357
- if (!COREF_SOURCE_LABELS.has(e.label)) continue;
358
- bestEntity = e;
359
- break;
360
- }
361
- if (bestEntity === null) continue;
362
- const gapText = fullText.slice(bestEntity.end, defPos);
363
- if (/(?:;|\.(?=\s*(?:["'„‚(]*\p{Lu}|$)))/u.test(gapText)) continue;
364
- if (!hasEntitySimilarity(alias, bestEntity.text)) continue;
365
- const key = `${alias.toLowerCase()}::${bestEntity.label}`;
366
- if (seen.has(key)) continue;
367
- seen.add(key);
368
- terms.push({
369
- alias,
370
- label: bestEntity.label,
371
- definitionStart: defPos,
372
- sourceText: bestEntity.text
373
- });
374
- }
375
- }
376
- return terms;
377
- };
378
- /**
379
- * Check if a character is a Unicode word character
380
- * (letter, digit, or combining mark). Used for word
381
- * boundary checks in coreference matching.
382
- */
383
- const isWordChar = (ch) => {
384
- if (ch === void 0) return false;
385
- return /[\p{L}\p{M}\p{N}]/u.test(ch);
386
- };
387
- /**
388
- * Find all occurrences of defined-term aliases in the
389
- * full text. Returns Entity spans for each match.
390
- *
391
- * Respects word boundaries: "Kupující" must not match
392
- * inside "Kupujícímu". A match is valid only if the
393
- * character before the start and after the end are NOT
394
- * word characters (letter/digit).
395
- *
396
- * Populates `ctx.corefSourceMap` with entries linking
397
- * each coref entity to its source entity text, for
398
- * consistent placeholder numbering.
399
- */
400
- const findCoreferenceSpans = (fullText, terms, ctx = defaultContext) => {
401
- const results = [];
402
- for (const term of terms) {
403
- let searchFrom = 0;
404
- while (searchFrom < fullText.length) {
405
- const idx = fullText.indexOf(term.alias, searchFrom);
406
- if (idx === -1) break;
407
- const matchEnd = idx + term.alias.length;
408
- const charBefore = idx > 0 ? fullText[idx - 1] : void 0;
409
- const charAfter = fullText[matchEnd];
410
- if (isWordChar(charBefore) || isWordChar(charAfter)) {
411
- searchFrom = idx + 1;
412
- continue;
413
- }
414
- const entity = {
415
- start: idx,
416
- end: matchEnd,
417
- label: term.label,
418
- text: term.alias,
419
- score: .95,
420
- source: DETECTION_SOURCES.COREFERENCE
421
- };
422
- ctx.corefSourceMap.set(corefKey(entity), term.sourceText);
423
- results.push(entity);
424
- searchFrom = matchEnd;
425
- }
426
- }
427
- return results;
428
- };
429
- //#endregion
430
- //#region src/detectors/gazetteer.ts
431
- const MAX_EDIT_DISTANCE = 2;
432
- const MIN_FUZZY_LENGTH = 4;
433
- const MAX_PREFIX_OVERSHOOT = 7;
434
- /**
435
- * Collect all searchable strings (canonical + variants)
436
- * from gazetteer entries, mapped to their labels and
437
- * entry IDs.
438
- */
439
- const buildSearchTerms = (entries) => {
440
- const terms = /* @__PURE__ */ new Map();
441
- for (const entry of entries) {
442
- const meta = {
443
- label: entry.label,
444
- entryId: entry.id
445
- };
446
- terms.set(entry.canonical, meta);
447
- for (const variant of entry.variants) terms.set(variant, meta);
448
- }
449
- return terms;
450
- };
451
- /**
452
- * Build TextSearch-compatible patterns from gazetteer
453
- * entries. Returns:
454
- * - Exact literal patterns for all terms
455
- * - Fuzzy patterns (distance: 2) for terms >= 4 chars
456
- * - Parallel metadata arrays for post-processing
457
- *
458
- * Patterns are ordered: all exact first, then all
459
- * fuzzy. The isFuzzy array marks which are which.
460
- */
461
- const buildGazetteerPatterns = (entries) => {
462
- const terms = buildSearchTerms(entries);
463
- const patterns = [];
464
- const labels = [];
465
- const isFuzzy = [];
466
- for (const [term, meta] of terms) {
467
- patterns.push({
468
- pattern: term,
469
- literal: true,
470
- wholeWords: false
471
- });
472
- labels.push(meta.label);
473
- isFuzzy.push(false);
474
- }
475
- for (const [term, meta] of terms) {
476
- if (term.length < MIN_FUZZY_LENGTH) continue;
477
- patterns.push({
478
- pattern: term,
479
- distance: MAX_EDIT_DISTANCE
480
- });
481
- labels.push(meta.label);
482
- isFuzzy.push(true);
483
- }
484
- return {
485
- patterns,
486
- data: {
487
- labels,
488
- isFuzzy
489
- }
490
- };
491
- };
492
- /**
493
- * Process gazetteer matches from the unified literal
494
- * search. Receives all matches; filters to the
495
- * gazetteer slice via sliceStart/sliceEnd.
496
- *
497
- * Exact matches get score 0.9; fuzzy matches get
498
- * 0.85. Fuzzy matches that overlap an exact match
499
- * are dropped.
500
- *
501
- * For exact matches, attempts prefix extension for
502
- * legal suffixes ("a.s.", "GmbH", "s.r.o." after
503
- * the matched term).
504
- */
505
- const processGazetteerMatches = (allMatches, sliceStart, sliceEnd, fullText, data) => {
506
- const results = [];
507
- const exactSpans = [];
508
- for (const match of allMatches) {
509
- const idx = match.pattern;
510
- if (idx < sliceStart || idx >= sliceEnd) continue;
511
- const localIdx = idx - sliceStart;
512
- if (data.isFuzzy[localIdx]) continue;
513
- const label = data.labels[localIdx];
514
- if (!label) continue;
515
- const extended = tryPrefixExtension(fullText, match.start, match.end);
516
- const end = extended?.end ?? match.end;
517
- const text = extended?.text ?? fullText.slice(match.start, match.end);
518
- exactSpans.push({
519
- start: match.start,
520
- end
521
- });
522
- results.push({
523
- start: match.start,
524
- end,
525
- label,
526
- text,
527
- score: .9,
528
- source: DETECTION_SOURCES.GAZETTEER
529
- });
530
- }
531
- for (const match of allMatches) {
532
- const idx = match.pattern;
533
- if (idx < sliceStart || idx >= sliceEnd) continue;
534
- const localIdx = idx - sliceStart;
535
- if (!data.isFuzzy[localIdx]) continue;
536
- if (match.distance === 0) continue;
537
- const label = data.labels[localIdx];
538
- if (!label) continue;
539
- if (exactSpans.some((e) => match.start < e.end && match.end > e.start)) continue;
540
- const matchText = fullText.slice(match.start, match.end);
541
- results.push({
542
- start: match.start,
543
- end: match.end,
544
- label,
545
- text: matchText,
546
- score: .85,
547
- source: DETECTION_SOURCES.GAZETTEER
548
- });
549
- }
550
- return results;
551
- };
552
- /**
553
- * Try to extend an exact match to capture one
554
- * trailing token (max 6 chars) that may be a legal
555
- * entity suffix (e.g., "a.s.", "GmbH", "s.r.o.").
556
- *
557
- * Does not validate the token against a legal-forms
558
- * list; false extensions are filtered by mergeAndDedup
559
- * when a legal-form detector produces a competing
560
- * entity with the correct span.
561
- */
562
- const tryPrefixExtension = (fullText, start, end) => {
563
- const maxEnd = Math.min(end + MAX_PREFIX_OVERSHOOT, fullText.length);
564
- if (maxEnd <= end + 1) return null;
565
- const after = fullText.slice(end, maxEnd);
566
- if (!after.startsWith(" ")) return null;
567
- const nextSpace = after.indexOf(" ", 1);
568
- const suffixEnd = nextSpace !== -1 ? nextSpace : after.length;
569
- if (suffixEnd <= 1) return null;
570
- const newEnd = end + suffixEnd;
571
- return {
572
- end: newEnd,
573
- text: fullText.slice(start, newEnd)
574
- };
575
- };
576
- //#endregion
577
- //#region src/util/text.ts
578
- /**
579
- * Shared text utilities for detectors.
580
- *
581
- * Extracted from names.ts and deny-list.ts to avoid
582
- * duplicating regex constants and helper functions.
583
- */
584
- /** Matches a string that starts with an uppercase letter. */
585
- const UPPER_START_RE = /^\p{Lu}/u;
586
- /** Matches a string consisting entirely of uppercase letters. */
587
- const ALL_UPPER_RE = /^\p{Lu}+$/u;
588
- const SENTENCE_END_RE = /[.!?]/;
589
- /**
590
- * Detect whether a position is at the start of a sentence.
591
- * Looks backward past whitespace for sentence-ending
592
- * punctuation (.!?). Position 0 and positions preceded
593
- * only by whitespace are considered sentence starts.
594
- */
595
- const isSentenceStart = (text, pos) => {
596
- if (pos === 0) return true;
597
- let i = pos - 1;
598
- while (i >= 0 && /\s/.test(text[i] ?? "")) i--;
599
- if (i < 0) return true;
600
- return SENTENCE_END_RE.test(text[i] ?? "");
601
- };
602
- //#endregion
603
- //#region src/detectors/names.ts
604
- const getCorpus = (ctx) => ctx.nameCorpus;
605
- const getNameCorpusFirstNames = (ctx = defaultContext) => ctx.nameCorpus?.firstNamesList ?? [];
606
- const getNameCorpusSurnames = (ctx = defaultContext) => ctx.nameCorpus?.surnamesList ?? [];
607
- const getNameCorpusTitles = (ctx = defaultContext) => ctx.nameCorpus?.titlesList ?? [];
608
- /**
609
- * Load name corpus data from injected dictionaries
610
- * and legacy config files. Merges all sources.
611
- *
612
- * Safe to call multiple times; only loads once per
613
- * context. Must be called before detectNameCorpus or
614
- * the getNameCorpus*() accessors are used.
615
- *
616
- * @param dictionaries Optional pre-loaded dictionaries
617
- * with per-language first names and surnames. When
618
- * omitted, only legacy config files are used.
619
- */
620
- const initNameCorpus = (ctx = defaultContext, dictionaries) => {
621
- if (ctx.nameCorpusPromise) return ctx.nameCorpusPromise;
622
- const promise = (async () => {
623
- try {
624
- const [legacyFirstMod, legacySurnameMod, titleMod, exclusionMod] = await Promise.all([
625
- import("./names-first.mjs"),
626
- import("./names-surnames.mjs"),
627
- import("./names-title-tokens.mjs"),
628
- import("./names-exclusions.mjs")
629
- ]);
630
- const firstNames = [...legacyFirstMod.default.names];
631
- if (dictionaries?.firstNames) for (const names of Object.values(dictionaries.firstNames)) for (const name of names) firstNames.push(name);
632
- const surnames = [...legacySurnameMod.default.names];
633
- if (dictionaries?.surnames) for (const names of Object.values(dictionaries.surnames)) for (const name of names) surnames.push(name);
634
- const dedup = (arr) => {
635
- const seen = /* @__PURE__ */ new Set();
636
- const result = [];
637
- for (const item of arr) {
638
- if (seen.has(item)) continue;
639
- seen.add(item);
640
- result.push(item);
641
- }
642
- return result;
643
- };
644
- const dedupFirst = dedup(firstNames);
645
- const dedupSurnames = dedup(surnames);
646
- const titles = titleMod.default.tokens;
647
- const exclusions = exclusionMod.default.words;
648
- ctx.nameCorpus = {
649
- firstNames: Object.freeze(new Set(dedupFirst)),
650
- surnames: Object.freeze(new Set(dedupSurnames)),
651
- titleTokens: Object.freeze(new Set(titles)),
652
- excludedWords: Object.freeze(new Set(exclusions)),
653
- firstNamesList: Object.freeze(dedupFirst),
654
- surnamesList: Object.freeze(dedupSurnames),
655
- titlesList: Object.freeze(titles),
656
- excludedList: Object.freeze(exclusions)
657
- };
658
- } catch (err) {
659
- ctx.nameCorpusPromise = null;
660
- console.warn("[anonymize] Failed to load name corpus JSON — name detection disabled:", err);
661
- }
662
- })();
663
- ctx.nameCorpusPromise = promise;
664
- return promise;
665
- };
666
- const INFLECTION_SUFFIXES = [
667
- "ovi",
668
- "em",
669
- "om",
670
- "ou",
671
- "é",
672
- "a",
673
- "u"
674
- ];
675
- /**
676
- * Strip common Czech/Slovak case suffixes from a token.
677
- * Returns candidate base forms if stripping produces a
678
- * plausible name (capitalised, length >= 3).
679
- *
680
- * For the "-ou" instrumental feminine suffix, also yields
681
- * base + "a" (e.g., "Editou" → "Edit" and "Edita")
682
- * because Czech feminine names decline -a → -ou.
683
- */
684
- const stripInflection = (token) => {
685
- const candidates = [];
686
- for (const suffix of INFLECTION_SUFFIXES) if (token.length > suffix.length + 2 && token.endsWith(suffix)) {
687
- const base = token.slice(0, -suffix.length);
688
- if (/^\p{Lu}/u.test(base)) {
689
- candidates.push(base);
690
- if (suffix === "ou" || suffix === "é" || suffix === "u") candidates.push(`${base}a`);
691
- }
692
- }
693
- return candidates;
694
- };
695
- const TOKEN_TYPE = {
696
- NAME: "name",
697
- SURNAME: "surname",
698
- TITLE: "title",
699
- ABBREVIATION: "abbreviation",
700
- CAPITALIZED: "capitalized",
701
- OTHER: "other"
702
- };
703
- const PERSON_CHAIN_BREAK_RE$1 = /[!?;:]/u;
704
- const isInitialContinuationGap$1 = (text, gap) => /^\p{Lu}$/u.test(text) && /^\.[^\S\n]{1,2}$/u.test(gap) || /^[^\S\n]{1,2}(?:\p{Lu}\.[^\S\n]{1,2})+$/u.test(gap);
705
- /**
706
- * Check if a token is in the first-name set, either
707
- * directly or after stripping Czech/Slovak inflection.
708
- */
709
- const isFirstNameToken = (token, corpus) => {
710
- if (corpus.firstNames.has(token)) return true;
711
- return stripInflection(token).some((b) => corpus.firstNames.has(b));
712
- };
713
- /**
714
- * Check if a token is in the surname set, either
715
- * directly or after stripping Czech/Slovak inflection.
716
- */
717
- const isSurnameToken = (token, corpus) => {
718
- if (corpus.surnames.has(token)) return true;
719
- return stripInflection(token).some((b) => corpus.surnames.has(b));
720
- };
721
- /**
722
- * Check if a token looks like a single-letter
723
- * abbreviation: "J.", "M.", etc.
724
- */
725
- const isAbbreviation = (token) => token.length === 2 && /^\p{Lu}$/u.test(token[0] ?? "") && token[1] === ".";
726
- const segmenter$1 = new Intl.Segmenter(void 0, { granularity: "word" });
727
- /**
728
- * Split text into word segments using Intl.Segmenter.
729
- * Only returns segments flagged as words.
730
- */
731
- const segmentWords = (fullText) => {
732
- const words = [];
733
- for (const seg of segmenter$1.segment(fullText)) if (seg.isWordLike) words.push({
734
- text: seg.segment,
735
- start: seg.index,
736
- end: seg.index + seg.segment.length
737
- });
738
- return words;
739
- };
740
- /** NAME or SURNAME — both represent corpus-matched tokens */
741
- const isCorpusMatch = (type) => type === TOKEN_TYPE.NAME || type === TOKEN_TYPE.SURNAME;
742
- const classifyToken = (word, corpus) => {
743
- const { text, start, end } = word;
744
- const lower = text.toLowerCase();
745
- const stripped = text.endsWith(".") ? text.slice(0, -1).toLowerCase() : lower;
746
- if (corpus.titleTokens.has(stripped)) return {
747
- text,
748
- type: TOKEN_TYPE.TITLE,
749
- start,
750
- end
751
- };
752
- if (isAbbreviation(text)) return {
753
- text,
754
- type: TOKEN_TYPE.ABBREVIATION,
755
- start,
756
- end
757
- };
758
- if (corpus.excludedWords.has(lower)) return {
759
- text,
760
- type: TOKEN_TYPE.OTHER,
761
- start,
762
- end
763
- };
764
- if (text.length < 3) return {
765
- text,
766
- type: TOKEN_TYPE.OTHER,
767
- start,
768
- end
769
- };
770
- if (text.length > 3 && ALL_UPPER_RE.test(text)) return {
771
- text,
772
- type: TOKEN_TYPE.OTHER,
773
- start,
774
- end
775
- };
776
- if (!UPPER_START_RE.test(text)) return {
777
- text,
778
- type: TOKEN_TYPE.OTHER,
779
- start,
780
- end
781
- };
782
- if (isFirstNameToken(text, corpus)) return {
783
- text,
784
- type: TOKEN_TYPE.NAME,
785
- start,
786
- end
787
- };
788
- if (isSurnameToken(text, corpus)) return {
789
- text,
790
- type: TOKEN_TYPE.SURNAME,
791
- start,
792
- end
793
- };
794
- return {
795
- text,
796
- type: TOKEN_TYPE.CAPITALIZED,
797
- start,
798
- end
799
- };
800
- };
801
- /**
802
- * Detect person names by looking up tokens against the
803
- * name corpus, then chaining adjacent name-like tokens.
804
- *
805
- * Requires initNameCorpus() to have been called first.
806
- * If not initialized, returns an empty array.
807
- *
808
- * Scoring:
809
- * TITLE + NAME/SURNAME → 0.95
810
- * NAME + NAME/SURNAME → 0.9
811
- * SURNAME + NAME/SURNAME → 0.9
812
- * NAME + CAPITALIZED → 0.7
813
- * ABBREVIATION + NAME → 0.7
814
- * Standalone NAME → 0.5 (low confidence)
815
- * Standalone SURNAME → skip (too ambiguous)
816
- */
817
- const detectNameCorpus = (fullText, ctx = defaultContext) => {
818
- const corpus = getCorpus(ctx);
819
- if (!corpus) return [];
820
- const tokens = segmentWords(fullText).map((w) => classifyToken(w, corpus));
821
- const entities = [];
822
- const consumed = /* @__PURE__ */ new Set();
823
- for (let i = 0; i < tokens.length; i++) {
824
- if (consumed.has(i)) continue;
825
- const token = tokens[i];
826
- if (!token) continue;
827
- if (token.type !== TOKEN_TYPE.TITLE && token.type !== TOKEN_TYPE.NAME && token.type !== TOKEN_TYPE.SURNAME && token.type !== TOKEN_TYPE.ABBREVIATION) continue;
828
- const MAX_CHAIN = 5;
829
- const chain = [token];
830
- let j = i + 1;
831
- while (j < tokens.length && chain.length < MAX_CHAIN) {
832
- const next = tokens[j];
833
- if (!next) break;
834
- const prev = chain.at(-1);
835
- if (prev) {
836
- const gap = fullText.slice(prev.end, next.start);
837
- const breaksOnPeriod = gap.includes(".") && !isInitialContinuationGap$1(prev.text, gap);
838
- if (gap.includes("\n") || PERSON_CHAIN_BREAK_RE$1.test(gap) || breaksOnPeriod) break;
839
- }
840
- if (next.type === TOKEN_TYPE.NAME || next.type === TOKEN_TYPE.SURNAME || next.type === TOKEN_TYPE.TITLE || next.type === TOKEN_TYPE.ABBREVIATION || next.type === TOKEN_TYPE.CAPITALIZED) {
841
- chain.push(next);
842
- j++;
843
- } else break;
844
- }
845
- const hasTitle = chain.some((t) => t.type === TOKEN_TYPE.TITLE);
846
- const hasCorpusName = chain.some((t) => isCorpusMatch(t.type));
847
- const hasFirstName = chain.some((t) => t.type === TOKEN_TYPE.NAME);
848
- const hasAbbreviation = chain.some((t) => t.type === TOKEN_TYPE.ABBREVIATION);
849
- const corpusCount = chain.filter((t) => isCorpusMatch(t.type)).length;
850
- const capitalizedCount = chain.filter((t) => t.type === TOKEN_TYPE.CAPITALIZED).length;
851
- let score = 0;
852
- if (hasTitle && hasCorpusName) score = .95;
853
- else if (corpusCount >= 2) score = .9;
854
- else if (hasCorpusName && capitalizedCount > 0) score = .7;
855
- else if (hasAbbreviation && hasCorpusName) score = .7;
856
- else if (hasFirstName && chain.length === 1) {
857
- if (isSentenceStart(fullText, token.start)) continue;
858
- score = .5;
859
- } else if (!hasFirstName && chain.length === 1 && chain[0]?.type === TOKEN_TYPE.SURNAME) continue;
860
- else if (hasTitle && chain.length === 1) continue;
861
- else {
862
- if (!hasCorpusName) continue;
863
- score = .5;
864
- }
865
- const first = chain.at(0);
866
- const last = chain.at(-1);
867
- if (!first || !last) continue;
868
- const start = first.start;
869
- const end = last.end;
870
- const text = fullText.slice(start, end);
871
- for (let k = i; k < i + chain.length; k++) consumed.add(k);
872
- entities.push({
873
- start,
874
- end,
875
- label: "person",
876
- text,
877
- score,
878
- source: DETECTION_SOURCES.REGEX
879
- });
880
- }
881
- return entities;
882
- };
883
- //#endregion
884
- //#region src/config/titles.ts
885
- /**
886
- * Academic and professional title prefixes.
887
- * Plain text; the detector auto-escapes for regex.
888
- * Sorted longest-first at build time.
889
- */
890
- const TITLE_PREFIXES = [
891
- "Ing.",
892
- "Mgr.",
893
- "MgA.",
894
- "Bc.",
895
- "BcA.",
896
- "JUDr.",
897
- "MUDr.",
898
- "MVDr.",
899
- "MDDr.",
900
- "PhDr.",
901
- "RNDr.",
902
- "PaedDr.",
903
- "ThDr.",
904
- "ThLic.",
905
- "ICDr.",
906
- "RSDr.",
907
- "PharmDr.",
908
- "artD.",
909
- "akad.",
910
- "doc.",
911
- "prof.",
912
- "ao. Univ.-Prof.",
913
- "o. Univ.-Prof.",
914
- "Univ.-Prof.",
915
- "Hon.-Prof.",
916
- "em. Prof.",
917
- "Dr. med. dent.",
918
- "Dr. med. vet.",
919
- "Dr. med.",
920
- "Dr. rer. nat.",
921
- "Dr. rer. soc.",
922
- "Dr. rer. pol.",
923
- "Dr. sc. tech.",
924
- "Dr. sc. nat.",
925
- "Dr. sc. hum.",
926
- "Dr. iur.",
927
- "Dr. jur.",
928
- "Dr. theol.",
929
- "Dr. oec.",
930
- "Dr. techn.",
931
- "Dr. h. c.",
932
- "Dr. phil.",
933
- "Dr.-Ing.",
934
- "Dr. Ing.",
935
- "Dr.",
936
- "Dipl.-Wirt.-Ing.",
937
- "Dipl.-Betriebsw.",
938
- "Dipl.-Inform.",
939
- "Dipl.-Volksw.",
940
- "Dipl.-Psych.",
941
- "Dipl.-Phys.",
942
- "Dipl.-Chem.",
943
- "Dipl.-Biol.",
944
- "Dipl.-Math.",
945
- "Dipl.-Päd.",
946
- "Dipl.-Soz.",
947
- "Dipl.-Kfm.",
948
- "Dipl.-Jur.",
949
- "Dipl. Ing.",
950
- "Dipl.-Ing.",
951
- "Mag. rer. soc. oec.",
952
- "Mag. rer. nat.",
953
- "Mag. phil.",
954
- "Mag. iur.",
955
- "Mag. arch.",
956
- "Mag. pharm.",
957
- "Mag. (FH)",
958
- "Mag.",
959
- "Bakk. rer. nat.",
960
- "Bakk. techn.",
961
- "Bakk. phil.",
962
- "Bakk.",
963
- "Lic. phil.",
964
- "Lic. iur.",
965
- "Lic. oec.",
966
- "Lic. theol.",
967
- "Lic.",
968
- "Priv.-Doz.",
969
- "PD",
970
- "RA"
971
- ];
972
- /**
973
- * Courtesy/honorific titles that precede a person's
974
- * name. Sorted alphabetically. The detector escapes
975
- * dots and adds \b for entries in HONORIFIC_BOUNDARY.
976
- */
977
- const HONORIFICS = [
978
- "Avv.",
979
- "Dame",
980
- "Doamna",
981
- "Domnul",
982
- "Don",
983
- "Doña",
984
- "Dott.",
985
- "Judge",
986
- "Justice",
987
- "Lady",
988
- "Lord",
989
- "M.",
990
- "Madame",
991
- "Mademoiselle",
992
- "Maître",
993
- "Me",
994
- "Messrs",
995
- "Miss",
996
- "Mlle",
997
- "Mme",
998
- "Monsieur",
999
- "Mr",
1000
- "Mrs",
1001
- "Ms",
1002
- "President",
1003
- "Señor",
1004
- "Señora",
1005
- "Sig.",
1006
- "Sig.ra",
1007
- "Signor",
1008
- "Signora",
1009
- "Signorina",
1010
- "Sir",
1011
- "Sr.",
1012
- "Sra."
1013
- ];
1014
- /**
1015
- * Honorifics that need \b word-boundary anchors
1016
- * (short or common words that could match mid-word).
1017
- */
1018
- const HONORIFIC_BOUNDARY = new Set([
1019
- "Don",
1020
- "Doña",
1021
- "M.",
1022
- "Me",
1023
- "Señor",
1024
- "Señora"
1025
- ]);
1026
- /**
1027
- * Post-nominal degrees (comma or space separated after name).
1028
- * Plain text; the detector auto-escapes for regex.
1029
- */
1030
- const POST_NOMINALS = [
1031
- "Ph.D.",
1032
- "Ph.D",
1033
- "CSc.",
1034
- "DrSc.",
1035
- "ArtD.",
1036
- "D.Phil.",
1037
- "DPhil.",
1038
- "MPhil.",
1039
- "MBA",
1040
- "MPA",
1041
- "LL.M.",
1042
- "LL.B.",
1043
- "M.Sc.",
1044
- "B.Sc.",
1045
- "MSc.",
1046
- "BSc.",
1047
- "M.Eng.",
1048
- "B.Eng.",
1049
- "M.A.",
1050
- "B.A.",
1051
- "JCD",
1052
- "JD",
1053
- "DiS.",
1054
- "ACCA",
1055
- "FCCA",
1056
- "CIPM",
1057
- "CIPT",
1058
- "CIPP/E",
1059
- "CIPP"
1060
- ];
206
+ const LEGAL_SUFFIXES = [...[
207
+ "spol. s r.o.",
208
+ "s.r.o.",
209
+ "s. r. o.",
210
+ "a.s.",
211
+ "a. s.",
212
+ "v.o.s.",
213
+ "v. o. s.",
214
+ "k.s.",
215
+ "k. s.",
216
+ "z.s.",
217
+ "z. s.",
218
+ "z.ú.",
219
+ "z. ú.",
220
+ "o.p.s.",
221
+ "o. p. s.",
222
+ "s.p.",
223
+ "s. p.",
224
+ "GmbH",
225
+ "AG",
226
+ "SE",
227
+ "KG",
228
+ "OHG",
229
+ "Ltd.",
230
+ "Ltd",
231
+ "LLC",
232
+ "LLP",
233
+ "Inc.",
234
+ "Corp.",
235
+ "Corporation",
236
+ "Co.",
237
+ "LP",
238
+ "L.P.",
239
+ "PLC",
240
+ "plc",
241
+ "N.A.",
242
+ "N.V.",
243
+ "B.V.",
244
+ "Pty Ltd.",
245
+ "Pty Ltd",
246
+ "S.A.",
247
+ "SA",
248
+ "SAS",
249
+ "SARL",
250
+ "S.p.A.",
251
+ "Sp. z o.o.",
252
+ "Sp. k.",
253
+ "Sp. j.",
254
+ "Ltda.",
255
+ "LTDA.",
256
+ "Ltda",
257
+ "LTDA",
258
+ "S/A",
259
+ "EIRELI",
260
+ "EPP",
261
+ "ME",
262
+ "MEI"
263
+ ]].sort((a, b) => b.length - a.length);
1061
264
  //#endregion
1062
265
  //#region src/data/char-groups.json
1063
266
  var char_groups_default = {
@@ -1432,220 +635,1503 @@ var char_groups_default = {
1432
635
  "code": "U+060C"
1433
636
  },
1434
637
  {
1435
- "char": "、",
1436
- "name": "ideographic comma",
1437
- "code": "U+3001"
638
+ "char": "、",
639
+ "name": "ideographic comma",
640
+ "code": "U+3001"
641
+ },
642
+ {
643
+ "char": ",",
644
+ "name": "full-width comma",
645
+ "code": "U+FF0C"
646
+ },
647
+ {
648
+ "char": "﹐",
649
+ "name": "small comma",
650
+ "code": "U+FE50"
651
+ },
652
+ {
653
+ "char": "՝",
654
+ "name": "Armenian comma",
655
+ "code": "U+055D"
656
+ }
657
+ ]
658
+ },
659
+ "ellipsis": {
660
+ "description": "Ellipsis characters",
661
+ "chars": [{
662
+ "char": "…",
663
+ "name": "horizontal ellipsis",
664
+ "code": "U+2026"
665
+ }]
666
+ },
667
+ "question-mark": {
668
+ "description": "Question mark variants",
669
+ "chars": [
670
+ {
671
+ "char": "?",
672
+ "name": "question mark",
673
+ "code": "U+003F"
674
+ },
675
+ {
676
+ "char": "?",
677
+ "name": "full-width question mark",
678
+ "code": "U+FF1F"
679
+ },
680
+ {
681
+ "char": "⁇",
682
+ "name": "double question mark",
683
+ "code": "U+2047"
684
+ },
685
+ {
686
+ "char": "⁈",
687
+ "name": "question exclamation mark",
688
+ "code": "U+2048"
689
+ },
690
+ {
691
+ "char": "؟",
692
+ "name": "Arabic question mark",
693
+ "code": "U+061F"
694
+ },
695
+ {
696
+ "char": "‽",
697
+ "name": "interrobang",
698
+ "code": "U+203D"
699
+ }
700
+ ]
701
+ },
702
+ "exclamation-mark": {
703
+ "description": "Exclamation mark variants",
704
+ "chars": [
705
+ {
706
+ "char": "!",
707
+ "name": "exclamation mark",
708
+ "code": "U+0021"
709
+ },
710
+ {
711
+ "char": "!",
712
+ "name": "full-width exclamation mark",
713
+ "code": "U+FF01"
714
+ },
715
+ {
716
+ "char": "‼",
717
+ "name": "double exclamation mark",
718
+ "code": "U+203C"
719
+ },
720
+ {
721
+ "char": "¡",
722
+ "name": "inverted exclamation mark",
723
+ "code": "U+00A1"
724
+ }
725
+ ]
726
+ },
727
+ "semicolon": {
728
+ "description": "Semicolon variants",
729
+ "chars": [
730
+ {
731
+ "char": ";",
732
+ "name": "semicolon",
733
+ "code": "U+003B"
1438
734
  },
1439
735
  {
1440
- "char": "",
1441
- "name": "full-width comma",
1442
- "code": "U+FF0C"
736
+ "char": "؛",
737
+ "name": "Arabic semicolon",
738
+ "code": "U+061B"
1443
739
  },
1444
740
  {
1445
- "char": "",
1446
- "name": "small comma",
1447
- "code": "U+FE50"
741
+ "char": "",
742
+ "name": "full-width semicolon",
743
+ "code": "U+FF1B"
1448
744
  },
1449
745
  {
1450
- "char": "՝",
1451
- "name": "Armenian comma",
1452
- "code": "U+055D"
746
+ "char": "",
747
+ "name": "Ethiopic semicolon",
748
+ "code": "U+1364"
749
+ }
750
+ ]
751
+ },
752
+ "underscore": {
753
+ "description": "Underscore variants",
754
+ "chars": [{
755
+ "char": "_",
756
+ "name": "low line",
757
+ "code": "U+005F"
758
+ }, {
759
+ "char": "_",
760
+ "name": "full-width low line",
761
+ "code": "U+FF3F"
762
+ }]
763
+ }
764
+ }
765
+ };
766
+ //#endregion
767
+ //#region src/util/char-groups.ts
768
+ /**
769
+ * Centralized Unicode character equivalence groups.
770
+ *
771
+ * Centralized Unicode character equivalence groups.
772
+ *
773
+ * Provides helpers to build regex character classes that
774
+ * match all typographic variants of a character type
775
+ * (dashes, spaces, quotes, etc.).
776
+ *
777
+ * The JSON is statically imported so the bundler inlines
778
+ * it, avoiding a runtime require() that breaks browsers.
779
+ */
780
+ /** Chars that need escaping inside a regex char class. */
781
+ const REGEX_CLASS_SPECIAL = /[\\\]^-]/;
782
+ const escapeForCharClass = (ch) => REGEX_CLASS_SPECIAL.test(ch) ? `\\${ch}` : ch;
783
+ const config = char_groups_default;
784
+ /**
785
+ * Get the raw characters for a named group.
786
+ * Throws if the group does not exist.
787
+ */
788
+ const charSet = (group) => {
789
+ const g = config.groups[group];
790
+ if (!g) throw new Error(`Unknown char group: "${group}"`);
791
+ return g.chars.map((entry) => entry.char);
792
+ };
793
+ /**
794
+ * Build a regex character class string for a named
795
+ * group. E.g., charClass("dash") returns a string
796
+ * like "[-\u2013\u2014\u2010\u2011\u2212\u2043]".
797
+ *
798
+ * The hyphen-minus is placed first so it is treated
799
+ * as a literal, not a range indicator.
800
+ */
801
+ const charClass = (group) => {
802
+ return `[${[...charSet(group)].sort((a, b) => {
803
+ if (a === "-") return -1;
804
+ if (b === "-") return 1;
805
+ return a.localeCompare(b);
806
+ }).map(escapeForCharClass).join("")}]`;
807
+ };
808
+ /**
809
+ * Return the inner content of a character class (without
810
+ * the surrounding brackets). Useful for embedding a group
811
+ * inside a larger character class, e.g.:
812
+ * `[\\s&,.${charClassInner("dash")}]`
813
+ */
814
+ const charClassInner = (group) => {
815
+ return [...charSet(group)].sort((a, b) => {
816
+ if (a === "-") return -1;
817
+ if (b === "-") return 1;
818
+ return a.localeCompare(b);
819
+ }).map(escapeForCharClass).join("");
820
+ };
821
+ /** Regex char class matching all dash variants. */
822
+ const DASH = charClass("dash");
823
+ /**
824
+ * Inner content of the dash char class (no brackets).
825
+ * For embedding inside a larger character class:
826
+ * `[\\s&,.${DASH_INNER}]`
827
+ */
828
+ const DASH_INNER = charClassInner("dash");
829
+ charClass("space");
830
+ charClass("quote-double");
831
+ /** Inner content of the double-quote char class (no brackets). */
832
+ const QUOTE_DOUBLE_INNER = charClassInner("quote-double");
833
+ charClass("quote-single");
834
+ /** Inner content of the single-quote char class (no brackets). */
835
+ const QUOTE_SINGLE_INNER = charClassInner("quote-single");
836
+ /**
837
+ * Inline opening-bracket characters that frequently introduce
838
+ * a defined-term parenthetical immediately after an address or
839
+ * organization span (e.g. `…GA 30326, USA (the "Premises")`).
840
+ * Hardcoded rather than data-driven because the set is tiny,
841
+ * regex-special, and not language-dependent.
842
+ */
843
+ const OPENING_BRACKETS_INNER = "(\\[{";
844
+ charClass("slash");
845
+ charClass("dot");
846
+ charClass("colon");
847
+ //#endregion
848
+ //#region src/detectors/legal-forms.ts
849
+ const SENTENCE_VERB_INDICATORS_SEED = new Set([
850
+ "je",
851
+ "jsou",
852
+ "is",
853
+ "are",
854
+ "ist",
855
+ "sind"
856
+ ]);
857
+ let sentenceVerbIndicatorsCache = null;
858
+ let sentenceVerbIndicatorsPromise = null;
859
+ const loadSentenceVerbIndicators = async () => {
860
+ if (sentenceVerbIndicatorsCache) return sentenceVerbIndicatorsCache;
861
+ if (sentenceVerbIndicatorsPromise) return sentenceVerbIndicatorsPromise;
862
+ sentenceVerbIndicatorsPromise = (async () => {
863
+ let data = {};
864
+ try {
865
+ const mod = await import("./sentence-verb-indicators.mjs");
866
+ data = mod.default ?? mod;
867
+ } catch (err) {
868
+ console.warn("[anonymize] legal-forms: failed to load sentence-verb-indicators.json, falling back to seed list:", err);
869
+ }
870
+ const all = new Set(SENTENCE_VERB_INDICATORS_SEED);
871
+ for (const [key, value] of Object.entries(data)) {
872
+ if (key.startsWith("_")) continue;
873
+ if (!Array.isArray(value)) continue;
874
+ for (const verb of value) {
875
+ if (typeof verb !== "string" || verb.length === 0) continue;
876
+ all.add(verb.toLowerCase());
877
+ }
878
+ }
879
+ sentenceVerbIndicatorsCache = all;
880
+ return all;
881
+ })();
882
+ return sentenceVerbIndicatorsPromise;
883
+ };
884
+ const getSentenceVerbIndicatorsSync = () => sentenceVerbIndicatorsCache ?? SENTENCE_VERB_INDICATORS_SEED;
885
+ const UPPER = "A-ZÁČĎÉĚÍŇÓŘŠŤÚŮÝŽÄÖÜÀÂÆÇÈÊËÎÏÔÙÛŸÑĄĆĘŁŃŚŹŻ\\u0130";
886
+ const LOWER = "a-záčďéěíňóřšťúůýžäöüßàâæçèêëîïôùûÿñąćęłńśźż\\u0131";
887
+ const CAP_WORD = `(?:[${UPPER}]{2,}|[${UPPER}][${LOWER}${UPPER}]+)`;
888
+ const ALLCAP_WORD = `[${UPPER}]{2,}`;
889
+ const ROMAN_NUMERAL_RE = /^(?=[IVXLCDM])M{0,3}(?:CM|CD|D?C{0,3})(?:XC|XL|L?X{0,3})(?:IX|IV|V?I{0,3})$/;
890
+ let legalRoleHeadsCache = null;
891
+ let legalRoleHeadsPromise = null;
892
+ const loadLegalRoleHeads = async () => {
893
+ if (legalRoleHeadsCache) return legalRoleHeadsCache;
894
+ if (legalRoleHeadsPromise) return legalRoleHeadsPromise;
895
+ legalRoleHeadsPromise = (async () => {
896
+ const sets = await loadLanguageConfigs("legalRoleHeads", (mod) => {
897
+ return mod.default ?? mod;
898
+ });
899
+ const all = /* @__PURE__ */ new Set();
900
+ for (const entry of sets) {
901
+ if (!entry || !Array.isArray(entry.words)) continue;
902
+ for (const word of entry.words) if (typeof word === "string" && word.length > 0) all.add(word.toLowerCase());
903
+ }
904
+ legalRoleHeadsCache = all;
905
+ return all;
906
+ })();
907
+ return legalRoleHeadsPromise;
908
+ };
909
+ const getLegalRoleHeadsSync = () => legalRoleHeadsCache ?? /* @__PURE__ */ new Set();
910
+ const warmLegalRoleHeads = async () => {
911
+ await Promise.all([
912
+ loadLegalRoleHeads(),
913
+ loadAllLegalSuffixes(),
914
+ loadSentenceVerbIndicators(),
915
+ loadClauseNounHeads()
916
+ ]);
917
+ };
918
+ let allLegalSuffixesCache = null;
919
+ let allLegalSuffixesPromise = null;
920
+ const loadAllLegalSuffixes = async () => {
921
+ if (allLegalSuffixesCache) return allLegalSuffixesCache;
922
+ if (allLegalSuffixesPromise) return allLegalSuffixesPromise;
923
+ allLegalSuffixesPromise = (async () => {
924
+ let data = {};
925
+ try {
926
+ data = (await import("./legal-forms.mjs")).default;
927
+ } catch {
928
+ data = {};
929
+ }
930
+ const seen = /* @__PURE__ */ new Set();
931
+ const out = [];
932
+ for (const list of Object.values(data)) for (const form of list) {
933
+ if (typeof form !== "string" || form.length === 0) continue;
934
+ if (seen.has(form)) continue;
935
+ seen.add(form);
936
+ out.push(form);
937
+ }
938
+ for (const form of LEGAL_SUFFIXES) if (!seen.has(form)) {
939
+ seen.add(form);
940
+ out.push(form);
941
+ }
942
+ out.sort((a, b) => b.length - a.length);
943
+ allLegalSuffixesCache = out;
944
+ return out;
945
+ })();
946
+ return allLegalSuffixesPromise;
947
+ };
948
+ const getAllLegalSuffixesSync = () => allLegalSuffixesCache ?? LEGAL_SUFFIXES;
949
+ /**
950
+ * Sync accessor for the full legal-form vocabulary
951
+ * (`data/legal-forms.json` plus `LEGAL_SUFFIXES`,
952
+ * longest-first). Falls back to `LEGAL_SUFFIXES` when
953
+ * `warmLegalRoleHeads()` has not run yet. Exposed so the
954
+ * trailing-period strip in `sanitizeEntities` can keep
955
+ * pace with the detector vocabulary rather than only the
956
+ * smaller `LEGAL_SUFFIXES` propagation list.
957
+ */
958
+ const getKnownLegalSuffixes = getAllLegalSuffixesSync;
959
+ const CLAUSE_NOUN_HEADS_SEED = new Set(["agreement", "contract"]);
960
+ let clauseNounHeadsCache = null;
961
+ let clauseNounHeadsPromise = null;
962
+ const loadClauseNounHeads = async () => {
963
+ if (clauseNounHeadsCache) return clauseNounHeadsCache;
964
+ if (clauseNounHeadsPromise) return clauseNounHeadsPromise;
965
+ clauseNounHeadsPromise = (async () => {
966
+ let data = {};
967
+ try {
968
+ const mod = await import("./clause-noun-heads.mjs");
969
+ data = mod.default ?? mod;
970
+ } catch (err) {
971
+ console.warn("[anonymize] legal-forms: failed to load clause-noun-heads.json, falling back to seed list:", err);
972
+ }
973
+ const all = new Set(CLAUSE_NOUN_HEADS_SEED);
974
+ for (const [key, value] of Object.entries(data)) {
975
+ if (key.startsWith("_")) continue;
976
+ if (!Array.isArray(value)) continue;
977
+ for (const noun of value) {
978
+ if (typeof noun !== "string" || noun.length === 0) continue;
979
+ all.add(noun.toLowerCase());
980
+ }
981
+ }
982
+ clauseNounHeadsCache = all;
983
+ return all;
984
+ })();
985
+ return clauseNounHeadsPromise;
986
+ };
987
+ const getClauseNounHeadsSync = () => clauseNounHeadsCache ?? CLAUSE_NOUN_HEADS_SEED;
988
+ const escapeForRegex = (form) => form.replace(/[.*+?^${}()|[\]\\]/g, "\\$&").replace(/\s+/g, "\\s+").replace(/\\\./g, "\\.[^\\S\\n]?");
989
+ const isShortForm = (form) => form.replace(/[.\s]/g, "").length <= 3 && !form.includes(" ");
990
+ const buildPatternString = (forms) => {
991
+ if (forms.length === 0) return null;
992
+ const alt = forms.toSorted((a, b) => b.length - a.length).map(escapeForRegex).join("|");
993
+ const HSPACE = "[^\\S\\n]";
994
+ const LOWER_CONNECTOR = `${HSPACE}+(?:a|and|und|et|e|y|i)${HSPACE}+(?=[${LOWER}])`;
995
+ const SIMPLE_SEP = `(?:${HSPACE}|[&,.${DASH_INNER}]){1,4}`;
996
+ const CAP_OR_NUM_WORD = `(?:${CAP_WORD}|\\d{1,4})`;
997
+ const LOWER_WORD = `(?:(?!(?:and|und|et)(?![${UPPER}${LOWER}]))[${LOWER}][${LOWER}${UPPER}]+)`;
998
+ const ANY_WORD_TAIL = `(?:(?!(?:and|und|et)(?![${UPPER}${LOWER}]))[${UPPER}${LOWER}][${LOWER}${UPPER}]+|[${UPPER}]{2,3}|\\d{1,4})`;
999
+ return `${`(?:${`(?:${CAP_WORD})(?:${SIMPLE_SEP}(?:${CAP_OR_NUM_WORD})){0,10}`})(?:${`${SIMPLE_SEP}(?:${LOWER_WORD})(?:(?:${LOWER_CONNECTOR}|${SIMPLE_SEP})(?:${ANY_WORD_TAIL})){0,10}`})?`}(?:\\s+|,\\s*)(?:${alt})(?![${LOWER}])`;
1000
+ };
1001
+ /**
1002
+ * Build legal form regex pattern strings.
1003
+ * Returns an array of regex strings for the unified
1004
+ * TextSearch builder. Empty if data package is not
1005
+ * installed.
1006
+ */
1007
+ const buildLegalFormPatterns = async () => {
1008
+ let data = {};
1009
+ try {
1010
+ data = (await import("./legal-forms.mjs")).default;
1011
+ } catch {
1012
+ return [];
1013
+ }
1014
+ const allForms = [];
1015
+ const seen = /* @__PURE__ */ new Set();
1016
+ for (const forms of Object.values(data)) for (const form of forms) if (!seen.has(form)) {
1017
+ seen.add(form);
1018
+ allForms.push(form);
1019
+ }
1020
+ for (const form of LEGAL_SUFFIXES) if (!seen.has(form)) {
1021
+ seen.add(form);
1022
+ allForms.push(form);
1023
+ }
1024
+ const patterns = [];
1025
+ const longPattern = buildPatternString(allForms.filter((f) => !isShortForm(f)));
1026
+ if (longPattern) patterns.push(longPattern);
1027
+ const shortPattern = buildPatternString(allForms.filter(isShortForm));
1028
+ if (shortPattern) patterns.push(shortPattern);
1029
+ const allcapPrefix = `(?:${ALLCAP_WORD})(?:[ \\t&,.${DASH_INNER}]{1,4}(?:${ALLCAP_WORD})){0,2}`;
1030
+ const allcapAlt = allForms.toSorted((a, b) => b.length - a.length).map(escapeForRegex).join("|");
1031
+ patterns.push(`${allcapPrefix}(?:[ \\t]+|,[ \\t]*)(?:${allcapAlt})(?![${LOWER}])`);
1032
+ return patterns;
1033
+ };
1034
+ const CONNECTOR_RE = /^(?:a|and|und|et|e|y|i|&)$/i;
1035
+ const AND_TYPE_CONNECTOR_RE = /^(?:and|und|et)$/i;
1036
+ const UPPER_LETTER_RE = /^\p{Lu}/u;
1037
+ const COMPANY_SUFFIX_WORDS_RE = /^(?:Company|Co|Bank|Brothers|Bros|Sons|Group|Holdings|Trust|Partners|Associates|Corporation|Industries|Enterprises|Solutions|Systems|Services|Foundation|Institute)$/;
1038
+ const IN_NAME_PREPOSITION_RE = /^(?:of|the)$/i;
1039
+ const ENTITY_HEAD_WORD_RE = /^[\p{L}\p{M}&]+/u;
1040
+ const LEADING_CLAUSE_RE = /(?:^|\s)(?:by\s+and\s+between|is\s+between)\s+/giu;
1041
+ /**
1042
+ * Find the word ending just before `pos` in `text`,
1043
+ * skipping any whitespace (not newlines).
1044
+ * Returns null if no word is found (e.g., at start
1045
+ * of text, or preceded by non-word chars like ".").
1046
+ */
1047
+ const findWordBefore = (text, pos) => {
1048
+ let scan = pos - 1;
1049
+ while (scan >= 0) {
1050
+ const ch = text.charAt(scan);
1051
+ if (ch === "\n" || !/\s/.test(ch)) break;
1052
+ scan--;
1053
+ }
1054
+ if (scan < 0 || text.charAt(scan) === "\n") return null;
1055
+ const wordEnd = scan + 1;
1056
+ while (scan >= 0 && /[\p{L}\p{M}&]/u.test(text.charAt(scan))) scan--;
1057
+ const wordStart = scan + 1;
1058
+ const word = text.slice(wordStart, wordEnd);
1059
+ if (word.length === 0) return null;
1060
+ return {
1061
+ word,
1062
+ start: wordStart
1063
+ };
1064
+ };
1065
+ /**
1066
+ * Count consecutive uppercase-starting words immediately
1067
+ * before `pos`. Stops at the first non-upper word or at
1068
+ * text/line start. Used to disambiguate "<First> <Last>
1069
+ * and <ORG>" from "<Multi-word Org> and <Continuation>".
1070
+ */
1071
+ const countUpperWordsBefore = (fullText, pos) => {
1072
+ let count = 0;
1073
+ let scan = pos;
1074
+ while (scan > 0) {
1075
+ const found = findWordBefore(fullText, scan);
1076
+ if (!found) break;
1077
+ if (!UPPER_LETTER_RE.test(found.word)) break;
1078
+ count++;
1079
+ scan = found.start;
1080
+ }
1081
+ return count;
1082
+ };
1083
+ /**
1084
+ * Extend a match backward through uppercase words and
1085
+ * lowercase connectors. Stops at start of text,
1086
+ * newline, or a word that doesn't qualify.
1087
+ *
1088
+ * Connectors (a, and, und, et, ...) are only consumed
1089
+ * when there is a valid word before them — a trailing
1090
+ * connector at an entity boundary is not consumed.
1091
+ * For multi-char "and"-type connectors we additionally
1092
+ * refuse to cross when exactly two uppercase words
1093
+ * precede them ("First Last and ORG, Inc." shape) —
1094
+ * unless the match itself begins with a known company-
1095
+ * suffix word ("…and Company, Inc."), in which case
1096
+ * the chain belongs to one organisation. In that
1097
+ * suffix-mode we also cross in-name prepositions
1098
+ * ("Bank of America and Trust Company, Inc.").
1099
+ */
1100
+ const extendBackward = (fullText, matchStart, options = {}) => {
1101
+ const headWord = ENTITY_HEAD_WORD_RE.exec(fullText.slice(matchStart))?.[0] ?? "";
1102
+ const suffixMode = options.forceSuffixMode === true || COMPANY_SUFFIX_WORDS_RE.test(headWord);
1103
+ let pos = matchStart;
1104
+ while (pos > 0) {
1105
+ const found = findWordBefore(fullText, pos);
1106
+ if (!found) break;
1107
+ const { word, start: wordStart } = found;
1108
+ const isUpper = UPPER_LETTER_RE.test(word);
1109
+ const isConnector = CONNECTOR_RE.test(word);
1110
+ const isInNamePrep = suffixMode && IN_NAME_PREPOSITION_RE.test(word);
1111
+ if (isUpper) pos = wordStart;
1112
+ else if (isConnector) {
1113
+ if (!suffixMode && AND_TYPE_CONNECTOR_RE.test(word) && countUpperWordsBefore(fullText, wordStart) === 2) break;
1114
+ const prev = findWordBefore(fullText, wordStart);
1115
+ if (!prev) break;
1116
+ if (!UPPER_LETTER_RE.test(prev.word)) break;
1117
+ pos = prev.start;
1118
+ } else if (isInNamePrep) {
1119
+ const prev = findWordBefore(fullText, wordStart);
1120
+ if (!prev) break;
1121
+ if (!UPPER_LETTER_RE.test(prev.word)) break;
1122
+ pos = prev.start;
1123
+ } else break;
1124
+ }
1125
+ return pos;
1126
+ };
1127
+ const trimLeadingClause = (text) => {
1128
+ let cut = -1;
1129
+ for (const match of text.matchAll(LEADING_CLAUSE_RE)) cut = match.index + match[0].length;
1130
+ if (cut <= 0) return {
1131
+ offset: 0,
1132
+ text
1133
+ };
1134
+ const trimmed = text.slice(cut);
1135
+ const leadingWs = trimmed.match(/^\s*/u)?.[0].length ?? 0;
1136
+ return {
1137
+ offset: cut + leadingWs,
1138
+ text: trimmed.slice(leadingWs)
1139
+ };
1140
+ };
1141
+ /**
1142
+ * Process legal form matches from the unified search.
1143
+ * Receives all matches; filters to the legal forms
1144
+ * slice via sliceStart/sliceEnd.
1145
+ *
1146
+ * The role-head trimming step reads per-language data from
1147
+ * a cache that `runPipeline` warms via `warmLegalRoleHeads()`
1148
+ * before calling this. Callers that invoke
1149
+ * `processLegalFormMatches` directly (without going through
1150
+ * `runPipeline`) must `await warmLegalRoleHeads()` first;
1151
+ * otherwise the trim falls back to a no-op and sentence-
1152
+ * fragment fixes do not apply.
1153
+ */
1154
+ const processLegalFormMatches = (allMatches, sliceStart, sliceEnd, fullText) => {
1155
+ const results = [];
1156
+ for (const match of allMatches) {
1157
+ const idx = match.pattern;
1158
+ if (idx < sliceStart || idx >= sliceEnd) continue;
1159
+ const text = match.text.trimEnd();
1160
+ if (text.length < 5) continue;
1161
+ const roleHeads = getLegalRoleHeadsSync();
1162
+ const firstWordMatch = /^[\p{L}\p{M}]+(?:-[\p{L}\p{M}]+)*/u.exec(text);
1163
+ let processedStart = match.start;
1164
+ let processedText = text;
1165
+ let trimmed = false;
1166
+ const firstWordText = firstWordMatch?.[0] ?? "";
1167
+ const firstWordLeading = /^[\p{L}\p{M}]+/u.exec(firstWordText)?.[0] ?? "";
1168
+ if (firstWordMatch !== null && (roleHeads.has(firstWordText.toLowerCase()) || firstWordLeading.length > 0 && roleHeads.has(firstWordLeading.toLowerCase()))) {
1169
+ let suffixOffset = -1;
1170
+ for (const suffix of getAllLegalSuffixesSync()) {
1171
+ const idx = text.lastIndexOf(suffix);
1172
+ if (idx !== -1 && idx + suffix.length >= text.length - 1) {
1173
+ suffixOffset = idx;
1174
+ break;
1453
1175
  }
1454
- ]
1455
- },
1456
- "ellipsis": {
1457
- "description": "Ellipsis characters",
1458
- "chars": [{
1459
- "char": "…",
1460
- "name": "horizontal ellipsis",
1461
- "code": "U+2026"
1462
- }]
1463
- },
1464
- "question-mark": {
1465
- "description": "Question mark variants",
1466
- "chars": [
1467
- {
1468
- "char": "?",
1469
- "name": "question mark",
1470
- "code": "U+003F"
1471
- },
1472
- {
1473
- "char": "?",
1474
- "name": "full-width question mark",
1475
- "code": "U+FF1F"
1476
- },
1477
- {
1478
- "char": "⁇",
1479
- "name": "double question mark",
1480
- "code": "U+2047"
1481
- },
1482
- {
1483
- "char": "⁈",
1484
- "name": "question exclamation mark",
1485
- "code": "U+2048"
1486
- },
1487
- {
1488
- "char": "؟",
1489
- "name": "Arabic question mark",
1490
- "code": "U+061F"
1491
- },
1492
- {
1493
- "char": "‽",
1494
- "name": "interrobang",
1495
- "code": "U+203D"
1176
+ }
1177
+ if (suffixOffset < 0) {} else {
1178
+ const midStart = firstWordMatch[0].length;
1179
+ const midEnd = suffixOffset;
1180
+ const midSection = text.slice(midStart, midEnd);
1181
+ const verbIndicators = getSentenceVerbIndicatorsSync();
1182
+ let lastVerbEndInMid = -1;
1183
+ for (const match of midSection.matchAll(/(?<![\p{L}\p{N}])[\p{L}\p{M}]+/gu)) if (match[0] !== void 0 && match.index !== void 0 && verbIndicators.has(match[0].toLowerCase())) lastVerbEndInMid = match.index + match[0].length;
1184
+ const digitAfterRole = /^\s+\d+(?:\.|\b)/u.test(midSection);
1185
+ let appositiveRoleHead = false;
1186
+ if (!digitAfterRole && lastVerbEndInMid === -1 && fullText) {
1187
+ const before = fullText.slice(Math.max(0, match.start - 40), match.start);
1188
+ const prevWord = /(?<![\p{L}\p{N}])(\p{L}[\p{L}\p{M}]*)\s*$/u.exec(before);
1189
+ if (prevWord !== null && getSentenceVerbIndicatorsSync().has(prevWord[1].toLowerCase())) appositiveRoleHead = true;
1496
1190
  }
1497
- ]
1498
- },
1499
- "exclamation-mark": {
1500
- "description": "Exclamation mark variants",
1501
- "chars": [
1502
- {
1503
- "char": "!",
1504
- "name": "exclamation mark",
1505
- "code": "U+0021"
1506
- },
1507
- {
1508
- "char": "!",
1509
- "name": "full-width exclamation mark",
1510
- "code": "U+FF01"
1511
- },
1512
- {
1513
- "char": "‼",
1514
- "name": "double exclamation mark",
1515
- "code": "U+203C"
1516
- },
1517
- {
1518
- "char": "¡",
1519
- "name": "inverted exclamation mark",
1520
- "code": "U+00A1"
1191
+ if (lastVerbEndInMid !== -1 || digitAfterRole || appositiveRoleHead) {
1192
+ const scanStart = lastVerbEndInMid !== -1 ? midStart + lastVerbEndInMid : midStart;
1193
+ const capRe = /(?<![\p{L}\p{N}])\p{Lu}[\p{L}\p{M}\p{N}]*/gu;
1194
+ capRe.lastIndex = scanStart;
1195
+ const clauseNouns = getClauseNounHeadsSync();
1196
+ let capMatch = null;
1197
+ for (let next = capRe.exec(text); next !== null; next = capRe.exec(text)) {
1198
+ if (next.index >= suffixOffset) break;
1199
+ const lc = next[0].toLowerCase();
1200
+ if (roleHeads.has(lc) || clauseNouns.has(lc)) continue;
1201
+ capMatch = next;
1202
+ break;
1203
+ }
1204
+ if (capMatch === null) continue;
1205
+ processedStart = match.start + capMatch.index;
1206
+ processedText = text.slice(capMatch.index);
1207
+ trimmed = true;
1521
1208
  }
1522
- ]
1523
- },
1524
- "semicolon": {
1525
- "description": "Semicolon variants",
1526
- "chars": [
1527
- {
1528
- "char": ";",
1529
- "name": "semicolon",
1530
- "code": "U+003B"
1531
- },
1532
- {
1533
- "char": "؛",
1534
- "name": "Arabic semicolon",
1535
- "code": "U+061B"
1536
- },
1537
- {
1538
- "char": ";",
1539
- "name": "full-width semicolon",
1540
- "code": "U+FF1B"
1541
- },
1542
- {
1543
- "char": "",
1544
- "name": "Ethiopic semicolon",
1545
- "code": "U+1364"
1209
+ }
1210
+ }
1211
+ if (processedText.includes("\n")) continue;
1212
+ let entityStart = processedStart;
1213
+ let entityText = processedText;
1214
+ if (fullText && !trimmed) {
1215
+ const extended = extendBackward(fullText, processedStart);
1216
+ if (extended < processedStart) {
1217
+ entityStart = extended;
1218
+ entityText = fullText.slice(extended, processedStart + processedText.length).trimEnd();
1219
+ }
1220
+ }
1221
+ const clauseTrim = trimLeadingClause(entityText);
1222
+ if (clauseTrim.offset > 0) {
1223
+ entityStart += clauseTrim.offset;
1224
+ entityText = clauseTrim.text;
1225
+ }
1226
+ const getPrefixInfo = (value) => {
1227
+ const prefixEnd = value.lastIndexOf(",") !== -1 ? value.lastIndexOf(",") : value.lastIndexOf(" ");
1228
+ return {
1229
+ prefixEnd,
1230
+ prefixPart: prefixEnd > 0 ? value.slice(0, prefixEnd).replace(/[^a-zA-ZÀ-ž]/g, "") : value.replace(/[^a-zA-ZÀ-ž]/g, "")
1231
+ };
1232
+ };
1233
+ let { prefixEnd, prefixPart } = getPrefixInfo(entityText);
1234
+ let isAllCapsMatch = prefixPart.length > 2 && prefixPart === prefixPart.toUpperCase();
1235
+ if (isAllCapsMatch && fullText) {
1236
+ const lineStart = fullText.lastIndexOf("\n", entityStart);
1237
+ const lineEnd = fullText.indexOf("\n", entityStart + entityText.length);
1238
+ const lineLetters = fullText.slice(lineStart + 1, lineEnd === -1 ? fullText.length : lineEnd).replace(/[^a-zA-ZÀ-ž]/g, "");
1239
+ const upperCount = [...lineLetters].filter((c) => c === c.toUpperCase()).length;
1240
+ if (lineLetters.length > 5 && upperCount / lineLetters.length >= .95) continue;
1241
+ if ((prefixPart.length > 0 ? entityText.slice(0, prefixEnd > 0 ? prefixEnd : entityText.length).trim().split(/\s+/).length : 0) > 3) {
1242
+ entityStart = match.start;
1243
+ entityText = text;
1244
+ ({prefixEnd, prefixPart} = getPrefixInfo(entityText));
1245
+ isAllCapsMatch = prefixPart.length > 2 && prefixPart === prefixPart.toUpperCase();
1246
+ }
1247
+ } else if (isAllCapsMatch) continue;
1248
+ const lastSpace = entityText.lastIndexOf(" ");
1249
+ const rawSuffix = lastSpace !== -1 ? entityText.slice(lastSpace + 1) : "";
1250
+ const suffixClean = rawSuffix.replace(/[.,]/g, "");
1251
+ if (suffixClean.length > 0 && ROMAN_NUMERAL_RE.test(suffixClean)) continue;
1252
+ if (suffixClean.length <= 2 && !/\./.test(rawSuffix) && /[^\x00-\x7F]/.test(entityText.slice(0, lastSpace !== -1 ? lastSpace : entityText.length))) continue;
1253
+ results.push({
1254
+ start: entityStart,
1255
+ end: entityStart + entityText.length,
1256
+ label: "organization",
1257
+ text: entityText,
1258
+ score: .95,
1259
+ source: DETECTION_SOURCES.LEGAL_FORM
1260
+ });
1261
+ }
1262
+ return results;
1263
+ };
1264
+ //#endregion
1265
+ //#region src/detectors/coreference.ts
1266
+ /**
1267
+ * Case-insensitive set of every known legal-form suffix
1268
+ * with whitespace normalized away. Used by the
1269
+ * coreference scan to reject parenthetical define-term
1270
+ * captures whose alias is only a legal-form
1271
+ * abbreviation (e.g. "(« SAS »)" after "ACME SAS"),
1272
+ * which would otherwise propagate generic suffix
1273
+ * mentions as document-wide org aliases.
1274
+ */
1275
+ const isLegalFormAlias = (alias) => {
1276
+ const normalized = alias.replace(/\s+/g, "").toLowerCase();
1277
+ if (normalized.length === 0) return false;
1278
+ for (const suffix of getKnownLegalSuffixes()) if (suffix.replace(/\s+/g, "").toLowerCase() === normalized) return true;
1279
+ return false;
1280
+ };
1281
+ /**
1282
+ * Load coreference definition patterns from per-language
1283
+ * JSON configs in src/data/. Uses the
1284
+ * language manifest for auto-discovery.
1285
+ */
1286
+ const loadDefinitionPatterns = async () => {
1287
+ const patterns = [];
1288
+ const allRows = await loadLanguageConfigs("coreference", (mod) => {
1289
+ return mod.default ?? mod;
1290
+ });
1291
+ for (const rows of allRows) {
1292
+ if (!Array.isArray(rows)) {
1293
+ console.warn("[anonymize] coreference: unexpected config shape, skipping");
1294
+ continue;
1295
+ }
1296
+ for (const row of rows) try {
1297
+ patterns.push({ pattern: new RegExp(row.pattern, row.flags) });
1298
+ } catch (err) {
1299
+ console.warn(`[anonymize] coreference: invalid regex "${row.pattern}":`, err);
1300
+ }
1301
+ }
1302
+ return patterns;
1303
+ };
1304
+ /**
1305
+ * Load generic role terms that should NOT be treated
1306
+ * as PII coreferences. "Prodávající" (Seller),
1307
+ * "Kupující" (Buyer), etc. are legal roles, not
1308
+ * identifying information.
1309
+ */
1310
+ const getRoleStopSet = async (ctx) => {
1311
+ if (ctx.roleStopSet) return ctx.roleStopSet;
1312
+ if (ctx.roleStopSetPromise) return ctx.roleStopSetPromise;
1313
+ const promise = (async () => {
1314
+ let result;
1315
+ try {
1316
+ const mod = await import("./generic-roles.mjs");
1317
+ const data = mod.default ?? mod;
1318
+ result = new Set(data.roles.map((r) => r.toLowerCase()));
1319
+ } catch {
1320
+ result = /* @__PURE__ */ new Set();
1321
+ }
1322
+ ctx.roleStopSet = result;
1323
+ return result;
1324
+ })();
1325
+ ctx.roleStopSetPromise = promise;
1326
+ return promise;
1327
+ };
1328
+ const getDefinitionPatterns = async (ctx) => {
1329
+ if (ctx.corefPatterns) return ctx.corefPatterns;
1330
+ if (ctx.corefPatternsPromise) return ctx.corefPatternsPromise;
1331
+ ctx.corefPatternsPromise = loadDefinitionPatterns();
1332
+ const patterns = await ctx.corefPatternsPromise;
1333
+ if (patterns.length === 0) {
1334
+ ctx.corefPatterns = patterns;
1335
+ if (!ctx.corefLoadAttempted) {
1336
+ ctx.corefLoadAttempted = true;
1337
+ console.warn("[anonymize] coreference: no definition patterns loaded; coreference detection will be inactive");
1338
+ }
1339
+ return patterns;
1340
+ }
1341
+ ctx.corefPatterns = patterns;
1342
+ return patterns;
1343
+ };
1344
+ const SEARCH_WINDOW = 200;
1345
+ /**
1346
+ * Check whether an alias has textual similarity to
1347
+ * the source entity. Prevents roles and structural
1348
+ * terms from being treated as name aliases.
1349
+ *
1350
+ * Three checks (any passes → similar):
1351
+ * 1. Word overlap: a word in the alias appears in the
1352
+ * entity (case-insensitive, min 2 chars)
1353
+ * 2. Initials: alias letters match first letters of
1354
+ * entity words ("TB" ↔ "Tomas Bata")
1355
+ * 3. Substring: alias is a substring of the entity
1356
+ * or vice versa (min 3 chars)
1357
+ */
1358
+ const hasEntitySimilarity = (alias, entityText) => {
1359
+ const aliasLower = alias.toLowerCase();
1360
+ const entityLower = entityText.toLowerCase();
1361
+ if (aliasLower.length >= 3 && entityLower.includes(aliasLower)) return true;
1362
+ if (entityLower.length >= 3 && aliasLower.includes(entityLower)) return true;
1363
+ const aliasWords = aliasLower.split(/[\s.,;:'"()/-]+/).filter((w) => w.length >= 2);
1364
+ const entityWords = entityLower.split(/[\s.,;:'"()/-]+/).filter((w) => w.length >= 2);
1365
+ const entityWordSet = new Set(entityWords);
1366
+ for (const word of aliasWords) if (entityWordSet.has(word)) return true;
1367
+ if (/^[\p{Lu}]+$/u.test(alias) && alias.length >= 2 && alias.length <= entityWords.length) {
1368
+ for (let start = 0; start <= entityWords.length - alias.length; start++) if (entityWords.slice(start, start + alias.length).map((w) => w.charAt(0)).join("") === aliasLower) return true;
1369
+ }
1370
+ return false;
1371
+ };
1372
+ /**
1373
+ * Scan for defined-term patterns near known entities.
1374
+ *
1375
+ * Legal documents universally follow:
1376
+ * "Dr. Heinrich Muller (hereinafter 'the Seller')..."
1377
+ *
1378
+ * After NER detects the entity, this function scans for
1379
+ * definitional patterns within +/-200 chars and extracts
1380
+ * the alias. Returns alias + label pairs that can be added
1381
+ * to the gazetteer for a full-text re-scan.
1382
+ */
1383
+ /**
1384
+ * Labels that can be the source of a coreference alias.
1385
+ * Only parties (person, organization) have defined-term
1386
+ * aliases in legal text. Dates, addresses, IDs do not.
1387
+ */
1388
+ const COREF_SOURCE_LABELS = new Set(["person", "organization"]);
1389
+ const extractDefinedTerms = async (fullText, entities, ctx = defaultContext) => {
1390
+ const [definitionPatterns, roleStops] = await Promise.all([getDefinitionPatterns(ctx), getRoleStopSet(ctx)]);
1391
+ const terms = [];
1392
+ const seen = /* @__PURE__ */ new Set();
1393
+ const sorted = [...entities].sort((a, b) => a.start - b.start);
1394
+ for (const { pattern } of definitionPatterns) {
1395
+ pattern.lastIndex = 0;
1396
+ for (let match = pattern.exec(fullText); match !== null; match = pattern.exec(fullText)) {
1397
+ const alias = match[1]?.trim();
1398
+ if (!alias || alias.length < 2) continue;
1399
+ if (roleStops.has(alias.toLowerCase())) continue;
1400
+ if (isLegalFormAlias(alias)) continue;
1401
+ const defPos = match.index;
1402
+ let bestEntity = null;
1403
+ for (let i = sorted.length - 1; i >= 0; i--) {
1404
+ const e = sorted[i];
1405
+ if (e === void 0) continue;
1406
+ if (e.end > defPos) continue;
1407
+ if (defPos - e.end > SEARCH_WINDOW) break;
1408
+ if (!COREF_SOURCE_LABELS.has(e.label)) continue;
1409
+ bestEntity = e;
1410
+ break;
1411
+ }
1412
+ if (bestEntity === null) continue;
1413
+ const gapText = fullText.slice(bestEntity.end, defPos);
1414
+ if (/(?:;|\.(?=\s*(?:["'„‚(]*\p{Lu}|$)))/u.test(gapText)) continue;
1415
+ if (!hasEntitySimilarity(alias, bestEntity.text)) continue;
1416
+ const key = `${alias.toLowerCase()}::${bestEntity.label}`;
1417
+ if (seen.has(key)) continue;
1418
+ seen.add(key);
1419
+ terms.push({
1420
+ alias,
1421
+ label: bestEntity.label,
1422
+ definitionStart: defPos,
1423
+ sourceText: bestEntity.text
1424
+ });
1425
+ }
1426
+ }
1427
+ return terms;
1428
+ };
1429
+ /**
1430
+ * Check if a character is a Unicode word character
1431
+ * (letter, digit, or combining mark). Used for word
1432
+ * boundary checks in coreference matching.
1433
+ */
1434
+ const isWordChar = (ch) => {
1435
+ if (ch === void 0) return false;
1436
+ return /[\p{L}\p{M}\p{N}]/u.test(ch);
1437
+ };
1438
+ /**
1439
+ * Find all occurrences of defined-term aliases in the
1440
+ * full text. Returns Entity spans for each match.
1441
+ *
1442
+ * Respects word boundaries: "Kupující" must not match
1443
+ * inside "Kupujícímu". A match is valid only if the
1444
+ * character before the start and after the end are NOT
1445
+ * word characters (letter/digit).
1446
+ *
1447
+ * Populates `ctx.corefSourceMap` with entries linking
1448
+ * each coref entity to its source entity text, for
1449
+ * consistent placeholder numbering.
1450
+ */
1451
+ const findCoreferenceSpans = (fullText, terms, ctx = defaultContext) => {
1452
+ const results = [];
1453
+ for (const term of terms) {
1454
+ let searchFrom = 0;
1455
+ while (searchFrom < fullText.length) {
1456
+ const idx = fullText.indexOf(term.alias, searchFrom);
1457
+ if (idx === -1) break;
1458
+ const matchEnd = idx + term.alias.length;
1459
+ const charBefore = idx > 0 ? fullText[idx - 1] : void 0;
1460
+ const charAfter = fullText[matchEnd];
1461
+ if (isWordChar(charBefore) || isWordChar(charAfter)) {
1462
+ searchFrom = idx + 1;
1463
+ continue;
1464
+ }
1465
+ const entity = {
1466
+ start: idx,
1467
+ end: matchEnd,
1468
+ label: term.label,
1469
+ text: term.alias,
1470
+ score: .95,
1471
+ source: DETECTION_SOURCES.COREFERENCE
1472
+ };
1473
+ ctx.corefSourceMap.set(corefKey(entity), term.sourceText);
1474
+ results.push(entity);
1475
+ searchFrom = matchEnd;
1476
+ }
1477
+ }
1478
+ return results;
1479
+ };
1480
+ //#endregion
1481
+ //#region src/detectors/gazetteer.ts
1482
+ const MAX_EDIT_DISTANCE = 2;
1483
+ const MIN_FUZZY_LENGTH = 4;
1484
+ const MAX_PREFIX_OVERSHOOT = 7;
1485
+ /**
1486
+ * Collect all searchable strings (canonical + variants)
1487
+ * from gazetteer entries, mapped to their labels and
1488
+ * entry IDs.
1489
+ */
1490
+ const buildSearchTerms = (entries) => {
1491
+ const terms = /* @__PURE__ */ new Map();
1492
+ for (const entry of entries) {
1493
+ const meta = {
1494
+ label: entry.label,
1495
+ entryId: entry.id
1496
+ };
1497
+ terms.set(entry.canonical, meta);
1498
+ for (const variant of entry.variants) terms.set(variant, meta);
1499
+ }
1500
+ return terms;
1501
+ };
1502
+ /**
1503
+ * Build TextSearch-compatible patterns from gazetteer
1504
+ * entries. Returns:
1505
+ * - Exact literal patterns for all terms
1506
+ * - Fuzzy patterns (distance: 2) for terms >= 4 chars
1507
+ * - Parallel metadata arrays for post-processing
1508
+ *
1509
+ * Patterns are ordered: all exact first, then all
1510
+ * fuzzy. The isFuzzy array marks which are which.
1511
+ */
1512
+ const buildGazetteerPatterns = (entries) => {
1513
+ const terms = buildSearchTerms(entries);
1514
+ const patterns = [];
1515
+ const labels = [];
1516
+ const isFuzzy = [];
1517
+ for (const [term, meta] of terms) {
1518
+ patterns.push({
1519
+ pattern: term,
1520
+ literal: true,
1521
+ wholeWords: false
1522
+ });
1523
+ labels.push(meta.label);
1524
+ isFuzzy.push(false);
1525
+ }
1526
+ for (const [term, meta] of terms) {
1527
+ if (term.length < MIN_FUZZY_LENGTH) continue;
1528
+ patterns.push({
1529
+ pattern: term,
1530
+ distance: MAX_EDIT_DISTANCE
1531
+ });
1532
+ labels.push(meta.label);
1533
+ isFuzzy.push(true);
1534
+ }
1535
+ return {
1536
+ patterns,
1537
+ data: {
1538
+ labels,
1539
+ isFuzzy
1540
+ }
1541
+ };
1542
+ };
1543
+ /**
1544
+ * Process gazetteer matches from the unified literal
1545
+ * search. Receives all matches; filters to the
1546
+ * gazetteer slice via sliceStart/sliceEnd.
1547
+ *
1548
+ * Exact matches get score 0.9; fuzzy matches get
1549
+ * 0.85. Fuzzy matches that overlap an exact match
1550
+ * are dropped.
1551
+ *
1552
+ * For exact matches, attempts prefix extension for
1553
+ * legal suffixes ("a.s.", "GmbH", "s.r.o." after
1554
+ * the matched term).
1555
+ */
1556
+ const processGazetteerMatches = (allMatches, sliceStart, sliceEnd, fullText, data) => {
1557
+ const results = [];
1558
+ const exactSpans = [];
1559
+ for (const match of allMatches) {
1560
+ const idx = match.pattern;
1561
+ if (idx < sliceStart || idx >= sliceEnd) continue;
1562
+ const localIdx = idx - sliceStart;
1563
+ if (data.isFuzzy[localIdx]) continue;
1564
+ const label = data.labels[localIdx];
1565
+ if (!label) continue;
1566
+ const extended = tryPrefixExtension(fullText, match.start, match.end);
1567
+ const end = extended?.end ?? match.end;
1568
+ const text = extended?.text ?? fullText.slice(match.start, match.end);
1569
+ exactSpans.push({
1570
+ start: match.start,
1571
+ end
1572
+ });
1573
+ results.push({
1574
+ start: match.start,
1575
+ end,
1576
+ label,
1577
+ text,
1578
+ score: .9,
1579
+ source: DETECTION_SOURCES.GAZETTEER
1580
+ });
1581
+ }
1582
+ for (const match of allMatches) {
1583
+ const idx = match.pattern;
1584
+ if (idx < sliceStart || idx >= sliceEnd) continue;
1585
+ const localIdx = idx - sliceStart;
1586
+ if (!data.isFuzzy[localIdx]) continue;
1587
+ if (match.distance === 0) continue;
1588
+ const label = data.labels[localIdx];
1589
+ if (!label) continue;
1590
+ if (exactSpans.some((e) => match.start < e.end && match.end > e.start)) continue;
1591
+ const matchText = fullText.slice(match.start, match.end);
1592
+ results.push({
1593
+ start: match.start,
1594
+ end: match.end,
1595
+ label,
1596
+ text: matchText,
1597
+ score: .85,
1598
+ source: DETECTION_SOURCES.GAZETTEER
1599
+ });
1600
+ }
1601
+ return results;
1602
+ };
1603
+ /**
1604
+ * Try to extend an exact match to capture one
1605
+ * trailing token (max 6 chars) that may be a legal
1606
+ * entity suffix (e.g., "a.s.", "GmbH", "s.r.o.").
1607
+ *
1608
+ * Does not validate the token against a legal-forms
1609
+ * list; false extensions are filtered by mergeAndDedup
1610
+ * when a legal-form detector produces a competing
1611
+ * entity with the correct span.
1612
+ */
1613
+ const tryPrefixExtension = (fullText, start, end) => {
1614
+ const maxEnd = Math.min(end + MAX_PREFIX_OVERSHOOT, fullText.length);
1615
+ if (maxEnd <= end + 1) return null;
1616
+ const after = fullText.slice(end, maxEnd);
1617
+ if (!after.startsWith(" ")) return null;
1618
+ const nextSpace = after.indexOf(" ", 1);
1619
+ const suffixEnd = nextSpace !== -1 ? nextSpace : after.length;
1620
+ if (suffixEnd <= 1) return null;
1621
+ const newEnd = end + suffixEnd;
1622
+ return {
1623
+ end: newEnd,
1624
+ text: fullText.slice(start, newEnd)
1625
+ };
1626
+ };
1627
+ //#endregion
1628
+ //#region src/util/text.ts
1629
+ /**
1630
+ * Shared text utilities for detectors.
1631
+ *
1632
+ * Extracted from names.ts and deny-list.ts to avoid
1633
+ * duplicating regex constants and helper functions.
1634
+ */
1635
+ /** Matches a string that starts with an uppercase letter. */
1636
+ const UPPER_START_RE = /^\p{Lu}/u;
1637
+ /** Matches a string consisting entirely of uppercase letters. */
1638
+ const ALL_UPPER_RE = /^\p{Lu}+$/u;
1639
+ const SENTENCE_END_RE = /[.!?]/;
1640
+ /**
1641
+ * Detect whether a position is at the start of a sentence.
1642
+ * Looks backward past whitespace for sentence-ending
1643
+ * punctuation (.!?). Position 0 and positions preceded
1644
+ * only by whitespace are considered sentence starts.
1645
+ */
1646
+ const isSentenceStart = (text, pos) => {
1647
+ if (pos === 0) return true;
1648
+ let i = pos - 1;
1649
+ while (i >= 0 && /\s/.test(text[i] ?? "")) i--;
1650
+ if (i < 0) return true;
1651
+ return SENTENCE_END_RE.test(text[i] ?? "");
1652
+ };
1653
+ //#endregion
1654
+ //#region src/detectors/names.ts
1655
+ const getCorpus = (ctx) => ctx.nameCorpus;
1656
+ const getNameCorpusFirstNames = (ctx = defaultContext) => ctx.nameCorpus?.firstNamesList ?? [];
1657
+ const getNameCorpusSurnames = (ctx = defaultContext) => ctx.nameCorpus?.surnamesList ?? [];
1658
+ const getNameCorpusTitles = (ctx = defaultContext) => ctx.nameCorpus?.titlesList ?? [];
1659
+ /**
1660
+ * Load name corpus data from injected dictionaries
1661
+ * and legacy config files. Merges all sources.
1662
+ *
1663
+ * Safe to call multiple times; only loads once per
1664
+ * context. Must be called before detectNameCorpus or
1665
+ * the getNameCorpus*() accessors are used.
1666
+ *
1667
+ * @param dictionaries Optional pre-loaded dictionaries
1668
+ * with per-language first names and surnames. When
1669
+ * omitted, only legacy config files are used.
1670
+ */
1671
+ const initNameCorpus = (ctx = defaultContext, dictionaries) => {
1672
+ if (ctx.nameCorpusPromise) return ctx.nameCorpusPromise;
1673
+ const promise = (async () => {
1674
+ try {
1675
+ const [legacyFirstMod, legacySurnameMod, titleMod, exclusionMod] = await Promise.all([
1676
+ import("./names-first.mjs"),
1677
+ import("./names-surnames.mjs"),
1678
+ import("./names-title-tokens.mjs"),
1679
+ import("./names-exclusions.mjs")
1680
+ ]);
1681
+ const firstNames = [...legacyFirstMod.default.names];
1682
+ if (dictionaries?.firstNames) for (const names of Object.values(dictionaries.firstNames)) for (const name of names) firstNames.push(name);
1683
+ const surnames = [...legacySurnameMod.default.names];
1684
+ if (dictionaries?.surnames) for (const names of Object.values(dictionaries.surnames)) for (const name of names) surnames.push(name);
1685
+ const dedup = (arr) => {
1686
+ const seen = /* @__PURE__ */ new Set();
1687
+ const result = [];
1688
+ for (const item of arr) {
1689
+ if (seen.has(item)) continue;
1690
+ seen.add(item);
1691
+ result.push(item);
1546
1692
  }
1547
- ]
1548
- },
1549
- "underscore": {
1550
- "description": "Underscore variants",
1551
- "chars": [{
1552
- "char": "_",
1553
- "name": "low line",
1554
- "code": "U+005F"
1555
- }, {
1556
- "char": "_",
1557
- "name": "full-width low line",
1558
- "code": "U+FF3F"
1559
- }]
1693
+ return result;
1694
+ };
1695
+ const dedupFirst = dedup(firstNames);
1696
+ const dedupSurnames = dedup(surnames);
1697
+ const titles = titleMod.default.tokens;
1698
+ const exclusions = exclusionMod.default.words;
1699
+ ctx.nameCorpus = {
1700
+ firstNames: Object.freeze(new Set(dedupFirst)),
1701
+ surnames: Object.freeze(new Set(dedupSurnames)),
1702
+ titleTokens: Object.freeze(new Set(titles)),
1703
+ excludedWords: Object.freeze(new Set(exclusions)),
1704
+ firstNamesList: Object.freeze(dedupFirst),
1705
+ surnamesList: Object.freeze(dedupSurnames),
1706
+ titlesList: Object.freeze(titles),
1707
+ excludedList: Object.freeze(exclusions)
1708
+ };
1709
+ } catch (err) {
1710
+ ctx.nameCorpusPromise = null;
1711
+ console.warn("[anonymize] Failed to load name corpus JSON — name detection disabled:", err);
1712
+ }
1713
+ })();
1714
+ ctx.nameCorpusPromise = promise;
1715
+ return promise;
1716
+ };
1717
+ const INFLECTION_SUFFIXES = [
1718
+ "ovi",
1719
+ "em",
1720
+ "om",
1721
+ "ou",
1722
+ "é",
1723
+ "a",
1724
+ "u"
1725
+ ];
1726
+ /**
1727
+ * Strip common Czech/Slovak case suffixes from a token.
1728
+ * Returns candidate base forms if stripping produces a
1729
+ * plausible name (capitalised, length >= 3).
1730
+ *
1731
+ * For the "-ou" instrumental feminine suffix, also yields
1732
+ * base + "a" (e.g., "Editou" → "Edit" and "Edita")
1733
+ * because Czech feminine names decline -a → -ou.
1734
+ */
1735
+ const stripInflection = (token) => {
1736
+ const candidates = [];
1737
+ for (const suffix of INFLECTION_SUFFIXES) if (token.length > suffix.length + 2 && token.endsWith(suffix)) {
1738
+ const base = token.slice(0, -suffix.length);
1739
+ if (/^\p{Lu}/u.test(base)) {
1740
+ candidates.push(base);
1741
+ if (suffix === "ou" || suffix === "é" || suffix === "u") candidates.push(`${base}a`);
1742
+ }
1743
+ }
1744
+ return candidates;
1745
+ };
1746
+ const TOKEN_TYPE = {
1747
+ NAME: "name",
1748
+ SURNAME: "surname",
1749
+ TITLE: "title",
1750
+ ABBREVIATION: "abbreviation",
1751
+ CAPITALIZED: "capitalized",
1752
+ OTHER: "other"
1753
+ };
1754
+ const PERSON_CHAIN_BREAK_RE$1 = /[!?;:]/u;
1755
+ const isInitialContinuationGap$1 = (text, gap) => /^\p{Lu}$/u.test(text) && /^\.[^\S\n]{1,2}$/u.test(gap) || /^[^\S\n]{1,2}(?:\p{Lu}\.[^\S\n]{1,2})+$/u.test(gap);
1756
+ /**
1757
+ * Check if a token is in the first-name set, either
1758
+ * directly or after stripping Czech/Slovak inflection.
1759
+ */
1760
+ const isFirstNameToken = (token, corpus) => {
1761
+ if (corpus.firstNames.has(token)) return true;
1762
+ return stripInflection(token).some((b) => corpus.firstNames.has(b));
1763
+ };
1764
+ /**
1765
+ * Check if a token is in the surname set, either
1766
+ * directly or after stripping Czech/Slovak inflection.
1767
+ */
1768
+ const isSurnameToken = (token, corpus) => {
1769
+ if (corpus.surnames.has(token)) return true;
1770
+ return stripInflection(token).some((b) => corpus.surnames.has(b));
1771
+ };
1772
+ /**
1773
+ * Check if a token looks like a single-letter
1774
+ * abbreviation: "J.", "M.", etc.
1775
+ */
1776
+ const isAbbreviation = (token) => token.length === 2 && /^\p{Lu}$/u.test(token[0] ?? "") && token[1] === ".";
1777
+ const segmenter$1 = new Intl.Segmenter(void 0, { granularity: "word" });
1778
+ /**
1779
+ * Split text into word segments using Intl.Segmenter.
1780
+ * Only returns segments flagged as words.
1781
+ */
1782
+ const segmentWords = (fullText) => {
1783
+ const words = [];
1784
+ for (const seg of segmenter$1.segment(fullText)) if (seg.isWordLike) words.push({
1785
+ text: seg.segment,
1786
+ start: seg.index,
1787
+ end: seg.index + seg.segment.length
1788
+ });
1789
+ return words;
1790
+ };
1791
+ /** NAME or SURNAME — both represent corpus-matched tokens */
1792
+ const isCorpusMatch = (type) => type === TOKEN_TYPE.NAME || type === TOKEN_TYPE.SURNAME;
1793
+ const classifyToken = (word, corpus) => {
1794
+ const { text, start, end } = word;
1795
+ const lower = text.toLowerCase();
1796
+ const stripped = text.endsWith(".") ? text.slice(0, -1).toLowerCase() : lower;
1797
+ if (corpus.titleTokens.has(stripped)) return {
1798
+ text,
1799
+ type: TOKEN_TYPE.TITLE,
1800
+ start,
1801
+ end
1802
+ };
1803
+ if (isAbbreviation(text)) return {
1804
+ text,
1805
+ type: TOKEN_TYPE.ABBREVIATION,
1806
+ start,
1807
+ end
1808
+ };
1809
+ if (corpus.excludedWords.has(lower)) return {
1810
+ text,
1811
+ type: TOKEN_TYPE.OTHER,
1812
+ start,
1813
+ end
1814
+ };
1815
+ if (text.length < 3) return {
1816
+ text,
1817
+ type: TOKEN_TYPE.OTHER,
1818
+ start,
1819
+ end
1820
+ };
1821
+ if (text.length > 3 && ALL_UPPER_RE.test(text)) return {
1822
+ text,
1823
+ type: TOKEN_TYPE.OTHER,
1824
+ start,
1825
+ end
1826
+ };
1827
+ if (!UPPER_START_RE.test(text)) return {
1828
+ text,
1829
+ type: TOKEN_TYPE.OTHER,
1830
+ start,
1831
+ end
1832
+ };
1833
+ if (isFirstNameToken(text, corpus)) return {
1834
+ text,
1835
+ type: TOKEN_TYPE.NAME,
1836
+ start,
1837
+ end
1838
+ };
1839
+ if (isSurnameToken(text, corpus)) return {
1840
+ text,
1841
+ type: TOKEN_TYPE.SURNAME,
1842
+ start,
1843
+ end
1844
+ };
1845
+ return {
1846
+ text,
1847
+ type: TOKEN_TYPE.CAPITALIZED,
1848
+ start,
1849
+ end
1850
+ };
1851
+ };
1852
+ /**
1853
+ * Detect person names by looking up tokens against the
1854
+ * name corpus, then chaining adjacent name-like tokens.
1855
+ *
1856
+ * Requires initNameCorpus() to have been called first.
1857
+ * If not initialized, returns an empty array.
1858
+ *
1859
+ * Scoring:
1860
+ * TITLE + NAME/SURNAME → 0.95
1861
+ * NAME + NAME/SURNAME → 0.9
1862
+ * SURNAME + NAME/SURNAME → 0.9
1863
+ * NAME + CAPITALIZED → 0.7
1864
+ * ABBREVIATION + NAME → 0.7
1865
+ * Standalone NAME → 0.5 (low confidence)
1866
+ * Standalone SURNAME → skip (too ambiguous)
1867
+ */
1868
+ const detectNameCorpus = (fullText, ctx = defaultContext) => {
1869
+ const corpus = getCorpus(ctx);
1870
+ if (!corpus) return [];
1871
+ const tokens = segmentWords(fullText).map((w) => classifyToken(w, corpus));
1872
+ const entities = [];
1873
+ const consumed = /* @__PURE__ */ new Set();
1874
+ for (let i = 0; i < tokens.length; i++) {
1875
+ if (consumed.has(i)) continue;
1876
+ const token = tokens[i];
1877
+ if (!token) continue;
1878
+ if (token.type !== TOKEN_TYPE.TITLE && token.type !== TOKEN_TYPE.NAME && token.type !== TOKEN_TYPE.SURNAME && token.type !== TOKEN_TYPE.ABBREVIATION) continue;
1879
+ const MAX_CHAIN = 5;
1880
+ const chain = [token];
1881
+ let j = i + 1;
1882
+ while (j < tokens.length && chain.length < MAX_CHAIN) {
1883
+ const next = tokens[j];
1884
+ if (!next) break;
1885
+ const prev = chain.at(-1);
1886
+ if (prev) {
1887
+ const gap = fullText.slice(prev.end, next.start);
1888
+ const breaksOnPeriod = gap.includes(".") && !isInitialContinuationGap$1(prev.text, gap);
1889
+ if (gap.includes("\n") || PERSON_CHAIN_BREAK_RE$1.test(gap) || breaksOnPeriod) break;
1890
+ }
1891
+ if (next.type === TOKEN_TYPE.NAME || next.type === TOKEN_TYPE.SURNAME || next.type === TOKEN_TYPE.TITLE || next.type === TOKEN_TYPE.ABBREVIATION || next.type === TOKEN_TYPE.CAPITALIZED) {
1892
+ chain.push(next);
1893
+ j++;
1894
+ } else break;
1895
+ }
1896
+ const hasTitle = chain.some((t) => t.type === TOKEN_TYPE.TITLE);
1897
+ const hasCorpusName = chain.some((t) => isCorpusMatch(t.type));
1898
+ const hasFirstName = chain.some((t) => t.type === TOKEN_TYPE.NAME);
1899
+ const hasAbbreviation = chain.some((t) => t.type === TOKEN_TYPE.ABBREVIATION);
1900
+ const corpusCount = chain.filter((t) => isCorpusMatch(t.type)).length;
1901
+ const capitalizedCount = chain.filter((t) => t.type === TOKEN_TYPE.CAPITALIZED).length;
1902
+ let score = 0;
1903
+ if (hasTitle && hasCorpusName) score = .95;
1904
+ else if (corpusCount >= 2) score = .9;
1905
+ else if (hasCorpusName && capitalizedCount > 0) score = .7;
1906
+ else if (hasAbbreviation && hasCorpusName) score = .7;
1907
+ else if (hasFirstName && chain.length === 1) {
1908
+ if (isSentenceStart(fullText, token.start)) continue;
1909
+ score = .5;
1910
+ } else if (!hasFirstName && chain.length === 1 && chain[0]?.type === TOKEN_TYPE.SURNAME) continue;
1911
+ else if (hasTitle && chain.length === 1) continue;
1912
+ else {
1913
+ if (!hasCorpusName) continue;
1914
+ score = .5;
1560
1915
  }
1916
+ const first = chain.at(0);
1917
+ const last = chain.at(-1);
1918
+ if (!first || !last) continue;
1919
+ const start = first.start;
1920
+ const end = last.end;
1921
+ const text = fullText.slice(start, end);
1922
+ for (let k = i; k < i + chain.length; k++) consumed.add(k);
1923
+ entities.push({
1924
+ start,
1925
+ end,
1926
+ label: "person",
1927
+ text,
1928
+ score,
1929
+ source: DETECTION_SOURCES.REGEX
1930
+ });
1561
1931
  }
1932
+ return entities;
1562
1933
  };
1563
1934
  //#endregion
1564
- //#region src/util/char-groups.ts
1565
- /**
1566
- * Centralized Unicode character equivalence groups.
1567
- *
1568
- * Centralized Unicode character equivalence groups.
1569
- *
1570
- * Provides helpers to build regex character classes that
1571
- * match all typographic variants of a character type
1572
- * (dashes, spaces, quotes, etc.).
1573
- *
1574
- * The JSON is statically imported so the bundler inlines
1575
- * it, avoiding a runtime require() that breaks browsers.
1576
- */
1577
- /** Chars that need escaping inside a regex char class. */
1578
- const REGEX_CLASS_SPECIAL = /[\\\]^-]/;
1579
- const escapeForCharClass = (ch) => REGEX_CLASS_SPECIAL.test(ch) ? `\\${ch}` : ch;
1580
- const config = char_groups_default;
1935
+ //#region src/config/titles.ts
1581
1936
  /**
1582
- * Get the raw characters for a named group.
1583
- * Throws if the group does not exist.
1937
+ * Academic and professional title prefixes.
1938
+ * Plain text; the detector auto-escapes for regex.
1939
+ * Sorted longest-first at build time.
1584
1940
  */
1585
- const charSet = (group) => {
1586
- const g = config.groups[group];
1587
- if (!g) throw new Error(`Unknown char group: "${group}"`);
1588
- return g.chars.map((entry) => entry.char);
1589
- };
1941
+ const TITLE_PREFIXES = [
1942
+ "Ing.",
1943
+ "Mgr.",
1944
+ "MgA.",
1945
+ "Bc.",
1946
+ "BcA.",
1947
+ "JUDr.",
1948
+ "MUDr.",
1949
+ "MVDr.",
1950
+ "MDDr.",
1951
+ "PhDr.",
1952
+ "RNDr.",
1953
+ "PaedDr.",
1954
+ "ThDr.",
1955
+ "ThLic.",
1956
+ "ICDr.",
1957
+ "RSDr.",
1958
+ "PharmDr.",
1959
+ "artD.",
1960
+ "akad.",
1961
+ "doc.",
1962
+ "prof.",
1963
+ "ao. Univ.-Prof.",
1964
+ "o. Univ.-Prof.",
1965
+ "Univ.-Prof.",
1966
+ "Hon.-Prof.",
1967
+ "em. Prof.",
1968
+ "Dr. med. dent.",
1969
+ "Dr. med. vet.",
1970
+ "Dr. med.",
1971
+ "Dr. rer. nat.",
1972
+ "Dr. rer. soc.",
1973
+ "Dr. rer. pol.",
1974
+ "Dr. sc. tech.",
1975
+ "Dr. sc. nat.",
1976
+ "Dr. sc. hum.",
1977
+ "Dr. iur.",
1978
+ "Dr. jur.",
1979
+ "Dr. theol.",
1980
+ "Dr. oec.",
1981
+ "Dr. techn.",
1982
+ "Dr. h. c.",
1983
+ "Dr. phil.",
1984
+ "Dr.-Ing.",
1985
+ "Dr. Ing.",
1986
+ "Dr.",
1987
+ "Dipl.-Wirt.-Ing.",
1988
+ "Dipl.-Betriebsw.",
1989
+ "Dipl.-Inform.",
1990
+ "Dipl.-Volksw.",
1991
+ "Dipl.-Psych.",
1992
+ "Dipl.-Phys.",
1993
+ "Dipl.-Chem.",
1994
+ "Dipl.-Biol.",
1995
+ "Dipl.-Math.",
1996
+ "Dipl.-Päd.",
1997
+ "Dipl.-Soz.",
1998
+ "Dipl.-Kfm.",
1999
+ "Dipl.-Jur.",
2000
+ "Dipl. Ing.",
2001
+ "Dipl.-Ing.",
2002
+ "Mag. rer. soc. oec.",
2003
+ "Mag. rer. nat.",
2004
+ "Mag. phil.",
2005
+ "Mag. iur.",
2006
+ "Mag. arch.",
2007
+ "Mag. pharm.",
2008
+ "Mag. (FH)",
2009
+ "Mag.",
2010
+ "Bakk. rer. nat.",
2011
+ "Bakk. techn.",
2012
+ "Bakk. phil.",
2013
+ "Bakk.",
2014
+ "Lic. phil.",
2015
+ "Lic. iur.",
2016
+ "Lic. oec.",
2017
+ "Lic. theol.",
2018
+ "Lic.",
2019
+ "Priv.-Doz.",
2020
+ "PD",
2021
+ "RA"
2022
+ ];
1590
2023
  /**
1591
- * Build a regex character class string for a named
1592
- * group. E.g., charClass("dash") returns a string
1593
- * like "[-\u2013\u2014\u2010\u2011\u2212\u2043]".
1594
- *
1595
- * The hyphen-minus is placed first so it is treated
1596
- * as a literal, not a range indicator.
2024
+ * Courtesy/honorific titles that precede a person's
2025
+ * name. Sorted alphabetically. The detector escapes
2026
+ * dots and adds \b for entries in HONORIFIC_BOUNDARY.
1597
2027
  */
1598
- const charClass = (group) => {
1599
- return `[${[...charSet(group)].sort((a, b) => {
1600
- if (a === "-") return -1;
1601
- if (b === "-") return 1;
1602
- return a.localeCompare(b);
1603
- }).map(escapeForCharClass).join("")}]`;
1604
- };
2028
+ const HONORIFICS = [
2029
+ "Avv.",
2030
+ "Dame",
2031
+ "Doamna",
2032
+ "Domnul",
2033
+ "Don",
2034
+ "Doña",
2035
+ "Dott.",
2036
+ "Judge",
2037
+ "Justice",
2038
+ "Lady",
2039
+ "Lord",
2040
+ "M.",
2041
+ "Madame",
2042
+ "Mademoiselle",
2043
+ "Maître",
2044
+ "Me",
2045
+ "Messrs",
2046
+ "Miss",
2047
+ "Mlle",
2048
+ "Mme",
2049
+ "Monsieur",
2050
+ "Mr",
2051
+ "Mrs",
2052
+ "Ms",
2053
+ "Pr",
2054
+ "Pr.",
2055
+ "President",
2056
+ "Señor",
2057
+ "Señora",
2058
+ "Sig.",
2059
+ "Sig.ra",
2060
+ "Signor",
2061
+ "Signora",
2062
+ "Signorina",
2063
+ "Sir",
2064
+ "Sr.",
2065
+ "Sra."
2066
+ ];
1605
2067
  /**
1606
- * Return the inner content of a character class (without
1607
- * the surrounding brackets). Useful for embedding a group
1608
- * inside a larger character class, e.g.:
1609
- * `[\\s&,.${charClassInner("dash")}]`
2068
+ * Honorifics that need \b word-boundary anchors
2069
+ * (short or common words that could match mid-word).
1610
2070
  */
1611
- const charClassInner = (group) => {
1612
- return [...charSet(group)].sort((a, b) => {
1613
- if (a === "-") return -1;
1614
- if (b === "-") return 1;
1615
- return a.localeCompare(b);
1616
- }).map(escapeForCharClass).join("");
1617
- };
1618
- /** Regex char class matching all dash variants. */
1619
- const DASH = charClass("dash");
2071
+ const HONORIFIC_BOUNDARY = new Set([
2072
+ "Don",
2073
+ "Doña",
2074
+ "M.",
2075
+ "Me",
2076
+ "Pr",
2077
+ "Pr.",
2078
+ "Señor",
2079
+ "Señora"
2080
+ ]);
1620
2081
  /**
1621
- * Inner content of the dash char class (no brackets).
1622
- * For embedding inside a larger character class:
1623
- * `[\\s&,.${DASH_INNER}]`
2082
+ * Post-nominal degrees (comma or space separated after name).
2083
+ * Plain text; the detector auto-escapes for regex.
1624
2084
  */
1625
- const DASH_INNER = charClassInner("dash");
1626
- charClass("space");
1627
- charClass("quote-double");
1628
- charClass("quote-single");
1629
- charClass("slash");
1630
- charClass("dot");
1631
- charClass("colon");
2085
+ const POST_NOMINALS = [
2086
+ "Ph.D.",
2087
+ "Ph.D",
2088
+ "CSc.",
2089
+ "DrSc.",
2090
+ "ArtD.",
2091
+ "D.Phil.",
2092
+ "DPhil.",
2093
+ "MPhil.",
2094
+ "MBA",
2095
+ "MPA",
2096
+ "LL.M.",
2097
+ "LL.B.",
2098
+ "M.Sc.",
2099
+ "B.Sc.",
2100
+ "MSc.",
2101
+ "BSc.",
2102
+ "M.Eng.",
2103
+ "B.Eng.",
2104
+ "M.A.",
2105
+ "B.A.",
2106
+ "JCD",
2107
+ "JD",
2108
+ "DiS.",
2109
+ "ACCA",
2110
+ "FCCA",
2111
+ "CIPM",
2112
+ "CIPT",
2113
+ "CIPP/E",
2114
+ "CIPP",
2115
+ "KC",
2116
+ "QC"
2117
+ ];
1632
2118
  //#endregion
1633
2119
  //#region src/detectors/regex.ts
1634
2120
  const MIN_PHONE_LENGTH = 7;
1635
2121
  const MIN_MONTH_NAME_LENGTH = 3;
1636
2122
  const escapeTitle = (title) => title.replace(/[.*+?^${}()|[\]\\]/g, "\\$&").replace(/\s+/g, "\\s*");
1637
2123
  /** Escape for use inside a regex alternation. */
1638
- const escapeRegex$1 = (s) => s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
2124
+ const escapeRegex$2 = (s) => s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
1639
2125
  /** Escape for use inside a regex character class. */
1640
2126
  const escapeCharClass = (s) => s.replace(/[\]\\^-]/g, "\\$&");
1641
2127
  const TITLE_PREFIX = TITLE_PREFIXES.toSorted((a, b) => b.length - a.length).map(escapeTitle).join("|");
1642
2128
  const POST_NOMINAL = POST_NOMINALS.toSorted((a, b) => b.length - a.length).map(escapeTitle).join("|");
1643
- const NAME_WORD = `[A-ZÁČĎÉĚÍŇÓŘŠŤÚŮÝŽ][a-záčďéěíňóřšťúůýžäöüß]+`;
2129
+ const NAME_WORD = `\\p{Lu}\\p{Ll}+`;
1644
2130
  const PARTICLE = "(?:van der|van den|de la|della|von|van|dos|ibn|ben|bin|del|zum|zur|ten|ter|da|de|di|al|el|le|la|zu|af|av)";
1645
2131
  const SP = "[^\\S\\n\\t]";
1646
2132
  /** Honorific alternation built from titles.ts config. */
1647
2133
  const HONORIFIC_ALT = [...HONORIFICS].toSorted((a, b) => b.length - a.length).map((h) => {
1648
- const escaped = escapeRegex$1(h);
2134
+ const escaped = escapeRegex$2(h);
1649
2135
  return HONORIFIC_BOUNDARY.has(h) ? `\\b${escaped}` : escaped;
1650
2136
  }).join("|");
1651
2137
  const toEntry = (validator, label, score) => {
@@ -1670,8 +2156,6 @@ const toEntry = (validator, label, score) => {
1670
2156
  const STDNUM_ENTRIES = [
1671
2157
  toEntry(hu.vat, "tax identification number", .95),
1672
2158
  toEntry(it.codiceFiscale, "national identification number", .95),
1673
- toEntry(es.dni, "national identification number", .9),
1674
- toEntry(es.nie, "national identification number", .95),
1675
2159
  toEntry(se.personnummer, "national identification number", .9),
1676
2160
  toEntry(ro.cnp, "national identification number", .95),
1677
2161
  toEntry(fr.nir, "social security number", .9),
@@ -1697,7 +2181,6 @@ const STDNUM_ENTRIES = [
1697
2181
  toEntry(fi.ytunnus, "registration number", .9),
1698
2182
  toEntry(bg.vat, "tax identification number", .95),
1699
2183
  toEntry(sk.dic, "tax identification number", .95),
1700
- toEntry(es.cif, "registration number", .95),
1701
2184
  toEntry(es.vat, "tax identification number", .95),
1702
2185
  toEntry(es.nss, "social security number", .9),
1703
2186
  toEntry(fr.tva, "tax identification number", .95),
@@ -1719,7 +2202,9 @@ const STDNUM_ENTRIES = [
1719
2202
  toEntry(ee.ik, "national identification number", .9),
1720
2203
  toEntry(cy.vat, "tax identification number", .95),
1721
2204
  toEntry(mt.vat, "tax identification number", .95),
1722
- toEntry(lu.vat, "tax identification number", .95)
2205
+ toEntry(lu.vat, "tax identification number", .95),
2206
+ toEntry(br.cpf, "tax identification number", .95),
2207
+ toEntry(br.cnpj, "tax identification number", .95)
1723
2208
  ].filter((e) => e !== null);
1724
2209
  const TITLED_PERSON = {
1725
2210
  pattern: `(?:${TITLE_PREFIX})(?:${SP}+(?:${TITLE_PREFIX}))*${SP}+(?:${NAME_WORD})(?:${SP}{1,4}(?:${PARTICLE}${SP}+)?${NAME_WORD}){1,3}(?:,?${SP}+(?:${POST_NOMINAL})(?:,?${SP}+(?:${POST_NOMINAL}))*)?`,
@@ -1731,6 +2216,11 @@ const HONORIFIC_PERSON = {
1731
2216
  label: "person",
1732
2217
  score: .95
1733
2218
  };
2219
+ const POSTNOMINAL_PERSON = {
2220
+ pattern: `${NAME_WORD}(?:(?:${SP}|-){1,2}(?:${PARTICLE}${SP}+)?${NAME_WORD}){1,3},?${SP}+(?:KC|QC)\\b`,
2221
+ label: "person",
2222
+ score: .95
2223
+ };
1734
2224
  const IBAN = {
1735
2225
  pattern: "\\b[A-Z]{2}\\d{2}\\s?[\\dA-Z]{4}\\s?[\\dA-Z]{4}\\s?[\\dA-Z]{4}\\s?[\\dA-Z]{4}\\s?[\\dA-Z]{0,14}\\b",
1736
2226
  label: "iban",
@@ -1768,7 +2258,7 @@ const CREDIT_CARD = {
1768
2258
  };
1769
2259
  const CZ_BIRTH_NUMBER = {
1770
2260
  pattern: `\\b\\d{6}/\\d{3,4}\\b`,
1771
- label: "czech birth number",
2261
+ label: "birth number",
1772
2262
  score: 1,
1773
2263
  validator: cz.rc
1774
2264
  };
@@ -1802,6 +2292,51 @@ const CZ_POSTAL = {
1802
2292
  label: "address",
1803
2293
  score: .7
1804
2294
  };
2295
+ const ES_POSTAL = {
2296
+ pattern: "\\b(?:C\\.?P\\.?|[Cc][óo]digo[^\\S\\n]+postal)[^\\S\\n]{0,3}:?[^\\S\\n]{0,3}\\d{5}\\b",
2297
+ label: "address",
2298
+ score: .7
2299
+ };
2300
+ const ES_DNI = {
2301
+ pattern: `\\b\\d{8}-?[A-Za-z]\\b`,
2302
+ label: "national identification number",
2303
+ score: .9,
2304
+ validator: es.dni
2305
+ };
2306
+ const ES_NIE = {
2307
+ pattern: `\\b[XYZxyz]-?\\d{7}-?[A-Za-z]\\b`,
2308
+ label: "national identification number",
2309
+ score: .95,
2310
+ validator: es.nie
2311
+ };
2312
+ const ES_CIF = {
2313
+ pattern: `\\b[A-HJNP-SUVWa-hjnp-suvw]-?\\d{7}-?[0-9A-Ja-j]\\b`,
2314
+ label: "registration number",
2315
+ score: .95,
2316
+ validator: es.cif
2317
+ };
2318
+ const BR_CPF_FORMATTED = {
2319
+ pattern: `\\b\\d{3}\\.\\d{3}\\.\\d{3}${DASH}\\d{2}\\b`,
2320
+ label: "tax identification number",
2321
+ score: .95,
2322
+ validator: br.cpf
2323
+ };
2324
+ const BR_CNPJ_FORMATTED = {
2325
+ pattern: `\\b\\d{2}\\.\\d{3}\\.\\d{3}/\\d{4}${DASH}\\d{2}\\b`,
2326
+ label: "tax identification number",
2327
+ score: .95,
2328
+ validator: br.cnpj
2329
+ };
2330
+ const BR_RG_WITH_SSP = {
2331
+ pattern: `\\b\\d{1,3}\\.?\\d{3}\\.?\\d{3}(?:${DASH}[0-9A-Za-z])?[^\\S\\n]+SSP(?:/[A-Z]{2})?\\b`,
2332
+ label: "national identification number",
2333
+ score: .95
2334
+ };
2335
+ const BR_OAB = {
2336
+ pattern: "\\bOAB/[A-Z]{2}[^\\S\\n]+(?:n[º°.][^\\S\\n]*)?(?:\\d{1,3}(?:\\.\\d{3})+|\\d{4,6})\\b",
2337
+ label: "registration number",
2338
+ score: .95
2339
+ };
1805
2340
  const URL = {
1806
2341
  pattern: "(?:https?://|https?:(?=[^\\s])|www\\.)[\\w\\-]+(?:\\.[\\w\\-]+)+(?::\\d+)?(?:[/?#][^\\s)\\]>]*[^\\s.,;:!?)\\]>])?",
1807
2342
  label: "url",
@@ -1830,6 +2365,7 @@ const PATH_SUFFIX = `(?:[/?#][^\\s)\\]>]*[^\\s.,;:!?)\\]>])?`;
1830
2365
  const ALL_REGEX_DEFS = [
1831
2366
  TITLED_PERSON,
1832
2367
  HONORIFIC_PERSON,
2368
+ POSTNOMINAL_PERSON,
1833
2369
  IBAN,
1834
2370
  EMAIL,
1835
2371
  INTL_PHONE,
@@ -1843,6 +2379,14 @@ const ALL_REGEX_DEFS = [
1843
2379
  CZ_BANK_ACCOUNT,
1844
2380
  HU_LANDLINE,
1845
2381
  CZ_POSTAL,
2382
+ ES_POSTAL,
2383
+ ES_DNI,
2384
+ ES_NIE,
2385
+ ES_CIF,
2386
+ BR_CPF_FORMATTED,
2387
+ BR_CNPJ_FORMATTED,
2388
+ BR_RG_WITH_SSP,
2389
+ BR_OAB,
1846
2390
  URL,
1847
2391
  {
1848
2392
  pattern: "\\b(?:[0-9a-fA-F]{1,4}:){7}[0-9a-fA-F]{1,4}\\b|\\b(?:[0-9a-fA-F]{1,4}:){1,7}:\\b|::(?:[0-9a-fA-F]{1,4}:){0,6}[0-9a-fA-F]{1,4}",
@@ -1859,6 +2403,17 @@ const ALL_REGEX_DEFS = [
1859
2403
  label: "url",
1860
2404
  score: .9
1861
2405
  },
2406
+ {
2407
+ pattern: "\\b(?:GIR[^\\S\\n]?0AA|[A-PR-UWYZ](?:[A-HK-Y][0-9](?:[0-9]|[ABEHMNPRV-Y])?|[0-9](?:[0-9]|[A-HJKPS-UW])?)[^\\S\\n]?[0-9][ABD-HJLNP-UW-Z]{2})\\b",
2408
+ label: "address",
2409
+ score: .9
2410
+ },
2411
+ {
2412
+ pattern: "\\b[A-CEGHJ-PR-TWXYZ][A-CEGHJ-NPR-TWXYZ][^\\S\\n]?\\d{2}[^\\S\\n]?\\d{2}[^\\S\\n]?\\d{2}[^\\S\\n]?[A-D]?\\b",
2413
+ label: "social security number",
2414
+ score: .95,
2415
+ validator: gb.nino
2416
+ },
1862
2417
  {
1863
2418
  pattern: "\\b(?:1[0-2]|0?[1-9]):[0-5]\\d[^\\S\\n]?(?:[aApP]\\.?[mM]\\.?)(?=[\\s,;!?)]|$)",
1864
2419
  label: "date",
@@ -1894,7 +2449,7 @@ const buildMonthAlternation = (months) => {
1894
2449
  if (clean.length >= MIN_MONTH_NAME_LENGTH) seen.add(clean);
1895
2450
  }
1896
2451
  }
1897
- return [...seen].toSorted((a, b) => b.length - a.length).map(escapeRegex$1).join("|");
2452
+ return [...seen].toSorted((a, b) => b.length - a.length).map(escapeRegex$2).join("|");
1898
2453
  };
1899
2454
  /**
1900
2455
  * Build date patterns from a month-name alternation.
@@ -1951,11 +2506,11 @@ const buildCurrencyPatterns = (data) => {
1951
2506
  const isAsciiAlpha = /^[a-zA-Z\s]+$/;
1952
2507
  const parts = data.codes.map((code) => ({
1953
2508
  len: code.length,
1954
- alt: escapeRegex$1(code)
2509
+ alt: escapeRegex$2(code)
1955
2510
  }));
1956
2511
  const MIN_CI_LENGTH = 3;
1957
2512
  if (data.localNames) for (const name of data.localNames) {
1958
- const escaped = escapeRegex$1(name);
2513
+ const escaped = escapeRegex$2(name);
1959
2514
  if (isAsciiAlpha.test(name) && name.length >= MIN_CI_LENGTH) parts.push({
1960
2515
  len: name.length,
1961
2516
  alt: `(?i:${escaped})`
@@ -1967,7 +2522,7 @@ const buildCurrencyPatterns = (data) => {
1967
2522
  }
1968
2523
  if (symbols) for (const ch of data.symbols) parts.push({
1969
2524
  len: ch.length,
1970
- alt: escapeRegex$1(ch)
2525
+ alt: escapeRegex$2(ch)
1971
2526
  });
1972
2527
  const trailingAlt = parts.toSorted((a, b) => b.len - a.len).map((p) => p.alt).join("|");
1973
2528
  if (!symbols && !trailingAlt) return [];
@@ -2019,290 +2574,70 @@ const processRegexMatches = (allMatches, sliceStart, sliceEnd, meta_) => {
2019
2574
  for (const match of allMatches) {
2020
2575
  const idx = match.pattern;
2021
2576
  if (idx < sliceStart || idx >= sliceEnd) continue;
2022
- const meta = meta_[idx - sliceStart];
2023
- if (!meta) continue;
2024
- if (meta.sourceDetail !== "custom-regex" && meta.label === "phone number" && match.text.length < MIN_PHONE_LENGTH) continue;
2025
- if (meta.validator) {
2026
- const compacted = meta.validator.compact(match.text);
2027
- if (!meta.validator.validate(compacted).valid) continue;
2028
- }
2029
- const entity = {
2030
- start: match.start,
2031
- end: match.end,
2032
- label: meta.label,
2033
- text: match.text,
2034
- score: meta.score,
2035
- source: DETECTION_SOURCES.REGEX
2036
- };
2037
- if (meta.sourceDetail) entity.sourceDetail = meta.sourceDetail;
2038
- results.push(entity);
2039
- }
2040
- return results;
2041
- };
2042
- /**
2043
- * Build signing clause place-name patterns from
2044
- * signing-clauses.json. Each pattern captures the
2045
- * city/place name from contract signing locations.
2046
- *
2047
- * The place name sub-pattern:
2048
- * \p{Lu}\p{Ll}+ (capitalized word)
2049
- * optionally followed by preposition + capitalized
2050
- * word (for "nad Nisou", "am Main", etc.)
2051
- * optionally followed by more capitalized words
2052
- * (for "Hradec Králové", "New York", etc.)
2053
- */
2054
- const buildSigningClausePatterns = (data) => {
2055
- const patterns = [];
2056
- for (const entry of data.patterns) {
2057
- const prepAlt = entry.prepositions.length > 0 ? entry.prepositions.join("|") : null;
2058
- const place = prepAlt ? `(\\p{Lu}\\p{Ll}+(?:\\s+(?:${prepAlt})\\s+\\p{Lu}\\p{Ll}+)*(?:\\s+\\p{Lu}\\p{Ll}+)*)` : `(\\p{Lu}\\p{Ll}+(?:[- ]\\p{Lu}\\p{Ll}+)*)`;
2059
- const full = `(?:^|\\n|[^\\S\\n])` + entry.prefix + place + (entry.suffix ? `(?:${entry.suffix})` : "");
2060
- patterns.push(full);
2061
- }
2062
- return patterns;
2063
- };
2064
- const SIGNING_CLAUSE_META = {
2065
- label: "address",
2066
- score: .9
2067
- };
2068
- let signingPatternPromise = null;
2069
- const loadSigningPatterns = async () => {
2070
- const mod = await import("./signing-clauses.mjs");
2071
- return buildSigningClausePatterns(mod.default ?? mod);
2072
- };
2073
- const getSigningClausePatterns = () => {
2074
- if (!signingPatternPromise) signingPatternPromise = loadSigningPatterns().catch((err) => {
2075
- signingPatternPromise = null;
2076
- throw err;
2077
- });
2078
- return signingPatternPromise;
2079
- };
2080
- //#endregion
2081
- //#region src/detectors/legal-forms.ts
2082
- const UPPER = "A-ZÁČĎÉĚÍŇÓŘŠŤÚŮÝŽÄÖÜÀÂÆÇÈÊËÎÏÔÙÛŸÑ\\u0130";
2083
- const LOWER = "a-záčďéěíňóřšťúůýžäöüßàâæçèêëîïôùûÿñ\\u0131";
2084
- const CAP_WORD = `(?:[${UPPER}]{2,}|[${UPPER}][${LOWER}${UPPER}]+)`;
2085
- const ANY_WORD = `(?:[${UPPER}${LOWER}][${LOWER}${UPPER}]+|[${UPPER}]{2,3}|\\d{1,4})`;
2086
- const ALLCAP_WORD = `[${UPPER}]{2,}`;
2087
- const ROMAN_NUMERAL_RE = /^(?=[IVXLCDM])M{0,3}(?:CM|CD|D?C{0,3})(?:XC|XL|L?X{0,3})(?:IX|IV|V?I{0,3})$/;
2088
- const escapeForRegex = (form) => form.replace(/[.*+?^${}()|[\]\\]/g, "\\$&").replace(/\s+/g, "\\s+").replace(/\\\./g, "\\.[^\\S\\n]?");
2089
- const isShortForm = (form) => form.replace(/[.\s]/g, "").length <= 3 && !form.includes(" ");
2090
- const buildPatternString = (forms) => {
2091
- if (forms.length === 0) return null;
2092
- const alt = forms.toSorted((a, b) => b.length - a.length).map(escapeForRegex).join("|");
2093
- return `${`(?:${CAP_WORD})(?:${`(?:[\\s&,.${DASH_INNER}]{1,4}|${`\\s+(?:a|and|und|et|e|y|i)\\s+(?=[${LOWER}])`})`}(?:${ANY_WORD})){0,10}`}(?:\\s+|,\\s*)(?:${alt})(?![${LOWER}])`;
2094
- };
2095
- /**
2096
- * Build legal form regex pattern strings.
2097
- * Returns an array of regex strings for the unified
2098
- * TextSearch builder. Empty if data package is not
2099
- * installed.
2100
- */
2101
- const buildLegalFormPatterns = async () => {
2102
- let data = {};
2103
- try {
2104
- data = (await import("./legal-forms.mjs")).default;
2105
- } catch {
2106
- return [];
2107
- }
2108
- const allForms = [];
2109
- const seen = /* @__PURE__ */ new Set();
2110
- for (const forms of Object.values(data)) for (const form of forms) {
2111
- const key = form.toLowerCase();
2112
- if (!seen.has(key)) {
2113
- seen.add(key);
2114
- allForms.push(form);
2115
- }
2116
- }
2117
- const patterns = [];
2118
- const longPattern = buildPatternString(allForms.filter((f) => !isShortForm(f)));
2119
- if (longPattern) patterns.push(longPattern);
2120
- const shortPattern = buildPatternString(allForms.filter(isShortForm));
2121
- if (shortPattern) patterns.push(shortPattern);
2122
- const allcapPrefix = `(?:${ALLCAP_WORD})(?:[\\s&,.${DASH_INNER}]{1,4}(?:${ALLCAP_WORD})){0,2}`;
2123
- const allcapAlt = allForms.toSorted((a, b) => b.length - a.length).map(escapeForRegex).join("|");
2124
- patterns.push(`${allcapPrefix}(?:\\s+|,\\s*)(?:${allcapAlt})(?![${LOWER}])`);
2125
- return patterns;
2126
- };
2127
- const CONNECTOR_RE = /^(?:a|and|und|et|e|y|i|&)$/i;
2128
- const LEADING_CLAUSE_RE = /(?:^|\s)(?:by\s+and\s+between|is\s+between)\s+/giu;
2129
- /**
2130
- * Find the word ending just before `pos` in `text`,
2131
- * skipping any whitespace (not newlines).
2132
- * Returns null if no word is found (e.g., at start
2133
- * of text, or preceded by non-word chars like ".").
2134
- */
2135
- const findWordBefore = (text, pos) => {
2136
- let scan = pos - 1;
2137
- while (scan >= 0) {
2138
- const ch = text.charAt(scan);
2139
- if (ch === "\n" || !/\s/.test(ch)) break;
2140
- scan--;
2141
- }
2142
- if (scan < 0 || text.charAt(scan) === "\n") return null;
2143
- const wordEnd = scan + 1;
2144
- while (scan >= 0 && /[\p{L}\p{M}&]/u.test(text.charAt(scan))) scan--;
2145
- const wordStart = scan + 1;
2146
- const word = text.slice(wordStart, wordEnd);
2147
- if (word.length === 0) return null;
2148
- return {
2149
- word,
2150
- start: wordStart
2151
- };
2152
- };
2153
- /**
2154
- * Extend a match backward through uppercase words and
2155
- * lowercase connectors. Stops at start of text,
2156
- * newline, or a word that doesn't qualify.
2157
- *
2158
- * Connectors (a, and, und, et, ...) are only consumed
2159
- * when there is a valid word before them — a trailing
2160
- * connector at an entity boundary is not consumed.
2161
- */
2162
- const extendBackward = (fullText, matchStart) => {
2163
- let pos = matchStart;
2164
- while (pos > 0) {
2165
- const found = findWordBefore(fullText, pos);
2166
- if (!found) break;
2167
- const { word, start: wordStart } = found;
2168
- const isUpper = /^\p{Lu}/u.test(word);
2169
- const isConnector = CONNECTOR_RE.test(word);
2170
- if (isUpper) pos = wordStart;
2171
- else if (isConnector) {
2172
- const prev = findWordBefore(fullText, wordStart);
2173
- if (!prev) break;
2174
- if (!/^\p{Lu}/u.test(prev.word)) break;
2175
- pos = prev.start;
2176
- } else break;
2177
- }
2178
- return pos;
2179
- };
2180
- const trimLeadingClause = (text) => {
2181
- let cut = -1;
2182
- for (const match of text.matchAll(LEADING_CLAUSE_RE)) cut = match.index + match[0].length;
2183
- if (cut <= 0) return {
2184
- offset: 0,
2185
- text
2186
- };
2187
- const trimmed = text.slice(cut);
2188
- const leadingWs = trimmed.match(/^\s*/u)?.[0].length ?? 0;
2189
- return {
2190
- offset: cut + leadingWs,
2191
- text: trimmed.slice(leadingWs)
2192
- };
2193
- };
2194
- /**
2195
- * Process legal form matches from the unified search.
2196
- * Receives all matches; filters to the legal forms
2197
- * slice via sliceStart/sliceEnd.
2198
- */
2199
- const processLegalFormMatches = (allMatches, sliceStart, sliceEnd, fullText) => {
2200
- const results = [];
2201
- for (const match of allMatches) {
2202
- const idx = match.pattern;
2203
- if (idx < sliceStart || idx >= sliceEnd) continue;
2204
- const text = match.text.trimEnd();
2205
- if (text.length < 5) continue;
2206
- if (text.includes("\n")) continue;
2207
- let entityStart = match.start;
2208
- let entityText = text;
2209
- if (fullText) {
2210
- const extended = extendBackward(fullText, match.start);
2211
- if (extended < match.start) {
2212
- entityStart = extended;
2213
- entityText = fullText.slice(extended, match.start + text.length).trimEnd();
2214
- }
2215
- }
2216
- const clauseTrim = trimLeadingClause(entityText);
2217
- if (clauseTrim.offset > 0) {
2218
- entityStart += clauseTrim.offset;
2219
- entityText = clauseTrim.text;
2220
- }
2221
- const getPrefixInfo = (value) => {
2222
- const prefixEnd = value.lastIndexOf(",") !== -1 ? value.lastIndexOf(",") : value.lastIndexOf(" ");
2223
- return {
2224
- prefixEnd,
2225
- prefixPart: prefixEnd > 0 ? value.slice(0, prefixEnd).replace(/[^a-zA-ZÀ-ž]/g, "") : value.replace(/[^a-zA-ZÀ-ž]/g, "")
2226
- };
2227
- };
2228
- let { prefixEnd, prefixPart } = getPrefixInfo(entityText);
2229
- let isAllCapsMatch = prefixPart.length > 2 && prefixPart === prefixPart.toUpperCase();
2230
- if (isAllCapsMatch && fullText) {
2231
- const lineStart = fullText.lastIndexOf("\n", entityStart);
2232
- const lineEnd = fullText.indexOf("\n", entityStart + entityText.length);
2233
- const lineLetters = fullText.slice(lineStart + 1, lineEnd === -1 ? fullText.length : lineEnd).replace(/[^a-zA-ZÀ-ž]/g, "");
2234
- const upperCount = [...lineLetters].filter((c) => c === c.toUpperCase()).length;
2235
- if (lineLetters.length > 5 && upperCount / lineLetters.length >= .95) continue;
2236
- if ((prefixPart.length > 0 ? entityText.slice(0, prefixEnd > 0 ? prefixEnd : entityText.length).trim().split(/\s+/).length : 0) > 3) {
2237
- entityStart = match.start;
2238
- entityText = text;
2239
- ({prefixEnd, prefixPart} = getPrefixInfo(entityText));
2240
- isAllCapsMatch = prefixPart.length > 2 && prefixPart === prefixPart.toUpperCase();
2241
- }
2242
- } else if (isAllCapsMatch) continue;
2243
- const lastSpace = entityText.lastIndexOf(" ");
2244
- const rawSuffix = lastSpace !== -1 ? entityText.slice(lastSpace + 1) : "";
2245
- const suffixClean = rawSuffix.replace(/[.,]/g, "");
2246
- if (suffixClean.length > 0 && ROMAN_NUMERAL_RE.test(suffixClean)) continue;
2247
- if (suffixClean.length <= 2 && !/\./.test(rawSuffix) && /[^\x00-\x7F]/.test(entityText.slice(0, lastSpace !== -1 ? lastSpace : entityText.length))) continue;
2248
- results.push({
2249
- start: entityStart,
2250
- end: entityStart + entityText.length,
2251
- label: "organization",
2252
- text: entityText,
2253
- score: .95,
2254
- source: DETECTION_SOURCES.LEGAL_FORM
2255
- });
2577
+ const meta = meta_[idx - sliceStart];
2578
+ if (!meta) continue;
2579
+ if (meta.sourceDetail !== "custom-regex" && meta.label === "phone number" && match.text.length < MIN_PHONE_LENGTH) continue;
2580
+ if (meta.validator) {
2581
+ const compacted = meta.validator.compact(match.text);
2582
+ if (!meta.validator.validate(compacted).valid) continue;
2583
+ }
2584
+ const entity = {
2585
+ start: match.start,
2586
+ end: match.end,
2587
+ label: meta.label,
2588
+ text: match.text,
2589
+ score: meta.score,
2590
+ source: DETECTION_SOURCES.REGEX
2591
+ };
2592
+ if (meta.sourceDetail) entity.sourceDetail = meta.sourceDetail;
2593
+ results.push(entity);
2256
2594
  }
2257
2595
  return results;
2258
2596
  };
2259
- //#endregion
2260
- //#region src/config/legal-forms.ts
2261
2597
  /**
2262
- * Known legal form suffixes. Shared between trigger
2263
- * detection (reclassification) and org-propagation
2264
- * (suffix stripping). Within each family of related
2265
- * forms, longer variants come first so that
2266
- * "spol. s r.o." matches before "s.r.o." and
2267
- * "s. r. o." matches before "s.r.o.".
2598
+ * Build signing clause place-name patterns from
2599
+ * signing-clauses.json. Each pattern captures the
2600
+ * city/place name from contract signing locations.
2601
+ *
2602
+ * The place name sub-pattern:
2603
+ * \p{Lu}\p{Ll}+ (capitalized word)
2604
+ * optionally followed by preposition + capitalized
2605
+ * word (for "nad Nisou", "am Main", etc.)
2606
+ * optionally followed by more capitalized words
2607
+ * (for "Hradec Králové", "New York", etc.)
2268
2608
  */
2269
- const LEGAL_SUFFIXES = [
2270
- "spol. s r.o.",
2271
- "s.r.o.",
2272
- "s. r. o.",
2273
- "a.s.",
2274
- "a. s.",
2275
- "v.o.s.",
2276
- "v. o. s.",
2277
- "k.s.",
2278
- "k. s.",
2279
- "z.s.",
2280
- "z. s.",
2281
- "z.ú.",
2282
- "z. ú.",
2283
- "o.p.s.",
2284
- "o. p. s.",
2285
- "s.p.",
2286
- "s. p.",
2287
- "GmbH",
2288
- "AG",
2289
- "SE",
2290
- "KG",
2291
- "OHG",
2292
- "Ltd.",
2293
- "Ltd",
2294
- "LLC",
2295
- "LLP",
2296
- "Inc.",
2297
- "S.A.",
2298
- "SA",
2299
- "SAS",
2300
- "SARL",
2301
- "Sp. z o.o.",
2302
- "S.p.A."
2303
- ];
2609
+ const buildSigningClausePatterns = (data) => {
2610
+ const patterns = [];
2611
+ for (const entry of data.patterns) {
2612
+ const prepAlt = entry.prepositions.length > 0 ? entry.prepositions.join("|") : null;
2613
+ const place = prepAlt ? `(\\p{Lu}\\p{Ll}+(?:\\s+(?:${prepAlt})\\s+\\p{Lu}\\p{Ll}+)*(?:\\s+\\p{Lu}\\p{Ll}+)*)` : `(\\p{Lu}\\p{Ll}+(?:[- ]\\p{Lu}\\p{Ll}+)*)`;
2614
+ const full = `(?:^|\\n|[^\\S\\n])` + entry.prefix + place + (entry.suffix ? `(?:${entry.suffix})` : "");
2615
+ patterns.push(full);
2616
+ }
2617
+ return patterns;
2618
+ };
2619
+ const SIGNING_CLAUSE_META = {
2620
+ label: "address",
2621
+ score: .9
2622
+ };
2623
+ let signingPatternPromise = null;
2624
+ const loadSigningPatterns = async () => {
2625
+ const mod = await import("./signing-clauses.mjs");
2626
+ return buildSigningClausePatterns(mod.default ?? mod);
2627
+ };
2628
+ const getSigningClausePatterns = () => {
2629
+ if (!signingPatternPromise) signingPatternPromise = loadSigningPatterns().catch((err) => {
2630
+ signingPatternPromise = null;
2631
+ throw err;
2632
+ });
2633
+ return signingPatternPromise;
2634
+ };
2304
2635
  //#endregion
2305
2636
  //#region src/detectors/triggers.ts
2637
+ const VALID_ID_VALIDATORS = {
2638
+ "br.cpf": br.cpf,
2639
+ "br.cnpj": br.cnpj
2640
+ };
2306
2641
  const TRIGGER_SCORE = .95;
2307
2642
  const WHITESPACE_RE$1 = /\s+/;
2308
2643
  const LETTER_RE = /\p{L}/u;
@@ -2317,12 +2652,23 @@ const DECIMAL_COMMA_RE = new RegExp(`^,(?:\\d|${DASH}{1,2})`);
2317
2652
  * etc.), skip the comma and degree, then continue.
2318
2653
  */
2319
2654
  const POST_NOMINAL_RE = new RegExp(`^,\\s*(?:${POST_NOMINALS.toSorted((a, b) => b.length - a.length).map((d) => d.replace(/[.*+?^${}()|[\]\\]/g, "\\$&").replace(/\\\./g, "\\.\\s*")).join("|")})\\.?`, "i");
2320
- const LEGAL_FORM_CHECK_RE = new RegExp(LEGAL_SUFFIXES.map((f) => {
2321
- const escaped = f.replace(/[.*+?^${}()|[\]\\]/g, "\\$&").replace(/\\\./g, "\\.\\s*");
2322
- const isDotFree = !f.includes(".");
2323
- const isShort = f.length <= 4;
2324
- return isDotFree && isShort ? `\\b${escaped}\\b` : escaped;
2325
- }).join("|"));
2655
+ const buildLegalFormCheckRe = (forms) => {
2656
+ const parts = forms.map((f) => {
2657
+ const escaped = f.replace(/[.*+?^${}()|[\]\\]/g, "\\$&").replace(/\\\./g, "\\.\\s*");
2658
+ return /^[\p{L}\p{M}]+$/u.test(f) ? `(?<![\\p{L}\\p{N}])${escaped}(?![\\p{L}\\p{N}])` : escaped;
2659
+ });
2660
+ return new RegExp(parts.join("|"), "u");
2661
+ };
2662
+ let cachedLegalFormCheckRe = null;
2663
+ let cachedLegalFormCheckSource = null;
2664
+ const getLegalFormCheckRe = () => {
2665
+ const source = getKnownLegalSuffixes();
2666
+ if (cachedLegalFormCheckSource !== source || cachedLegalFormCheckRe === null) {
2667
+ cachedLegalFormCheckSource = source;
2668
+ cachedLegalFormCheckRe = buildLegalFormCheckRe(source);
2669
+ }
2670
+ return cachedLegalFormCheckRe;
2671
+ };
2326
2672
  const compileValidations = (validations) => validations.map((v) => {
2327
2673
  switch (v.type) {
2328
2674
  case "starts-uppercase": return {
@@ -2349,6 +2695,17 @@ const compileValidations = (validations) => validations.map((v) => {
2349
2695
  type: "matches-pattern",
2350
2696
  re: new RegExp(v.pattern, (v.flags ?? "").replace(/[gy]/g, ""))
2351
2697
  };
2698
+ case "valid-id": {
2699
+ const validator = VALID_ID_VALIDATORS[v.validator];
2700
+ if (!validator) throw new Error(`Unknown valid-id validator: ${JSON.stringify(v.validator)}`);
2701
+ return {
2702
+ type: "valid-id",
2703
+ check: (value) => {
2704
+ const compact = value.replace(/[\s.\-/]/g, "");
2705
+ return validator.validate(compact).valid;
2706
+ }
2707
+ };
2708
+ }
2352
2709
  default: throw new Error(`Unknown validation type: ${JSON.stringify(v)}`);
2353
2710
  }
2354
2711
  });
@@ -2372,6 +2729,9 @@ const applyValidations = (text, validations) => {
2372
2729
  case "matches-pattern":
2373
2730
  if (!v.re.test(text)) return false;
2374
2731
  break;
2732
+ case "valid-id":
2733
+ if (!v.check(text)) return false;
2734
+ break;
2375
2735
  default: throw new Error(`Unknown compiled validation type: ${JSON.stringify(v)}`);
2376
2736
  }
2377
2737
  return true;
@@ -2466,8 +2826,10 @@ const buildTriggerPatterns = async () => {
2466
2826
  strategy: rule.strategy.type
2467
2827
  });
2468
2828
  }
2829
+ const patterns = rules.map((r) => r.trigger.toLowerCase());
2830
+ await loadAddressStopKeywords();
2469
2831
  return {
2470
- patterns: rules.map((r) => r.trigger.toLowerCase()),
2832
+ patterns,
2471
2833
  rules
2472
2834
  };
2473
2835
  };
@@ -2488,14 +2850,103 @@ const stripQuotes = (value) => {
2488
2850
  const COMMA_STOP_CHARS = new Set([
2489
2851
  "\n",
2490
2852
  "(",
2491
- " "
2853
+ " ",
2854
+ ";"
2492
2855
  ]);
2493
2856
  /**
2857
+ * Sentence-terminator detection: a period that genuinely ends
2858
+ * one clause and starts another. Used by `to-next-comma` so
2859
+ * governing-law clauses ("…State of New York. SECTION 2…")
2860
+ * don't sweep across the period.
2861
+ *
2862
+ * Three positive signals (any one terminates):
2863
+ * 1. Long lowercase tail before the dot (>= 5 letters) —
2864
+ * catches "…construction. SECTION 2…", "…vykonává.".
2865
+ * 2. Currency/amount tail (zł, Kč, USD, €) — catches
2866
+ * "…w kwocie 1000 zł. Termin płatności…".
2867
+ * 3. Proper-noun head: a capitalized word of >= 4 letters
2868
+ * ending in lowercase, with no internal uppercase, AND
2869
+ * a substantial next clause (Capital + >= 2 lowercase).
2870
+ * Catches short city names that the lowercase-tail rule
2871
+ * misses: "…z siedzibą w Łódź. Kapitał zakładowy…",
2872
+ * "…seat in Brno. Section 2…".
2873
+ *
2874
+ * The rule must NOT fire on:
2875
+ * - title abbreviations: "Mr.", "Mrs.", "Dr.", "Hon.",
2876
+ * "Sr.", "Jr." (head <= 3 chars or insufficient Ll tail)
2877
+ * - degree abbreviations: "Ph.D.", "RNDr.", "MUDr.",
2878
+ * "Ing." (internal periods or internal uppercase block
2879
+ * the proper-noun pattern)
2880
+ * - street-type abbreviations: "Ste.", "Ave.", "Inc.",
2881
+ * "ul.", "al.", "nábř." (lowercase initials or
2882
+ * insufficient Ll tail)
2883
+ * - small-word lowercase abbreviations: "prof.", "inż.",
2884
+ * "hab." (no leading uppercase, so the proper-noun rule
2885
+ * can't fire)
2886
+ */
2887
+ const NEXT_IS_SENTENCE_START_RE = /^\.(?:\s+\p{Lu}|\s*$)/u;
2888
+ const SENTENCE_TAIL_RE = /\p{Ll}{5,}$/u;
2889
+ /**
2890
+ * Proper-noun tail: capital letter + >= 3 lowercase letters,
2891
+ * preceded by a non-letter and non-period (so we don't slice
2892
+ * into the middle of an acronym or a multi-dot abbreviation).
2893
+ * The 4-character minimum excludes 2–3-char titles ("Mr",
2894
+ * "Mrs", "Dr", "Inc", "Ste"); the all-lowercase tail
2895
+ * excludes mixed-case degrees ("RNDr", "MUDr").
2896
+ */
2897
+ const PROPER_NOUN_HEAD_RE = /(?:^|[^\p{L}.])\p{Lu}\p{Ll}{3,}$/u;
2898
+ /**
2899
+ * Next clause begins with a real word: capital + >= 2
2900
+ * lowercase letters. Filters cases where a capitalized
2901
+ * abbreviation (e.g., "Smith Inc.") follows a proper noun,
2902
+ * which would otherwise look sentence-like.
2903
+ */
2904
+ const NEXT_IS_REAL_SENTENCE_RE = /^\.\s+\p{Lu}\p{Ll}{2,}/u;
2905
+ /**
2906
+ * Short currency-abbreviation tail (zł, Kč, gr, Ft, kr,
2907
+ * лв, USD, PLN, EUR, …). When a period follows one of
2908
+ * these and the next token starts uppercase, treat it as
2909
+ * a sentence boundary even though the tail is too short
2910
+ * to satisfy `SENTENCE_TAIL_RE`. Without this, amount
2911
+ * triggers using `to-next-comma` swallow the following
2912
+ * clause: `"w kwocie 1000 zł. Termin płatności…"`.
2913
+ *
2914
+ * Currency codes are typically uppercase (`USD`, `PLN`)
2915
+ * but appear lowercased in informal writing (`pln`, `eur`);
2916
+ * local names mix case (`zł`, `Kč`, `Ft`); symbols
2917
+ * (`€`, `$`, `£`) appear after the amount as well. The
2918
+ * negative lookbehind on a letter ensures the abbreviation
2919
+ * is matched only as a standalone token; symbols are
2920
+ * matched unconditionally since they are not letters.
2921
+ */
2922
+ const CURRENCY_TAIL_RE = /(?:(?<![\p{L}])(?:zł|Kč|gr|Ft|kr|лв|USD|PLN|EUR|CZK|GBP|CHF|HUF|RON|SEK|NOK|DKK)|[€$£])$/iu;
2923
+ const NUMERIC_SENTENCE_TAIL_RE = /\d$/;
2924
+ const isSentenceTerminator = (text, periodIndex) => {
2925
+ const tail = text.slice(periodIndex);
2926
+ if (!NEXT_IS_SENTENCE_START_RE.test(tail)) return false;
2927
+ const head = text.slice(0, periodIndex);
2928
+ if (SENTENCE_TAIL_RE.test(head) || CURRENCY_TAIL_RE.test(head) || NUMERIC_SENTENCE_TAIL_RE.test(head)) return true;
2929
+ return PROPER_NOUN_HEAD_RE.test(head) && NEXT_IS_REAL_SENTENCE_RE.test(tail);
2930
+ };
2931
+ /**
2494
2932
  * Field-label keywords that terminate address scanning.
2495
2933
  * When a comma in the address strategy is followed by
2496
2934
  * one of these, the address stops before the keyword.
2935
+ *
2936
+ * The list is sourced from
2937
+ * `data/address-stop-keywords.json` (per-language so new
2938
+ * languages can drop in their own labels without
2939
+ * touching this file). `loadAddressStopKeywords` unions
2940
+ * every language into a single longest-first array;
2941
+ * `getAddressStopKeywordsSync` returns the cached union
2942
+ * and falls back to a seed list so the strategy keeps
2943
+ * working before the warmup promise resolves.
2944
+ *
2945
+ * The address strategy doesn't know the document
2946
+ * language, so a flat union is intentional: any
2947
+ * language's labels can appear in any address.
2497
2948
  */
2498
- const ADDRESS_STOP_KEYWORDS = [
2949
+ const ADDRESS_STOP_KEYWORDS_SEED = [
2499
2950
  "číslo účtu",
2500
2951
  "registrační",
2501
2952
  "zastoupen",
@@ -2518,6 +2969,52 @@ const ADDRESS_STOP_KEYWORDS = [
2518
2969
  "bic",
2519
2970
  "ič"
2520
2971
  ];
2972
+ let addressStopKeywordsCache = null;
2973
+ let addressStopKeywordsPromise = null;
2974
+ const loadAddressStopKeywords = async () => {
2975
+ if (addressStopKeywordsCache) return addressStopKeywordsCache;
2976
+ if (addressStopKeywordsPromise) return addressStopKeywordsPromise;
2977
+ addressStopKeywordsPromise = (async () => {
2978
+ let data = {};
2979
+ try {
2980
+ const mod = await import("./address-stop-keywords.mjs");
2981
+ data = mod.default ?? mod;
2982
+ } catch (err) {
2983
+ console.warn("[anonymize] triggers: failed to load address-stop-keywords.json, falling back to seed list:", err);
2984
+ }
2985
+ const seen = /* @__PURE__ */ new Set();
2986
+ const out = [];
2987
+ const addAll = (list) => {
2988
+ for (const kw of list) {
2989
+ if (typeof kw !== "string" || kw.length === 0) continue;
2990
+ const lower = kw.toLowerCase();
2991
+ if (seen.has(lower)) continue;
2992
+ seen.add(lower);
2993
+ out.push(lower);
2994
+ }
2995
+ };
2996
+ for (const [key, value] of Object.entries(data)) {
2997
+ if (key.startsWith("_")) continue;
2998
+ if (!Array.isArray(value)) continue;
2999
+ addAll(value);
3000
+ }
3001
+ addAll(ADDRESS_STOP_KEYWORDS_SEED);
3002
+ out.sort((a, b) => b.length - a.length);
3003
+ addressStopKeywordsCache = out;
3004
+ return out;
3005
+ })();
3006
+ return addressStopKeywordsPromise;
3007
+ };
3008
+ const getAddressStopKeywordsSync = () => addressStopKeywordsCache ?? ADDRESS_STOP_KEYWORDS_SEED;
3009
+ /**
3010
+ * Warm the address-stop-keywords cache. Pipeline callers
3011
+ * await this before invoking trigger detection so the
3012
+ * synchronous `extractValue` path uses the merged list
3013
+ * instead of the seed fallback.
3014
+ */
3015
+ const warmAddressStopKeywords = async () => {
3016
+ await loadAddressStopKeywords();
3017
+ };
2521
3018
  const extractValue = (text, triggerEnd, strategy, label) => {
2522
3019
  const remaining = text.slice(triggerEnd);
2523
3020
  const stripped = remaining.replace(/^[\s:;]+/, "");
@@ -2526,6 +3023,8 @@ const extractValue = (text, triggerEnd, strategy, label) => {
2526
3023
  if (valueText.length === 0) return null;
2527
3024
  switch (strategy.type) {
2528
3025
  case "to-next-comma": {
3026
+ const stopWords = strategy.stopWords ?? [];
3027
+ const stopWordRe = stopWords.length > 0 ? new RegExp(`^(?:${stopWords.map((w) => w.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")).join("|")})(?![\\p{L}\\p{N}])`, "iu") : null;
2529
3028
  let end = 0;
2530
3029
  let foundStop = false;
2531
3030
  while (end < valueText.length) {
@@ -2534,6 +3033,14 @@ const extractValue = (text, triggerEnd, strategy, label) => {
2534
3033
  foundStop = true;
2535
3034
  break;
2536
3035
  }
3036
+ if (ch === "." && isSentenceTerminator(valueText, end)) {
3037
+ foundStop = true;
3038
+ break;
3039
+ }
3040
+ if (stopWordRe !== null && (end === 0 || !/[\p{L}\p{N}]/u.test(valueText[end - 1] ?? "")) && stopWordRe.test(valueText.slice(end))) {
3041
+ foundStop = true;
3042
+ break;
3043
+ }
2537
3044
  if (ch === ",") {
2538
3045
  const afterComma = valueText.slice(end);
2539
3046
  if (DECIMAL_COMMA_RE.test(afterComma)) {
@@ -2562,6 +3069,8 @@ const extractValue = (text, triggerEnd, strategy, label) => {
2562
3069
  };
2563
3070
  }
2564
3071
  case "to-end-of-line": {
3072
+ const consumed = remaining.length - valueText.length;
3073
+ if (consumed > 0 && remaining.slice(0, consumed).includes("\n")) return null;
2565
3074
  const LINE_STOPS = ["\n"];
2566
3075
  let end = valueText.length;
2567
3076
  for (const ch of LINE_STOPS) {
@@ -2582,7 +3091,8 @@ const extractValue = (text, triggerEnd, strategy, label) => {
2582
3091
  const tabIdx = valueText.indexOf(" ");
2583
3092
  const cellText = tabIdx !== -1 ? valueText.slice(0, tabIdx) : valueText;
2584
3093
  const PUNCT_ONLY = /^[\p{P}\p{S}]+$/u;
2585
- const words = cellText.split(WHITESPACE_RE$1).filter((w) => !PUNCT_ONLY.test(w)).slice(0, strategy.count);
3094
+ const NUMBER_MARKER = /^(?:n[ºo°.]|№)$/i;
3095
+ const words = cellText.split(WHITESPACE_RE$1).filter((w) => !PUNCT_ONLY.test(w) && !NUMBER_MARKER.test(w)).slice(0, strategy.count);
2586
3096
  if (words.length === 0) return null;
2587
3097
  const firstWord = words[0];
2588
3098
  if (firstWord === void 0) return null;
@@ -2605,14 +3115,22 @@ const extractValue = (text, triggerEnd, strategy, label) => {
2605
3115
  }
2606
3116
  case "company-id-value": {
2607
3117
  const raw = text.slice(triggerEnd);
2608
- const sepMatch = /^(?:\s*:\s*|\s+)/.exec(raw);
3118
+ const triggerLastChar = text[triggerEnd - 1] ?? "";
3119
+ const sepMatch = (triggerLastChar === "°" || triggerLastChar === "º" || triggerLastChar === "№" || triggerLastChar === "#" ? /^(?:\s*:\s*|\s+|)/ : /^(?:\s*:\s*|\s+)/).exec(raw);
2609
3120
  if (!sepMatch) return null;
2610
- const afterSep = raw.slice(sepMatch[0].length);
2611
- const idMatch = /^[A-Z]{0,6}\s?\d[\d\s\-/]{4,}/i.exec(afterSep);
3121
+ let afterSep = raw.slice(sepMatch[0].length);
3122
+ const labelMatch = /^(?:nr\.?|numer|n[ºo°.]|№|no\.?)(?:\s*:\s*|\s+)/i.exec(afterSep);
3123
+ let labelOffset = 0;
3124
+ if (labelMatch) {
3125
+ labelOffset = labelMatch[0].length;
3126
+ afterSep = afterSep.slice(labelOffset);
3127
+ }
3128
+ const idMatch = /^[A-Z]{0,6}\s?(?:[A-Z0-9]{2}\s?)?\d{2}(?:(?<=\d)[A-Z]|[\d\s.\-/]){3,}/i.exec(afterSep);
2612
3129
  if (!idMatch) return null;
2613
- const idText = idMatch[0].trim();
3130
+ const trailingLetterMatch = /\d$/.test(idMatch[0]) ? /^[A-Z]/i.exec(afterSep.slice(idMatch[0].length)) : null;
3131
+ const idText = (trailingLetterMatch ? idMatch[0] + trailingLetterMatch[0] : idMatch[0]).trim();
2614
3132
  const leadingSpaces = idMatch[0].length - idMatch[0].trimStart().length;
2615
- const idStart = triggerEnd + sepMatch[0].length + leadingSpaces;
3133
+ const idStart = triggerEnd + sepMatch[0].length + labelOffset + leadingSpaces;
2616
3134
  return {
2617
3135
  start: idStart,
2618
3136
  end: idStart + idText.length,
@@ -2622,18 +3140,36 @@ const extractValue = (text, triggerEnd, strategy, label) => {
2622
3140
  case "address": {
2623
3141
  const maxLen = strategy.maxChars ?? 120;
2624
3142
  const UPPER_RE = /\p{Lu}/u;
3143
+ const stopKeywords = getAddressStopKeywordsSync();
2625
3144
  let end = 0;
2626
3145
  while (end < valueText.length && end < maxLen) {
2627
3146
  const ch = valueText[end];
2628
3147
  if (ch === "\n" || ch === "(") break;
3148
+ if (ch === " " || ch === " ") {
3149
+ const afterWs = valueText.slice(end).trimStart().toLowerCase();
3150
+ if (stopKeywords.some((kw) => {
3151
+ if (!afterWs.startsWith(kw)) return false;
3152
+ const next = afterWs[kw.length];
3153
+ return next === void 0 || /[\s:;.,!?()\d]/.test(next);
3154
+ })) break;
3155
+ }
2629
3156
  if (ch === ".") {
2630
3157
  const next = valueText[end + 1];
2631
3158
  const afterNext = valueText[end + 2];
3159
+ const afterPeriod = valueText.slice(end + 1).replace(/^\s+/, "").toLowerCase();
3160
+ if (afterPeriod.length > 0) {
3161
+ if (getAddressStopKeywordsSync().some((kw) => {
3162
+ if (!afterPeriod.startsWith(kw)) return false;
3163
+ const afterKw = afterPeriod[kw.length];
3164
+ return afterKw === void 0 || /[\s:;.,!?()\d]/.test(afterKw);
3165
+ })) break;
3166
+ }
2632
3167
  if (next !== void 0 && (/\p{L}/u.test(next) || /\d/.test(next))) {
2633
3168
  end++;
2634
3169
  continue;
2635
3170
  }
2636
3171
  if (next === " " && afterNext !== void 0 && (/\p{L}/u.test(afterNext) || /\d/.test(afterNext))) {
3172
+ if (isSentenceTerminator(valueText, end)) break;
2637
3173
  end++;
2638
3174
  continue;
2639
3175
  }
@@ -2645,7 +3181,7 @@ const extractValue = (text, triggerEnd, strategy, label) => {
2645
3181
  const peekCh = valueText[peek];
2646
3182
  if (peekCh === void 0) break;
2647
3183
  const afterComma = valueText.slice(end + 1).trimStart().toLowerCase();
2648
- if (ADDRESS_STOP_KEYWORDS.some((kw) => {
3184
+ if (getAddressStopKeywordsSync().some((kw) => {
2649
3185
  if (!afterComma.startsWith(kw)) return false;
2650
3186
  const next = afterComma[kw.length];
2651
3187
  return next === void 0 || /[\s:;.,!?()\d]/.test(next);
@@ -2672,6 +3208,28 @@ const extractValue = (text, triggerEnd, strategy, label) => {
2672
3208
  text: extracted
2673
3209
  };
2674
3210
  }
3211
+ case "match-pattern": {
3212
+ const nlIdx = valueText.indexOf("\n");
3213
+ const searchText = nlIdx === -1 ? valueText : valueText.slice(0, nlIdx);
3214
+ if (searchText.length === 0) return null;
3215
+ const flags = (strategy.flags ?? "").replace(/[gy]/g, "");
3216
+ const anchoredSource = `^(?:${strategy.pattern})`;
3217
+ let re;
3218
+ try {
3219
+ re = new RegExp(anchoredSource, flags);
3220
+ } catch {
3221
+ return null;
3222
+ }
3223
+ const m = re.exec(searchText);
3224
+ if (!m || m[0].length === 0) return null;
3225
+ const matchStart = m.index;
3226
+ const matchEnd = matchStart + m[0].length;
3227
+ return {
3228
+ start: valueStart + matchStart,
3229
+ end: valueStart + matchEnd,
3230
+ text: m[0]
3231
+ };
3232
+ }
2675
3233
  default: return null;
2676
3234
  }
2677
3235
  };
@@ -2691,7 +3249,8 @@ const processTriggerMatches = (allMatches, sliceStart, sliceEnd, fullText, rules
2691
3249
  if (match.start > 0 && LETTER_RE.test(fullText[match.start - 1] ?? "")) continue;
2692
3250
  const rule = rules[localIdx];
2693
3251
  if (!rule) continue;
2694
- if (!rule.trigger.endsWith(" ") && LETTER_RE.test(fullText[match.end] ?? "")) continue;
3252
+ const triggerLastChar = rule.trigger.at(-1) ?? "";
3253
+ if (LETTER_RE.test(triggerLastChar) && LETTER_RE.test(fullText[match.end] ?? "")) continue;
2695
3254
  const triggerEnd = match.end;
2696
3255
  const rawValue = extractValue(fullText, triggerEnd, rule.strategy, rule.label);
2697
3256
  const value = rawValue ? stripQuotes(rawValue) : null;
@@ -2700,7 +3259,7 @@ const processTriggerMatches = (allMatches, sliceStart, sliceEnd, fullText, rules
2700
3259
  const entityStart = rule.includeTrigger ? match.start : value.start;
2701
3260
  const entityEnd = value.end;
2702
3261
  const entityText = fullText.slice(entityStart, entityEnd);
2703
- const effectiveLabel = rule.label === "person" && LEGAL_FORM_CHECK_RE.test(entityText) ? "organization" : rule.label;
3262
+ const effectiveLabel = rule.label === "person" && getLegalFormCheckRe().test(entityText) ? "organization" : rule.label;
2704
3263
  results.push({
2705
3264
  start: entityStart,
2706
3265
  end: entityEnd,
@@ -2970,11 +3529,82 @@ const resolveCountries = (regions, countries) => {
2970
3529
  return result;
2971
3530
  };
2972
3531
  //#endregion
3532
+ //#region src/util/homoglyphs.ts
3533
+ const CHAR_MAP$1 = new Map([
3534
+ [1040, 65],
3535
+ [1042, 66],
3536
+ [1045, 69],
3537
+ [1050, 75],
3538
+ [1052, 77],
3539
+ [1053, 72],
3540
+ [1054, 79],
3541
+ [1056, 80],
3542
+ [1057, 67],
3543
+ [1058, 84],
3544
+ [1061, 88],
3545
+ [1072, 97],
3546
+ [1077, 101],
3547
+ [1086, 111],
3548
+ [1088, 112],
3549
+ [1089, 99],
3550
+ [1091, 121],
3551
+ [1093, 120],
3552
+ [1110, 105],
3553
+ [1112, 106],
3554
+ [913, 65],
3555
+ [914, 66],
3556
+ [917, 69],
3557
+ [919, 72],
3558
+ [921, 73],
3559
+ [922, 75],
3560
+ [924, 77],
3561
+ [925, 78],
3562
+ [927, 79],
3563
+ [929, 80],
3564
+ [932, 84],
3565
+ [933, 89],
3566
+ [935, 88],
3567
+ [918, 90],
3568
+ [945, 97],
3569
+ [954, 107],
3570
+ [959, 111],
3571
+ [961, 112]
3572
+ ]);
3573
+ /** Chunk size for `String.fromCharCode` to avoid hitting
3574
+ * the call-stack limit on very large strings. */
3575
+ const CHUNK_SIZE$1 = 8192;
3576
+ const normalizeHomoglyphs = (text) => {
3577
+ let hasSpecial = false;
3578
+ for (let i = 0; i < text.length; i++) if (CHAR_MAP$1.has(text.charCodeAt(i))) {
3579
+ hasSpecial = true;
3580
+ break;
3581
+ }
3582
+ if (!hasSpecial) return text;
3583
+ const len = text.length;
3584
+ const codes = new Uint16Array(len);
3585
+ for (let i = 0; i < len; i++) {
3586
+ const code = text.charCodeAt(i);
3587
+ codes[i] = CHAR_MAP$1.get(code) ?? code;
3588
+ }
3589
+ if (len <= CHUNK_SIZE$1) return String.fromCharCode(...codes);
3590
+ let result = "";
3591
+ for (let offset = 0; offset < len; offset += CHUNK_SIZE$1) {
3592
+ const end = Math.min(offset + CHUNK_SIZE$1, len);
3593
+ result += String.fromCharCode(...codes.subarray(offset, end));
3594
+ }
3595
+ return result;
3596
+ };
3597
+ //#endregion
2973
3598
  //#region src/filters/false-positives.ts
2974
3599
  const TEMPLATE_PLACEHOLDER_RE = /^(?:\.{3,}|_{3,}|\[[\w\s]+\]|\{[\w\s]+\})$/;
2975
3600
  const POSTAL_CODE_RE = /\d{3}\s?\d{2}/;
2976
3601
  const HAS_DIGIT_RE = /\d/;
2977
- const ADDRESS_COMPONENTS_RE = /(?:^|\s)(?:ul\.|ulice|nám\.|náměstí|tř\.|třída|nábř\.|nábřeží|č\.p\.|č\.ev\.|č\.|sídliště|bulvár)(?=[\s,./]|$)/i;
3602
+ const ADDRESS_COMPONENT_EXTRA_RE = /(?:^|\s)(?:č\.p\.|č\.ev\.|č\.|sídliště)(?=[\s,./]|$)/i;
3603
+ const BARE_COURS_PROSE_RE = /(?:^|\s)cours(?!\s+\p{Lu})(?=[\s,./]|$)/u;
3604
+ const isOnlyAmbiguousCours = (text) => {
3605
+ if (!BARE_COURS_PROSE_RE.test(text)) return false;
3606
+ return !hasAddressComponent(text.replace(/(?:^|\s)cours(?=[\s,./]|$)/gi, " "));
3607
+ };
2978
3608
  const JURISDICTION_RE = /^(?:state|commonwealth|district|territory)\s+of\s+/i;
2979
3609
  const MAX_ENTITY_LENGTH = {
2980
3610
  organization: 80,
@@ -3057,6 +3687,41 @@ const loadGenericRoles = (ctx = defaultContext) => {
3057
3687
  };
3058
3688
  /** Sync accessor — returns empty set before init. */
3059
3689
  const getGenericRoles = (ctx) => ctx.genericRoles ?? EMPTY_GENERIC_ROLES;
3690
+ const STREET_TYPES_SEED_RE = /(?:^|\s)(?:ul\.|ulice|nám\.|náměstí|tř\.|třída|nábř\.|nábřeží|bulvár)(?=[\s,./]|$)/i;
3691
+ let _streetTypesRe = STREET_TYPES_SEED_RE;
3692
+ let _streetTypesPromise = null;
3693
+ const escapeRegex$1 = (s) => s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
3694
+ const loadStreetTypeRegex = async () => {
3695
+ try {
3696
+ const data = (await import("./address-street-types.mjs")).default ?? {};
3697
+ const words = /* @__PURE__ */ new Set();
3698
+ for (const [key, val] of Object.entries(data)) {
3699
+ if (key.startsWith("_")) continue;
3700
+ if (!Array.isArray(val)) continue;
3701
+ for (const w of val) if (typeof w === "string" && w.length > 0) words.add(w.toLowerCase());
3702
+ }
3703
+ if (words.size === 0) {
3704
+ _streetTypesRe = STREET_TYPES_SEED_RE;
3705
+ return;
3706
+ }
3707
+ const alternation = [...words].sort((a, b) => b.length - a.length).map(escapeRegex$1).join("|");
3708
+ _streetTypesRe = new RegExp(`(?:^|\\s)(?:${alternation})(?=[\\s,./]|$)`, "iu");
3709
+ } catch {
3710
+ _streetTypesRe = STREET_TYPES_SEED_RE;
3711
+ }
3712
+ };
3713
+ /** Ensure street-type vocabulary is loaded. */
3714
+ const initAddressComponents = () => {
3715
+ if (!_streetTypesPromise) _streetTypesPromise = loadStreetTypeRegex();
3716
+ return _streetTypesPromise;
3717
+ };
3718
+ /**
3719
+ * True when `text` contains a recognised address
3720
+ * component: either a per-language street word (loaded
3721
+ * from JSON) or a language-agnostic anchor such as a
3722
+ * Czech house number form.
3723
+ */
3724
+ const hasAddressComponent = (text) => _streetTypesRe.test(text) || ADDRESS_COMPONENT_EXTRA_RE.test(text);
3060
3725
  const isCallerOwnedEntity$3 = (entity) => entity.sourceDetail === "custom-deny-list" || entity.sourceDetail === "custom-regex";
3061
3726
  /**
3062
3727
  * Filter out entities that are likely false positives:
@@ -3091,13 +3756,15 @@ const filterFalsePositives = (entities, ctx = defaultContext, fullText) => {
3091
3756
  if (normalized.label === "person" && HAS_DIGIT_RE.test(trimmed)) continue;
3092
3757
  if (normalized.label === "person") {
3093
3758
  const tokens = trimmed.split(/\s+/u);
3094
- const last = tokens.at(-1)?.replace(/[.,;:!?]+$/u, "").toLowerCase();
3095
- if (tokens.length > 1 && last && PERSON_TRAILING_NOUNS.has(last)) continue;
3759
+ const last = tokens.at(-1)?.replace(/[.,;:!?]+$/u, "");
3760
+ const lastFolded = last ? normalizeHomoglyphs(last).toLowerCase() : void 0;
3761
+ if (tokens.length > 1 && lastFolded && PERSON_TRAILING_NOUNS.has(lastFolded)) continue;
3096
3762
  }
3097
- if ((normalized.label === "person" || normalized.label === "organization") && roles.has(trimmed.toLowerCase())) continue;
3763
+ if ((normalized.label === "person" || normalized.label === "organization") && roles.has(normalizeHomoglyphs(trimmed).toLowerCase())) continue;
3098
3764
  if (normalized.label === "organization" && normalized.source === "legal-form" && trimmed === trimmed.toUpperCase() && LEGAL_FORM_HEADING_RE.test(trimmed)) continue;
3099
- if (normalized.label === "address" && trimmed.length > 40 && !POSTAL_CODE_RE.test(trimmed) && !HAS_DIGIT_RE.test(trimmed) && !ADDRESS_COMPONENTS_RE.test(trimmed) && !JURISDICTION_RE.test(trimmed)) continue;
3100
- if (normalized.label === "address" && normalized.source === "trigger" && !HAS_DIGIT_RE.test(trimmed) && !ADDRESS_COMPONENTS_RE.test(trimmed) && !JURISDICTION_RE.test(trimmed)) continue;
3765
+ if (normalized.label === "address" && trimmed.length > 40 && !POSTAL_CODE_RE.test(trimmed) && !HAS_DIGIT_RE.test(trimmed) && !hasAddressComponent(trimmed) && !JURISDICTION_RE.test(trimmed)) continue;
3766
+ if (normalized.label === "address" && normalized.source === "trigger" && !HAS_DIGIT_RE.test(trimmed) && !hasAddressComponent(trimmed) && !JURISDICTION_RE.test(trimmed)) continue;
3767
+ if (normalized.label === "address" && normalized.source === "trigger" && !HAS_DIGIT_RE.test(trimmed) && isOnlyAmbiguousCours(trimmed)) continue;
3101
3768
  if (normalized.label === "address" && SIGNING_CLAUSE_ADDRESS_RE.test(trimmed)) continue;
3102
3769
  filtered.push(normalized);
3103
3770
  }
@@ -3254,6 +3921,122 @@ const loadPersonStopwords = (ctx) => {
3254
3921
  const EMPTY_PERSON_STOPWORDS = /* @__PURE__ */ new Set();
3255
3922
  /** Sync accessor — returns empty set before init. */
3256
3923
  const getPersonStopwords = (ctx) => ctx.personStopwords ?? EMPTY_PERSON_STOPWORDS;
3924
+ const loadAddressStopwords = (ctx) => {
3925
+ if (ctx.addressStopwordsPromise) return ctx.addressStopwordsPromise;
3926
+ ctx.addressStopwordsPromise = (async () => {
3927
+ try {
3928
+ const mod = await import("./address-stopwords.mjs");
3929
+ const set = new Set(mod.default?.words ?? []);
3930
+ ctx.addressStopwords = set;
3931
+ return set;
3932
+ } catch {
3933
+ const empty = /* @__PURE__ */ new Set();
3934
+ ctx.addressStopwords = empty;
3935
+ return empty;
3936
+ }
3937
+ })();
3938
+ return ctx.addressStopwordsPromise;
3939
+ };
3940
+ const EMPTY_ADDRESS_STOPWORDS = /* @__PURE__ */ new Set();
3941
+ const getAddressStopwords = (ctx) => ctx.addressStopwords ?? EMPTY_ADDRESS_STOPWORDS;
3942
+ /**
3943
+ * Word characters in unicode property notation. The check is
3944
+ * "single-token" — no internal whitespace — and we keep dashes
3945
+ * tokens like "K-12" out by requiring the surface to be a single
3946
+ * uninterrupted run of letters or marks.
3947
+ */
3948
+ const SINGLE_WORD_RE = /^\p{L}+$/u;
3949
+ /**
3950
+ * Format-level address signals — structurally numeric or
3951
+ * 2-letter-state patterns, language-agnostic. The street-type
3952
+ * vocabulary is loaded from `address-street-types.json` so new
3953
+ * languages contribute via data, not code.
3954
+ * - `,\s*[A-Z]{2}\b` → US state abbreviation after a comma
3955
+ * - `\b\d{5}(?:-\d{4})?\b` → US ZIP / ZIP+4
3956
+ * - `\b\d{3}\s\d{2}\b` → Czech/Slovak postal block (140 00)
3957
+ * - `\b\d{2}-\d{3}\b` → Polish postal code (00-950)
3958
+ */
3959
+ const ADDRESS_FORMAT_RE = /,\s*\p{Lu}{2}\b|\b\d{5}(?:-\d{4})?\b|\b\d{3}\s\d{2}\b|\b\d{2}-\d{3}\b/u;
3960
+ let cachedStreetTypeRe = null;
3961
+ let streetTypeReLoaded = false;
3962
+ const loadStreetTypeRe = async () => {
3963
+ if (streetTypeReLoaded) return cachedStreetTypeRe;
3964
+ try {
3965
+ const config = (await import("./address-street-types.mjs")).default ?? {};
3966
+ const words = [];
3967
+ for (const value of Object.values(config)) {
3968
+ if (!Array.isArray(value)) continue;
3969
+ for (const word of value) if (typeof word === "string" && word.length > 0) words.push(word);
3970
+ }
3971
+ if (words.length === 0) cachedStreetTypeRe = null;
3972
+ else {
3973
+ words.sort((a, b) => b.length - a.length);
3974
+ const isLetterDigit = (c) => /[\p{L}\p{N}]/u.test(c);
3975
+ const wordLikeTail = [];
3976
+ const punctTail = [];
3977
+ for (const w of words) {
3978
+ const escaped = w.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
3979
+ if (isLetterDigit(w.at(-1) ?? "")) wordLikeTail.push(escaped);
3980
+ else punctTail.push(escaped);
3981
+ }
3982
+ const branches = [];
3983
+ if (wordLikeTail.length > 0) branches.push(`(?:${wordLikeTail.join("|")})(?![\\p{L}\\p{N}])`);
3984
+ if (punctTail.length > 0) branches.push(`(?:${punctTail.join("|")})`);
3985
+ cachedStreetTypeRe = new RegExp(`(?<![\\p{L}\\p{N}])(?:${branches.join("|")})`, "iu");
3986
+ }
3987
+ } catch {
3988
+ cachedStreetTypeRe = null;
3989
+ }
3990
+ streetTypeReLoaded = true;
3991
+ return cachedStreetTypeRe;
3992
+ };
3993
+ const getStreetTypeRe = () => streetTypeReLoaded ? cachedStreetTypeRe : null;
3994
+ const hasAdjacentAddressEvidence = (fullText, start, end) => {
3995
+ const window = fullText.slice(Math.max(0, start - 40), Math.min(fullText.length, end + 40));
3996
+ if (ADDRESS_FORMAT_RE.test(window)) return true;
3997
+ const streetRe = getStreetTypeRe();
3998
+ return streetRe !== null && streetRe.test(window);
3999
+ };
4000
+ /**
4001
+ * Capitalised words that almost never start a person name. When a
4002
+ * single-token surname candidate is immediately followed by one of
4003
+ * these, the "next-word is uppercase" promotion heuristic would
4004
+ * otherwise turn section headings ("Purchase Price↵The Purchaser
4005
+ * undertakes…") into spurious person hits. Kept narrow on purpose;
4006
+ * the surrounding pipeline still chains real names via the deny-list
4007
+ * cascade when both halves are surnames.
4008
+ */
4009
+ const SENTENCE_STARTER_WORDS = new Set([
4010
+ "The",
4011
+ "This",
4012
+ "These",
4013
+ "Those",
4014
+ "An",
4015
+ "Any",
4016
+ "All",
4017
+ "Each",
4018
+ "Every",
4019
+ "No",
4020
+ "Now",
4021
+ "Whereas",
4022
+ "Whereby",
4023
+ "Wherein",
4024
+ "Whereof",
4025
+ "Notwithstanding",
4026
+ "Subject",
4027
+ "In",
4028
+ "On",
4029
+ "At",
4030
+ "By",
4031
+ "For",
4032
+ "If",
4033
+ "Upon",
4034
+ "Unless",
4035
+ "Until",
4036
+ "Provided",
4037
+ "Pursuant",
4038
+ "Such"
4039
+ ]);
3257
4040
  const PERSON_CHAIN_BREAK_RE = /[!?;:]|,/u;
3258
4041
  const WORD_CHAR_RE$1 = /[\p{L}\p{N}]/u;
3259
4042
  const isInitialContinuationGap = (text, gap) => /^\p{Lu}$/u.test(text) && /^\.[^\S\n]{1,2}$/u.test(gap) || /^[^\S\n]{1,2}(?:\p{Lu}\.[^\S\n]{1,2})+$/u.test(gap);
@@ -3273,6 +4056,8 @@ const buildDenyList = async (config, ctx = defaultContext) => {
3273
4056
  loadStopwords(ctx),
3274
4057
  loadAllowList(ctx),
3275
4058
  loadPersonStopwords(ctx),
4059
+ loadAddressStopwords(ctx),
4060
+ loadStreetTypeRe(),
3276
4061
  loadGenericRoles(ctx)
3277
4062
  ]);
3278
4063
  const dictionaries = config.dictionaries;
@@ -3419,6 +4204,8 @@ const ensureDenyListData = async (ctx = defaultContext, dictionaries) => {
3419
4204
  loadStopwords(ctx),
3420
4205
  loadAllowList(ctx),
3421
4206
  loadPersonStopwords(ctx),
4207
+ loadAddressStopwords(ctx),
4208
+ loadStreetTypeRe(),
3422
4209
  loadGenericRoles(ctx)
3423
4210
  ]);
3424
4211
  };
@@ -3486,14 +4273,18 @@ const processDenyListMatches = (allMatches, sliceStart, sliceEnd, fullText, data
3486
4273
  if (!getPersonStopwords(ctx).has(keyword)) nameHits.push(m);
3487
4274
  }
3488
4275
  const nonPersonLabels = m.labels.filter((l) => l !== "person");
3489
- for (const label of nonPersonLabels) results.push({
3490
- start: m.start,
3491
- end: m.end,
3492
- label,
3493
- text: m.text,
3494
- score: .9,
3495
- source: DETECTION_SOURCES.DENY_LIST
3496
- });
4276
+ const suppressAddress = SINGLE_WORD_RE.test(m.text) && getAddressStopwords(ctx).has(m.text.toLowerCase()) && !hasAdjacentAddressEvidence(fullText, m.start, m.end);
4277
+ for (const label of nonPersonLabels) {
4278
+ if (label === "address" && suppressAddress) continue;
4279
+ results.push({
4280
+ start: m.start,
4281
+ end: m.end,
4282
+ label,
4283
+ text: m.text,
4284
+ score: .9,
4285
+ source: DETECTION_SOURCES.DENY_LIST
4286
+ });
4287
+ }
3497
4288
  }
3498
4289
  }
3499
4290
  nameHits.sort((a, b) => a.start - b.start);
@@ -3525,6 +4316,8 @@ const processDenyListMatches = (allMatches, sliceStart, sliceEnd, fullText, data
3525
4316
  const afterEnd = last.end;
3526
4317
  const rest = fullText.slice(afterEnd).trimStart();
3527
4318
  if (!(rest.length > 1 && /^\p{Lu}\p{Ll}/u.test(rest))) continue;
4319
+ const nextWord = /^\p{L}+/u.exec(rest)?.[0] ?? "";
4320
+ if (SENTENCE_STARTER_WORDS.has(nextWord)) continue;
3528
4321
  }
3529
4322
  results.push({
3530
4323
  start: first.start,
@@ -3636,6 +4429,15 @@ const extendPersonName = (text, start, end, ctx) => {
3636
4429
  };
3637
4430
  //#endregion
3638
4431
  //#region src/detectors/address-seeds.ts
4432
+ /**
4433
+ * Trailing chars that should never end an address span. Combines
4434
+ * structural separators, opening brackets, and typographic quote
4435
+ * variants so a span like `… GA 30326, USA (the "Premises")`
4436
+ * does not pull the parenthetical opener into the address.
4437
+ * The prime (`′`) catches measurement tails (`5′`) that the
4438
+ * cluster occasionally absorbs.
4439
+ */
4440
+ const ADDRESS_TRAILING_TRIM_RE = new RegExp(`[,;:\\s${OPENING_BRACKETS_INNER}${QUOTE_DOUBLE_INNER}${QUOTE_SINGLE_INNER}′]`, "u");
3639
4441
  let cachedBoundaryRe = null;
3640
4442
  const loadBoundaryWords = async () => {
3641
4443
  try {
@@ -3644,6 +4446,56 @@ const loadBoundaryWords = async () => {
3644
4446
  return {};
3645
4447
  }
3646
4448
  };
4449
+ const BR_CEP_CONTEXT_WINDOW = 200;
4450
+ let cachedBrCepContextRe = null;
4451
+ let cachedBrCepContextPromise = null;
4452
+ const loadBrCueWords = async () => {
4453
+ const sources = await Promise.all([(async () => {
4454
+ try {
4455
+ return (await import("./address-street-types.mjs")).default["pt-br"];
4456
+ } catch {
4457
+ return;
4458
+ }
4459
+ })(), (async () => {
4460
+ try {
4461
+ return (await import("./address-boundaries.mjs")).default["pt-br"];
4462
+ } catch {
4463
+ return;
4464
+ }
4465
+ })()]);
4466
+ const out = [];
4467
+ const seen = /* @__PURE__ */ new Set();
4468
+ for (const entry of sources) {
4469
+ if (!Array.isArray(entry)) continue;
4470
+ for (const word of entry) {
4471
+ if (typeof word !== "string" || word.length === 0) continue;
4472
+ const key = word.toLowerCase();
4473
+ if (seen.has(key)) continue;
4474
+ seen.add(key);
4475
+ out.push(word);
4476
+ }
4477
+ }
4478
+ return out;
4479
+ };
4480
+ const getBrCepContextRe = async () => {
4481
+ if (cachedBrCepContextRe !== null) return cachedBrCepContextRe;
4482
+ if (cachedBrCepContextPromise) return cachedBrCepContextPromise;
4483
+ cachedBrCepContextPromise = (async () => {
4484
+ const words = await loadBrCueWords();
4485
+ if (words.length === 0) return null;
4486
+ const escaped = words.toSorted((a, b) => b.length - a.length).map((w) => w.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"));
4487
+ const re = new RegExp(`(?<![\\p{L}\\p{N}])(?:${escaped.join("|")})(?![\\p{L}\\p{N}])`, "iu");
4488
+ cachedBrCepContextRe = re;
4489
+ return re;
4490
+ })();
4491
+ return cachedBrCepContextPromise;
4492
+ };
4493
+ const hasBrCueNearby = (fullText, start, end, re) => {
4494
+ const windowStart = Math.max(0, start - BR_CEP_CONTEXT_WINDOW);
4495
+ const windowEnd = Math.min(fullText.length, end + BR_CEP_CONTEXT_WINDOW);
4496
+ const window = fullText.slice(windowStart, windowEnd);
4497
+ return new RegExp(re.source, re.flags.replace("g", "")).test(window);
4498
+ };
3647
4499
  /**
3648
4500
  * Build regex for boundary words. Matches any
3649
4501
  * boundary word preceded by a word boundary.
@@ -3657,7 +4509,7 @@ const getBoundaryRe = async () => {
3657
4509
  for (const word of entries) words.push(word.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"));
3658
4510
  }
3659
4511
  words.sort((a, b) => b.length - a.length);
3660
- cachedBoundaryRe = words.length > 0 ? new RegExp(`\\b(?:${words.join("|")})\\b`, "i") : /(?!)/;
4512
+ cachedBoundaryRe = words.length > 0 ? new RegExp(`(?<![\\p{L}\\p{N}])(?:${words.join("|")})(?![\\p{L}\\p{N}])`, "iu") : /(?!)/;
3661
4513
  return cachedBoundaryRe;
3662
4514
  };
3663
4515
  /**
@@ -3679,7 +4531,7 @@ const buildStreetTypePatterns = async () => {
3679
4531
  }
3680
4532
  return words;
3681
4533
  };
3682
- const collectSeeds = (allMatches, sliceStart, sliceEnd, fullText, existingEntities) => {
4534
+ const collectSeeds = (allMatches, sliceStart, sliceEnd, fullText, existingEntities, brCepContextRe) => {
3683
4535
  const seeds = [];
3684
4536
  for (const match of allMatches) {
3685
4537
  const idx = match.pattern;
@@ -3713,18 +4565,41 @@ const collectSeeds = (allMatches, sliceStart, sliceEnd, fullText, existingEntiti
3713
4565
  text: e.text
3714
4566
  });
3715
4567
  }
3716
- const postalRe = /\b(?:\d{3}\s\d{2}|\d{2}-\d{3})\b/g;
4568
+ const postalRe = /\b(?:\d{3}\s\d{2}|\d{2}-\d{3}|\d{5}-\d{3})\b/g;
3717
4569
  let postalMatch;
3718
4570
  while ((postalMatch = postalRe.exec(fullText)) !== null) {
3719
4571
  const start = postalMatch.index;
3720
4572
  const end = start + postalMatch[0].length;
3721
- if (!seeds.some((s) => s.start <= start && s.end >= end)) seeds.push({
4573
+ if (seeds.some((s) => s.start <= start && s.end >= end)) continue;
4574
+ if (/^\d{5}-\d{3}$/.test(postalMatch[0]) && (brCepContextRe === null || !hasBrCueNearby(fullText, start, end, brCepContextRe))) continue;
4575
+ seeds.push({
3722
4576
  type: "postal-code",
3723
4577
  start,
3724
4578
  end,
3725
4579
  text: postalMatch[0]
3726
4580
  });
3727
4581
  }
4582
+ const itCapRe = /\b\d{5}(?=\s+\p{Lu}\p{L}+)/gu;
4583
+ let itCapMatch;
4584
+ while ((itCapMatch = itCapRe.exec(fullText)) !== null) {
4585
+ const start = itCapMatch.index;
4586
+ const end = start + itCapMatch[0].length;
4587
+ if (seeds.some((s) => s.start <= start && s.end >= end)) continue;
4588
+ if (!seeds.some((s) => {
4589
+ if (Math.abs(s.start - start) > 80) return false;
4590
+ if (s.type === "address-trigger") return true;
4591
+ if (s.type === "city") return true;
4592
+ if (s.type === "postal-code") return true;
4593
+ if (s.type === "street-word") return s.text.toLowerCase() !== "via";
4594
+ return false;
4595
+ })) continue;
4596
+ seeds.push({
4597
+ type: "postal-code",
4598
+ start,
4599
+ end,
4600
+ text: itCapMatch[0]
4601
+ });
4602
+ }
3728
4603
  const streetNumRe = /\b(\p{Lu}\p{Ll}{2,})\s+(\d{1,5}(?:\/\d{1,5})?)\s*[,\n]/gu;
3729
4604
  let streetMatch;
3730
4605
  while ((streetMatch = streetNumRe.exec(fullText)) !== null) {
@@ -3782,6 +4657,10 @@ const scoreCluster = (cluster) => {
3782
4657
  const NON_ADDRESS_LABELS = new Set([
3783
4658
  "registration number",
3784
4659
  "tax identification number",
4660
+ "national identification number",
4661
+ "social security number",
4662
+ "birth number",
4663
+ "identity card number",
3785
4664
  "person",
3786
4665
  "bank account number",
3787
4666
  "email address",
@@ -3818,7 +4697,7 @@ const expandCluster = async (fullText, cluster, existingEntities) => {
3818
4697
  const doubleNewline = remaining.indexOf("\n\n");
3819
4698
  if (doubleNewline !== -1 && doubleNewline < nearestBoundary) nearestBoundary = doubleNewline;
3820
4699
  rightPos = end + remaining.slice(0, nearestBoundary).trimEnd().length;
3821
- while (rightPos > end && /[,;:\s]/.test(fullText[rightPos - 1] ?? "")) rightPos--;
4700
+ while (rightPos > end && ADDRESS_TRAILING_TRIM_RE.test(fullText[rightPos - 1] ?? "")) rightPos--;
3822
4701
  return {
3823
4702
  start: Math.min(leftPos, start),
3824
4703
  end: Math.max(rightPos, end)
@@ -3835,7 +4714,7 @@ const expandCluster = async (fullText, cluster, existingEntities) => {
3835
4714
  * using their output as seed sources.
3836
4715
  */
3837
4716
  const processAddressSeeds = async (allMatches, sliceStart, sliceEnd, fullText, existingEntities) => {
3838
- const clusters = clusterSeeds(collectSeeds(allMatches, sliceStart, sliceEnd, fullText, existingEntities), 150);
4717
+ const clusters = clusterSeeds(collectSeeds(allMatches, sliceStart, sliceEnd, fullText, existingEntities, await getBrCepContextRe()), 150);
3839
4718
  const results = [];
3840
4719
  for (const cluster of clusters) {
3841
4720
  const score = scoreCluster(cluster);
@@ -3860,6 +4739,7 @@ const processAddressSeeds = async (allMatches, sliceStart, sliceEnd, fullText, e
3860
4739
  const TRAILING_SEP = /[,\s]+$/;
3861
4740
  const WORD_CHAR_RE = /[\p{L}\p{N}]/u;
3862
4741
  const ORG_PROPAGATION_SCORE = .9;
4742
+ const ORG_DETERMINER_RE = /(?<![\p{L}\p{N}])(společnost(?:i|í|em|u)?|spolecnost(?:i|em|u)?|the\s+(?:company|corporation|firm)|die\s+(?:gesellschaft|firma)|la\s+(?:société|empresa|sociedad)|el\s+(?:empresa|sociedad))\s+$/iu;
3863
4743
  const isCallerOwnedEntity$2 = (entity) => entity.sourceDetail === "custom-deny-list" || entity.sourceDetail === "custom-regex";
3864
4744
  /**
3865
4745
  * After the main detection pass, collect organization
@@ -3903,16 +4783,25 @@ const propagateOrgNames = (entities, fullText) => {
3903
4783
  searchFrom = idx + 1;
3904
4784
  continue;
3905
4785
  }
3906
- if (!isOverlapping(idx, matchEnd)) {
4786
+ let spanStart = idx;
4787
+ const lookbackStart = Math.max(0, idx - 40);
4788
+ const lookback = fullText.slice(lookbackStart, idx);
4789
+ const determinerMatch = ORG_DETERMINER_RE.exec(lookback);
4790
+ if (determinerMatch !== null) {
4791
+ const determiner = determinerMatch[1] ?? "";
4792
+ const offsetInMatch = determinerMatch[0].indexOf(determiner);
4793
+ spanStart = lookbackStart + determinerMatch.index + offsetInMatch;
4794
+ }
4795
+ if (!isOverlapping(spanStart, matchEnd)) {
3907
4796
  results.push({
3908
- start: idx,
4797
+ start: spanStart,
3909
4798
  end: matchEnd,
3910
4799
  label,
3911
- text: baseName,
4800
+ text: fullText.slice(spanStart, matchEnd),
3912
4801
  score: ORG_PROPAGATION_SCORE,
3913
4802
  source: DETECTION_SOURCES.COREFERENCE
3914
4803
  });
3915
- covered.push([idx, matchEnd]);
4804
+ covered.push([spanStart, matchEnd]);
3916
4805
  }
3917
4806
  searchFrom = matchEnd;
3918
4807
  }
@@ -5077,13 +5966,89 @@ const shouldReplace = (a, b) => {
5077
5966
  };
5078
5967
  /** Labels where colons are structurally significant. */
5079
5968
  const COLON_LABELS = new Set(["ip address", "mac address"]);
5969
+ /**
5970
+ * Labels whose detectors emit precise, evidence-backed spans. When
5971
+ * one of these fires at the exact same offsets as a fuzzier
5972
+ * `address` hit (city dictionary lookup, address-seed cluster), the
5973
+ * `address` label is almost always a dictionary collision — "March
5974
+ * 15" the date getting tagged as an address because "March" appears
5975
+ * in a city corpus, or "Brent Phillips" emitted as both `person`
5976
+ * and `address` because "Brent" is a UK city. The precise detector
5977
+ * wins.
5978
+ */
5979
+ const PRECISE_OVER_ADDRESS = new Set([
5980
+ "person",
5981
+ "date",
5982
+ "date of birth",
5983
+ "phone number",
5984
+ "email address",
5985
+ "monetary amount",
5986
+ "iban",
5987
+ "bank account number",
5988
+ "tax identification number",
5989
+ "registration number",
5990
+ "identity card number",
5991
+ "national identification number",
5992
+ "passport number",
5993
+ "credit card number"
5994
+ ]);
5995
+ /**
5996
+ * Labels the `person` chain wins against at identical offsets. The
5997
+ * chain carries adjacent-name evidence (deny-list surname plus a
5998
+ * capitalised follow-up) a single-token dictionary collision does
5999
+ * not. Kept narrow: organizations are NOT here — "Morgan Stanley"
6000
+ * legitimately appears in both the org and name dictionaries, and
6001
+ * the existing detector priority is the right tie-breaker there.
6002
+ */
6003
+ const PERSON_PREFERRED_OVER = new Set(["address", "land parcel"]);
6004
+ const resolveSameSpanLabelConflicts = (entities) => {
6005
+ if (entities.length < 2) return entities;
6006
+ const byOffsets = /* @__PURE__ */ new Map();
6007
+ for (const entity of entities) {
6008
+ const key = `${entity.start}:${entity.end}`;
6009
+ const list = byOffsets.get(key);
6010
+ if (list) list.push(entity);
6011
+ else byOffsets.set(key, [entity]);
6012
+ }
6013
+ const dropped = /* @__PURE__ */ new Set();
6014
+ for (const [, group] of byOffsets) {
6015
+ if (group.length < 2) continue;
6016
+ const labels = new Set(group.map((e) => e.label));
6017
+ if (labels.size < 2) continue;
6018
+ const hasPerson = labels.has("person");
6019
+ const hasPreciseNonAddress = [...labels].some((l) => l !== "address" && PRECISE_OVER_ADDRESS.has(l));
6020
+ let maxPriority = -1;
6021
+ for (const e of group) {
6022
+ if (hasLockedBoundary(e)) continue;
6023
+ const pri = DETECTOR_PRIORITY[e.source] ?? 0;
6024
+ if (pri > maxPriority) maxPriority = pri;
6025
+ }
6026
+ for (const e of group) {
6027
+ if (hasLockedBoundary(e)) continue;
6028
+ if ((DETECTOR_PRIORITY[e.source] ?? 0) < maxPriority) {
6029
+ dropped.add(e);
6030
+ continue;
6031
+ }
6032
+ if (hasPerson && PERSON_PREFERRED_OVER.has(e.label)) {
6033
+ dropped.add(e);
6034
+ continue;
6035
+ }
6036
+ if (hasPreciseNonAddress && e.label === "address") dropped.add(e);
6037
+ }
6038
+ }
6039
+ if (dropped.size === 0) return entities;
6040
+ return entities.filter((e) => !dropped.has(e));
6041
+ };
5080
6042
  /** Strip leading/trailing whitespace and punctuation. */
5081
6043
  const sanitizeEntities = (entities) => entities.flatMap((e) => {
5082
6044
  if (hasLockedBoundary(e)) return [e];
5083
6045
  const strip = COLON_LABELS.has(e.label) ? /[\s,;]+/ : /[\s:,;]+/;
5084
6046
  const leadTrimmed = e.text.replace(/^(?:\.\s)+/, "").replace(new RegExp(`^${strip.source}`, strip.flags), "");
5085
6047
  const lead = e.text.length - leadTrimmed.length;
5086
- const cleaned = leadTrimmed.replace(new RegExp(`${strip.source}$`, strip.flags), "");
6048
+ let cleaned = leadTrimmed.replace(new RegExp(`${strip.source}$`, strip.flags), "");
6049
+ if (e.label === "organization" && cleaned.endsWith(".") && !LITERAL_SOURCES.has(e.source)) {
6050
+ if (!getKnownLegalSuffixes().some((suffix) => cleaned.endsWith(suffix))) cleaned = cleaned.slice(0, -1).trimEnd();
6051
+ }
5087
6052
  if (cleaned.length === 0) return [];
5088
6053
  if (!/[\p{L}\p{N}]/u.test(cleaned)) return [];
5089
6054
  const collapsed = cleaned.replace(/\s*\n\s*/g, " ").replace(/\s{2,}/g, " ");
@@ -5132,7 +6097,7 @@ const mergeAndDedup = (...layers) => {
5132
6097
  }
5133
6098
  if (overlaps.every((existing) => shouldReplace(entity, existing))) merged.splice(overlapStart, overlaps.length, { ...entity });
5134
6099
  }
5135
- return sanitizeEntities(merged);
6100
+ return resolveSameSpanLabelConflicts(sanitizeEntities(merged));
5136
6101
  };
5137
6102
  const escapeRegex = (s) => s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
5138
6103
  let amountWordsRe = null;
@@ -5168,9 +6133,10 @@ const filterAllowedLabels = (entities, allowedLabels) => {
5168
6133
  return entities.filter((e) => allowedLabels.has(e.label));
5169
6134
  };
5170
6135
  const labelIsAllowed = (label, allowedLabels) => !allowedLabels || allowedLabels.has(label);
6136
+ const NON_NER_LABELS = new Set(["misc"]);
5171
6137
  const getRequestedNerLabels = (config, expandForHotwords = false) => {
5172
6138
  const labels = config.labels.length > 0 ? config.labels : DEFAULT_ENTITY_LABELS;
5173
- return expandForHotwords ? expandLabelsForHotwordRules(labels) : labels;
6139
+ return (expandForHotwords ? expandLabelsForHotwordRules(labels) : labels).filter((label) => !NON_NER_LABELS.has(label));
5174
6140
  };
5175
6141
  const checkAbort = (signal) => {
5176
6142
  if (signal?.aborted) throw new DOMException("Pipeline aborted", "AbortError");
@@ -5251,6 +6217,8 @@ const runPipeline = async (options) => {
5251
6217
  loadGenericRoles(ctx),
5252
6218
  initPrepositions(),
5253
6219
  initStreetAbbrevs(),
6220
+ initAddressComponents(),
6221
+ warmAddressStopKeywords(),
5254
6222
  zoneInit,
5255
6223
  hotwordInit
5256
6224
  ]);
@@ -5258,6 +6226,8 @@ const runPipeline = async (options) => {
5258
6226
  loadGenericRoles(ctx),
5259
6227
  initPrepositions(),
5260
6228
  initStreetAbbrevs(),
6229
+ initAddressComponents(),
6230
+ warmAddressStopKeywords(),
5261
6231
  hotwordInit
5262
6232
  ]);
5263
6233
  if (cachedSearch && config.enableDenyList) await ensureDenyListData(ctx, config.dictionaries);
@@ -5277,9 +6247,11 @@ const runPipeline = async (options) => {
5277
6247
  const customRegexEntities = filterAllowedLabels(config.enableRegex ? processRegexMatches(customRegexMatches, slices.customRegex.start, slices.customRegex.end, search.customRegexMeta) : [], preHotwordAllowedLabels);
5278
6248
  const regexEntities = filterAllowedLabels([...rawRegexEntities, ...customRegexEntities], preHotwordAllowedLabels);
5279
6249
  if (regexEntities.length > 0) log("regex", `${regexEntities.length} matches`);
6250
+ if (legalFormsEnabled || config.enableTriggerPhrases) await warmLegalRoleHeads();
5280
6251
  const rawLegalFormEntities = legalFormsEnabled ? processLegalFormMatches(regexMatches, slices.legalForms.start, slices.legalForms.end, fullText) : [];
5281
6252
  const legalFormEntities = filterAllowedLabels(rawLegalFormEntities, preHotwordAllowedLabels);
5282
6253
  if (legalFormEntities.length > 0) log("legal-forms", `${legalFormEntities.length} matches`);
6254
+ if (config.enableTriggerPhrases) await warmAddressStopKeywords();
5283
6255
  const rawTriggerEntities = config.enableTriggerPhrases ? processTriggerMatches(regexMatches, slices.triggers.start, slices.triggers.end, fullText, search.triggerRules) : [];
5284
6256
  const triggerEntities = filterAllowedLabels(rawTriggerEntities, preHotwordAllowedLabels);
5285
6257
  if (triggerEntities.length > 0) log("trigger-phrases", `${triggerEntities.length} matches`);
@@ -5325,10 +6297,11 @@ const runPipeline = async (options) => {
5325
6297
  ];
5326
6298
  let rawNerEntities = [];
5327
6299
  let nerEntities = [];
5328
- if (config.enableNer && nerInference) {
6300
+ const requestedNerLabels = getRequestedNerLabels(config, hotwordsActive);
6301
+ if (config.enableNer && nerInference && requestedNerLabels.length > 0) {
5329
6302
  const maskResult = maskDetectedSpans(fullText, nerMaskEntities);
5330
6303
  log("ner", "running inference...");
5331
- const rawNer = await nerInference(maskResult.maskedText, [...getRequestedNerLabels(config, hotwordsActive)], config.threshold, signal);
6304
+ const rawNer = await nerInference(maskResult.maskedText, [...requestedNerLabels], config.threshold, signal);
5332
6305
  rawNerEntities = unmaskNerEntities(rawNer, maskResult, fullText);
5333
6306
  nerEntities = filterAllowedLabels(rawNerEntities, preHotwordAllowedLabels);
5334
6307
  const masked = rawNer.length - rawNerEntities.length;
@@ -5389,6 +6362,7 @@ const runPipeline = async (options) => {
5389
6362
  checkAbort(signal);
5390
6363
  ctx.corefSourceMap.clear();
5391
6364
  if (config.enableCoreference) {
6365
+ if (!legalFormsEnabled) await warmLegalRoleHeads();
5392
6366
  const terms = await extractDefinedTerms(fullText, merged.filter((entity) => !isCallerOwnedEntity(entity)), ctx);
5393
6367
  if (terms.length > 0) {
5394
6368
  log("coreference", `${terms.length} defined terms`);
@@ -5429,7 +6403,7 @@ const resolveOperator = (config, label) => config.operators[label] ?? "replace";
5429
6403
  //#region src/redact.ts
5430
6404
  const WHITESPACE_RE = /\s+/g;
5431
6405
  const PHONE_NOISE_RE = /[()\s-]/g;
5432
- const SPACE_DASH_RE = /[\s-]/g;
6406
+ const ID_SEPARATOR_RE = /[\s\-/.]/g;
5433
6407
  /**
5434
6408
  * Normalize entity text so that surface-form variations
5435
6409
  * of the same real-world value map to a single canonical
@@ -5439,8 +6413,8 @@ const normalizeEntityText = (label, text) => {
5439
6413
  const upper = label.toUpperCase().replace(WHITESPACE_RE, "_");
5440
6414
  if (upper === "EMAIL_ADDRESS" || upper === "EMAIL") return text.toLowerCase().trim();
5441
6415
  if (upper === "PHONE_NUMBER" || upper === "PHONE") return text.replace(PHONE_NOISE_RE, "");
5442
- if (upper === "IBAN" || upper === "BANK_ACCOUNT_NUMBER" || upper === "TAX_IDENTIFICATION_NUMBER" || upper === "REGISTRATION_NUMBER") return text.replace(SPACE_DASH_RE, "").toUpperCase();
5443
- if (upper === "PERSON" || upper === "ORGANIZATION" || upper === "ADDRESS" || upper === "LAND_PARCEL") return text.replace(WHITESPACE_RE, " ").toLowerCase().trim();
6416
+ if (upper === "IBAN" || upper === "BANK_ACCOUNT_NUMBER" || upper === "TAX_IDENTIFICATION_NUMBER" || upper === "REGISTRATION_NUMBER" || upper === "NATIONAL_IDENTIFICATION_NUMBER" || upper === "SOCIAL_SECURITY_NUMBER" || upper === "BIRTH_NUMBER" || upper === "IDENTITY_CARD_NUMBER" || upper === "PASSPORT_NUMBER" || upper === "CREDIT_CARD_NUMBER") return text.replace(ID_SEPARATOR_RE, "").toUpperCase();
6417
+ if (upper === "PERSON" || upper === "ORGANIZATION" || upper === "ADDRESS" || upper === "LAND_PARCEL" || upper === "MISC") return text.replace(WHITESPACE_RE, " ").toLowerCase().trim();
5444
6418
  return text.trim();
5445
6419
  };
5446
6420
  /**
@@ -5984,6 +6958,6 @@ const levenshtein = (rawA, rawB) => {
5984
6958
  //#region src/wasm.ts
5985
6959
  initTextSearch(TextSearch);
5986
6960
  //#endregion
5987
- export { CURRENCY_PATTERN_META, DATE_PATTERN_META, DEFAULT_ENTITY_LABELS, DEFAULT_OPERATOR_CONFIG, DETECTION_SOURCES, DETECTOR_PRIORITY, OPERATOR_REGISTRY, OPERATOR_TYPES, REGEX_META, REGEX_PATTERNS, REGIONS, ZONE_SCORE_ADJUSTMENTS, applyHotwordRules, applyZoneAdjustments, boostNearMissEntities, buildDenyList, buildGazetteerPatterns, buildLegalFormPatterns, buildPlaceholderMap, buildStreetTypePatterns, buildTriggerPatterns, buildUnifiedSearch, chunkText, classifyZones, computeChunkOffsets, corefKey, createPipelineContext, deanonymise, decodeSpans, decodeTokenSpans, detectNameCorpus, ensureDenyListData, exportRedactionKey, extractDefinedTerms, filterFalsePositives, findCoreferenceSpans, getCurrencyPatterns, getDatePatterns, initHotwordRules, initNameCorpus, initZoneClassifier, levenshtein, mergeAndDedup, mergeChunkEntities, normalizeForSearch, prepareBatch, processAddressSeeds, processDenyListMatches, processGazetteerMatches, processLegalFormMatches, processRegexMatches, processTriggerMatches, propagateOrgNames, redactText, resolveCountries, resolveOperator, runPipeline, runUnifiedSearch, sanitizeEntities, tokenizeText };
6961
+ export { CURRENCY_PATTERN_META, DATE_PATTERN_META, DEFAULT_ENTITY_LABELS, DEFAULT_OPERATOR_CONFIG, DETECTION_SOURCES, DETECTOR_PRIORITY, OPERATOR_REGISTRY, OPERATOR_TYPES, REGEX_META, REGEX_PATTERNS, REGIONS, ZONE_SCORE_ADJUSTMENTS, applyHotwordRules, applyZoneAdjustments, boostNearMissEntities, buildDenyList, buildGazetteerPatterns, buildLegalFormPatterns, buildPlaceholderMap, buildStreetTypePatterns, buildTriggerPatterns, buildUnifiedSearch, chunkText, classifyZones, computeChunkOffsets, corefKey, createPipelineContext, deanonymise, decodeSpans, decodeTokenSpans, detectNameCorpus, ensureDenyListData, exportRedactionKey, extractDefinedTerms, filterFalsePositives, findCoreferenceSpans, getCurrencyPatterns, getDatePatterns, initAddressComponents, initHotwordRules, initNameCorpus, initZoneClassifier, levenshtein, mergeAndDedup, mergeChunkEntities, normalizeForSearch, prepareBatch, processAddressSeeds, processDenyListMatches, processGazetteerMatches, processLegalFormMatches, processRegexMatches, processTriggerMatches, propagateOrgNames, redactText, resolveCountries, resolveOperator, runPipeline, runUnifiedSearch, sanitizeEntities, tokenizeText, warmLegalRoleHeads };
5988
6962
 
5989
6963
  //# sourceMappingURL=wasm.mjs.map