@stll/anonymize 0.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js ADDED
@@ -0,0 +1,4969 @@
1
+ import { at, be, bg, cy, cz, de, dk, ee, es, fi, fr, gb, gr, hr, hu, ie, it, lt, lu, lv, mt, nl, pl, pt, ro, se, si, sk } from "@stll/stdnum";
2
+ import { toRegex } from "@stll/stdnum/patterns";
3
+ import { TextSearch } from "@stll/text-search";
4
+ //#region src/types.ts
5
+ /**
6
+ * Source of a detected entity span.
7
+ * Ordered by detection layer in the pipeline.
8
+ */
9
+ const DETECTION_SOURCES = {
10
+ TRIGGER: "trigger",
11
+ REGEX: "regex",
12
+ DENY_LIST: "deny-list",
13
+ LEGAL_FORM: "legal-form",
14
+ GAZETTEER: "gazetteer",
15
+ NER: "ner",
16
+ COREFERENCE: "coreference"
17
+ };
18
+ /**
19
+ * Priority levels for detection sources.
20
+ * Higher = more structurally reliable. Used during
21
+ * overlap resolution so deterministic detectors beat
22
+ * probabilistic ones regardless of raw score.
23
+ */
24
+ const DETECTOR_PRIORITY = {
25
+ gazetteer: 5,
26
+ trigger: 4,
27
+ "legal-form": 3,
28
+ regex: 3,
29
+ "deny-list": 2,
30
+ coreference: 2,
31
+ ner: 1
32
+ };
33
+ /**
34
+ * Anonymisation operator types. Each operator defines
35
+ * how a confirmed entity is replaced in the output.
36
+ */
37
+ const OPERATOR_TYPES = ["replace", "redact"];
38
+ /**
39
+ * Canonical entity labels used across the pipeline.
40
+ * NER models may use different native labels; the bench
41
+ * NER wrapper maps model output to these canonical names.
42
+ *
43
+ * These labels are ephemeral: entities are regenerated on
44
+ * every pipeline run and never persisted to the database.
45
+ * Renaming a label here requires no migration.
46
+ */
47
+ const DEFAULT_ENTITY_LABELS = [
48
+ "person",
49
+ "organization",
50
+ "phone number",
51
+ "address",
52
+ "email address",
53
+ "date",
54
+ "date of birth",
55
+ "bank account number",
56
+ "iban",
57
+ "tax identification number",
58
+ "identity card number",
59
+ "registration number",
60
+ "credit card number",
61
+ "passport number",
62
+ "monetary amount"
63
+ ];
64
+ //#endregion
65
+ //#region src/context.ts
66
+ /**
67
+ * Build a stable cache key for an entity that survives
68
+ * shallow copies (spread). Uses position + label so the
69
+ * key is identical for the original object and any
70
+ * `{ ...entity }` copy produced by mergeAndDedup.
71
+ */
72
+ const corefKey = (e) => `${e.start}:${e.end}:${e.label}`;
73
+ /** Create a fresh, empty pipeline context. */
74
+ const createPipelineContext = () => ({
75
+ search: null,
76
+ searchKey: "",
77
+ searchPromise: null,
78
+ nameCorpus: null,
79
+ nameCorpusPromise: null,
80
+ stopwords: null,
81
+ stopwordsPromise: null,
82
+ allowList: null,
83
+ allowListPromise: null,
84
+ personStopwords: null,
85
+ personStopwordsPromise: null,
86
+ firstNameExclusions: null,
87
+ firstNameExclusionCorpusLen: 0,
88
+ genericRoles: null,
89
+ genericRolesPromise: null,
90
+ corefPatterns: null,
91
+ corefPatternsPromise: null,
92
+ corefLoadAttempted: false,
93
+ roleStopSet: null,
94
+ roleStopSetPromise: null,
95
+ zoneHeadingPatterns: null,
96
+ zoneSigningPatterns: null,
97
+ zoneInitPromise: null,
98
+ corefSourceMap: /* @__PURE__ */ new Map()
99
+ });
100
+ /**
101
+ * Module-level default context. Used when callers
102
+ * don't provide an explicit context, preserving full
103
+ * backward compatibility with the existing API.
104
+ */
105
+ const defaultContext = createPipelineContext();
106
+ //#endregion
107
+ //#region src/util/lang-loader.ts
108
+ let _manifest = null;
109
+ let _manifestPromise = null;
110
+ const loadManifest = () => {
111
+ if (_manifest) return Promise.resolve(_manifest);
112
+ if (_manifestPromise) return _manifestPromise;
113
+ _manifestPromise = (async () => {
114
+ try {
115
+ const mod = await import("@stll/anonymize-data/config/manifest.json");
116
+ const parsed = mod.default ?? mod;
117
+ if (!parsed || typeof parsed.languages !== "object" || parsed.languages === null || Array.isArray(parsed.languages)) {
118
+ console.warn("[anonymize] lang-loader: manifest has unexpected structure, falling back to hardcoded list");
119
+ _manifest = { languages: {} };
120
+ return _manifest;
121
+ }
122
+ _manifest = parsed;
123
+ return _manifest;
124
+ } catch (err) {
125
+ console.warn("[anonymize] lang-loader: manifest not available, falling back to hardcoded language list:", err);
126
+ _manifest = { languages: {} };
127
+ return _manifest;
128
+ }
129
+ })();
130
+ return _manifestPromise;
131
+ };
132
+ const LOADER_REGISTRIES = {
133
+ triggers: {
134
+ cs: () => import("@stll/anonymize-data/config/triggers.cs.json"),
135
+ de: () => import("@stll/anonymize-data/config/triggers.de.json"),
136
+ en: () => import("@stll/anonymize-data/config/triggers.en.json"),
137
+ es: () => import("@stll/anonymize-data/config/triggers.es.json"),
138
+ fr: () => import("@stll/anonymize-data/config/triggers.fr.json"),
139
+ hu: () => import("@stll/anonymize-data/config/triggers.hu.json"),
140
+ it: () => import("@stll/anonymize-data/config/triggers.it.json"),
141
+ pl: () => import("@stll/anonymize-data/config/triggers.pl.json"),
142
+ ro: () => import("@stll/anonymize-data/config/triggers.ro.json"),
143
+ sv: () => import("@stll/anonymize-data/config/triggers.sv.json")
144
+ },
145
+ coreference: {
146
+ cs: () => import("@stll/anonymize-data/config/coreference.cs.json"),
147
+ de: () => import("@stll/anonymize-data/config/coreference.de.json"),
148
+ en: () => import("@stll/anonymize-data/config/coreference.en.json"),
149
+ sk: () => import("@stll/anonymize-data/config/coreference.sk.json")
150
+ }
151
+ };
152
+ const FALLBACK_LANGUAGES = {
153
+ triggers: [
154
+ "cs",
155
+ "de",
156
+ "en",
157
+ "es",
158
+ "fr",
159
+ "hu",
160
+ "it",
161
+ "pl",
162
+ "ro",
163
+ "sv"
164
+ ],
165
+ coreference: [
166
+ "cs",
167
+ "de",
168
+ "en",
169
+ "sk"
170
+ ]
171
+ };
172
+ /**
173
+ * Load all config files of a given type for all
174
+ * languages enabled in the manifest.
175
+ *
176
+ * Falls back to the hardcoded language list when the
177
+ * manifest is unavailable (backward compatibility).
178
+ */
179
+ const loadLanguageConfigs = async (configType, mapFn) => {
180
+ const manifest = await loadManifest();
181
+ const registry = LOADER_REGISTRIES[configType];
182
+ const codes = Object.keys(manifest.languages).length > 0 ? Object.entries(manifest.languages).filter(([code, lang]) => {
183
+ if (!lang || typeof lang !== "object") {
184
+ console.warn(`[anonymize] lang-loader: manifest entry for "${code}" is not an object, skipping`);
185
+ return false;
186
+ }
187
+ return lang[configType] === true;
188
+ }).map(([code]) => code) : [...FALLBACK_LANGUAGES[configType]];
189
+ const results = new Array(codes.length);
190
+ const loads = codes.map(async (code, i) => {
191
+ const loader = registry[code];
192
+ if (!loader) {
193
+ console.warn(`[anonymize] lang-loader: language "${code}" is enabled in the manifest for "${configType}" but has no loader in the static registry`);
194
+ return;
195
+ }
196
+ let mod;
197
+ try {
198
+ mod = await loader();
199
+ } catch (err) {
200
+ console.warn(`[anonymize] lang-loader: failed to import "${configType}" config for "${code}":`, err);
201
+ return;
202
+ }
203
+ let result;
204
+ try {
205
+ result = mapFn(mod);
206
+ } catch (err) {
207
+ console.warn(`[anonymize] lang-loader: mapFn failed for "${code}" (${configType}):`, err);
208
+ return;
209
+ }
210
+ results[i] = result;
211
+ });
212
+ await Promise.all(loads);
213
+ return results.filter((r) => r !== void 0);
214
+ };
215
+ //#endregion
216
+ //#region src/detectors/coreference.ts
217
+ /**
218
+ * Load coreference definition patterns from per-language
219
+ * JSON configs in @stll/anonymize-data. Uses the
220
+ * language manifest for auto-discovery.
221
+ */
222
+ const loadDefinitionPatterns = async () => {
223
+ const patterns = [];
224
+ const allRows = await loadLanguageConfigs("coreference", (mod) => {
225
+ return mod.default ?? mod;
226
+ });
227
+ for (const rows of allRows) {
228
+ if (!Array.isArray(rows)) {
229
+ console.warn("[anonymize] coreference: unexpected config shape, skipping");
230
+ continue;
231
+ }
232
+ for (const row of rows) try {
233
+ patterns.push({ pattern: new RegExp(row.pattern, row.flags) });
234
+ } catch (err) {
235
+ console.warn(`[anonymize] coreference: invalid regex "${row.pattern}":`, err);
236
+ }
237
+ }
238
+ return patterns;
239
+ };
240
+ /**
241
+ * Load generic role terms that should NOT be treated
242
+ * as PII coreferences. "Prodávající" (Seller),
243
+ * "Kupující" (Buyer), etc. are legal roles, not
244
+ * identifying information.
245
+ */
246
+ const getRoleStopSet = async (ctx) => {
247
+ if (ctx.roleStopSet) return ctx.roleStopSet;
248
+ if (ctx.roleStopSetPromise) return ctx.roleStopSetPromise;
249
+ const promise = (async () => {
250
+ let result;
251
+ try {
252
+ const mod = await import("@stll/anonymize-data/config/generic-roles.json");
253
+ const data = mod.default ?? mod;
254
+ result = new Set(data.roles.map((r) => r.toLowerCase()));
255
+ } catch {
256
+ result = /* @__PURE__ */ new Set();
257
+ }
258
+ ctx.roleStopSet = result;
259
+ return result;
260
+ })();
261
+ ctx.roleStopSetPromise = promise;
262
+ return promise;
263
+ };
264
+ const getDefinitionPatterns = async (ctx) => {
265
+ if (ctx.corefPatterns) return ctx.corefPatterns;
266
+ if (ctx.corefPatternsPromise) return ctx.corefPatternsPromise;
267
+ ctx.corefPatternsPromise = loadDefinitionPatterns();
268
+ const patterns = await ctx.corefPatternsPromise;
269
+ if (patterns.length === 0) {
270
+ ctx.corefPatterns = patterns;
271
+ if (!ctx.corefLoadAttempted) {
272
+ ctx.corefLoadAttempted = true;
273
+ console.warn("[anonymize] coreference: no definition patterns loaded; coreference detection will be inactive");
274
+ }
275
+ return patterns;
276
+ }
277
+ ctx.corefPatterns = patterns;
278
+ return patterns;
279
+ };
280
+ const SEARCH_WINDOW = 200;
281
+ /**
282
+ * Scan for defined-term patterns near known entities.
283
+ *
284
+ * Legal documents universally follow:
285
+ * "Dr. Heinrich Muller (hereinafter 'the Seller')..."
286
+ *
287
+ * After NER detects the entity, this function scans for
288
+ * definitional patterns within +/-200 chars and extracts
289
+ * the alias. Returns alias + label pairs that can be added
290
+ * to the gazetteer for a full-text re-scan.
291
+ */
292
+ /**
293
+ * Labels that can be the source of a coreference alias.
294
+ * Only parties (person, organization) have defined-term
295
+ * aliases in legal text. Dates, addresses, IDs do not.
296
+ */
297
+ const COREF_SOURCE_LABELS = new Set(["person", "organization"]);
298
+ const extractDefinedTerms = async (fullText, entities, ctx = defaultContext) => {
299
+ const [definitionPatterns, roleStops] = await Promise.all([getDefinitionPatterns(ctx), getRoleStopSet(ctx)]);
300
+ const terms = [];
301
+ const seen = /* @__PURE__ */ new Set();
302
+ const sorted = [...entities].sort((a, b) => a.start - b.start);
303
+ for (const { pattern } of definitionPatterns) {
304
+ pattern.lastIndex = 0;
305
+ for (let match = pattern.exec(fullText); match !== null; match = pattern.exec(fullText)) {
306
+ const alias = match[1]?.trim();
307
+ if (!alias || alias.length < 2) continue;
308
+ if (roleStops.has(alias.toLowerCase())) continue;
309
+ const defPos = match.index;
310
+ let bestEntity = null;
311
+ for (let i = sorted.length - 1; i >= 0; i--) {
312
+ const e = sorted[i];
313
+ if (e === void 0) continue;
314
+ if (e.end > defPos) continue;
315
+ if (defPos - e.end > SEARCH_WINDOW) break;
316
+ if (!COREF_SOURCE_LABELS.has(e.label)) continue;
317
+ bestEntity = e;
318
+ break;
319
+ }
320
+ if (bestEntity === null) continue;
321
+ const key = `${alias.toLowerCase()}::${bestEntity.label}`;
322
+ if (seen.has(key)) continue;
323
+ seen.add(key);
324
+ terms.push({
325
+ alias,
326
+ label: bestEntity.label,
327
+ definitionStart: defPos,
328
+ sourceText: bestEntity.text
329
+ });
330
+ }
331
+ }
332
+ return terms;
333
+ };
334
+ /**
335
+ * Check if a character is a Unicode word character
336
+ * (letter, digit, or combining mark). Used for word
337
+ * boundary checks in coreference matching.
338
+ */
339
+ const isWordChar = (ch) => {
340
+ if (ch === void 0) return false;
341
+ return /[\p{L}\p{M}\p{N}]/u.test(ch);
342
+ };
343
+ /**
344
+ * Find all occurrences of defined-term aliases in the
345
+ * full text. Returns Entity spans for each match.
346
+ *
347
+ * Respects word boundaries: "Kupující" must not match
348
+ * inside "Kupujícímu". A match is valid only if the
349
+ * character before the start and after the end are NOT
350
+ * word characters (letter/digit).
351
+ *
352
+ * Populates `ctx.corefSourceMap` with entries linking
353
+ * each coref entity to its source entity text, for
354
+ * consistent placeholder numbering.
355
+ */
356
+ const findCoreferenceSpans = (fullText, terms, ctx = defaultContext) => {
357
+ const results = [];
358
+ for (const term of terms) {
359
+ let searchFrom = 0;
360
+ while (searchFrom < fullText.length) {
361
+ const idx = fullText.indexOf(term.alias, searchFrom);
362
+ if (idx === -1) break;
363
+ const matchEnd = idx + term.alias.length;
364
+ const charBefore = idx > 0 ? fullText[idx - 1] : void 0;
365
+ const charAfter = fullText[matchEnd];
366
+ if (isWordChar(charBefore) || isWordChar(charAfter)) {
367
+ searchFrom = idx + 1;
368
+ continue;
369
+ }
370
+ const entity = {
371
+ start: idx,
372
+ end: matchEnd,
373
+ label: term.label,
374
+ text: term.alias,
375
+ score: .95,
376
+ source: DETECTION_SOURCES.COREFERENCE
377
+ };
378
+ ctx.corefSourceMap.set(corefKey(entity), term.sourceText);
379
+ results.push(entity);
380
+ searchFrom = matchEnd;
381
+ }
382
+ }
383
+ return results;
384
+ };
385
+ //#endregion
386
+ //#region src/detectors/gazetteer.ts
387
+ const MAX_EDIT_DISTANCE = 2;
388
+ const MIN_FUZZY_LENGTH = 4;
389
+ const MAX_PREFIX_OVERSHOOT = 7;
390
+ /**
391
+ * Collect all searchable strings (canonical + variants)
392
+ * from gazetteer entries, mapped to their labels and
393
+ * entry IDs.
394
+ */
395
+ const buildSearchTerms = (entries) => {
396
+ const terms = /* @__PURE__ */ new Map();
397
+ for (const entry of entries) {
398
+ const meta = {
399
+ label: entry.label,
400
+ entryId: entry.id
401
+ };
402
+ terms.set(entry.canonical, meta);
403
+ for (const variant of entry.variants) terms.set(variant, meta);
404
+ }
405
+ return terms;
406
+ };
407
+ /**
408
+ * Build TextSearch-compatible patterns from gazetteer
409
+ * entries. Returns:
410
+ * - Exact literal patterns for all terms
411
+ * - Fuzzy patterns (distance: 2) for terms >= 4 chars
412
+ * - Parallel metadata arrays for post-processing
413
+ *
414
+ * Patterns are ordered: all exact first, then all
415
+ * fuzzy. The isFuzzy array marks which are which.
416
+ */
417
+ const buildGazetteerPatterns = (entries) => {
418
+ const terms = buildSearchTerms(entries);
419
+ const patterns = [];
420
+ const labels = [];
421
+ const isFuzzy = [];
422
+ for (const [term, meta] of terms) {
423
+ patterns.push({
424
+ pattern: term,
425
+ literal: true,
426
+ wholeWords: false
427
+ });
428
+ labels.push(meta.label);
429
+ isFuzzy.push(false);
430
+ }
431
+ for (const [term, meta] of terms) {
432
+ if (term.length < MIN_FUZZY_LENGTH) continue;
433
+ patterns.push({
434
+ pattern: term,
435
+ distance: MAX_EDIT_DISTANCE
436
+ });
437
+ labels.push(meta.label);
438
+ isFuzzy.push(true);
439
+ }
440
+ return {
441
+ patterns,
442
+ data: {
443
+ labels,
444
+ isFuzzy
445
+ }
446
+ };
447
+ };
448
+ /**
449
+ * Process gazetteer matches from the unified literal
450
+ * search. Receives all matches; filters to the
451
+ * gazetteer slice via sliceStart/sliceEnd.
452
+ *
453
+ * Exact matches get score 0.9; fuzzy matches get
454
+ * 0.85. Fuzzy matches that overlap an exact match
455
+ * are dropped.
456
+ *
457
+ * For exact matches, attempts prefix extension for
458
+ * legal suffixes ("a.s.", "GmbH", "s.r.o." after
459
+ * the matched term).
460
+ */
461
+ const processGazetteerMatches = (allMatches, sliceStart, sliceEnd, fullText, data) => {
462
+ const results = [];
463
+ const exactSpans = [];
464
+ for (const match of allMatches) {
465
+ const idx = match.pattern;
466
+ if (idx < sliceStart || idx >= sliceEnd) continue;
467
+ const localIdx = idx - sliceStart;
468
+ if (data.isFuzzy[localIdx]) continue;
469
+ const label = data.labels[localIdx];
470
+ if (!label) continue;
471
+ const extended = tryPrefixExtension(fullText, match.start, match.end);
472
+ const end = extended?.end ?? match.end;
473
+ const text = extended?.text ?? fullText.slice(match.start, match.end);
474
+ exactSpans.push({
475
+ start: match.start,
476
+ end
477
+ });
478
+ results.push({
479
+ start: match.start,
480
+ end,
481
+ label,
482
+ text,
483
+ score: .9,
484
+ source: DETECTION_SOURCES.GAZETTEER
485
+ });
486
+ }
487
+ for (const match of allMatches) {
488
+ const idx = match.pattern;
489
+ if (idx < sliceStart || idx >= sliceEnd) continue;
490
+ const localIdx = idx - sliceStart;
491
+ if (!data.isFuzzy[localIdx]) continue;
492
+ if (match.distance === 0) continue;
493
+ const label = data.labels[localIdx];
494
+ if (!label) continue;
495
+ if (exactSpans.some((e) => match.start < e.end && match.end > e.start)) continue;
496
+ const matchText = fullText.slice(match.start, match.end);
497
+ results.push({
498
+ start: match.start,
499
+ end: match.end,
500
+ label,
501
+ text: matchText,
502
+ score: .85,
503
+ source: DETECTION_SOURCES.GAZETTEER
504
+ });
505
+ }
506
+ return results;
507
+ };
508
+ /**
509
+ * Try to extend an exact match to capture one
510
+ * trailing token (max 6 chars) that may be a legal
511
+ * entity suffix (e.g., "a.s.", "GmbH", "s.r.o.").
512
+ *
513
+ * Does not validate the token against a legal-forms
514
+ * list; false extensions are filtered by mergeAndDedup
515
+ * when a legal-form detector produces a competing
516
+ * entity with the correct span.
517
+ */
518
+ const tryPrefixExtension = (fullText, start, end) => {
519
+ const maxEnd = Math.min(end + MAX_PREFIX_OVERSHOOT, fullText.length);
520
+ if (maxEnd <= end + 1) return null;
521
+ const after = fullText.slice(end, maxEnd);
522
+ if (!after.startsWith(" ")) return null;
523
+ const nextSpace = after.indexOf(" ", 1);
524
+ const suffixEnd = nextSpace !== -1 ? nextSpace : after.length;
525
+ if (suffixEnd <= 1) return null;
526
+ const newEnd = end + suffixEnd;
527
+ return {
528
+ end: newEnd,
529
+ text: fullText.slice(start, newEnd)
530
+ };
531
+ };
532
+ //#endregion
533
+ //#region src/util/text.ts
534
+ /**
535
+ * Shared text utilities for detectors.
536
+ *
537
+ * Extracted from names.ts and deny-list.ts to avoid
538
+ * duplicating regex constants and helper functions.
539
+ */
540
+ /** Matches a string that starts with an uppercase letter. */
541
+ const UPPER_START_RE = /^\p{Lu}/u;
542
+ /** Matches a string consisting entirely of uppercase letters. */
543
+ const ALL_UPPER_RE = /^\p{Lu}+$/u;
544
+ const SENTENCE_END_RE = /[.!?]/;
545
+ /**
546
+ * Detect whether a position is at the start of a sentence.
547
+ * Looks backward past whitespace for sentence-ending
548
+ * punctuation (.!?). Position 0 and positions preceded
549
+ * only by whitespace are considered sentence starts.
550
+ */
551
+ const isSentenceStart = (text, pos) => {
552
+ if (pos === 0) return true;
553
+ let i = pos - 1;
554
+ while (i >= 0 && /\s/.test(text[i] ?? "")) i--;
555
+ if (i < 0) return true;
556
+ return SENTENCE_END_RE.test(text[i] ?? "");
557
+ };
558
+ //#endregion
559
+ //#region src/detectors/names.ts
560
+ const getCorpus = (ctx) => ctx.nameCorpus;
561
+ const getNameCorpusFirstNames = (ctx = defaultContext) => ctx.nameCorpus?.firstNamesList ?? [];
562
+ const getNameCorpusSurnames = (ctx = defaultContext) => ctx.nameCorpus?.surnamesList ?? [];
563
+ const getNameCorpusTitles = (ctx = defaultContext) => ctx.nameCorpus?.titlesList ?? [];
564
+ /**
565
+ * Load name corpus data from JSON config files.
566
+ * Safe to call multiple times; only loads once per
567
+ * context. Must be called before detectNameCorpus or
568
+ * the getNameCorpus*() accessors are used.
569
+ */
570
+ const initNameCorpus = (ctx = defaultContext) => {
571
+ if (ctx.nameCorpusPromise) return ctx.nameCorpusPromise;
572
+ const promise = (async () => {
573
+ try {
574
+ const [firstMod, surnameMod, titleMod, exclusionMod] = await Promise.all([
575
+ import("@stll/anonymize-data/config/names-first.json"),
576
+ import("@stll/anonymize-data/config/names-surnames.json"),
577
+ import("@stll/anonymize-data/config/names-title-tokens.json"),
578
+ import("@stll/anonymize-data/config/names-exclusions.json")
579
+ ]);
580
+ const firstNames = firstMod.default.names;
581
+ const surnames = surnameMod.default.names;
582
+ const titles = titleMod.default.tokens;
583
+ const exclusions = exclusionMod.default.words;
584
+ ctx.nameCorpus = {
585
+ firstNames: Object.freeze(new Set(firstNames)),
586
+ surnames: Object.freeze(new Set(surnames)),
587
+ titleTokens: Object.freeze(new Set(titles)),
588
+ excludedWords: Object.freeze(new Set(exclusions)),
589
+ firstNamesList: Object.freeze(firstNames),
590
+ surnamesList: Object.freeze(surnames),
591
+ titlesList: Object.freeze(titles),
592
+ excludedList: Object.freeze(exclusions)
593
+ };
594
+ } catch (err) {
595
+ ctx.nameCorpusPromise = null;
596
+ console.warn("[anonymize] Failed to load name corpus JSON — name detection disabled:", err);
597
+ }
598
+ })();
599
+ ctx.nameCorpusPromise = promise;
600
+ return promise;
601
+ };
602
+ const INFLECTION_SUFFIXES = [
603
+ "ovi",
604
+ "em",
605
+ "om",
606
+ "ou",
607
+ "é",
608
+ "a",
609
+ "u"
610
+ ];
611
+ /**
612
+ * Strip common Czech/Slovak case suffixes from a token.
613
+ * Returns candidate base forms if stripping produces a
614
+ * plausible name (capitalised, length >= 3).
615
+ *
616
+ * For the "-ou" instrumental feminine suffix, also yields
617
+ * base + "a" (e.g., "Editou" → "Edit" and "Edita")
618
+ * because Czech feminine names decline -a → -ou.
619
+ */
620
+ const stripInflection = (token) => {
621
+ const candidates = [];
622
+ for (const suffix of INFLECTION_SUFFIXES) if (token.length > suffix.length + 2 && token.endsWith(suffix)) {
623
+ const base = token.slice(0, -suffix.length);
624
+ if (/^\p{Lu}/u.test(base)) {
625
+ candidates.push(base);
626
+ if (suffix === "ou" || suffix === "é" || suffix === "u") candidates.push(`${base}a`);
627
+ }
628
+ }
629
+ return candidates;
630
+ };
631
+ const TOKEN_TYPE = {
632
+ NAME: "name",
633
+ SURNAME: "surname",
634
+ TITLE: "title",
635
+ ABBREVIATION: "abbreviation",
636
+ CAPITALIZED: "capitalized",
637
+ OTHER: "other"
638
+ };
639
+ /**
640
+ * Check if a token is in the first-name set, either
641
+ * directly or after stripping Czech/Slovak inflection.
642
+ */
643
+ const isFirstNameToken = (token, corpus) => {
644
+ if (corpus.firstNames.has(token)) return true;
645
+ return stripInflection(token).some((b) => corpus.firstNames.has(b));
646
+ };
647
+ /**
648
+ * Check if a token is in the surname set, either
649
+ * directly or after stripping Czech/Slovak inflection.
650
+ */
651
+ const isSurnameToken = (token, corpus) => {
652
+ if (corpus.surnames.has(token)) return true;
653
+ return stripInflection(token).some((b) => corpus.surnames.has(b));
654
+ };
655
+ /**
656
+ * Check if a token looks like a single-letter
657
+ * abbreviation: "J.", "M.", etc.
658
+ */
659
+ const isAbbreviation = (token) => token.length === 2 && /^\p{Lu}$/u.test(token[0] ?? "") && token[1] === ".";
660
+ const segmenter$1 = new Intl.Segmenter(void 0, { granularity: "word" });
661
+ /**
662
+ * Split text into word segments using Intl.Segmenter.
663
+ * Only returns segments flagged as words.
664
+ */
665
+ const segmentWords = (fullText) => {
666
+ const words = [];
667
+ for (const seg of segmenter$1.segment(fullText)) if (seg.isWordLike) words.push({
668
+ text: seg.segment,
669
+ start: seg.index,
670
+ end: seg.index + seg.segment.length
671
+ });
672
+ return words;
673
+ };
674
+ /** NAME or SURNAME — both represent corpus-matched tokens */
675
+ const isCorpusMatch = (type) => type === TOKEN_TYPE.NAME || type === TOKEN_TYPE.SURNAME;
676
+ const classifyToken = (word, corpus) => {
677
+ const { text, start, end } = word;
678
+ const lower = text.toLowerCase();
679
+ const stripped = text.endsWith(".") ? text.slice(0, -1).toLowerCase() : lower;
680
+ if (corpus.titleTokens.has(stripped)) return {
681
+ text,
682
+ type: TOKEN_TYPE.TITLE,
683
+ start,
684
+ end
685
+ };
686
+ if (isAbbreviation(text)) return {
687
+ text,
688
+ type: TOKEN_TYPE.ABBREVIATION,
689
+ start,
690
+ end
691
+ };
692
+ if (corpus.excludedWords.has(lower)) return {
693
+ text,
694
+ type: TOKEN_TYPE.OTHER,
695
+ start,
696
+ end
697
+ };
698
+ if (text.length < 3) return {
699
+ text,
700
+ type: TOKEN_TYPE.OTHER,
701
+ start,
702
+ end
703
+ };
704
+ if (text.length > 3 && ALL_UPPER_RE.test(text)) return {
705
+ text,
706
+ type: TOKEN_TYPE.OTHER,
707
+ start,
708
+ end
709
+ };
710
+ if (!UPPER_START_RE.test(text)) return {
711
+ text,
712
+ type: TOKEN_TYPE.OTHER,
713
+ start,
714
+ end
715
+ };
716
+ if (isFirstNameToken(text, corpus)) return {
717
+ text,
718
+ type: TOKEN_TYPE.NAME,
719
+ start,
720
+ end
721
+ };
722
+ if (isSurnameToken(text, corpus)) return {
723
+ text,
724
+ type: TOKEN_TYPE.SURNAME,
725
+ start,
726
+ end
727
+ };
728
+ return {
729
+ text,
730
+ type: TOKEN_TYPE.CAPITALIZED,
731
+ start,
732
+ end
733
+ };
734
+ };
735
+ /**
736
+ * Detect person names by looking up tokens against the
737
+ * name corpus, then chaining adjacent name-like tokens.
738
+ *
739
+ * Requires initNameCorpus() to have been called first.
740
+ * If not initialized, returns an empty array.
741
+ *
742
+ * Scoring:
743
+ * TITLE + NAME/SURNAME → 0.95
744
+ * NAME + NAME/SURNAME → 0.9
745
+ * SURNAME + NAME/SURNAME → 0.9
746
+ * NAME + CAPITALIZED → 0.7
747
+ * ABBREVIATION + NAME → 0.7
748
+ * Standalone NAME → 0.5 (low confidence)
749
+ * Standalone SURNAME → skip (too ambiguous)
750
+ */
751
+ const detectNameCorpus = (fullText, ctx = defaultContext) => {
752
+ const corpus = getCorpus(ctx);
753
+ if (!corpus) return [];
754
+ const tokens = segmentWords(fullText).map((w) => classifyToken(w, corpus));
755
+ const entities = [];
756
+ const consumed = /* @__PURE__ */ new Set();
757
+ for (let i = 0; i < tokens.length; i++) {
758
+ if (consumed.has(i)) continue;
759
+ const token = tokens[i];
760
+ if (!token) continue;
761
+ if (token.type !== TOKEN_TYPE.TITLE && token.type !== TOKEN_TYPE.NAME && token.type !== TOKEN_TYPE.SURNAME && token.type !== TOKEN_TYPE.ABBREVIATION) continue;
762
+ const MAX_CHAIN = 5;
763
+ const chain = [token];
764
+ let j = i + 1;
765
+ while (j < tokens.length && chain.length < MAX_CHAIN) {
766
+ const next = tokens[j];
767
+ if (!next) break;
768
+ const prev = chain.at(-1);
769
+ if (prev) {
770
+ if (fullText.slice(prev.end, next.start).includes("\n")) break;
771
+ }
772
+ if (next.type === TOKEN_TYPE.NAME || next.type === TOKEN_TYPE.SURNAME || next.type === TOKEN_TYPE.TITLE || next.type === TOKEN_TYPE.ABBREVIATION || next.type === TOKEN_TYPE.CAPITALIZED) {
773
+ chain.push(next);
774
+ j++;
775
+ } else break;
776
+ }
777
+ const hasTitle = chain.some((t) => t.type === TOKEN_TYPE.TITLE);
778
+ const hasCorpusName = chain.some((t) => isCorpusMatch(t.type));
779
+ const hasFirstName = chain.some((t) => t.type === TOKEN_TYPE.NAME);
780
+ const hasAbbreviation = chain.some((t) => t.type === TOKEN_TYPE.ABBREVIATION);
781
+ const corpusCount = chain.filter((t) => isCorpusMatch(t.type)).length;
782
+ const capitalizedCount = chain.filter((t) => t.type === TOKEN_TYPE.CAPITALIZED).length;
783
+ let score = 0;
784
+ if (hasTitle && hasCorpusName) score = .95;
785
+ else if (corpusCount >= 2) score = .9;
786
+ else if (hasCorpusName && capitalizedCount > 0) score = .7;
787
+ else if (hasAbbreviation && hasCorpusName) score = .7;
788
+ else if (hasFirstName && chain.length === 1) {
789
+ if (isSentenceStart(fullText, token.start)) continue;
790
+ score = .5;
791
+ } else if (!hasFirstName && chain.length === 1 && chain[0]?.type === TOKEN_TYPE.SURNAME) continue;
792
+ else if (hasTitle && chain.length === 1) continue;
793
+ else {
794
+ if (!hasCorpusName) continue;
795
+ score = .5;
796
+ }
797
+ const first = chain.at(0);
798
+ const last = chain.at(-1);
799
+ if (!first || !last) continue;
800
+ const start = first.start;
801
+ const end = last.end;
802
+ const text = fullText.slice(start, end);
803
+ for (let k = i; k < i + chain.length; k++) consumed.add(k);
804
+ entities.push({
805
+ start,
806
+ end,
807
+ label: "person",
808
+ text,
809
+ score,
810
+ source: DETECTION_SOURCES.REGEX
811
+ });
812
+ }
813
+ return entities;
814
+ };
815
+ //#endregion
816
+ //#region src/config/titles.ts
817
+ /**
818
+ * Academic and professional title prefixes.
819
+ * Plain text; the detector auto-escapes for regex.
820
+ * Sorted longest-first at build time.
821
+ */
822
+ const TITLE_PREFIXES = [
823
+ "Ing.",
824
+ "Mgr.",
825
+ "MgA.",
826
+ "Bc.",
827
+ "BcA.",
828
+ "JUDr.",
829
+ "MUDr.",
830
+ "MVDr.",
831
+ "MDDr.",
832
+ "PhDr.",
833
+ "RNDr.",
834
+ "PaedDr.",
835
+ "ThDr.",
836
+ "ThLic.",
837
+ "ICDr.",
838
+ "RSDr.",
839
+ "PharmDr.",
840
+ "artD.",
841
+ "akad.",
842
+ "doc.",
843
+ "prof.",
844
+ "ao. Univ.-Prof.",
845
+ "o. Univ.-Prof.",
846
+ "Univ.-Prof.",
847
+ "Hon.-Prof.",
848
+ "em. Prof.",
849
+ "Dr. med. dent.",
850
+ "Dr. med. vet.",
851
+ "Dr. med.",
852
+ "Dr. rer. nat.",
853
+ "Dr. rer. soc.",
854
+ "Dr. rer. pol.",
855
+ "Dr. sc. tech.",
856
+ "Dr. sc. nat.",
857
+ "Dr. sc. hum.",
858
+ "Dr. iur.",
859
+ "Dr. jur.",
860
+ "Dr. theol.",
861
+ "Dr. oec.",
862
+ "Dr. techn.",
863
+ "Dr. h. c.",
864
+ "Dr. phil.",
865
+ "Dr.-Ing.",
866
+ "Dr. Ing.",
867
+ "Dr.",
868
+ "Dipl.-Wirt.-Ing.",
869
+ "Dipl.-Betriebsw.",
870
+ "Dipl.-Inform.",
871
+ "Dipl.-Volksw.",
872
+ "Dipl.-Psych.",
873
+ "Dipl.-Phys.",
874
+ "Dipl.-Chem.",
875
+ "Dipl.-Biol.",
876
+ "Dipl.-Math.",
877
+ "Dipl.-Päd.",
878
+ "Dipl.-Soz.",
879
+ "Dipl.-Kfm.",
880
+ "Dipl.-Jur.",
881
+ "Dipl. Ing.",
882
+ "Dipl.-Ing.",
883
+ "Mag. rer. soc. oec.",
884
+ "Mag. rer. nat.",
885
+ "Mag. phil.",
886
+ "Mag. iur.",
887
+ "Mag. arch.",
888
+ "Mag. pharm.",
889
+ "Mag. (FH)",
890
+ "Mag.",
891
+ "Bakk. rer. nat.",
892
+ "Bakk. techn.",
893
+ "Bakk. phil.",
894
+ "Bakk.",
895
+ "Lic. phil.",
896
+ "Lic. iur.",
897
+ "Lic. oec.",
898
+ "Lic. theol.",
899
+ "Lic.",
900
+ "Priv.-Doz.",
901
+ "PD",
902
+ "RA"
903
+ ];
904
+ /**
905
+ * Courtesy/honorific titles that precede a person's
906
+ * name. Sorted alphabetically. The detector escapes
907
+ * dots and adds \b for entries in HONORIFIC_BOUNDARY.
908
+ */
909
+ const HONORIFICS = [
910
+ "Avv.",
911
+ "Dame",
912
+ "Doamna",
913
+ "Domnul",
914
+ "Don",
915
+ "Doña",
916
+ "Dott.",
917
+ "Judge",
918
+ "Justice",
919
+ "Lady",
920
+ "Lord",
921
+ "M.",
922
+ "Madame",
923
+ "Mademoiselle",
924
+ "Maître",
925
+ "Me",
926
+ "Messrs",
927
+ "Miss",
928
+ "Mlle",
929
+ "Mme",
930
+ "Monsieur",
931
+ "Mr",
932
+ "Mrs",
933
+ "Ms",
934
+ "President",
935
+ "Señor",
936
+ "Señora",
937
+ "Sig.",
938
+ "Sig.ra",
939
+ "Signor",
940
+ "Signora",
941
+ "Signorina",
942
+ "Sir",
943
+ "Sr.",
944
+ "Sra."
945
+ ];
946
+ /**
947
+ * Honorifics that need \b word-boundary anchors
948
+ * (short or common words that could match mid-word).
949
+ */
950
+ const HONORIFIC_BOUNDARY = new Set([
951
+ "Don",
952
+ "Doña",
953
+ "M.",
954
+ "Me",
955
+ "Señor",
956
+ "Señora"
957
+ ]);
958
+ /**
959
+ * Post-nominal degrees (comma or space separated after name).
960
+ * Plain text; the detector auto-escapes for regex.
961
+ */
962
+ const POST_NOMINALS = [
963
+ "Ph.D.",
964
+ "Ph.D",
965
+ "CSc.",
966
+ "DrSc.",
967
+ "ArtD.",
968
+ "D.Phil.",
969
+ "DPhil.",
970
+ "MPhil.",
971
+ "MBA",
972
+ "MPA",
973
+ "LL.M.",
974
+ "LL.B.",
975
+ "M.Sc.",
976
+ "B.Sc.",
977
+ "MSc.",
978
+ "BSc.",
979
+ "M.Eng.",
980
+ "B.Eng.",
981
+ "M.A.",
982
+ "B.A.",
983
+ "JCD",
984
+ "JD",
985
+ "DiS.",
986
+ "ACCA",
987
+ "FCCA",
988
+ "CIPM",
989
+ "CIPT",
990
+ "CIPP/E",
991
+ "CIPP"
992
+ ];
993
+ //#endregion
994
+ //#region src/detectors/regex.ts
995
+ const MIN_PHONE_LENGTH = 7;
996
+ const MIN_MONTH_NAME_LENGTH = 3;
997
+ const escapeTitle = (title) => title.replace(/[.*+?^${}()|[\]\\]/g, "\\$&").replace(/\s+/g, "\\s*");
998
+ /** Escape for use inside a regex alternation. */
999
+ const escapeRegex$1 = (s) => s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
1000
+ /** Escape for use inside a regex character class. */
1001
+ const escapeCharClass = (s) => s.replace(/[\]\\^-]/g, "\\$&");
1002
+ const TITLE_PREFIX = TITLE_PREFIXES.toSorted((a, b) => b.length - a.length).map(escapeTitle).join("|");
1003
+ const POST_NOMINAL = POST_NOMINALS.toSorted((a, b) => b.length - a.length).map(escapeTitle).join("|");
1004
+ const NAME_WORD = `[A-ZÁČĎÉĚÍŇÓŘŠŤÚŮÝŽ][a-záčďéěíňóřšťúůýžäöüß]+`;
1005
+ const PARTICLE = "(?:van der|van den|de la|della|von|van|dos|ibn|ben|bin|del|zum|zur|ten|ter|da|de|di|al|el|le|la|zu|af|av)";
1006
+ const SP = "[^\\S\\n\\t]";
1007
+ /** Honorific alternation built from titles.ts config. */
1008
+ const HONORIFIC_ALT = [...HONORIFICS].toSorted((a, b) => b.length - a.length).map((h) => {
1009
+ const escaped = escapeRegex$1(h);
1010
+ return HONORIFIC_BOUNDARY.has(h) ? `\\b${escaped}` : escaped;
1011
+ }).join("|");
1012
+ const toEntry = (validator, label, score) => {
1013
+ const pattern = toRegex(validator).source;
1014
+ if (!pattern) return null;
1015
+ return {
1016
+ validator,
1017
+ label,
1018
+ score,
1019
+ pattern
1020
+ };
1021
+ };
1022
+ /**
1023
+ * Stdnum validators for national/company IDs.
1024
+ *
1025
+ * Selection criteria: only patterns specific enough
1026
+ * to avoid excessive false positives (country-prefixed
1027
+ * VAT numbers, structured personal IDs). Generic
1028
+ * digit-only patterns (e.g. \d{8}) are excluded unless
1029
+ * the validator's checksum is strong enough to filter.
1030
+ */
1031
+ const STDNUM_ENTRIES = [
1032
+ toEntry(hu.vat, "tax identification number", .95),
1033
+ toEntry(it.codiceFiscale, "national identification number", .95),
1034
+ toEntry(es.dni, "national identification number", .9),
1035
+ toEntry(es.nie, "national identification number", .95),
1036
+ toEntry(se.personnummer, "national identification number", .9),
1037
+ toEntry(ro.cnp, "national identification number", .95),
1038
+ toEntry(fr.nir, "social security number", .9),
1039
+ toEntry(cz.dic, "tax identification number", .95),
1040
+ toEntry(de.vat, "tax identification number", .95),
1041
+ toEntry(de.idnr, "tax identification number", .9),
1042
+ toEntry(de.stnr, "tax identification number", .9),
1043
+ toEntry(de.svnr, "social security number", .9),
1044
+ toEntry(pl.nip, "tax identification number", .95),
1045
+ toEntry(pl.pesel, "national identification number", .9),
1046
+ toEntry(gb.vat, "tax identification number", .95),
1047
+ toEntry(gb.nino, "social security number", .95),
1048
+ toEntry(at.uid, "tax identification number", .95),
1049
+ toEntry(at.tin, "tax identification number", .9),
1050
+ toEntry(at.businessid, "registration number", .95),
1051
+ toEntry(be.vat, "tax identification number", .95),
1052
+ toEntry(be.nn, "national identification number", .9),
1053
+ toEntry(nl.vat, "tax identification number", .95),
1054
+ toEntry(dk.vat, "tax identification number", .95),
1055
+ toEntry(dk.cpr, "national identification number", .9),
1056
+ toEntry(fi.vat, "tax identification number", .95),
1057
+ toEntry(fi.hetu, "national identification number", .95),
1058
+ toEntry(fi.ytunnus, "registration number", .9),
1059
+ toEntry(bg.vat, "tax identification number", .95),
1060
+ toEntry(sk.dic, "tax identification number", .95),
1061
+ toEntry(es.cif, "registration number", .95),
1062
+ toEntry(es.vat, "tax identification number", .95),
1063
+ toEntry(es.nss, "social security number", .9),
1064
+ toEntry(fr.tva, "tax identification number", .95),
1065
+ toEntry(fr.siren, "registration number", .9),
1066
+ toEntry(fr.siret, "registration number", .9),
1067
+ toEntry(it.iva, "tax identification number", .95),
1068
+ toEntry(ie.vat, "tax identification number", .95),
1069
+ toEntry(ie.pps, "national identification number", .9),
1070
+ toEntry(pt.vat, "tax identification number", .95),
1071
+ toEntry(pt.cc, "national identification number", .9),
1072
+ toEntry(ro.vat, "tax identification number", .95),
1073
+ toEntry(gr.vat, "tax identification number", .95),
1074
+ toEntry(hr.vat, "tax identification number", .95),
1075
+ toEntry(si.vat, "tax identification number", .95),
1076
+ toEntry(lt.vat, "tax identification number", .95),
1077
+ toEntry(lt.asmens, "national identification number", .9),
1078
+ toEntry(lv.vat, "tax identification number", .95),
1079
+ toEntry(ee.vat, "tax identification number", .95),
1080
+ toEntry(ee.ik, "national identification number", .9),
1081
+ toEntry(cy.vat, "tax identification number", .95),
1082
+ toEntry(mt.vat, "tax identification number", .95),
1083
+ toEntry(lu.vat, "tax identification number", .95)
1084
+ ].filter((e) => e !== null);
1085
+ const TITLED_PERSON = {
1086
+ pattern: `(?:${TITLE_PREFIX})(?:${SP}+(?:${TITLE_PREFIX}))*${SP}+(?:${NAME_WORD})(?:${SP}{1,4}(?:${PARTICLE}${SP}+)?${NAME_WORD}){1,3}(?:,?${SP}+(?:${POST_NOMINAL})(?:,?${SP}+(?:${POST_NOMINAL}))*)?`,
1087
+ label: "person",
1088
+ score: .95
1089
+ };
1090
+ const HONORIFIC_PERSON = {
1091
+ pattern: `(?:${HONORIFIC_ALT})\\.?${SP}+${NAME_WORD}(?:(?:${SP}|-){1,2}(?:${PARTICLE}${SP}+)?${NAME_WORD}){0,3}(?:${SP}+(?:QC|KC|SC|LJ|AG))?`,
1092
+ label: "person",
1093
+ score: .95
1094
+ };
1095
+ const IBAN = {
1096
+ pattern: "\\b[A-Z]{2}\\d{2}\\s?[\\dA-Z]{4}\\s?[\\dA-Z]{4}\\s?[\\dA-Z]{4}\\s?[\\dA-Z]{4}\\s?[\\dA-Z]{0,14}\\b",
1097
+ label: "iban",
1098
+ score: 1
1099
+ };
1100
+ const EMAIL = {
1101
+ pattern: `\\b[\\w.+\\-]+@[\\w\\-]+(?:\\.[\\w\\-]+)+\\b`,
1102
+ label: "email address",
1103
+ score: 1
1104
+ };
1105
+ const INTL_PHONE = {
1106
+ pattern: "\\+\\d{1,3}(?:[^\\S\\n]|[.\\-])?\\(?\\d{2,4}\\)?(?:[^\\S\\n]|[.\\-])?\\d{3}(?:[^\\S\\n]|[.\\-])?\\d{2,4}(?:[^\\S\\n]|[.\\-])?\\d{0,4}\\b",
1107
+ label: "phone number",
1108
+ score: 1
1109
+ };
1110
+ const CZ_PHONE = {
1111
+ pattern: "\\b[2-7]\\d{2}(?:[^\\S\\n]|[.\\-])?\\d{3}(?:[^\\S\\n]|[.\\-])?\\d{3}(?!(?:[^\\S\\n]|[.\\-])?\\d*/\\d)(?![^\\S\\n]*(?:Kč|,-|korun|EUR|USD|€|\\$))\\b",
1112
+ label: "phone number",
1113
+ score: .85
1114
+ };
1115
+ /**
1116
+ * Phone numbers prefixed with "tel.:" or "telefon:".
1117
+ * Captures the number after the prefix, including
1118
+ * optional international code (+420).
1119
+ */
1120
+ const TEL_PREFIX_PHONE = {
1121
+ pattern: "(?:\\b[Tt]el(?:efon)?\\.?\\s*:?\\s*)(?:\\+?\\d{1,3}[^\\S\\n]?)?\\d{3}(?:[^\\S\\n]|[.\\-])?\\d{3}(?:[^\\S\\n]|[.\\-])?\\d{3}\\b",
1122
+ label: "phone number",
1123
+ score: .95
1124
+ };
1125
+ const CREDIT_CARD = {
1126
+ pattern: "\\b(?:4\\d{3}|5[1-5]\\d{2}|3[47]\\d{2})(?:[^\\S\\n]|[.\\-])?\\d{4}(?:[^\\S\\n]|[.\\-])?\\d{4}(?:[^\\S\\n]|[.\\-])?\\d{2,4}\\b",
1127
+ label: "credit card number",
1128
+ score: 1
1129
+ };
1130
+ const CZ_BIRTH_NUMBER = {
1131
+ pattern: `\\b\\d{6}/\\d{3,4}\\b`,
1132
+ label: "czech birth number",
1133
+ score: 1,
1134
+ validator: cz.rc
1135
+ };
1136
+ const DATE_NUMERIC = {
1137
+ pattern: "\\b(?:\\d{1,2}[./]\\d{1,2}[./]\\d{2,4}|\\d{4}-\\d{2}-\\d{2})\\b",
1138
+ label: "date",
1139
+ score: 1
1140
+ };
1141
+ const DATE_CZ_SPACED = {
1142
+ pattern: `\\b\\d{1,2}\\.[^\\S\\n]+\\d{1,2}\\.[^\\S\\n]+\\d{4}\\b`,
1143
+ label: "date",
1144
+ score: 1
1145
+ };
1146
+ const IP_ADDRESS = {
1147
+ pattern: "\\b(?:(?:25[0-5]|2[0-4]\\d|[01]?\\d\\d?)\\.){3}(?:25[0-5]|2[0-4]\\d|[01]?\\d\\d?)\\b",
1148
+ label: "ip address",
1149
+ score: 1
1150
+ };
1151
+ const CZ_BANK_ACCOUNT = {
1152
+ pattern: `\\b(?:\\d{1,6}-)?\\d{6,10}/\\d{4}(?!\\d)`,
1153
+ label: "bank account number",
1154
+ score: .95
1155
+ };
1156
+ const HU_LANDLINE = {
1157
+ pattern: "\\+36(?:[^\\S\\n]|[.\\-])?1(?:[^\\S\\n]|[.\\-])?\\d{3}(?:[^\\S\\n]|[.\\-])?\\d{4}\\b",
1158
+ label: "phone number",
1159
+ score: .9
1160
+ };
1161
+ const CZ_POSTAL = {
1162
+ pattern: `\\b\\d{3}[^\\S\\n]\\d{2}\\b`,
1163
+ label: "address",
1164
+ score: .7
1165
+ };
1166
+ const URL = {
1167
+ pattern: "(?:https?://|https?:(?=[^\\s])|www\\.)[\\w\\-]+(?:\\.[\\w\\-]+)+(?::\\d+)?(?:[/?#][^\\s)\\]>]*[^\\s.,;:!?)\\]>])?",
1168
+ label: "url",
1169
+ score: 1
1170
+ };
1171
+ const LONG_TLDS = "com|org|net|eu|cz|sk|pl|hu|ro|fr|es|co\\.uk|nl|ch|info|io|dev";
1172
+ const SHORT_TLDS = "de|at|be|se|fi|dk|no|it|uk";
1173
+ const HOST_LABEL = `[a-zA-Z0-9](?:[a-zA-Z0-9\\-]*[a-zA-Z0-9])?`;
1174
+ const BARE_HOST = `\\b[a-zA-Z0-9][a-zA-Z0-9\\-]+[a-zA-Z0-9]`;
1175
+ const PATH_SUFFIX = `(?:[/?#][^\\s)\\]>]*[^\\s.,;:!?)\\]>])?`;
1176
+ /**
1177
+ * All static PII regex definitions. Scanned in a
1178
+ * single pass by @stll/regex-set (Rust DFA).
1179
+ *
1180
+ * Hand-written patterns (0-17) followed by
1181
+ * stdnum-derived patterns (18+). Each stdnum entry
1182
+ * has a post-match validator for confirmation.
1183
+ *
1184
+ * Monetary amount patterns are built dynamically from
1185
+ * currencies.json via `getCurrencyPatterns()`.
1186
+ *
1187
+ * Date patterns using written month names are built
1188
+ * dynamically from date-months.json via
1189
+ * `getDatePatterns()`.
1190
+ */
1191
+ const ALL_REGEX_DEFS = [
1192
+ TITLED_PERSON,
1193
+ HONORIFIC_PERSON,
1194
+ IBAN,
1195
+ EMAIL,
1196
+ INTL_PHONE,
1197
+ CZ_PHONE,
1198
+ TEL_PREFIX_PHONE,
1199
+ CREDIT_CARD,
1200
+ CZ_BIRTH_NUMBER,
1201
+ DATE_NUMERIC,
1202
+ DATE_CZ_SPACED,
1203
+ IP_ADDRESS,
1204
+ CZ_BANK_ACCOUNT,
1205
+ HU_LANDLINE,
1206
+ CZ_POSTAL,
1207
+ URL,
1208
+ {
1209
+ pattern: "\\b(?:[0-9a-fA-F]{1,4}:){7}[0-9a-fA-F]{1,4}\\b|\\b(?:[0-9a-fA-F]{1,4}:){1,7}:\\b|::(?:[0-9a-fA-F]{1,4}:){0,6}[0-9a-fA-F]{1,4}",
1210
+ label: "ip address",
1211
+ score: 1
1212
+ },
1213
+ {
1214
+ pattern: "\\b(?:[0-9a-fA-F]{2}:){5}[0-9a-fA-F]{2}\\b|\\b(?:[0-9a-fA-F]{2}-){5}[0-9a-fA-F]{2}\\b",
1215
+ label: "mac address",
1216
+ score: 1
1217
+ },
1218
+ {
1219
+ pattern: `${BARE_HOST}(?:\\.${HOST_LABEL})*\\.(?:${LONG_TLDS})\\b${PATH_SUFFIX}|${BARE_HOST}(?:\\.${HOST_LABEL})+\\.(?:${SHORT_TLDS})\\b${PATH_SUFFIX}`,
1220
+ label: "url",
1221
+ score: .9
1222
+ },
1223
+ ...STDNUM_ENTRIES
1224
+ ];
1225
+ /** Flat pattern array for text-search. */
1226
+ const REGEX_PATTERNS = ALL_REGEX_DEFS.map((d) => d.pattern);
1227
+ /** Parallel metadata. Index = pattern index. */
1228
+ const REGEX_META = ALL_REGEX_DEFS.map((d) => {
1229
+ const meta = {
1230
+ label: d.label,
1231
+ score: d.score
1232
+ };
1233
+ if (d.validator) meta.validator = d.validator;
1234
+ return meta;
1235
+ });
1236
+ /**
1237
+ * Build month-name alternation from date-months.json.
1238
+ * Deduplicates across all 22 languages, filters names
1239
+ * shorter than 3 chars (too many false positives), and
1240
+ * sorts longest-first so the regex engine prefers the
1241
+ * longest match.
1242
+ */
1243
+ const buildMonthAlternation = (months) => {
1244
+ const seen = /* @__PURE__ */ new Set();
1245
+ for (const [key, value] of Object.entries(months)) {
1246
+ if (key.startsWith("_")) continue;
1247
+ const names = Array.isArray(value) ? value : [value];
1248
+ for (const name of names) {
1249
+ const clean = name.replace(/\.$/, "").toLowerCase();
1250
+ if (clean.length >= MIN_MONTH_NAME_LENGTH) seen.add(clean);
1251
+ }
1252
+ }
1253
+ return [...seen].toSorted((a, b) => b.length - a.length).map(escapeRegex$1).join("|");
1254
+ };
1255
+ /**
1256
+ * Build date patterns from a month-name alternation.
1257
+ * Returns 6 patterns covering the major written-date
1258
+ * formats across all supported languages.
1259
+ */
1260
+ const buildDatePatternsFromMonths = (alt) => {
1261
+ if (!alt) return [];
1262
+ return [
1263
+ `(?i)\\b\\d{1,2}\\.?\\s+(?:${alt})\\.?\\s+\\d{4}(?:\\s+\\d{1,2}:\\d{2}(?::\\d{2})?)?\\b`,
1264
+ `(?i)\\b(?:${alt})\\.?\\s+\\d{1,2},?\\s+\\d{4}\\b`,
1265
+ `(?i)\\b\\d{1,2}(?:st|nd|rd|th)\\s+(?:${alt})\\.?(?:\\s+\\d{4})?(?=\\s|[.,;!?)]|$)`,
1266
+ `(?i)\\b(?:${alt})\\.?\\s+\\d{4}\\b`,
1267
+ `(?i)\\b\\d{4}\\.\\s+(?:${alt})\\.?\\s+\\d{1,2}\\.?(?=\\s|[.,;!?)]|$)`,
1268
+ `(?i)\\b\\d{1,2}\\s+de\\s+(?:${alt})\\.?(?:\\s+de)?\\s+\\d{4}\\b`
1269
+ ];
1270
+ };
1271
+ /** Cached promise for date patterns. Loaded once. */
1272
+ let datePatternPromise = null;
1273
+ const loadDatePatterns = async () => {
1274
+ const mod = await import("@stll/anonymize-data/config/date-months.json");
1275
+ return buildDatePatternsFromMonths(buildMonthAlternation(mod.default ?? mod));
1276
+ };
1277
+ /**
1278
+ * Get dynamically built date patterns from
1279
+ * date-months.json. Returns a cached promise; the JSON
1280
+ * is loaded only once.
1281
+ */
1282
+ const getDatePatterns = () => {
1283
+ if (!datePatternPromise) datePatternPromise = loadDatePatterns().catch((err) => {
1284
+ datePatternPromise = null;
1285
+ throw err;
1286
+ });
1287
+ return datePatternPromise;
1288
+ };
1289
+ /** Date pattern metadata (all are score 1 dates). */
1290
+ const DATE_PATTERN_META = Object.freeze({
1291
+ label: "date",
1292
+ score: 1
1293
+ });
1294
+ /**
1295
+ * Build symbol character class, code alternation,
1296
+ * and local-name alternation from currencies.json,
1297
+ * then return two monetary amount patterns: leading
1298
+ * symbol and trailing code/name.
1299
+ *
1300
+ * The number sub-pattern accepts both grouped
1301
+ * thousands (1,000) and plain integers (100000)
1302
+ * via `\d{1,9}` to catch unformatted amounts.
1303
+ */
1304
+ const buildCurrencyPatterns = (data) => {
1305
+ const symbols = data.symbols.map(escapeCharClass).join("");
1306
+ const isAsciiAlpha = /^[a-zA-Z\s]+$/;
1307
+ const parts = data.codes.map((code) => ({
1308
+ len: code.length,
1309
+ alt: escapeRegex$1(code)
1310
+ }));
1311
+ const MIN_CI_LENGTH = 3;
1312
+ if (data.localNames) for (const name of data.localNames) {
1313
+ const escaped = escapeRegex$1(name);
1314
+ if (isAsciiAlpha.test(name) && name.length >= MIN_CI_LENGTH) parts.push({
1315
+ len: name.length,
1316
+ alt: `(?i:${escaped})`
1317
+ });
1318
+ else parts.push({
1319
+ len: name.length,
1320
+ alt: escaped
1321
+ });
1322
+ }
1323
+ if (symbols) for (const ch of data.symbols) parts.push({
1324
+ len: ch.length,
1325
+ alt: escapeRegex$1(ch)
1326
+ });
1327
+ const trailingAlt = parts.toSorted((a, b) => b.len - a.len).map((p) => p.alt).join("|");
1328
+ if (!symbols && !trailingAlt) return [];
1329
+ const NUM = "(?:\\d{1,3}(?:[,.'[^\\S\\n\\t]]\\d{3})+|\\d{1,9})";
1330
+ const patterns = [];
1331
+ if (symbols) patterns.push(`(?:[${symbols}])[^\\S\\n\\t]?${NUM}(?:[.,](?:\\d{1,2}[-–—]?|[-–—]{1,2})?)?\\b`);
1332
+ if (trailingAlt) patterns.push(`\\b(?:${trailingAlt})[^\\S\\n\\t]{0,2}${NUM}(?:[.,](?:\\d{1,2}[-–—]?|[-–—]{1,2})?)?\\b`);
1333
+ if (trailingAlt) patterns.push(`\\b${NUM}(?:[.,](?:\\d{1,2}[-–—]?|[-–—]{1,2})?)?[^\\S\\n\\t]{0,4}(?:${trailingAlt})(?:\\b|(?=\\s|[.,;!?)]|$))`);
1334
+ return patterns;
1335
+ };
1336
+ /** Cached promise for currency patterns. Loaded once. */
1337
+ let currencyPatternPromise = null;
1338
+ const loadCurrencyPatterns = async () => {
1339
+ const mod = await import("@stll/anonymize-data/config/currencies.json");
1340
+ return buildCurrencyPatterns(mod.default ?? mod);
1341
+ };
1342
+ /**
1343
+ * Get dynamically built monetary amount patterns from
1344
+ * currencies.json. Returns a cached promise; the JSON
1345
+ * is loaded only once.
1346
+ */
1347
+ const getCurrencyPatterns = () => {
1348
+ if (!currencyPatternPromise) currencyPatternPromise = loadCurrencyPatterns().catch((err) => {
1349
+ currencyPatternPromise = null;
1350
+ throw err;
1351
+ });
1352
+ return currencyPatternPromise;
1353
+ };
1354
+ /** Currency pattern metadata (score 0.9). */
1355
+ const CURRENCY_PATTERN_META = Object.freeze({
1356
+ label: "monetary amount",
1357
+ score: .9
1358
+ });
1359
+ /**
1360
+ * Process regex matches from the unified search.
1361
+ * Receives all matches; filters to the regex slice
1362
+ * via sliceStart/sliceEnd. Local index into META is
1363
+ * match.pattern - sliceStart.
1364
+ *
1365
+ * For stdnum-derived patterns (those with a validator
1366
+ * in META), the matched text is passed through the
1367
+ * validator's validate() method. If validation fails,
1368
+ * the match is discarded as a false positive.
1369
+ */
1370
+ const processRegexMatches = (allMatches, sliceStart, sliceEnd, meta_) => {
1371
+ const results = [];
1372
+ for (const match of allMatches) {
1373
+ const idx = match.pattern;
1374
+ if (idx < sliceStart || idx >= sliceEnd) continue;
1375
+ const meta = meta_[idx - sliceStart];
1376
+ if (!meta) continue;
1377
+ if (meta.label === "phone number" && match.text.length < MIN_PHONE_LENGTH) continue;
1378
+ if (meta.validator) {
1379
+ const compacted = meta.validator.compact(match.text);
1380
+ if (!meta.validator.validate(compacted).valid) continue;
1381
+ }
1382
+ results.push({
1383
+ start: match.start,
1384
+ end: match.end,
1385
+ label: meta.label,
1386
+ text: match.text,
1387
+ score: meta.score,
1388
+ source: DETECTION_SOURCES.REGEX
1389
+ });
1390
+ }
1391
+ return results;
1392
+ };
1393
+ /**
1394
+ * Build signing clause place-name patterns from
1395
+ * signing-clauses.json. Each pattern captures the
1396
+ * city/place name from contract signing locations.
1397
+ *
1398
+ * The place name sub-pattern:
1399
+ * \p{Lu}\p{Ll}+ (capitalized word)
1400
+ * optionally followed by preposition + capitalized
1401
+ * word (for "nad Nisou", "am Main", etc.)
1402
+ * optionally followed by more capitalized words
1403
+ * (for "Hradec Králové", "New York", etc.)
1404
+ */
1405
+ const buildSigningClausePatterns = (data) => {
1406
+ const patterns = [];
1407
+ for (const entry of data.patterns) {
1408
+ const prepAlt = entry.prepositions.length > 0 ? entry.prepositions.join("|") : null;
1409
+ const place = prepAlt ? `(\\p{Lu}\\p{Ll}+(?:\\s+(?:${prepAlt})\\s+\\p{Lu}\\p{Ll}+)*(?:\\s+\\p{Lu}\\p{Ll}+)*)` : `(\\p{Lu}\\p{Ll}+(?:[- ]\\p{Lu}\\p{Ll}+)*)`;
1410
+ const full = `(?:^|\\n|[^\\S\\n])` + entry.prefix + place + (entry.suffix ? `(?:${entry.suffix})` : "");
1411
+ patterns.push(full);
1412
+ }
1413
+ return patterns;
1414
+ };
1415
+ const SIGNING_CLAUSE_META = {
1416
+ label: "address",
1417
+ score: .9
1418
+ };
1419
+ let signingPatternPromise = null;
1420
+ const loadSigningPatterns = async () => {
1421
+ const mod = await import("@stll/anonymize-data/config/signing-clauses.json");
1422
+ return buildSigningClausePatterns(mod.default ?? mod);
1423
+ };
1424
+ const getSigningClausePatterns = () => {
1425
+ if (!signingPatternPromise) signingPatternPromise = loadSigningPatterns().catch((err) => {
1426
+ signingPatternPromise = null;
1427
+ throw err;
1428
+ });
1429
+ return signingPatternPromise;
1430
+ };
1431
+ //#endregion
1432
+ //#region src/detectors/legal-forms.ts
1433
+ const UPPER = "A-ZÁČĎÉĚÍŇÓŘŠŤÚŮÝŽÄÖÜÀÂÆÇÈÊËÎÏÔÙÛŸÑ\\u0130";
1434
+ const LOWER = "a-záčďéěíňóřšťúůýžäöüßàâæçèêëîïôùûÿñ\\u0131";
1435
+ const CAP_WORD = `(?:[${UPPER}]{2,}|[${UPPER}][${LOWER}${UPPER}]+)`;
1436
+ const ANY_WORD = `(?:[${UPPER}${LOWER}][${LOWER}${UPPER}]+|[${UPPER}]{2,3}|\\d{1,4})`;
1437
+ const ALLCAP_WORD = `[${UPPER}]{2,}`;
1438
+ const ROMAN_NUMERAL_RE = /^(?=[IVXLCDM])M{0,3}(?:CM|CD|D?C{0,3})(?:XC|XL|L?X{0,3})(?:IX|IV|V?I{0,3})$/;
1439
+ const escapeForRegex = (form) => form.replace(/[.*+?^${}()|[\]\\]/g, "\\$&").replace(/\s+/g, "\\s+").replace(/\\\./g, "\\.[^\\S\\n]?");
1440
+ const isShortForm = (form) => form.replace(/[.\s]/g, "").length <= 3 && !form.includes(" ");
1441
+ const buildPatternString = (forms, requireCapBefore) => {
1442
+ if (forms.length === 0) return null;
1443
+ const alt = forms.toSorted((a, b) => b.length - a.length).map(escapeForRegex).join("|");
1444
+ return `${`(?:${CAP_WORD})(?:(?:[\\s&,.\\-–—]{1,4}|\\s+(?:a|and|und|et|e|y|i)\\s+)(?:${ANY_WORD})){0,10}`}${requireCapBefore ? `(?:\\s+|,\\s*)` : `\\s+`}(?:${alt})(?![${LOWER}])`;
1445
+ };
1446
+ /**
1447
+ * Build legal form regex pattern strings.
1448
+ * Returns an array of regex strings for the unified
1449
+ * TextSearch builder. Empty if data package is not
1450
+ * installed.
1451
+ */
1452
+ const buildLegalFormPatterns = async () => {
1453
+ let data = {};
1454
+ try {
1455
+ data = (await import("@stll/anonymize-data/config/legal-forms.json")).default;
1456
+ } catch {
1457
+ return [];
1458
+ }
1459
+ const allForms = [];
1460
+ const seen = /* @__PURE__ */ new Set();
1461
+ for (const forms of Object.values(data)) for (const form of forms) {
1462
+ const key = form.toLowerCase();
1463
+ if (!seen.has(key)) {
1464
+ seen.add(key);
1465
+ allForms.push(form);
1466
+ }
1467
+ }
1468
+ const patterns = [];
1469
+ const longPattern = buildPatternString(allForms.filter((f) => !isShortForm(f)), false);
1470
+ if (longPattern) patterns.push(longPattern);
1471
+ const shortPattern = buildPatternString(allForms.filter(isShortForm), true);
1472
+ if (shortPattern) patterns.push(shortPattern);
1473
+ const allcapPrefix = `(?:${ALLCAP_WORD})(?:[\\s&,.\\-–—]{1,4}(?:${ALLCAP_WORD})){0,2}`;
1474
+ const allcapAlt = allForms.toSorted((a, b) => b.length - a.length).map(escapeForRegex).join("|");
1475
+ patterns.push(`${allcapPrefix}(?:\\s+|,\\s*)(?:${allcapAlt})(?![${LOWER}])`);
1476
+ return patterns;
1477
+ };
1478
+ /**
1479
+ * Process legal form matches from the unified search.
1480
+ * Receives all matches; filters to the legal forms
1481
+ * slice via sliceStart/sliceEnd.
1482
+ */
1483
+ const processLegalFormMatches = (allMatches, sliceStart, sliceEnd, fullText) => {
1484
+ const results = [];
1485
+ for (const match of allMatches) {
1486
+ const idx = match.pattern;
1487
+ if (idx < sliceStart || idx >= sliceEnd) continue;
1488
+ const text = match.text.trimEnd();
1489
+ if (text.length < 5) continue;
1490
+ if (text.includes("\n")) continue;
1491
+ const prefixEnd = text.lastIndexOf(",") !== -1 ? text.lastIndexOf(",") : text.lastIndexOf(" ");
1492
+ const prefixPart = prefixEnd > 0 ? text.slice(0, prefixEnd).replace(/[^a-zA-ZÀ-ž]/g, "") : text.replace(/[^a-zA-ZÀ-ž]/g, "");
1493
+ const isAllCapsMatch = prefixPart.length > 2 && prefixPart === prefixPart.toUpperCase();
1494
+ if (isAllCapsMatch && fullText) {
1495
+ const lineStart = fullText.lastIndexOf("\n", match.start);
1496
+ const lineEnd = fullText.indexOf("\n", match.end);
1497
+ const lineLetters = fullText.slice(lineStart + 1, lineEnd === -1 ? fullText.length : lineEnd).replace(/[^a-zA-ZÀ-ž]/g, "");
1498
+ const upperCount = [...lineLetters].filter((c) => c === c.toUpperCase()).length;
1499
+ if (lineLetters.length > 5 && upperCount / lineLetters.length >= .95) continue;
1500
+ if ((prefixPart.length > 0 ? text.slice(0, prefixEnd > 0 ? prefixEnd : text.length).trim().split(/\s+/).length : 0) > 3) continue;
1501
+ } else if (isAllCapsMatch) continue;
1502
+ const lastSpace = text.lastIndexOf(" ");
1503
+ const rawSuffix = lastSpace !== -1 ? text.slice(lastSpace + 1) : "";
1504
+ const suffixClean = rawSuffix.replace(/[.,]/g, "");
1505
+ if (suffixClean.length > 0 && ROMAN_NUMERAL_RE.test(suffixClean)) continue;
1506
+ if (suffixClean.length <= 2 && !/\./.test(rawSuffix) && /[^\x00-\x7F]/.test(text.slice(0, lastSpace !== -1 ? lastSpace : text.length))) continue;
1507
+ results.push({
1508
+ start: match.start,
1509
+ end: match.start + text.length,
1510
+ label: "organization",
1511
+ text,
1512
+ score: .95,
1513
+ source: DETECTION_SOURCES.LEGAL_FORM
1514
+ });
1515
+ }
1516
+ return results;
1517
+ };
1518
+ //#endregion
1519
+ //#region src/config/legal-forms.ts
1520
+ /**
1521
+ * Known legal form suffixes. Shared between trigger
1522
+ * detection (reclassification) and org-propagation
1523
+ * (suffix stripping). Within each family of related
1524
+ * forms, longer variants come first so that
1525
+ * "spol. s r.o." matches before "s.r.o." and
1526
+ * "s. r. o." matches before "s.r.o.".
1527
+ */
1528
+ const LEGAL_SUFFIXES = [
1529
+ "spol. s r.o.",
1530
+ "s.r.o.",
1531
+ "s. r. o.",
1532
+ "a.s.",
1533
+ "a. s.",
1534
+ "v.o.s.",
1535
+ "v. o. s.",
1536
+ "k.s.",
1537
+ "k. s.",
1538
+ "z.s.",
1539
+ "z. s.",
1540
+ "z.ú.",
1541
+ "z. ú.",
1542
+ "o.p.s.",
1543
+ "o. p. s.",
1544
+ "s.p.",
1545
+ "s. p.",
1546
+ "GmbH",
1547
+ "AG",
1548
+ "SE",
1549
+ "KG",
1550
+ "OHG",
1551
+ "Ltd.",
1552
+ "Ltd",
1553
+ "LLC",
1554
+ "LLP",
1555
+ "Inc.",
1556
+ "S.A.",
1557
+ "SA",
1558
+ "SAS",
1559
+ "SARL",
1560
+ "Sp. z o.o.",
1561
+ "S.p.A."
1562
+ ];
1563
+ //#endregion
1564
+ //#region src/detectors/triggers.ts
1565
+ const TRIGGER_SCORE = .95;
1566
+ const WHITESPACE_RE$1 = /\s+/;
1567
+ const LETTER_RE = /\p{L}/u;
1568
+ /**
1569
+ * Post-nominal degree regex. When a comma-stop is
1570
+ * followed by a known post-nominal (Ph.D., CSc., MBA
1571
+ * etc.), skip the comma and degree, then continue.
1572
+ */
1573
+ const POST_NOMINAL_RE = new RegExp(`^,\\s*(?:${POST_NOMINALS.toSorted((a, b) => b.length - a.length).map((d) => d.replace(/[.*+?^${}()|[\]\\]/g, "\\$&").replace(/\\\./g, "\\.\\s*")).join("|")})\\.?`, "i");
1574
+ const LEGAL_FORM_CHECK_RE = new RegExp(LEGAL_SUFFIXES.map((f) => {
1575
+ const escaped = f.replace(/[.*+?^${}()|[\]\\]/g, "\\$&").replace(/\\\./g, "\\.\\s*");
1576
+ const isDotFree = !f.includes(".");
1577
+ const isShort = f.length <= 4;
1578
+ return isDotFree && isShort ? `\\b${escaped}\\b` : escaped;
1579
+ }).join("|"));
1580
+ const compileValidations = (validations) => validations.map((v) => {
1581
+ switch (v.type) {
1582
+ case "starts-uppercase": return {
1583
+ type: "starts-uppercase",
1584
+ re: /^\p{Lu}/u
1585
+ };
1586
+ case "min-length": return {
1587
+ type: "min-length",
1588
+ min: v.min
1589
+ };
1590
+ case "max-length": return {
1591
+ type: "max-length",
1592
+ max: v.max
1593
+ };
1594
+ case "no-digits": return {
1595
+ type: "no-digits",
1596
+ re: /\d/
1597
+ };
1598
+ case "has-digits": return {
1599
+ type: "has-digits",
1600
+ re: /\d/
1601
+ };
1602
+ case "matches-pattern": return {
1603
+ type: "matches-pattern",
1604
+ re: new RegExp(v.pattern, (v.flags ?? "").replace(/[gy]/g, ""))
1605
+ };
1606
+ default: throw new Error(`Unknown validation type: ${JSON.stringify(v)}`);
1607
+ }
1608
+ });
1609
+ const applyValidations = (text, validations) => {
1610
+ for (const v of validations) switch (v.type) {
1611
+ case "starts-uppercase":
1612
+ if (!v.re.test(text)) return false;
1613
+ break;
1614
+ case "min-length":
1615
+ if (text.length < v.min) return false;
1616
+ break;
1617
+ case "max-length":
1618
+ if (text.length > v.max) return false;
1619
+ break;
1620
+ case "no-digits":
1621
+ if (v.re.test(text)) return false;
1622
+ break;
1623
+ case "has-digits":
1624
+ if (!v.re.test(text)) return false;
1625
+ break;
1626
+ case "matches-pattern":
1627
+ if (!v.re.test(text)) return false;
1628
+ break;
1629
+ default: throw new Error(`Unknown compiled validation type: ${JSON.stringify(v)}`);
1630
+ }
1631
+ return true;
1632
+ };
1633
+ const expandTriggerGroups = (groups) => {
1634
+ const rules = [];
1635
+ for (const group of groups) {
1636
+ const extensions = group.extensions ?? [];
1637
+ const compiled = compileValidations(group.validations ?? []);
1638
+ const allTriggers = new Set(group.triggers);
1639
+ for (const trigger of group.triggers) {
1640
+ if (extensions.includes("add-colon") && !trigger.endsWith(":")) allTriggers.add(`${trigger}:`);
1641
+ if (extensions.includes("add-trailing-space") && !trigger.endsWith(" ")) allTriggers.add(`${trigger} `);
1642
+ if (extensions.includes("add-colon-space") && !trigger.endsWith(": ") && !trigger.endsWith(":")) allTriggers.add(`${trigger}: `);
1643
+ if (extensions.includes("normalize-spaces")) {
1644
+ if (trigger.includes(" ")) allTriggers.add(trigger.replace(/ /g, "\xA0"));
1645
+ }
1646
+ }
1647
+ const includeTrigger = group.includeTrigger ?? false;
1648
+ for (const trigger of allTriggers) rules.push({
1649
+ trigger,
1650
+ label: group.label,
1651
+ strategy: group.strategy,
1652
+ validations: compiled,
1653
+ includeTrigger
1654
+ });
1655
+ }
1656
+ return rules;
1657
+ };
1658
+ /**
1659
+ * Build trigger patterns and rules from data configs.
1660
+ * Returns string[] for the unified TextSearch
1661
+ * builder and the parallel rules array.
1662
+ */
1663
+ const buildTriggerPatterns = async () => {
1664
+ const rules = [];
1665
+ const allGroups = await loadLanguageConfigs("triggers", (mod) => {
1666
+ return mod.default ?? mod;
1667
+ });
1668
+ for (const groups of allGroups) {
1669
+ if (!Array.isArray(groups)) {
1670
+ console.warn("[anonymize] triggers: unexpected config shape, skipping");
1671
+ continue;
1672
+ }
1673
+ rules.push(...expandTriggerGroups(groups));
1674
+ }
1675
+ try {
1676
+ const globalMod = await import("@stll/anonymize-data/config/triggers.global.json");
1677
+ const globalGroups = globalMod.default ?? globalMod;
1678
+ if (Array.isArray(globalGroups)) rules.push(...expandTriggerGroups(globalGroups));
1679
+ } catch (err) {
1680
+ if (!(err instanceof Error) || !err.message.includes("Cannot find module")) throw err;
1681
+ }
1682
+ const seen = /* @__PURE__ */ new Map();
1683
+ for (const rule of rules) {
1684
+ const key = rule.trigger.toLowerCase();
1685
+ const prev = seen.get(key);
1686
+ if (prev !== void 0) {
1687
+ const labelDiff = prev.label !== rule.label;
1688
+ const stratDiff = prev.strategy !== rule.strategy.type;
1689
+ if (labelDiff || stratDiff) console.warn(`[anonymize] duplicate trigger "${rule.trigger}":` + (labelDiff ? ` labels "${prev.label}" vs "${rule.label}"` : "") + (stratDiff ? ` strategies "${prev.strategy}" vs "${rule.strategy.type}"` : ""));
1690
+ }
1691
+ seen.set(key, {
1692
+ label: rule.label,
1693
+ strategy: rule.strategy.type
1694
+ });
1695
+ }
1696
+ return {
1697
+ patterns: rules.map((r) => r.trigger.toLowerCase()),
1698
+ rules
1699
+ };
1700
+ };
1701
+ const LEADING_PUNCT = /^[„""»«'"()\s]+/;
1702
+ const TRAILING_PUNCT = /[""»«'"()\s]+$/;
1703
+ const stripQuotes = (value) => {
1704
+ const leadingMatch = LEADING_PUNCT.exec(value.text);
1705
+ const leadingLen = leadingMatch ? leadingMatch[0].length : 0;
1706
+ const stripped = value.text.slice(leadingLen).replace(TRAILING_PUNCT, "");
1707
+ if (stripped.length === 0) return null;
1708
+ return {
1709
+ start: value.start + leadingLen,
1710
+ end: value.start + leadingLen + stripped.length,
1711
+ text: stripped
1712
+ };
1713
+ };
1714
+ /** Hard stop characters for to-next-comma scanning. */
1715
+ const COMMA_STOP_CHARS = new Set(["\n", "("]);
1716
+ /**
1717
+ * Field-label keywords that terminate address scanning.
1718
+ * When a comma in the address strategy is followed by
1719
+ * one of these, the address stops before the keyword.
1720
+ */
1721
+ const ADDRESS_STOP_KEYWORDS = [
1722
+ "číslo účtu",
1723
+ "registrační",
1724
+ "zastoupen",
1725
+ "bankovní",
1726
+ "e-mail",
1727
+ "telefon",
1728
+ "jednatel",
1729
+ "ředitel",
1730
+ "datová",
1731
+ "vložka",
1732
+ "sp.zn.",
1733
+ "oddíl",
1734
+ "swift",
1735
+ "email",
1736
+ "iban",
1737
+ "dič",
1738
+ "ičo",
1739
+ "tel",
1740
+ "č.ú.",
1741
+ "bic",
1742
+ "ič"
1743
+ ];
1744
+ const extractValue = (text, triggerEnd, strategy, label) => {
1745
+ const remaining = text.slice(triggerEnd);
1746
+ const stripped = remaining.replace(/^[\s:;]+/, "");
1747
+ const valueStart = triggerEnd + (remaining.length - stripped.length);
1748
+ const valueText = stripped;
1749
+ if (valueText.length === 0) return null;
1750
+ switch (strategy.type) {
1751
+ case "to-next-comma": {
1752
+ let end = 0;
1753
+ let foundStop = false;
1754
+ while (end < valueText.length) {
1755
+ const ch = valueText[end];
1756
+ if (ch !== void 0 && COMMA_STOP_CHARS.has(ch)) {
1757
+ foundStop = true;
1758
+ break;
1759
+ }
1760
+ if (ch === ",") {
1761
+ const afterComma = valueText.slice(end);
1762
+ const degreeMatch = label === "person" ? POST_NOMINAL_RE.exec(afterComma) : null;
1763
+ if (degreeMatch) {
1764
+ end += degreeMatch[0].length;
1765
+ continue;
1766
+ }
1767
+ foundStop = true;
1768
+ break;
1769
+ }
1770
+ end++;
1771
+ }
1772
+ if (!foundStop) end = Math.min(end, 100);
1773
+ const rawSlice = valueText.slice(0, end);
1774
+ const extracted = rawSlice.trim();
1775
+ if (extracted.length === 0) return null;
1776
+ const trailingSpaces = rawSlice.length - rawSlice.trimEnd().length;
1777
+ return {
1778
+ start: valueStart,
1779
+ end: valueStart + end - trailingSpaces,
1780
+ text: extracted
1781
+ };
1782
+ }
1783
+ case "to-end-of-line": {
1784
+ const LINE_STOPS = ["\n"];
1785
+ let end = valueText.length;
1786
+ for (const ch of LINE_STOPS) {
1787
+ const idx = valueText.indexOf(ch);
1788
+ if (idx !== -1 && idx < end) end = idx;
1789
+ }
1790
+ const rawSlice = valueText.slice(0, end);
1791
+ const extracted = rawSlice.trim();
1792
+ if (extracted.length === 0) return null;
1793
+ const trailingSpaces = rawSlice.length - rawSlice.trimEnd().length;
1794
+ return {
1795
+ start: valueStart,
1796
+ end: valueStart + end - trailingSpaces,
1797
+ text: extracted
1798
+ };
1799
+ }
1800
+ case "n-words": {
1801
+ const tabIdx = valueText.indexOf(" ");
1802
+ const cellText = tabIdx !== -1 ? valueText.slice(0, tabIdx) : valueText;
1803
+ const PUNCT_ONLY = /^[\p{P}\p{S}]+$/u;
1804
+ const words = cellText.split(WHITESPACE_RE$1).filter((w) => !PUNCT_ONLY.test(w)).slice(0, strategy.count);
1805
+ if (words.length === 0) return null;
1806
+ const firstWord = words[0];
1807
+ if (firstWord === void 0) return null;
1808
+ const firstIdx = cellText.indexOf(firstWord);
1809
+ let actualEnd = firstIdx + firstWord.length;
1810
+ let searchPos = actualEnd;
1811
+ for (let wi = 1; wi < words.length; wi++) {
1812
+ const w = words[wi];
1813
+ if (w === void 0) break;
1814
+ const wIdx = cellText.indexOf(w, searchPos);
1815
+ if (wIdx === -1) break;
1816
+ actualEnd = wIdx + w.length;
1817
+ searchPos = actualEnd;
1818
+ }
1819
+ return {
1820
+ start: valueStart + firstIdx,
1821
+ end: valueStart + actualEnd,
1822
+ text: cellText.slice(firstIdx, actualEnd)
1823
+ };
1824
+ }
1825
+ case "company-id-value": {
1826
+ const raw = text.slice(triggerEnd);
1827
+ const sepMatch = /^(?:\s*:\s*|\s+)/.exec(raw);
1828
+ if (!sepMatch) return null;
1829
+ const afterSep = raw.slice(sepMatch[0].length);
1830
+ const idMatch = /^[A-Z]{0,6}\s?\d[\d\s\-/]{4,}/i.exec(afterSep);
1831
+ if (!idMatch) return null;
1832
+ const idText = idMatch[0].trim();
1833
+ const leadingSpaces = idMatch[0].length - idMatch[0].trimStart().length;
1834
+ const idStart = triggerEnd + sepMatch[0].length + leadingSpaces;
1835
+ return {
1836
+ start: idStart,
1837
+ end: idStart + idText.length,
1838
+ text: idText
1839
+ };
1840
+ }
1841
+ case "address": {
1842
+ const maxLen = strategy.maxChars ?? 120;
1843
+ const UPPER_RE = /\p{Lu}/u;
1844
+ let end = 0;
1845
+ while (end < valueText.length && end < maxLen) {
1846
+ const ch = valueText[end];
1847
+ if (ch === "\n" || ch === "(") break;
1848
+ if (ch === ".") {
1849
+ const next = valueText[end + 1];
1850
+ const afterNext = valueText[end + 2];
1851
+ if (next !== void 0 && (/\p{L}/u.test(next) || /\d/.test(next))) {
1852
+ end++;
1853
+ continue;
1854
+ }
1855
+ if (next === " " && afterNext !== void 0 && (/\p{L}/u.test(afterNext) || /\d/.test(afterNext))) {
1856
+ end++;
1857
+ continue;
1858
+ }
1859
+ break;
1860
+ }
1861
+ if (ch === ",") {
1862
+ let peek = end + 1;
1863
+ while (peek < valueText.length && (valueText[peek] === " " || valueText[peek] === " ")) peek++;
1864
+ const peekCh = valueText[peek];
1865
+ if (peekCh === void 0) break;
1866
+ const afterComma = valueText.slice(end + 1).trimStart().toLowerCase();
1867
+ if (ADDRESS_STOP_KEYWORDS.some((kw) => {
1868
+ if (!afterComma.startsWith(kw)) return false;
1869
+ const next = afterComma[kw.length];
1870
+ return next === void 0 || /[\s:;.,!?()\d]/.test(next);
1871
+ })) break;
1872
+ if (/\d/.test(peekCh) || UPPER_RE.test(peekCh)) {
1873
+ end++;
1874
+ continue;
1875
+ }
1876
+ break;
1877
+ }
1878
+ end++;
1879
+ }
1880
+ if (end >= maxLen) {
1881
+ const lastSpace = valueText.lastIndexOf(" ", end - 1);
1882
+ if (lastSpace > 0) end = lastSpace;
1883
+ }
1884
+ const rawSlice = valueText.slice(0, end);
1885
+ const extracted = rawSlice.trim();
1886
+ if (extracted.length === 0) return null;
1887
+ const trailingSpaces = rawSlice.length - rawSlice.trimEnd().length;
1888
+ return {
1889
+ start: valueStart,
1890
+ end: valueStart + end - trailingSpaces,
1891
+ text: extracted
1892
+ };
1893
+ }
1894
+ default: return null;
1895
+ }
1896
+ };
1897
+ /**
1898
+ * Process trigger matches from the unified search.
1899
+ * Receives all matches; filters to the trigger slice
1900
+ * via sliceStart/sliceEnd. Uses fullText for value
1901
+ * extraction (the unified search runs on lowercased
1902
+ * text, but extraction needs original casing).
1903
+ */
1904
+ const processTriggerMatches = (allMatches, sliceStart, sliceEnd, fullText, rules) => {
1905
+ const results = [];
1906
+ for (const match of allMatches) {
1907
+ const idx = match.pattern;
1908
+ if (idx < sliceStart || idx >= sliceEnd) continue;
1909
+ const localIdx = idx - sliceStart;
1910
+ if (match.start > 0 && LETTER_RE.test(fullText[match.start - 1] ?? "")) continue;
1911
+ const rule = rules[localIdx];
1912
+ if (!rule) continue;
1913
+ if (!rule.trigger.endsWith(" ") && LETTER_RE.test(fullText[match.end] ?? "")) continue;
1914
+ const triggerEnd = match.end;
1915
+ const rawValue = extractValue(fullText, triggerEnd, rule.strategy, rule.label);
1916
+ const value = rawValue ? stripQuotes(rawValue) : null;
1917
+ if (value) {
1918
+ if (!applyValidations(value.text, rule.validations)) continue;
1919
+ const entityStart = rule.includeTrigger ? match.start : value.start;
1920
+ const entityEnd = value.end;
1921
+ const entityText = fullText.slice(entityStart, entityEnd);
1922
+ const effectiveLabel = rule.label === "person" && LEGAL_FORM_CHECK_RE.test(entityText) ? "organization" : rule.label;
1923
+ results.push({
1924
+ start: entityStart,
1925
+ end: entityEnd,
1926
+ label: effectiveLabel,
1927
+ text: entityText,
1928
+ score: TRIGGER_SCORE,
1929
+ source: DETECTION_SOURCES.TRIGGER
1930
+ });
1931
+ }
1932
+ }
1933
+ return results;
1934
+ };
1935
+ //#endregion
1936
+ //#region src/regions.ts
1937
+ /**
1938
+ * Geographic regions and country code mappings for
1939
+ * scoping deny list dictionaries.
1940
+ */
1941
+ const REGIONS = {
1942
+ Global: null,
1943
+ International: null,
1944
+ Europe: [
1945
+ "AL",
1946
+ "AD",
1947
+ "AT",
1948
+ "BE",
1949
+ "BA",
1950
+ "BG",
1951
+ "HR",
1952
+ "CY",
1953
+ "CZ",
1954
+ "DK",
1955
+ "EE",
1956
+ "FI",
1957
+ "FR",
1958
+ "DE",
1959
+ "GR",
1960
+ "HU",
1961
+ "IS",
1962
+ "IE",
1963
+ "IT",
1964
+ "XK",
1965
+ "LV",
1966
+ "LI",
1967
+ "LT",
1968
+ "LU",
1969
+ "MD",
1970
+ "ME",
1971
+ "MK",
1972
+ "MT",
1973
+ "MC",
1974
+ "NL",
1975
+ "NO",
1976
+ "PL",
1977
+ "PT",
1978
+ "RO",
1979
+ "RS",
1980
+ "SK",
1981
+ "SI",
1982
+ "ES",
1983
+ "SE",
1984
+ "CH",
1985
+ "UA",
1986
+ "GB"
1987
+ ],
1988
+ Americas: [
1989
+ "US",
1990
+ "CA",
1991
+ "MX",
1992
+ "BR",
1993
+ "AR",
1994
+ "CL",
1995
+ "CO",
1996
+ "PE",
1997
+ "EC",
1998
+ "VE",
1999
+ "UY",
2000
+ "PY",
2001
+ "BO",
2002
+ "CR",
2003
+ "PA",
2004
+ "DO",
2005
+ "GT",
2006
+ "HN",
2007
+ "SV",
2008
+ "NI",
2009
+ "CU"
2010
+ ],
2011
+ AsiaPacific: [
2012
+ "AU",
2013
+ "NZ",
2014
+ "JP",
2015
+ "KR",
2016
+ "CN",
2017
+ "TW",
2018
+ "SG",
2019
+ "MY",
2020
+ "TH",
2021
+ "VN",
2022
+ "PH",
2023
+ "ID",
2024
+ "IN",
2025
+ "PK",
2026
+ "BD",
2027
+ "LK",
2028
+ "NP",
2029
+ "HK",
2030
+ "MO"
2031
+ ],
2032
+ MENA: [
2033
+ "AE",
2034
+ "SA",
2035
+ "IL",
2036
+ "TR",
2037
+ "EG",
2038
+ "JO",
2039
+ "LB",
2040
+ "IQ",
2041
+ "IR",
2042
+ "QA",
2043
+ "KW",
2044
+ "BH",
2045
+ "OM",
2046
+ "MA",
2047
+ "TN",
2048
+ "DZ",
2049
+ "LY",
2050
+ "SY",
2051
+ "YE",
2052
+ "PS"
2053
+ ],
2054
+ SubSaharanAfrica: [
2055
+ "ZA",
2056
+ "NG",
2057
+ "KE",
2058
+ "GH",
2059
+ "TZ",
2060
+ "ET",
2061
+ "SN",
2062
+ "CI",
2063
+ "CM",
2064
+ "UG",
2065
+ "RW",
2066
+ "MZ",
2067
+ "AO",
2068
+ "ZW",
2069
+ "BW",
2070
+ "NA",
2071
+ "MU"
2072
+ ],
2073
+ EU: [
2074
+ "AT",
2075
+ "BE",
2076
+ "BG",
2077
+ "HR",
2078
+ "CY",
2079
+ "CZ",
2080
+ "DK",
2081
+ "EE",
2082
+ "FI",
2083
+ "FR",
2084
+ "DE",
2085
+ "GR",
2086
+ "HU",
2087
+ "IE",
2088
+ "IT",
2089
+ "LV",
2090
+ "LT",
2091
+ "LU",
2092
+ "MT",
2093
+ "NL",
2094
+ "PL",
2095
+ "PT",
2096
+ "RO",
2097
+ "SK",
2098
+ "SI",
2099
+ "ES",
2100
+ "SE"
2101
+ ],
2102
+ DACH: [
2103
+ "DE",
2104
+ "AT",
2105
+ "CH"
2106
+ ],
2107
+ Nordics: [
2108
+ "DK",
2109
+ "SE",
2110
+ "NO",
2111
+ "FI",
2112
+ "IS"
2113
+ ],
2114
+ CEE: [
2115
+ "CZ",
2116
+ "SK",
2117
+ "PL",
2118
+ "HU",
2119
+ "RO",
2120
+ "BG",
2121
+ "HR",
2122
+ "SI",
2123
+ "LT",
2124
+ "LV",
2125
+ "EE"
2126
+ ],
2127
+ Anglosphere: [
2128
+ "GB",
2129
+ "US",
2130
+ "CA",
2131
+ "AU",
2132
+ "NZ",
2133
+ "IE"
2134
+ ],
2135
+ Benelux: [
2136
+ "BE",
2137
+ "NL",
2138
+ "LU"
2139
+ ],
2140
+ GulfStates: [
2141
+ "AE",
2142
+ "SA",
2143
+ "QA",
2144
+ "KW",
2145
+ "BH",
2146
+ "OM"
2147
+ ],
2148
+ SouthAsia: [
2149
+ "IN",
2150
+ "PK",
2151
+ "BD",
2152
+ "LK",
2153
+ "NP"
2154
+ ],
2155
+ EastAsia: [
2156
+ "CN",
2157
+ "JP",
2158
+ "KR",
2159
+ "TW"
2160
+ ],
2161
+ SoutheastAsia: [
2162
+ "SG",
2163
+ "MY",
2164
+ "TH",
2165
+ "VN",
2166
+ "PH",
2167
+ "ID"
2168
+ ],
2169
+ Oceania: ["AU", "NZ"]
2170
+ };
2171
+ /**
2172
+ * Expand region names to country codes and merge with
2173
+ * explicit country codes. Returns null when both inputs
2174
+ * are empty/undefined (meaning "match all countries").
2175
+ */
2176
+ const resolveCountries = (regions, countries) => {
2177
+ const hasRegions = regions && regions.length > 0;
2178
+ const hasCountries = countries && countries.length > 0;
2179
+ if (!hasRegions && !hasCountries) return null;
2180
+ const result = /* @__PURE__ */ new Set();
2181
+ const isRegion = (name) => name in REGIONS;
2182
+ if (hasRegions) for (const name of regions) {
2183
+ if (!isRegion(name)) continue;
2184
+ const codes = REGIONS[name];
2185
+ if (codes === null) return null;
2186
+ for (const code of codes) result.add(code);
2187
+ }
2188
+ if (hasCountries) for (const code of countries) result.add(code);
2189
+ return result;
2190
+ };
2191
+ //#endregion
2192
+ //#region src/filters/false-positives.ts
2193
+ const TEMPLATE_PLACEHOLDER_RE = /^(?:\.{3,}|_{3,}|\[[\w\s]+\]|\{[\w\s]+\})$/;
2194
+ const POSTAL_CODE_RE = /\d{3}\s?\d{2}/;
2195
+ const HAS_DIGIT_RE = /\d/;
2196
+ const ADDRESS_COMPONENTS_RE = /(?:^|\s)(?:ul\.|ulice|nám\.|náměstí|tř\.|třída|nábř\.|nábřeží|č\.p\.|č\.ev\.|č\.|sídliště|bulvár)(?=[\s,./]|$)/i;
2197
+ const MAX_ENTITY_LENGTH = {
2198
+ organization: 80,
2199
+ person: 60
2200
+ };
2201
+ const SECTION_NUMBER_RE = /^(?:§\s*)?\d{1,3}(?:\.\d{1,3}){0,4}\.?$/;
2202
+ const STANDALONE_YEAR_RE = /^(?:19|20)\d{2}$/;
2203
+ const EMPTY_GENERIC_ROLES = /* @__PURE__ */ new Set();
2204
+ /**
2205
+ * Load generic-roles.json and cache the result on the
2206
+ * given context. Must be awaited during pipeline init
2207
+ * so the sync accessor is populated before
2208
+ * filterFalsePositives runs.
2209
+ */
2210
+ const loadGenericRoles = (ctx = defaultContext) => {
2211
+ if (ctx.genericRolesPromise) return ctx.genericRolesPromise;
2212
+ ctx.genericRolesPromise = (async () => {
2213
+ try {
2214
+ const mod = await import("@stll/anonymize-data/config/generic-roles.json");
2215
+ const set = new Set(mod.default?.roles ?? []);
2216
+ ctx.genericRoles = set;
2217
+ return set;
2218
+ } catch {
2219
+ const empty = /* @__PURE__ */ new Set();
2220
+ ctx.genericRoles = empty;
2221
+ return empty;
2222
+ }
2223
+ })();
2224
+ return ctx.genericRolesPromise;
2225
+ };
2226
+ /** Sync accessor — returns empty set before init. */
2227
+ const getGenericRoles = (ctx) => ctx.genericRoles ?? EMPTY_GENERIC_ROLES;
2228
+ /**
2229
+ * Filter out entities that are likely false positives:
2230
+ * template placeholders, clause/section numbers,
2231
+ * standalone years, and generic legal role terms.
2232
+ *
2233
+ * Runs as a post-processing step after all detection
2234
+ * layers have merged.
2235
+ */
2236
+ const filterFalsePositives = (entities, ctx = defaultContext) => {
2237
+ const filtered = [];
2238
+ const roles = getGenericRoles(ctx);
2239
+ for (const entity of entities) {
2240
+ const trimmed = entity.text.trim();
2241
+ if (TEMPLATE_PLACEHOLDER_RE.test(trimmed)) continue;
2242
+ const maxLen = MAX_ENTITY_LENGTH[entity.label];
2243
+ if (maxLen && trimmed.length > maxLen && entity.source !== "legal-form") continue;
2244
+ if (SECTION_NUMBER_RE.test(trimmed) && entity.source !== "trigger") continue;
2245
+ if (STANDALONE_YEAR_RE.test(trimmed)) continue;
2246
+ if ((entity.label === "person" || entity.label === "organization") && roles.has(trimmed.toLowerCase())) continue;
2247
+ if (entity.label === "address" && trimmed.length > 40 && !POSTAL_CODE_RE.test(trimmed) && !HAS_DIGIT_RE.test(trimmed) && !ADDRESS_COMPONENTS_RE.test(trimmed)) continue;
2248
+ if (entity.label === "address" && entity.source === "trigger" && !HAS_DIGIT_RE.test(trimmed) && !ADDRESS_COMPONENTS_RE.test(trimmed)) continue;
2249
+ filtered.push(entity);
2250
+ }
2251
+ return filtered;
2252
+ };
2253
+ //#endregion
2254
+ //#region src/util/normalize.ts
2255
+ const CHAR_MAP = new Map([
2256
+ [160, 32],
2257
+ [8199, 32],
2258
+ [8239, 32],
2259
+ [8211, 45],
2260
+ [8212, 45],
2261
+ [8220, 34],
2262
+ [8221, 34]
2263
+ ]);
2264
+ /** Chunk size for `String.fromCharCode` to avoid
2265
+ * hitting the call-stack limit on very large strings. */
2266
+ const CHUNK_SIZE = 8192;
2267
+ const normalizeForSearch = (text) => {
2268
+ let hasSpecial = false;
2269
+ for (let i = 0; i < text.length; i++) if (CHAR_MAP.has(text.charCodeAt(i))) {
2270
+ hasSpecial = true;
2271
+ break;
2272
+ }
2273
+ if (!hasSpecial) return text;
2274
+ const len = text.length;
2275
+ const codes = new Uint16Array(len);
2276
+ for (let i = 0; i < len; i++) {
2277
+ const code = text.charCodeAt(i);
2278
+ codes[i] = CHAR_MAP.get(code) ?? code;
2279
+ }
2280
+ if (len <= CHUNK_SIZE) return String.fromCharCode(...codes);
2281
+ let result = "";
2282
+ for (let offset = 0; offset < len; offset += CHUNK_SIZE) {
2283
+ const end = Math.min(offset + CHUNK_SIZE, len);
2284
+ result += String.fromCharCode(...codes.subarray(offset, end));
2285
+ }
2286
+ return result;
2287
+ };
2288
+ //#endregion
2289
+ //#region src/detectors/deny-list.ts
2290
+ /**
2291
+ * Try to load the optional @stll/anonymize-data package.
2292
+ * Returns null if not installed.
2293
+ */
2294
+ const loadDataModule = async () => {
2295
+ try {
2296
+ return await import("@stll/anonymize-data");
2297
+ } catch {
2298
+ return null;
2299
+ }
2300
+ };
2301
+ const loadAllowList = (ctx) => {
2302
+ if (ctx.allowListPromise) return ctx.allowListPromise;
2303
+ ctx.allowListPromise = (async () => {
2304
+ try {
2305
+ const mod = await import("@stll/anonymize-data/config/allow-list.json");
2306
+ const set = new Set(mod.default?.words ?? []);
2307
+ ctx.allowList = set;
2308
+ return set;
2309
+ } catch {
2310
+ const empty = /* @__PURE__ */ new Set();
2311
+ ctx.allowList = empty;
2312
+ return empty;
2313
+ }
2314
+ })();
2315
+ return ctx.allowListPromise;
2316
+ };
2317
+ const EMPTY_ALLOW_LIST = /* @__PURE__ */ new Set();
2318
+ /** Sync accessor — returns empty set before init. */
2319
+ const getAllowList = (ctx) => ctx.allowList ?? EMPTY_ALLOW_LIST;
2320
+ /**
2321
+ * Common EU given names present in the stopwords-iso dataset
2322
+ * but absent from the first-name corpus. Without this
2323
+ * supplementary set, these names would pass through the
2324
+ * corpus-based filter and remain in the stopwords, silently
2325
+ * suppressing person detection.
2326
+ *
2327
+ * Sourced from EU member state birth registries (top-100
2328
+ * names) cross-referenced with stopwords.json.
2329
+ */
2330
+ const SUPPLEMENTARY_NAME_EXCLUSIONS = new Set([
2331
+ "ana",
2332
+ "ben",
2333
+ "dan",
2334
+ "eden",
2335
+ "ella",
2336
+ "ina",
2337
+ "jo",
2338
+ "kai",
2339
+ "lena",
2340
+ "may",
2341
+ "mia",
2342
+ "sam",
2343
+ "sara",
2344
+ "sue",
2345
+ "tim",
2346
+ "tom"
2347
+ ]);
2348
+ /**
2349
+ * Names from the first-name corpus (lowercased) that also
2350
+ * appear in the stopwords-iso dataset, plus supplementary
2351
+ * common EU given names not in the corpus. These must be
2352
+ * kept out of global STOPWORDS so that person detection is
2353
+ * not silently suppressed for real given names.
2354
+ *
2355
+ * Computed lazily after initNameCorpus() has populated
2356
+ * the first-name corpus. Re-builds if corpus size changes.
2357
+ */
2358
+ const getFirstNameExclusions = (ctx) => {
2359
+ const corpus = getNameCorpusFirstNames(ctx);
2360
+ if (ctx.firstNameExclusions && corpus.length === ctx.firstNameExclusionCorpusLen) return ctx.firstNameExclusions;
2361
+ ctx.firstNameExclusionCorpusLen = corpus.length;
2362
+ const set = new Set([...corpus.map((n) => n.toLowerCase()), ...SUPPLEMENTARY_NAME_EXCLUSIONS]);
2363
+ ctx.firstNameExclusions = set;
2364
+ return set;
2365
+ };
2366
+ /**
2367
+ * Global stopwords: common words across 23 EU languages
2368
+ * sourced from the stopwords-iso dataset (MIT license).
2369
+ * Checked case-insensitively against matches.
2370
+ *
2371
+ * Entries that collide with the first-name corpus are
2372
+ * excluded so they can still be detected as person names.
2373
+ *
2374
+ * Regenerate: bun packages/data/scripts/generate-stopwords.ts
2375
+ */
2376
+ const loadStopwords = (ctx) => {
2377
+ if (ctx.stopwordsPromise) return ctx.stopwordsPromise;
2378
+ ctx.stopwordsPromise = (async () => {
2379
+ try {
2380
+ const list = ((await import("@stll/anonymize-data/config/stopwords.json")).default ?? []).filter((w) => !getFirstNameExclusions(ctx).has(w));
2381
+ const set = new Set(list);
2382
+ ctx.stopwords = set;
2383
+ return set;
2384
+ } catch (err) {
2385
+ console.warn("[anonymize] Failed to load stopwords.json — stopword filtering disabled:", err);
2386
+ const empty = /* @__PURE__ */ new Set();
2387
+ ctx.stopwords = empty;
2388
+ return empty;
2389
+ }
2390
+ })();
2391
+ return ctx.stopwordsPromise;
2392
+ };
2393
+ const EMPTY_STOPWORDS = /* @__PURE__ */ new Set();
2394
+ /** Sync accessor — returns empty set before init. */
2395
+ const getStopwords = (ctx) => ctx.stopwords ?? EMPTY_STOPWORDS;
2396
+ const loadPersonStopwords = (ctx) => {
2397
+ if (ctx.personStopwordsPromise) return ctx.personStopwordsPromise;
2398
+ ctx.personStopwordsPromise = (async () => {
2399
+ try {
2400
+ const mod = await import("@stll/anonymize-data/config/person-stopwords.json");
2401
+ const set = new Set(mod.default?.words ?? []);
2402
+ ctx.personStopwords = set;
2403
+ return set;
2404
+ } catch {
2405
+ const empty = /* @__PURE__ */ new Set();
2406
+ ctx.personStopwords = empty;
2407
+ return empty;
2408
+ }
2409
+ })();
2410
+ return ctx.personStopwordsPromise;
2411
+ };
2412
+ const EMPTY_PERSON_STOPWORDS = /* @__PURE__ */ new Set();
2413
+ /** Sync accessor — returns empty set before init. */
2414
+ const getPersonStopwords = (ctx) => ctx.personStopwords ?? EMPTY_PERSON_STOPWORDS;
2415
+ /**
2416
+ * Resolve which dictionaries to load based on country
2417
+ * and category filters, load them, and build the deny
2418
+ * list data. The returned data provides PatternEntry[]
2419
+ * for the unified builder and parallel arrays for
2420
+ * post-processing.
2421
+ *
2422
+ * Requires `@stll/anonymize-data` to be installed.
2423
+ * Returns null if the data package is not available.
2424
+ */
2425
+ const buildDenyList = async (config, ctx = defaultContext) => {
2426
+ await initNameCorpus(ctx);
2427
+ await Promise.all([
2428
+ loadStopwords(ctx),
2429
+ loadAllowList(ctx),
2430
+ loadPersonStopwords(ctx),
2431
+ loadGenericRoles(ctx)
2432
+ ]);
2433
+ const dataModule = await loadDataModule();
2434
+ if (!dataModule) return null;
2435
+ const allowedCountries = resolveCountries(config.denyListRegions, config.denyListCountries);
2436
+ const excluded = config.denyListExcludeCategories;
2437
+ const excludeCategories = excluded ? new Set(excluded) : /* @__PURE__ */ new Set();
2438
+ const ids = [...dataModule.ALL_DICTIONARY_IDS].filter((id) => {
2439
+ const meta = dataModule.DICTIONARY_META[id];
2440
+ if (!meta) return false;
2441
+ if (excludeCategories.has(meta.category)) return false;
2442
+ if (allowedCountries === null) return true;
2443
+ if (meta.country === null) return true;
2444
+ return allowedCountries.has(meta.country);
2445
+ });
2446
+ const patternList = [];
2447
+ const labelList = [];
2448
+ const sourceList = [];
2449
+ const patternIndex = /* @__PURE__ */ new Map();
2450
+ const results = await Promise.all(ids.map(async (id) => {
2451
+ return {
2452
+ id,
2453
+ entries: await dataModule.loadDictionary(id)
2454
+ };
2455
+ }));
2456
+ const addDenyListEntry = (entry, label) => {
2457
+ const normalized = normalizeForSearch(entry).replace(/[|\\]/g, "");
2458
+ if (normalized.length === 0) return;
2459
+ const lower = normalized.toLowerCase();
2460
+ const existing = patternIndex.get(lower);
2461
+ if (existing !== void 0) {
2462
+ if (!labelList[existing].includes(label)) labelList[existing].push(label);
2463
+ if (!sourceList[existing].includes("deny-list")) sourceList[existing].push("deny-list");
2464
+ } else {
2465
+ patternIndex.set(lower, patternList.length);
2466
+ patternList.push(normalized);
2467
+ labelList.push([label]);
2468
+ sourceList.push(["deny-list"]);
2469
+ }
2470
+ };
2471
+ for (const { id, entries } of results) {
2472
+ const meta = dataModule.DICTIONARY_META[id];
2473
+ if (!meta) continue;
2474
+ for (const entry of entries) addDenyListEntry(entry, meta.label);
2475
+ }
2476
+ if (!excludeCategories.has("Places")) {
2477
+ const cityCountries = allowedCountries !== null ? [...allowedCountries] : [
2478
+ "AT",
2479
+ "AU",
2480
+ "BE",
2481
+ "BG",
2482
+ "BR",
2483
+ "CA",
2484
+ "CH",
2485
+ "CZ",
2486
+ "DE",
2487
+ "DK",
2488
+ "ES",
2489
+ "FI",
2490
+ "FR",
2491
+ "GB",
2492
+ "GR",
2493
+ "HR",
2494
+ "HU",
2495
+ "IE",
2496
+ "IT",
2497
+ "LU",
2498
+ "NL",
2499
+ "NO",
2500
+ "NZ",
2501
+ "PL",
2502
+ "PT",
2503
+ "RO",
2504
+ "SE",
2505
+ "SI",
2506
+ "SK",
2507
+ "US"
2508
+ ];
2509
+ const cityEntries = await dataModule.loadCityDictionaries(cityCountries);
2510
+ for (const entry of cityEntries) addDenyListEntry(entry, "address");
2511
+ }
2512
+ const addNameEntry = (name, source) => {
2513
+ const normalized = normalizeForSearch(name).replace(/[|\\]/g, "");
2514
+ if (normalized.length === 0) return;
2515
+ const lower = normalized.toLowerCase();
2516
+ const existing = patternIndex.get(lower);
2517
+ if (existing !== void 0) {
2518
+ if (!labelList[existing].includes("person")) labelList[existing].push("person");
2519
+ if (!sourceList[existing].includes(source)) sourceList[existing].push(source);
2520
+ } else {
2521
+ patternIndex.set(lower, patternList.length);
2522
+ patternList.push(normalized);
2523
+ labelList.push(["person"]);
2524
+ sourceList.push([source]);
2525
+ }
2526
+ };
2527
+ for (const name of getNameCorpusFirstNames(ctx)) addNameEntry(name, "first-name");
2528
+ for (const name of getNameCorpusSurnames(ctx)) addNameEntry(name, "surname");
2529
+ for (const title of getNameCorpusTitles(ctx)) {
2530
+ const norm = normalizeForSearch(title).replace(/[|\\]/g, "");
2531
+ if (norm.length === 0) continue;
2532
+ const lower = norm.toLowerCase();
2533
+ const existing = patternIndex.get(lower);
2534
+ if (existing !== void 0) {
2535
+ if (!sourceList[existing].includes("title")) sourceList[existing].push("title");
2536
+ } else {
2537
+ patternIndex.set(lower, patternList.length);
2538
+ patternList.push(norm);
2539
+ labelList.push(["person"]);
2540
+ sourceList.push(["title"]);
2541
+ }
2542
+ }
2543
+ if (patternList.length === 0) return null;
2544
+ return {
2545
+ labels: labelList,
2546
+ originals: patternList,
2547
+ sources: sourceList
2548
+ };
2549
+ };
2550
+ /**
2551
+ * Ensure all deny-list support data (stopwords, allow
2552
+ * list, person stopwords, generic roles) is loaded on
2553
+ * the given context. Call this before
2554
+ * processDenyListMatches / filterFalsePositives when
2555
+ * the search instance was built on a different context
2556
+ * (e.g. cachedSearch).
2557
+ */
2558
+ const ensureDenyListData = async (ctx = defaultContext) => {
2559
+ await initNameCorpus(ctx);
2560
+ await Promise.all([
2561
+ loadStopwords(ctx),
2562
+ loadAllowList(ctx),
2563
+ loadPersonStopwords(ctx),
2564
+ loadGenericRoles(ctx)
2565
+ ]);
2566
+ };
2567
+ /**
2568
+ * Process deny list matches from the unified search.
2569
+ * Receives all matches; filters to the deny list slice
2570
+ * via sliceStart/sliceEnd. Local index into data.labels,
2571
+ * data.originals, data.sources is match.pattern - sliceStart.
2572
+ *
2573
+ * Two-pass approach to reduce false positives:
2574
+ * 1. Collect all matches (case-insensitive,
2575
+ * whole-word via Rust automaton)
2576
+ * 2. Require uppercase start in source text
2577
+ * 3. For person names, require at least one
2578
+ * mid-sentence occurrence to prove proper noun
2579
+ * 4. Return all occurrences of validated terms
2580
+ */
2581
+ const processDenyListMatches = (allMatches, sliceStart, sliceEnd, fullText, data, ctx = defaultContext) => {
2582
+ const matchesByPattern = /* @__PURE__ */ new Map();
2583
+ for (const match of allMatches) {
2584
+ const idx = match.pattern;
2585
+ if (idx < sliceStart || idx >= sliceEnd) continue;
2586
+ const localIdx = idx - sliceStart;
2587
+ const sourceChar = fullText[match.start] ?? "";
2588
+ if (!UPPER_START_RE.test(sourceChar)) continue;
2589
+ const matchText = fullText.slice(match.start, match.end);
2590
+ const keyword = matchText.toLowerCase();
2591
+ if (getStopwords(ctx).has(keyword) || getAllowList(ctx).has(keyword)) continue;
2592
+ if (ALL_UPPER_RE.test(matchText)) continue;
2593
+ const labels = data.labels[localIdx];
2594
+ if (!labels || labels.length === 0) continue;
2595
+ const entry = {
2596
+ start: match.start,
2597
+ end: match.end,
2598
+ labels,
2599
+ text: matchText,
2600
+ patternIdx: localIdx
2601
+ };
2602
+ const existing = matchesByPattern.get(localIdx);
2603
+ if (existing) existing.push(entry);
2604
+ else matchesByPattern.set(localIdx, [entry]);
2605
+ }
2606
+ const results = [];
2607
+ const nameHits = [];
2608
+ for (const [, matches] of Array.from(matchesByPattern)) {
2609
+ const first = matches[0];
2610
+ if (!first) continue;
2611
+ const hasPerson = first.labels.includes("person");
2612
+ const nonPersonLabels = first.labels.filter((l) => l !== "person");
2613
+ if (hasPerson) {
2614
+ const keyword = first.text.toLowerCase();
2615
+ if (!getPersonStopwords(ctx).has(keyword)) for (const m of matches) nameHits.push(m);
2616
+ }
2617
+ for (const m of matches) for (const label of nonPersonLabels) results.push({
2618
+ start: m.start,
2619
+ end: m.end,
2620
+ label,
2621
+ text: m.text,
2622
+ score: .9,
2623
+ source: DETECTION_SOURCES.DENY_LIST
2624
+ });
2625
+ }
2626
+ nameHits.sort((a, b) => a.start - b.start);
2627
+ const nameConsumed = /* @__PURE__ */ new Set();
2628
+ for (let i = 0; i < nameHits.length; i++) {
2629
+ if (nameConsumed.has(i)) continue;
2630
+ const hit = nameHits[i];
2631
+ if (!hit) continue;
2632
+ const chain = [hit];
2633
+ let j = i + 1;
2634
+ while (j < nameHits.length && chain.length < 5) {
2635
+ const next = nameHits[j];
2636
+ if (!next) break;
2637
+ const prev = chain.at(-1);
2638
+ if (!prev) break;
2639
+ const gap = fullText.slice(prev.end, next.start);
2640
+ if (gap.length > 4 || gap.length === 0 || gap.includes("\n") || gap.includes(" ")) break;
2641
+ chain.push(next);
2642
+ j++;
2643
+ }
2644
+ for (let k = i; k < i + chain.length; k++) nameConsumed.add(k);
2645
+ const first = chain.at(0);
2646
+ const last = chain.at(-1);
2647
+ if (!first || !last) continue;
2648
+ const extended = extendPersonName(fullText, first.start, last.end, ctx);
2649
+ const score = chain.length >= 2 ? .9 : .5;
2650
+ if (chain.length === 1 && isSentenceStart(fullText, first.start)) continue;
2651
+ results.push({
2652
+ start: first.start,
2653
+ end: extended.end,
2654
+ label: "person",
2655
+ text: extended.text,
2656
+ score,
2657
+ source: DETECTION_SOURCES.DENY_LIST
2658
+ });
2659
+ }
2660
+ extendCityDistricts(results, fullText);
2661
+ return results;
2662
+ };
2663
+ const DISTRICT_SUFFIX_RE = new RegExp(`^ (\\d{1,2}(?!\\d)|(?:XXX|XXIX|XXVIII|XXVII|XXVI|XXV|XXIV|XXIII|XXII|XXI|XX|XIX|XVIII|XVII|XVI|XV|XIV|XIII|XII|XI|X|IX|VIII|VII|VI|IV|III|II))(?=[\\s,;.)"\\n]|$)`);
2664
+ const POSTAL_PREFIX_RE = /(?:\d{5}|\d{3}\s\d{2})\s+$/;
2665
+ const TRAILING_WORD_EXCLUSIONS = new Set([
2666
+ "nájemce",
2667
+ "pronajímatel",
2668
+ "kupující",
2669
+ "prodávající",
2670
+ "objednatel",
2671
+ "zhotovitel",
2672
+ "dodavatel",
2673
+ "odběratel",
2674
+ "věřitel",
2675
+ "dlužník",
2676
+ "zadavatel",
2677
+ "uchazeč",
2678
+ "příjemce",
2679
+ "plátce",
2680
+ "správa",
2681
+ "sekretariát",
2682
+ "kancelář",
2683
+ "odbor",
2684
+ "oddělení",
2685
+ "úřad",
2686
+ "inspekce",
2687
+ "agentura",
2688
+ "článek",
2689
+ "smlouva",
2690
+ "dodatek",
2691
+ "příloha",
2692
+ "předmět",
2693
+ "podmínky",
2694
+ "ustanovení"
2695
+ ]);
2696
+ const extendCityDistricts = (entities, fullText) => {
2697
+ for (const entity of entities) {
2698
+ if (entity.label !== "address") continue;
2699
+ const afterMatch = fullText.slice(entity.end);
2700
+ const suffixM = DISTRICT_SUFFIX_RE.exec(afterMatch);
2701
+ if (suffixM) {
2702
+ entity.end += suffixM[0].length;
2703
+ entity.text = fullText.slice(entity.start, entity.end);
2704
+ }
2705
+ const afterDistrict = fullText.slice(entity.end);
2706
+ const dashDistrictM = /^[\s]*[-–][\s]*(\p{Lu}\p{Ll}+)/u.exec(afterDistrict);
2707
+ if (dashDistrictM && !dashDistrictM[0].includes("\n")) {
2708
+ entity.end += dashDistrictM[0].length;
2709
+ entity.text = fullText.slice(entity.start, entity.end);
2710
+ }
2711
+ const beforeMatch = fullText.slice(Math.max(0, entity.start - 10), entity.start);
2712
+ const prefixM = POSTAL_PREFIX_RE.exec(beforeMatch);
2713
+ if (prefixM) {
2714
+ entity.start -= prefixM[0].length;
2715
+ entity.text = fullText.slice(entity.start, entity.end);
2716
+ }
2717
+ const afterExt = fullText.slice(entity.end);
2718
+ const trailingWordM = /^[\s]+(\p{Lu}\p{Ll}+)/u.exec(afterExt);
2719
+ if (trailingWordM && !trailingWordM[0].includes("\n")) {
2720
+ const candidate = (trailingWordM[1] ?? "").toLowerCase();
2721
+ if (!TRAILING_WORD_EXCLUSIONS.has(candidate)) {
2722
+ entity.end += trailingWordM[0].length;
2723
+ entity.text = fullText.slice(entity.start, entity.end);
2724
+ }
2725
+ }
2726
+ }
2727
+ };
2728
+ /**
2729
+ * Extend a person name match to include subsequent
2730
+ * capitalized words. "Pavel" + " Heřmánek" → "Pavel
2731
+ * Heřmánek". Stops at lowercase words, punctuation,
2732
+ * or end of text. Also extends backward if preceded
2733
+ * by a capitalized word (for "Miroslav Braňka" when
2734
+ * only "Braňka" matched).
2735
+ */
2736
+ const extendPersonName = (text, start, end, ctx) => {
2737
+ let newEnd = end;
2738
+ let pos = newEnd;
2739
+ while (pos < text.length) if (pos < text.length && text[pos] === " ") {
2740
+ const wordStart = pos + 1;
2741
+ if (wordStart >= text.length) break;
2742
+ const char = text[wordStart] ?? "";
2743
+ if (!UPPER_START_RE.test(char)) break;
2744
+ let wordEnd = wordStart;
2745
+ while (wordEnd < text.length && !/\s/.test(text[wordEnd] ?? "")) wordEnd++;
2746
+ const stripped = text.slice(wordStart, wordEnd).replace(/[,;.]+$/, "");
2747
+ if (stripped.length < 2) break;
2748
+ const lower = stripped.toLowerCase();
2749
+ if (getStopwords(ctx).has(lower) || getPersonStopwords(ctx).has(lower)) break;
2750
+ newEnd = wordStart + stripped.length;
2751
+ pos = newEnd;
2752
+ } else break;
2753
+ return {
2754
+ end: newEnd,
2755
+ text: text.slice(start, newEnd)
2756
+ };
2757
+ };
2758
+ //#endregion
2759
+ //#region src/detectors/address-seeds.ts
2760
+ let cachedBoundaryRe = null;
2761
+ const loadBoundaryWords = async () => {
2762
+ try {
2763
+ return (await import("@stll/anonymize-data/config/address-boundaries.json")).default;
2764
+ } catch {
2765
+ return {};
2766
+ }
2767
+ };
2768
+ /**
2769
+ * Build regex for boundary words. Matches any
2770
+ * boundary word preceded by a word boundary.
2771
+ */
2772
+ const getBoundaryRe = async () => {
2773
+ if (cachedBoundaryRe) return cachedBoundaryRe;
2774
+ const config = await loadBoundaryWords();
2775
+ const words = [];
2776
+ for (const entries of Object.values(config)) {
2777
+ if (!Array.isArray(entries)) continue;
2778
+ for (const word of entries) words.push(word.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"));
2779
+ }
2780
+ words.sort((a, b) => b.length - a.length);
2781
+ cachedBoundaryRe = words.length > 0 ? new RegExp(`\\b(?:${words.join("|")})\\b`, "i") : /(?!)/;
2782
+ return cachedBoundaryRe;
2783
+ };
2784
+ /**
2785
+ * Build street type patterns for the unified search.
2786
+ * Returns string[] for the unified TextSearch
2787
+ * builder. Empty if data package is not installed.
2788
+ */
2789
+ const buildStreetTypePatterns = async () => {
2790
+ let config = {};
2791
+ try {
2792
+ config = (await import("@stll/anonymize-data/config/address-street-types.json")).default;
2793
+ } catch {
2794
+ return [];
2795
+ }
2796
+ const words = [];
2797
+ for (const values of Object.values(config)) {
2798
+ if (!Array.isArray(values)) continue;
2799
+ for (const word of values) words.push(word);
2800
+ }
2801
+ return words;
2802
+ };
2803
+ const collectSeeds = (allMatches, sliceStart, sliceEnd, fullText, existingEntities) => {
2804
+ const seeds = [];
2805
+ for (const match of allMatches) {
2806
+ const idx = match.pattern;
2807
+ if (idx < sliceStart || idx >= sliceEnd) continue;
2808
+ seeds.push({
2809
+ type: "street-word",
2810
+ start: match.start,
2811
+ end: match.end,
2812
+ text: match.text
2813
+ });
2814
+ }
2815
+ for (const e of existingEntities) {
2816
+ if (e.label !== "address") continue;
2817
+ if (e.source === "deny-list") seeds.push({
2818
+ type: "city",
2819
+ start: e.start,
2820
+ end: e.end,
2821
+ text: e.text
2822
+ });
2823
+ else if (e.source === "trigger" && /^\d/.test(e.text)) seeds.push({
2824
+ type: "postal-code",
2825
+ start: e.start,
2826
+ end: e.end,
2827
+ text: e.text
2828
+ });
2829
+ else if (e.source === "trigger") seeds.push({
2830
+ type: "address-trigger",
2831
+ start: e.start,
2832
+ end: e.end,
2833
+ text: e.text
2834
+ });
2835
+ }
2836
+ const postalRe = /\b(?:\d{3}\s\d{2}|\d{2}-\d{3})\b/g;
2837
+ let postalMatch;
2838
+ while ((postalMatch = postalRe.exec(fullText)) !== null) {
2839
+ const start = postalMatch.index;
2840
+ const end = start + postalMatch[0].length;
2841
+ if (!seeds.some((s) => s.start <= start && s.end >= end)) seeds.push({
2842
+ type: "postal-code",
2843
+ start,
2844
+ end,
2845
+ text: postalMatch[0]
2846
+ });
2847
+ }
2848
+ const streetNumRe = /\b(\p{Lu}\p{Ll}{2,})\s+(\d{1,5}(?:\/\d{1,5})?)\s*[,\n]/gu;
2849
+ let streetMatch;
2850
+ while ((streetMatch = streetNumRe.exec(fullText)) !== null) {
2851
+ const matchedStreet = streetMatch[1];
2852
+ const matchedNum = streetMatch[2];
2853
+ if (!matchedStreet || !matchedNum) continue;
2854
+ const start = streetMatch.index;
2855
+ const end = start + matchedStreet.length + 1 + matchedNum.length;
2856
+ seeds.push({
2857
+ type: "street-word",
2858
+ start,
2859
+ end,
2860
+ text: fullText.slice(start, end)
2861
+ });
2862
+ }
2863
+ return seeds.sort((a, b) => a.start - b.start);
2864
+ };
2865
+ const clusterSeeds = (seeds, maxGap) => {
2866
+ const first = seeds[0];
2867
+ if (!first) return [];
2868
+ const clusters = [];
2869
+ let current = {
2870
+ seeds: [first],
2871
+ start: first.start,
2872
+ end: first.end
2873
+ };
2874
+ for (let i = 1; i < seeds.length; i++) {
2875
+ const seed = seeds.at(i);
2876
+ if (!seed) continue;
2877
+ if (seed.start - current.end <= maxGap) {
2878
+ current.seeds.push(seed);
2879
+ current.end = Math.max(current.end, seed.end);
2880
+ } else {
2881
+ clusters.push(current);
2882
+ current = {
2883
+ seeds: [seed],
2884
+ start: seed.start,
2885
+ end: seed.end
2886
+ };
2887
+ }
2888
+ }
2889
+ clusters.push(current);
2890
+ return clusters;
2891
+ };
2892
+ const scoreCluster = (cluster) => {
2893
+ const types = new Set(cluster.seeds.map((s) => s.type));
2894
+ if (types.size < 2) return 0;
2895
+ let score = .5;
2896
+ if (types.has("postal-code")) score += .15;
2897
+ if (types.has("city")) score += .15;
2898
+ if (types.has("street-word")) score += .15;
2899
+ if (types.has("address-trigger")) score += .1;
2900
+ return Math.min(score, .95);
2901
+ };
2902
+ const NON_ADDRESS_LABELS = new Set([
2903
+ "registration number",
2904
+ "tax identification number",
2905
+ "person",
2906
+ "bank account number",
2907
+ "email address",
2908
+ "phone number",
2909
+ "organization",
2910
+ "iban"
2911
+ ]);
2912
+ const expandCluster = async (fullText, cluster, existingEntities) => {
2913
+ const { start, end } = cluster;
2914
+ let leftBound = 0;
2915
+ for (const e of existingEntities) if (NON_ADDRESS_LABELS.has(e.label) && e.end <= start && e.end > leftBound) leftBound = e.end;
2916
+ let leftPos = start;
2917
+ while (leftPos > leftBound) {
2918
+ let p = leftPos - 1;
2919
+ while (p >= 0 && (fullText[p] === " " || fullText[p] === ",")) p--;
2920
+ if (p < 0) break;
2921
+ let wordEnd = p + 1;
2922
+ while (p >= 0 && /\S/.test(fullText[p] ?? "")) p--;
2923
+ const word = fullText.slice(p + 1, wordEnd);
2924
+ if (word.length < 2 || !/^\p{Lu}/u.test(word) && !/^\d/.test(word)) break;
2925
+ if (fullText.slice(p + 1, leftPos).includes("\n")) break;
2926
+ leftPos = p + 1;
2927
+ }
2928
+ let rightPos = end;
2929
+ const remaining = fullText.slice(rightPos);
2930
+ let nearestBoundary = Math.min(remaining.length, 200);
2931
+ const boundaryMatch = (await getBoundaryRe()).exec(remaining);
2932
+ if (boundaryMatch && boundaryMatch.index < nearestBoundary) nearestBoundary = boundaryMatch.index;
2933
+ for (const e of existingEntities) {
2934
+ if (!NON_ADDRESS_LABELS.has(e.label)) continue;
2935
+ const offset = e.start - rightPos;
2936
+ if (offset > 0 && offset < nearestBoundary) nearestBoundary = offset;
2937
+ }
2938
+ const doubleNewline = remaining.indexOf("\n\n");
2939
+ if (doubleNewline !== -1 && doubleNewline < nearestBoundary) nearestBoundary = doubleNewline;
2940
+ rightPos = end + remaining.slice(0, nearestBoundary).trimEnd().length;
2941
+ while (rightPos > end && /[,;:\s]/.test(fullText[rightPos - 1] ?? "")) rightPos--;
2942
+ return {
2943
+ start: Math.min(leftPos, start),
2944
+ end: Math.max(rightPos, end)
2945
+ };
2946
+ };
2947
+ /**
2948
+ * Process address seeds from the unified search.
2949
+ * Receives all matches; filters to the street types
2950
+ * slice via sliceStart/sliceEnd. Uses fullText and
2951
+ * existingEntities for seed collection, clustering,
2952
+ * expansion, and scoring.
2953
+ *
2954
+ * Runs as a post-processor after all other detectors,
2955
+ * using their output as seed sources.
2956
+ */
2957
+ const processAddressSeeds = async (allMatches, sliceStart, sliceEnd, fullText, existingEntities) => {
2958
+ const clusters = clusterSeeds(collectSeeds(allMatches, sliceStart, sliceEnd, fullText, existingEntities), 150);
2959
+ const results = [];
2960
+ for (const cluster of clusters) {
2961
+ const score = scoreCluster(cluster);
2962
+ if (score < .6) continue;
2963
+ const { start, end } = await expandCluster(fullText, cluster, existingEntities);
2964
+ const text = fullText.slice(start, end).trim();
2965
+ if (text.length < 5 || text.length > 300) continue;
2966
+ if (text.includes("\n")) continue;
2967
+ results.push({
2968
+ start,
2969
+ end: start + text.length,
2970
+ label: "address",
2971
+ text,
2972
+ score,
2973
+ source: DETECTION_SOURCES.REGEX
2974
+ });
2975
+ }
2976
+ return results;
2977
+ };
2978
+ //#endregion
2979
+ //#region src/detectors/org-propagation.ts
2980
+ const TRAILING_SEP = /[,\s]+$/;
2981
+ const WORD_CHAR_RE = /[\p{L}\p{N}]/u;
2982
+ const ORG_PROPAGATION_SCORE = .9;
2983
+ /**
2984
+ * After the main detection pass, collect organization
2985
+ * entities with a legal form suffix, strip the suffix
2986
+ * to get the base name, and re-scan the full text for
2987
+ * bare mentions of that base name. Returns new entities
2988
+ * for occurrences not already covered.
2989
+ */
2990
+ const propagateOrgNames = (entities, fullText) => {
2991
+ const seeds = [];
2992
+ const seenBases = /* @__PURE__ */ new Set();
2993
+ for (const e of entities) {
2994
+ if (e.label !== "organization") continue;
2995
+ for (const suffix of LEGAL_SUFFIXES) if (e.text.endsWith(suffix)) {
2996
+ const base = e.text.slice(0, -suffix.length).replace(TRAILING_SEP, "").trim();
2997
+ if (base.length >= 3 && !seenBases.has(base)) {
2998
+ seenBases.add(base);
2999
+ seeds.push({
3000
+ baseName: base,
3001
+ label: e.label
3002
+ });
3003
+ }
3004
+ break;
3005
+ }
3006
+ }
3007
+ if (seeds.length === 0) return [];
3008
+ const covered = entities.map((e) => [e.start, e.end]);
3009
+ const isOverlapping = (start, end) => covered.some(([cs, ce]) => start < ce && end > cs);
3010
+ const results = [];
3011
+ for (const seed of seeds) {
3012
+ const { baseName, label } = seed;
3013
+ let searchFrom = 0;
3014
+ while (searchFrom < fullText.length) {
3015
+ const idx = fullText.indexOf(baseName, searchFrom);
3016
+ if (idx === -1) break;
3017
+ const matchEnd = idx + baseName.length;
3018
+ const prevCh = fullText[idx - 1] ?? "";
3019
+ const nextCh = fullText[matchEnd] ?? "";
3020
+ if (WORD_CHAR_RE.test(prevCh) || WORD_CHAR_RE.test(nextCh)) {
3021
+ searchFrom = idx + 1;
3022
+ continue;
3023
+ }
3024
+ if (!isOverlapping(idx, matchEnd)) {
3025
+ results.push({
3026
+ start: idx,
3027
+ end: matchEnd,
3028
+ label,
3029
+ text: baseName,
3030
+ score: ORG_PROPAGATION_SCORE,
3031
+ source: DETECTION_SOURCES.COREFERENCE
3032
+ });
3033
+ covered.push([idx, matchEnd]);
3034
+ }
3035
+ searchFrom = matchEnd;
3036
+ }
3037
+ }
3038
+ return results;
3039
+ };
3040
+ //#endregion
3041
+ //#region src/filters/confidence-boost.ts
3042
+ const BARE_STOPWORDS = new Set([
3043
+ "Příloha",
3044
+ "Smlouva",
3045
+ "Článek",
3046
+ "Dodatek",
3047
+ "Celkem",
3048
+ "Strana",
3049
+ "Faktura",
3050
+ "Částka",
3051
+ "Položka",
3052
+ "Kapitola",
3053
+ "Zákon",
3054
+ "Vyhláška",
3055
+ "Nařízení",
3056
+ "Usnesení",
3057
+ "Rozsudek",
3058
+ "Bod",
3059
+ "Odstavec",
3060
+ "Záloha",
3061
+ "Zbývá",
3062
+ "Dne",
3063
+ "Platba",
3064
+ "Datum",
3065
+ "Splatnost",
3066
+ "Variabilní",
3067
+ "Konstantní",
3068
+ "Specifický"
3069
+ ]);
3070
+ const NEAR_MISS_BAND = .15;
3071
+ const BOOST_PER_NEIGHBOUR = .05;
3072
+ const CONTEXT_WINDOW_CHARS = 150;
3073
+ const HIGH_CONFIDENCE_FLOOR = .9;
3074
+ /**
3075
+ * Boost confidence of near-miss NER entities that appear
3076
+ * near high-confidence detections (regex, trigger phrase).
3077
+ *
3078
+ * If an NER entity scored between (threshold - 0.15) and
3079
+ * threshold, count how many confirmed entities exist within
3080
+ * a 150-char window. Add +0.05 per co-located entity.
3081
+ * If the boosted score crosses the threshold, include it.
3082
+ *
3083
+ * Only mutates score on near-miss entities; high-confidence
3084
+ * entities pass through unchanged.
3085
+ */
3086
+ const boostNearMissEntities = (entities, threshold) => {
3087
+ const nearMissBand = Math.max(0, threshold - NEAR_MISS_BAND);
3088
+ const confirmed = entities.filter((e) => e.score >= HIGH_CONFIDENCE_FLOOR);
3089
+ const boosted = [];
3090
+ for (const entity of entities) {
3091
+ if (entity.score >= threshold) {
3092
+ boosted.push(entity);
3093
+ continue;
3094
+ }
3095
+ if (entity.score < nearMissBand) continue;
3096
+ const midpoint = (entity.start + entity.end) / 2;
3097
+ let neighbourCount = 0;
3098
+ for (const anchor of confirmed) {
3099
+ const anchorMid = (anchor.start + anchor.end) / 2;
3100
+ if (Math.abs(midpoint - anchorMid) <= CONTEXT_WINDOW_CHARS) neighbourCount++;
3101
+ }
3102
+ const boostedScore = entity.score + neighbourCount * BOOST_PER_NEIGHBOUR;
3103
+ if (boostedScore >= threshold) boosted.push({
3104
+ ...entity,
3105
+ score: boostedScore
3106
+ });
3107
+ }
3108
+ return boosted;
3109
+ };
3110
+ const UPPER_WORD_RE = /\p{Lu}/u;
3111
+ /** Header zone: top 15% of document */
3112
+ const HEADER_ZONE_FRACTION = .15;
3113
+ /** Context window for address adjacency */
3114
+ const STREET_CONTEXT_WINDOW = 200;
3115
+ let _addressPreps = null;
3116
+ let _temporalPreps = null;
3117
+ let _prepsPromise = null;
3118
+ const loadPrepositions = async () => {
3119
+ try {
3120
+ const mod = await import("@stll/anonymize-data/config/address-prepositions.json");
3121
+ const data = mod.default ?? mod;
3122
+ const addr = /* @__PURE__ */ new Set();
3123
+ const temp = /* @__PURE__ */ new Set();
3124
+ for (const words of Object.values(data.address)) if (Array.isArray(words)) for (const w of words) addr.add(w.toLowerCase());
3125
+ for (const words of Object.values(data.temporal)) if (Array.isArray(words)) for (const w of words) temp.add(w.toLowerCase());
3126
+ _addressPreps = addr;
3127
+ _temporalPreps = temp;
3128
+ } catch {
3129
+ _addressPreps = /* @__PURE__ */ new Set();
3130
+ _temporalPreps = /* @__PURE__ */ new Set();
3131
+ }
3132
+ };
3133
+ /** Ensure preposition data is loaded. */
3134
+ const initPrepositions = () => {
3135
+ if (!_prepsPromise) _prepsPromise = loadPrepositions();
3136
+ return _prepsPromise;
3137
+ };
3138
+ const getAddressPreps = () => _addressPreps ?? /* @__PURE__ */ new Set();
3139
+ const getTemporalPreps = () => _temporalPreps ?? /* @__PURE__ */ new Set();
3140
+ let _streetAbbrevs = null;
3141
+ let _streetAbbrevsPromise = null;
3142
+ const loadStreetAbbrevs = async () => {
3143
+ try {
3144
+ const mod = await import("@stll/anonymize-data/config/address-street-types.json");
3145
+ const data = mod.default ?? mod;
3146
+ const abbrevs = /* @__PURE__ */ new Set();
3147
+ for (const [key, words] of Object.entries(data)) {
3148
+ if (key.startsWith("_")) continue;
3149
+ if (!Array.isArray(words)) continue;
3150
+ for (const w of words) if (w.includes(".")) abbrevs.add(w.toLowerCase());
3151
+ }
3152
+ _streetAbbrevs = abbrevs;
3153
+ } catch {
3154
+ _streetAbbrevs = /* @__PURE__ */ new Set();
3155
+ }
3156
+ };
3157
+ /** Ensure street abbreviation data is loaded. */
3158
+ const initStreetAbbrevs = () => {
3159
+ if (!_streetAbbrevsPromise) _streetAbbrevsPromise = loadStreetAbbrevs();
3160
+ return _streetAbbrevsPromise;
3161
+ };
3162
+ const getStreetAbbrevs = () => _streetAbbrevs ?? /* @__PURE__ */ new Set();
3163
+ /**
3164
+ * Scan backwards from known address entities and
3165
+ * house number patterns to find street names.
3166
+ *
3167
+ * Strategy (a): from a house number like "2512/2a",
3168
+ * walk left to find the first uppercase word — that's
3169
+ * the street name start. "Mezi úvozy 2512/2a" →
3170
+ * captures "Mezi úvozy 2512/2a".
3171
+ *
3172
+ * Strategy (b): if a colon ":" appears within 3 chars
3173
+ * before the street start, boost confidence. Colons
3174
+ * signal "label: value" pairs universal in contracts.
3175
+ *
3176
+ * Strategy (c): in the header zone (top 15% of doc),
3177
+ * be more aggressive — detect street patterns even
3178
+ * without nearby address entities.
3179
+ */
3180
+ const detectStreetPatternsNearAddresses = (fullText, existingEntities) => {
3181
+ const results = [];
3182
+ const addressEntities = existingEntities.filter((e) => e.label === "address");
3183
+ const headerEnd = Math.floor(fullText.length * HEADER_ZONE_FRACTION);
3184
+ const houseNumRe = /\b(?:\d{1,4}\/\d+[a-zA-Z]\b|\d{3,4}\/\d+\b|(?:1[3-9]|[2-9]\d)\/\d{3,}\b)/g;
3185
+ houseNumRe.lastIndex = 0;
3186
+ for (let m = houseNumRe.exec(fullText); m !== null; m = houseNumRe.exec(fullText)) {
3187
+ const numStart = m.index;
3188
+ const numEnd = numStart + m[0].length;
3189
+ if (existingEntities.some((e) => e.start <= numStart && e.end >= numEnd)) continue;
3190
+ const inHeader = numStart < headerEnd;
3191
+ const nearAddress = addressEntities.some((e) => Math.abs(e.start - numEnd) < STREET_CONTEXT_WINDOW || Math.abs(e.end - numStart) < STREET_CONTEXT_WINDOW);
3192
+ if (!inHeader && !nearAddress) continue;
3193
+ let scanPos = numStart - 1;
3194
+ while (scanPos >= 0 && /[\s\u00A0]/.test(fullText[scanPos] ?? "")) scanPos--;
3195
+ if (scanPos < 0) continue;
3196
+ let hasTemporalPrep = false;
3197
+ let streetStart = scanPos + 1;
3198
+ let wordCount = 0;
3199
+ const MAX_WORDS = 5;
3200
+ while (scanPos >= 0 && wordCount < MAX_WORDS) {
3201
+ let wordEnd = scanPos + 1;
3202
+ const hasDot = fullText[scanPos] === ".";
3203
+ if (hasDot) scanPos--;
3204
+ while (scanPos >= 0 && /[\p{L}\p{M}\d]/u.test(fullText[scanPos] ?? "")) scanPos--;
3205
+ const wordStart = scanPos + 1;
3206
+ const rawWord = fullText.slice(wordStart, wordEnd);
3207
+ const word = hasDot ? rawWord.slice(0, -1) : rawWord;
3208
+ if (word.length === 0) break;
3209
+ const isStreetAbbrev = hasDot && getStreetAbbrevs().has(rawWord.toLowerCase());
3210
+ const isUpper = UPPER_WORD_RE.test(word[0] ?? "");
3211
+ const isPrep = getAddressPreps().has(word.toLowerCase());
3212
+ const isDigitToken = /^\d{1,2}$/.test(word);
3213
+ if (!isUpper && !isPrep && !isStreetAbbrev && !isDigitToken) break;
3214
+ if (isPrep && getTemporalPreps().has(word.toLowerCase())) hasTemporalPrep = true;
3215
+ streetStart = wordStart;
3216
+ wordCount++;
3217
+ while (scanPos >= 0 && /[\s\u00A0]/.test(fullText[scanPos] ?? "")) scanPos--;
3218
+ const prevCh = fullText[scanPos];
3219
+ if (prevCh === "\n" || prevCh === " " || prevCh === ";" || prevCh === void 0) break;
3220
+ if (prevCh === ",") break;
3221
+ }
3222
+ if (wordCount === 0) continue;
3223
+ const streetText = fullText.slice(streetStart, numEnd);
3224
+ if (streetText.length < 4) continue;
3225
+ if (hasTemporalPrep) continue;
3226
+ if (existingEntities.some((e) => e.start <= streetStart && e.end >= numEnd)) continue;
3227
+ const score = fullText.slice(Math.max(0, streetStart - 5), streetStart).includes(":") ? .95 : inHeader ? .85 : .8;
3228
+ results.push({
3229
+ start: streetStart,
3230
+ end: numEnd,
3231
+ label: "address",
3232
+ text: streetText,
3233
+ score,
3234
+ source: "regex"
3235
+ });
3236
+ }
3237
+ const bareHouseRe = /(?<=\s|^)(\p{Lu}\p{Ll}[\p{Ll}\p{Lu}]+\s+\d{1,3})\b/gu;
3238
+ bareHouseRe.lastIndex = 0;
3239
+ const allAddr = [...addressEntities, ...results];
3240
+ for (let m = bareHouseRe.exec(fullText); m !== null; m = bareHouseRe.exec(fullText)) {
3241
+ const captured = m[1];
3242
+ if (captured === void 0) continue;
3243
+ const start = m.index;
3244
+ const end = start + captured.length;
3245
+ if (!allAddr.some((e) => {
3246
+ if (Math.min(Math.abs(e.start - end), Math.abs(e.end - start)) > 50) return false;
3247
+ const lo = Math.min(e.start, start);
3248
+ const hi = Math.max(e.end, end);
3249
+ return !fullText.slice(lo, hi).includes("\n");
3250
+ })) continue;
3251
+ const spaceIdx = captured.search(/\s+\d/);
3252
+ const word = spaceIdx > 0 ? captured.slice(0, spaceIdx) : captured;
3253
+ if (BARE_STOPWORDS.has(word)) continue;
3254
+ if ([...existingEntities, ...results].some((e) => e.start < end && e.end > start)) continue;
3255
+ results.push({
3256
+ start,
3257
+ end,
3258
+ label: "address",
3259
+ text: captured,
3260
+ score: .75,
3261
+ source: "regex"
3262
+ });
3263
+ }
3264
+ return results;
3265
+ };
3266
+ const ORPHAN_STREET_RE = /^\s*(\p{Lu}[\p{Ll}\p{Lu}]+(?:\s+[\p{Lu}\p{Ll}][\p{Ll}]+)*\s+\d{2,4}[a-zA-Z]?)\s*$/gmu;
3267
+ /**
3268
+ * In the header zone (top 15%), find standalone lines
3269
+ * matching "[Uppercase word(s)] [number]" that sit
3270
+ * between other detected entities. These are almost
3271
+ * certainly street addresses in party definitions.
3272
+ *
3273
+ * Example: "Evropská 710" on its own line between
3274
+ * an organization entity and a postal code entity.
3275
+ */
3276
+ const detectOrphanStreetLines = (fullText, existingEntities) => {
3277
+ const headerEnd = Math.floor(fullText.length * HEADER_ZONE_FRACTION);
3278
+ const results = [];
3279
+ ORPHAN_STREET_RE.lastIndex = 0;
3280
+ for (let m = ORPHAN_STREET_RE.exec(fullText); m !== null; m = ORPHAN_STREET_RE.exec(fullText)) {
3281
+ const captured = m[1];
3282
+ if (captured === void 0) continue;
3283
+ const start = m.index + m[0].indexOf(captured);
3284
+ const end = start + captured.length;
3285
+ if (start >= headerEnd) continue;
3286
+ if (existingEntities.some((e) => e.start <= start && e.end >= end)) continue;
3287
+ if (!existingEntities.some((e) => Math.abs(e.start - end) < 200 || Math.abs(e.end - start) < 200)) continue;
3288
+ results.push({
3289
+ start,
3290
+ end,
3291
+ label: "address",
3292
+ text: captured,
3293
+ score: .85,
3294
+ source: "regex"
3295
+ });
3296
+ }
3297
+ return results;
3298
+ };
3299
+ //#endregion
3300
+ //#region src/filters/zone-classifier.ts
3301
+ /**
3302
+ * Additive score adjustments per document zone.
3303
+ * Header and signature blocks are dense with PII;
3304
+ * tables often contain structured identifying data.
3305
+ */
3306
+ const ZONE_SCORE_ADJUSTMENTS = {
3307
+ header: .1,
3308
+ signature: .15,
3309
+ body: 0,
3310
+ table: .05
3311
+ };
3312
+ const loadSectionHeadings = async () => {
3313
+ const mod = await import("@stll/anonymize-data/config/section-headings.json");
3314
+ return (mod.default ?? mod).patterns.map((p) => new RegExp(p.re, p.flags));
3315
+ };
3316
+ const loadSigningClauses = async () => {
3317
+ const mod = await import("@stll/anonymize-data/config/signing-clauses.json");
3318
+ return (mod.default ?? mod).patterns.map((p) => {
3319
+ const prefix = p.prefix || "";
3320
+ const suffix = p.suffix || "";
3321
+ const prepAlt = p.prepositions.length > 0 ? p.prepositions.join("|") : null;
3322
+ const combined = `^\\s*(?:${prefix}${prepAlt ? `\\p{Lu}\\p{Ll}+(?:\\s+(?:${prepAlt})\\s+\\p{Lu}\\p{Ll}+)*(?:\\s+\\p{Lu}\\p{Ll}+)*` : "\\p{Lu}\\p{Ll}+(?:[- ]\\p{Lu}\\p{Ll}+)*"}${suffix})`;
3323
+ return new RegExp(combined, "u");
3324
+ });
3325
+ };
3326
+ /**
3327
+ * Ensure config data is loaded. Call once before
3328
+ * classifyZones. Safe to call multiple times.
3329
+ */
3330
+ const initZoneClassifier = (ctx = defaultContext) => {
3331
+ if (ctx.zoneInitPromise) return ctx.zoneInitPromise;
3332
+ ctx.zoneInitPromise = Promise.all([loadSectionHeadings(), loadSigningClauses()]).then(([headings, clauses]) => {
3333
+ ctx.zoneHeadingPatterns = headings;
3334
+ ctx.zoneSigningPatterns = clauses;
3335
+ }).catch((err) => {
3336
+ ctx.zoneInitPromise = null;
3337
+ throw err;
3338
+ });
3339
+ return ctx.zoneInitPromise;
3340
+ };
3341
+ const MIN_TABS_FOR_TABLE = 2;
3342
+ const isTableLine = (line) => {
3343
+ let tabCount = 0;
3344
+ for (const ch of line) {
3345
+ if (ch === " ") tabCount++;
3346
+ if (tabCount >= MIN_TABS_FOR_TABLE) return true;
3347
+ }
3348
+ return false;
3349
+ };
3350
+ /**
3351
+ * Classify a document into zones based on
3352
+ * structural heuristics. Zones are non-overlapping
3353
+ * and cover the entire text.
3354
+ *
3355
+ * Must call `initZoneClassifier()` first.
3356
+ */
3357
+ const classifyZones = (fullText, ctx = defaultContext) => {
3358
+ if (fullText.length === 0) return [];
3359
+ const headingRes = ctx.zoneHeadingPatterns;
3360
+ const signingRes = ctx.zoneSigningPatterns;
3361
+ if (!headingRes || !signingRes) {
3362
+ console.warn("[anonymize] classifyZones called before initZoneClassifier(); returning body-only");
3363
+ return [{
3364
+ zone: "body",
3365
+ start: 0,
3366
+ end: fullText.length
3367
+ }];
3368
+ }
3369
+ const lines = fullText.split("\n");
3370
+ const zones = [];
3371
+ let headerEndLine = -1;
3372
+ for (let i = 0; i < lines.length; i++) {
3373
+ const line = lines[i];
3374
+ if (line === void 0) continue;
3375
+ for (const re of headingRes) if (re.test(line)) {
3376
+ headerEndLine = i;
3377
+ break;
3378
+ }
3379
+ if (headerEndLine !== -1) break;
3380
+ }
3381
+ let signatureStartLine = -1;
3382
+ for (let i = lines.length - 1; i >= 0; i--) {
3383
+ const line = lines[i];
3384
+ if (line === void 0) continue;
3385
+ for (const re of signingRes) if (re.test(line)) {
3386
+ signatureStartLine = i;
3387
+ break;
3388
+ }
3389
+ if (signatureStartLine !== -1) break;
3390
+ }
3391
+ const lineOffsets = [];
3392
+ let offset = 0;
3393
+ for (const line of lines) {
3394
+ lineOffsets.push(offset);
3395
+ offset += line.length + 1;
3396
+ }
3397
+ let headerEndOffset = headerEndLine >= 0 ? lineOffsets[headerEndLine] ?? 0 : 0;
3398
+ const signatureStartOffset = signatureStartLine >= 0 ? lineOffsets[signatureStartLine] ?? fullText.length : fullText.length;
3399
+ if (headerEndLine > 0 && signatureStartLine >= 0 && headerEndOffset > signatureStartOffset) {
3400
+ headerEndLine = -1;
3401
+ headerEndOffset = 0;
3402
+ }
3403
+ if (headerEndLine > 0) zones.push({
3404
+ zone: "header",
3405
+ start: 0,
3406
+ end: headerEndOffset
3407
+ });
3408
+ const bodyStart = headerEndLine > 0 ? headerEndOffset : 0;
3409
+ const bodyEnd = signatureStartLine >= 0 ? signatureStartOffset : fullText.length;
3410
+ let tableStart = -1;
3411
+ for (let i = Math.max(headerEndLine, 0); i < (signatureStartLine >= 0 ? signatureStartLine : lines.length); i++) {
3412
+ const line = lines[i];
3413
+ if (line === void 0) continue;
3414
+ const lineStart = lineOffsets[i] ?? 0;
3415
+ const lineEnd = lineStart + line.length;
3416
+ if (isTableLine(line)) {
3417
+ if (tableStart === -1) tableStart = lineStart;
3418
+ } else if (tableStart !== -1) {
3419
+ zones.push({
3420
+ zone: "table",
3421
+ start: tableStart,
3422
+ end: lineStart
3423
+ });
3424
+ tableStart = -1;
3425
+ }
3426
+ if (i === (signatureStartLine >= 0 ? signatureStartLine - 1 : lines.length - 1) && tableStart !== -1) {
3427
+ zones.push({
3428
+ zone: "table",
3429
+ start: tableStart,
3430
+ end: Math.min(lineEnd + 1, bodyEnd)
3431
+ });
3432
+ tableStart = -1;
3433
+ }
3434
+ }
3435
+ const sortedSpecial = zones.toSorted((a, b) => a.start - b.start);
3436
+ let cursor = bodyStart;
3437
+ const bodyZones = [];
3438
+ for (const span of sortedSpecial) {
3439
+ if (span.zone === "header") continue;
3440
+ if (span.start > cursor) bodyZones.push({
3441
+ zone: "body",
3442
+ start: cursor,
3443
+ end: span.start
3444
+ });
3445
+ cursor = Math.max(cursor, span.end);
3446
+ }
3447
+ if (cursor < bodyEnd) bodyZones.push({
3448
+ zone: "body",
3449
+ start: cursor,
3450
+ end: bodyEnd
3451
+ });
3452
+ for (const z of bodyZones) zones.push(z);
3453
+ if (signatureStartLine >= 0) zones.push({
3454
+ zone: "signature",
3455
+ start: signatureStartOffset,
3456
+ end: fullText.length
3457
+ });
3458
+ return zones.toSorted((a, b) => a.start - b.start);
3459
+ };
3460
+ /**
3461
+ * Find which zone an entity's midpoint falls in.
3462
+ * Returns "body" if no zone matches (defensive).
3463
+ */
3464
+ const findZone = (midpoint, zones) => {
3465
+ for (const span of zones) if (midpoint >= span.start && midpoint < span.end) return span.zone;
3466
+ return "body";
3467
+ };
3468
+ /**
3469
+ * Apply zone-based score adjustments to entities.
3470
+ * Entities in header/signature/table zones get a
3471
+ * small additive boost reflecting the higher PII
3472
+ * density in those regions.
3473
+ *
3474
+ * Returns a new array; does not mutate inputs.
3475
+ */
3476
+ const applyZoneAdjustments = (entities, zones) => {
3477
+ if (zones.length === 0) return entities.map((e) => ({ ...e }));
3478
+ const result = [];
3479
+ for (const entity of entities) {
3480
+ const adjustment = ZONE_SCORE_ADJUSTMENTS[findZone((entity.start + entity.end) / 2, zones)];
3481
+ if (adjustment > 0) result.push({
3482
+ ...entity,
3483
+ score: Math.min(1, entity.score + adjustment)
3484
+ });
3485
+ else result.push({ ...entity });
3486
+ }
3487
+ return result;
3488
+ };
3489
+ //#endregion
3490
+ //#region src/filters/hotword-rules.ts
3491
+ let rules = null;
3492
+ let search = null;
3493
+ /**
3494
+ * Maps each TextSearch pattern index back to the
3495
+ * rule index that owns it, so a single AC scan
3496
+ * resolves all hotword hits to their rule.
3497
+ */
3498
+ let patternToRule = null;
3499
+ let initPromise = null;
3500
+ const loadRules = async () => {
3501
+ const mod = await import("@stll/anonymize-data/config/hotword-rules.json");
3502
+ const loaded = (mod.default ?? mod).rules;
3503
+ const patterns = [];
3504
+ const mapping = [];
3505
+ for (let ruleIdx = 0; ruleIdx < loaded.length; ruleIdx++) {
3506
+ const rule = loaded[ruleIdx];
3507
+ if (!rule) continue;
3508
+ for (const hw of rule.hotwords) {
3509
+ patterns.push({
3510
+ pattern: hw,
3511
+ literal: true,
3512
+ caseInsensitive: true
3513
+ });
3514
+ mapping.push(ruleIdx);
3515
+ }
3516
+ }
3517
+ const builtSearch = patterns.length > 0 ? new TextSearch(patterns, {
3518
+ overlapStrategy: "all",
3519
+ caseInsensitive: true,
3520
+ wholeWords: true
3521
+ }) : null;
3522
+ patternToRule = mapping;
3523
+ search = builtSearch;
3524
+ rules = loaded;
3525
+ };
3526
+ /**
3527
+ * Load hotword rules from the data package.
3528
+ * Safe to call multiple times; subsequent calls
3529
+ * are no-ops.
3530
+ */
3531
+ const initHotwordRules = async () => {
3532
+ if (rules !== null) return;
3533
+ if (initPromise !== null) return initPromise;
3534
+ initPromise = loadRules().catch((err) => {
3535
+ initPromise = null;
3536
+ throw err;
3537
+ });
3538
+ return initPromise;
3539
+ };
3540
+ /**
3541
+ * Apply hotword context rules to detected entities.
3542
+ *
3543
+ * Scans `fullText` once with a single AC automaton
3544
+ * for all hotwords across all rules, then checks
3545
+ * proximity to each entity. Distance-decayed
3546
+ * adjustment: closer hotwords give a stronger boost.
3547
+ *
3548
+ * Returns a new array; input entities are not mutated.
3549
+ */
3550
+ const applyHotwordRules = (entities, fullText) => {
3551
+ if (rules === null || rules.length === 0 || search === null || patternToRule === null) return entities;
3552
+ const hits = search.findIter(fullText);
3553
+ if (hits.length === 0) return entities;
3554
+ const hitsByRule = /* @__PURE__ */ new Map();
3555
+ for (const hit of hits) {
3556
+ const ruleIdx = patternToRule[hit.pattern];
3557
+ if (ruleIdx === void 0) continue;
3558
+ let bucket = hitsByRule.get(ruleIdx);
3559
+ if (bucket === void 0) {
3560
+ bucket = [];
3561
+ hitsByRule.set(ruleIdx, bucket);
3562
+ }
3563
+ bucket.push(hit);
3564
+ }
3565
+ const result = [];
3566
+ for (const entity of entities) {
3567
+ let bestAdjustment = 0;
3568
+ let bestReclassify;
3569
+ for (let ruleIdx = 0; ruleIdx < rules.length; ruleIdx++) {
3570
+ const rule = rules[ruleIdx];
3571
+ if (!rule) continue;
3572
+ if (!rule.targetLabels.includes(entity.label)) continue;
3573
+ const ruleHits = hitsByRule.get(ruleIdx);
3574
+ if (ruleHits === void 0) continue;
3575
+ for (const hit of ruleHits) {
3576
+ let distance;
3577
+ let maxDistance;
3578
+ if (hit.end <= entity.start) {
3579
+ distance = entity.start - hit.end;
3580
+ maxDistance = rule.proximityBefore;
3581
+ } else if (hit.start >= entity.end) {
3582
+ distance = hit.start - entity.end;
3583
+ maxDistance = rule.proximityAfter;
3584
+ } else {
3585
+ distance = 0;
3586
+ maxDistance = Math.max(rule.proximityBefore, rule.proximityAfter);
3587
+ }
3588
+ if (distance > maxDistance) continue;
3589
+ const decay = maxDistance === 0 ? 1 : 1 - distance / maxDistance;
3590
+ const adj = rule.scoreAdjustment * decay;
3591
+ if (Math.abs(adj) > Math.abs(bestAdjustment)) {
3592
+ bestAdjustment = adj;
3593
+ if (adj > 0) bestReclassify = rule.reclassifyTo;
3594
+ else bestReclassify = void 0;
3595
+ }
3596
+ }
3597
+ }
3598
+ if (bestAdjustment === 0) {
3599
+ result.push(entity);
3600
+ continue;
3601
+ }
3602
+ const newScore = Math.min(1, Math.max(0, entity.score + bestAdjustment));
3603
+ const newLabel = bestReclassify !== void 0 ? bestReclassify : entity.label;
3604
+ result.push({
3605
+ ...entity,
3606
+ score: newScore,
3607
+ label: newLabel
3608
+ });
3609
+ }
3610
+ return result;
3611
+ };
3612
+ //#endregion
3613
+ //#region src/filters/boundary-consistency.ts
3614
+ /** Max gap (in chars) between entities to merge. */
3615
+ const MAX_GAP = 3;
3616
+ /**
3617
+ * Characters allowed in the gap between two adjacent
3618
+ * same-label entities that should be merged: spaces,
3619
+ * tabs, commas, and hyphens. Uses `[ \t,\-]` instead
3620
+ * of `\s` to avoid merging entities across newlines.
3621
+ */
3622
+ const GAP_PATTERN = /^[ \t,\-]+$/;
3623
+ /**
3624
+ * Build a set of word boundary offsets for the full
3625
+ * text using `Intl.Segmenter`. Returns a sorted array
3626
+ * of offsets where words start and end.
3627
+ */
3628
+ const buildWordBoundaries = (text) => {
3629
+ const segmenter = new Intl.Segmenter("und", { granularity: "word" });
3630
+ const boundaries = /* @__PURE__ */ new Set();
3631
+ for (const seg of segmenter.segment(text)) {
3632
+ if (!seg.isWordLike) continue;
3633
+ boundaries.add(seg.index);
3634
+ boundaries.add(seg.index + seg.segment.length);
3635
+ }
3636
+ return boundaries;
3637
+ };
3638
+ /**
3639
+ * Characters that act as hard stops when scanning
3640
+ * backward for a word boundary. Entity boundaries
3641
+ * should never extend past these.
3642
+ */
3643
+ const WORD_START_STOPS = new Set([
3644
+ "\n",
3645
+ "\r",
3646
+ ",",
3647
+ ";",
3648
+ "(",
3649
+ ")",
3650
+ "[",
3651
+ "]"
3652
+ ]);
3653
+ /**
3654
+ * Find the word-start offset at or before `pos`.
3655
+ * Scans left until a word boundary is found.
3656
+ */
3657
+ const wordStartAt = (pos, boundaries, text) => {
3658
+ let p = pos;
3659
+ while (p > 0 && !boundaries.has(p)) {
3660
+ const prev = text[p - 1];
3661
+ if (prev !== void 0 && WORD_START_STOPS.has(prev)) return p;
3662
+ p--;
3663
+ }
3664
+ return p;
3665
+ };
3666
+ /**
3667
+ * Characters that act as hard stops when scanning
3668
+ * forward for a word boundary. Entity boundaries
3669
+ * should never extend past these.
3670
+ */
3671
+ const WORD_END_STOPS = new Set([
3672
+ "\n",
3673
+ "\r",
3674
+ ",",
3675
+ ";",
3676
+ ".",
3677
+ "(",
3678
+ ")",
3679
+ "[",
3680
+ "]"
3681
+ ]);
3682
+ /**
3683
+ * Find the word-end offset at or after `pos`.
3684
+ * Scans right until a word boundary is found.
3685
+ */
3686
+ const wordEndAt = (pos, boundaries, text) => {
3687
+ let p = pos;
3688
+ while (p < text.length && !boundaries.has(p)) {
3689
+ const ch = text[p];
3690
+ if (ch !== void 0 && WORD_END_STOPS.has(ch)) return p;
3691
+ p++;
3692
+ }
3693
+ return p;
3694
+ };
3695
+ /**
3696
+ * Binary search: find the leftmost index in `arr`
3697
+ * where `arr[index].start >= value`.
3698
+ */
3699
+ const lowerBound = (arr, value) => {
3700
+ let lo = 0;
3701
+ let hi = arr.length;
3702
+ while (lo < hi) {
3703
+ const mid = lo + hi >>> 1;
3704
+ const el = arr[mid];
3705
+ if (el && el.start < value) lo = mid + 1;
3706
+ else hi = mid;
3707
+ }
3708
+ return lo;
3709
+ };
3710
+ /**
3711
+ * Merge adjacent same-label entities separated only by
3712
+ * whitespace, comma, or hyphen (max 3 chars). Also
3713
+ * merges same-label entities that partially overlap
3714
+ * (which can happen after word-boundary expansion).
3715
+ *
3716
+ * Looks for the last same-label entity in the result
3717
+ * (not just the very last entity) so that an
3718
+ * intervening different-label entity does not prevent
3719
+ * merging.
3720
+ *
3721
+ * Uses binary search for the `gapOccupied` check and
3722
+ * a Map for O(1) same-label prev lookup. O(n log n).
3723
+ */
3724
+ const mergeAdjacent = (entities, fullText) => {
3725
+ const sorted = entities.toSorted((a, b) => a.start - b.start);
3726
+ const result = [];
3727
+ const lastByLabel = /* @__PURE__ */ new Map();
3728
+ for (const entity of sorted) {
3729
+ const prev = lastByLabel.get(entity.label);
3730
+ if (!prev) {
3731
+ const copy = { ...entity };
3732
+ result.push(copy);
3733
+ lastByLabel.set(entity.label, copy);
3734
+ continue;
3735
+ }
3736
+ if (entity.start < prev.end) {
3737
+ prev.end = Math.max(prev.end, entity.end);
3738
+ prev.text = fullText.slice(prev.start, prev.end);
3739
+ prev.score = Math.max(prev.score, entity.score);
3740
+ continue;
3741
+ }
3742
+ const gap = fullText.slice(prev.end, entity.start);
3743
+ const gapStart = prev.end;
3744
+ const gapEnd = entity.start;
3745
+ const searchIdx = lowerBound(sorted, gapStart);
3746
+ let gapOccupied = false;
3747
+ for (let k = searchIdx; k < sorted.length; k++) {
3748
+ const other = sorted[k];
3749
+ if (!other || other.start >= gapEnd) break;
3750
+ if (other.label !== entity.label && other.end > gapStart) {
3751
+ gapOccupied = true;
3752
+ break;
3753
+ }
3754
+ }
3755
+ if (!gapOccupied && gap.length <= MAX_GAP && GAP_PATTERN.test(gap)) {
3756
+ prev.end = entity.end;
3757
+ prev.text = fullText.slice(prev.start, prev.end);
3758
+ prev.score = Math.max(prev.score, entity.score);
3759
+ } else {
3760
+ const copy = { ...entity };
3761
+ result.push(copy);
3762
+ lastByLabel.set(entity.label, copy);
3763
+ }
3764
+ }
3765
+ return result;
3766
+ };
3767
+ /**
3768
+ * Fix partial-word boundaries by extending entity
3769
+ * start/end to the nearest word boundary. Does not
3770
+ * extend across newlines or into spans occupied by
3771
+ * different-label entities.
3772
+ *
3773
+ * Uses binary search to skip irrelevant entries when
3774
+ * clamping at cross-label neighbors. O(n log n) in
3775
+ * the common case; O(n^2) worst case when many
3776
+ * same-label entities precede a cross-label boundary.
3777
+ */
3778
+ const fixPartialWords = (entities, fullText) => {
3779
+ const boundaries = buildWordBoundaries(fullText);
3780
+ const sorted = entities.toSorted((a, b) => a.start - b.start);
3781
+ const byEnd = sorted.map((e, idx) => ({
3782
+ entity: e,
3783
+ idx
3784
+ })).sort((a, b) => a.entity.end - b.entity.end);
3785
+ const endPositions = byEnd.map((x) => x.entity.end);
3786
+ return sorted.map((e, eIdx) => {
3787
+ let newStart = wordStartAt(e.start, boundaries, fullText);
3788
+ let newEnd = wordEndAt(e.end, boundaries, fullText);
3789
+ let lo = 0;
3790
+ let hi = endPositions.length;
3791
+ while (lo < hi) {
3792
+ const mid = lo + hi >>> 1;
3793
+ if ((endPositions[mid] ?? Number.POSITIVE_INFINITY) <= newStart) lo = mid + 1;
3794
+ else hi = mid;
3795
+ }
3796
+ for (let k = lo; k < byEnd.length; k++) {
3797
+ const entry = byEnd[k];
3798
+ if (!entry || entry.entity.end > e.start) break;
3799
+ if (entry.idx === eIdx) continue;
3800
+ if (entry.entity.label === e.label) continue;
3801
+ newStart = Math.max(newStart, entry.entity.end);
3802
+ }
3803
+ const startIdx = lowerBound(sorted, e.end);
3804
+ for (let k = startIdx; k < sorted.length; k++) {
3805
+ const other = sorted[k];
3806
+ if (!other || other.start >= newEnd) break;
3807
+ if (other === e) continue;
3808
+ if (other.label === e.label) continue;
3809
+ newEnd = Math.min(newEnd, other.start);
3810
+ }
3811
+ if (newStart === e.start && newEnd === e.end) return e;
3812
+ return {
3813
+ ...e,
3814
+ start: newStart,
3815
+ end: newEnd,
3816
+ text: fullText.slice(newStart, newEnd)
3817
+ };
3818
+ });
3819
+ };
3820
+ /**
3821
+ * Deduplicate entities with identical [start, end, label].
3822
+ * Keeps the entry with the highest score.
3823
+ */
3824
+ const deduplicateSpans = (entities) => {
3825
+ const seen = /* @__PURE__ */ new Map();
3826
+ for (const entity of entities) {
3827
+ const key = `${entity.start}:${entity.end}:${entity.label}`;
3828
+ const existing = seen.get(key);
3829
+ if (!existing || entity.score > existing.score) seen.set(key, entity);
3830
+ }
3831
+ return [...seen.values()];
3832
+ };
3833
+ /**
3834
+ * Remove nested same-label entities. If a shorter
3835
+ * entity is fully contained within a longer entity
3836
+ * of the same label, drop the shorter one.
3837
+ *
3838
+ * Uses a "max end seen" sweep per label. O(n) after
3839
+ * the initial sort.
3840
+ */
3841
+ const removeNestedSameLabel = (entities) => {
3842
+ const sorted = entities.toSorted((a, b) => {
3843
+ if (a.start !== b.start) return a.start - b.start;
3844
+ return b.end - a.end;
3845
+ });
3846
+ const result = [];
3847
+ const maxEndByLabel = /* @__PURE__ */ new Map();
3848
+ for (const entity of sorted) {
3849
+ const maxEnd = maxEndByLabel.get(entity.label);
3850
+ if (maxEnd !== void 0 && entity.end <= maxEnd) continue;
3851
+ maxEndByLabel.set(entity.label, entity.end);
3852
+ result.push(entity);
3853
+ }
3854
+ return result;
3855
+ };
3856
+ /**
3857
+ * Resolve cross-label overlaps that can arise when
3858
+ * `fixPartialWords` independently expands two
3859
+ * different-label entities toward the same word
3860
+ * boundary. The entity with the higher score (or
3861
+ * longer span on tie) keeps its boundary; the other
3862
+ * is trimmed so the overlap disappears.
3863
+ *
3864
+ * Preserved existing structure: sorted + early break
3865
+ * already gives good amortized behavior. O(n^2)
3866
+ * worst case but rare in practice.
3867
+ */
3868
+ const resolveCrossLabelOverlaps = (entities, fullText) => {
3869
+ const sorted = entities.map((e) => ({ ...e })).sort((a, b) => a.start - b.start);
3870
+ for (let i = 0; i < sorted.length; i++) for (let j = i + 1; j < sorted.length; j++) {
3871
+ const a = sorted[i];
3872
+ const b = sorted[j];
3873
+ if (!a || !b) continue;
3874
+ if (b.start >= a.end) break;
3875
+ if (a.label === b.label) continue;
3876
+ const aContainsB = a.start <= b.start && a.end >= b.end;
3877
+ const bContainsA = b.start <= a.start && b.end >= a.end;
3878
+ if (aContainsB || bContainsA) continue;
3879
+ const aLen = a.end - a.start;
3880
+ const bLen = b.end - b.start;
3881
+ if (a.score > b.score || a.score === b.score && aLen >= bLen) {
3882
+ b.start = a.end;
3883
+ b.text = fullText.slice(b.start, b.end);
3884
+ } else {
3885
+ a.end = b.start;
3886
+ a.text = fullText.slice(a.start, a.end);
3887
+ }
3888
+ }
3889
+ return sorted.filter((e) => e.start < e.end);
3890
+ };
3891
+ /**
3892
+ * Post-processing pass for entity boundary consistency.
3893
+ * Runs after mergeAndDedup, before false-positive
3894
+ * filtering.
3895
+ *
3896
+ * 1. Fix partial-word boundaries (respects cross-label
3897
+ * neighbors to avoid introducing new overlaps)
3898
+ * 2. Resolve any remaining cross-label overlaps
3899
+ * 3. Deduplicate identical [start, end, label] spans
3900
+ * 4. Merge adjacent same-label entities (catches any
3901
+ * new adjacency/overlap from step 1)
3902
+ * 5. Remove nested same-label entities
3903
+ */
3904
+ const enforceBoundaryConsistency = (entities, fullText) => {
3905
+ return removeNestedSameLabel(mergeAdjacent(deduplicateSpans(resolveCrossLabelOverlaps(fixPartialWords(entities, fullText), fullText)), fullText));
3906
+ };
3907
+ //#endregion
3908
+ //#region src/build-unified-search.ts
3909
+ /**
3910
+ * Build the unified search instances from all
3911
+ * detector pattern sources.
3912
+ *
3913
+ * Two TextSearch instances (not one) to avoid
3914
+ * 200K per-pattern object allocations:
3915
+ * 1. regex + triggers + legal-forms (mixed, ~140
3916
+ * patterns, caseInsensitive for trigger AC)
3917
+ * 2. deny-list + street-types + gazetteer
3918
+ * (caseInsensitive, overlap "all";
3919
+ * deny-list/street-type use per-pattern
3920
+ * wholeWords: true; gazetteer exact use
3921
+ * wholeWords: false; gazetteer fuzzy use
3922
+ * distance: 2 via @stll/fuzzy-search)
3923
+ *
3924
+ * All patterns are PatternEntry objects with
3925
+ * per-pattern literal/wholeWords settings.
3926
+ */
3927
+ const buildUnifiedSearch = async (config, gazetteerEntries = [], ctx = defaultContext) => {
3928
+ const [legalForms, triggers, denyListData, streetTypes, currencyPatterns, datePatterns, signingPatterns] = await Promise.all([
3929
+ buildLegalFormPatterns(),
3930
+ config.enableTriggerPhrases ? buildTriggerPatterns() : Promise.resolve({
3931
+ patterns: [],
3932
+ rules: []
3933
+ }),
3934
+ config.enableDenyList ? buildDenyList(config, ctx) : Promise.resolve(null),
3935
+ buildStreetTypePatterns(),
3936
+ getCurrencyPatterns(),
3937
+ getDatePatterns(),
3938
+ getSigningClausePatterns()
3939
+ ]);
3940
+ const allRegex = [
3941
+ ...REGEX_PATTERNS,
3942
+ ...currencyPatterns,
3943
+ ...datePatterns,
3944
+ ...signingPatterns
3945
+ ];
3946
+ const regexMeta = [
3947
+ ...REGEX_META,
3948
+ ...currencyPatterns.map(() => CURRENCY_PATTERN_META),
3949
+ ...datePatterns.map(() => DATE_PATTERN_META),
3950
+ ...signingPatterns.map(() => SIGNING_CLAUSE_META)
3951
+ ];
3952
+ let offset = 0;
3953
+ const regexSlice = {
3954
+ start: offset,
3955
+ end: offset + allRegex.length
3956
+ };
3957
+ offset = regexSlice.end;
3958
+ const legalFormsSlice = {
3959
+ start: offset,
3960
+ end: offset + legalForms.length
3961
+ };
3962
+ offset = legalFormsSlice.end;
3963
+ const triggersSlice = {
3964
+ start: offset,
3965
+ end: offset + triggers.patterns.length
3966
+ };
3967
+ const triggerEntries = triggers.patterns.map((p) => ({
3968
+ pattern: p,
3969
+ literal: true,
3970
+ caseInsensitive: true
3971
+ }));
3972
+ const tsRegex = new TextSearch([
3973
+ ...allRegex,
3974
+ ...legalForms,
3975
+ ...triggerEntries
3976
+ ]);
3977
+ offset = 0;
3978
+ const denyListOriginals = denyListData?.originals ?? [];
3979
+ const denyListSlice = {
3980
+ start: offset,
3981
+ end: offset + denyListOriginals.length
3982
+ };
3983
+ offset = denyListSlice.end;
3984
+ const streetTypesSlice = {
3985
+ start: offset,
3986
+ end: offset + streetTypes.length
3987
+ };
3988
+ offset = streetTypesSlice.end;
3989
+ const gazResult = config.enableGazetteer && gazetteerEntries.length > 0 ? buildGazetteerPatterns(gazetteerEntries) : null;
3990
+ const gazetteerSlice = {
3991
+ start: offset,
3992
+ end: offset + (gazResult?.patterns.length ?? 0)
3993
+ };
3994
+ offset = gazetteerSlice.end;
3995
+ const wrapWholeWord = (s) => ({
3996
+ pattern: s,
3997
+ literal: true,
3998
+ wholeWords: true
3999
+ });
4000
+ const literalAllPatterns = [
4001
+ ...denyListOriginals.map(wrapWholeWord),
4002
+ ...streetTypes.map(wrapWholeWord),
4003
+ ...gazResult?.patterns ?? []
4004
+ ];
4005
+ return {
4006
+ tsRegex,
4007
+ tsLiterals: literalAllPatterns.length > 0 ? new TextSearch(literalAllPatterns, {
4008
+ caseInsensitive: true,
4009
+ overlapStrategy: "all"
4010
+ }) : new TextSearch([]),
4011
+ slices: {
4012
+ regex: regexSlice,
4013
+ legalForms: legalFormsSlice,
4014
+ triggers: triggersSlice,
4015
+ denyList: denyListSlice,
4016
+ streetTypes: streetTypesSlice,
4017
+ gazetteer: gazetteerSlice
4018
+ },
4019
+ regexMeta,
4020
+ triggerRules: triggers.rules,
4021
+ denyListData,
4022
+ gazetteerData: gazResult?.data ?? null
4023
+ };
4024
+ };
4025
+ //#endregion
4026
+ //#region src/unified-search.ts
4027
+ const runUnifiedSearch = (instance, fullText) => {
4028
+ const regexMatches = instance.tsRegex.findIter(fullText);
4029
+ const normalized = normalizeForSearch(fullText);
4030
+ return {
4031
+ regexMatches,
4032
+ literalMatches: instance.tsLiterals.findIter(normalized)
4033
+ };
4034
+ };
4035
+ //#endregion
4036
+ //#region src/util/entity-masking.ts
4037
+ const MASK_TOKEN = "[MASKED]";
4038
+ const MASK_LEN = 8;
4039
+ /**
4040
+ * Replace detected entity spans with placeholder tokens.
4041
+ * Each entity span is replaced with "[MASKED]" (fixed
4042
+ * length). Returns the masked text and an offset mapping
4043
+ * function.
4044
+ */
4045
+ const maskDetectedSpans = (fullText, entities) => {
4046
+ if (entities.length === 0) return {
4047
+ maskedText: fullText,
4048
+ offsetMap: (s, e) => ({
4049
+ start: s,
4050
+ end: e
4051
+ })
4052
+ };
4053
+ const sorted = entities.toSorted((a, b) => a.start - b.start || b.end - a.end);
4054
+ const spans = [];
4055
+ const first = sorted[0];
4056
+ if (!first) return {
4057
+ maskedText: fullText,
4058
+ offsetMap: (s, e) => ({
4059
+ start: s,
4060
+ end: e
4061
+ })
4062
+ };
4063
+ let cur = {
4064
+ start: first.start,
4065
+ end: first.end
4066
+ };
4067
+ for (let i = 1; i < sorted.length; i++) {
4068
+ const s = sorted[i];
4069
+ if (!s) continue;
4070
+ if (s.start < cur.end) cur.end = Math.max(cur.end, s.end);
4071
+ else {
4072
+ spans.push(cur);
4073
+ cur = {
4074
+ start: s.start,
4075
+ end: s.end
4076
+ };
4077
+ }
4078
+ }
4079
+ spans.push(cur);
4080
+ const segments = [];
4081
+ const parts = [];
4082
+ let prev = 0;
4083
+ let cumulativeShift = 0;
4084
+ for (const span of spans) {
4085
+ parts.push(fullText.slice(prev, span.start));
4086
+ parts.push(MASK_TOKEN);
4087
+ const delta = span.end - span.start - MASK_LEN;
4088
+ cumulativeShift += delta;
4089
+ const maskedStart = span.start - (cumulativeShift - delta);
4090
+ const maskedEnd = maskedStart + MASK_LEN;
4091
+ segments.push({
4092
+ maskedStart,
4093
+ maskedEnd,
4094
+ shift: cumulativeShift,
4095
+ origStart: span.start,
4096
+ origEnd: span.end
4097
+ });
4098
+ prev = span.end;
4099
+ }
4100
+ parts.push(fullText.slice(prev));
4101
+ const maskedText = parts.join("");
4102
+ const offsetMap = (maskedStart, maskedEnd) => {
4103
+ for (const seg of segments) if (maskedStart < seg.maskedEnd && maskedEnd > seg.maskedStart) return null;
4104
+ let shift = 0;
4105
+ for (const seg of segments) if (maskedStart >= seg.maskedEnd) shift = seg.shift;
4106
+ else break;
4107
+ return {
4108
+ start: maskedStart + shift,
4109
+ end: maskedEnd + shift
4110
+ };
4111
+ };
4112
+ return {
4113
+ maskedText,
4114
+ offsetMap
4115
+ };
4116
+ };
4117
+ /**
4118
+ * Map NER entities from masked-text offsets back to
4119
+ * original-text offsets. Discards any NER entity whose
4120
+ * mapped span overlaps a masked (rule-detected) region.
4121
+ */
4122
+ const unmaskNerEntities = (nerEntities, maskResult, fullText) => {
4123
+ const result = [];
4124
+ for (const ner of nerEntities) {
4125
+ const mapped = maskResult.offsetMap(ner.start, ner.end);
4126
+ if (mapped === null) continue;
4127
+ result.push({
4128
+ ...ner,
4129
+ start: mapped.start,
4130
+ end: mapped.end,
4131
+ text: fullText.slice(mapped.start, mapped.end)
4132
+ });
4133
+ }
4134
+ return result;
4135
+ };
4136
+ //#endregion
4137
+ //#region src/pipeline.ts
4138
+ const shouldReplace = (a, b) => {
4139
+ const aPri = DETECTOR_PRIORITY[a.source] ?? 0;
4140
+ const bPri = DETECTOR_PRIORITY[b.source] ?? 0;
4141
+ if (aPri !== bPri) return aPri > bPri;
4142
+ return a.score > b.score || a.score === b.score && a.end - a.start > b.end - b.start;
4143
+ };
4144
+ /** Labels where colons are structurally significant. */
4145
+ const COLON_LABELS = new Set(["ip address", "mac address"]);
4146
+ /** Strip leading/trailing whitespace and punctuation. */
4147
+ const sanitizeEntities = (entities) => entities.flatMap((e) => {
4148
+ const strip = COLON_LABELS.has(e.label) ? /[\s,;]+/ : /[\s:,;]+/;
4149
+ const leadTrimmed = e.text.replace(new RegExp(`^${strip.source}`, strip.flags), "");
4150
+ const lead = e.text.length - leadTrimmed.length;
4151
+ const cleaned = leadTrimmed.replace(new RegExp(`${strip.source}$`, strip.flags), "");
4152
+ if (cleaned.length === 0) return [];
4153
+ if (!/[\p{L}\p{N}]/u.test(cleaned)) return [];
4154
+ if (cleaned === e.text) return [e];
4155
+ return [{
4156
+ ...e,
4157
+ start: e.start + lead,
4158
+ end: e.start + lead + cleaned.length,
4159
+ text: cleaned
4160
+ }];
4161
+ });
4162
+ const mergeAndDedup = (...layers) => {
4163
+ const all = [];
4164
+ for (const layer of layers) for (const entity of layer) all.push(entity);
4165
+ if (all.length === 0) return [];
4166
+ const sorted = all.toSorted((a, b) => a.start - b.start);
4167
+ const first = sorted[0];
4168
+ if (!first) return [];
4169
+ const merged = [{ ...first }];
4170
+ for (let i = 1; i < sorted.length; i++) {
4171
+ const entity = sorted[i];
4172
+ const last = merged[merged.length - 1];
4173
+ if (!entity || !last) continue;
4174
+ if (last.end <= entity.start) merged.push({ ...entity });
4175
+ else if (shouldReplace(entity, last)) merged[merged.length - 1] = { ...entity };
4176
+ }
4177
+ return sanitizeEntities(merged);
4178
+ };
4179
+ const escapeRegex = (s) => s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
4180
+ let amountWordsRe = null;
4181
+ let amountWordsLoaded = false;
4182
+ const getAmountWordsRe = async () => {
4183
+ if (amountWordsLoaded && amountWordsRe) return amountWordsRe;
4184
+ try {
4185
+ const alt = (await import("@stll/anonymize-data/config/amount-words.json")).default.patterns.flatMap((p) => p.keywords).map(escapeRegex).join("|");
4186
+ amountWordsRe = new RegExp(`^[,;]?[^\\S\\n]*(\\((?:${alt})[:\\s][^)\\n]{1,120}\\))`, "i");
4187
+ } catch {
4188
+ amountWordsRe = /^[,;]?[^\S\n]*(\((?:slovy|slovně)[:\s][^)\n]{1,120}\))/i;
4189
+ }
4190
+ amountWordsLoaded = true;
4191
+ return amountWordsRe;
4192
+ };
4193
+ const extendMonetaryAmountWords = (entities, fullText, re) => entities.map((e) => {
4194
+ if (e.label !== "monetary amount") return e;
4195
+ const after = fullText.slice(e.end);
4196
+ const m = re.exec(after);
4197
+ if (!m) return e;
4198
+ const newEnd = e.end + m[0].length;
4199
+ return {
4200
+ ...e,
4201
+ end: newEnd,
4202
+ text: fullText.slice(e.start, newEnd)
4203
+ };
4204
+ });
4205
+ const checkAbort = (signal) => {
4206
+ if (signal?.aborted) throw new DOMException("Pipeline aborted", "AbortError");
4207
+ };
4208
+ const configKey = (config, gazetteerEntries) => {
4209
+ const gazFingerprint = config.enableGazetteer && gazetteerEntries.length > 0 ? gazetteerEntries.map((e) => `${e.id}:${e.canonical}:${e.label}:${[...e.variants].sort().join(",")}`).toSorted().join(";") : "";
4210
+ return `${config.enableDenyList}:${config.enableTriggerPhrases}:${config.denyListCountries?.toSorted().join(",") ?? ""}:${config.denyListRegions?.toSorted().join(",") ?? ""}:${config.denyListExcludeCategories?.toSorted().join(",") ?? ""}:${config.enableGazetteer}:${gazFingerprint}`;
4211
+ };
4212
+ /**
4213
+ * Get or build a cached search instance. Cache state
4214
+ * lives on the provided PipelineContext, not at module
4215
+ * level.
4216
+ */
4217
+ const getCachedSearch = async (config, gazetteerEntries, ctx) => {
4218
+ const key = configKey(config, gazetteerEntries);
4219
+ if (ctx.search && ctx.searchKey === key) return ctx.search;
4220
+ if (ctx.searchPromise && ctx.searchKey === key) return ctx.searchPromise;
4221
+ ctx.search = null;
4222
+ ctx.searchKey = key;
4223
+ const promise = buildUnifiedSearch(config, gazetteerEntries, ctx);
4224
+ ctx.searchPromise = promise;
4225
+ const result = await promise;
4226
+ if (ctx.searchKey === key) ctx.search = result;
4227
+ return result;
4228
+ };
4229
+ /**
4230
+ * Run the full detection pipeline.
4231
+ *
4232
+ * Two TextSearch instances scan the text (regex +
4233
+ * literals). Results are dispatched to each
4234
+ * detector's post-processor by pattern index range.
4235
+ *
4236
+ * Pass an AbortSignal to cancel the pipeline between
4237
+ * stages. Throws a DOMException with name "AbortError"
4238
+ * when cancelled.
4239
+ *
4240
+ * Pass an optional `context` to isolate cached state
4241
+ * from other pipeline runs. If omitted, a module-level
4242
+ * default context is used (backward compatible).
4243
+ */
4244
+ const runPipeline = async (options) => {
4245
+ const { fullText, config, gazetteerEntries, nerInference = null, onProgress, cachedSearch, signal, context } = options;
4246
+ const ctx = context ?? defaultContext;
4247
+ const log = (step, detail) => {
4248
+ onProgress?.(step, detail);
4249
+ };
4250
+ checkAbort(signal);
4251
+ let zoneInitOk = false;
4252
+ const enableHotwords = config.enableHotwordRules === true;
4253
+ let hotwordInitOk = false;
4254
+ const hotwordInit = enableHotwords ? initHotwordRules().then(() => {
4255
+ hotwordInitOk = true;
4256
+ }).catch((err) => {
4257
+ log("hotwords", "init failed; skipping");
4258
+ console.warn("[anonymize] hotword rules init failed", err);
4259
+ }) : Promise.resolve();
4260
+ if (config.enableZoneClassification) {
4261
+ const zoneInit = initZoneClassifier(ctx).then(() => {
4262
+ zoneInitOk = true;
4263
+ }).catch((err) => {
4264
+ log("zones", "init failed; skipping");
4265
+ console.warn("[anonymize] zone classifier init failed", err);
4266
+ });
4267
+ await Promise.all([
4268
+ loadGenericRoles(ctx),
4269
+ initPrepositions(),
4270
+ initStreetAbbrevs(),
4271
+ zoneInit,
4272
+ hotwordInit
4273
+ ]);
4274
+ } else await Promise.all([
4275
+ loadGenericRoles(ctx),
4276
+ initPrepositions(),
4277
+ initStreetAbbrevs(),
4278
+ hotwordInit
4279
+ ]);
4280
+ if (cachedSearch && config.enableDenyList) await ensureDenyListData(ctx);
4281
+ let zones = [];
4282
+ if (config.enableZoneClassification && zoneInitOk) {
4283
+ zones = classifyZones(fullText, ctx);
4284
+ if (zones.length > 0) log("zones", [...new Set(zones.map((z) => z.zone))].join(", "));
4285
+ }
4286
+ checkAbort(signal);
4287
+ const search = cachedSearch ?? await getCachedSearch(config, gazetteerEntries, ctx);
4288
+ checkAbort(signal);
4289
+ const { regexMatches, literalMatches } = runUnifiedSearch(search, fullText);
4290
+ const { slices } = search;
4291
+ const regexEntities = config.enableRegex ? processRegexMatches(regexMatches, slices.regex.start, slices.regex.end, search.regexMeta) : [];
4292
+ if (regexEntities.length > 0) log("regex", `${regexEntities.length} matches`);
4293
+ const legalFormEntities = processLegalFormMatches(regexMatches, slices.legalForms.start, slices.legalForms.end, fullText);
4294
+ if (legalFormEntities.length > 0) log("legal-forms", `${legalFormEntities.length} matches`);
4295
+ const triggerEntities = config.enableTriggerPhrases ? processTriggerMatches(regexMatches, slices.triggers.start, slices.triggers.end, fullText, search.triggerRules) : [];
4296
+ if (triggerEntities.length > 0) log("trigger-phrases", `${triggerEntities.length} matches`);
4297
+ checkAbort(signal);
4298
+ let nameCorpusEntities = [];
4299
+ if (config.enableNameCorpus && !config.enableDenyList) {
4300
+ await initNameCorpus(ctx);
4301
+ checkAbort(signal);
4302
+ nameCorpusEntities = detectNameCorpus(fullText, ctx);
4303
+ log("name-corpus", `${nameCorpusEntities.length} matches`);
4304
+ }
4305
+ const denyListEntities = config.enableDenyList && search.denyListData ? processDenyListMatches(literalMatches, slices.denyList.start, slices.denyList.end, fullText, search.denyListData, ctx) : [];
4306
+ if (denyListEntities.length > 0) log("deny-list", `${denyListEntities.length} matches`);
4307
+ const gazetteerEntities = config.enableGazetteer && search.gazetteerData ? processGazetteerMatches(literalMatches, slices.gazetteer.start, slices.gazetteer.end, fullText, search.gazetteerData) : [];
4308
+ if (gazetteerEntities.length > 0) log("gazetteer", `${gazetteerEntities.length} matches`);
4309
+ checkAbort(signal);
4310
+ let nerEntities = [];
4311
+ if (config.enableNer && nerInference) {
4312
+ const maskResult = maskDetectedSpans(fullText, [
4313
+ ...triggerEntities,
4314
+ ...regexEntities,
4315
+ ...legalFormEntities,
4316
+ ...nameCorpusEntities,
4317
+ ...denyListEntities,
4318
+ ...gazetteerEntities
4319
+ ]);
4320
+ log("ner", "running inference...");
4321
+ const rawNer = await nerInference(maskResult.maskedText, config.labels, config.threshold, signal);
4322
+ nerEntities = unmaskNerEntities(rawNer, maskResult, fullText);
4323
+ const dropped = rawNer.length - nerEntities.length;
4324
+ log("ner", `${nerEntities.length} entities` + (dropped > 0 ? ` (${dropped} masked)` : ""));
4325
+ }
4326
+ checkAbort(signal);
4327
+ const preAddressEntities = [
4328
+ ...triggerEntities,
4329
+ ...regexEntities,
4330
+ ...legalFormEntities,
4331
+ ...nameCorpusEntities,
4332
+ ...denyListEntities,
4333
+ ...gazetteerEntities,
4334
+ ...nerEntities
4335
+ ];
4336
+ const addressSeedEntities = await processAddressSeeds(literalMatches, slices.streetTypes.start, slices.streetTypes.end, fullText, preAddressEntities);
4337
+ if (addressSeedEntities.length > 0) log("address-seeds", `${addressSeedEntities.length} expanded`);
4338
+ checkAbort(signal);
4339
+ const zoneAdjusted = applyZoneAdjustments([...preAddressEntities, ...addressSeedEntities], zones);
4340
+ const preBoostEntities = enableHotwords && hotwordInitOk ? applyHotwordRules(zoneAdjusted, fullText) : zoneAdjusted;
4341
+ let allEntities;
4342
+ if (config.enableConfidenceBoost) {
4343
+ allEntities = boostNearMissEntities(preBoostEntities, config.threshold);
4344
+ const boosted = allEntities.length - preBoostEntities.filter((e) => e.score >= config.threshold).length;
4345
+ if (boosted > 0) log("confidence-boost", `${boosted} near-miss promoted`);
4346
+ } else allEntities = preBoostEntities.filter((e) => e.score >= config.threshold);
4347
+ const streetPatterns = detectStreetPatternsNearAddresses(fullText, allEntities);
4348
+ if (streetPatterns.length > 0) {
4349
+ allEntities = [...allEntities, ...streetPatterns];
4350
+ log("street-context", `${streetPatterns.length} street patterns near addresses`);
4351
+ }
4352
+ const orphanStreets = detectOrphanStreetLines(fullText, allEntities);
4353
+ if (orphanStreets.length > 0) {
4354
+ allEntities = [...allEntities, ...orphanStreets];
4355
+ log("orphan-streets", `${orphanStreets.length} header street lines`);
4356
+ }
4357
+ const rawMerged = mergeAndDedup(allEntities);
4358
+ log("merge", `${rawMerged.length} after dedup`);
4359
+ const mergedExtended = extendMonetaryAmountWords(rawMerged, fullText, await getAmountWordsRe());
4360
+ const consistent = enforceBoundaryConsistency(mergedExtended, fullText);
4361
+ if (consistent.length < mergedExtended.length) log("boundary", `${mergedExtended.length - consistent.length} consolidated`);
4362
+ let postOrgEntities = consistent;
4363
+ if (config.enableCoreference) {
4364
+ const thresholded = propagateOrgNames(consistent, fullText).filter((e) => e.score >= config.threshold);
4365
+ if (thresholded.length > 0) {
4366
+ postOrgEntities = mergeAndDedup(consistent, thresholded);
4367
+ log("org-propagation", `${thresholded.length} base names`);
4368
+ }
4369
+ }
4370
+ const merged = filterFalsePositives(postOrgEntities, ctx);
4371
+ if (merged.length < postOrgEntities.length) log("filter", `removed ${postOrgEntities.length - merged.length} FPs`);
4372
+ checkAbort(signal);
4373
+ ctx.corefSourceMap.clear();
4374
+ if (config.enableCoreference) {
4375
+ const terms = await extractDefinedTerms(fullText, merged, ctx);
4376
+ if (terms.length > 0) {
4377
+ log("coreference", `${terms.length} defined terms`);
4378
+ const corefSpans = findCoreferenceSpans(fullText, terms, ctx);
4379
+ if (corefSpans.length > 0) {
4380
+ log("coreference-rescan", `${corefSpans.length} aliases`);
4381
+ return sanitizeEntities(filterFalsePositives(enforceBoundaryConsistency(mergeAndDedup(merged, corefSpans), fullText), ctx));
4382
+ }
4383
+ }
4384
+ }
4385
+ return sanitizeEntities(merged);
4386
+ };
4387
+ const OPERATOR_REGISTRY = {
4388
+ replace: {
4389
+ type: "replace",
4390
+ reversibility: "reversible",
4391
+ apply: (_text, _label, placeholder) => placeholder
4392
+ },
4393
+ redact: {
4394
+ type: "redact",
4395
+ reversibility: "irreversible",
4396
+ apply: (_text, _label, _placeholder, redactString) => redactString
4397
+ }
4398
+ };
4399
+ /**
4400
+ * Default operator config: replace for all labels.
4401
+ * Preserves existing pipeline behaviour.
4402
+ */
4403
+ const DEFAULT_OPERATOR_CONFIG = {
4404
+ operators: {},
4405
+ redactString: "[REDACTED]"
4406
+ };
4407
+ /**
4408
+ * Resolve the operator for a label, falling back to "replace".
4409
+ */
4410
+ const resolveOperator = (config, label) => config.operators[label] ?? "replace";
4411
+ //#endregion
4412
+ //#region src/redact.ts
4413
+ const WHITESPACE_RE = /\s+/g;
4414
+ const PHONE_NOISE_RE = /[()\s-]/g;
4415
+ const SPACE_DASH_RE = /[\s-]/g;
4416
+ /**
4417
+ * Normalize entity text so that surface-form variations
4418
+ * of the same real-world value map to a single canonical
4419
+ * key. Lowercased emails, stripped phone formatting, etc.
4420
+ */
4421
+ const normalizeEntityText = (label, text) => {
4422
+ const upper = label.toUpperCase().replace(WHITESPACE_RE, "_");
4423
+ if (upper === "EMAIL_ADDRESS" || upper === "EMAIL") return text.toLowerCase().trim();
4424
+ if (upper === "PHONE_NUMBER" || upper === "PHONE") return text.replace(PHONE_NOISE_RE, "");
4425
+ if (upper === "IBAN" || upper === "BANK_ACCOUNT_NUMBER" || upper === "TAX_IDENTIFICATION_NUMBER" || upper === "REGISTRATION_NUMBER") return text.replace(SPACE_DASH_RE, "").toUpperCase();
4426
+ if (upper === "PERSON" || upper === "ORGANIZATION" || upper === "ADDRESS") return text.replace(WHITESPACE_RE, " ").toLowerCase().trim();
4427
+ return text.trim();
4428
+ };
4429
+ /**
4430
+ * Build a stable mapping from entity text to numbered
4431
+ * placeholders. Same real-world value always maps to the
4432
+ * same placeholder (e.g., "Dr. Muller" and "Dr. Muller"
4433
+ * both become [PERSON_1]).
4434
+ *
4435
+ * Placeholder format: [LABEL_N] where LABEL is uppercase
4436
+ * and N is a 1-based counter per label.
4437
+ *
4438
+ * @param ctx Pipeline context. Must be the same instance
4439
+ * passed to `runPipeline` (or `findCoreferenceSpans`)
4440
+ * so coreference placeholder links are preserved.
4441
+ * Defaults to `defaultContext` for single-tenant usage.
4442
+ */
4443
+ const buildPlaceholderMap = (entities, ctx = defaultContext) => {
4444
+ const counters = /* @__PURE__ */ new Map();
4445
+ const textLabelToPlaceholder = /* @__PURE__ */ new Map();
4446
+ const normalizedToPlaceholder = /* @__PURE__ */ new Map();
4447
+ const sorted = entities.toSorted((a, b) => a.start - b.start);
4448
+ for (const entity of sorted) {
4449
+ const compositeKey = `${entity.label}\0${entity.text}`;
4450
+ if (textLabelToPlaceholder.has(compositeKey)) continue;
4451
+ const labelKey = entity.label.toUpperCase().replace(WHITESPACE_RE, "_");
4452
+ const sourceText = ctx.corefSourceMap.get(corefKey(entity));
4453
+ if (sourceText !== void 0) {
4454
+ const sourceNormalizedKey = `${labelKey}\0${normalizeEntityText(entity.label, sourceText)}`;
4455
+ const sourceExisting = normalizedToPlaceholder.get(sourceNormalizedKey);
4456
+ if (sourceExisting) {
4457
+ textLabelToPlaceholder.set(compositeKey, sourceExisting);
4458
+ continue;
4459
+ }
4460
+ }
4461
+ const normalizedKey = `${labelKey}\0${normalizeEntityText(entity.label, entity.text)}`;
4462
+ const existing = normalizedToPlaceholder.get(normalizedKey);
4463
+ if (existing) {
4464
+ textLabelToPlaceholder.set(compositeKey, existing);
4465
+ continue;
4466
+ }
4467
+ const count = (counters.get(labelKey) ?? 0) + 1;
4468
+ counters.set(labelKey, count);
4469
+ const placeholder = `[${labelKey}_${count}]`;
4470
+ textLabelToPlaceholder.set(compositeKey, placeholder);
4471
+ normalizedToPlaceholder.set(normalizedKey, placeholder);
4472
+ }
4473
+ return textLabelToPlaceholder;
4474
+ };
4475
+ /**
4476
+ * Apply redactions to the source text, replacing each
4477
+ * confirmed entity span using the configured operator.
4478
+ *
4479
+ * Co-references are consistent: if the same text appears
4480
+ * multiple times, all occurrences get the same placeholder.
4481
+ *
4482
+ * @param ctx Pipeline context. Must be the same instance
4483
+ * passed to `runPipeline` (or `findCoreferenceSpans`)
4484
+ * so coreference placeholder links are preserved.
4485
+ * Defaults to `defaultContext` for single-tenant usage.
4486
+ */
4487
+ const redactText = (fullText, entities, config = DEFAULT_OPERATOR_CONFIG, ctx = defaultContext) => {
4488
+ if (entities.length === 0) return {
4489
+ redactedText: fullText,
4490
+ redactionMap: /* @__PURE__ */ new Map(),
4491
+ operatorMap: /* @__PURE__ */ new Map(),
4492
+ entityCount: 0
4493
+ };
4494
+ const placeholderMap = buildPlaceholderMap(entities, ctx);
4495
+ const sorted = entities.toSorted((a, b) => a.start - b.start);
4496
+ const nonOverlapping = [];
4497
+ let lastEnd = 0;
4498
+ for (const entity of sorted) if (entity.start >= lastEnd) {
4499
+ nonOverlapping.push(entity);
4500
+ lastEnd = entity.end;
4501
+ }
4502
+ const parts = [];
4503
+ const redactionMap = /* @__PURE__ */ new Map();
4504
+ const operatorMap = /* @__PURE__ */ new Map();
4505
+ let cursor = 0;
4506
+ for (const entity of nonOverlapping) {
4507
+ if (entity.start > cursor) parts.push(fullText.slice(cursor, entity.start));
4508
+ const placeholder = placeholderMap.get(`${entity.label}\0${entity.text}`) ?? `[${entity.label.toUpperCase().replace(/\s+/g, "_")}]`;
4509
+ const opType = resolveOperator(config, entity.label);
4510
+ const operator = OPERATOR_REGISTRY[opType];
4511
+ const replacement = operator.apply(entity.text, entity.label, placeholder, config.redactString);
4512
+ parts.push(replacement);
4513
+ operatorMap.set(placeholder, opType);
4514
+ if (operator.reversibility === "reversible" && !redactionMap.has(placeholder)) redactionMap.set(placeholder, entity.text);
4515
+ cursor = entity.end;
4516
+ }
4517
+ if (cursor < fullText.length) parts.push(fullText.slice(cursor));
4518
+ return {
4519
+ redactedText: parts.join(""),
4520
+ redactionMap,
4521
+ operatorMap,
4522
+ entityCount: nonOverlapping.length
4523
+ };
4524
+ };
4525
+ /**
4526
+ * Serialize the redaction key to JSON for export.
4527
+ * Includes operator metadata so the export is self-describing.
4528
+ */
4529
+ const exportRedactionKey = (redactionMap, operatorMap) => {
4530
+ const entries = {};
4531
+ for (const [placeholder, value] of redactionMap) entries[placeholder] = {
4532
+ original: value,
4533
+ operator: operatorMap.get(placeholder) ?? "replace"
4534
+ };
4535
+ return JSON.stringify({ entries }, null, 2);
4536
+ };
4537
+ /**
4538
+ * De-anonymise text using a redaction key.
4539
+ * Replaces placeholders back with original values.
4540
+ * Only works for reversible operators (replace).
4541
+ */
4542
+ const deanonymise = (redactedText, redactionMap) => {
4543
+ let result = redactedText;
4544
+ for (const [placeholder, original] of redactionMap) result = result.replaceAll(placeholder, original);
4545
+ return result;
4546
+ };
4547
+ //#endregion
4548
+ //#region src/gliner/decoder.ts
4549
+ const sigmoid$1 = (x) => 1 / (1 + Math.exp(-x));
4550
+ /** Check if two spans overlap (optionally allowing multi-label). */
4551
+ const hasOverlapping = (a, b, multiLabel) => {
4552
+ const a0 = a[0] ?? 0;
4553
+ const a1 = a[1] ?? 0;
4554
+ const b0 = b[0] ?? 0;
4555
+ const b1 = b[1] ?? 0;
4556
+ if (a0 === b0 && a1 === b1) return !multiLabel;
4557
+ return !(a0 > b1 || b0 > a1);
4558
+ };
4559
+ /**
4560
+ * Greedy non-overlapping span selection.
4561
+ * Sorts by score descending, keeps each span only if it
4562
+ * doesn't overlap with already-selected spans.
4563
+ */
4564
+ const greedySearch = (spans, flatNer, multiLabel) => {
4565
+ const sorted = spans.toSorted((a, b) => b[4] - a[4]);
4566
+ const selected = [];
4567
+ for (const span of sorted) if (!selected.some((s) => {
4568
+ if (flatNer) return hasOverlapping([span[1], span[2]], [s[1], s[2]], multiLabel);
4569
+ if (span[1] <= s[1] && span[2] >= s[2] || s[1] <= span[1] && s[2] >= span[2]) return false;
4570
+ return hasOverlapping([span[1], span[2]], [s[1], s[2]], multiLabel);
4571
+ })) selected.push(span);
4572
+ return selected.toSorted((a, b) => a[1] - b[1]);
4573
+ };
4574
+ /**
4575
+ * Decode span-level model logits into entity results.
4576
+ */
4577
+ const decodeSpans = (batchSize, inputLength, maxWidth, numEntities, texts, batchIds, batchWordsStartIdx, batchWordsEndIdx, idToClass, modelOutput, flatNer, threshold, multiLabel) => {
4578
+ const spans = Array.from({ length: batchSize }, () => []);
4579
+ const batchPadding = inputLength * maxWidth * numEntities;
4580
+ const startTokenPadding = maxWidth * numEntities;
4581
+ const endTokenPadding = numEntities;
4582
+ for (let id = 0; id < modelOutput.length; id++) {
4583
+ const batch = Math.floor(id / batchPadding);
4584
+ const startToken = Math.floor(id / startTokenPadding) % inputLength;
4585
+ const endToken = startToken + Math.floor(id / endTokenPadding) % maxWidth;
4586
+ const entity = id % numEntities;
4587
+ const prob = sigmoid$1(modelOutput[id] ?? 0);
4588
+ const batchStarts = batchWordsStartIdx[batch];
4589
+ const batchEnds = batchWordsEndIdx[batch];
4590
+ const batchSpans = spans[batch];
4591
+ if (!batchStarts || !batchEnds || !batchSpans) continue;
4592
+ if (prob >= threshold && startToken < batchStarts.length && endToken < batchEnds.length) {
4593
+ const globalBatch = batchIds[batch] ?? 0;
4594
+ const startIdx = batchStarts[startToken] ?? 0;
4595
+ const endIdx = batchEnds[endToken] ?? 0;
4596
+ const spanText = (texts[globalBatch] ?? "").slice(startIdx, endIdx);
4597
+ batchSpans.push([
4598
+ spanText,
4599
+ startIdx,
4600
+ endIdx,
4601
+ idToClass[entity + 1] ?? "",
4602
+ prob
4603
+ ]);
4604
+ }
4605
+ }
4606
+ return spans.map((batchSpans) => greedySearch(batchSpans, flatNer, multiLabel));
4607
+ };
4608
+ //#endregion
4609
+ //#region src/gliner/token-decoder.ts
4610
+ const sigmoid = (x) => 1 / (1 + Math.exp(-x));
4611
+ const B_TAG = 0;
4612
+ const I_TAG = 1;
4613
+ /**
4614
+ * Decode token-level BIO logits into entity spans.
4615
+ *
4616
+ * For each word, checks if the B(egin) logit for any class
4617
+ * exceeds the threshold. If so, extends the span by consuming
4618
+ * subsequent I(nside) tokens of the same class.
4619
+ */
4620
+ const decodeTokenSpans = (batchSize, numWords, numEntities, texts, batchIds, batchWordsStartIdx, batchWordsEndIdx, idToClass, modelOutput, threshold) => {
4621
+ const results = Array.from({ length: batchSize }, () => []);
4622
+ const wordStride = numEntities * 3;
4623
+ const batchStride = numWords * wordStride;
4624
+ for (let b = 0; b < batchSize; b++) {
4625
+ const batchOffset = b * batchStride;
4626
+ const starts = batchWordsStartIdx[b];
4627
+ const ends = batchWordsEndIdx[b];
4628
+ const text = texts[batchIds[b] ?? 0] ?? "";
4629
+ const batchSpans = results[b];
4630
+ if (!starts || !ends || !batchSpans) continue;
4631
+ const actualWords = starts.length;
4632
+ for (let e = 0; e < numEntities; e++) {
4633
+ let w = 0;
4634
+ while (w < actualWords) {
4635
+ const bScore = sigmoid(modelOutput[batchOffset + w * wordStride + e * 3 + B_TAG] ?? 0);
4636
+ if (bScore < threshold) {
4637
+ w++;
4638
+ continue;
4639
+ }
4640
+ const spanStart = w;
4641
+ let spanEnd = w;
4642
+ let maxScore = bScore;
4643
+ while (spanEnd + 1 < actualWords) {
4644
+ const iScore = sigmoid(modelOutput[batchOffset + (spanEnd + 1) * wordStride + e * 3 + I_TAG] ?? 0);
4645
+ if (iScore < threshold) break;
4646
+ spanEnd++;
4647
+ maxScore = Math.max(maxScore, iScore);
4648
+ }
4649
+ const charStart = starts[spanStart] ?? 0;
4650
+ const charEnd = ends[spanEnd] ?? 0;
4651
+ const spanText = text.slice(charStart, charEnd);
4652
+ const label = idToClass[e + 1] ?? "";
4653
+ if (spanText.trim().length > 0 && label) batchSpans.push([
4654
+ spanText,
4655
+ charStart,
4656
+ charEnd,
4657
+ label,
4658
+ maxScore
4659
+ ]);
4660
+ w = spanEnd + 1;
4661
+ }
4662
+ }
4663
+ const selected = [];
4664
+ for (const span of batchSpans.toSorted((x, y) => y[4] - x[4])) if (!selected.some((s) => span[1] < s[2] && span[2] > s[1])) selected.push(span);
4665
+ results[b] = selected.toSorted((x, y) => x[1] - y[1]);
4666
+ }
4667
+ return results;
4668
+ };
4669
+ //#endregion
4670
+ //#region src/gliner/processor.ts
4671
+ const segmenter = new Intl.Segmenter(void 0, { granularity: "word" });
4672
+ /** Tokenize text into words with character offsets. */
4673
+ const tokenizeText = (text) => {
4674
+ const words = [];
4675
+ const starts = [];
4676
+ const ends = [];
4677
+ for (const { segment, index, isWordLike } of segmenter.segment(text)) {
4678
+ if (!isWordLike && !/\d/u.test(segment)) continue;
4679
+ words.push(segment);
4680
+ starts.push(index);
4681
+ ends.push(index + segment.length);
4682
+ }
4683
+ return [
4684
+ words,
4685
+ starts,
4686
+ ends
4687
+ ];
4688
+ };
4689
+ /** Build entity label <-> id mappings. */
4690
+ const createMappings = (labels) => {
4691
+ const classToId = {};
4692
+ const idToClass = {};
4693
+ for (let i = 0; i < labels.length; i++) {
4694
+ const label = labels[i];
4695
+ if (label === void 0) continue;
4696
+ const id = i + 1;
4697
+ classToId[label] = id;
4698
+ idToClass[id] = label;
4699
+ }
4700
+ return {
4701
+ classToId,
4702
+ idToClass
4703
+ };
4704
+ };
4705
+ /** Prepend entity prompt tokens to word tokens. */
4706
+ const prepareTextInputs = (batchTokens, entities) => {
4707
+ const inputTexts = [];
4708
+ const promptLengths = [];
4709
+ const textLengths = [];
4710
+ for (const tokens of batchTokens) {
4711
+ textLengths.push(tokens.length);
4712
+ const prompt = [];
4713
+ for (const ent of entities) {
4714
+ prompt.push("<<ENT>>");
4715
+ prompt.push(ent);
4716
+ }
4717
+ prompt.push("<<SEP>>");
4718
+ promptLengths.push(prompt.length);
4719
+ inputTexts.push([...prompt, ...tokens]);
4720
+ }
4721
+ return [
4722
+ inputTexts,
4723
+ textLengths,
4724
+ promptLengths
4725
+ ];
4726
+ };
4727
+ /** Encode word sequences into token IDs with masks. */
4728
+ const encodeInputs = (tokenizer, texts, promptLengths) => {
4729
+ const clsTokenId = tokenizer.token_to_id("[CLS]") ?? 1;
4730
+ const sepTokenId = tokenizer.token_to_id("[SEP]") ?? 2;
4731
+ const allInputIds = [];
4732
+ const allAttentionMasks = [];
4733
+ const allWordsMasks = [];
4734
+ for (let idx = 0; idx < texts.length; idx++) {
4735
+ const promptLength = promptLengths[idx] ?? 0;
4736
+ const words = texts[idx];
4737
+ if (!words) continue;
4738
+ const wordsMask = [0];
4739
+ const inputIds = [clsTokenId];
4740
+ const attentionMask = [1];
4741
+ let wordCounter = 1;
4742
+ for (let wordId = 0; wordId < words.length; wordId++) {
4743
+ const word = words[wordId];
4744
+ if (word === void 0) continue;
4745
+ const wordTokens = tokenizer.encode(word).ids.slice(1, -1);
4746
+ for (let tokenId = 0; tokenId < wordTokens.length; tokenId++) {
4747
+ attentionMask.push(1);
4748
+ if (wordId < promptLength) wordsMask.push(0);
4749
+ else if (tokenId === 0) {
4750
+ wordsMask.push(wordCounter);
4751
+ wordCounter++;
4752
+ } else wordsMask.push(0);
4753
+ inputIds.push(wordTokens[tokenId] ?? 0);
4754
+ }
4755
+ }
4756
+ inputIds.push(sepTokenId);
4757
+ wordsMask.push(0);
4758
+ attentionMask.push(1);
4759
+ allInputIds.push(inputIds);
4760
+ allAttentionMasks.push(attentionMask);
4761
+ allWordsMasks.push(wordsMask);
4762
+ }
4763
+ return [
4764
+ allInputIds,
4765
+ allAttentionMasks,
4766
+ allWordsMasks
4767
+ ];
4768
+ };
4769
+ /** Build span index pairs and masks for the model. */
4770
+ const prepareSpans = (batchTokens, maxWidth) => {
4771
+ const spanIdxs = [];
4772
+ const spanMasks = [];
4773
+ for (const tokens of batchTokens) {
4774
+ const len = tokens.length;
4775
+ const idx = [];
4776
+ const mask = [];
4777
+ for (let i = 0; i < len; i++) for (let j = 0; j < maxWidth; j++) {
4778
+ const endIdx = Math.min(i + j, len - 1);
4779
+ idx.push([i, endIdx]);
4780
+ mask.push(endIdx < len);
4781
+ }
4782
+ spanIdxs.push(idx);
4783
+ spanMasks.push(mask);
4784
+ }
4785
+ return {
4786
+ spanIdxs,
4787
+ spanMasks
4788
+ };
4789
+ };
4790
+ /** Pad a 2D or 3D array to uniform inner length. */
4791
+ const padArray = (arr, dimensions = 2) => {
4792
+ if (arr.length === 0) return [];
4793
+ const maxLength = Math.max(...arr.map((sub) => sub.length));
4794
+ let finalDim = 0;
4795
+ if (dimensions === 3) {
4796
+ for (const sub of arr) if (sub.length > 0) {
4797
+ finalDim = sub[0].length;
4798
+ break;
4799
+ }
4800
+ }
4801
+ return arr.map((sub) => {
4802
+ const padCount = maxLength - sub.length;
4803
+ const fill = dimensions === 3 ? Array.from({ length: padCount }, () => Array.from({ length: finalDim }).fill(0)) : Array.from({ length: padCount }).fill(0);
4804
+ return [...sub, ...fill];
4805
+ });
4806
+ };
4807
+ /** Prepare a complete batch for ONNX inference. */
4808
+ const prepareBatch = (tokenizer, texts, entities, maxWidth) => {
4809
+ const batchTokens = [];
4810
+ const batchWordsStartIdx = [];
4811
+ const batchWordsEndIdx = [];
4812
+ for (const text of texts) {
4813
+ const [words, starts, ends] = tokenizeText(text);
4814
+ batchTokens.push(words);
4815
+ batchWordsStartIdx.push(starts);
4816
+ batchWordsEndIdx.push(ends);
4817
+ }
4818
+ const { idToClass } = createMappings(entities);
4819
+ const [inputTokens, textLengths, promptLengths] = prepareTextInputs(batchTokens, entities);
4820
+ let [inputsIds, attentionMasks, wordsMasks] = encodeInputs(tokenizer, inputTokens, promptLengths);
4821
+ inputsIds = padArray(inputsIds);
4822
+ attentionMasks = padArray(attentionMasks);
4823
+ wordsMasks = padArray(wordsMasks);
4824
+ let { spanIdxs, spanMasks } = prepareSpans(batchTokens, maxWidth);
4825
+ spanIdxs = padArray(spanIdxs, 3);
4826
+ spanMasks = padArray(spanMasks);
4827
+ return {
4828
+ inputsIds,
4829
+ attentionMasks,
4830
+ wordsMasks,
4831
+ textLengths,
4832
+ spanIdxs,
4833
+ spanMasks,
4834
+ idToClass,
4835
+ batchTokens,
4836
+ batchWordsStartIdx,
4837
+ batchWordsEndIdx
4838
+ };
4839
+ };
4840
+ //#endregion
4841
+ //#region src/util/chunker.ts
4842
+ const MAX_CHUNK_CHARS = 1500;
4843
+ const OVERLAP_CHARS = 50;
4844
+ const MIN_CHUNK_LENGTH = 10;
4845
+ /**
4846
+ * Split text into overlapping chunks for GLiNER's
4847
+ * ~512 token context window. Character-based splitting
4848
+ * (rough approximation of token limits).
4849
+ *
4850
+ * Tries to break at sentence boundaries when possible.
4851
+ */
4852
+ const chunkText = (text) => {
4853
+ const chunks = [];
4854
+ let offset = 0;
4855
+ while (offset < text.length) {
4856
+ let end = Math.min(offset + MAX_CHUNK_CHARS, text.length);
4857
+ if (end < text.length) {
4858
+ const lastPeriod = text.slice(offset, end).lastIndexOf(". ");
4859
+ if (lastPeriod > MAX_CHUNK_CHARS * .5) end = offset + lastPeriod + 2;
4860
+ }
4861
+ const chunk = text.slice(offset, end);
4862
+ if (chunk.trim().length > MIN_CHUNK_LENGTH) chunks.push(chunk);
4863
+ offset = Math.max(offset + 1, end - OVERLAP_CHARS);
4864
+ }
4865
+ return chunks;
4866
+ };
4867
+ /**
4868
+ * Compute the byte offset of each chunk within the
4869
+ * original document text.
4870
+ */
4871
+ const computeChunkOffsets = (fullText, chunks) => {
4872
+ const offsets = [];
4873
+ let searchFrom = 0;
4874
+ for (const chunk of chunks) {
4875
+ const idx = fullText.indexOf(chunk, searchFrom);
4876
+ offsets.push(idx !== -1 ? idx : searchFrom);
4877
+ searchFrom = idx !== -1 ? idx + Math.max(1, chunk.length - OVERLAP_CHARS) : searchFrom;
4878
+ }
4879
+ return offsets;
4880
+ };
4881
+ const POSITION_THRESHOLD = 5;
4882
+ /**
4883
+ * Merge entities from overlapping chunks back to
4884
+ * document-level offsets. Deduplicates entities that
4885
+ * appear in overlap regions (keeps highest score).
4886
+ *
4887
+ * Dedup invariant: each incoming entity is compared
4888
+ * against the highest-scored same-label near-dup in
4889
+ * its proximity window. If it loses, it is dropped.
4890
+ * This does NOT guarantee that all pairwise near-dup
4891
+ * relationships in the output are resolved; a lower-
4892
+ * scored entity can survive if the bridging entity
4893
+ * that would have replaced it was itself dropped by
4894
+ * a higher-scored match.
4895
+ *
4896
+ * Uses a reverse-scan over the sorted merged array
4897
+ * so each entity only compares against nearby
4898
+ * predecessors — O(n * w) average where w is the max
4899
+ * entities per POSITION_THRESHOLD window, O(n²) worst
4900
+ * case when replacements dominate (splice is O(n)).
4901
+ */
4902
+ const mergeChunkEntities = (chunkOffsets, chunkResults) => {
4903
+ const allEntities = [];
4904
+ for (let i = 0; i < chunkResults.length; i++) {
4905
+ const offset = chunkOffsets[i] ?? 0;
4906
+ const entities = chunkResults[i];
4907
+ if (!entities) continue;
4908
+ for (const entity of entities) allEntities.push({
4909
+ ...entity,
4910
+ start: entity.start + offset,
4911
+ end: entity.end + offset
4912
+ });
4913
+ }
4914
+ const sorted = allEntities.toSorted((a, b) => a.start - b.start);
4915
+ const merged = [];
4916
+ for (const entity of sorted) {
4917
+ let bestDupIndex = -1;
4918
+ let bestDupScore = -1;
4919
+ for (let j = merged.length - 1; j >= 0; j--) {
4920
+ const existing = merged[j];
4921
+ if (existing === void 0) continue;
4922
+ if (entity.start - existing.start >= POSITION_THRESHOLD) break;
4923
+ if (existing.label === entity.label && Math.abs(existing.end - entity.end) < POSITION_THRESHOLD && existing.score > bestDupScore) {
4924
+ bestDupIndex = j;
4925
+ bestDupScore = existing.score;
4926
+ }
4927
+ }
4928
+ if (bestDupIndex !== -1) {
4929
+ const existing = merged[bestDupIndex];
4930
+ if (existing !== void 0 && entity.score > existing.score) {
4931
+ merged.splice(bestDupIndex, 1);
4932
+ merged.push({ ...entity });
4933
+ }
4934
+ } else merged.push({ ...entity });
4935
+ }
4936
+ return merged;
4937
+ };
4938
+ //#endregion
4939
+ //#region src/util/levenshtein.ts
4940
+ /**
4941
+ * Compute the Levenshtein edit distance between two
4942
+ * strings. O(n*m) time, O(min(n,m)) space using a
4943
+ * single-row DP approach.
4944
+ */
4945
+ const levenshtein = (rawA, rawB) => {
4946
+ if (rawA === rawB) return 0;
4947
+ if (rawA.length === 0) return rawB.length;
4948
+ if (rawB.length === 0) return rawA.length;
4949
+ const [shorter, longer] = rawA.length <= rawB.length ? [rawA, rawB] : [rawB, rawA];
4950
+ const aLen = shorter.length;
4951
+ const bLen = longer.length;
4952
+ const row = new Uint16Array(aLen + 1);
4953
+ for (let i = 0; i <= aLen; i++) row[i] = i;
4954
+ for (let j = 1; j <= bLen; j++) {
4955
+ let prev = row[0] ?? 0;
4956
+ row[0] = j;
4957
+ for (let i = 1; i <= aLen; i++) {
4958
+ const cost = shorter[i - 1] === longer[j - 1] ? 0 : 1;
4959
+ const temp = row[i] ?? 0;
4960
+ row[i] = Math.min((row[i] ?? 0) + 1, (row[i - 1] ?? 0) + 1, prev + cost);
4961
+ prev = temp;
4962
+ }
4963
+ }
4964
+ return row[aLen] ?? 0;
4965
+ };
4966
+ //#endregion
4967
+ export { CURRENCY_PATTERN_META, DATE_PATTERN_META, DEFAULT_ENTITY_LABELS, DEFAULT_OPERATOR_CONFIG, DETECTION_SOURCES, DETECTOR_PRIORITY, OPERATOR_REGISTRY, OPERATOR_TYPES, REGEX_META, REGEX_PATTERNS, REGIONS, ZONE_SCORE_ADJUSTMENTS, applyHotwordRules, applyZoneAdjustments, boostNearMissEntities, buildDenyList, buildGazetteerPatterns, buildLegalFormPatterns, buildPlaceholderMap, buildStreetTypePatterns, buildTriggerPatterns, buildUnifiedSearch, chunkText, classifyZones, computeChunkOffsets, corefKey, createPipelineContext, deanonymise, decodeSpans, decodeTokenSpans, detectNameCorpus, ensureDenyListData, exportRedactionKey, extractDefinedTerms, filterFalsePositives, findCoreferenceSpans, getCurrencyPatterns, getDatePatterns, initHotwordRules, initNameCorpus, initZoneClassifier, levenshtein, mergeAndDedup, mergeChunkEntities, normalizeForSearch, prepareBatch, processAddressSeeds, processDenyListMatches, processGazetteerMatches, processLegalFormMatches, processRegexMatches, processTriggerMatches, propagateOrgNames, redactText, resolveCountries, resolveOperator, runPipeline, runUnifiedSearch, sanitizeEntities, tokenizeText };
4968
+
4969
+ //# sourceMappingURL=index.js.map