@stll/anonymize 0.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +27 -0
- package/README.md +79 -0
- package/dist/index.d.ts +939 -0
- package/dist/index.js +4969 -0
- package/dist/index.js.map +1 -0
- package/package.json +49 -0
package/dist/index.js
ADDED
|
@@ -0,0 +1,4969 @@
|
|
|
1
|
+
import { at, be, bg, cy, cz, de, dk, ee, es, fi, fr, gb, gr, hr, hu, ie, it, lt, lu, lv, mt, nl, pl, pt, ro, se, si, sk } from "@stll/stdnum";
|
|
2
|
+
import { toRegex } from "@stll/stdnum/patterns";
|
|
3
|
+
import { TextSearch } from "@stll/text-search";
|
|
4
|
+
//#region src/types.ts
|
|
5
|
+
/**
|
|
6
|
+
* Source of a detected entity span.
|
|
7
|
+
* Ordered by detection layer in the pipeline.
|
|
8
|
+
*/
|
|
9
|
+
const DETECTION_SOURCES = {
|
|
10
|
+
TRIGGER: "trigger",
|
|
11
|
+
REGEX: "regex",
|
|
12
|
+
DENY_LIST: "deny-list",
|
|
13
|
+
LEGAL_FORM: "legal-form",
|
|
14
|
+
GAZETTEER: "gazetteer",
|
|
15
|
+
NER: "ner",
|
|
16
|
+
COREFERENCE: "coreference"
|
|
17
|
+
};
|
|
18
|
+
/**
|
|
19
|
+
* Priority levels for detection sources.
|
|
20
|
+
* Higher = more structurally reliable. Used during
|
|
21
|
+
* overlap resolution so deterministic detectors beat
|
|
22
|
+
* probabilistic ones regardless of raw score.
|
|
23
|
+
*/
|
|
24
|
+
const DETECTOR_PRIORITY = {
|
|
25
|
+
gazetteer: 5,
|
|
26
|
+
trigger: 4,
|
|
27
|
+
"legal-form": 3,
|
|
28
|
+
regex: 3,
|
|
29
|
+
"deny-list": 2,
|
|
30
|
+
coreference: 2,
|
|
31
|
+
ner: 1
|
|
32
|
+
};
|
|
33
|
+
/**
|
|
34
|
+
* Anonymisation operator types. Each operator defines
|
|
35
|
+
* how a confirmed entity is replaced in the output.
|
|
36
|
+
*/
|
|
37
|
+
const OPERATOR_TYPES = ["replace", "redact"];
|
|
38
|
+
/**
|
|
39
|
+
* Canonical entity labels used across the pipeline.
|
|
40
|
+
* NER models may use different native labels; the bench
|
|
41
|
+
* NER wrapper maps model output to these canonical names.
|
|
42
|
+
*
|
|
43
|
+
* These labels are ephemeral: entities are regenerated on
|
|
44
|
+
* every pipeline run and never persisted to the database.
|
|
45
|
+
* Renaming a label here requires no migration.
|
|
46
|
+
*/
|
|
47
|
+
const DEFAULT_ENTITY_LABELS = [
|
|
48
|
+
"person",
|
|
49
|
+
"organization",
|
|
50
|
+
"phone number",
|
|
51
|
+
"address",
|
|
52
|
+
"email address",
|
|
53
|
+
"date",
|
|
54
|
+
"date of birth",
|
|
55
|
+
"bank account number",
|
|
56
|
+
"iban",
|
|
57
|
+
"tax identification number",
|
|
58
|
+
"identity card number",
|
|
59
|
+
"registration number",
|
|
60
|
+
"credit card number",
|
|
61
|
+
"passport number",
|
|
62
|
+
"monetary amount"
|
|
63
|
+
];
|
|
64
|
+
//#endregion
|
|
65
|
+
//#region src/context.ts
|
|
66
|
+
/**
|
|
67
|
+
* Build a stable cache key for an entity that survives
|
|
68
|
+
* shallow copies (spread). Uses position + label so the
|
|
69
|
+
* key is identical for the original object and any
|
|
70
|
+
* `{ ...entity }` copy produced by mergeAndDedup.
|
|
71
|
+
*/
|
|
72
|
+
const corefKey = (e) => `${e.start}:${e.end}:${e.label}`;
|
|
73
|
+
/** Create a fresh, empty pipeline context. */
|
|
74
|
+
const createPipelineContext = () => ({
|
|
75
|
+
search: null,
|
|
76
|
+
searchKey: "",
|
|
77
|
+
searchPromise: null,
|
|
78
|
+
nameCorpus: null,
|
|
79
|
+
nameCorpusPromise: null,
|
|
80
|
+
stopwords: null,
|
|
81
|
+
stopwordsPromise: null,
|
|
82
|
+
allowList: null,
|
|
83
|
+
allowListPromise: null,
|
|
84
|
+
personStopwords: null,
|
|
85
|
+
personStopwordsPromise: null,
|
|
86
|
+
firstNameExclusions: null,
|
|
87
|
+
firstNameExclusionCorpusLen: 0,
|
|
88
|
+
genericRoles: null,
|
|
89
|
+
genericRolesPromise: null,
|
|
90
|
+
corefPatterns: null,
|
|
91
|
+
corefPatternsPromise: null,
|
|
92
|
+
corefLoadAttempted: false,
|
|
93
|
+
roleStopSet: null,
|
|
94
|
+
roleStopSetPromise: null,
|
|
95
|
+
zoneHeadingPatterns: null,
|
|
96
|
+
zoneSigningPatterns: null,
|
|
97
|
+
zoneInitPromise: null,
|
|
98
|
+
corefSourceMap: /* @__PURE__ */ new Map()
|
|
99
|
+
});
|
|
100
|
+
/**
|
|
101
|
+
* Module-level default context. Used when callers
|
|
102
|
+
* don't provide an explicit context, preserving full
|
|
103
|
+
* backward compatibility with the existing API.
|
|
104
|
+
*/
|
|
105
|
+
const defaultContext = createPipelineContext();
|
|
106
|
+
//#endregion
|
|
107
|
+
//#region src/util/lang-loader.ts
|
|
108
|
+
let _manifest = null;
|
|
109
|
+
let _manifestPromise = null;
|
|
110
|
+
const loadManifest = () => {
|
|
111
|
+
if (_manifest) return Promise.resolve(_manifest);
|
|
112
|
+
if (_manifestPromise) return _manifestPromise;
|
|
113
|
+
_manifestPromise = (async () => {
|
|
114
|
+
try {
|
|
115
|
+
const mod = await import("@stll/anonymize-data/config/manifest.json");
|
|
116
|
+
const parsed = mod.default ?? mod;
|
|
117
|
+
if (!parsed || typeof parsed.languages !== "object" || parsed.languages === null || Array.isArray(parsed.languages)) {
|
|
118
|
+
console.warn("[anonymize] lang-loader: manifest has unexpected structure, falling back to hardcoded list");
|
|
119
|
+
_manifest = { languages: {} };
|
|
120
|
+
return _manifest;
|
|
121
|
+
}
|
|
122
|
+
_manifest = parsed;
|
|
123
|
+
return _manifest;
|
|
124
|
+
} catch (err) {
|
|
125
|
+
console.warn("[anonymize] lang-loader: manifest not available, falling back to hardcoded language list:", err);
|
|
126
|
+
_manifest = { languages: {} };
|
|
127
|
+
return _manifest;
|
|
128
|
+
}
|
|
129
|
+
})();
|
|
130
|
+
return _manifestPromise;
|
|
131
|
+
};
|
|
132
|
+
const LOADER_REGISTRIES = {
|
|
133
|
+
triggers: {
|
|
134
|
+
cs: () => import("@stll/anonymize-data/config/triggers.cs.json"),
|
|
135
|
+
de: () => import("@stll/anonymize-data/config/triggers.de.json"),
|
|
136
|
+
en: () => import("@stll/anonymize-data/config/triggers.en.json"),
|
|
137
|
+
es: () => import("@stll/anonymize-data/config/triggers.es.json"),
|
|
138
|
+
fr: () => import("@stll/anonymize-data/config/triggers.fr.json"),
|
|
139
|
+
hu: () => import("@stll/anonymize-data/config/triggers.hu.json"),
|
|
140
|
+
it: () => import("@stll/anonymize-data/config/triggers.it.json"),
|
|
141
|
+
pl: () => import("@stll/anonymize-data/config/triggers.pl.json"),
|
|
142
|
+
ro: () => import("@stll/anonymize-data/config/triggers.ro.json"),
|
|
143
|
+
sv: () => import("@stll/anonymize-data/config/triggers.sv.json")
|
|
144
|
+
},
|
|
145
|
+
coreference: {
|
|
146
|
+
cs: () => import("@stll/anonymize-data/config/coreference.cs.json"),
|
|
147
|
+
de: () => import("@stll/anonymize-data/config/coreference.de.json"),
|
|
148
|
+
en: () => import("@stll/anonymize-data/config/coreference.en.json"),
|
|
149
|
+
sk: () => import("@stll/anonymize-data/config/coreference.sk.json")
|
|
150
|
+
}
|
|
151
|
+
};
|
|
152
|
+
const FALLBACK_LANGUAGES = {
|
|
153
|
+
triggers: [
|
|
154
|
+
"cs",
|
|
155
|
+
"de",
|
|
156
|
+
"en",
|
|
157
|
+
"es",
|
|
158
|
+
"fr",
|
|
159
|
+
"hu",
|
|
160
|
+
"it",
|
|
161
|
+
"pl",
|
|
162
|
+
"ro",
|
|
163
|
+
"sv"
|
|
164
|
+
],
|
|
165
|
+
coreference: [
|
|
166
|
+
"cs",
|
|
167
|
+
"de",
|
|
168
|
+
"en",
|
|
169
|
+
"sk"
|
|
170
|
+
]
|
|
171
|
+
};
|
|
172
|
+
/**
|
|
173
|
+
* Load all config files of a given type for all
|
|
174
|
+
* languages enabled in the manifest.
|
|
175
|
+
*
|
|
176
|
+
* Falls back to the hardcoded language list when the
|
|
177
|
+
* manifest is unavailable (backward compatibility).
|
|
178
|
+
*/
|
|
179
|
+
const loadLanguageConfigs = async (configType, mapFn) => {
|
|
180
|
+
const manifest = await loadManifest();
|
|
181
|
+
const registry = LOADER_REGISTRIES[configType];
|
|
182
|
+
const codes = Object.keys(manifest.languages).length > 0 ? Object.entries(manifest.languages).filter(([code, lang]) => {
|
|
183
|
+
if (!lang || typeof lang !== "object") {
|
|
184
|
+
console.warn(`[anonymize] lang-loader: manifest entry for "${code}" is not an object, skipping`);
|
|
185
|
+
return false;
|
|
186
|
+
}
|
|
187
|
+
return lang[configType] === true;
|
|
188
|
+
}).map(([code]) => code) : [...FALLBACK_LANGUAGES[configType]];
|
|
189
|
+
const results = new Array(codes.length);
|
|
190
|
+
const loads = codes.map(async (code, i) => {
|
|
191
|
+
const loader = registry[code];
|
|
192
|
+
if (!loader) {
|
|
193
|
+
console.warn(`[anonymize] lang-loader: language "${code}" is enabled in the manifest for "${configType}" but has no loader in the static registry`);
|
|
194
|
+
return;
|
|
195
|
+
}
|
|
196
|
+
let mod;
|
|
197
|
+
try {
|
|
198
|
+
mod = await loader();
|
|
199
|
+
} catch (err) {
|
|
200
|
+
console.warn(`[anonymize] lang-loader: failed to import "${configType}" config for "${code}":`, err);
|
|
201
|
+
return;
|
|
202
|
+
}
|
|
203
|
+
let result;
|
|
204
|
+
try {
|
|
205
|
+
result = mapFn(mod);
|
|
206
|
+
} catch (err) {
|
|
207
|
+
console.warn(`[anonymize] lang-loader: mapFn failed for "${code}" (${configType}):`, err);
|
|
208
|
+
return;
|
|
209
|
+
}
|
|
210
|
+
results[i] = result;
|
|
211
|
+
});
|
|
212
|
+
await Promise.all(loads);
|
|
213
|
+
return results.filter((r) => r !== void 0);
|
|
214
|
+
};
|
|
215
|
+
//#endregion
|
|
216
|
+
//#region src/detectors/coreference.ts
|
|
217
|
+
/**
|
|
218
|
+
* Load coreference definition patterns from per-language
|
|
219
|
+
* JSON configs in @stll/anonymize-data. Uses the
|
|
220
|
+
* language manifest for auto-discovery.
|
|
221
|
+
*/
|
|
222
|
+
const loadDefinitionPatterns = async () => {
|
|
223
|
+
const patterns = [];
|
|
224
|
+
const allRows = await loadLanguageConfigs("coreference", (mod) => {
|
|
225
|
+
return mod.default ?? mod;
|
|
226
|
+
});
|
|
227
|
+
for (const rows of allRows) {
|
|
228
|
+
if (!Array.isArray(rows)) {
|
|
229
|
+
console.warn("[anonymize] coreference: unexpected config shape, skipping");
|
|
230
|
+
continue;
|
|
231
|
+
}
|
|
232
|
+
for (const row of rows) try {
|
|
233
|
+
patterns.push({ pattern: new RegExp(row.pattern, row.flags) });
|
|
234
|
+
} catch (err) {
|
|
235
|
+
console.warn(`[anonymize] coreference: invalid regex "${row.pattern}":`, err);
|
|
236
|
+
}
|
|
237
|
+
}
|
|
238
|
+
return patterns;
|
|
239
|
+
};
|
|
240
|
+
/**
|
|
241
|
+
* Load generic role terms that should NOT be treated
|
|
242
|
+
* as PII coreferences. "Prodávající" (Seller),
|
|
243
|
+
* "Kupující" (Buyer), etc. are legal roles, not
|
|
244
|
+
* identifying information.
|
|
245
|
+
*/
|
|
246
|
+
const getRoleStopSet = async (ctx) => {
|
|
247
|
+
if (ctx.roleStopSet) return ctx.roleStopSet;
|
|
248
|
+
if (ctx.roleStopSetPromise) return ctx.roleStopSetPromise;
|
|
249
|
+
const promise = (async () => {
|
|
250
|
+
let result;
|
|
251
|
+
try {
|
|
252
|
+
const mod = await import("@stll/anonymize-data/config/generic-roles.json");
|
|
253
|
+
const data = mod.default ?? mod;
|
|
254
|
+
result = new Set(data.roles.map((r) => r.toLowerCase()));
|
|
255
|
+
} catch {
|
|
256
|
+
result = /* @__PURE__ */ new Set();
|
|
257
|
+
}
|
|
258
|
+
ctx.roleStopSet = result;
|
|
259
|
+
return result;
|
|
260
|
+
})();
|
|
261
|
+
ctx.roleStopSetPromise = promise;
|
|
262
|
+
return promise;
|
|
263
|
+
};
|
|
264
|
+
const getDefinitionPatterns = async (ctx) => {
|
|
265
|
+
if (ctx.corefPatterns) return ctx.corefPatterns;
|
|
266
|
+
if (ctx.corefPatternsPromise) return ctx.corefPatternsPromise;
|
|
267
|
+
ctx.corefPatternsPromise = loadDefinitionPatterns();
|
|
268
|
+
const patterns = await ctx.corefPatternsPromise;
|
|
269
|
+
if (patterns.length === 0) {
|
|
270
|
+
ctx.corefPatterns = patterns;
|
|
271
|
+
if (!ctx.corefLoadAttempted) {
|
|
272
|
+
ctx.corefLoadAttempted = true;
|
|
273
|
+
console.warn("[anonymize] coreference: no definition patterns loaded; coreference detection will be inactive");
|
|
274
|
+
}
|
|
275
|
+
return patterns;
|
|
276
|
+
}
|
|
277
|
+
ctx.corefPatterns = patterns;
|
|
278
|
+
return patterns;
|
|
279
|
+
};
|
|
280
|
+
const SEARCH_WINDOW = 200;
|
|
281
|
+
/**
|
|
282
|
+
* Scan for defined-term patterns near known entities.
|
|
283
|
+
*
|
|
284
|
+
* Legal documents universally follow:
|
|
285
|
+
* "Dr. Heinrich Muller (hereinafter 'the Seller')..."
|
|
286
|
+
*
|
|
287
|
+
* After NER detects the entity, this function scans for
|
|
288
|
+
* definitional patterns within +/-200 chars and extracts
|
|
289
|
+
* the alias. Returns alias + label pairs that can be added
|
|
290
|
+
* to the gazetteer for a full-text re-scan.
|
|
291
|
+
*/
|
|
292
|
+
/**
|
|
293
|
+
* Labels that can be the source of a coreference alias.
|
|
294
|
+
* Only parties (person, organization) have defined-term
|
|
295
|
+
* aliases in legal text. Dates, addresses, IDs do not.
|
|
296
|
+
*/
|
|
297
|
+
const COREF_SOURCE_LABELS = new Set(["person", "organization"]);
|
|
298
|
+
const extractDefinedTerms = async (fullText, entities, ctx = defaultContext) => {
|
|
299
|
+
const [definitionPatterns, roleStops] = await Promise.all([getDefinitionPatterns(ctx), getRoleStopSet(ctx)]);
|
|
300
|
+
const terms = [];
|
|
301
|
+
const seen = /* @__PURE__ */ new Set();
|
|
302
|
+
const sorted = [...entities].sort((a, b) => a.start - b.start);
|
|
303
|
+
for (const { pattern } of definitionPatterns) {
|
|
304
|
+
pattern.lastIndex = 0;
|
|
305
|
+
for (let match = pattern.exec(fullText); match !== null; match = pattern.exec(fullText)) {
|
|
306
|
+
const alias = match[1]?.trim();
|
|
307
|
+
if (!alias || alias.length < 2) continue;
|
|
308
|
+
if (roleStops.has(alias.toLowerCase())) continue;
|
|
309
|
+
const defPos = match.index;
|
|
310
|
+
let bestEntity = null;
|
|
311
|
+
for (let i = sorted.length - 1; i >= 0; i--) {
|
|
312
|
+
const e = sorted[i];
|
|
313
|
+
if (e === void 0) continue;
|
|
314
|
+
if (e.end > defPos) continue;
|
|
315
|
+
if (defPos - e.end > SEARCH_WINDOW) break;
|
|
316
|
+
if (!COREF_SOURCE_LABELS.has(e.label)) continue;
|
|
317
|
+
bestEntity = e;
|
|
318
|
+
break;
|
|
319
|
+
}
|
|
320
|
+
if (bestEntity === null) continue;
|
|
321
|
+
const key = `${alias.toLowerCase()}::${bestEntity.label}`;
|
|
322
|
+
if (seen.has(key)) continue;
|
|
323
|
+
seen.add(key);
|
|
324
|
+
terms.push({
|
|
325
|
+
alias,
|
|
326
|
+
label: bestEntity.label,
|
|
327
|
+
definitionStart: defPos,
|
|
328
|
+
sourceText: bestEntity.text
|
|
329
|
+
});
|
|
330
|
+
}
|
|
331
|
+
}
|
|
332
|
+
return terms;
|
|
333
|
+
};
|
|
334
|
+
/**
|
|
335
|
+
* Check if a character is a Unicode word character
|
|
336
|
+
* (letter, digit, or combining mark). Used for word
|
|
337
|
+
* boundary checks in coreference matching.
|
|
338
|
+
*/
|
|
339
|
+
const isWordChar = (ch) => {
|
|
340
|
+
if (ch === void 0) return false;
|
|
341
|
+
return /[\p{L}\p{M}\p{N}]/u.test(ch);
|
|
342
|
+
};
|
|
343
|
+
/**
|
|
344
|
+
* Find all occurrences of defined-term aliases in the
|
|
345
|
+
* full text. Returns Entity spans for each match.
|
|
346
|
+
*
|
|
347
|
+
* Respects word boundaries: "Kupující" must not match
|
|
348
|
+
* inside "Kupujícímu". A match is valid only if the
|
|
349
|
+
* character before the start and after the end are NOT
|
|
350
|
+
* word characters (letter/digit).
|
|
351
|
+
*
|
|
352
|
+
* Populates `ctx.corefSourceMap` with entries linking
|
|
353
|
+
* each coref entity to its source entity text, for
|
|
354
|
+
* consistent placeholder numbering.
|
|
355
|
+
*/
|
|
356
|
+
const findCoreferenceSpans = (fullText, terms, ctx = defaultContext) => {
|
|
357
|
+
const results = [];
|
|
358
|
+
for (const term of terms) {
|
|
359
|
+
let searchFrom = 0;
|
|
360
|
+
while (searchFrom < fullText.length) {
|
|
361
|
+
const idx = fullText.indexOf(term.alias, searchFrom);
|
|
362
|
+
if (idx === -1) break;
|
|
363
|
+
const matchEnd = idx + term.alias.length;
|
|
364
|
+
const charBefore = idx > 0 ? fullText[idx - 1] : void 0;
|
|
365
|
+
const charAfter = fullText[matchEnd];
|
|
366
|
+
if (isWordChar(charBefore) || isWordChar(charAfter)) {
|
|
367
|
+
searchFrom = idx + 1;
|
|
368
|
+
continue;
|
|
369
|
+
}
|
|
370
|
+
const entity = {
|
|
371
|
+
start: idx,
|
|
372
|
+
end: matchEnd,
|
|
373
|
+
label: term.label,
|
|
374
|
+
text: term.alias,
|
|
375
|
+
score: .95,
|
|
376
|
+
source: DETECTION_SOURCES.COREFERENCE
|
|
377
|
+
};
|
|
378
|
+
ctx.corefSourceMap.set(corefKey(entity), term.sourceText);
|
|
379
|
+
results.push(entity);
|
|
380
|
+
searchFrom = matchEnd;
|
|
381
|
+
}
|
|
382
|
+
}
|
|
383
|
+
return results;
|
|
384
|
+
};
|
|
385
|
+
//#endregion
|
|
386
|
+
//#region src/detectors/gazetteer.ts
|
|
387
|
+
const MAX_EDIT_DISTANCE = 2;
|
|
388
|
+
const MIN_FUZZY_LENGTH = 4;
|
|
389
|
+
const MAX_PREFIX_OVERSHOOT = 7;
|
|
390
|
+
/**
|
|
391
|
+
* Collect all searchable strings (canonical + variants)
|
|
392
|
+
* from gazetteer entries, mapped to their labels and
|
|
393
|
+
* entry IDs.
|
|
394
|
+
*/
|
|
395
|
+
const buildSearchTerms = (entries) => {
|
|
396
|
+
const terms = /* @__PURE__ */ new Map();
|
|
397
|
+
for (const entry of entries) {
|
|
398
|
+
const meta = {
|
|
399
|
+
label: entry.label,
|
|
400
|
+
entryId: entry.id
|
|
401
|
+
};
|
|
402
|
+
terms.set(entry.canonical, meta);
|
|
403
|
+
for (const variant of entry.variants) terms.set(variant, meta);
|
|
404
|
+
}
|
|
405
|
+
return terms;
|
|
406
|
+
};
|
|
407
|
+
/**
|
|
408
|
+
* Build TextSearch-compatible patterns from gazetteer
|
|
409
|
+
* entries. Returns:
|
|
410
|
+
* - Exact literal patterns for all terms
|
|
411
|
+
* - Fuzzy patterns (distance: 2) for terms >= 4 chars
|
|
412
|
+
* - Parallel metadata arrays for post-processing
|
|
413
|
+
*
|
|
414
|
+
* Patterns are ordered: all exact first, then all
|
|
415
|
+
* fuzzy. The isFuzzy array marks which are which.
|
|
416
|
+
*/
|
|
417
|
+
const buildGazetteerPatterns = (entries) => {
|
|
418
|
+
const terms = buildSearchTerms(entries);
|
|
419
|
+
const patterns = [];
|
|
420
|
+
const labels = [];
|
|
421
|
+
const isFuzzy = [];
|
|
422
|
+
for (const [term, meta] of terms) {
|
|
423
|
+
patterns.push({
|
|
424
|
+
pattern: term,
|
|
425
|
+
literal: true,
|
|
426
|
+
wholeWords: false
|
|
427
|
+
});
|
|
428
|
+
labels.push(meta.label);
|
|
429
|
+
isFuzzy.push(false);
|
|
430
|
+
}
|
|
431
|
+
for (const [term, meta] of terms) {
|
|
432
|
+
if (term.length < MIN_FUZZY_LENGTH) continue;
|
|
433
|
+
patterns.push({
|
|
434
|
+
pattern: term,
|
|
435
|
+
distance: MAX_EDIT_DISTANCE
|
|
436
|
+
});
|
|
437
|
+
labels.push(meta.label);
|
|
438
|
+
isFuzzy.push(true);
|
|
439
|
+
}
|
|
440
|
+
return {
|
|
441
|
+
patterns,
|
|
442
|
+
data: {
|
|
443
|
+
labels,
|
|
444
|
+
isFuzzy
|
|
445
|
+
}
|
|
446
|
+
};
|
|
447
|
+
};
|
|
448
|
+
/**
|
|
449
|
+
* Process gazetteer matches from the unified literal
|
|
450
|
+
* search. Receives all matches; filters to the
|
|
451
|
+
* gazetteer slice via sliceStart/sliceEnd.
|
|
452
|
+
*
|
|
453
|
+
* Exact matches get score 0.9; fuzzy matches get
|
|
454
|
+
* 0.85. Fuzzy matches that overlap an exact match
|
|
455
|
+
* are dropped.
|
|
456
|
+
*
|
|
457
|
+
* For exact matches, attempts prefix extension for
|
|
458
|
+
* legal suffixes ("a.s.", "GmbH", "s.r.o." after
|
|
459
|
+
* the matched term).
|
|
460
|
+
*/
|
|
461
|
+
const processGazetteerMatches = (allMatches, sliceStart, sliceEnd, fullText, data) => {
|
|
462
|
+
const results = [];
|
|
463
|
+
const exactSpans = [];
|
|
464
|
+
for (const match of allMatches) {
|
|
465
|
+
const idx = match.pattern;
|
|
466
|
+
if (idx < sliceStart || idx >= sliceEnd) continue;
|
|
467
|
+
const localIdx = idx - sliceStart;
|
|
468
|
+
if (data.isFuzzy[localIdx]) continue;
|
|
469
|
+
const label = data.labels[localIdx];
|
|
470
|
+
if (!label) continue;
|
|
471
|
+
const extended = tryPrefixExtension(fullText, match.start, match.end);
|
|
472
|
+
const end = extended?.end ?? match.end;
|
|
473
|
+
const text = extended?.text ?? fullText.slice(match.start, match.end);
|
|
474
|
+
exactSpans.push({
|
|
475
|
+
start: match.start,
|
|
476
|
+
end
|
|
477
|
+
});
|
|
478
|
+
results.push({
|
|
479
|
+
start: match.start,
|
|
480
|
+
end,
|
|
481
|
+
label,
|
|
482
|
+
text,
|
|
483
|
+
score: .9,
|
|
484
|
+
source: DETECTION_SOURCES.GAZETTEER
|
|
485
|
+
});
|
|
486
|
+
}
|
|
487
|
+
for (const match of allMatches) {
|
|
488
|
+
const idx = match.pattern;
|
|
489
|
+
if (idx < sliceStart || idx >= sliceEnd) continue;
|
|
490
|
+
const localIdx = idx - sliceStart;
|
|
491
|
+
if (!data.isFuzzy[localIdx]) continue;
|
|
492
|
+
if (match.distance === 0) continue;
|
|
493
|
+
const label = data.labels[localIdx];
|
|
494
|
+
if (!label) continue;
|
|
495
|
+
if (exactSpans.some((e) => match.start < e.end && match.end > e.start)) continue;
|
|
496
|
+
const matchText = fullText.slice(match.start, match.end);
|
|
497
|
+
results.push({
|
|
498
|
+
start: match.start,
|
|
499
|
+
end: match.end,
|
|
500
|
+
label,
|
|
501
|
+
text: matchText,
|
|
502
|
+
score: .85,
|
|
503
|
+
source: DETECTION_SOURCES.GAZETTEER
|
|
504
|
+
});
|
|
505
|
+
}
|
|
506
|
+
return results;
|
|
507
|
+
};
|
|
508
|
+
/**
|
|
509
|
+
* Try to extend an exact match to capture one
|
|
510
|
+
* trailing token (max 6 chars) that may be a legal
|
|
511
|
+
* entity suffix (e.g., "a.s.", "GmbH", "s.r.o.").
|
|
512
|
+
*
|
|
513
|
+
* Does not validate the token against a legal-forms
|
|
514
|
+
* list; false extensions are filtered by mergeAndDedup
|
|
515
|
+
* when a legal-form detector produces a competing
|
|
516
|
+
* entity with the correct span.
|
|
517
|
+
*/
|
|
518
|
+
const tryPrefixExtension = (fullText, start, end) => {
|
|
519
|
+
const maxEnd = Math.min(end + MAX_PREFIX_OVERSHOOT, fullText.length);
|
|
520
|
+
if (maxEnd <= end + 1) return null;
|
|
521
|
+
const after = fullText.slice(end, maxEnd);
|
|
522
|
+
if (!after.startsWith(" ")) return null;
|
|
523
|
+
const nextSpace = after.indexOf(" ", 1);
|
|
524
|
+
const suffixEnd = nextSpace !== -1 ? nextSpace : after.length;
|
|
525
|
+
if (suffixEnd <= 1) return null;
|
|
526
|
+
const newEnd = end + suffixEnd;
|
|
527
|
+
return {
|
|
528
|
+
end: newEnd,
|
|
529
|
+
text: fullText.slice(start, newEnd)
|
|
530
|
+
};
|
|
531
|
+
};
|
|
532
|
+
//#endregion
|
|
533
|
+
//#region src/util/text.ts
|
|
534
|
+
/**
|
|
535
|
+
* Shared text utilities for detectors.
|
|
536
|
+
*
|
|
537
|
+
* Extracted from names.ts and deny-list.ts to avoid
|
|
538
|
+
* duplicating regex constants and helper functions.
|
|
539
|
+
*/
|
|
540
|
+
/** Matches a string that starts with an uppercase letter. */
|
|
541
|
+
const UPPER_START_RE = /^\p{Lu}/u;
|
|
542
|
+
/** Matches a string consisting entirely of uppercase letters. */
|
|
543
|
+
const ALL_UPPER_RE = /^\p{Lu}+$/u;
|
|
544
|
+
const SENTENCE_END_RE = /[.!?]/;
|
|
545
|
+
/**
|
|
546
|
+
* Detect whether a position is at the start of a sentence.
|
|
547
|
+
* Looks backward past whitespace for sentence-ending
|
|
548
|
+
* punctuation (.!?). Position 0 and positions preceded
|
|
549
|
+
* only by whitespace are considered sentence starts.
|
|
550
|
+
*/
|
|
551
|
+
const isSentenceStart = (text, pos) => {
|
|
552
|
+
if (pos === 0) return true;
|
|
553
|
+
let i = pos - 1;
|
|
554
|
+
while (i >= 0 && /\s/.test(text[i] ?? "")) i--;
|
|
555
|
+
if (i < 0) return true;
|
|
556
|
+
return SENTENCE_END_RE.test(text[i] ?? "");
|
|
557
|
+
};
|
|
558
|
+
//#endregion
|
|
559
|
+
//#region src/detectors/names.ts
|
|
560
|
+
const getCorpus = (ctx) => ctx.nameCorpus;
|
|
561
|
+
const getNameCorpusFirstNames = (ctx = defaultContext) => ctx.nameCorpus?.firstNamesList ?? [];
|
|
562
|
+
const getNameCorpusSurnames = (ctx = defaultContext) => ctx.nameCorpus?.surnamesList ?? [];
|
|
563
|
+
const getNameCorpusTitles = (ctx = defaultContext) => ctx.nameCorpus?.titlesList ?? [];
|
|
564
|
+
/**
|
|
565
|
+
* Load name corpus data from JSON config files.
|
|
566
|
+
* Safe to call multiple times; only loads once per
|
|
567
|
+
* context. Must be called before detectNameCorpus or
|
|
568
|
+
* the getNameCorpus*() accessors are used.
|
|
569
|
+
*/
|
|
570
|
+
const initNameCorpus = (ctx = defaultContext) => {
|
|
571
|
+
if (ctx.nameCorpusPromise) return ctx.nameCorpusPromise;
|
|
572
|
+
const promise = (async () => {
|
|
573
|
+
try {
|
|
574
|
+
const [firstMod, surnameMod, titleMod, exclusionMod] = await Promise.all([
|
|
575
|
+
import("@stll/anonymize-data/config/names-first.json"),
|
|
576
|
+
import("@stll/anonymize-data/config/names-surnames.json"),
|
|
577
|
+
import("@stll/anonymize-data/config/names-title-tokens.json"),
|
|
578
|
+
import("@stll/anonymize-data/config/names-exclusions.json")
|
|
579
|
+
]);
|
|
580
|
+
const firstNames = firstMod.default.names;
|
|
581
|
+
const surnames = surnameMod.default.names;
|
|
582
|
+
const titles = titleMod.default.tokens;
|
|
583
|
+
const exclusions = exclusionMod.default.words;
|
|
584
|
+
ctx.nameCorpus = {
|
|
585
|
+
firstNames: Object.freeze(new Set(firstNames)),
|
|
586
|
+
surnames: Object.freeze(new Set(surnames)),
|
|
587
|
+
titleTokens: Object.freeze(new Set(titles)),
|
|
588
|
+
excludedWords: Object.freeze(new Set(exclusions)),
|
|
589
|
+
firstNamesList: Object.freeze(firstNames),
|
|
590
|
+
surnamesList: Object.freeze(surnames),
|
|
591
|
+
titlesList: Object.freeze(titles),
|
|
592
|
+
excludedList: Object.freeze(exclusions)
|
|
593
|
+
};
|
|
594
|
+
} catch (err) {
|
|
595
|
+
ctx.nameCorpusPromise = null;
|
|
596
|
+
console.warn("[anonymize] Failed to load name corpus JSON — name detection disabled:", err);
|
|
597
|
+
}
|
|
598
|
+
})();
|
|
599
|
+
ctx.nameCorpusPromise = promise;
|
|
600
|
+
return promise;
|
|
601
|
+
};
|
|
602
|
+
const INFLECTION_SUFFIXES = [
|
|
603
|
+
"ovi",
|
|
604
|
+
"em",
|
|
605
|
+
"om",
|
|
606
|
+
"ou",
|
|
607
|
+
"é",
|
|
608
|
+
"a",
|
|
609
|
+
"u"
|
|
610
|
+
];
|
|
611
|
+
/**
|
|
612
|
+
* Strip common Czech/Slovak case suffixes from a token.
|
|
613
|
+
* Returns candidate base forms if stripping produces a
|
|
614
|
+
* plausible name (capitalised, length >= 3).
|
|
615
|
+
*
|
|
616
|
+
* For the "-ou" instrumental feminine suffix, also yields
|
|
617
|
+
* base + "a" (e.g., "Editou" → "Edit" and "Edita")
|
|
618
|
+
* because Czech feminine names decline -a → -ou.
|
|
619
|
+
*/
|
|
620
|
+
const stripInflection = (token) => {
|
|
621
|
+
const candidates = [];
|
|
622
|
+
for (const suffix of INFLECTION_SUFFIXES) if (token.length > suffix.length + 2 && token.endsWith(suffix)) {
|
|
623
|
+
const base = token.slice(0, -suffix.length);
|
|
624
|
+
if (/^\p{Lu}/u.test(base)) {
|
|
625
|
+
candidates.push(base);
|
|
626
|
+
if (suffix === "ou" || suffix === "é" || suffix === "u") candidates.push(`${base}a`);
|
|
627
|
+
}
|
|
628
|
+
}
|
|
629
|
+
return candidates;
|
|
630
|
+
};
|
|
631
|
+
const TOKEN_TYPE = {
|
|
632
|
+
NAME: "name",
|
|
633
|
+
SURNAME: "surname",
|
|
634
|
+
TITLE: "title",
|
|
635
|
+
ABBREVIATION: "abbreviation",
|
|
636
|
+
CAPITALIZED: "capitalized",
|
|
637
|
+
OTHER: "other"
|
|
638
|
+
};
|
|
639
|
+
/**
|
|
640
|
+
* Check if a token is in the first-name set, either
|
|
641
|
+
* directly or after stripping Czech/Slovak inflection.
|
|
642
|
+
*/
|
|
643
|
+
const isFirstNameToken = (token, corpus) => {
|
|
644
|
+
if (corpus.firstNames.has(token)) return true;
|
|
645
|
+
return stripInflection(token).some((b) => corpus.firstNames.has(b));
|
|
646
|
+
};
|
|
647
|
+
/**
|
|
648
|
+
* Check if a token is in the surname set, either
|
|
649
|
+
* directly or after stripping Czech/Slovak inflection.
|
|
650
|
+
*/
|
|
651
|
+
const isSurnameToken = (token, corpus) => {
|
|
652
|
+
if (corpus.surnames.has(token)) return true;
|
|
653
|
+
return stripInflection(token).some((b) => corpus.surnames.has(b));
|
|
654
|
+
};
|
|
655
|
+
/**
|
|
656
|
+
* Check if a token looks like a single-letter
|
|
657
|
+
* abbreviation: "J.", "M.", etc.
|
|
658
|
+
*/
|
|
659
|
+
const isAbbreviation = (token) => token.length === 2 && /^\p{Lu}$/u.test(token[0] ?? "") && token[1] === ".";
|
|
660
|
+
const segmenter$1 = new Intl.Segmenter(void 0, { granularity: "word" });
|
|
661
|
+
/**
|
|
662
|
+
* Split text into word segments using Intl.Segmenter.
|
|
663
|
+
* Only returns segments flagged as words.
|
|
664
|
+
*/
|
|
665
|
+
const segmentWords = (fullText) => {
|
|
666
|
+
const words = [];
|
|
667
|
+
for (const seg of segmenter$1.segment(fullText)) if (seg.isWordLike) words.push({
|
|
668
|
+
text: seg.segment,
|
|
669
|
+
start: seg.index,
|
|
670
|
+
end: seg.index + seg.segment.length
|
|
671
|
+
});
|
|
672
|
+
return words;
|
|
673
|
+
};
|
|
674
|
+
/** NAME or SURNAME — both represent corpus-matched tokens */
|
|
675
|
+
const isCorpusMatch = (type) => type === TOKEN_TYPE.NAME || type === TOKEN_TYPE.SURNAME;
|
|
676
|
+
const classifyToken = (word, corpus) => {
|
|
677
|
+
const { text, start, end } = word;
|
|
678
|
+
const lower = text.toLowerCase();
|
|
679
|
+
const stripped = text.endsWith(".") ? text.slice(0, -1).toLowerCase() : lower;
|
|
680
|
+
if (corpus.titleTokens.has(stripped)) return {
|
|
681
|
+
text,
|
|
682
|
+
type: TOKEN_TYPE.TITLE,
|
|
683
|
+
start,
|
|
684
|
+
end
|
|
685
|
+
};
|
|
686
|
+
if (isAbbreviation(text)) return {
|
|
687
|
+
text,
|
|
688
|
+
type: TOKEN_TYPE.ABBREVIATION,
|
|
689
|
+
start,
|
|
690
|
+
end
|
|
691
|
+
};
|
|
692
|
+
if (corpus.excludedWords.has(lower)) return {
|
|
693
|
+
text,
|
|
694
|
+
type: TOKEN_TYPE.OTHER,
|
|
695
|
+
start,
|
|
696
|
+
end
|
|
697
|
+
};
|
|
698
|
+
if (text.length < 3) return {
|
|
699
|
+
text,
|
|
700
|
+
type: TOKEN_TYPE.OTHER,
|
|
701
|
+
start,
|
|
702
|
+
end
|
|
703
|
+
};
|
|
704
|
+
if (text.length > 3 && ALL_UPPER_RE.test(text)) return {
|
|
705
|
+
text,
|
|
706
|
+
type: TOKEN_TYPE.OTHER,
|
|
707
|
+
start,
|
|
708
|
+
end
|
|
709
|
+
};
|
|
710
|
+
if (!UPPER_START_RE.test(text)) return {
|
|
711
|
+
text,
|
|
712
|
+
type: TOKEN_TYPE.OTHER,
|
|
713
|
+
start,
|
|
714
|
+
end
|
|
715
|
+
};
|
|
716
|
+
if (isFirstNameToken(text, corpus)) return {
|
|
717
|
+
text,
|
|
718
|
+
type: TOKEN_TYPE.NAME,
|
|
719
|
+
start,
|
|
720
|
+
end
|
|
721
|
+
};
|
|
722
|
+
if (isSurnameToken(text, corpus)) return {
|
|
723
|
+
text,
|
|
724
|
+
type: TOKEN_TYPE.SURNAME,
|
|
725
|
+
start,
|
|
726
|
+
end
|
|
727
|
+
};
|
|
728
|
+
return {
|
|
729
|
+
text,
|
|
730
|
+
type: TOKEN_TYPE.CAPITALIZED,
|
|
731
|
+
start,
|
|
732
|
+
end
|
|
733
|
+
};
|
|
734
|
+
};
|
|
735
|
+
/**
|
|
736
|
+
* Detect person names by looking up tokens against the
|
|
737
|
+
* name corpus, then chaining adjacent name-like tokens.
|
|
738
|
+
*
|
|
739
|
+
* Requires initNameCorpus() to have been called first.
|
|
740
|
+
* If not initialized, returns an empty array.
|
|
741
|
+
*
|
|
742
|
+
* Scoring:
|
|
743
|
+
* TITLE + NAME/SURNAME → 0.95
|
|
744
|
+
* NAME + NAME/SURNAME → 0.9
|
|
745
|
+
* SURNAME + NAME/SURNAME → 0.9
|
|
746
|
+
* NAME + CAPITALIZED → 0.7
|
|
747
|
+
* ABBREVIATION + NAME → 0.7
|
|
748
|
+
* Standalone NAME → 0.5 (low confidence)
|
|
749
|
+
* Standalone SURNAME → skip (too ambiguous)
|
|
750
|
+
*/
|
|
751
|
+
const detectNameCorpus = (fullText, ctx = defaultContext) => {
|
|
752
|
+
const corpus = getCorpus(ctx);
|
|
753
|
+
if (!corpus) return [];
|
|
754
|
+
const tokens = segmentWords(fullText).map((w) => classifyToken(w, corpus));
|
|
755
|
+
const entities = [];
|
|
756
|
+
const consumed = /* @__PURE__ */ new Set();
|
|
757
|
+
for (let i = 0; i < tokens.length; i++) {
|
|
758
|
+
if (consumed.has(i)) continue;
|
|
759
|
+
const token = tokens[i];
|
|
760
|
+
if (!token) continue;
|
|
761
|
+
if (token.type !== TOKEN_TYPE.TITLE && token.type !== TOKEN_TYPE.NAME && token.type !== TOKEN_TYPE.SURNAME && token.type !== TOKEN_TYPE.ABBREVIATION) continue;
|
|
762
|
+
const MAX_CHAIN = 5;
|
|
763
|
+
const chain = [token];
|
|
764
|
+
let j = i + 1;
|
|
765
|
+
while (j < tokens.length && chain.length < MAX_CHAIN) {
|
|
766
|
+
const next = tokens[j];
|
|
767
|
+
if (!next) break;
|
|
768
|
+
const prev = chain.at(-1);
|
|
769
|
+
if (prev) {
|
|
770
|
+
if (fullText.slice(prev.end, next.start).includes("\n")) break;
|
|
771
|
+
}
|
|
772
|
+
if (next.type === TOKEN_TYPE.NAME || next.type === TOKEN_TYPE.SURNAME || next.type === TOKEN_TYPE.TITLE || next.type === TOKEN_TYPE.ABBREVIATION || next.type === TOKEN_TYPE.CAPITALIZED) {
|
|
773
|
+
chain.push(next);
|
|
774
|
+
j++;
|
|
775
|
+
} else break;
|
|
776
|
+
}
|
|
777
|
+
const hasTitle = chain.some((t) => t.type === TOKEN_TYPE.TITLE);
|
|
778
|
+
const hasCorpusName = chain.some((t) => isCorpusMatch(t.type));
|
|
779
|
+
const hasFirstName = chain.some((t) => t.type === TOKEN_TYPE.NAME);
|
|
780
|
+
const hasAbbreviation = chain.some((t) => t.type === TOKEN_TYPE.ABBREVIATION);
|
|
781
|
+
const corpusCount = chain.filter((t) => isCorpusMatch(t.type)).length;
|
|
782
|
+
const capitalizedCount = chain.filter((t) => t.type === TOKEN_TYPE.CAPITALIZED).length;
|
|
783
|
+
let score = 0;
|
|
784
|
+
if (hasTitle && hasCorpusName) score = .95;
|
|
785
|
+
else if (corpusCount >= 2) score = .9;
|
|
786
|
+
else if (hasCorpusName && capitalizedCount > 0) score = .7;
|
|
787
|
+
else if (hasAbbreviation && hasCorpusName) score = .7;
|
|
788
|
+
else if (hasFirstName && chain.length === 1) {
|
|
789
|
+
if (isSentenceStart(fullText, token.start)) continue;
|
|
790
|
+
score = .5;
|
|
791
|
+
} else if (!hasFirstName && chain.length === 1 && chain[0]?.type === TOKEN_TYPE.SURNAME) continue;
|
|
792
|
+
else if (hasTitle && chain.length === 1) continue;
|
|
793
|
+
else {
|
|
794
|
+
if (!hasCorpusName) continue;
|
|
795
|
+
score = .5;
|
|
796
|
+
}
|
|
797
|
+
const first = chain.at(0);
|
|
798
|
+
const last = chain.at(-1);
|
|
799
|
+
if (!first || !last) continue;
|
|
800
|
+
const start = first.start;
|
|
801
|
+
const end = last.end;
|
|
802
|
+
const text = fullText.slice(start, end);
|
|
803
|
+
for (let k = i; k < i + chain.length; k++) consumed.add(k);
|
|
804
|
+
entities.push({
|
|
805
|
+
start,
|
|
806
|
+
end,
|
|
807
|
+
label: "person",
|
|
808
|
+
text,
|
|
809
|
+
score,
|
|
810
|
+
source: DETECTION_SOURCES.REGEX
|
|
811
|
+
});
|
|
812
|
+
}
|
|
813
|
+
return entities;
|
|
814
|
+
};
|
|
815
|
+
//#endregion
|
|
816
|
+
//#region src/config/titles.ts
|
|
817
|
+
/**
|
|
818
|
+
* Academic and professional title prefixes.
|
|
819
|
+
* Plain text; the detector auto-escapes for regex.
|
|
820
|
+
* Sorted longest-first at build time.
|
|
821
|
+
*/
|
|
822
|
+
const TITLE_PREFIXES = [
|
|
823
|
+
"Ing.",
|
|
824
|
+
"Mgr.",
|
|
825
|
+
"MgA.",
|
|
826
|
+
"Bc.",
|
|
827
|
+
"BcA.",
|
|
828
|
+
"JUDr.",
|
|
829
|
+
"MUDr.",
|
|
830
|
+
"MVDr.",
|
|
831
|
+
"MDDr.",
|
|
832
|
+
"PhDr.",
|
|
833
|
+
"RNDr.",
|
|
834
|
+
"PaedDr.",
|
|
835
|
+
"ThDr.",
|
|
836
|
+
"ThLic.",
|
|
837
|
+
"ICDr.",
|
|
838
|
+
"RSDr.",
|
|
839
|
+
"PharmDr.",
|
|
840
|
+
"artD.",
|
|
841
|
+
"akad.",
|
|
842
|
+
"doc.",
|
|
843
|
+
"prof.",
|
|
844
|
+
"ao. Univ.-Prof.",
|
|
845
|
+
"o. Univ.-Prof.",
|
|
846
|
+
"Univ.-Prof.",
|
|
847
|
+
"Hon.-Prof.",
|
|
848
|
+
"em. Prof.",
|
|
849
|
+
"Dr. med. dent.",
|
|
850
|
+
"Dr. med. vet.",
|
|
851
|
+
"Dr. med.",
|
|
852
|
+
"Dr. rer. nat.",
|
|
853
|
+
"Dr. rer. soc.",
|
|
854
|
+
"Dr. rer. pol.",
|
|
855
|
+
"Dr. sc. tech.",
|
|
856
|
+
"Dr. sc. nat.",
|
|
857
|
+
"Dr. sc. hum.",
|
|
858
|
+
"Dr. iur.",
|
|
859
|
+
"Dr. jur.",
|
|
860
|
+
"Dr. theol.",
|
|
861
|
+
"Dr. oec.",
|
|
862
|
+
"Dr. techn.",
|
|
863
|
+
"Dr. h. c.",
|
|
864
|
+
"Dr. phil.",
|
|
865
|
+
"Dr.-Ing.",
|
|
866
|
+
"Dr. Ing.",
|
|
867
|
+
"Dr.",
|
|
868
|
+
"Dipl.-Wirt.-Ing.",
|
|
869
|
+
"Dipl.-Betriebsw.",
|
|
870
|
+
"Dipl.-Inform.",
|
|
871
|
+
"Dipl.-Volksw.",
|
|
872
|
+
"Dipl.-Psych.",
|
|
873
|
+
"Dipl.-Phys.",
|
|
874
|
+
"Dipl.-Chem.",
|
|
875
|
+
"Dipl.-Biol.",
|
|
876
|
+
"Dipl.-Math.",
|
|
877
|
+
"Dipl.-Päd.",
|
|
878
|
+
"Dipl.-Soz.",
|
|
879
|
+
"Dipl.-Kfm.",
|
|
880
|
+
"Dipl.-Jur.",
|
|
881
|
+
"Dipl. Ing.",
|
|
882
|
+
"Dipl.-Ing.",
|
|
883
|
+
"Mag. rer. soc. oec.",
|
|
884
|
+
"Mag. rer. nat.",
|
|
885
|
+
"Mag. phil.",
|
|
886
|
+
"Mag. iur.",
|
|
887
|
+
"Mag. arch.",
|
|
888
|
+
"Mag. pharm.",
|
|
889
|
+
"Mag. (FH)",
|
|
890
|
+
"Mag.",
|
|
891
|
+
"Bakk. rer. nat.",
|
|
892
|
+
"Bakk. techn.",
|
|
893
|
+
"Bakk. phil.",
|
|
894
|
+
"Bakk.",
|
|
895
|
+
"Lic. phil.",
|
|
896
|
+
"Lic. iur.",
|
|
897
|
+
"Lic. oec.",
|
|
898
|
+
"Lic. theol.",
|
|
899
|
+
"Lic.",
|
|
900
|
+
"Priv.-Doz.",
|
|
901
|
+
"PD",
|
|
902
|
+
"RA"
|
|
903
|
+
];
|
|
904
|
+
/**
|
|
905
|
+
* Courtesy/honorific titles that precede a person's
|
|
906
|
+
* name. Sorted alphabetically. The detector escapes
|
|
907
|
+
* dots and adds \b for entries in HONORIFIC_BOUNDARY.
|
|
908
|
+
*/
|
|
909
|
+
const HONORIFICS = [
|
|
910
|
+
"Avv.",
|
|
911
|
+
"Dame",
|
|
912
|
+
"Doamna",
|
|
913
|
+
"Domnul",
|
|
914
|
+
"Don",
|
|
915
|
+
"Doña",
|
|
916
|
+
"Dott.",
|
|
917
|
+
"Judge",
|
|
918
|
+
"Justice",
|
|
919
|
+
"Lady",
|
|
920
|
+
"Lord",
|
|
921
|
+
"M.",
|
|
922
|
+
"Madame",
|
|
923
|
+
"Mademoiselle",
|
|
924
|
+
"Maître",
|
|
925
|
+
"Me",
|
|
926
|
+
"Messrs",
|
|
927
|
+
"Miss",
|
|
928
|
+
"Mlle",
|
|
929
|
+
"Mme",
|
|
930
|
+
"Monsieur",
|
|
931
|
+
"Mr",
|
|
932
|
+
"Mrs",
|
|
933
|
+
"Ms",
|
|
934
|
+
"President",
|
|
935
|
+
"Señor",
|
|
936
|
+
"Señora",
|
|
937
|
+
"Sig.",
|
|
938
|
+
"Sig.ra",
|
|
939
|
+
"Signor",
|
|
940
|
+
"Signora",
|
|
941
|
+
"Signorina",
|
|
942
|
+
"Sir",
|
|
943
|
+
"Sr.",
|
|
944
|
+
"Sra."
|
|
945
|
+
];
|
|
946
|
+
/**
|
|
947
|
+
* Honorifics that need \b word-boundary anchors
|
|
948
|
+
* (short or common words that could match mid-word).
|
|
949
|
+
*/
|
|
950
|
+
const HONORIFIC_BOUNDARY = new Set([
|
|
951
|
+
"Don",
|
|
952
|
+
"Doña",
|
|
953
|
+
"M.",
|
|
954
|
+
"Me",
|
|
955
|
+
"Señor",
|
|
956
|
+
"Señora"
|
|
957
|
+
]);
|
|
958
|
+
/**
|
|
959
|
+
* Post-nominal degrees (comma or space separated after name).
|
|
960
|
+
* Plain text; the detector auto-escapes for regex.
|
|
961
|
+
*/
|
|
962
|
+
const POST_NOMINALS = [
|
|
963
|
+
"Ph.D.",
|
|
964
|
+
"Ph.D",
|
|
965
|
+
"CSc.",
|
|
966
|
+
"DrSc.",
|
|
967
|
+
"ArtD.",
|
|
968
|
+
"D.Phil.",
|
|
969
|
+
"DPhil.",
|
|
970
|
+
"MPhil.",
|
|
971
|
+
"MBA",
|
|
972
|
+
"MPA",
|
|
973
|
+
"LL.M.",
|
|
974
|
+
"LL.B.",
|
|
975
|
+
"M.Sc.",
|
|
976
|
+
"B.Sc.",
|
|
977
|
+
"MSc.",
|
|
978
|
+
"BSc.",
|
|
979
|
+
"M.Eng.",
|
|
980
|
+
"B.Eng.",
|
|
981
|
+
"M.A.",
|
|
982
|
+
"B.A.",
|
|
983
|
+
"JCD",
|
|
984
|
+
"JD",
|
|
985
|
+
"DiS.",
|
|
986
|
+
"ACCA",
|
|
987
|
+
"FCCA",
|
|
988
|
+
"CIPM",
|
|
989
|
+
"CIPT",
|
|
990
|
+
"CIPP/E",
|
|
991
|
+
"CIPP"
|
|
992
|
+
];
|
|
993
|
+
//#endregion
|
|
994
|
+
//#region src/detectors/regex.ts
|
|
995
|
+
const MIN_PHONE_LENGTH = 7;
|
|
996
|
+
const MIN_MONTH_NAME_LENGTH = 3;
|
|
997
|
+
const escapeTitle = (title) => title.replace(/[.*+?^${}()|[\]\\]/g, "\\$&").replace(/\s+/g, "\\s*");
|
|
998
|
+
/** Escape for use inside a regex alternation. */
|
|
999
|
+
const escapeRegex$1 = (s) => s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
1000
|
+
/** Escape for use inside a regex character class. */
|
|
1001
|
+
const escapeCharClass = (s) => s.replace(/[\]\\^-]/g, "\\$&");
|
|
1002
|
+
const TITLE_PREFIX = TITLE_PREFIXES.toSorted((a, b) => b.length - a.length).map(escapeTitle).join("|");
|
|
1003
|
+
const POST_NOMINAL = POST_NOMINALS.toSorted((a, b) => b.length - a.length).map(escapeTitle).join("|");
|
|
1004
|
+
const NAME_WORD = `[A-ZÁČĎÉĚÍŇÓŘŠŤÚŮÝŽ][a-záčďéěíňóřšťúůýžäöüß]+`;
|
|
1005
|
+
const PARTICLE = "(?:van der|van den|de la|della|von|van|dos|ibn|ben|bin|del|zum|zur|ten|ter|da|de|di|al|el|le|la|zu|af|av)";
|
|
1006
|
+
const SP = "[^\\S\\n\\t]";
|
|
1007
|
+
/** Honorific alternation built from titles.ts config. */
|
|
1008
|
+
const HONORIFIC_ALT = [...HONORIFICS].toSorted((a, b) => b.length - a.length).map((h) => {
|
|
1009
|
+
const escaped = escapeRegex$1(h);
|
|
1010
|
+
return HONORIFIC_BOUNDARY.has(h) ? `\\b${escaped}` : escaped;
|
|
1011
|
+
}).join("|");
|
|
1012
|
+
const toEntry = (validator, label, score) => {
|
|
1013
|
+
const pattern = toRegex(validator).source;
|
|
1014
|
+
if (!pattern) return null;
|
|
1015
|
+
return {
|
|
1016
|
+
validator,
|
|
1017
|
+
label,
|
|
1018
|
+
score,
|
|
1019
|
+
pattern
|
|
1020
|
+
};
|
|
1021
|
+
};
|
|
1022
|
+
/**
|
|
1023
|
+
* Stdnum validators for national/company IDs.
|
|
1024
|
+
*
|
|
1025
|
+
* Selection criteria: only patterns specific enough
|
|
1026
|
+
* to avoid excessive false positives (country-prefixed
|
|
1027
|
+
* VAT numbers, structured personal IDs). Generic
|
|
1028
|
+
* digit-only patterns (e.g. \d{8}) are excluded unless
|
|
1029
|
+
* the validator's checksum is strong enough to filter.
|
|
1030
|
+
*/
|
|
1031
|
+
const STDNUM_ENTRIES = [
|
|
1032
|
+
toEntry(hu.vat, "tax identification number", .95),
|
|
1033
|
+
toEntry(it.codiceFiscale, "national identification number", .95),
|
|
1034
|
+
toEntry(es.dni, "national identification number", .9),
|
|
1035
|
+
toEntry(es.nie, "national identification number", .95),
|
|
1036
|
+
toEntry(se.personnummer, "national identification number", .9),
|
|
1037
|
+
toEntry(ro.cnp, "national identification number", .95),
|
|
1038
|
+
toEntry(fr.nir, "social security number", .9),
|
|
1039
|
+
toEntry(cz.dic, "tax identification number", .95),
|
|
1040
|
+
toEntry(de.vat, "tax identification number", .95),
|
|
1041
|
+
toEntry(de.idnr, "tax identification number", .9),
|
|
1042
|
+
toEntry(de.stnr, "tax identification number", .9),
|
|
1043
|
+
toEntry(de.svnr, "social security number", .9),
|
|
1044
|
+
toEntry(pl.nip, "tax identification number", .95),
|
|
1045
|
+
toEntry(pl.pesel, "national identification number", .9),
|
|
1046
|
+
toEntry(gb.vat, "tax identification number", .95),
|
|
1047
|
+
toEntry(gb.nino, "social security number", .95),
|
|
1048
|
+
toEntry(at.uid, "tax identification number", .95),
|
|
1049
|
+
toEntry(at.tin, "tax identification number", .9),
|
|
1050
|
+
toEntry(at.businessid, "registration number", .95),
|
|
1051
|
+
toEntry(be.vat, "tax identification number", .95),
|
|
1052
|
+
toEntry(be.nn, "national identification number", .9),
|
|
1053
|
+
toEntry(nl.vat, "tax identification number", .95),
|
|
1054
|
+
toEntry(dk.vat, "tax identification number", .95),
|
|
1055
|
+
toEntry(dk.cpr, "national identification number", .9),
|
|
1056
|
+
toEntry(fi.vat, "tax identification number", .95),
|
|
1057
|
+
toEntry(fi.hetu, "national identification number", .95),
|
|
1058
|
+
toEntry(fi.ytunnus, "registration number", .9),
|
|
1059
|
+
toEntry(bg.vat, "tax identification number", .95),
|
|
1060
|
+
toEntry(sk.dic, "tax identification number", .95),
|
|
1061
|
+
toEntry(es.cif, "registration number", .95),
|
|
1062
|
+
toEntry(es.vat, "tax identification number", .95),
|
|
1063
|
+
toEntry(es.nss, "social security number", .9),
|
|
1064
|
+
toEntry(fr.tva, "tax identification number", .95),
|
|
1065
|
+
toEntry(fr.siren, "registration number", .9),
|
|
1066
|
+
toEntry(fr.siret, "registration number", .9),
|
|
1067
|
+
toEntry(it.iva, "tax identification number", .95),
|
|
1068
|
+
toEntry(ie.vat, "tax identification number", .95),
|
|
1069
|
+
toEntry(ie.pps, "national identification number", .9),
|
|
1070
|
+
toEntry(pt.vat, "tax identification number", .95),
|
|
1071
|
+
toEntry(pt.cc, "national identification number", .9),
|
|
1072
|
+
toEntry(ro.vat, "tax identification number", .95),
|
|
1073
|
+
toEntry(gr.vat, "tax identification number", .95),
|
|
1074
|
+
toEntry(hr.vat, "tax identification number", .95),
|
|
1075
|
+
toEntry(si.vat, "tax identification number", .95),
|
|
1076
|
+
toEntry(lt.vat, "tax identification number", .95),
|
|
1077
|
+
toEntry(lt.asmens, "national identification number", .9),
|
|
1078
|
+
toEntry(lv.vat, "tax identification number", .95),
|
|
1079
|
+
toEntry(ee.vat, "tax identification number", .95),
|
|
1080
|
+
toEntry(ee.ik, "national identification number", .9),
|
|
1081
|
+
toEntry(cy.vat, "tax identification number", .95),
|
|
1082
|
+
toEntry(mt.vat, "tax identification number", .95),
|
|
1083
|
+
toEntry(lu.vat, "tax identification number", .95)
|
|
1084
|
+
].filter((e) => e !== null);
|
|
1085
|
+
const TITLED_PERSON = {
|
|
1086
|
+
pattern: `(?:${TITLE_PREFIX})(?:${SP}+(?:${TITLE_PREFIX}))*${SP}+(?:${NAME_WORD})(?:${SP}{1,4}(?:${PARTICLE}${SP}+)?${NAME_WORD}){1,3}(?:,?${SP}+(?:${POST_NOMINAL})(?:,?${SP}+(?:${POST_NOMINAL}))*)?`,
|
|
1087
|
+
label: "person",
|
|
1088
|
+
score: .95
|
|
1089
|
+
};
|
|
1090
|
+
const HONORIFIC_PERSON = {
|
|
1091
|
+
pattern: `(?:${HONORIFIC_ALT})\\.?${SP}+${NAME_WORD}(?:(?:${SP}|-){1,2}(?:${PARTICLE}${SP}+)?${NAME_WORD}){0,3}(?:${SP}+(?:QC|KC|SC|LJ|AG))?`,
|
|
1092
|
+
label: "person",
|
|
1093
|
+
score: .95
|
|
1094
|
+
};
|
|
1095
|
+
const IBAN = {
|
|
1096
|
+
pattern: "\\b[A-Z]{2}\\d{2}\\s?[\\dA-Z]{4}\\s?[\\dA-Z]{4}\\s?[\\dA-Z]{4}\\s?[\\dA-Z]{4}\\s?[\\dA-Z]{0,14}\\b",
|
|
1097
|
+
label: "iban",
|
|
1098
|
+
score: 1
|
|
1099
|
+
};
|
|
1100
|
+
const EMAIL = {
|
|
1101
|
+
pattern: `\\b[\\w.+\\-]+@[\\w\\-]+(?:\\.[\\w\\-]+)+\\b`,
|
|
1102
|
+
label: "email address",
|
|
1103
|
+
score: 1
|
|
1104
|
+
};
|
|
1105
|
+
const INTL_PHONE = {
|
|
1106
|
+
pattern: "\\+\\d{1,3}(?:[^\\S\\n]|[.\\-])?\\(?\\d{2,4}\\)?(?:[^\\S\\n]|[.\\-])?\\d{3}(?:[^\\S\\n]|[.\\-])?\\d{2,4}(?:[^\\S\\n]|[.\\-])?\\d{0,4}\\b",
|
|
1107
|
+
label: "phone number",
|
|
1108
|
+
score: 1
|
|
1109
|
+
};
|
|
1110
|
+
const CZ_PHONE = {
|
|
1111
|
+
pattern: "\\b[2-7]\\d{2}(?:[^\\S\\n]|[.\\-])?\\d{3}(?:[^\\S\\n]|[.\\-])?\\d{3}(?!(?:[^\\S\\n]|[.\\-])?\\d*/\\d)(?![^\\S\\n]*(?:Kč|,-|korun|EUR|USD|€|\\$))\\b",
|
|
1112
|
+
label: "phone number",
|
|
1113
|
+
score: .85
|
|
1114
|
+
};
|
|
1115
|
+
/**
|
|
1116
|
+
* Phone numbers prefixed with "tel.:" or "telefon:".
|
|
1117
|
+
* Captures the number after the prefix, including
|
|
1118
|
+
* optional international code (+420).
|
|
1119
|
+
*/
|
|
1120
|
+
const TEL_PREFIX_PHONE = {
|
|
1121
|
+
pattern: "(?:\\b[Tt]el(?:efon)?\\.?\\s*:?\\s*)(?:\\+?\\d{1,3}[^\\S\\n]?)?\\d{3}(?:[^\\S\\n]|[.\\-])?\\d{3}(?:[^\\S\\n]|[.\\-])?\\d{3}\\b",
|
|
1122
|
+
label: "phone number",
|
|
1123
|
+
score: .95
|
|
1124
|
+
};
|
|
1125
|
+
const CREDIT_CARD = {
|
|
1126
|
+
pattern: "\\b(?:4\\d{3}|5[1-5]\\d{2}|3[47]\\d{2})(?:[^\\S\\n]|[.\\-])?\\d{4}(?:[^\\S\\n]|[.\\-])?\\d{4}(?:[^\\S\\n]|[.\\-])?\\d{2,4}\\b",
|
|
1127
|
+
label: "credit card number",
|
|
1128
|
+
score: 1
|
|
1129
|
+
};
|
|
1130
|
+
const CZ_BIRTH_NUMBER = {
|
|
1131
|
+
pattern: `\\b\\d{6}/\\d{3,4}\\b`,
|
|
1132
|
+
label: "czech birth number",
|
|
1133
|
+
score: 1,
|
|
1134
|
+
validator: cz.rc
|
|
1135
|
+
};
|
|
1136
|
+
const DATE_NUMERIC = {
|
|
1137
|
+
pattern: "\\b(?:\\d{1,2}[./]\\d{1,2}[./]\\d{2,4}|\\d{4}-\\d{2}-\\d{2})\\b",
|
|
1138
|
+
label: "date",
|
|
1139
|
+
score: 1
|
|
1140
|
+
};
|
|
1141
|
+
const DATE_CZ_SPACED = {
|
|
1142
|
+
pattern: `\\b\\d{1,2}\\.[^\\S\\n]+\\d{1,2}\\.[^\\S\\n]+\\d{4}\\b`,
|
|
1143
|
+
label: "date",
|
|
1144
|
+
score: 1
|
|
1145
|
+
};
|
|
1146
|
+
const IP_ADDRESS = {
|
|
1147
|
+
pattern: "\\b(?:(?:25[0-5]|2[0-4]\\d|[01]?\\d\\d?)\\.){3}(?:25[0-5]|2[0-4]\\d|[01]?\\d\\d?)\\b",
|
|
1148
|
+
label: "ip address",
|
|
1149
|
+
score: 1
|
|
1150
|
+
};
|
|
1151
|
+
const CZ_BANK_ACCOUNT = {
|
|
1152
|
+
pattern: `\\b(?:\\d{1,6}-)?\\d{6,10}/\\d{4}(?!\\d)`,
|
|
1153
|
+
label: "bank account number",
|
|
1154
|
+
score: .95
|
|
1155
|
+
};
|
|
1156
|
+
const HU_LANDLINE = {
|
|
1157
|
+
pattern: "\\+36(?:[^\\S\\n]|[.\\-])?1(?:[^\\S\\n]|[.\\-])?\\d{3}(?:[^\\S\\n]|[.\\-])?\\d{4}\\b",
|
|
1158
|
+
label: "phone number",
|
|
1159
|
+
score: .9
|
|
1160
|
+
};
|
|
1161
|
+
const CZ_POSTAL = {
|
|
1162
|
+
pattern: `\\b\\d{3}[^\\S\\n]\\d{2}\\b`,
|
|
1163
|
+
label: "address",
|
|
1164
|
+
score: .7
|
|
1165
|
+
};
|
|
1166
|
+
const URL = {
|
|
1167
|
+
pattern: "(?:https?://|https?:(?=[^\\s])|www\\.)[\\w\\-]+(?:\\.[\\w\\-]+)+(?::\\d+)?(?:[/?#][^\\s)\\]>]*[^\\s.,;:!?)\\]>])?",
|
|
1168
|
+
label: "url",
|
|
1169
|
+
score: 1
|
|
1170
|
+
};
|
|
1171
|
+
const LONG_TLDS = "com|org|net|eu|cz|sk|pl|hu|ro|fr|es|co\\.uk|nl|ch|info|io|dev";
|
|
1172
|
+
const SHORT_TLDS = "de|at|be|se|fi|dk|no|it|uk";
|
|
1173
|
+
const HOST_LABEL = `[a-zA-Z0-9](?:[a-zA-Z0-9\\-]*[a-zA-Z0-9])?`;
|
|
1174
|
+
const BARE_HOST = `\\b[a-zA-Z0-9][a-zA-Z0-9\\-]+[a-zA-Z0-9]`;
|
|
1175
|
+
const PATH_SUFFIX = `(?:[/?#][^\\s)\\]>]*[^\\s.,;:!?)\\]>])?`;
|
|
1176
|
+
/**
|
|
1177
|
+
* All static PII regex definitions. Scanned in a
|
|
1178
|
+
* single pass by @stll/regex-set (Rust DFA).
|
|
1179
|
+
*
|
|
1180
|
+
* Hand-written patterns (0-17) followed by
|
|
1181
|
+
* stdnum-derived patterns (18+). Each stdnum entry
|
|
1182
|
+
* has a post-match validator for confirmation.
|
|
1183
|
+
*
|
|
1184
|
+
* Monetary amount patterns are built dynamically from
|
|
1185
|
+
* currencies.json via `getCurrencyPatterns()`.
|
|
1186
|
+
*
|
|
1187
|
+
* Date patterns using written month names are built
|
|
1188
|
+
* dynamically from date-months.json via
|
|
1189
|
+
* `getDatePatterns()`.
|
|
1190
|
+
*/
|
|
1191
|
+
const ALL_REGEX_DEFS = [
|
|
1192
|
+
TITLED_PERSON,
|
|
1193
|
+
HONORIFIC_PERSON,
|
|
1194
|
+
IBAN,
|
|
1195
|
+
EMAIL,
|
|
1196
|
+
INTL_PHONE,
|
|
1197
|
+
CZ_PHONE,
|
|
1198
|
+
TEL_PREFIX_PHONE,
|
|
1199
|
+
CREDIT_CARD,
|
|
1200
|
+
CZ_BIRTH_NUMBER,
|
|
1201
|
+
DATE_NUMERIC,
|
|
1202
|
+
DATE_CZ_SPACED,
|
|
1203
|
+
IP_ADDRESS,
|
|
1204
|
+
CZ_BANK_ACCOUNT,
|
|
1205
|
+
HU_LANDLINE,
|
|
1206
|
+
CZ_POSTAL,
|
|
1207
|
+
URL,
|
|
1208
|
+
{
|
|
1209
|
+
pattern: "\\b(?:[0-9a-fA-F]{1,4}:){7}[0-9a-fA-F]{1,4}\\b|\\b(?:[0-9a-fA-F]{1,4}:){1,7}:\\b|::(?:[0-9a-fA-F]{1,4}:){0,6}[0-9a-fA-F]{1,4}",
|
|
1210
|
+
label: "ip address",
|
|
1211
|
+
score: 1
|
|
1212
|
+
},
|
|
1213
|
+
{
|
|
1214
|
+
pattern: "\\b(?:[0-9a-fA-F]{2}:){5}[0-9a-fA-F]{2}\\b|\\b(?:[0-9a-fA-F]{2}-){5}[0-9a-fA-F]{2}\\b",
|
|
1215
|
+
label: "mac address",
|
|
1216
|
+
score: 1
|
|
1217
|
+
},
|
|
1218
|
+
{
|
|
1219
|
+
pattern: `${BARE_HOST}(?:\\.${HOST_LABEL})*\\.(?:${LONG_TLDS})\\b${PATH_SUFFIX}|${BARE_HOST}(?:\\.${HOST_LABEL})+\\.(?:${SHORT_TLDS})\\b${PATH_SUFFIX}`,
|
|
1220
|
+
label: "url",
|
|
1221
|
+
score: .9
|
|
1222
|
+
},
|
|
1223
|
+
...STDNUM_ENTRIES
|
|
1224
|
+
];
|
|
1225
|
+
/** Flat pattern array for text-search. */
|
|
1226
|
+
const REGEX_PATTERNS = ALL_REGEX_DEFS.map((d) => d.pattern);
|
|
1227
|
+
/** Parallel metadata. Index = pattern index. */
|
|
1228
|
+
const REGEX_META = ALL_REGEX_DEFS.map((d) => {
|
|
1229
|
+
const meta = {
|
|
1230
|
+
label: d.label,
|
|
1231
|
+
score: d.score
|
|
1232
|
+
};
|
|
1233
|
+
if (d.validator) meta.validator = d.validator;
|
|
1234
|
+
return meta;
|
|
1235
|
+
});
|
|
1236
|
+
/**
|
|
1237
|
+
* Build month-name alternation from date-months.json.
|
|
1238
|
+
* Deduplicates across all 22 languages, filters names
|
|
1239
|
+
* shorter than 3 chars (too many false positives), and
|
|
1240
|
+
* sorts longest-first so the regex engine prefers the
|
|
1241
|
+
* longest match.
|
|
1242
|
+
*/
|
|
1243
|
+
const buildMonthAlternation = (months) => {
|
|
1244
|
+
const seen = /* @__PURE__ */ new Set();
|
|
1245
|
+
for (const [key, value] of Object.entries(months)) {
|
|
1246
|
+
if (key.startsWith("_")) continue;
|
|
1247
|
+
const names = Array.isArray(value) ? value : [value];
|
|
1248
|
+
for (const name of names) {
|
|
1249
|
+
const clean = name.replace(/\.$/, "").toLowerCase();
|
|
1250
|
+
if (clean.length >= MIN_MONTH_NAME_LENGTH) seen.add(clean);
|
|
1251
|
+
}
|
|
1252
|
+
}
|
|
1253
|
+
return [...seen].toSorted((a, b) => b.length - a.length).map(escapeRegex$1).join("|");
|
|
1254
|
+
};
|
|
1255
|
+
/**
|
|
1256
|
+
* Build date patterns from a month-name alternation.
|
|
1257
|
+
* Returns 6 patterns covering the major written-date
|
|
1258
|
+
* formats across all supported languages.
|
|
1259
|
+
*/
|
|
1260
|
+
const buildDatePatternsFromMonths = (alt) => {
|
|
1261
|
+
if (!alt) return [];
|
|
1262
|
+
return [
|
|
1263
|
+
`(?i)\\b\\d{1,2}\\.?\\s+(?:${alt})\\.?\\s+\\d{4}(?:\\s+\\d{1,2}:\\d{2}(?::\\d{2})?)?\\b`,
|
|
1264
|
+
`(?i)\\b(?:${alt})\\.?\\s+\\d{1,2},?\\s+\\d{4}\\b`,
|
|
1265
|
+
`(?i)\\b\\d{1,2}(?:st|nd|rd|th)\\s+(?:${alt})\\.?(?:\\s+\\d{4})?(?=\\s|[.,;!?)]|$)`,
|
|
1266
|
+
`(?i)\\b(?:${alt})\\.?\\s+\\d{4}\\b`,
|
|
1267
|
+
`(?i)\\b\\d{4}\\.\\s+(?:${alt})\\.?\\s+\\d{1,2}\\.?(?=\\s|[.,;!?)]|$)`,
|
|
1268
|
+
`(?i)\\b\\d{1,2}\\s+de\\s+(?:${alt})\\.?(?:\\s+de)?\\s+\\d{4}\\b`
|
|
1269
|
+
];
|
|
1270
|
+
};
|
|
1271
|
+
/** Cached promise for date patterns. Loaded once. */
|
|
1272
|
+
let datePatternPromise = null;
|
|
1273
|
+
const loadDatePatterns = async () => {
|
|
1274
|
+
const mod = await import("@stll/anonymize-data/config/date-months.json");
|
|
1275
|
+
return buildDatePatternsFromMonths(buildMonthAlternation(mod.default ?? mod));
|
|
1276
|
+
};
|
|
1277
|
+
/**
|
|
1278
|
+
* Get dynamically built date patterns from
|
|
1279
|
+
* date-months.json. Returns a cached promise; the JSON
|
|
1280
|
+
* is loaded only once.
|
|
1281
|
+
*/
|
|
1282
|
+
const getDatePatterns = () => {
|
|
1283
|
+
if (!datePatternPromise) datePatternPromise = loadDatePatterns().catch((err) => {
|
|
1284
|
+
datePatternPromise = null;
|
|
1285
|
+
throw err;
|
|
1286
|
+
});
|
|
1287
|
+
return datePatternPromise;
|
|
1288
|
+
};
|
|
1289
|
+
/** Date pattern metadata (all are score 1 dates). */
|
|
1290
|
+
const DATE_PATTERN_META = Object.freeze({
|
|
1291
|
+
label: "date",
|
|
1292
|
+
score: 1
|
|
1293
|
+
});
|
|
1294
|
+
/**
|
|
1295
|
+
* Build symbol character class, code alternation,
|
|
1296
|
+
* and local-name alternation from currencies.json,
|
|
1297
|
+
* then return two monetary amount patterns: leading
|
|
1298
|
+
* symbol and trailing code/name.
|
|
1299
|
+
*
|
|
1300
|
+
* The number sub-pattern accepts both grouped
|
|
1301
|
+
* thousands (1,000) and plain integers (100000)
|
|
1302
|
+
* via `\d{1,9}` to catch unformatted amounts.
|
|
1303
|
+
*/
|
|
1304
|
+
const buildCurrencyPatterns = (data) => {
|
|
1305
|
+
const symbols = data.symbols.map(escapeCharClass).join("");
|
|
1306
|
+
const isAsciiAlpha = /^[a-zA-Z\s]+$/;
|
|
1307
|
+
const parts = data.codes.map((code) => ({
|
|
1308
|
+
len: code.length,
|
|
1309
|
+
alt: escapeRegex$1(code)
|
|
1310
|
+
}));
|
|
1311
|
+
const MIN_CI_LENGTH = 3;
|
|
1312
|
+
if (data.localNames) for (const name of data.localNames) {
|
|
1313
|
+
const escaped = escapeRegex$1(name);
|
|
1314
|
+
if (isAsciiAlpha.test(name) && name.length >= MIN_CI_LENGTH) parts.push({
|
|
1315
|
+
len: name.length,
|
|
1316
|
+
alt: `(?i:${escaped})`
|
|
1317
|
+
});
|
|
1318
|
+
else parts.push({
|
|
1319
|
+
len: name.length,
|
|
1320
|
+
alt: escaped
|
|
1321
|
+
});
|
|
1322
|
+
}
|
|
1323
|
+
if (symbols) for (const ch of data.symbols) parts.push({
|
|
1324
|
+
len: ch.length,
|
|
1325
|
+
alt: escapeRegex$1(ch)
|
|
1326
|
+
});
|
|
1327
|
+
const trailingAlt = parts.toSorted((a, b) => b.len - a.len).map((p) => p.alt).join("|");
|
|
1328
|
+
if (!symbols && !trailingAlt) return [];
|
|
1329
|
+
const NUM = "(?:\\d{1,3}(?:[,.'[^\\S\\n\\t]]\\d{3})+|\\d{1,9})";
|
|
1330
|
+
const patterns = [];
|
|
1331
|
+
if (symbols) patterns.push(`(?:[${symbols}])[^\\S\\n\\t]?${NUM}(?:[.,](?:\\d{1,2}[-–—]?|[-–—]{1,2})?)?\\b`);
|
|
1332
|
+
if (trailingAlt) patterns.push(`\\b(?:${trailingAlt})[^\\S\\n\\t]{0,2}${NUM}(?:[.,](?:\\d{1,2}[-–—]?|[-–—]{1,2})?)?\\b`);
|
|
1333
|
+
if (trailingAlt) patterns.push(`\\b${NUM}(?:[.,](?:\\d{1,2}[-–—]?|[-–—]{1,2})?)?[^\\S\\n\\t]{0,4}(?:${trailingAlt})(?:\\b|(?=\\s|[.,;!?)]|$))`);
|
|
1334
|
+
return patterns;
|
|
1335
|
+
};
|
|
1336
|
+
/** Cached promise for currency patterns. Loaded once. */
|
|
1337
|
+
let currencyPatternPromise = null;
|
|
1338
|
+
const loadCurrencyPatterns = async () => {
|
|
1339
|
+
const mod = await import("@stll/anonymize-data/config/currencies.json");
|
|
1340
|
+
return buildCurrencyPatterns(mod.default ?? mod);
|
|
1341
|
+
};
|
|
1342
|
+
/**
|
|
1343
|
+
* Get dynamically built monetary amount patterns from
|
|
1344
|
+
* currencies.json. Returns a cached promise; the JSON
|
|
1345
|
+
* is loaded only once.
|
|
1346
|
+
*/
|
|
1347
|
+
const getCurrencyPatterns = () => {
|
|
1348
|
+
if (!currencyPatternPromise) currencyPatternPromise = loadCurrencyPatterns().catch((err) => {
|
|
1349
|
+
currencyPatternPromise = null;
|
|
1350
|
+
throw err;
|
|
1351
|
+
});
|
|
1352
|
+
return currencyPatternPromise;
|
|
1353
|
+
};
|
|
1354
|
+
/** Currency pattern metadata (score 0.9). */
|
|
1355
|
+
const CURRENCY_PATTERN_META = Object.freeze({
|
|
1356
|
+
label: "monetary amount",
|
|
1357
|
+
score: .9
|
|
1358
|
+
});
|
|
1359
|
+
/**
|
|
1360
|
+
* Process regex matches from the unified search.
|
|
1361
|
+
* Receives all matches; filters to the regex slice
|
|
1362
|
+
* via sliceStart/sliceEnd. Local index into META is
|
|
1363
|
+
* match.pattern - sliceStart.
|
|
1364
|
+
*
|
|
1365
|
+
* For stdnum-derived patterns (those with a validator
|
|
1366
|
+
* in META), the matched text is passed through the
|
|
1367
|
+
* validator's validate() method. If validation fails,
|
|
1368
|
+
* the match is discarded as a false positive.
|
|
1369
|
+
*/
|
|
1370
|
+
const processRegexMatches = (allMatches, sliceStart, sliceEnd, meta_) => {
|
|
1371
|
+
const results = [];
|
|
1372
|
+
for (const match of allMatches) {
|
|
1373
|
+
const idx = match.pattern;
|
|
1374
|
+
if (idx < sliceStart || idx >= sliceEnd) continue;
|
|
1375
|
+
const meta = meta_[idx - sliceStart];
|
|
1376
|
+
if (!meta) continue;
|
|
1377
|
+
if (meta.label === "phone number" && match.text.length < MIN_PHONE_LENGTH) continue;
|
|
1378
|
+
if (meta.validator) {
|
|
1379
|
+
const compacted = meta.validator.compact(match.text);
|
|
1380
|
+
if (!meta.validator.validate(compacted).valid) continue;
|
|
1381
|
+
}
|
|
1382
|
+
results.push({
|
|
1383
|
+
start: match.start,
|
|
1384
|
+
end: match.end,
|
|
1385
|
+
label: meta.label,
|
|
1386
|
+
text: match.text,
|
|
1387
|
+
score: meta.score,
|
|
1388
|
+
source: DETECTION_SOURCES.REGEX
|
|
1389
|
+
});
|
|
1390
|
+
}
|
|
1391
|
+
return results;
|
|
1392
|
+
};
|
|
1393
|
+
/**
|
|
1394
|
+
* Build signing clause place-name patterns from
|
|
1395
|
+
* signing-clauses.json. Each pattern captures the
|
|
1396
|
+
* city/place name from contract signing locations.
|
|
1397
|
+
*
|
|
1398
|
+
* The place name sub-pattern:
|
|
1399
|
+
* \p{Lu}\p{Ll}+ (capitalized word)
|
|
1400
|
+
* optionally followed by preposition + capitalized
|
|
1401
|
+
* word (for "nad Nisou", "am Main", etc.)
|
|
1402
|
+
* optionally followed by more capitalized words
|
|
1403
|
+
* (for "Hradec Králové", "New York", etc.)
|
|
1404
|
+
*/
|
|
1405
|
+
const buildSigningClausePatterns = (data) => {
|
|
1406
|
+
const patterns = [];
|
|
1407
|
+
for (const entry of data.patterns) {
|
|
1408
|
+
const prepAlt = entry.prepositions.length > 0 ? entry.prepositions.join("|") : null;
|
|
1409
|
+
const place = prepAlt ? `(\\p{Lu}\\p{Ll}+(?:\\s+(?:${prepAlt})\\s+\\p{Lu}\\p{Ll}+)*(?:\\s+\\p{Lu}\\p{Ll}+)*)` : `(\\p{Lu}\\p{Ll}+(?:[- ]\\p{Lu}\\p{Ll}+)*)`;
|
|
1410
|
+
const full = `(?:^|\\n|[^\\S\\n])` + entry.prefix + place + (entry.suffix ? `(?:${entry.suffix})` : "");
|
|
1411
|
+
patterns.push(full);
|
|
1412
|
+
}
|
|
1413
|
+
return patterns;
|
|
1414
|
+
};
|
|
1415
|
+
const SIGNING_CLAUSE_META = {
|
|
1416
|
+
label: "address",
|
|
1417
|
+
score: .9
|
|
1418
|
+
};
|
|
1419
|
+
let signingPatternPromise = null;
|
|
1420
|
+
const loadSigningPatterns = async () => {
|
|
1421
|
+
const mod = await import("@stll/anonymize-data/config/signing-clauses.json");
|
|
1422
|
+
return buildSigningClausePatterns(mod.default ?? mod);
|
|
1423
|
+
};
|
|
1424
|
+
const getSigningClausePatterns = () => {
|
|
1425
|
+
if (!signingPatternPromise) signingPatternPromise = loadSigningPatterns().catch((err) => {
|
|
1426
|
+
signingPatternPromise = null;
|
|
1427
|
+
throw err;
|
|
1428
|
+
});
|
|
1429
|
+
return signingPatternPromise;
|
|
1430
|
+
};
|
|
1431
|
+
//#endregion
|
|
1432
|
+
//#region src/detectors/legal-forms.ts
|
|
1433
|
+
const UPPER = "A-ZÁČĎÉĚÍŇÓŘŠŤÚŮÝŽÄÖÜÀÂÆÇÈÊËÎÏÔÙÛŸÑ\\u0130";
|
|
1434
|
+
const LOWER = "a-záčďéěíňóřšťúůýžäöüßàâæçèêëîïôùûÿñ\\u0131";
|
|
1435
|
+
const CAP_WORD = `(?:[${UPPER}]{2,}|[${UPPER}][${LOWER}${UPPER}]+)`;
|
|
1436
|
+
const ANY_WORD = `(?:[${UPPER}${LOWER}][${LOWER}${UPPER}]+|[${UPPER}]{2,3}|\\d{1,4})`;
|
|
1437
|
+
const ALLCAP_WORD = `[${UPPER}]{2,}`;
|
|
1438
|
+
const ROMAN_NUMERAL_RE = /^(?=[IVXLCDM])M{0,3}(?:CM|CD|D?C{0,3})(?:XC|XL|L?X{0,3})(?:IX|IV|V?I{0,3})$/;
|
|
1439
|
+
const escapeForRegex = (form) => form.replace(/[.*+?^${}()|[\]\\]/g, "\\$&").replace(/\s+/g, "\\s+").replace(/\\\./g, "\\.[^\\S\\n]?");
|
|
1440
|
+
const isShortForm = (form) => form.replace(/[.\s]/g, "").length <= 3 && !form.includes(" ");
|
|
1441
|
+
const buildPatternString = (forms, requireCapBefore) => {
|
|
1442
|
+
if (forms.length === 0) return null;
|
|
1443
|
+
const alt = forms.toSorted((a, b) => b.length - a.length).map(escapeForRegex).join("|");
|
|
1444
|
+
return `${`(?:${CAP_WORD})(?:(?:[\\s&,.\\-–—]{1,4}|\\s+(?:a|and|und|et|e|y|i)\\s+)(?:${ANY_WORD})){0,10}`}${requireCapBefore ? `(?:\\s+|,\\s*)` : `\\s+`}(?:${alt})(?![${LOWER}])`;
|
|
1445
|
+
};
|
|
1446
|
+
/**
|
|
1447
|
+
* Build legal form regex pattern strings.
|
|
1448
|
+
* Returns an array of regex strings for the unified
|
|
1449
|
+
* TextSearch builder. Empty if data package is not
|
|
1450
|
+
* installed.
|
|
1451
|
+
*/
|
|
1452
|
+
const buildLegalFormPatterns = async () => {
|
|
1453
|
+
let data = {};
|
|
1454
|
+
try {
|
|
1455
|
+
data = (await import("@stll/anonymize-data/config/legal-forms.json")).default;
|
|
1456
|
+
} catch {
|
|
1457
|
+
return [];
|
|
1458
|
+
}
|
|
1459
|
+
const allForms = [];
|
|
1460
|
+
const seen = /* @__PURE__ */ new Set();
|
|
1461
|
+
for (const forms of Object.values(data)) for (const form of forms) {
|
|
1462
|
+
const key = form.toLowerCase();
|
|
1463
|
+
if (!seen.has(key)) {
|
|
1464
|
+
seen.add(key);
|
|
1465
|
+
allForms.push(form);
|
|
1466
|
+
}
|
|
1467
|
+
}
|
|
1468
|
+
const patterns = [];
|
|
1469
|
+
const longPattern = buildPatternString(allForms.filter((f) => !isShortForm(f)), false);
|
|
1470
|
+
if (longPattern) patterns.push(longPattern);
|
|
1471
|
+
const shortPattern = buildPatternString(allForms.filter(isShortForm), true);
|
|
1472
|
+
if (shortPattern) patterns.push(shortPattern);
|
|
1473
|
+
const allcapPrefix = `(?:${ALLCAP_WORD})(?:[\\s&,.\\-–—]{1,4}(?:${ALLCAP_WORD})){0,2}`;
|
|
1474
|
+
const allcapAlt = allForms.toSorted((a, b) => b.length - a.length).map(escapeForRegex).join("|");
|
|
1475
|
+
patterns.push(`${allcapPrefix}(?:\\s+|,\\s*)(?:${allcapAlt})(?![${LOWER}])`);
|
|
1476
|
+
return patterns;
|
|
1477
|
+
};
|
|
1478
|
+
/**
|
|
1479
|
+
* Process legal form matches from the unified search.
|
|
1480
|
+
* Receives all matches; filters to the legal forms
|
|
1481
|
+
* slice via sliceStart/sliceEnd.
|
|
1482
|
+
*/
|
|
1483
|
+
const processLegalFormMatches = (allMatches, sliceStart, sliceEnd, fullText) => {
|
|
1484
|
+
const results = [];
|
|
1485
|
+
for (const match of allMatches) {
|
|
1486
|
+
const idx = match.pattern;
|
|
1487
|
+
if (idx < sliceStart || idx >= sliceEnd) continue;
|
|
1488
|
+
const text = match.text.trimEnd();
|
|
1489
|
+
if (text.length < 5) continue;
|
|
1490
|
+
if (text.includes("\n")) continue;
|
|
1491
|
+
const prefixEnd = text.lastIndexOf(",") !== -1 ? text.lastIndexOf(",") : text.lastIndexOf(" ");
|
|
1492
|
+
const prefixPart = prefixEnd > 0 ? text.slice(0, prefixEnd).replace(/[^a-zA-ZÀ-ž]/g, "") : text.replace(/[^a-zA-ZÀ-ž]/g, "");
|
|
1493
|
+
const isAllCapsMatch = prefixPart.length > 2 && prefixPart === prefixPart.toUpperCase();
|
|
1494
|
+
if (isAllCapsMatch && fullText) {
|
|
1495
|
+
const lineStart = fullText.lastIndexOf("\n", match.start);
|
|
1496
|
+
const lineEnd = fullText.indexOf("\n", match.end);
|
|
1497
|
+
const lineLetters = fullText.slice(lineStart + 1, lineEnd === -1 ? fullText.length : lineEnd).replace(/[^a-zA-ZÀ-ž]/g, "");
|
|
1498
|
+
const upperCount = [...lineLetters].filter((c) => c === c.toUpperCase()).length;
|
|
1499
|
+
if (lineLetters.length > 5 && upperCount / lineLetters.length >= .95) continue;
|
|
1500
|
+
if ((prefixPart.length > 0 ? text.slice(0, prefixEnd > 0 ? prefixEnd : text.length).trim().split(/\s+/).length : 0) > 3) continue;
|
|
1501
|
+
} else if (isAllCapsMatch) continue;
|
|
1502
|
+
const lastSpace = text.lastIndexOf(" ");
|
|
1503
|
+
const rawSuffix = lastSpace !== -1 ? text.slice(lastSpace + 1) : "";
|
|
1504
|
+
const suffixClean = rawSuffix.replace(/[.,]/g, "");
|
|
1505
|
+
if (suffixClean.length > 0 && ROMAN_NUMERAL_RE.test(suffixClean)) continue;
|
|
1506
|
+
if (suffixClean.length <= 2 && !/\./.test(rawSuffix) && /[^\x00-\x7F]/.test(text.slice(0, lastSpace !== -1 ? lastSpace : text.length))) continue;
|
|
1507
|
+
results.push({
|
|
1508
|
+
start: match.start,
|
|
1509
|
+
end: match.start + text.length,
|
|
1510
|
+
label: "organization",
|
|
1511
|
+
text,
|
|
1512
|
+
score: .95,
|
|
1513
|
+
source: DETECTION_SOURCES.LEGAL_FORM
|
|
1514
|
+
});
|
|
1515
|
+
}
|
|
1516
|
+
return results;
|
|
1517
|
+
};
|
|
1518
|
+
//#endregion
|
|
1519
|
+
//#region src/config/legal-forms.ts
|
|
1520
|
+
/**
|
|
1521
|
+
* Known legal form suffixes. Shared between trigger
|
|
1522
|
+
* detection (reclassification) and org-propagation
|
|
1523
|
+
* (suffix stripping). Within each family of related
|
|
1524
|
+
* forms, longer variants come first so that
|
|
1525
|
+
* "spol. s r.o." matches before "s.r.o." and
|
|
1526
|
+
* "s. r. o." matches before "s.r.o.".
|
|
1527
|
+
*/
|
|
1528
|
+
const LEGAL_SUFFIXES = [
|
|
1529
|
+
"spol. s r.o.",
|
|
1530
|
+
"s.r.o.",
|
|
1531
|
+
"s. r. o.",
|
|
1532
|
+
"a.s.",
|
|
1533
|
+
"a. s.",
|
|
1534
|
+
"v.o.s.",
|
|
1535
|
+
"v. o. s.",
|
|
1536
|
+
"k.s.",
|
|
1537
|
+
"k. s.",
|
|
1538
|
+
"z.s.",
|
|
1539
|
+
"z. s.",
|
|
1540
|
+
"z.ú.",
|
|
1541
|
+
"z. ú.",
|
|
1542
|
+
"o.p.s.",
|
|
1543
|
+
"o. p. s.",
|
|
1544
|
+
"s.p.",
|
|
1545
|
+
"s. p.",
|
|
1546
|
+
"GmbH",
|
|
1547
|
+
"AG",
|
|
1548
|
+
"SE",
|
|
1549
|
+
"KG",
|
|
1550
|
+
"OHG",
|
|
1551
|
+
"Ltd.",
|
|
1552
|
+
"Ltd",
|
|
1553
|
+
"LLC",
|
|
1554
|
+
"LLP",
|
|
1555
|
+
"Inc.",
|
|
1556
|
+
"S.A.",
|
|
1557
|
+
"SA",
|
|
1558
|
+
"SAS",
|
|
1559
|
+
"SARL",
|
|
1560
|
+
"Sp. z o.o.",
|
|
1561
|
+
"S.p.A."
|
|
1562
|
+
];
|
|
1563
|
+
//#endregion
|
|
1564
|
+
//#region src/detectors/triggers.ts
|
|
1565
|
+
const TRIGGER_SCORE = .95;
|
|
1566
|
+
const WHITESPACE_RE$1 = /\s+/;
|
|
1567
|
+
const LETTER_RE = /\p{L}/u;
|
|
1568
|
+
/**
|
|
1569
|
+
* Post-nominal degree regex. When a comma-stop is
|
|
1570
|
+
* followed by a known post-nominal (Ph.D., CSc., MBA
|
|
1571
|
+
* etc.), skip the comma and degree, then continue.
|
|
1572
|
+
*/
|
|
1573
|
+
const POST_NOMINAL_RE = new RegExp(`^,\\s*(?:${POST_NOMINALS.toSorted((a, b) => b.length - a.length).map((d) => d.replace(/[.*+?^${}()|[\]\\]/g, "\\$&").replace(/\\\./g, "\\.\\s*")).join("|")})\\.?`, "i");
|
|
1574
|
+
const LEGAL_FORM_CHECK_RE = new RegExp(LEGAL_SUFFIXES.map((f) => {
|
|
1575
|
+
const escaped = f.replace(/[.*+?^${}()|[\]\\]/g, "\\$&").replace(/\\\./g, "\\.\\s*");
|
|
1576
|
+
const isDotFree = !f.includes(".");
|
|
1577
|
+
const isShort = f.length <= 4;
|
|
1578
|
+
return isDotFree && isShort ? `\\b${escaped}\\b` : escaped;
|
|
1579
|
+
}).join("|"));
|
|
1580
|
+
const compileValidations = (validations) => validations.map((v) => {
|
|
1581
|
+
switch (v.type) {
|
|
1582
|
+
case "starts-uppercase": return {
|
|
1583
|
+
type: "starts-uppercase",
|
|
1584
|
+
re: /^\p{Lu}/u
|
|
1585
|
+
};
|
|
1586
|
+
case "min-length": return {
|
|
1587
|
+
type: "min-length",
|
|
1588
|
+
min: v.min
|
|
1589
|
+
};
|
|
1590
|
+
case "max-length": return {
|
|
1591
|
+
type: "max-length",
|
|
1592
|
+
max: v.max
|
|
1593
|
+
};
|
|
1594
|
+
case "no-digits": return {
|
|
1595
|
+
type: "no-digits",
|
|
1596
|
+
re: /\d/
|
|
1597
|
+
};
|
|
1598
|
+
case "has-digits": return {
|
|
1599
|
+
type: "has-digits",
|
|
1600
|
+
re: /\d/
|
|
1601
|
+
};
|
|
1602
|
+
case "matches-pattern": return {
|
|
1603
|
+
type: "matches-pattern",
|
|
1604
|
+
re: new RegExp(v.pattern, (v.flags ?? "").replace(/[gy]/g, ""))
|
|
1605
|
+
};
|
|
1606
|
+
default: throw new Error(`Unknown validation type: ${JSON.stringify(v)}`);
|
|
1607
|
+
}
|
|
1608
|
+
});
|
|
1609
|
+
const applyValidations = (text, validations) => {
|
|
1610
|
+
for (const v of validations) switch (v.type) {
|
|
1611
|
+
case "starts-uppercase":
|
|
1612
|
+
if (!v.re.test(text)) return false;
|
|
1613
|
+
break;
|
|
1614
|
+
case "min-length":
|
|
1615
|
+
if (text.length < v.min) return false;
|
|
1616
|
+
break;
|
|
1617
|
+
case "max-length":
|
|
1618
|
+
if (text.length > v.max) return false;
|
|
1619
|
+
break;
|
|
1620
|
+
case "no-digits":
|
|
1621
|
+
if (v.re.test(text)) return false;
|
|
1622
|
+
break;
|
|
1623
|
+
case "has-digits":
|
|
1624
|
+
if (!v.re.test(text)) return false;
|
|
1625
|
+
break;
|
|
1626
|
+
case "matches-pattern":
|
|
1627
|
+
if (!v.re.test(text)) return false;
|
|
1628
|
+
break;
|
|
1629
|
+
default: throw new Error(`Unknown compiled validation type: ${JSON.stringify(v)}`);
|
|
1630
|
+
}
|
|
1631
|
+
return true;
|
|
1632
|
+
};
|
|
1633
|
+
const expandTriggerGroups = (groups) => {
|
|
1634
|
+
const rules = [];
|
|
1635
|
+
for (const group of groups) {
|
|
1636
|
+
const extensions = group.extensions ?? [];
|
|
1637
|
+
const compiled = compileValidations(group.validations ?? []);
|
|
1638
|
+
const allTriggers = new Set(group.triggers);
|
|
1639
|
+
for (const trigger of group.triggers) {
|
|
1640
|
+
if (extensions.includes("add-colon") && !trigger.endsWith(":")) allTriggers.add(`${trigger}:`);
|
|
1641
|
+
if (extensions.includes("add-trailing-space") && !trigger.endsWith(" ")) allTriggers.add(`${trigger} `);
|
|
1642
|
+
if (extensions.includes("add-colon-space") && !trigger.endsWith(": ") && !trigger.endsWith(":")) allTriggers.add(`${trigger}: `);
|
|
1643
|
+
if (extensions.includes("normalize-spaces")) {
|
|
1644
|
+
if (trigger.includes(" ")) allTriggers.add(trigger.replace(/ /g, "\xA0"));
|
|
1645
|
+
}
|
|
1646
|
+
}
|
|
1647
|
+
const includeTrigger = group.includeTrigger ?? false;
|
|
1648
|
+
for (const trigger of allTriggers) rules.push({
|
|
1649
|
+
trigger,
|
|
1650
|
+
label: group.label,
|
|
1651
|
+
strategy: group.strategy,
|
|
1652
|
+
validations: compiled,
|
|
1653
|
+
includeTrigger
|
|
1654
|
+
});
|
|
1655
|
+
}
|
|
1656
|
+
return rules;
|
|
1657
|
+
};
|
|
1658
|
+
/**
|
|
1659
|
+
* Build trigger patterns and rules from data configs.
|
|
1660
|
+
* Returns string[] for the unified TextSearch
|
|
1661
|
+
* builder and the parallel rules array.
|
|
1662
|
+
*/
|
|
1663
|
+
const buildTriggerPatterns = async () => {
|
|
1664
|
+
const rules = [];
|
|
1665
|
+
const allGroups = await loadLanguageConfigs("triggers", (mod) => {
|
|
1666
|
+
return mod.default ?? mod;
|
|
1667
|
+
});
|
|
1668
|
+
for (const groups of allGroups) {
|
|
1669
|
+
if (!Array.isArray(groups)) {
|
|
1670
|
+
console.warn("[anonymize] triggers: unexpected config shape, skipping");
|
|
1671
|
+
continue;
|
|
1672
|
+
}
|
|
1673
|
+
rules.push(...expandTriggerGroups(groups));
|
|
1674
|
+
}
|
|
1675
|
+
try {
|
|
1676
|
+
const globalMod = await import("@stll/anonymize-data/config/triggers.global.json");
|
|
1677
|
+
const globalGroups = globalMod.default ?? globalMod;
|
|
1678
|
+
if (Array.isArray(globalGroups)) rules.push(...expandTriggerGroups(globalGroups));
|
|
1679
|
+
} catch (err) {
|
|
1680
|
+
if (!(err instanceof Error) || !err.message.includes("Cannot find module")) throw err;
|
|
1681
|
+
}
|
|
1682
|
+
const seen = /* @__PURE__ */ new Map();
|
|
1683
|
+
for (const rule of rules) {
|
|
1684
|
+
const key = rule.trigger.toLowerCase();
|
|
1685
|
+
const prev = seen.get(key);
|
|
1686
|
+
if (prev !== void 0) {
|
|
1687
|
+
const labelDiff = prev.label !== rule.label;
|
|
1688
|
+
const stratDiff = prev.strategy !== rule.strategy.type;
|
|
1689
|
+
if (labelDiff || stratDiff) console.warn(`[anonymize] duplicate trigger "${rule.trigger}":` + (labelDiff ? ` labels "${prev.label}" vs "${rule.label}"` : "") + (stratDiff ? ` strategies "${prev.strategy}" vs "${rule.strategy.type}"` : ""));
|
|
1690
|
+
}
|
|
1691
|
+
seen.set(key, {
|
|
1692
|
+
label: rule.label,
|
|
1693
|
+
strategy: rule.strategy.type
|
|
1694
|
+
});
|
|
1695
|
+
}
|
|
1696
|
+
return {
|
|
1697
|
+
patterns: rules.map((r) => r.trigger.toLowerCase()),
|
|
1698
|
+
rules
|
|
1699
|
+
};
|
|
1700
|
+
};
|
|
1701
|
+
const LEADING_PUNCT = /^[„""»«'"()\s]+/;
|
|
1702
|
+
const TRAILING_PUNCT = /[""»«'"()\s]+$/;
|
|
1703
|
+
const stripQuotes = (value) => {
|
|
1704
|
+
const leadingMatch = LEADING_PUNCT.exec(value.text);
|
|
1705
|
+
const leadingLen = leadingMatch ? leadingMatch[0].length : 0;
|
|
1706
|
+
const stripped = value.text.slice(leadingLen).replace(TRAILING_PUNCT, "");
|
|
1707
|
+
if (stripped.length === 0) return null;
|
|
1708
|
+
return {
|
|
1709
|
+
start: value.start + leadingLen,
|
|
1710
|
+
end: value.start + leadingLen + stripped.length,
|
|
1711
|
+
text: stripped
|
|
1712
|
+
};
|
|
1713
|
+
};
|
|
1714
|
+
/** Hard stop characters for to-next-comma scanning. */
|
|
1715
|
+
const COMMA_STOP_CHARS = new Set(["\n", "("]);
|
|
1716
|
+
/**
|
|
1717
|
+
* Field-label keywords that terminate address scanning.
|
|
1718
|
+
* When a comma in the address strategy is followed by
|
|
1719
|
+
* one of these, the address stops before the keyword.
|
|
1720
|
+
*/
|
|
1721
|
+
const ADDRESS_STOP_KEYWORDS = [
|
|
1722
|
+
"číslo účtu",
|
|
1723
|
+
"registrační",
|
|
1724
|
+
"zastoupen",
|
|
1725
|
+
"bankovní",
|
|
1726
|
+
"e-mail",
|
|
1727
|
+
"telefon",
|
|
1728
|
+
"jednatel",
|
|
1729
|
+
"ředitel",
|
|
1730
|
+
"datová",
|
|
1731
|
+
"vložka",
|
|
1732
|
+
"sp.zn.",
|
|
1733
|
+
"oddíl",
|
|
1734
|
+
"swift",
|
|
1735
|
+
"email",
|
|
1736
|
+
"iban",
|
|
1737
|
+
"dič",
|
|
1738
|
+
"ičo",
|
|
1739
|
+
"tel",
|
|
1740
|
+
"č.ú.",
|
|
1741
|
+
"bic",
|
|
1742
|
+
"ič"
|
|
1743
|
+
];
|
|
1744
|
+
const extractValue = (text, triggerEnd, strategy, label) => {
|
|
1745
|
+
const remaining = text.slice(triggerEnd);
|
|
1746
|
+
const stripped = remaining.replace(/^[\s:;]+/, "");
|
|
1747
|
+
const valueStart = triggerEnd + (remaining.length - stripped.length);
|
|
1748
|
+
const valueText = stripped;
|
|
1749
|
+
if (valueText.length === 0) return null;
|
|
1750
|
+
switch (strategy.type) {
|
|
1751
|
+
case "to-next-comma": {
|
|
1752
|
+
let end = 0;
|
|
1753
|
+
let foundStop = false;
|
|
1754
|
+
while (end < valueText.length) {
|
|
1755
|
+
const ch = valueText[end];
|
|
1756
|
+
if (ch !== void 0 && COMMA_STOP_CHARS.has(ch)) {
|
|
1757
|
+
foundStop = true;
|
|
1758
|
+
break;
|
|
1759
|
+
}
|
|
1760
|
+
if (ch === ",") {
|
|
1761
|
+
const afterComma = valueText.slice(end);
|
|
1762
|
+
const degreeMatch = label === "person" ? POST_NOMINAL_RE.exec(afterComma) : null;
|
|
1763
|
+
if (degreeMatch) {
|
|
1764
|
+
end += degreeMatch[0].length;
|
|
1765
|
+
continue;
|
|
1766
|
+
}
|
|
1767
|
+
foundStop = true;
|
|
1768
|
+
break;
|
|
1769
|
+
}
|
|
1770
|
+
end++;
|
|
1771
|
+
}
|
|
1772
|
+
if (!foundStop) end = Math.min(end, 100);
|
|
1773
|
+
const rawSlice = valueText.slice(0, end);
|
|
1774
|
+
const extracted = rawSlice.trim();
|
|
1775
|
+
if (extracted.length === 0) return null;
|
|
1776
|
+
const trailingSpaces = rawSlice.length - rawSlice.trimEnd().length;
|
|
1777
|
+
return {
|
|
1778
|
+
start: valueStart,
|
|
1779
|
+
end: valueStart + end - trailingSpaces,
|
|
1780
|
+
text: extracted
|
|
1781
|
+
};
|
|
1782
|
+
}
|
|
1783
|
+
case "to-end-of-line": {
|
|
1784
|
+
const LINE_STOPS = ["\n"];
|
|
1785
|
+
let end = valueText.length;
|
|
1786
|
+
for (const ch of LINE_STOPS) {
|
|
1787
|
+
const idx = valueText.indexOf(ch);
|
|
1788
|
+
if (idx !== -1 && idx < end) end = idx;
|
|
1789
|
+
}
|
|
1790
|
+
const rawSlice = valueText.slice(0, end);
|
|
1791
|
+
const extracted = rawSlice.trim();
|
|
1792
|
+
if (extracted.length === 0) return null;
|
|
1793
|
+
const trailingSpaces = rawSlice.length - rawSlice.trimEnd().length;
|
|
1794
|
+
return {
|
|
1795
|
+
start: valueStart,
|
|
1796
|
+
end: valueStart + end - trailingSpaces,
|
|
1797
|
+
text: extracted
|
|
1798
|
+
};
|
|
1799
|
+
}
|
|
1800
|
+
case "n-words": {
|
|
1801
|
+
const tabIdx = valueText.indexOf(" ");
|
|
1802
|
+
const cellText = tabIdx !== -1 ? valueText.slice(0, tabIdx) : valueText;
|
|
1803
|
+
const PUNCT_ONLY = /^[\p{P}\p{S}]+$/u;
|
|
1804
|
+
const words = cellText.split(WHITESPACE_RE$1).filter((w) => !PUNCT_ONLY.test(w)).slice(0, strategy.count);
|
|
1805
|
+
if (words.length === 0) return null;
|
|
1806
|
+
const firstWord = words[0];
|
|
1807
|
+
if (firstWord === void 0) return null;
|
|
1808
|
+
const firstIdx = cellText.indexOf(firstWord);
|
|
1809
|
+
let actualEnd = firstIdx + firstWord.length;
|
|
1810
|
+
let searchPos = actualEnd;
|
|
1811
|
+
for (let wi = 1; wi < words.length; wi++) {
|
|
1812
|
+
const w = words[wi];
|
|
1813
|
+
if (w === void 0) break;
|
|
1814
|
+
const wIdx = cellText.indexOf(w, searchPos);
|
|
1815
|
+
if (wIdx === -1) break;
|
|
1816
|
+
actualEnd = wIdx + w.length;
|
|
1817
|
+
searchPos = actualEnd;
|
|
1818
|
+
}
|
|
1819
|
+
return {
|
|
1820
|
+
start: valueStart + firstIdx,
|
|
1821
|
+
end: valueStart + actualEnd,
|
|
1822
|
+
text: cellText.slice(firstIdx, actualEnd)
|
|
1823
|
+
};
|
|
1824
|
+
}
|
|
1825
|
+
case "company-id-value": {
|
|
1826
|
+
const raw = text.slice(triggerEnd);
|
|
1827
|
+
const sepMatch = /^(?:\s*:\s*|\s+)/.exec(raw);
|
|
1828
|
+
if (!sepMatch) return null;
|
|
1829
|
+
const afterSep = raw.slice(sepMatch[0].length);
|
|
1830
|
+
const idMatch = /^[A-Z]{0,6}\s?\d[\d\s\-/]{4,}/i.exec(afterSep);
|
|
1831
|
+
if (!idMatch) return null;
|
|
1832
|
+
const idText = idMatch[0].trim();
|
|
1833
|
+
const leadingSpaces = idMatch[0].length - idMatch[0].trimStart().length;
|
|
1834
|
+
const idStart = triggerEnd + sepMatch[0].length + leadingSpaces;
|
|
1835
|
+
return {
|
|
1836
|
+
start: idStart,
|
|
1837
|
+
end: idStart + idText.length,
|
|
1838
|
+
text: idText
|
|
1839
|
+
};
|
|
1840
|
+
}
|
|
1841
|
+
case "address": {
|
|
1842
|
+
const maxLen = strategy.maxChars ?? 120;
|
|
1843
|
+
const UPPER_RE = /\p{Lu}/u;
|
|
1844
|
+
let end = 0;
|
|
1845
|
+
while (end < valueText.length && end < maxLen) {
|
|
1846
|
+
const ch = valueText[end];
|
|
1847
|
+
if (ch === "\n" || ch === "(") break;
|
|
1848
|
+
if (ch === ".") {
|
|
1849
|
+
const next = valueText[end + 1];
|
|
1850
|
+
const afterNext = valueText[end + 2];
|
|
1851
|
+
if (next !== void 0 && (/\p{L}/u.test(next) || /\d/.test(next))) {
|
|
1852
|
+
end++;
|
|
1853
|
+
continue;
|
|
1854
|
+
}
|
|
1855
|
+
if (next === " " && afterNext !== void 0 && (/\p{L}/u.test(afterNext) || /\d/.test(afterNext))) {
|
|
1856
|
+
end++;
|
|
1857
|
+
continue;
|
|
1858
|
+
}
|
|
1859
|
+
break;
|
|
1860
|
+
}
|
|
1861
|
+
if (ch === ",") {
|
|
1862
|
+
let peek = end + 1;
|
|
1863
|
+
while (peek < valueText.length && (valueText[peek] === " " || valueText[peek] === " ")) peek++;
|
|
1864
|
+
const peekCh = valueText[peek];
|
|
1865
|
+
if (peekCh === void 0) break;
|
|
1866
|
+
const afterComma = valueText.slice(end + 1).trimStart().toLowerCase();
|
|
1867
|
+
if (ADDRESS_STOP_KEYWORDS.some((kw) => {
|
|
1868
|
+
if (!afterComma.startsWith(kw)) return false;
|
|
1869
|
+
const next = afterComma[kw.length];
|
|
1870
|
+
return next === void 0 || /[\s:;.,!?()\d]/.test(next);
|
|
1871
|
+
})) break;
|
|
1872
|
+
if (/\d/.test(peekCh) || UPPER_RE.test(peekCh)) {
|
|
1873
|
+
end++;
|
|
1874
|
+
continue;
|
|
1875
|
+
}
|
|
1876
|
+
break;
|
|
1877
|
+
}
|
|
1878
|
+
end++;
|
|
1879
|
+
}
|
|
1880
|
+
if (end >= maxLen) {
|
|
1881
|
+
const lastSpace = valueText.lastIndexOf(" ", end - 1);
|
|
1882
|
+
if (lastSpace > 0) end = lastSpace;
|
|
1883
|
+
}
|
|
1884
|
+
const rawSlice = valueText.slice(0, end);
|
|
1885
|
+
const extracted = rawSlice.trim();
|
|
1886
|
+
if (extracted.length === 0) return null;
|
|
1887
|
+
const trailingSpaces = rawSlice.length - rawSlice.trimEnd().length;
|
|
1888
|
+
return {
|
|
1889
|
+
start: valueStart,
|
|
1890
|
+
end: valueStart + end - trailingSpaces,
|
|
1891
|
+
text: extracted
|
|
1892
|
+
};
|
|
1893
|
+
}
|
|
1894
|
+
default: return null;
|
|
1895
|
+
}
|
|
1896
|
+
};
|
|
1897
|
+
/**
|
|
1898
|
+
* Process trigger matches from the unified search.
|
|
1899
|
+
* Receives all matches; filters to the trigger slice
|
|
1900
|
+
* via sliceStart/sliceEnd. Uses fullText for value
|
|
1901
|
+
* extraction (the unified search runs on lowercased
|
|
1902
|
+
* text, but extraction needs original casing).
|
|
1903
|
+
*/
|
|
1904
|
+
const processTriggerMatches = (allMatches, sliceStart, sliceEnd, fullText, rules) => {
|
|
1905
|
+
const results = [];
|
|
1906
|
+
for (const match of allMatches) {
|
|
1907
|
+
const idx = match.pattern;
|
|
1908
|
+
if (idx < sliceStart || idx >= sliceEnd) continue;
|
|
1909
|
+
const localIdx = idx - sliceStart;
|
|
1910
|
+
if (match.start > 0 && LETTER_RE.test(fullText[match.start - 1] ?? "")) continue;
|
|
1911
|
+
const rule = rules[localIdx];
|
|
1912
|
+
if (!rule) continue;
|
|
1913
|
+
if (!rule.trigger.endsWith(" ") && LETTER_RE.test(fullText[match.end] ?? "")) continue;
|
|
1914
|
+
const triggerEnd = match.end;
|
|
1915
|
+
const rawValue = extractValue(fullText, triggerEnd, rule.strategy, rule.label);
|
|
1916
|
+
const value = rawValue ? stripQuotes(rawValue) : null;
|
|
1917
|
+
if (value) {
|
|
1918
|
+
if (!applyValidations(value.text, rule.validations)) continue;
|
|
1919
|
+
const entityStart = rule.includeTrigger ? match.start : value.start;
|
|
1920
|
+
const entityEnd = value.end;
|
|
1921
|
+
const entityText = fullText.slice(entityStart, entityEnd);
|
|
1922
|
+
const effectiveLabel = rule.label === "person" && LEGAL_FORM_CHECK_RE.test(entityText) ? "organization" : rule.label;
|
|
1923
|
+
results.push({
|
|
1924
|
+
start: entityStart,
|
|
1925
|
+
end: entityEnd,
|
|
1926
|
+
label: effectiveLabel,
|
|
1927
|
+
text: entityText,
|
|
1928
|
+
score: TRIGGER_SCORE,
|
|
1929
|
+
source: DETECTION_SOURCES.TRIGGER
|
|
1930
|
+
});
|
|
1931
|
+
}
|
|
1932
|
+
}
|
|
1933
|
+
return results;
|
|
1934
|
+
};
|
|
1935
|
+
//#endregion
|
|
1936
|
+
//#region src/regions.ts
|
|
1937
|
+
/**
|
|
1938
|
+
* Geographic regions and country code mappings for
|
|
1939
|
+
* scoping deny list dictionaries.
|
|
1940
|
+
*/
|
|
1941
|
+
const REGIONS = {
|
|
1942
|
+
Global: null,
|
|
1943
|
+
International: null,
|
|
1944
|
+
Europe: [
|
|
1945
|
+
"AL",
|
|
1946
|
+
"AD",
|
|
1947
|
+
"AT",
|
|
1948
|
+
"BE",
|
|
1949
|
+
"BA",
|
|
1950
|
+
"BG",
|
|
1951
|
+
"HR",
|
|
1952
|
+
"CY",
|
|
1953
|
+
"CZ",
|
|
1954
|
+
"DK",
|
|
1955
|
+
"EE",
|
|
1956
|
+
"FI",
|
|
1957
|
+
"FR",
|
|
1958
|
+
"DE",
|
|
1959
|
+
"GR",
|
|
1960
|
+
"HU",
|
|
1961
|
+
"IS",
|
|
1962
|
+
"IE",
|
|
1963
|
+
"IT",
|
|
1964
|
+
"XK",
|
|
1965
|
+
"LV",
|
|
1966
|
+
"LI",
|
|
1967
|
+
"LT",
|
|
1968
|
+
"LU",
|
|
1969
|
+
"MD",
|
|
1970
|
+
"ME",
|
|
1971
|
+
"MK",
|
|
1972
|
+
"MT",
|
|
1973
|
+
"MC",
|
|
1974
|
+
"NL",
|
|
1975
|
+
"NO",
|
|
1976
|
+
"PL",
|
|
1977
|
+
"PT",
|
|
1978
|
+
"RO",
|
|
1979
|
+
"RS",
|
|
1980
|
+
"SK",
|
|
1981
|
+
"SI",
|
|
1982
|
+
"ES",
|
|
1983
|
+
"SE",
|
|
1984
|
+
"CH",
|
|
1985
|
+
"UA",
|
|
1986
|
+
"GB"
|
|
1987
|
+
],
|
|
1988
|
+
Americas: [
|
|
1989
|
+
"US",
|
|
1990
|
+
"CA",
|
|
1991
|
+
"MX",
|
|
1992
|
+
"BR",
|
|
1993
|
+
"AR",
|
|
1994
|
+
"CL",
|
|
1995
|
+
"CO",
|
|
1996
|
+
"PE",
|
|
1997
|
+
"EC",
|
|
1998
|
+
"VE",
|
|
1999
|
+
"UY",
|
|
2000
|
+
"PY",
|
|
2001
|
+
"BO",
|
|
2002
|
+
"CR",
|
|
2003
|
+
"PA",
|
|
2004
|
+
"DO",
|
|
2005
|
+
"GT",
|
|
2006
|
+
"HN",
|
|
2007
|
+
"SV",
|
|
2008
|
+
"NI",
|
|
2009
|
+
"CU"
|
|
2010
|
+
],
|
|
2011
|
+
AsiaPacific: [
|
|
2012
|
+
"AU",
|
|
2013
|
+
"NZ",
|
|
2014
|
+
"JP",
|
|
2015
|
+
"KR",
|
|
2016
|
+
"CN",
|
|
2017
|
+
"TW",
|
|
2018
|
+
"SG",
|
|
2019
|
+
"MY",
|
|
2020
|
+
"TH",
|
|
2021
|
+
"VN",
|
|
2022
|
+
"PH",
|
|
2023
|
+
"ID",
|
|
2024
|
+
"IN",
|
|
2025
|
+
"PK",
|
|
2026
|
+
"BD",
|
|
2027
|
+
"LK",
|
|
2028
|
+
"NP",
|
|
2029
|
+
"HK",
|
|
2030
|
+
"MO"
|
|
2031
|
+
],
|
|
2032
|
+
MENA: [
|
|
2033
|
+
"AE",
|
|
2034
|
+
"SA",
|
|
2035
|
+
"IL",
|
|
2036
|
+
"TR",
|
|
2037
|
+
"EG",
|
|
2038
|
+
"JO",
|
|
2039
|
+
"LB",
|
|
2040
|
+
"IQ",
|
|
2041
|
+
"IR",
|
|
2042
|
+
"QA",
|
|
2043
|
+
"KW",
|
|
2044
|
+
"BH",
|
|
2045
|
+
"OM",
|
|
2046
|
+
"MA",
|
|
2047
|
+
"TN",
|
|
2048
|
+
"DZ",
|
|
2049
|
+
"LY",
|
|
2050
|
+
"SY",
|
|
2051
|
+
"YE",
|
|
2052
|
+
"PS"
|
|
2053
|
+
],
|
|
2054
|
+
SubSaharanAfrica: [
|
|
2055
|
+
"ZA",
|
|
2056
|
+
"NG",
|
|
2057
|
+
"KE",
|
|
2058
|
+
"GH",
|
|
2059
|
+
"TZ",
|
|
2060
|
+
"ET",
|
|
2061
|
+
"SN",
|
|
2062
|
+
"CI",
|
|
2063
|
+
"CM",
|
|
2064
|
+
"UG",
|
|
2065
|
+
"RW",
|
|
2066
|
+
"MZ",
|
|
2067
|
+
"AO",
|
|
2068
|
+
"ZW",
|
|
2069
|
+
"BW",
|
|
2070
|
+
"NA",
|
|
2071
|
+
"MU"
|
|
2072
|
+
],
|
|
2073
|
+
EU: [
|
|
2074
|
+
"AT",
|
|
2075
|
+
"BE",
|
|
2076
|
+
"BG",
|
|
2077
|
+
"HR",
|
|
2078
|
+
"CY",
|
|
2079
|
+
"CZ",
|
|
2080
|
+
"DK",
|
|
2081
|
+
"EE",
|
|
2082
|
+
"FI",
|
|
2083
|
+
"FR",
|
|
2084
|
+
"DE",
|
|
2085
|
+
"GR",
|
|
2086
|
+
"HU",
|
|
2087
|
+
"IE",
|
|
2088
|
+
"IT",
|
|
2089
|
+
"LV",
|
|
2090
|
+
"LT",
|
|
2091
|
+
"LU",
|
|
2092
|
+
"MT",
|
|
2093
|
+
"NL",
|
|
2094
|
+
"PL",
|
|
2095
|
+
"PT",
|
|
2096
|
+
"RO",
|
|
2097
|
+
"SK",
|
|
2098
|
+
"SI",
|
|
2099
|
+
"ES",
|
|
2100
|
+
"SE"
|
|
2101
|
+
],
|
|
2102
|
+
DACH: [
|
|
2103
|
+
"DE",
|
|
2104
|
+
"AT",
|
|
2105
|
+
"CH"
|
|
2106
|
+
],
|
|
2107
|
+
Nordics: [
|
|
2108
|
+
"DK",
|
|
2109
|
+
"SE",
|
|
2110
|
+
"NO",
|
|
2111
|
+
"FI",
|
|
2112
|
+
"IS"
|
|
2113
|
+
],
|
|
2114
|
+
CEE: [
|
|
2115
|
+
"CZ",
|
|
2116
|
+
"SK",
|
|
2117
|
+
"PL",
|
|
2118
|
+
"HU",
|
|
2119
|
+
"RO",
|
|
2120
|
+
"BG",
|
|
2121
|
+
"HR",
|
|
2122
|
+
"SI",
|
|
2123
|
+
"LT",
|
|
2124
|
+
"LV",
|
|
2125
|
+
"EE"
|
|
2126
|
+
],
|
|
2127
|
+
Anglosphere: [
|
|
2128
|
+
"GB",
|
|
2129
|
+
"US",
|
|
2130
|
+
"CA",
|
|
2131
|
+
"AU",
|
|
2132
|
+
"NZ",
|
|
2133
|
+
"IE"
|
|
2134
|
+
],
|
|
2135
|
+
Benelux: [
|
|
2136
|
+
"BE",
|
|
2137
|
+
"NL",
|
|
2138
|
+
"LU"
|
|
2139
|
+
],
|
|
2140
|
+
GulfStates: [
|
|
2141
|
+
"AE",
|
|
2142
|
+
"SA",
|
|
2143
|
+
"QA",
|
|
2144
|
+
"KW",
|
|
2145
|
+
"BH",
|
|
2146
|
+
"OM"
|
|
2147
|
+
],
|
|
2148
|
+
SouthAsia: [
|
|
2149
|
+
"IN",
|
|
2150
|
+
"PK",
|
|
2151
|
+
"BD",
|
|
2152
|
+
"LK",
|
|
2153
|
+
"NP"
|
|
2154
|
+
],
|
|
2155
|
+
EastAsia: [
|
|
2156
|
+
"CN",
|
|
2157
|
+
"JP",
|
|
2158
|
+
"KR",
|
|
2159
|
+
"TW"
|
|
2160
|
+
],
|
|
2161
|
+
SoutheastAsia: [
|
|
2162
|
+
"SG",
|
|
2163
|
+
"MY",
|
|
2164
|
+
"TH",
|
|
2165
|
+
"VN",
|
|
2166
|
+
"PH",
|
|
2167
|
+
"ID"
|
|
2168
|
+
],
|
|
2169
|
+
Oceania: ["AU", "NZ"]
|
|
2170
|
+
};
|
|
2171
|
+
/**
|
|
2172
|
+
* Expand region names to country codes and merge with
|
|
2173
|
+
* explicit country codes. Returns null when both inputs
|
|
2174
|
+
* are empty/undefined (meaning "match all countries").
|
|
2175
|
+
*/
|
|
2176
|
+
const resolveCountries = (regions, countries) => {
|
|
2177
|
+
const hasRegions = regions && regions.length > 0;
|
|
2178
|
+
const hasCountries = countries && countries.length > 0;
|
|
2179
|
+
if (!hasRegions && !hasCountries) return null;
|
|
2180
|
+
const result = /* @__PURE__ */ new Set();
|
|
2181
|
+
const isRegion = (name) => name in REGIONS;
|
|
2182
|
+
if (hasRegions) for (const name of regions) {
|
|
2183
|
+
if (!isRegion(name)) continue;
|
|
2184
|
+
const codes = REGIONS[name];
|
|
2185
|
+
if (codes === null) return null;
|
|
2186
|
+
for (const code of codes) result.add(code);
|
|
2187
|
+
}
|
|
2188
|
+
if (hasCountries) for (const code of countries) result.add(code);
|
|
2189
|
+
return result;
|
|
2190
|
+
};
|
|
2191
|
+
//#endregion
|
|
2192
|
+
//#region src/filters/false-positives.ts
|
|
2193
|
+
const TEMPLATE_PLACEHOLDER_RE = /^(?:\.{3,}|_{3,}|\[[\w\s]+\]|\{[\w\s]+\})$/;
|
|
2194
|
+
const POSTAL_CODE_RE = /\d{3}\s?\d{2}/;
|
|
2195
|
+
const HAS_DIGIT_RE = /\d/;
|
|
2196
|
+
const ADDRESS_COMPONENTS_RE = /(?:^|\s)(?:ul\.|ulice|nám\.|náměstí|tř\.|třída|nábř\.|nábřeží|č\.p\.|č\.ev\.|č\.|sídliště|bulvár)(?=[\s,./]|$)/i;
|
|
2197
|
+
const MAX_ENTITY_LENGTH = {
|
|
2198
|
+
organization: 80,
|
|
2199
|
+
person: 60
|
|
2200
|
+
};
|
|
2201
|
+
const SECTION_NUMBER_RE = /^(?:§\s*)?\d{1,3}(?:\.\d{1,3}){0,4}\.?$/;
|
|
2202
|
+
const STANDALONE_YEAR_RE = /^(?:19|20)\d{2}$/;
|
|
2203
|
+
const EMPTY_GENERIC_ROLES = /* @__PURE__ */ new Set();
|
|
2204
|
+
/**
|
|
2205
|
+
* Load generic-roles.json and cache the result on the
|
|
2206
|
+
* given context. Must be awaited during pipeline init
|
|
2207
|
+
* so the sync accessor is populated before
|
|
2208
|
+
* filterFalsePositives runs.
|
|
2209
|
+
*/
|
|
2210
|
+
const loadGenericRoles = (ctx = defaultContext) => {
|
|
2211
|
+
if (ctx.genericRolesPromise) return ctx.genericRolesPromise;
|
|
2212
|
+
ctx.genericRolesPromise = (async () => {
|
|
2213
|
+
try {
|
|
2214
|
+
const mod = await import("@stll/anonymize-data/config/generic-roles.json");
|
|
2215
|
+
const set = new Set(mod.default?.roles ?? []);
|
|
2216
|
+
ctx.genericRoles = set;
|
|
2217
|
+
return set;
|
|
2218
|
+
} catch {
|
|
2219
|
+
const empty = /* @__PURE__ */ new Set();
|
|
2220
|
+
ctx.genericRoles = empty;
|
|
2221
|
+
return empty;
|
|
2222
|
+
}
|
|
2223
|
+
})();
|
|
2224
|
+
return ctx.genericRolesPromise;
|
|
2225
|
+
};
|
|
2226
|
+
/** Sync accessor — returns empty set before init. */
|
|
2227
|
+
const getGenericRoles = (ctx) => ctx.genericRoles ?? EMPTY_GENERIC_ROLES;
|
|
2228
|
+
/**
|
|
2229
|
+
* Filter out entities that are likely false positives:
|
|
2230
|
+
* template placeholders, clause/section numbers,
|
|
2231
|
+
* standalone years, and generic legal role terms.
|
|
2232
|
+
*
|
|
2233
|
+
* Runs as a post-processing step after all detection
|
|
2234
|
+
* layers have merged.
|
|
2235
|
+
*/
|
|
2236
|
+
const filterFalsePositives = (entities, ctx = defaultContext) => {
|
|
2237
|
+
const filtered = [];
|
|
2238
|
+
const roles = getGenericRoles(ctx);
|
|
2239
|
+
for (const entity of entities) {
|
|
2240
|
+
const trimmed = entity.text.trim();
|
|
2241
|
+
if (TEMPLATE_PLACEHOLDER_RE.test(trimmed)) continue;
|
|
2242
|
+
const maxLen = MAX_ENTITY_LENGTH[entity.label];
|
|
2243
|
+
if (maxLen && trimmed.length > maxLen && entity.source !== "legal-form") continue;
|
|
2244
|
+
if (SECTION_NUMBER_RE.test(trimmed) && entity.source !== "trigger") continue;
|
|
2245
|
+
if (STANDALONE_YEAR_RE.test(trimmed)) continue;
|
|
2246
|
+
if ((entity.label === "person" || entity.label === "organization") && roles.has(trimmed.toLowerCase())) continue;
|
|
2247
|
+
if (entity.label === "address" && trimmed.length > 40 && !POSTAL_CODE_RE.test(trimmed) && !HAS_DIGIT_RE.test(trimmed) && !ADDRESS_COMPONENTS_RE.test(trimmed)) continue;
|
|
2248
|
+
if (entity.label === "address" && entity.source === "trigger" && !HAS_DIGIT_RE.test(trimmed) && !ADDRESS_COMPONENTS_RE.test(trimmed)) continue;
|
|
2249
|
+
filtered.push(entity);
|
|
2250
|
+
}
|
|
2251
|
+
return filtered;
|
|
2252
|
+
};
|
|
2253
|
+
//#endregion
|
|
2254
|
+
//#region src/util/normalize.ts
|
|
2255
|
+
const CHAR_MAP = new Map([
|
|
2256
|
+
[160, 32],
|
|
2257
|
+
[8199, 32],
|
|
2258
|
+
[8239, 32],
|
|
2259
|
+
[8211, 45],
|
|
2260
|
+
[8212, 45],
|
|
2261
|
+
[8220, 34],
|
|
2262
|
+
[8221, 34]
|
|
2263
|
+
]);
|
|
2264
|
+
/** Chunk size for `String.fromCharCode` to avoid
|
|
2265
|
+
* hitting the call-stack limit on very large strings. */
|
|
2266
|
+
const CHUNK_SIZE = 8192;
|
|
2267
|
+
const normalizeForSearch = (text) => {
|
|
2268
|
+
let hasSpecial = false;
|
|
2269
|
+
for (let i = 0; i < text.length; i++) if (CHAR_MAP.has(text.charCodeAt(i))) {
|
|
2270
|
+
hasSpecial = true;
|
|
2271
|
+
break;
|
|
2272
|
+
}
|
|
2273
|
+
if (!hasSpecial) return text;
|
|
2274
|
+
const len = text.length;
|
|
2275
|
+
const codes = new Uint16Array(len);
|
|
2276
|
+
for (let i = 0; i < len; i++) {
|
|
2277
|
+
const code = text.charCodeAt(i);
|
|
2278
|
+
codes[i] = CHAR_MAP.get(code) ?? code;
|
|
2279
|
+
}
|
|
2280
|
+
if (len <= CHUNK_SIZE) return String.fromCharCode(...codes);
|
|
2281
|
+
let result = "";
|
|
2282
|
+
for (let offset = 0; offset < len; offset += CHUNK_SIZE) {
|
|
2283
|
+
const end = Math.min(offset + CHUNK_SIZE, len);
|
|
2284
|
+
result += String.fromCharCode(...codes.subarray(offset, end));
|
|
2285
|
+
}
|
|
2286
|
+
return result;
|
|
2287
|
+
};
|
|
2288
|
+
//#endregion
|
|
2289
|
+
//#region src/detectors/deny-list.ts
|
|
2290
|
+
/**
|
|
2291
|
+
* Try to load the optional @stll/anonymize-data package.
|
|
2292
|
+
* Returns null if not installed.
|
|
2293
|
+
*/
|
|
2294
|
+
const loadDataModule = async () => {
|
|
2295
|
+
try {
|
|
2296
|
+
return await import("@stll/anonymize-data");
|
|
2297
|
+
} catch {
|
|
2298
|
+
return null;
|
|
2299
|
+
}
|
|
2300
|
+
};
|
|
2301
|
+
const loadAllowList = (ctx) => {
|
|
2302
|
+
if (ctx.allowListPromise) return ctx.allowListPromise;
|
|
2303
|
+
ctx.allowListPromise = (async () => {
|
|
2304
|
+
try {
|
|
2305
|
+
const mod = await import("@stll/anonymize-data/config/allow-list.json");
|
|
2306
|
+
const set = new Set(mod.default?.words ?? []);
|
|
2307
|
+
ctx.allowList = set;
|
|
2308
|
+
return set;
|
|
2309
|
+
} catch {
|
|
2310
|
+
const empty = /* @__PURE__ */ new Set();
|
|
2311
|
+
ctx.allowList = empty;
|
|
2312
|
+
return empty;
|
|
2313
|
+
}
|
|
2314
|
+
})();
|
|
2315
|
+
return ctx.allowListPromise;
|
|
2316
|
+
};
|
|
2317
|
+
const EMPTY_ALLOW_LIST = /* @__PURE__ */ new Set();
|
|
2318
|
+
/** Sync accessor — returns empty set before init. */
|
|
2319
|
+
const getAllowList = (ctx) => ctx.allowList ?? EMPTY_ALLOW_LIST;
|
|
2320
|
+
/**
|
|
2321
|
+
* Common EU given names present in the stopwords-iso dataset
|
|
2322
|
+
* but absent from the first-name corpus. Without this
|
|
2323
|
+
* supplementary set, these names would pass through the
|
|
2324
|
+
* corpus-based filter and remain in the stopwords, silently
|
|
2325
|
+
* suppressing person detection.
|
|
2326
|
+
*
|
|
2327
|
+
* Sourced from EU member state birth registries (top-100
|
|
2328
|
+
* names) cross-referenced with stopwords.json.
|
|
2329
|
+
*/
|
|
2330
|
+
const SUPPLEMENTARY_NAME_EXCLUSIONS = new Set([
|
|
2331
|
+
"ana",
|
|
2332
|
+
"ben",
|
|
2333
|
+
"dan",
|
|
2334
|
+
"eden",
|
|
2335
|
+
"ella",
|
|
2336
|
+
"ina",
|
|
2337
|
+
"jo",
|
|
2338
|
+
"kai",
|
|
2339
|
+
"lena",
|
|
2340
|
+
"may",
|
|
2341
|
+
"mia",
|
|
2342
|
+
"sam",
|
|
2343
|
+
"sara",
|
|
2344
|
+
"sue",
|
|
2345
|
+
"tim",
|
|
2346
|
+
"tom"
|
|
2347
|
+
]);
|
|
2348
|
+
/**
|
|
2349
|
+
* Names from the first-name corpus (lowercased) that also
|
|
2350
|
+
* appear in the stopwords-iso dataset, plus supplementary
|
|
2351
|
+
* common EU given names not in the corpus. These must be
|
|
2352
|
+
* kept out of global STOPWORDS so that person detection is
|
|
2353
|
+
* not silently suppressed for real given names.
|
|
2354
|
+
*
|
|
2355
|
+
* Computed lazily after initNameCorpus() has populated
|
|
2356
|
+
* the first-name corpus. Re-builds if corpus size changes.
|
|
2357
|
+
*/
|
|
2358
|
+
const getFirstNameExclusions = (ctx) => {
|
|
2359
|
+
const corpus = getNameCorpusFirstNames(ctx);
|
|
2360
|
+
if (ctx.firstNameExclusions && corpus.length === ctx.firstNameExclusionCorpusLen) return ctx.firstNameExclusions;
|
|
2361
|
+
ctx.firstNameExclusionCorpusLen = corpus.length;
|
|
2362
|
+
const set = new Set([...corpus.map((n) => n.toLowerCase()), ...SUPPLEMENTARY_NAME_EXCLUSIONS]);
|
|
2363
|
+
ctx.firstNameExclusions = set;
|
|
2364
|
+
return set;
|
|
2365
|
+
};
|
|
2366
|
+
/**
|
|
2367
|
+
* Global stopwords: common words across 23 EU languages
|
|
2368
|
+
* sourced from the stopwords-iso dataset (MIT license).
|
|
2369
|
+
* Checked case-insensitively against matches.
|
|
2370
|
+
*
|
|
2371
|
+
* Entries that collide with the first-name corpus are
|
|
2372
|
+
* excluded so they can still be detected as person names.
|
|
2373
|
+
*
|
|
2374
|
+
* Regenerate: bun packages/data/scripts/generate-stopwords.ts
|
|
2375
|
+
*/
|
|
2376
|
+
const loadStopwords = (ctx) => {
|
|
2377
|
+
if (ctx.stopwordsPromise) return ctx.stopwordsPromise;
|
|
2378
|
+
ctx.stopwordsPromise = (async () => {
|
|
2379
|
+
try {
|
|
2380
|
+
const list = ((await import("@stll/anonymize-data/config/stopwords.json")).default ?? []).filter((w) => !getFirstNameExclusions(ctx).has(w));
|
|
2381
|
+
const set = new Set(list);
|
|
2382
|
+
ctx.stopwords = set;
|
|
2383
|
+
return set;
|
|
2384
|
+
} catch (err) {
|
|
2385
|
+
console.warn("[anonymize] Failed to load stopwords.json — stopword filtering disabled:", err);
|
|
2386
|
+
const empty = /* @__PURE__ */ new Set();
|
|
2387
|
+
ctx.stopwords = empty;
|
|
2388
|
+
return empty;
|
|
2389
|
+
}
|
|
2390
|
+
})();
|
|
2391
|
+
return ctx.stopwordsPromise;
|
|
2392
|
+
};
|
|
2393
|
+
const EMPTY_STOPWORDS = /* @__PURE__ */ new Set();
|
|
2394
|
+
/** Sync accessor — returns empty set before init. */
|
|
2395
|
+
const getStopwords = (ctx) => ctx.stopwords ?? EMPTY_STOPWORDS;
|
|
2396
|
+
const loadPersonStopwords = (ctx) => {
|
|
2397
|
+
if (ctx.personStopwordsPromise) return ctx.personStopwordsPromise;
|
|
2398
|
+
ctx.personStopwordsPromise = (async () => {
|
|
2399
|
+
try {
|
|
2400
|
+
const mod = await import("@stll/anonymize-data/config/person-stopwords.json");
|
|
2401
|
+
const set = new Set(mod.default?.words ?? []);
|
|
2402
|
+
ctx.personStopwords = set;
|
|
2403
|
+
return set;
|
|
2404
|
+
} catch {
|
|
2405
|
+
const empty = /* @__PURE__ */ new Set();
|
|
2406
|
+
ctx.personStopwords = empty;
|
|
2407
|
+
return empty;
|
|
2408
|
+
}
|
|
2409
|
+
})();
|
|
2410
|
+
return ctx.personStopwordsPromise;
|
|
2411
|
+
};
|
|
2412
|
+
const EMPTY_PERSON_STOPWORDS = /* @__PURE__ */ new Set();
|
|
2413
|
+
/** Sync accessor — returns empty set before init. */
|
|
2414
|
+
const getPersonStopwords = (ctx) => ctx.personStopwords ?? EMPTY_PERSON_STOPWORDS;
|
|
2415
|
+
/**
|
|
2416
|
+
* Resolve which dictionaries to load based on country
|
|
2417
|
+
* and category filters, load them, and build the deny
|
|
2418
|
+
* list data. The returned data provides PatternEntry[]
|
|
2419
|
+
* for the unified builder and parallel arrays for
|
|
2420
|
+
* post-processing.
|
|
2421
|
+
*
|
|
2422
|
+
* Requires `@stll/anonymize-data` to be installed.
|
|
2423
|
+
* Returns null if the data package is not available.
|
|
2424
|
+
*/
|
|
2425
|
+
const buildDenyList = async (config, ctx = defaultContext) => {
|
|
2426
|
+
await initNameCorpus(ctx);
|
|
2427
|
+
await Promise.all([
|
|
2428
|
+
loadStopwords(ctx),
|
|
2429
|
+
loadAllowList(ctx),
|
|
2430
|
+
loadPersonStopwords(ctx),
|
|
2431
|
+
loadGenericRoles(ctx)
|
|
2432
|
+
]);
|
|
2433
|
+
const dataModule = await loadDataModule();
|
|
2434
|
+
if (!dataModule) return null;
|
|
2435
|
+
const allowedCountries = resolveCountries(config.denyListRegions, config.denyListCountries);
|
|
2436
|
+
const excluded = config.denyListExcludeCategories;
|
|
2437
|
+
const excludeCategories = excluded ? new Set(excluded) : /* @__PURE__ */ new Set();
|
|
2438
|
+
const ids = [...dataModule.ALL_DICTIONARY_IDS].filter((id) => {
|
|
2439
|
+
const meta = dataModule.DICTIONARY_META[id];
|
|
2440
|
+
if (!meta) return false;
|
|
2441
|
+
if (excludeCategories.has(meta.category)) return false;
|
|
2442
|
+
if (allowedCountries === null) return true;
|
|
2443
|
+
if (meta.country === null) return true;
|
|
2444
|
+
return allowedCountries.has(meta.country);
|
|
2445
|
+
});
|
|
2446
|
+
const patternList = [];
|
|
2447
|
+
const labelList = [];
|
|
2448
|
+
const sourceList = [];
|
|
2449
|
+
const patternIndex = /* @__PURE__ */ new Map();
|
|
2450
|
+
const results = await Promise.all(ids.map(async (id) => {
|
|
2451
|
+
return {
|
|
2452
|
+
id,
|
|
2453
|
+
entries: await dataModule.loadDictionary(id)
|
|
2454
|
+
};
|
|
2455
|
+
}));
|
|
2456
|
+
const addDenyListEntry = (entry, label) => {
|
|
2457
|
+
const normalized = normalizeForSearch(entry).replace(/[|\\]/g, "");
|
|
2458
|
+
if (normalized.length === 0) return;
|
|
2459
|
+
const lower = normalized.toLowerCase();
|
|
2460
|
+
const existing = patternIndex.get(lower);
|
|
2461
|
+
if (existing !== void 0) {
|
|
2462
|
+
if (!labelList[existing].includes(label)) labelList[existing].push(label);
|
|
2463
|
+
if (!sourceList[existing].includes("deny-list")) sourceList[existing].push("deny-list");
|
|
2464
|
+
} else {
|
|
2465
|
+
patternIndex.set(lower, patternList.length);
|
|
2466
|
+
patternList.push(normalized);
|
|
2467
|
+
labelList.push([label]);
|
|
2468
|
+
sourceList.push(["deny-list"]);
|
|
2469
|
+
}
|
|
2470
|
+
};
|
|
2471
|
+
for (const { id, entries } of results) {
|
|
2472
|
+
const meta = dataModule.DICTIONARY_META[id];
|
|
2473
|
+
if (!meta) continue;
|
|
2474
|
+
for (const entry of entries) addDenyListEntry(entry, meta.label);
|
|
2475
|
+
}
|
|
2476
|
+
if (!excludeCategories.has("Places")) {
|
|
2477
|
+
const cityCountries = allowedCountries !== null ? [...allowedCountries] : [
|
|
2478
|
+
"AT",
|
|
2479
|
+
"AU",
|
|
2480
|
+
"BE",
|
|
2481
|
+
"BG",
|
|
2482
|
+
"BR",
|
|
2483
|
+
"CA",
|
|
2484
|
+
"CH",
|
|
2485
|
+
"CZ",
|
|
2486
|
+
"DE",
|
|
2487
|
+
"DK",
|
|
2488
|
+
"ES",
|
|
2489
|
+
"FI",
|
|
2490
|
+
"FR",
|
|
2491
|
+
"GB",
|
|
2492
|
+
"GR",
|
|
2493
|
+
"HR",
|
|
2494
|
+
"HU",
|
|
2495
|
+
"IE",
|
|
2496
|
+
"IT",
|
|
2497
|
+
"LU",
|
|
2498
|
+
"NL",
|
|
2499
|
+
"NO",
|
|
2500
|
+
"NZ",
|
|
2501
|
+
"PL",
|
|
2502
|
+
"PT",
|
|
2503
|
+
"RO",
|
|
2504
|
+
"SE",
|
|
2505
|
+
"SI",
|
|
2506
|
+
"SK",
|
|
2507
|
+
"US"
|
|
2508
|
+
];
|
|
2509
|
+
const cityEntries = await dataModule.loadCityDictionaries(cityCountries);
|
|
2510
|
+
for (const entry of cityEntries) addDenyListEntry(entry, "address");
|
|
2511
|
+
}
|
|
2512
|
+
const addNameEntry = (name, source) => {
|
|
2513
|
+
const normalized = normalizeForSearch(name).replace(/[|\\]/g, "");
|
|
2514
|
+
if (normalized.length === 0) return;
|
|
2515
|
+
const lower = normalized.toLowerCase();
|
|
2516
|
+
const existing = patternIndex.get(lower);
|
|
2517
|
+
if (existing !== void 0) {
|
|
2518
|
+
if (!labelList[existing].includes("person")) labelList[existing].push("person");
|
|
2519
|
+
if (!sourceList[existing].includes(source)) sourceList[existing].push(source);
|
|
2520
|
+
} else {
|
|
2521
|
+
patternIndex.set(lower, patternList.length);
|
|
2522
|
+
patternList.push(normalized);
|
|
2523
|
+
labelList.push(["person"]);
|
|
2524
|
+
sourceList.push([source]);
|
|
2525
|
+
}
|
|
2526
|
+
};
|
|
2527
|
+
for (const name of getNameCorpusFirstNames(ctx)) addNameEntry(name, "first-name");
|
|
2528
|
+
for (const name of getNameCorpusSurnames(ctx)) addNameEntry(name, "surname");
|
|
2529
|
+
for (const title of getNameCorpusTitles(ctx)) {
|
|
2530
|
+
const norm = normalizeForSearch(title).replace(/[|\\]/g, "");
|
|
2531
|
+
if (norm.length === 0) continue;
|
|
2532
|
+
const lower = norm.toLowerCase();
|
|
2533
|
+
const existing = patternIndex.get(lower);
|
|
2534
|
+
if (existing !== void 0) {
|
|
2535
|
+
if (!sourceList[existing].includes("title")) sourceList[existing].push("title");
|
|
2536
|
+
} else {
|
|
2537
|
+
patternIndex.set(lower, patternList.length);
|
|
2538
|
+
patternList.push(norm);
|
|
2539
|
+
labelList.push(["person"]);
|
|
2540
|
+
sourceList.push(["title"]);
|
|
2541
|
+
}
|
|
2542
|
+
}
|
|
2543
|
+
if (patternList.length === 0) return null;
|
|
2544
|
+
return {
|
|
2545
|
+
labels: labelList,
|
|
2546
|
+
originals: patternList,
|
|
2547
|
+
sources: sourceList
|
|
2548
|
+
};
|
|
2549
|
+
};
|
|
2550
|
+
/**
|
|
2551
|
+
* Ensure all deny-list support data (stopwords, allow
|
|
2552
|
+
* list, person stopwords, generic roles) is loaded on
|
|
2553
|
+
* the given context. Call this before
|
|
2554
|
+
* processDenyListMatches / filterFalsePositives when
|
|
2555
|
+
* the search instance was built on a different context
|
|
2556
|
+
* (e.g. cachedSearch).
|
|
2557
|
+
*/
|
|
2558
|
+
const ensureDenyListData = async (ctx = defaultContext) => {
|
|
2559
|
+
await initNameCorpus(ctx);
|
|
2560
|
+
await Promise.all([
|
|
2561
|
+
loadStopwords(ctx),
|
|
2562
|
+
loadAllowList(ctx),
|
|
2563
|
+
loadPersonStopwords(ctx),
|
|
2564
|
+
loadGenericRoles(ctx)
|
|
2565
|
+
]);
|
|
2566
|
+
};
|
|
2567
|
+
/**
|
|
2568
|
+
* Process deny list matches from the unified search.
|
|
2569
|
+
* Receives all matches; filters to the deny list slice
|
|
2570
|
+
* via sliceStart/sliceEnd. Local index into data.labels,
|
|
2571
|
+
* data.originals, data.sources is match.pattern - sliceStart.
|
|
2572
|
+
*
|
|
2573
|
+
* Two-pass approach to reduce false positives:
|
|
2574
|
+
* 1. Collect all matches (case-insensitive,
|
|
2575
|
+
* whole-word via Rust automaton)
|
|
2576
|
+
* 2. Require uppercase start in source text
|
|
2577
|
+
* 3. For person names, require at least one
|
|
2578
|
+
* mid-sentence occurrence to prove proper noun
|
|
2579
|
+
* 4. Return all occurrences of validated terms
|
|
2580
|
+
*/
|
|
2581
|
+
const processDenyListMatches = (allMatches, sliceStart, sliceEnd, fullText, data, ctx = defaultContext) => {
|
|
2582
|
+
const matchesByPattern = /* @__PURE__ */ new Map();
|
|
2583
|
+
for (const match of allMatches) {
|
|
2584
|
+
const idx = match.pattern;
|
|
2585
|
+
if (idx < sliceStart || idx >= sliceEnd) continue;
|
|
2586
|
+
const localIdx = idx - sliceStart;
|
|
2587
|
+
const sourceChar = fullText[match.start] ?? "";
|
|
2588
|
+
if (!UPPER_START_RE.test(sourceChar)) continue;
|
|
2589
|
+
const matchText = fullText.slice(match.start, match.end);
|
|
2590
|
+
const keyword = matchText.toLowerCase();
|
|
2591
|
+
if (getStopwords(ctx).has(keyword) || getAllowList(ctx).has(keyword)) continue;
|
|
2592
|
+
if (ALL_UPPER_RE.test(matchText)) continue;
|
|
2593
|
+
const labels = data.labels[localIdx];
|
|
2594
|
+
if (!labels || labels.length === 0) continue;
|
|
2595
|
+
const entry = {
|
|
2596
|
+
start: match.start,
|
|
2597
|
+
end: match.end,
|
|
2598
|
+
labels,
|
|
2599
|
+
text: matchText,
|
|
2600
|
+
patternIdx: localIdx
|
|
2601
|
+
};
|
|
2602
|
+
const existing = matchesByPattern.get(localIdx);
|
|
2603
|
+
if (existing) existing.push(entry);
|
|
2604
|
+
else matchesByPattern.set(localIdx, [entry]);
|
|
2605
|
+
}
|
|
2606
|
+
const results = [];
|
|
2607
|
+
const nameHits = [];
|
|
2608
|
+
for (const [, matches] of Array.from(matchesByPattern)) {
|
|
2609
|
+
const first = matches[0];
|
|
2610
|
+
if (!first) continue;
|
|
2611
|
+
const hasPerson = first.labels.includes("person");
|
|
2612
|
+
const nonPersonLabels = first.labels.filter((l) => l !== "person");
|
|
2613
|
+
if (hasPerson) {
|
|
2614
|
+
const keyword = first.text.toLowerCase();
|
|
2615
|
+
if (!getPersonStopwords(ctx).has(keyword)) for (const m of matches) nameHits.push(m);
|
|
2616
|
+
}
|
|
2617
|
+
for (const m of matches) for (const label of nonPersonLabels) results.push({
|
|
2618
|
+
start: m.start,
|
|
2619
|
+
end: m.end,
|
|
2620
|
+
label,
|
|
2621
|
+
text: m.text,
|
|
2622
|
+
score: .9,
|
|
2623
|
+
source: DETECTION_SOURCES.DENY_LIST
|
|
2624
|
+
});
|
|
2625
|
+
}
|
|
2626
|
+
nameHits.sort((a, b) => a.start - b.start);
|
|
2627
|
+
const nameConsumed = /* @__PURE__ */ new Set();
|
|
2628
|
+
for (let i = 0; i < nameHits.length; i++) {
|
|
2629
|
+
if (nameConsumed.has(i)) continue;
|
|
2630
|
+
const hit = nameHits[i];
|
|
2631
|
+
if (!hit) continue;
|
|
2632
|
+
const chain = [hit];
|
|
2633
|
+
let j = i + 1;
|
|
2634
|
+
while (j < nameHits.length && chain.length < 5) {
|
|
2635
|
+
const next = nameHits[j];
|
|
2636
|
+
if (!next) break;
|
|
2637
|
+
const prev = chain.at(-1);
|
|
2638
|
+
if (!prev) break;
|
|
2639
|
+
const gap = fullText.slice(prev.end, next.start);
|
|
2640
|
+
if (gap.length > 4 || gap.length === 0 || gap.includes("\n") || gap.includes(" ")) break;
|
|
2641
|
+
chain.push(next);
|
|
2642
|
+
j++;
|
|
2643
|
+
}
|
|
2644
|
+
for (let k = i; k < i + chain.length; k++) nameConsumed.add(k);
|
|
2645
|
+
const first = chain.at(0);
|
|
2646
|
+
const last = chain.at(-1);
|
|
2647
|
+
if (!first || !last) continue;
|
|
2648
|
+
const extended = extendPersonName(fullText, first.start, last.end, ctx);
|
|
2649
|
+
const score = chain.length >= 2 ? .9 : .5;
|
|
2650
|
+
if (chain.length === 1 && isSentenceStart(fullText, first.start)) continue;
|
|
2651
|
+
results.push({
|
|
2652
|
+
start: first.start,
|
|
2653
|
+
end: extended.end,
|
|
2654
|
+
label: "person",
|
|
2655
|
+
text: extended.text,
|
|
2656
|
+
score,
|
|
2657
|
+
source: DETECTION_SOURCES.DENY_LIST
|
|
2658
|
+
});
|
|
2659
|
+
}
|
|
2660
|
+
extendCityDistricts(results, fullText);
|
|
2661
|
+
return results;
|
|
2662
|
+
};
|
|
2663
|
+
const DISTRICT_SUFFIX_RE = new RegExp(`^ (\\d{1,2}(?!\\d)|(?:XXX|XXIX|XXVIII|XXVII|XXVI|XXV|XXIV|XXIII|XXII|XXI|XX|XIX|XVIII|XVII|XVI|XV|XIV|XIII|XII|XI|X|IX|VIII|VII|VI|IV|III|II))(?=[\\s,;.)"\\n]|$)`);
|
|
2664
|
+
const POSTAL_PREFIX_RE = /(?:\d{5}|\d{3}\s\d{2})\s+$/;
|
|
2665
|
+
const TRAILING_WORD_EXCLUSIONS = new Set([
|
|
2666
|
+
"nájemce",
|
|
2667
|
+
"pronajímatel",
|
|
2668
|
+
"kupující",
|
|
2669
|
+
"prodávající",
|
|
2670
|
+
"objednatel",
|
|
2671
|
+
"zhotovitel",
|
|
2672
|
+
"dodavatel",
|
|
2673
|
+
"odběratel",
|
|
2674
|
+
"věřitel",
|
|
2675
|
+
"dlužník",
|
|
2676
|
+
"zadavatel",
|
|
2677
|
+
"uchazeč",
|
|
2678
|
+
"příjemce",
|
|
2679
|
+
"plátce",
|
|
2680
|
+
"správa",
|
|
2681
|
+
"sekretariát",
|
|
2682
|
+
"kancelář",
|
|
2683
|
+
"odbor",
|
|
2684
|
+
"oddělení",
|
|
2685
|
+
"úřad",
|
|
2686
|
+
"inspekce",
|
|
2687
|
+
"agentura",
|
|
2688
|
+
"článek",
|
|
2689
|
+
"smlouva",
|
|
2690
|
+
"dodatek",
|
|
2691
|
+
"příloha",
|
|
2692
|
+
"předmět",
|
|
2693
|
+
"podmínky",
|
|
2694
|
+
"ustanovení"
|
|
2695
|
+
]);
|
|
2696
|
+
const extendCityDistricts = (entities, fullText) => {
|
|
2697
|
+
for (const entity of entities) {
|
|
2698
|
+
if (entity.label !== "address") continue;
|
|
2699
|
+
const afterMatch = fullText.slice(entity.end);
|
|
2700
|
+
const suffixM = DISTRICT_SUFFIX_RE.exec(afterMatch);
|
|
2701
|
+
if (suffixM) {
|
|
2702
|
+
entity.end += suffixM[0].length;
|
|
2703
|
+
entity.text = fullText.slice(entity.start, entity.end);
|
|
2704
|
+
}
|
|
2705
|
+
const afterDistrict = fullText.slice(entity.end);
|
|
2706
|
+
const dashDistrictM = /^[\s]*[-–][\s]*(\p{Lu}\p{Ll}+)/u.exec(afterDistrict);
|
|
2707
|
+
if (dashDistrictM && !dashDistrictM[0].includes("\n")) {
|
|
2708
|
+
entity.end += dashDistrictM[0].length;
|
|
2709
|
+
entity.text = fullText.slice(entity.start, entity.end);
|
|
2710
|
+
}
|
|
2711
|
+
const beforeMatch = fullText.slice(Math.max(0, entity.start - 10), entity.start);
|
|
2712
|
+
const prefixM = POSTAL_PREFIX_RE.exec(beforeMatch);
|
|
2713
|
+
if (prefixM) {
|
|
2714
|
+
entity.start -= prefixM[0].length;
|
|
2715
|
+
entity.text = fullText.slice(entity.start, entity.end);
|
|
2716
|
+
}
|
|
2717
|
+
const afterExt = fullText.slice(entity.end);
|
|
2718
|
+
const trailingWordM = /^[\s]+(\p{Lu}\p{Ll}+)/u.exec(afterExt);
|
|
2719
|
+
if (trailingWordM && !trailingWordM[0].includes("\n")) {
|
|
2720
|
+
const candidate = (trailingWordM[1] ?? "").toLowerCase();
|
|
2721
|
+
if (!TRAILING_WORD_EXCLUSIONS.has(candidate)) {
|
|
2722
|
+
entity.end += trailingWordM[0].length;
|
|
2723
|
+
entity.text = fullText.slice(entity.start, entity.end);
|
|
2724
|
+
}
|
|
2725
|
+
}
|
|
2726
|
+
}
|
|
2727
|
+
};
|
|
2728
|
+
/**
|
|
2729
|
+
* Extend a person name match to include subsequent
|
|
2730
|
+
* capitalized words. "Pavel" + " Heřmánek" → "Pavel
|
|
2731
|
+
* Heřmánek". Stops at lowercase words, punctuation,
|
|
2732
|
+
* or end of text. Also extends backward if preceded
|
|
2733
|
+
* by a capitalized word (for "Miroslav Braňka" when
|
|
2734
|
+
* only "Braňka" matched).
|
|
2735
|
+
*/
|
|
2736
|
+
const extendPersonName = (text, start, end, ctx) => {
|
|
2737
|
+
let newEnd = end;
|
|
2738
|
+
let pos = newEnd;
|
|
2739
|
+
while (pos < text.length) if (pos < text.length && text[pos] === " ") {
|
|
2740
|
+
const wordStart = pos + 1;
|
|
2741
|
+
if (wordStart >= text.length) break;
|
|
2742
|
+
const char = text[wordStart] ?? "";
|
|
2743
|
+
if (!UPPER_START_RE.test(char)) break;
|
|
2744
|
+
let wordEnd = wordStart;
|
|
2745
|
+
while (wordEnd < text.length && !/\s/.test(text[wordEnd] ?? "")) wordEnd++;
|
|
2746
|
+
const stripped = text.slice(wordStart, wordEnd).replace(/[,;.]+$/, "");
|
|
2747
|
+
if (stripped.length < 2) break;
|
|
2748
|
+
const lower = stripped.toLowerCase();
|
|
2749
|
+
if (getStopwords(ctx).has(lower) || getPersonStopwords(ctx).has(lower)) break;
|
|
2750
|
+
newEnd = wordStart + stripped.length;
|
|
2751
|
+
pos = newEnd;
|
|
2752
|
+
} else break;
|
|
2753
|
+
return {
|
|
2754
|
+
end: newEnd,
|
|
2755
|
+
text: text.slice(start, newEnd)
|
|
2756
|
+
};
|
|
2757
|
+
};
|
|
2758
|
+
//#endregion
|
|
2759
|
+
//#region src/detectors/address-seeds.ts
|
|
2760
|
+
let cachedBoundaryRe = null;
|
|
2761
|
+
const loadBoundaryWords = async () => {
|
|
2762
|
+
try {
|
|
2763
|
+
return (await import("@stll/anonymize-data/config/address-boundaries.json")).default;
|
|
2764
|
+
} catch {
|
|
2765
|
+
return {};
|
|
2766
|
+
}
|
|
2767
|
+
};
|
|
2768
|
+
/**
|
|
2769
|
+
* Build regex for boundary words. Matches any
|
|
2770
|
+
* boundary word preceded by a word boundary.
|
|
2771
|
+
*/
|
|
2772
|
+
const getBoundaryRe = async () => {
|
|
2773
|
+
if (cachedBoundaryRe) return cachedBoundaryRe;
|
|
2774
|
+
const config = await loadBoundaryWords();
|
|
2775
|
+
const words = [];
|
|
2776
|
+
for (const entries of Object.values(config)) {
|
|
2777
|
+
if (!Array.isArray(entries)) continue;
|
|
2778
|
+
for (const word of entries) words.push(word.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"));
|
|
2779
|
+
}
|
|
2780
|
+
words.sort((a, b) => b.length - a.length);
|
|
2781
|
+
cachedBoundaryRe = words.length > 0 ? new RegExp(`\\b(?:${words.join("|")})\\b`, "i") : /(?!)/;
|
|
2782
|
+
return cachedBoundaryRe;
|
|
2783
|
+
};
|
|
2784
|
+
/**
|
|
2785
|
+
* Build street type patterns for the unified search.
|
|
2786
|
+
* Returns string[] for the unified TextSearch
|
|
2787
|
+
* builder. Empty if data package is not installed.
|
|
2788
|
+
*/
|
|
2789
|
+
const buildStreetTypePatterns = async () => {
|
|
2790
|
+
let config = {};
|
|
2791
|
+
try {
|
|
2792
|
+
config = (await import("@stll/anonymize-data/config/address-street-types.json")).default;
|
|
2793
|
+
} catch {
|
|
2794
|
+
return [];
|
|
2795
|
+
}
|
|
2796
|
+
const words = [];
|
|
2797
|
+
for (const values of Object.values(config)) {
|
|
2798
|
+
if (!Array.isArray(values)) continue;
|
|
2799
|
+
for (const word of values) words.push(word);
|
|
2800
|
+
}
|
|
2801
|
+
return words;
|
|
2802
|
+
};
|
|
2803
|
+
const collectSeeds = (allMatches, sliceStart, sliceEnd, fullText, existingEntities) => {
|
|
2804
|
+
const seeds = [];
|
|
2805
|
+
for (const match of allMatches) {
|
|
2806
|
+
const idx = match.pattern;
|
|
2807
|
+
if (idx < sliceStart || idx >= sliceEnd) continue;
|
|
2808
|
+
seeds.push({
|
|
2809
|
+
type: "street-word",
|
|
2810
|
+
start: match.start,
|
|
2811
|
+
end: match.end,
|
|
2812
|
+
text: match.text
|
|
2813
|
+
});
|
|
2814
|
+
}
|
|
2815
|
+
for (const e of existingEntities) {
|
|
2816
|
+
if (e.label !== "address") continue;
|
|
2817
|
+
if (e.source === "deny-list") seeds.push({
|
|
2818
|
+
type: "city",
|
|
2819
|
+
start: e.start,
|
|
2820
|
+
end: e.end,
|
|
2821
|
+
text: e.text
|
|
2822
|
+
});
|
|
2823
|
+
else if (e.source === "trigger" && /^\d/.test(e.text)) seeds.push({
|
|
2824
|
+
type: "postal-code",
|
|
2825
|
+
start: e.start,
|
|
2826
|
+
end: e.end,
|
|
2827
|
+
text: e.text
|
|
2828
|
+
});
|
|
2829
|
+
else if (e.source === "trigger") seeds.push({
|
|
2830
|
+
type: "address-trigger",
|
|
2831
|
+
start: e.start,
|
|
2832
|
+
end: e.end,
|
|
2833
|
+
text: e.text
|
|
2834
|
+
});
|
|
2835
|
+
}
|
|
2836
|
+
const postalRe = /\b(?:\d{3}\s\d{2}|\d{2}-\d{3})\b/g;
|
|
2837
|
+
let postalMatch;
|
|
2838
|
+
while ((postalMatch = postalRe.exec(fullText)) !== null) {
|
|
2839
|
+
const start = postalMatch.index;
|
|
2840
|
+
const end = start + postalMatch[0].length;
|
|
2841
|
+
if (!seeds.some((s) => s.start <= start && s.end >= end)) seeds.push({
|
|
2842
|
+
type: "postal-code",
|
|
2843
|
+
start,
|
|
2844
|
+
end,
|
|
2845
|
+
text: postalMatch[0]
|
|
2846
|
+
});
|
|
2847
|
+
}
|
|
2848
|
+
const streetNumRe = /\b(\p{Lu}\p{Ll}{2,})\s+(\d{1,5}(?:\/\d{1,5})?)\s*[,\n]/gu;
|
|
2849
|
+
let streetMatch;
|
|
2850
|
+
while ((streetMatch = streetNumRe.exec(fullText)) !== null) {
|
|
2851
|
+
const matchedStreet = streetMatch[1];
|
|
2852
|
+
const matchedNum = streetMatch[2];
|
|
2853
|
+
if (!matchedStreet || !matchedNum) continue;
|
|
2854
|
+
const start = streetMatch.index;
|
|
2855
|
+
const end = start + matchedStreet.length + 1 + matchedNum.length;
|
|
2856
|
+
seeds.push({
|
|
2857
|
+
type: "street-word",
|
|
2858
|
+
start,
|
|
2859
|
+
end,
|
|
2860
|
+
text: fullText.slice(start, end)
|
|
2861
|
+
});
|
|
2862
|
+
}
|
|
2863
|
+
return seeds.sort((a, b) => a.start - b.start);
|
|
2864
|
+
};
|
|
2865
|
+
const clusterSeeds = (seeds, maxGap) => {
|
|
2866
|
+
const first = seeds[0];
|
|
2867
|
+
if (!first) return [];
|
|
2868
|
+
const clusters = [];
|
|
2869
|
+
let current = {
|
|
2870
|
+
seeds: [first],
|
|
2871
|
+
start: first.start,
|
|
2872
|
+
end: first.end
|
|
2873
|
+
};
|
|
2874
|
+
for (let i = 1; i < seeds.length; i++) {
|
|
2875
|
+
const seed = seeds.at(i);
|
|
2876
|
+
if (!seed) continue;
|
|
2877
|
+
if (seed.start - current.end <= maxGap) {
|
|
2878
|
+
current.seeds.push(seed);
|
|
2879
|
+
current.end = Math.max(current.end, seed.end);
|
|
2880
|
+
} else {
|
|
2881
|
+
clusters.push(current);
|
|
2882
|
+
current = {
|
|
2883
|
+
seeds: [seed],
|
|
2884
|
+
start: seed.start,
|
|
2885
|
+
end: seed.end
|
|
2886
|
+
};
|
|
2887
|
+
}
|
|
2888
|
+
}
|
|
2889
|
+
clusters.push(current);
|
|
2890
|
+
return clusters;
|
|
2891
|
+
};
|
|
2892
|
+
const scoreCluster = (cluster) => {
|
|
2893
|
+
const types = new Set(cluster.seeds.map((s) => s.type));
|
|
2894
|
+
if (types.size < 2) return 0;
|
|
2895
|
+
let score = .5;
|
|
2896
|
+
if (types.has("postal-code")) score += .15;
|
|
2897
|
+
if (types.has("city")) score += .15;
|
|
2898
|
+
if (types.has("street-word")) score += .15;
|
|
2899
|
+
if (types.has("address-trigger")) score += .1;
|
|
2900
|
+
return Math.min(score, .95);
|
|
2901
|
+
};
|
|
2902
|
+
const NON_ADDRESS_LABELS = new Set([
|
|
2903
|
+
"registration number",
|
|
2904
|
+
"tax identification number",
|
|
2905
|
+
"person",
|
|
2906
|
+
"bank account number",
|
|
2907
|
+
"email address",
|
|
2908
|
+
"phone number",
|
|
2909
|
+
"organization",
|
|
2910
|
+
"iban"
|
|
2911
|
+
]);
|
|
2912
|
+
const expandCluster = async (fullText, cluster, existingEntities) => {
|
|
2913
|
+
const { start, end } = cluster;
|
|
2914
|
+
let leftBound = 0;
|
|
2915
|
+
for (const e of existingEntities) if (NON_ADDRESS_LABELS.has(e.label) && e.end <= start && e.end > leftBound) leftBound = e.end;
|
|
2916
|
+
let leftPos = start;
|
|
2917
|
+
while (leftPos > leftBound) {
|
|
2918
|
+
let p = leftPos - 1;
|
|
2919
|
+
while (p >= 0 && (fullText[p] === " " || fullText[p] === ",")) p--;
|
|
2920
|
+
if (p < 0) break;
|
|
2921
|
+
let wordEnd = p + 1;
|
|
2922
|
+
while (p >= 0 && /\S/.test(fullText[p] ?? "")) p--;
|
|
2923
|
+
const word = fullText.slice(p + 1, wordEnd);
|
|
2924
|
+
if (word.length < 2 || !/^\p{Lu}/u.test(word) && !/^\d/.test(word)) break;
|
|
2925
|
+
if (fullText.slice(p + 1, leftPos).includes("\n")) break;
|
|
2926
|
+
leftPos = p + 1;
|
|
2927
|
+
}
|
|
2928
|
+
let rightPos = end;
|
|
2929
|
+
const remaining = fullText.slice(rightPos);
|
|
2930
|
+
let nearestBoundary = Math.min(remaining.length, 200);
|
|
2931
|
+
const boundaryMatch = (await getBoundaryRe()).exec(remaining);
|
|
2932
|
+
if (boundaryMatch && boundaryMatch.index < nearestBoundary) nearestBoundary = boundaryMatch.index;
|
|
2933
|
+
for (const e of existingEntities) {
|
|
2934
|
+
if (!NON_ADDRESS_LABELS.has(e.label)) continue;
|
|
2935
|
+
const offset = e.start - rightPos;
|
|
2936
|
+
if (offset > 0 && offset < nearestBoundary) nearestBoundary = offset;
|
|
2937
|
+
}
|
|
2938
|
+
const doubleNewline = remaining.indexOf("\n\n");
|
|
2939
|
+
if (doubleNewline !== -1 && doubleNewline < nearestBoundary) nearestBoundary = doubleNewline;
|
|
2940
|
+
rightPos = end + remaining.slice(0, nearestBoundary).trimEnd().length;
|
|
2941
|
+
while (rightPos > end && /[,;:\s]/.test(fullText[rightPos - 1] ?? "")) rightPos--;
|
|
2942
|
+
return {
|
|
2943
|
+
start: Math.min(leftPos, start),
|
|
2944
|
+
end: Math.max(rightPos, end)
|
|
2945
|
+
};
|
|
2946
|
+
};
|
|
2947
|
+
/**
|
|
2948
|
+
* Process address seeds from the unified search.
|
|
2949
|
+
* Receives all matches; filters to the street types
|
|
2950
|
+
* slice via sliceStart/sliceEnd. Uses fullText and
|
|
2951
|
+
* existingEntities for seed collection, clustering,
|
|
2952
|
+
* expansion, and scoring.
|
|
2953
|
+
*
|
|
2954
|
+
* Runs as a post-processor after all other detectors,
|
|
2955
|
+
* using their output as seed sources.
|
|
2956
|
+
*/
|
|
2957
|
+
const processAddressSeeds = async (allMatches, sliceStart, sliceEnd, fullText, existingEntities) => {
|
|
2958
|
+
const clusters = clusterSeeds(collectSeeds(allMatches, sliceStart, sliceEnd, fullText, existingEntities), 150);
|
|
2959
|
+
const results = [];
|
|
2960
|
+
for (const cluster of clusters) {
|
|
2961
|
+
const score = scoreCluster(cluster);
|
|
2962
|
+
if (score < .6) continue;
|
|
2963
|
+
const { start, end } = await expandCluster(fullText, cluster, existingEntities);
|
|
2964
|
+
const text = fullText.slice(start, end).trim();
|
|
2965
|
+
if (text.length < 5 || text.length > 300) continue;
|
|
2966
|
+
if (text.includes("\n")) continue;
|
|
2967
|
+
results.push({
|
|
2968
|
+
start,
|
|
2969
|
+
end: start + text.length,
|
|
2970
|
+
label: "address",
|
|
2971
|
+
text,
|
|
2972
|
+
score,
|
|
2973
|
+
source: DETECTION_SOURCES.REGEX
|
|
2974
|
+
});
|
|
2975
|
+
}
|
|
2976
|
+
return results;
|
|
2977
|
+
};
|
|
2978
|
+
//#endregion
|
|
2979
|
+
//#region src/detectors/org-propagation.ts
|
|
2980
|
+
const TRAILING_SEP = /[,\s]+$/;
|
|
2981
|
+
const WORD_CHAR_RE = /[\p{L}\p{N}]/u;
|
|
2982
|
+
const ORG_PROPAGATION_SCORE = .9;
|
|
2983
|
+
/**
|
|
2984
|
+
* After the main detection pass, collect organization
|
|
2985
|
+
* entities with a legal form suffix, strip the suffix
|
|
2986
|
+
* to get the base name, and re-scan the full text for
|
|
2987
|
+
* bare mentions of that base name. Returns new entities
|
|
2988
|
+
* for occurrences not already covered.
|
|
2989
|
+
*/
|
|
2990
|
+
const propagateOrgNames = (entities, fullText) => {
|
|
2991
|
+
const seeds = [];
|
|
2992
|
+
const seenBases = /* @__PURE__ */ new Set();
|
|
2993
|
+
for (const e of entities) {
|
|
2994
|
+
if (e.label !== "organization") continue;
|
|
2995
|
+
for (const suffix of LEGAL_SUFFIXES) if (e.text.endsWith(suffix)) {
|
|
2996
|
+
const base = e.text.slice(0, -suffix.length).replace(TRAILING_SEP, "").trim();
|
|
2997
|
+
if (base.length >= 3 && !seenBases.has(base)) {
|
|
2998
|
+
seenBases.add(base);
|
|
2999
|
+
seeds.push({
|
|
3000
|
+
baseName: base,
|
|
3001
|
+
label: e.label
|
|
3002
|
+
});
|
|
3003
|
+
}
|
|
3004
|
+
break;
|
|
3005
|
+
}
|
|
3006
|
+
}
|
|
3007
|
+
if (seeds.length === 0) return [];
|
|
3008
|
+
const covered = entities.map((e) => [e.start, e.end]);
|
|
3009
|
+
const isOverlapping = (start, end) => covered.some(([cs, ce]) => start < ce && end > cs);
|
|
3010
|
+
const results = [];
|
|
3011
|
+
for (const seed of seeds) {
|
|
3012
|
+
const { baseName, label } = seed;
|
|
3013
|
+
let searchFrom = 0;
|
|
3014
|
+
while (searchFrom < fullText.length) {
|
|
3015
|
+
const idx = fullText.indexOf(baseName, searchFrom);
|
|
3016
|
+
if (idx === -1) break;
|
|
3017
|
+
const matchEnd = idx + baseName.length;
|
|
3018
|
+
const prevCh = fullText[idx - 1] ?? "";
|
|
3019
|
+
const nextCh = fullText[matchEnd] ?? "";
|
|
3020
|
+
if (WORD_CHAR_RE.test(prevCh) || WORD_CHAR_RE.test(nextCh)) {
|
|
3021
|
+
searchFrom = idx + 1;
|
|
3022
|
+
continue;
|
|
3023
|
+
}
|
|
3024
|
+
if (!isOverlapping(idx, matchEnd)) {
|
|
3025
|
+
results.push({
|
|
3026
|
+
start: idx,
|
|
3027
|
+
end: matchEnd,
|
|
3028
|
+
label,
|
|
3029
|
+
text: baseName,
|
|
3030
|
+
score: ORG_PROPAGATION_SCORE,
|
|
3031
|
+
source: DETECTION_SOURCES.COREFERENCE
|
|
3032
|
+
});
|
|
3033
|
+
covered.push([idx, matchEnd]);
|
|
3034
|
+
}
|
|
3035
|
+
searchFrom = matchEnd;
|
|
3036
|
+
}
|
|
3037
|
+
}
|
|
3038
|
+
return results;
|
|
3039
|
+
};
|
|
3040
|
+
//#endregion
|
|
3041
|
+
//#region src/filters/confidence-boost.ts
|
|
3042
|
+
const BARE_STOPWORDS = new Set([
|
|
3043
|
+
"Příloha",
|
|
3044
|
+
"Smlouva",
|
|
3045
|
+
"Článek",
|
|
3046
|
+
"Dodatek",
|
|
3047
|
+
"Celkem",
|
|
3048
|
+
"Strana",
|
|
3049
|
+
"Faktura",
|
|
3050
|
+
"Částka",
|
|
3051
|
+
"Položka",
|
|
3052
|
+
"Kapitola",
|
|
3053
|
+
"Zákon",
|
|
3054
|
+
"Vyhláška",
|
|
3055
|
+
"Nařízení",
|
|
3056
|
+
"Usnesení",
|
|
3057
|
+
"Rozsudek",
|
|
3058
|
+
"Bod",
|
|
3059
|
+
"Odstavec",
|
|
3060
|
+
"Záloha",
|
|
3061
|
+
"Zbývá",
|
|
3062
|
+
"Dne",
|
|
3063
|
+
"Platba",
|
|
3064
|
+
"Datum",
|
|
3065
|
+
"Splatnost",
|
|
3066
|
+
"Variabilní",
|
|
3067
|
+
"Konstantní",
|
|
3068
|
+
"Specifický"
|
|
3069
|
+
]);
|
|
3070
|
+
const NEAR_MISS_BAND = .15;
|
|
3071
|
+
const BOOST_PER_NEIGHBOUR = .05;
|
|
3072
|
+
const CONTEXT_WINDOW_CHARS = 150;
|
|
3073
|
+
const HIGH_CONFIDENCE_FLOOR = .9;
|
|
3074
|
+
/**
|
|
3075
|
+
* Boost confidence of near-miss NER entities that appear
|
|
3076
|
+
* near high-confidence detections (regex, trigger phrase).
|
|
3077
|
+
*
|
|
3078
|
+
* If an NER entity scored between (threshold - 0.15) and
|
|
3079
|
+
* threshold, count how many confirmed entities exist within
|
|
3080
|
+
* a 150-char window. Add +0.05 per co-located entity.
|
|
3081
|
+
* If the boosted score crosses the threshold, include it.
|
|
3082
|
+
*
|
|
3083
|
+
* Only mutates score on near-miss entities; high-confidence
|
|
3084
|
+
* entities pass through unchanged.
|
|
3085
|
+
*/
|
|
3086
|
+
const boostNearMissEntities = (entities, threshold) => {
|
|
3087
|
+
const nearMissBand = Math.max(0, threshold - NEAR_MISS_BAND);
|
|
3088
|
+
const confirmed = entities.filter((e) => e.score >= HIGH_CONFIDENCE_FLOOR);
|
|
3089
|
+
const boosted = [];
|
|
3090
|
+
for (const entity of entities) {
|
|
3091
|
+
if (entity.score >= threshold) {
|
|
3092
|
+
boosted.push(entity);
|
|
3093
|
+
continue;
|
|
3094
|
+
}
|
|
3095
|
+
if (entity.score < nearMissBand) continue;
|
|
3096
|
+
const midpoint = (entity.start + entity.end) / 2;
|
|
3097
|
+
let neighbourCount = 0;
|
|
3098
|
+
for (const anchor of confirmed) {
|
|
3099
|
+
const anchorMid = (anchor.start + anchor.end) / 2;
|
|
3100
|
+
if (Math.abs(midpoint - anchorMid) <= CONTEXT_WINDOW_CHARS) neighbourCount++;
|
|
3101
|
+
}
|
|
3102
|
+
const boostedScore = entity.score + neighbourCount * BOOST_PER_NEIGHBOUR;
|
|
3103
|
+
if (boostedScore >= threshold) boosted.push({
|
|
3104
|
+
...entity,
|
|
3105
|
+
score: boostedScore
|
|
3106
|
+
});
|
|
3107
|
+
}
|
|
3108
|
+
return boosted;
|
|
3109
|
+
};
|
|
3110
|
+
const UPPER_WORD_RE = /\p{Lu}/u;
|
|
3111
|
+
/** Header zone: top 15% of document */
|
|
3112
|
+
const HEADER_ZONE_FRACTION = .15;
|
|
3113
|
+
/** Context window for address adjacency */
|
|
3114
|
+
const STREET_CONTEXT_WINDOW = 200;
|
|
3115
|
+
let _addressPreps = null;
|
|
3116
|
+
let _temporalPreps = null;
|
|
3117
|
+
let _prepsPromise = null;
|
|
3118
|
+
const loadPrepositions = async () => {
|
|
3119
|
+
try {
|
|
3120
|
+
const mod = await import("@stll/anonymize-data/config/address-prepositions.json");
|
|
3121
|
+
const data = mod.default ?? mod;
|
|
3122
|
+
const addr = /* @__PURE__ */ new Set();
|
|
3123
|
+
const temp = /* @__PURE__ */ new Set();
|
|
3124
|
+
for (const words of Object.values(data.address)) if (Array.isArray(words)) for (const w of words) addr.add(w.toLowerCase());
|
|
3125
|
+
for (const words of Object.values(data.temporal)) if (Array.isArray(words)) for (const w of words) temp.add(w.toLowerCase());
|
|
3126
|
+
_addressPreps = addr;
|
|
3127
|
+
_temporalPreps = temp;
|
|
3128
|
+
} catch {
|
|
3129
|
+
_addressPreps = /* @__PURE__ */ new Set();
|
|
3130
|
+
_temporalPreps = /* @__PURE__ */ new Set();
|
|
3131
|
+
}
|
|
3132
|
+
};
|
|
3133
|
+
/** Ensure preposition data is loaded. */
|
|
3134
|
+
const initPrepositions = () => {
|
|
3135
|
+
if (!_prepsPromise) _prepsPromise = loadPrepositions();
|
|
3136
|
+
return _prepsPromise;
|
|
3137
|
+
};
|
|
3138
|
+
const getAddressPreps = () => _addressPreps ?? /* @__PURE__ */ new Set();
|
|
3139
|
+
const getTemporalPreps = () => _temporalPreps ?? /* @__PURE__ */ new Set();
|
|
3140
|
+
let _streetAbbrevs = null;
|
|
3141
|
+
let _streetAbbrevsPromise = null;
|
|
3142
|
+
const loadStreetAbbrevs = async () => {
|
|
3143
|
+
try {
|
|
3144
|
+
const mod = await import("@stll/anonymize-data/config/address-street-types.json");
|
|
3145
|
+
const data = mod.default ?? mod;
|
|
3146
|
+
const abbrevs = /* @__PURE__ */ new Set();
|
|
3147
|
+
for (const [key, words] of Object.entries(data)) {
|
|
3148
|
+
if (key.startsWith("_")) continue;
|
|
3149
|
+
if (!Array.isArray(words)) continue;
|
|
3150
|
+
for (const w of words) if (w.includes(".")) abbrevs.add(w.toLowerCase());
|
|
3151
|
+
}
|
|
3152
|
+
_streetAbbrevs = abbrevs;
|
|
3153
|
+
} catch {
|
|
3154
|
+
_streetAbbrevs = /* @__PURE__ */ new Set();
|
|
3155
|
+
}
|
|
3156
|
+
};
|
|
3157
|
+
/** Ensure street abbreviation data is loaded. */
|
|
3158
|
+
const initStreetAbbrevs = () => {
|
|
3159
|
+
if (!_streetAbbrevsPromise) _streetAbbrevsPromise = loadStreetAbbrevs();
|
|
3160
|
+
return _streetAbbrevsPromise;
|
|
3161
|
+
};
|
|
3162
|
+
const getStreetAbbrevs = () => _streetAbbrevs ?? /* @__PURE__ */ new Set();
|
|
3163
|
+
/**
|
|
3164
|
+
* Scan backwards from known address entities and
|
|
3165
|
+
* house number patterns to find street names.
|
|
3166
|
+
*
|
|
3167
|
+
* Strategy (a): from a house number like "2512/2a",
|
|
3168
|
+
* walk left to find the first uppercase word — that's
|
|
3169
|
+
* the street name start. "Mezi úvozy 2512/2a" →
|
|
3170
|
+
* captures "Mezi úvozy 2512/2a".
|
|
3171
|
+
*
|
|
3172
|
+
* Strategy (b): if a colon ":" appears within 3 chars
|
|
3173
|
+
* before the street start, boost confidence. Colons
|
|
3174
|
+
* signal "label: value" pairs universal in contracts.
|
|
3175
|
+
*
|
|
3176
|
+
* Strategy (c): in the header zone (top 15% of doc),
|
|
3177
|
+
* be more aggressive — detect street patterns even
|
|
3178
|
+
* without nearby address entities.
|
|
3179
|
+
*/
|
|
3180
|
+
const detectStreetPatternsNearAddresses = (fullText, existingEntities) => {
|
|
3181
|
+
const results = [];
|
|
3182
|
+
const addressEntities = existingEntities.filter((e) => e.label === "address");
|
|
3183
|
+
const headerEnd = Math.floor(fullText.length * HEADER_ZONE_FRACTION);
|
|
3184
|
+
const houseNumRe = /\b(?:\d{1,4}\/\d+[a-zA-Z]\b|\d{3,4}\/\d+\b|(?:1[3-9]|[2-9]\d)\/\d{3,}\b)/g;
|
|
3185
|
+
houseNumRe.lastIndex = 0;
|
|
3186
|
+
for (let m = houseNumRe.exec(fullText); m !== null; m = houseNumRe.exec(fullText)) {
|
|
3187
|
+
const numStart = m.index;
|
|
3188
|
+
const numEnd = numStart + m[0].length;
|
|
3189
|
+
if (existingEntities.some((e) => e.start <= numStart && e.end >= numEnd)) continue;
|
|
3190
|
+
const inHeader = numStart < headerEnd;
|
|
3191
|
+
const nearAddress = addressEntities.some((e) => Math.abs(e.start - numEnd) < STREET_CONTEXT_WINDOW || Math.abs(e.end - numStart) < STREET_CONTEXT_WINDOW);
|
|
3192
|
+
if (!inHeader && !nearAddress) continue;
|
|
3193
|
+
let scanPos = numStart - 1;
|
|
3194
|
+
while (scanPos >= 0 && /[\s\u00A0]/.test(fullText[scanPos] ?? "")) scanPos--;
|
|
3195
|
+
if (scanPos < 0) continue;
|
|
3196
|
+
let hasTemporalPrep = false;
|
|
3197
|
+
let streetStart = scanPos + 1;
|
|
3198
|
+
let wordCount = 0;
|
|
3199
|
+
const MAX_WORDS = 5;
|
|
3200
|
+
while (scanPos >= 0 && wordCount < MAX_WORDS) {
|
|
3201
|
+
let wordEnd = scanPos + 1;
|
|
3202
|
+
const hasDot = fullText[scanPos] === ".";
|
|
3203
|
+
if (hasDot) scanPos--;
|
|
3204
|
+
while (scanPos >= 0 && /[\p{L}\p{M}\d]/u.test(fullText[scanPos] ?? "")) scanPos--;
|
|
3205
|
+
const wordStart = scanPos + 1;
|
|
3206
|
+
const rawWord = fullText.slice(wordStart, wordEnd);
|
|
3207
|
+
const word = hasDot ? rawWord.slice(0, -1) : rawWord;
|
|
3208
|
+
if (word.length === 0) break;
|
|
3209
|
+
const isStreetAbbrev = hasDot && getStreetAbbrevs().has(rawWord.toLowerCase());
|
|
3210
|
+
const isUpper = UPPER_WORD_RE.test(word[0] ?? "");
|
|
3211
|
+
const isPrep = getAddressPreps().has(word.toLowerCase());
|
|
3212
|
+
const isDigitToken = /^\d{1,2}$/.test(word);
|
|
3213
|
+
if (!isUpper && !isPrep && !isStreetAbbrev && !isDigitToken) break;
|
|
3214
|
+
if (isPrep && getTemporalPreps().has(word.toLowerCase())) hasTemporalPrep = true;
|
|
3215
|
+
streetStart = wordStart;
|
|
3216
|
+
wordCount++;
|
|
3217
|
+
while (scanPos >= 0 && /[\s\u00A0]/.test(fullText[scanPos] ?? "")) scanPos--;
|
|
3218
|
+
const prevCh = fullText[scanPos];
|
|
3219
|
+
if (prevCh === "\n" || prevCh === " " || prevCh === ";" || prevCh === void 0) break;
|
|
3220
|
+
if (prevCh === ",") break;
|
|
3221
|
+
}
|
|
3222
|
+
if (wordCount === 0) continue;
|
|
3223
|
+
const streetText = fullText.slice(streetStart, numEnd);
|
|
3224
|
+
if (streetText.length < 4) continue;
|
|
3225
|
+
if (hasTemporalPrep) continue;
|
|
3226
|
+
if (existingEntities.some((e) => e.start <= streetStart && e.end >= numEnd)) continue;
|
|
3227
|
+
const score = fullText.slice(Math.max(0, streetStart - 5), streetStart).includes(":") ? .95 : inHeader ? .85 : .8;
|
|
3228
|
+
results.push({
|
|
3229
|
+
start: streetStart,
|
|
3230
|
+
end: numEnd,
|
|
3231
|
+
label: "address",
|
|
3232
|
+
text: streetText,
|
|
3233
|
+
score,
|
|
3234
|
+
source: "regex"
|
|
3235
|
+
});
|
|
3236
|
+
}
|
|
3237
|
+
const bareHouseRe = /(?<=\s|^)(\p{Lu}\p{Ll}[\p{Ll}\p{Lu}]+\s+\d{1,3})\b/gu;
|
|
3238
|
+
bareHouseRe.lastIndex = 0;
|
|
3239
|
+
const allAddr = [...addressEntities, ...results];
|
|
3240
|
+
for (let m = bareHouseRe.exec(fullText); m !== null; m = bareHouseRe.exec(fullText)) {
|
|
3241
|
+
const captured = m[1];
|
|
3242
|
+
if (captured === void 0) continue;
|
|
3243
|
+
const start = m.index;
|
|
3244
|
+
const end = start + captured.length;
|
|
3245
|
+
if (!allAddr.some((e) => {
|
|
3246
|
+
if (Math.min(Math.abs(e.start - end), Math.abs(e.end - start)) > 50) return false;
|
|
3247
|
+
const lo = Math.min(e.start, start);
|
|
3248
|
+
const hi = Math.max(e.end, end);
|
|
3249
|
+
return !fullText.slice(lo, hi).includes("\n");
|
|
3250
|
+
})) continue;
|
|
3251
|
+
const spaceIdx = captured.search(/\s+\d/);
|
|
3252
|
+
const word = spaceIdx > 0 ? captured.slice(0, spaceIdx) : captured;
|
|
3253
|
+
if (BARE_STOPWORDS.has(word)) continue;
|
|
3254
|
+
if ([...existingEntities, ...results].some((e) => e.start < end && e.end > start)) continue;
|
|
3255
|
+
results.push({
|
|
3256
|
+
start,
|
|
3257
|
+
end,
|
|
3258
|
+
label: "address",
|
|
3259
|
+
text: captured,
|
|
3260
|
+
score: .75,
|
|
3261
|
+
source: "regex"
|
|
3262
|
+
});
|
|
3263
|
+
}
|
|
3264
|
+
return results;
|
|
3265
|
+
};
|
|
3266
|
+
const ORPHAN_STREET_RE = /^\s*(\p{Lu}[\p{Ll}\p{Lu}]+(?:\s+[\p{Lu}\p{Ll}][\p{Ll}]+)*\s+\d{2,4}[a-zA-Z]?)\s*$/gmu;
|
|
3267
|
+
/**
|
|
3268
|
+
* In the header zone (top 15%), find standalone lines
|
|
3269
|
+
* matching "[Uppercase word(s)] [number]" that sit
|
|
3270
|
+
* between other detected entities. These are almost
|
|
3271
|
+
* certainly street addresses in party definitions.
|
|
3272
|
+
*
|
|
3273
|
+
* Example: "Evropská 710" on its own line between
|
|
3274
|
+
* an organization entity and a postal code entity.
|
|
3275
|
+
*/
|
|
3276
|
+
const detectOrphanStreetLines = (fullText, existingEntities) => {
|
|
3277
|
+
const headerEnd = Math.floor(fullText.length * HEADER_ZONE_FRACTION);
|
|
3278
|
+
const results = [];
|
|
3279
|
+
ORPHAN_STREET_RE.lastIndex = 0;
|
|
3280
|
+
for (let m = ORPHAN_STREET_RE.exec(fullText); m !== null; m = ORPHAN_STREET_RE.exec(fullText)) {
|
|
3281
|
+
const captured = m[1];
|
|
3282
|
+
if (captured === void 0) continue;
|
|
3283
|
+
const start = m.index + m[0].indexOf(captured);
|
|
3284
|
+
const end = start + captured.length;
|
|
3285
|
+
if (start >= headerEnd) continue;
|
|
3286
|
+
if (existingEntities.some((e) => e.start <= start && e.end >= end)) continue;
|
|
3287
|
+
if (!existingEntities.some((e) => Math.abs(e.start - end) < 200 || Math.abs(e.end - start) < 200)) continue;
|
|
3288
|
+
results.push({
|
|
3289
|
+
start,
|
|
3290
|
+
end,
|
|
3291
|
+
label: "address",
|
|
3292
|
+
text: captured,
|
|
3293
|
+
score: .85,
|
|
3294
|
+
source: "regex"
|
|
3295
|
+
});
|
|
3296
|
+
}
|
|
3297
|
+
return results;
|
|
3298
|
+
};
|
|
3299
|
+
//#endregion
|
|
3300
|
+
//#region src/filters/zone-classifier.ts
|
|
3301
|
+
/**
|
|
3302
|
+
* Additive score adjustments per document zone.
|
|
3303
|
+
* Header and signature blocks are dense with PII;
|
|
3304
|
+
* tables often contain structured identifying data.
|
|
3305
|
+
*/
|
|
3306
|
+
const ZONE_SCORE_ADJUSTMENTS = {
|
|
3307
|
+
header: .1,
|
|
3308
|
+
signature: .15,
|
|
3309
|
+
body: 0,
|
|
3310
|
+
table: .05
|
|
3311
|
+
};
|
|
3312
|
+
const loadSectionHeadings = async () => {
|
|
3313
|
+
const mod = await import("@stll/anonymize-data/config/section-headings.json");
|
|
3314
|
+
return (mod.default ?? mod).patterns.map((p) => new RegExp(p.re, p.flags));
|
|
3315
|
+
};
|
|
3316
|
+
const loadSigningClauses = async () => {
|
|
3317
|
+
const mod = await import("@stll/anonymize-data/config/signing-clauses.json");
|
|
3318
|
+
return (mod.default ?? mod).patterns.map((p) => {
|
|
3319
|
+
const prefix = p.prefix || "";
|
|
3320
|
+
const suffix = p.suffix || "";
|
|
3321
|
+
const prepAlt = p.prepositions.length > 0 ? p.prepositions.join("|") : null;
|
|
3322
|
+
const combined = `^\\s*(?:${prefix}${prepAlt ? `\\p{Lu}\\p{Ll}+(?:\\s+(?:${prepAlt})\\s+\\p{Lu}\\p{Ll}+)*(?:\\s+\\p{Lu}\\p{Ll}+)*` : "\\p{Lu}\\p{Ll}+(?:[- ]\\p{Lu}\\p{Ll}+)*"}${suffix})`;
|
|
3323
|
+
return new RegExp(combined, "u");
|
|
3324
|
+
});
|
|
3325
|
+
};
|
|
3326
|
+
/**
|
|
3327
|
+
* Ensure config data is loaded. Call once before
|
|
3328
|
+
* classifyZones. Safe to call multiple times.
|
|
3329
|
+
*/
|
|
3330
|
+
const initZoneClassifier = (ctx = defaultContext) => {
|
|
3331
|
+
if (ctx.zoneInitPromise) return ctx.zoneInitPromise;
|
|
3332
|
+
ctx.zoneInitPromise = Promise.all([loadSectionHeadings(), loadSigningClauses()]).then(([headings, clauses]) => {
|
|
3333
|
+
ctx.zoneHeadingPatterns = headings;
|
|
3334
|
+
ctx.zoneSigningPatterns = clauses;
|
|
3335
|
+
}).catch((err) => {
|
|
3336
|
+
ctx.zoneInitPromise = null;
|
|
3337
|
+
throw err;
|
|
3338
|
+
});
|
|
3339
|
+
return ctx.zoneInitPromise;
|
|
3340
|
+
};
|
|
3341
|
+
const MIN_TABS_FOR_TABLE = 2;
|
|
3342
|
+
const isTableLine = (line) => {
|
|
3343
|
+
let tabCount = 0;
|
|
3344
|
+
for (const ch of line) {
|
|
3345
|
+
if (ch === " ") tabCount++;
|
|
3346
|
+
if (tabCount >= MIN_TABS_FOR_TABLE) return true;
|
|
3347
|
+
}
|
|
3348
|
+
return false;
|
|
3349
|
+
};
|
|
3350
|
+
/**
|
|
3351
|
+
* Classify a document into zones based on
|
|
3352
|
+
* structural heuristics. Zones are non-overlapping
|
|
3353
|
+
* and cover the entire text.
|
|
3354
|
+
*
|
|
3355
|
+
* Must call `initZoneClassifier()` first.
|
|
3356
|
+
*/
|
|
3357
|
+
const classifyZones = (fullText, ctx = defaultContext) => {
|
|
3358
|
+
if (fullText.length === 0) return [];
|
|
3359
|
+
const headingRes = ctx.zoneHeadingPatterns;
|
|
3360
|
+
const signingRes = ctx.zoneSigningPatterns;
|
|
3361
|
+
if (!headingRes || !signingRes) {
|
|
3362
|
+
console.warn("[anonymize] classifyZones called before initZoneClassifier(); returning body-only");
|
|
3363
|
+
return [{
|
|
3364
|
+
zone: "body",
|
|
3365
|
+
start: 0,
|
|
3366
|
+
end: fullText.length
|
|
3367
|
+
}];
|
|
3368
|
+
}
|
|
3369
|
+
const lines = fullText.split("\n");
|
|
3370
|
+
const zones = [];
|
|
3371
|
+
let headerEndLine = -1;
|
|
3372
|
+
for (let i = 0; i < lines.length; i++) {
|
|
3373
|
+
const line = lines[i];
|
|
3374
|
+
if (line === void 0) continue;
|
|
3375
|
+
for (const re of headingRes) if (re.test(line)) {
|
|
3376
|
+
headerEndLine = i;
|
|
3377
|
+
break;
|
|
3378
|
+
}
|
|
3379
|
+
if (headerEndLine !== -1) break;
|
|
3380
|
+
}
|
|
3381
|
+
let signatureStartLine = -1;
|
|
3382
|
+
for (let i = lines.length - 1; i >= 0; i--) {
|
|
3383
|
+
const line = lines[i];
|
|
3384
|
+
if (line === void 0) continue;
|
|
3385
|
+
for (const re of signingRes) if (re.test(line)) {
|
|
3386
|
+
signatureStartLine = i;
|
|
3387
|
+
break;
|
|
3388
|
+
}
|
|
3389
|
+
if (signatureStartLine !== -1) break;
|
|
3390
|
+
}
|
|
3391
|
+
const lineOffsets = [];
|
|
3392
|
+
let offset = 0;
|
|
3393
|
+
for (const line of lines) {
|
|
3394
|
+
lineOffsets.push(offset);
|
|
3395
|
+
offset += line.length + 1;
|
|
3396
|
+
}
|
|
3397
|
+
let headerEndOffset = headerEndLine >= 0 ? lineOffsets[headerEndLine] ?? 0 : 0;
|
|
3398
|
+
const signatureStartOffset = signatureStartLine >= 0 ? lineOffsets[signatureStartLine] ?? fullText.length : fullText.length;
|
|
3399
|
+
if (headerEndLine > 0 && signatureStartLine >= 0 && headerEndOffset > signatureStartOffset) {
|
|
3400
|
+
headerEndLine = -1;
|
|
3401
|
+
headerEndOffset = 0;
|
|
3402
|
+
}
|
|
3403
|
+
if (headerEndLine > 0) zones.push({
|
|
3404
|
+
zone: "header",
|
|
3405
|
+
start: 0,
|
|
3406
|
+
end: headerEndOffset
|
|
3407
|
+
});
|
|
3408
|
+
const bodyStart = headerEndLine > 0 ? headerEndOffset : 0;
|
|
3409
|
+
const bodyEnd = signatureStartLine >= 0 ? signatureStartOffset : fullText.length;
|
|
3410
|
+
let tableStart = -1;
|
|
3411
|
+
for (let i = Math.max(headerEndLine, 0); i < (signatureStartLine >= 0 ? signatureStartLine : lines.length); i++) {
|
|
3412
|
+
const line = lines[i];
|
|
3413
|
+
if (line === void 0) continue;
|
|
3414
|
+
const lineStart = lineOffsets[i] ?? 0;
|
|
3415
|
+
const lineEnd = lineStart + line.length;
|
|
3416
|
+
if (isTableLine(line)) {
|
|
3417
|
+
if (tableStart === -1) tableStart = lineStart;
|
|
3418
|
+
} else if (tableStart !== -1) {
|
|
3419
|
+
zones.push({
|
|
3420
|
+
zone: "table",
|
|
3421
|
+
start: tableStart,
|
|
3422
|
+
end: lineStart
|
|
3423
|
+
});
|
|
3424
|
+
tableStart = -1;
|
|
3425
|
+
}
|
|
3426
|
+
if (i === (signatureStartLine >= 0 ? signatureStartLine - 1 : lines.length - 1) && tableStart !== -1) {
|
|
3427
|
+
zones.push({
|
|
3428
|
+
zone: "table",
|
|
3429
|
+
start: tableStart,
|
|
3430
|
+
end: Math.min(lineEnd + 1, bodyEnd)
|
|
3431
|
+
});
|
|
3432
|
+
tableStart = -1;
|
|
3433
|
+
}
|
|
3434
|
+
}
|
|
3435
|
+
const sortedSpecial = zones.toSorted((a, b) => a.start - b.start);
|
|
3436
|
+
let cursor = bodyStart;
|
|
3437
|
+
const bodyZones = [];
|
|
3438
|
+
for (const span of sortedSpecial) {
|
|
3439
|
+
if (span.zone === "header") continue;
|
|
3440
|
+
if (span.start > cursor) bodyZones.push({
|
|
3441
|
+
zone: "body",
|
|
3442
|
+
start: cursor,
|
|
3443
|
+
end: span.start
|
|
3444
|
+
});
|
|
3445
|
+
cursor = Math.max(cursor, span.end);
|
|
3446
|
+
}
|
|
3447
|
+
if (cursor < bodyEnd) bodyZones.push({
|
|
3448
|
+
zone: "body",
|
|
3449
|
+
start: cursor,
|
|
3450
|
+
end: bodyEnd
|
|
3451
|
+
});
|
|
3452
|
+
for (const z of bodyZones) zones.push(z);
|
|
3453
|
+
if (signatureStartLine >= 0) zones.push({
|
|
3454
|
+
zone: "signature",
|
|
3455
|
+
start: signatureStartOffset,
|
|
3456
|
+
end: fullText.length
|
|
3457
|
+
});
|
|
3458
|
+
return zones.toSorted((a, b) => a.start - b.start);
|
|
3459
|
+
};
|
|
3460
|
+
/**
|
|
3461
|
+
* Find which zone an entity's midpoint falls in.
|
|
3462
|
+
* Returns "body" if no zone matches (defensive).
|
|
3463
|
+
*/
|
|
3464
|
+
const findZone = (midpoint, zones) => {
|
|
3465
|
+
for (const span of zones) if (midpoint >= span.start && midpoint < span.end) return span.zone;
|
|
3466
|
+
return "body";
|
|
3467
|
+
};
|
|
3468
|
+
/**
|
|
3469
|
+
* Apply zone-based score adjustments to entities.
|
|
3470
|
+
* Entities in header/signature/table zones get a
|
|
3471
|
+
* small additive boost reflecting the higher PII
|
|
3472
|
+
* density in those regions.
|
|
3473
|
+
*
|
|
3474
|
+
* Returns a new array; does not mutate inputs.
|
|
3475
|
+
*/
|
|
3476
|
+
const applyZoneAdjustments = (entities, zones) => {
|
|
3477
|
+
if (zones.length === 0) return entities.map((e) => ({ ...e }));
|
|
3478
|
+
const result = [];
|
|
3479
|
+
for (const entity of entities) {
|
|
3480
|
+
const adjustment = ZONE_SCORE_ADJUSTMENTS[findZone((entity.start + entity.end) / 2, zones)];
|
|
3481
|
+
if (adjustment > 0) result.push({
|
|
3482
|
+
...entity,
|
|
3483
|
+
score: Math.min(1, entity.score + adjustment)
|
|
3484
|
+
});
|
|
3485
|
+
else result.push({ ...entity });
|
|
3486
|
+
}
|
|
3487
|
+
return result;
|
|
3488
|
+
};
|
|
3489
|
+
//#endregion
|
|
3490
|
+
//#region src/filters/hotword-rules.ts
|
|
3491
|
+
let rules = null;
|
|
3492
|
+
let search = null;
|
|
3493
|
+
/**
|
|
3494
|
+
* Maps each TextSearch pattern index back to the
|
|
3495
|
+
* rule index that owns it, so a single AC scan
|
|
3496
|
+
* resolves all hotword hits to their rule.
|
|
3497
|
+
*/
|
|
3498
|
+
let patternToRule = null;
|
|
3499
|
+
let initPromise = null;
|
|
3500
|
+
const loadRules = async () => {
|
|
3501
|
+
const mod = await import("@stll/anonymize-data/config/hotword-rules.json");
|
|
3502
|
+
const loaded = (mod.default ?? mod).rules;
|
|
3503
|
+
const patterns = [];
|
|
3504
|
+
const mapping = [];
|
|
3505
|
+
for (let ruleIdx = 0; ruleIdx < loaded.length; ruleIdx++) {
|
|
3506
|
+
const rule = loaded[ruleIdx];
|
|
3507
|
+
if (!rule) continue;
|
|
3508
|
+
for (const hw of rule.hotwords) {
|
|
3509
|
+
patterns.push({
|
|
3510
|
+
pattern: hw,
|
|
3511
|
+
literal: true,
|
|
3512
|
+
caseInsensitive: true
|
|
3513
|
+
});
|
|
3514
|
+
mapping.push(ruleIdx);
|
|
3515
|
+
}
|
|
3516
|
+
}
|
|
3517
|
+
const builtSearch = patterns.length > 0 ? new TextSearch(patterns, {
|
|
3518
|
+
overlapStrategy: "all",
|
|
3519
|
+
caseInsensitive: true,
|
|
3520
|
+
wholeWords: true
|
|
3521
|
+
}) : null;
|
|
3522
|
+
patternToRule = mapping;
|
|
3523
|
+
search = builtSearch;
|
|
3524
|
+
rules = loaded;
|
|
3525
|
+
};
|
|
3526
|
+
/**
|
|
3527
|
+
* Load hotword rules from the data package.
|
|
3528
|
+
* Safe to call multiple times; subsequent calls
|
|
3529
|
+
* are no-ops.
|
|
3530
|
+
*/
|
|
3531
|
+
const initHotwordRules = async () => {
|
|
3532
|
+
if (rules !== null) return;
|
|
3533
|
+
if (initPromise !== null) return initPromise;
|
|
3534
|
+
initPromise = loadRules().catch((err) => {
|
|
3535
|
+
initPromise = null;
|
|
3536
|
+
throw err;
|
|
3537
|
+
});
|
|
3538
|
+
return initPromise;
|
|
3539
|
+
};
|
|
3540
|
+
/**
|
|
3541
|
+
* Apply hotword context rules to detected entities.
|
|
3542
|
+
*
|
|
3543
|
+
* Scans `fullText` once with a single AC automaton
|
|
3544
|
+
* for all hotwords across all rules, then checks
|
|
3545
|
+
* proximity to each entity. Distance-decayed
|
|
3546
|
+
* adjustment: closer hotwords give a stronger boost.
|
|
3547
|
+
*
|
|
3548
|
+
* Returns a new array; input entities are not mutated.
|
|
3549
|
+
*/
|
|
3550
|
+
const applyHotwordRules = (entities, fullText) => {
|
|
3551
|
+
if (rules === null || rules.length === 0 || search === null || patternToRule === null) return entities;
|
|
3552
|
+
const hits = search.findIter(fullText);
|
|
3553
|
+
if (hits.length === 0) return entities;
|
|
3554
|
+
const hitsByRule = /* @__PURE__ */ new Map();
|
|
3555
|
+
for (const hit of hits) {
|
|
3556
|
+
const ruleIdx = patternToRule[hit.pattern];
|
|
3557
|
+
if (ruleIdx === void 0) continue;
|
|
3558
|
+
let bucket = hitsByRule.get(ruleIdx);
|
|
3559
|
+
if (bucket === void 0) {
|
|
3560
|
+
bucket = [];
|
|
3561
|
+
hitsByRule.set(ruleIdx, bucket);
|
|
3562
|
+
}
|
|
3563
|
+
bucket.push(hit);
|
|
3564
|
+
}
|
|
3565
|
+
const result = [];
|
|
3566
|
+
for (const entity of entities) {
|
|
3567
|
+
let bestAdjustment = 0;
|
|
3568
|
+
let bestReclassify;
|
|
3569
|
+
for (let ruleIdx = 0; ruleIdx < rules.length; ruleIdx++) {
|
|
3570
|
+
const rule = rules[ruleIdx];
|
|
3571
|
+
if (!rule) continue;
|
|
3572
|
+
if (!rule.targetLabels.includes(entity.label)) continue;
|
|
3573
|
+
const ruleHits = hitsByRule.get(ruleIdx);
|
|
3574
|
+
if (ruleHits === void 0) continue;
|
|
3575
|
+
for (const hit of ruleHits) {
|
|
3576
|
+
let distance;
|
|
3577
|
+
let maxDistance;
|
|
3578
|
+
if (hit.end <= entity.start) {
|
|
3579
|
+
distance = entity.start - hit.end;
|
|
3580
|
+
maxDistance = rule.proximityBefore;
|
|
3581
|
+
} else if (hit.start >= entity.end) {
|
|
3582
|
+
distance = hit.start - entity.end;
|
|
3583
|
+
maxDistance = rule.proximityAfter;
|
|
3584
|
+
} else {
|
|
3585
|
+
distance = 0;
|
|
3586
|
+
maxDistance = Math.max(rule.proximityBefore, rule.proximityAfter);
|
|
3587
|
+
}
|
|
3588
|
+
if (distance > maxDistance) continue;
|
|
3589
|
+
const decay = maxDistance === 0 ? 1 : 1 - distance / maxDistance;
|
|
3590
|
+
const adj = rule.scoreAdjustment * decay;
|
|
3591
|
+
if (Math.abs(adj) > Math.abs(bestAdjustment)) {
|
|
3592
|
+
bestAdjustment = adj;
|
|
3593
|
+
if (adj > 0) bestReclassify = rule.reclassifyTo;
|
|
3594
|
+
else bestReclassify = void 0;
|
|
3595
|
+
}
|
|
3596
|
+
}
|
|
3597
|
+
}
|
|
3598
|
+
if (bestAdjustment === 0) {
|
|
3599
|
+
result.push(entity);
|
|
3600
|
+
continue;
|
|
3601
|
+
}
|
|
3602
|
+
const newScore = Math.min(1, Math.max(0, entity.score + bestAdjustment));
|
|
3603
|
+
const newLabel = bestReclassify !== void 0 ? bestReclassify : entity.label;
|
|
3604
|
+
result.push({
|
|
3605
|
+
...entity,
|
|
3606
|
+
score: newScore,
|
|
3607
|
+
label: newLabel
|
|
3608
|
+
});
|
|
3609
|
+
}
|
|
3610
|
+
return result;
|
|
3611
|
+
};
|
|
3612
|
+
//#endregion
|
|
3613
|
+
//#region src/filters/boundary-consistency.ts
|
|
3614
|
+
/** Max gap (in chars) between entities to merge. */
|
|
3615
|
+
const MAX_GAP = 3;
|
|
3616
|
+
/**
|
|
3617
|
+
* Characters allowed in the gap between two adjacent
|
|
3618
|
+
* same-label entities that should be merged: spaces,
|
|
3619
|
+
* tabs, commas, and hyphens. Uses `[ \t,\-]` instead
|
|
3620
|
+
* of `\s` to avoid merging entities across newlines.
|
|
3621
|
+
*/
|
|
3622
|
+
const GAP_PATTERN = /^[ \t,\-]+$/;
|
|
3623
|
+
/**
|
|
3624
|
+
* Build a set of word boundary offsets for the full
|
|
3625
|
+
* text using `Intl.Segmenter`. Returns a sorted array
|
|
3626
|
+
* of offsets where words start and end.
|
|
3627
|
+
*/
|
|
3628
|
+
const buildWordBoundaries = (text) => {
|
|
3629
|
+
const segmenter = new Intl.Segmenter("und", { granularity: "word" });
|
|
3630
|
+
const boundaries = /* @__PURE__ */ new Set();
|
|
3631
|
+
for (const seg of segmenter.segment(text)) {
|
|
3632
|
+
if (!seg.isWordLike) continue;
|
|
3633
|
+
boundaries.add(seg.index);
|
|
3634
|
+
boundaries.add(seg.index + seg.segment.length);
|
|
3635
|
+
}
|
|
3636
|
+
return boundaries;
|
|
3637
|
+
};
|
|
3638
|
+
/**
|
|
3639
|
+
* Characters that act as hard stops when scanning
|
|
3640
|
+
* backward for a word boundary. Entity boundaries
|
|
3641
|
+
* should never extend past these.
|
|
3642
|
+
*/
|
|
3643
|
+
const WORD_START_STOPS = new Set([
|
|
3644
|
+
"\n",
|
|
3645
|
+
"\r",
|
|
3646
|
+
",",
|
|
3647
|
+
";",
|
|
3648
|
+
"(",
|
|
3649
|
+
")",
|
|
3650
|
+
"[",
|
|
3651
|
+
"]"
|
|
3652
|
+
]);
|
|
3653
|
+
/**
|
|
3654
|
+
* Find the word-start offset at or before `pos`.
|
|
3655
|
+
* Scans left until a word boundary is found.
|
|
3656
|
+
*/
|
|
3657
|
+
const wordStartAt = (pos, boundaries, text) => {
|
|
3658
|
+
let p = pos;
|
|
3659
|
+
while (p > 0 && !boundaries.has(p)) {
|
|
3660
|
+
const prev = text[p - 1];
|
|
3661
|
+
if (prev !== void 0 && WORD_START_STOPS.has(prev)) return p;
|
|
3662
|
+
p--;
|
|
3663
|
+
}
|
|
3664
|
+
return p;
|
|
3665
|
+
};
|
|
3666
|
+
/**
|
|
3667
|
+
* Characters that act as hard stops when scanning
|
|
3668
|
+
* forward for a word boundary. Entity boundaries
|
|
3669
|
+
* should never extend past these.
|
|
3670
|
+
*/
|
|
3671
|
+
const WORD_END_STOPS = new Set([
|
|
3672
|
+
"\n",
|
|
3673
|
+
"\r",
|
|
3674
|
+
",",
|
|
3675
|
+
";",
|
|
3676
|
+
".",
|
|
3677
|
+
"(",
|
|
3678
|
+
")",
|
|
3679
|
+
"[",
|
|
3680
|
+
"]"
|
|
3681
|
+
]);
|
|
3682
|
+
/**
|
|
3683
|
+
* Find the word-end offset at or after `pos`.
|
|
3684
|
+
* Scans right until a word boundary is found.
|
|
3685
|
+
*/
|
|
3686
|
+
const wordEndAt = (pos, boundaries, text) => {
|
|
3687
|
+
let p = pos;
|
|
3688
|
+
while (p < text.length && !boundaries.has(p)) {
|
|
3689
|
+
const ch = text[p];
|
|
3690
|
+
if (ch !== void 0 && WORD_END_STOPS.has(ch)) return p;
|
|
3691
|
+
p++;
|
|
3692
|
+
}
|
|
3693
|
+
return p;
|
|
3694
|
+
};
|
|
3695
|
+
/**
|
|
3696
|
+
* Binary search: find the leftmost index in `arr`
|
|
3697
|
+
* where `arr[index].start >= value`.
|
|
3698
|
+
*/
|
|
3699
|
+
const lowerBound = (arr, value) => {
|
|
3700
|
+
let lo = 0;
|
|
3701
|
+
let hi = arr.length;
|
|
3702
|
+
while (lo < hi) {
|
|
3703
|
+
const mid = lo + hi >>> 1;
|
|
3704
|
+
const el = arr[mid];
|
|
3705
|
+
if (el && el.start < value) lo = mid + 1;
|
|
3706
|
+
else hi = mid;
|
|
3707
|
+
}
|
|
3708
|
+
return lo;
|
|
3709
|
+
};
|
|
3710
|
+
/**
|
|
3711
|
+
* Merge adjacent same-label entities separated only by
|
|
3712
|
+
* whitespace, comma, or hyphen (max 3 chars). Also
|
|
3713
|
+
* merges same-label entities that partially overlap
|
|
3714
|
+
* (which can happen after word-boundary expansion).
|
|
3715
|
+
*
|
|
3716
|
+
* Looks for the last same-label entity in the result
|
|
3717
|
+
* (not just the very last entity) so that an
|
|
3718
|
+
* intervening different-label entity does not prevent
|
|
3719
|
+
* merging.
|
|
3720
|
+
*
|
|
3721
|
+
* Uses binary search for the `gapOccupied` check and
|
|
3722
|
+
* a Map for O(1) same-label prev lookup. O(n log n).
|
|
3723
|
+
*/
|
|
3724
|
+
const mergeAdjacent = (entities, fullText) => {
|
|
3725
|
+
const sorted = entities.toSorted((a, b) => a.start - b.start);
|
|
3726
|
+
const result = [];
|
|
3727
|
+
const lastByLabel = /* @__PURE__ */ new Map();
|
|
3728
|
+
for (const entity of sorted) {
|
|
3729
|
+
const prev = lastByLabel.get(entity.label);
|
|
3730
|
+
if (!prev) {
|
|
3731
|
+
const copy = { ...entity };
|
|
3732
|
+
result.push(copy);
|
|
3733
|
+
lastByLabel.set(entity.label, copy);
|
|
3734
|
+
continue;
|
|
3735
|
+
}
|
|
3736
|
+
if (entity.start < prev.end) {
|
|
3737
|
+
prev.end = Math.max(prev.end, entity.end);
|
|
3738
|
+
prev.text = fullText.slice(prev.start, prev.end);
|
|
3739
|
+
prev.score = Math.max(prev.score, entity.score);
|
|
3740
|
+
continue;
|
|
3741
|
+
}
|
|
3742
|
+
const gap = fullText.slice(prev.end, entity.start);
|
|
3743
|
+
const gapStart = prev.end;
|
|
3744
|
+
const gapEnd = entity.start;
|
|
3745
|
+
const searchIdx = lowerBound(sorted, gapStart);
|
|
3746
|
+
let gapOccupied = false;
|
|
3747
|
+
for (let k = searchIdx; k < sorted.length; k++) {
|
|
3748
|
+
const other = sorted[k];
|
|
3749
|
+
if (!other || other.start >= gapEnd) break;
|
|
3750
|
+
if (other.label !== entity.label && other.end > gapStart) {
|
|
3751
|
+
gapOccupied = true;
|
|
3752
|
+
break;
|
|
3753
|
+
}
|
|
3754
|
+
}
|
|
3755
|
+
if (!gapOccupied && gap.length <= MAX_GAP && GAP_PATTERN.test(gap)) {
|
|
3756
|
+
prev.end = entity.end;
|
|
3757
|
+
prev.text = fullText.slice(prev.start, prev.end);
|
|
3758
|
+
prev.score = Math.max(prev.score, entity.score);
|
|
3759
|
+
} else {
|
|
3760
|
+
const copy = { ...entity };
|
|
3761
|
+
result.push(copy);
|
|
3762
|
+
lastByLabel.set(entity.label, copy);
|
|
3763
|
+
}
|
|
3764
|
+
}
|
|
3765
|
+
return result;
|
|
3766
|
+
};
|
|
3767
|
+
/**
|
|
3768
|
+
* Fix partial-word boundaries by extending entity
|
|
3769
|
+
* start/end to the nearest word boundary. Does not
|
|
3770
|
+
* extend across newlines or into spans occupied by
|
|
3771
|
+
* different-label entities.
|
|
3772
|
+
*
|
|
3773
|
+
* Uses binary search to skip irrelevant entries when
|
|
3774
|
+
* clamping at cross-label neighbors. O(n log n) in
|
|
3775
|
+
* the common case; O(n^2) worst case when many
|
|
3776
|
+
* same-label entities precede a cross-label boundary.
|
|
3777
|
+
*/
|
|
3778
|
+
const fixPartialWords = (entities, fullText) => {
|
|
3779
|
+
const boundaries = buildWordBoundaries(fullText);
|
|
3780
|
+
const sorted = entities.toSorted((a, b) => a.start - b.start);
|
|
3781
|
+
const byEnd = sorted.map((e, idx) => ({
|
|
3782
|
+
entity: e,
|
|
3783
|
+
idx
|
|
3784
|
+
})).sort((a, b) => a.entity.end - b.entity.end);
|
|
3785
|
+
const endPositions = byEnd.map((x) => x.entity.end);
|
|
3786
|
+
return sorted.map((e, eIdx) => {
|
|
3787
|
+
let newStart = wordStartAt(e.start, boundaries, fullText);
|
|
3788
|
+
let newEnd = wordEndAt(e.end, boundaries, fullText);
|
|
3789
|
+
let lo = 0;
|
|
3790
|
+
let hi = endPositions.length;
|
|
3791
|
+
while (lo < hi) {
|
|
3792
|
+
const mid = lo + hi >>> 1;
|
|
3793
|
+
if ((endPositions[mid] ?? Number.POSITIVE_INFINITY) <= newStart) lo = mid + 1;
|
|
3794
|
+
else hi = mid;
|
|
3795
|
+
}
|
|
3796
|
+
for (let k = lo; k < byEnd.length; k++) {
|
|
3797
|
+
const entry = byEnd[k];
|
|
3798
|
+
if (!entry || entry.entity.end > e.start) break;
|
|
3799
|
+
if (entry.idx === eIdx) continue;
|
|
3800
|
+
if (entry.entity.label === e.label) continue;
|
|
3801
|
+
newStart = Math.max(newStart, entry.entity.end);
|
|
3802
|
+
}
|
|
3803
|
+
const startIdx = lowerBound(sorted, e.end);
|
|
3804
|
+
for (let k = startIdx; k < sorted.length; k++) {
|
|
3805
|
+
const other = sorted[k];
|
|
3806
|
+
if (!other || other.start >= newEnd) break;
|
|
3807
|
+
if (other === e) continue;
|
|
3808
|
+
if (other.label === e.label) continue;
|
|
3809
|
+
newEnd = Math.min(newEnd, other.start);
|
|
3810
|
+
}
|
|
3811
|
+
if (newStart === e.start && newEnd === e.end) return e;
|
|
3812
|
+
return {
|
|
3813
|
+
...e,
|
|
3814
|
+
start: newStart,
|
|
3815
|
+
end: newEnd,
|
|
3816
|
+
text: fullText.slice(newStart, newEnd)
|
|
3817
|
+
};
|
|
3818
|
+
});
|
|
3819
|
+
};
|
|
3820
|
+
/**
|
|
3821
|
+
* Deduplicate entities with identical [start, end, label].
|
|
3822
|
+
* Keeps the entry with the highest score.
|
|
3823
|
+
*/
|
|
3824
|
+
const deduplicateSpans = (entities) => {
|
|
3825
|
+
const seen = /* @__PURE__ */ new Map();
|
|
3826
|
+
for (const entity of entities) {
|
|
3827
|
+
const key = `${entity.start}:${entity.end}:${entity.label}`;
|
|
3828
|
+
const existing = seen.get(key);
|
|
3829
|
+
if (!existing || entity.score > existing.score) seen.set(key, entity);
|
|
3830
|
+
}
|
|
3831
|
+
return [...seen.values()];
|
|
3832
|
+
};
|
|
3833
|
+
/**
|
|
3834
|
+
* Remove nested same-label entities. If a shorter
|
|
3835
|
+
* entity is fully contained within a longer entity
|
|
3836
|
+
* of the same label, drop the shorter one.
|
|
3837
|
+
*
|
|
3838
|
+
* Uses a "max end seen" sweep per label. O(n) after
|
|
3839
|
+
* the initial sort.
|
|
3840
|
+
*/
|
|
3841
|
+
const removeNestedSameLabel = (entities) => {
|
|
3842
|
+
const sorted = entities.toSorted((a, b) => {
|
|
3843
|
+
if (a.start !== b.start) return a.start - b.start;
|
|
3844
|
+
return b.end - a.end;
|
|
3845
|
+
});
|
|
3846
|
+
const result = [];
|
|
3847
|
+
const maxEndByLabel = /* @__PURE__ */ new Map();
|
|
3848
|
+
for (const entity of sorted) {
|
|
3849
|
+
const maxEnd = maxEndByLabel.get(entity.label);
|
|
3850
|
+
if (maxEnd !== void 0 && entity.end <= maxEnd) continue;
|
|
3851
|
+
maxEndByLabel.set(entity.label, entity.end);
|
|
3852
|
+
result.push(entity);
|
|
3853
|
+
}
|
|
3854
|
+
return result;
|
|
3855
|
+
};
|
|
3856
|
+
/**
|
|
3857
|
+
* Resolve cross-label overlaps that can arise when
|
|
3858
|
+
* `fixPartialWords` independently expands two
|
|
3859
|
+
* different-label entities toward the same word
|
|
3860
|
+
* boundary. The entity with the higher score (or
|
|
3861
|
+
* longer span on tie) keeps its boundary; the other
|
|
3862
|
+
* is trimmed so the overlap disappears.
|
|
3863
|
+
*
|
|
3864
|
+
* Preserved existing structure: sorted + early break
|
|
3865
|
+
* already gives good amortized behavior. O(n^2)
|
|
3866
|
+
* worst case but rare in practice.
|
|
3867
|
+
*/
|
|
3868
|
+
const resolveCrossLabelOverlaps = (entities, fullText) => {
|
|
3869
|
+
const sorted = entities.map((e) => ({ ...e })).sort((a, b) => a.start - b.start);
|
|
3870
|
+
for (let i = 0; i < sorted.length; i++) for (let j = i + 1; j < sorted.length; j++) {
|
|
3871
|
+
const a = sorted[i];
|
|
3872
|
+
const b = sorted[j];
|
|
3873
|
+
if (!a || !b) continue;
|
|
3874
|
+
if (b.start >= a.end) break;
|
|
3875
|
+
if (a.label === b.label) continue;
|
|
3876
|
+
const aContainsB = a.start <= b.start && a.end >= b.end;
|
|
3877
|
+
const bContainsA = b.start <= a.start && b.end >= a.end;
|
|
3878
|
+
if (aContainsB || bContainsA) continue;
|
|
3879
|
+
const aLen = a.end - a.start;
|
|
3880
|
+
const bLen = b.end - b.start;
|
|
3881
|
+
if (a.score > b.score || a.score === b.score && aLen >= bLen) {
|
|
3882
|
+
b.start = a.end;
|
|
3883
|
+
b.text = fullText.slice(b.start, b.end);
|
|
3884
|
+
} else {
|
|
3885
|
+
a.end = b.start;
|
|
3886
|
+
a.text = fullText.slice(a.start, a.end);
|
|
3887
|
+
}
|
|
3888
|
+
}
|
|
3889
|
+
return sorted.filter((e) => e.start < e.end);
|
|
3890
|
+
};
|
|
3891
|
+
/**
|
|
3892
|
+
* Post-processing pass for entity boundary consistency.
|
|
3893
|
+
* Runs after mergeAndDedup, before false-positive
|
|
3894
|
+
* filtering.
|
|
3895
|
+
*
|
|
3896
|
+
* 1. Fix partial-word boundaries (respects cross-label
|
|
3897
|
+
* neighbors to avoid introducing new overlaps)
|
|
3898
|
+
* 2. Resolve any remaining cross-label overlaps
|
|
3899
|
+
* 3. Deduplicate identical [start, end, label] spans
|
|
3900
|
+
* 4. Merge adjacent same-label entities (catches any
|
|
3901
|
+
* new adjacency/overlap from step 1)
|
|
3902
|
+
* 5. Remove nested same-label entities
|
|
3903
|
+
*/
|
|
3904
|
+
const enforceBoundaryConsistency = (entities, fullText) => {
|
|
3905
|
+
return removeNestedSameLabel(mergeAdjacent(deduplicateSpans(resolveCrossLabelOverlaps(fixPartialWords(entities, fullText), fullText)), fullText));
|
|
3906
|
+
};
|
|
3907
|
+
//#endregion
|
|
3908
|
+
//#region src/build-unified-search.ts
|
|
3909
|
+
/**
|
|
3910
|
+
* Build the unified search instances from all
|
|
3911
|
+
* detector pattern sources.
|
|
3912
|
+
*
|
|
3913
|
+
* Two TextSearch instances (not one) to avoid
|
|
3914
|
+
* 200K per-pattern object allocations:
|
|
3915
|
+
* 1. regex + triggers + legal-forms (mixed, ~140
|
|
3916
|
+
* patterns, caseInsensitive for trigger AC)
|
|
3917
|
+
* 2. deny-list + street-types + gazetteer
|
|
3918
|
+
* (caseInsensitive, overlap "all";
|
|
3919
|
+
* deny-list/street-type use per-pattern
|
|
3920
|
+
* wholeWords: true; gazetteer exact use
|
|
3921
|
+
* wholeWords: false; gazetteer fuzzy use
|
|
3922
|
+
* distance: 2 via @stll/fuzzy-search)
|
|
3923
|
+
*
|
|
3924
|
+
* All patterns are PatternEntry objects with
|
|
3925
|
+
* per-pattern literal/wholeWords settings.
|
|
3926
|
+
*/
|
|
3927
|
+
const buildUnifiedSearch = async (config, gazetteerEntries = [], ctx = defaultContext) => {
|
|
3928
|
+
const [legalForms, triggers, denyListData, streetTypes, currencyPatterns, datePatterns, signingPatterns] = await Promise.all([
|
|
3929
|
+
buildLegalFormPatterns(),
|
|
3930
|
+
config.enableTriggerPhrases ? buildTriggerPatterns() : Promise.resolve({
|
|
3931
|
+
patterns: [],
|
|
3932
|
+
rules: []
|
|
3933
|
+
}),
|
|
3934
|
+
config.enableDenyList ? buildDenyList(config, ctx) : Promise.resolve(null),
|
|
3935
|
+
buildStreetTypePatterns(),
|
|
3936
|
+
getCurrencyPatterns(),
|
|
3937
|
+
getDatePatterns(),
|
|
3938
|
+
getSigningClausePatterns()
|
|
3939
|
+
]);
|
|
3940
|
+
const allRegex = [
|
|
3941
|
+
...REGEX_PATTERNS,
|
|
3942
|
+
...currencyPatterns,
|
|
3943
|
+
...datePatterns,
|
|
3944
|
+
...signingPatterns
|
|
3945
|
+
];
|
|
3946
|
+
const regexMeta = [
|
|
3947
|
+
...REGEX_META,
|
|
3948
|
+
...currencyPatterns.map(() => CURRENCY_PATTERN_META),
|
|
3949
|
+
...datePatterns.map(() => DATE_PATTERN_META),
|
|
3950
|
+
...signingPatterns.map(() => SIGNING_CLAUSE_META)
|
|
3951
|
+
];
|
|
3952
|
+
let offset = 0;
|
|
3953
|
+
const regexSlice = {
|
|
3954
|
+
start: offset,
|
|
3955
|
+
end: offset + allRegex.length
|
|
3956
|
+
};
|
|
3957
|
+
offset = regexSlice.end;
|
|
3958
|
+
const legalFormsSlice = {
|
|
3959
|
+
start: offset,
|
|
3960
|
+
end: offset + legalForms.length
|
|
3961
|
+
};
|
|
3962
|
+
offset = legalFormsSlice.end;
|
|
3963
|
+
const triggersSlice = {
|
|
3964
|
+
start: offset,
|
|
3965
|
+
end: offset + triggers.patterns.length
|
|
3966
|
+
};
|
|
3967
|
+
const triggerEntries = triggers.patterns.map((p) => ({
|
|
3968
|
+
pattern: p,
|
|
3969
|
+
literal: true,
|
|
3970
|
+
caseInsensitive: true
|
|
3971
|
+
}));
|
|
3972
|
+
const tsRegex = new TextSearch([
|
|
3973
|
+
...allRegex,
|
|
3974
|
+
...legalForms,
|
|
3975
|
+
...triggerEntries
|
|
3976
|
+
]);
|
|
3977
|
+
offset = 0;
|
|
3978
|
+
const denyListOriginals = denyListData?.originals ?? [];
|
|
3979
|
+
const denyListSlice = {
|
|
3980
|
+
start: offset,
|
|
3981
|
+
end: offset + denyListOriginals.length
|
|
3982
|
+
};
|
|
3983
|
+
offset = denyListSlice.end;
|
|
3984
|
+
const streetTypesSlice = {
|
|
3985
|
+
start: offset,
|
|
3986
|
+
end: offset + streetTypes.length
|
|
3987
|
+
};
|
|
3988
|
+
offset = streetTypesSlice.end;
|
|
3989
|
+
const gazResult = config.enableGazetteer && gazetteerEntries.length > 0 ? buildGazetteerPatterns(gazetteerEntries) : null;
|
|
3990
|
+
const gazetteerSlice = {
|
|
3991
|
+
start: offset,
|
|
3992
|
+
end: offset + (gazResult?.patterns.length ?? 0)
|
|
3993
|
+
};
|
|
3994
|
+
offset = gazetteerSlice.end;
|
|
3995
|
+
const wrapWholeWord = (s) => ({
|
|
3996
|
+
pattern: s,
|
|
3997
|
+
literal: true,
|
|
3998
|
+
wholeWords: true
|
|
3999
|
+
});
|
|
4000
|
+
const literalAllPatterns = [
|
|
4001
|
+
...denyListOriginals.map(wrapWholeWord),
|
|
4002
|
+
...streetTypes.map(wrapWholeWord),
|
|
4003
|
+
...gazResult?.patterns ?? []
|
|
4004
|
+
];
|
|
4005
|
+
return {
|
|
4006
|
+
tsRegex,
|
|
4007
|
+
tsLiterals: literalAllPatterns.length > 0 ? new TextSearch(literalAllPatterns, {
|
|
4008
|
+
caseInsensitive: true,
|
|
4009
|
+
overlapStrategy: "all"
|
|
4010
|
+
}) : new TextSearch([]),
|
|
4011
|
+
slices: {
|
|
4012
|
+
regex: regexSlice,
|
|
4013
|
+
legalForms: legalFormsSlice,
|
|
4014
|
+
triggers: triggersSlice,
|
|
4015
|
+
denyList: denyListSlice,
|
|
4016
|
+
streetTypes: streetTypesSlice,
|
|
4017
|
+
gazetteer: gazetteerSlice
|
|
4018
|
+
},
|
|
4019
|
+
regexMeta,
|
|
4020
|
+
triggerRules: triggers.rules,
|
|
4021
|
+
denyListData,
|
|
4022
|
+
gazetteerData: gazResult?.data ?? null
|
|
4023
|
+
};
|
|
4024
|
+
};
|
|
4025
|
+
//#endregion
|
|
4026
|
+
//#region src/unified-search.ts
|
|
4027
|
+
const runUnifiedSearch = (instance, fullText) => {
|
|
4028
|
+
const regexMatches = instance.tsRegex.findIter(fullText);
|
|
4029
|
+
const normalized = normalizeForSearch(fullText);
|
|
4030
|
+
return {
|
|
4031
|
+
regexMatches,
|
|
4032
|
+
literalMatches: instance.tsLiterals.findIter(normalized)
|
|
4033
|
+
};
|
|
4034
|
+
};
|
|
4035
|
+
//#endregion
|
|
4036
|
+
//#region src/util/entity-masking.ts
|
|
4037
|
+
const MASK_TOKEN = "[MASKED]";
|
|
4038
|
+
const MASK_LEN = 8;
|
|
4039
|
+
/**
|
|
4040
|
+
* Replace detected entity spans with placeholder tokens.
|
|
4041
|
+
* Each entity span is replaced with "[MASKED]" (fixed
|
|
4042
|
+
* length). Returns the masked text and an offset mapping
|
|
4043
|
+
* function.
|
|
4044
|
+
*/
|
|
4045
|
+
const maskDetectedSpans = (fullText, entities) => {
|
|
4046
|
+
if (entities.length === 0) return {
|
|
4047
|
+
maskedText: fullText,
|
|
4048
|
+
offsetMap: (s, e) => ({
|
|
4049
|
+
start: s,
|
|
4050
|
+
end: e
|
|
4051
|
+
})
|
|
4052
|
+
};
|
|
4053
|
+
const sorted = entities.toSorted((a, b) => a.start - b.start || b.end - a.end);
|
|
4054
|
+
const spans = [];
|
|
4055
|
+
const first = sorted[0];
|
|
4056
|
+
if (!first) return {
|
|
4057
|
+
maskedText: fullText,
|
|
4058
|
+
offsetMap: (s, e) => ({
|
|
4059
|
+
start: s,
|
|
4060
|
+
end: e
|
|
4061
|
+
})
|
|
4062
|
+
};
|
|
4063
|
+
let cur = {
|
|
4064
|
+
start: first.start,
|
|
4065
|
+
end: first.end
|
|
4066
|
+
};
|
|
4067
|
+
for (let i = 1; i < sorted.length; i++) {
|
|
4068
|
+
const s = sorted[i];
|
|
4069
|
+
if (!s) continue;
|
|
4070
|
+
if (s.start < cur.end) cur.end = Math.max(cur.end, s.end);
|
|
4071
|
+
else {
|
|
4072
|
+
spans.push(cur);
|
|
4073
|
+
cur = {
|
|
4074
|
+
start: s.start,
|
|
4075
|
+
end: s.end
|
|
4076
|
+
};
|
|
4077
|
+
}
|
|
4078
|
+
}
|
|
4079
|
+
spans.push(cur);
|
|
4080
|
+
const segments = [];
|
|
4081
|
+
const parts = [];
|
|
4082
|
+
let prev = 0;
|
|
4083
|
+
let cumulativeShift = 0;
|
|
4084
|
+
for (const span of spans) {
|
|
4085
|
+
parts.push(fullText.slice(prev, span.start));
|
|
4086
|
+
parts.push(MASK_TOKEN);
|
|
4087
|
+
const delta = span.end - span.start - MASK_LEN;
|
|
4088
|
+
cumulativeShift += delta;
|
|
4089
|
+
const maskedStart = span.start - (cumulativeShift - delta);
|
|
4090
|
+
const maskedEnd = maskedStart + MASK_LEN;
|
|
4091
|
+
segments.push({
|
|
4092
|
+
maskedStart,
|
|
4093
|
+
maskedEnd,
|
|
4094
|
+
shift: cumulativeShift,
|
|
4095
|
+
origStart: span.start,
|
|
4096
|
+
origEnd: span.end
|
|
4097
|
+
});
|
|
4098
|
+
prev = span.end;
|
|
4099
|
+
}
|
|
4100
|
+
parts.push(fullText.slice(prev));
|
|
4101
|
+
const maskedText = parts.join("");
|
|
4102
|
+
const offsetMap = (maskedStart, maskedEnd) => {
|
|
4103
|
+
for (const seg of segments) if (maskedStart < seg.maskedEnd && maskedEnd > seg.maskedStart) return null;
|
|
4104
|
+
let shift = 0;
|
|
4105
|
+
for (const seg of segments) if (maskedStart >= seg.maskedEnd) shift = seg.shift;
|
|
4106
|
+
else break;
|
|
4107
|
+
return {
|
|
4108
|
+
start: maskedStart + shift,
|
|
4109
|
+
end: maskedEnd + shift
|
|
4110
|
+
};
|
|
4111
|
+
};
|
|
4112
|
+
return {
|
|
4113
|
+
maskedText,
|
|
4114
|
+
offsetMap
|
|
4115
|
+
};
|
|
4116
|
+
};
|
|
4117
|
+
/**
|
|
4118
|
+
* Map NER entities from masked-text offsets back to
|
|
4119
|
+
* original-text offsets. Discards any NER entity whose
|
|
4120
|
+
* mapped span overlaps a masked (rule-detected) region.
|
|
4121
|
+
*/
|
|
4122
|
+
const unmaskNerEntities = (nerEntities, maskResult, fullText) => {
|
|
4123
|
+
const result = [];
|
|
4124
|
+
for (const ner of nerEntities) {
|
|
4125
|
+
const mapped = maskResult.offsetMap(ner.start, ner.end);
|
|
4126
|
+
if (mapped === null) continue;
|
|
4127
|
+
result.push({
|
|
4128
|
+
...ner,
|
|
4129
|
+
start: mapped.start,
|
|
4130
|
+
end: mapped.end,
|
|
4131
|
+
text: fullText.slice(mapped.start, mapped.end)
|
|
4132
|
+
});
|
|
4133
|
+
}
|
|
4134
|
+
return result;
|
|
4135
|
+
};
|
|
4136
|
+
//#endregion
|
|
4137
|
+
//#region src/pipeline.ts
|
|
4138
|
+
const shouldReplace = (a, b) => {
|
|
4139
|
+
const aPri = DETECTOR_PRIORITY[a.source] ?? 0;
|
|
4140
|
+
const bPri = DETECTOR_PRIORITY[b.source] ?? 0;
|
|
4141
|
+
if (aPri !== bPri) return aPri > bPri;
|
|
4142
|
+
return a.score > b.score || a.score === b.score && a.end - a.start > b.end - b.start;
|
|
4143
|
+
};
|
|
4144
|
+
/** Labels where colons are structurally significant. */
|
|
4145
|
+
const COLON_LABELS = new Set(["ip address", "mac address"]);
|
|
4146
|
+
/** Strip leading/trailing whitespace and punctuation. */
|
|
4147
|
+
const sanitizeEntities = (entities) => entities.flatMap((e) => {
|
|
4148
|
+
const strip = COLON_LABELS.has(e.label) ? /[\s,;]+/ : /[\s:,;]+/;
|
|
4149
|
+
const leadTrimmed = e.text.replace(new RegExp(`^${strip.source}`, strip.flags), "");
|
|
4150
|
+
const lead = e.text.length - leadTrimmed.length;
|
|
4151
|
+
const cleaned = leadTrimmed.replace(new RegExp(`${strip.source}$`, strip.flags), "");
|
|
4152
|
+
if (cleaned.length === 0) return [];
|
|
4153
|
+
if (!/[\p{L}\p{N}]/u.test(cleaned)) return [];
|
|
4154
|
+
if (cleaned === e.text) return [e];
|
|
4155
|
+
return [{
|
|
4156
|
+
...e,
|
|
4157
|
+
start: e.start + lead,
|
|
4158
|
+
end: e.start + lead + cleaned.length,
|
|
4159
|
+
text: cleaned
|
|
4160
|
+
}];
|
|
4161
|
+
});
|
|
4162
|
+
const mergeAndDedup = (...layers) => {
|
|
4163
|
+
const all = [];
|
|
4164
|
+
for (const layer of layers) for (const entity of layer) all.push(entity);
|
|
4165
|
+
if (all.length === 0) return [];
|
|
4166
|
+
const sorted = all.toSorted((a, b) => a.start - b.start);
|
|
4167
|
+
const first = sorted[0];
|
|
4168
|
+
if (!first) return [];
|
|
4169
|
+
const merged = [{ ...first }];
|
|
4170
|
+
for (let i = 1; i < sorted.length; i++) {
|
|
4171
|
+
const entity = sorted[i];
|
|
4172
|
+
const last = merged[merged.length - 1];
|
|
4173
|
+
if (!entity || !last) continue;
|
|
4174
|
+
if (last.end <= entity.start) merged.push({ ...entity });
|
|
4175
|
+
else if (shouldReplace(entity, last)) merged[merged.length - 1] = { ...entity };
|
|
4176
|
+
}
|
|
4177
|
+
return sanitizeEntities(merged);
|
|
4178
|
+
};
|
|
4179
|
+
const escapeRegex = (s) => s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
4180
|
+
let amountWordsRe = null;
|
|
4181
|
+
let amountWordsLoaded = false;
|
|
4182
|
+
const getAmountWordsRe = async () => {
|
|
4183
|
+
if (amountWordsLoaded && amountWordsRe) return amountWordsRe;
|
|
4184
|
+
try {
|
|
4185
|
+
const alt = (await import("@stll/anonymize-data/config/amount-words.json")).default.patterns.flatMap((p) => p.keywords).map(escapeRegex).join("|");
|
|
4186
|
+
amountWordsRe = new RegExp(`^[,;]?[^\\S\\n]*(\\((?:${alt})[:\\s][^)\\n]{1,120}\\))`, "i");
|
|
4187
|
+
} catch {
|
|
4188
|
+
amountWordsRe = /^[,;]?[^\S\n]*(\((?:slovy|slovně)[:\s][^)\n]{1,120}\))/i;
|
|
4189
|
+
}
|
|
4190
|
+
amountWordsLoaded = true;
|
|
4191
|
+
return amountWordsRe;
|
|
4192
|
+
};
|
|
4193
|
+
const extendMonetaryAmountWords = (entities, fullText, re) => entities.map((e) => {
|
|
4194
|
+
if (e.label !== "monetary amount") return e;
|
|
4195
|
+
const after = fullText.slice(e.end);
|
|
4196
|
+
const m = re.exec(after);
|
|
4197
|
+
if (!m) return e;
|
|
4198
|
+
const newEnd = e.end + m[0].length;
|
|
4199
|
+
return {
|
|
4200
|
+
...e,
|
|
4201
|
+
end: newEnd,
|
|
4202
|
+
text: fullText.slice(e.start, newEnd)
|
|
4203
|
+
};
|
|
4204
|
+
});
|
|
4205
|
+
const checkAbort = (signal) => {
|
|
4206
|
+
if (signal?.aborted) throw new DOMException("Pipeline aborted", "AbortError");
|
|
4207
|
+
};
|
|
4208
|
+
const configKey = (config, gazetteerEntries) => {
|
|
4209
|
+
const gazFingerprint = config.enableGazetteer && gazetteerEntries.length > 0 ? gazetteerEntries.map((e) => `${e.id}:${e.canonical}:${e.label}:${[...e.variants].sort().join(",")}`).toSorted().join(";") : "";
|
|
4210
|
+
return `${config.enableDenyList}:${config.enableTriggerPhrases}:${config.denyListCountries?.toSorted().join(",") ?? ""}:${config.denyListRegions?.toSorted().join(",") ?? ""}:${config.denyListExcludeCategories?.toSorted().join(",") ?? ""}:${config.enableGazetteer}:${gazFingerprint}`;
|
|
4211
|
+
};
|
|
4212
|
+
/**
|
|
4213
|
+
* Get or build a cached search instance. Cache state
|
|
4214
|
+
* lives on the provided PipelineContext, not at module
|
|
4215
|
+
* level.
|
|
4216
|
+
*/
|
|
4217
|
+
const getCachedSearch = async (config, gazetteerEntries, ctx) => {
|
|
4218
|
+
const key = configKey(config, gazetteerEntries);
|
|
4219
|
+
if (ctx.search && ctx.searchKey === key) return ctx.search;
|
|
4220
|
+
if (ctx.searchPromise && ctx.searchKey === key) return ctx.searchPromise;
|
|
4221
|
+
ctx.search = null;
|
|
4222
|
+
ctx.searchKey = key;
|
|
4223
|
+
const promise = buildUnifiedSearch(config, gazetteerEntries, ctx);
|
|
4224
|
+
ctx.searchPromise = promise;
|
|
4225
|
+
const result = await promise;
|
|
4226
|
+
if (ctx.searchKey === key) ctx.search = result;
|
|
4227
|
+
return result;
|
|
4228
|
+
};
|
|
4229
|
+
/**
|
|
4230
|
+
* Run the full detection pipeline.
|
|
4231
|
+
*
|
|
4232
|
+
* Two TextSearch instances scan the text (regex +
|
|
4233
|
+
* literals). Results are dispatched to each
|
|
4234
|
+
* detector's post-processor by pattern index range.
|
|
4235
|
+
*
|
|
4236
|
+
* Pass an AbortSignal to cancel the pipeline between
|
|
4237
|
+
* stages. Throws a DOMException with name "AbortError"
|
|
4238
|
+
* when cancelled.
|
|
4239
|
+
*
|
|
4240
|
+
* Pass an optional `context` to isolate cached state
|
|
4241
|
+
* from other pipeline runs. If omitted, a module-level
|
|
4242
|
+
* default context is used (backward compatible).
|
|
4243
|
+
*/
|
|
4244
|
+
const runPipeline = async (options) => {
|
|
4245
|
+
const { fullText, config, gazetteerEntries, nerInference = null, onProgress, cachedSearch, signal, context } = options;
|
|
4246
|
+
const ctx = context ?? defaultContext;
|
|
4247
|
+
const log = (step, detail) => {
|
|
4248
|
+
onProgress?.(step, detail);
|
|
4249
|
+
};
|
|
4250
|
+
checkAbort(signal);
|
|
4251
|
+
let zoneInitOk = false;
|
|
4252
|
+
const enableHotwords = config.enableHotwordRules === true;
|
|
4253
|
+
let hotwordInitOk = false;
|
|
4254
|
+
const hotwordInit = enableHotwords ? initHotwordRules().then(() => {
|
|
4255
|
+
hotwordInitOk = true;
|
|
4256
|
+
}).catch((err) => {
|
|
4257
|
+
log("hotwords", "init failed; skipping");
|
|
4258
|
+
console.warn("[anonymize] hotword rules init failed", err);
|
|
4259
|
+
}) : Promise.resolve();
|
|
4260
|
+
if (config.enableZoneClassification) {
|
|
4261
|
+
const zoneInit = initZoneClassifier(ctx).then(() => {
|
|
4262
|
+
zoneInitOk = true;
|
|
4263
|
+
}).catch((err) => {
|
|
4264
|
+
log("zones", "init failed; skipping");
|
|
4265
|
+
console.warn("[anonymize] zone classifier init failed", err);
|
|
4266
|
+
});
|
|
4267
|
+
await Promise.all([
|
|
4268
|
+
loadGenericRoles(ctx),
|
|
4269
|
+
initPrepositions(),
|
|
4270
|
+
initStreetAbbrevs(),
|
|
4271
|
+
zoneInit,
|
|
4272
|
+
hotwordInit
|
|
4273
|
+
]);
|
|
4274
|
+
} else await Promise.all([
|
|
4275
|
+
loadGenericRoles(ctx),
|
|
4276
|
+
initPrepositions(),
|
|
4277
|
+
initStreetAbbrevs(),
|
|
4278
|
+
hotwordInit
|
|
4279
|
+
]);
|
|
4280
|
+
if (cachedSearch && config.enableDenyList) await ensureDenyListData(ctx);
|
|
4281
|
+
let zones = [];
|
|
4282
|
+
if (config.enableZoneClassification && zoneInitOk) {
|
|
4283
|
+
zones = classifyZones(fullText, ctx);
|
|
4284
|
+
if (zones.length > 0) log("zones", [...new Set(zones.map((z) => z.zone))].join(", "));
|
|
4285
|
+
}
|
|
4286
|
+
checkAbort(signal);
|
|
4287
|
+
const search = cachedSearch ?? await getCachedSearch(config, gazetteerEntries, ctx);
|
|
4288
|
+
checkAbort(signal);
|
|
4289
|
+
const { regexMatches, literalMatches } = runUnifiedSearch(search, fullText);
|
|
4290
|
+
const { slices } = search;
|
|
4291
|
+
const regexEntities = config.enableRegex ? processRegexMatches(regexMatches, slices.regex.start, slices.regex.end, search.regexMeta) : [];
|
|
4292
|
+
if (regexEntities.length > 0) log("regex", `${regexEntities.length} matches`);
|
|
4293
|
+
const legalFormEntities = processLegalFormMatches(regexMatches, slices.legalForms.start, slices.legalForms.end, fullText);
|
|
4294
|
+
if (legalFormEntities.length > 0) log("legal-forms", `${legalFormEntities.length} matches`);
|
|
4295
|
+
const triggerEntities = config.enableTriggerPhrases ? processTriggerMatches(regexMatches, slices.triggers.start, slices.triggers.end, fullText, search.triggerRules) : [];
|
|
4296
|
+
if (triggerEntities.length > 0) log("trigger-phrases", `${triggerEntities.length} matches`);
|
|
4297
|
+
checkAbort(signal);
|
|
4298
|
+
let nameCorpusEntities = [];
|
|
4299
|
+
if (config.enableNameCorpus && !config.enableDenyList) {
|
|
4300
|
+
await initNameCorpus(ctx);
|
|
4301
|
+
checkAbort(signal);
|
|
4302
|
+
nameCorpusEntities = detectNameCorpus(fullText, ctx);
|
|
4303
|
+
log("name-corpus", `${nameCorpusEntities.length} matches`);
|
|
4304
|
+
}
|
|
4305
|
+
const denyListEntities = config.enableDenyList && search.denyListData ? processDenyListMatches(literalMatches, slices.denyList.start, slices.denyList.end, fullText, search.denyListData, ctx) : [];
|
|
4306
|
+
if (denyListEntities.length > 0) log("deny-list", `${denyListEntities.length} matches`);
|
|
4307
|
+
const gazetteerEntities = config.enableGazetteer && search.gazetteerData ? processGazetteerMatches(literalMatches, slices.gazetteer.start, slices.gazetteer.end, fullText, search.gazetteerData) : [];
|
|
4308
|
+
if (gazetteerEntities.length > 0) log("gazetteer", `${gazetteerEntities.length} matches`);
|
|
4309
|
+
checkAbort(signal);
|
|
4310
|
+
let nerEntities = [];
|
|
4311
|
+
if (config.enableNer && nerInference) {
|
|
4312
|
+
const maskResult = maskDetectedSpans(fullText, [
|
|
4313
|
+
...triggerEntities,
|
|
4314
|
+
...regexEntities,
|
|
4315
|
+
...legalFormEntities,
|
|
4316
|
+
...nameCorpusEntities,
|
|
4317
|
+
...denyListEntities,
|
|
4318
|
+
...gazetteerEntities
|
|
4319
|
+
]);
|
|
4320
|
+
log("ner", "running inference...");
|
|
4321
|
+
const rawNer = await nerInference(maskResult.maskedText, config.labels, config.threshold, signal);
|
|
4322
|
+
nerEntities = unmaskNerEntities(rawNer, maskResult, fullText);
|
|
4323
|
+
const dropped = rawNer.length - nerEntities.length;
|
|
4324
|
+
log("ner", `${nerEntities.length} entities` + (dropped > 0 ? ` (${dropped} masked)` : ""));
|
|
4325
|
+
}
|
|
4326
|
+
checkAbort(signal);
|
|
4327
|
+
const preAddressEntities = [
|
|
4328
|
+
...triggerEntities,
|
|
4329
|
+
...regexEntities,
|
|
4330
|
+
...legalFormEntities,
|
|
4331
|
+
...nameCorpusEntities,
|
|
4332
|
+
...denyListEntities,
|
|
4333
|
+
...gazetteerEntities,
|
|
4334
|
+
...nerEntities
|
|
4335
|
+
];
|
|
4336
|
+
const addressSeedEntities = await processAddressSeeds(literalMatches, slices.streetTypes.start, slices.streetTypes.end, fullText, preAddressEntities);
|
|
4337
|
+
if (addressSeedEntities.length > 0) log("address-seeds", `${addressSeedEntities.length} expanded`);
|
|
4338
|
+
checkAbort(signal);
|
|
4339
|
+
const zoneAdjusted = applyZoneAdjustments([...preAddressEntities, ...addressSeedEntities], zones);
|
|
4340
|
+
const preBoostEntities = enableHotwords && hotwordInitOk ? applyHotwordRules(zoneAdjusted, fullText) : zoneAdjusted;
|
|
4341
|
+
let allEntities;
|
|
4342
|
+
if (config.enableConfidenceBoost) {
|
|
4343
|
+
allEntities = boostNearMissEntities(preBoostEntities, config.threshold);
|
|
4344
|
+
const boosted = allEntities.length - preBoostEntities.filter((e) => e.score >= config.threshold).length;
|
|
4345
|
+
if (boosted > 0) log("confidence-boost", `${boosted} near-miss promoted`);
|
|
4346
|
+
} else allEntities = preBoostEntities.filter((e) => e.score >= config.threshold);
|
|
4347
|
+
const streetPatterns = detectStreetPatternsNearAddresses(fullText, allEntities);
|
|
4348
|
+
if (streetPatterns.length > 0) {
|
|
4349
|
+
allEntities = [...allEntities, ...streetPatterns];
|
|
4350
|
+
log("street-context", `${streetPatterns.length} street patterns near addresses`);
|
|
4351
|
+
}
|
|
4352
|
+
const orphanStreets = detectOrphanStreetLines(fullText, allEntities);
|
|
4353
|
+
if (orphanStreets.length > 0) {
|
|
4354
|
+
allEntities = [...allEntities, ...orphanStreets];
|
|
4355
|
+
log("orphan-streets", `${orphanStreets.length} header street lines`);
|
|
4356
|
+
}
|
|
4357
|
+
const rawMerged = mergeAndDedup(allEntities);
|
|
4358
|
+
log("merge", `${rawMerged.length} after dedup`);
|
|
4359
|
+
const mergedExtended = extendMonetaryAmountWords(rawMerged, fullText, await getAmountWordsRe());
|
|
4360
|
+
const consistent = enforceBoundaryConsistency(mergedExtended, fullText);
|
|
4361
|
+
if (consistent.length < mergedExtended.length) log("boundary", `${mergedExtended.length - consistent.length} consolidated`);
|
|
4362
|
+
let postOrgEntities = consistent;
|
|
4363
|
+
if (config.enableCoreference) {
|
|
4364
|
+
const thresholded = propagateOrgNames(consistent, fullText).filter((e) => e.score >= config.threshold);
|
|
4365
|
+
if (thresholded.length > 0) {
|
|
4366
|
+
postOrgEntities = mergeAndDedup(consistent, thresholded);
|
|
4367
|
+
log("org-propagation", `${thresholded.length} base names`);
|
|
4368
|
+
}
|
|
4369
|
+
}
|
|
4370
|
+
const merged = filterFalsePositives(postOrgEntities, ctx);
|
|
4371
|
+
if (merged.length < postOrgEntities.length) log("filter", `removed ${postOrgEntities.length - merged.length} FPs`);
|
|
4372
|
+
checkAbort(signal);
|
|
4373
|
+
ctx.corefSourceMap.clear();
|
|
4374
|
+
if (config.enableCoreference) {
|
|
4375
|
+
const terms = await extractDefinedTerms(fullText, merged, ctx);
|
|
4376
|
+
if (terms.length > 0) {
|
|
4377
|
+
log("coreference", `${terms.length} defined terms`);
|
|
4378
|
+
const corefSpans = findCoreferenceSpans(fullText, terms, ctx);
|
|
4379
|
+
if (corefSpans.length > 0) {
|
|
4380
|
+
log("coreference-rescan", `${corefSpans.length} aliases`);
|
|
4381
|
+
return sanitizeEntities(filterFalsePositives(enforceBoundaryConsistency(mergeAndDedup(merged, corefSpans), fullText), ctx));
|
|
4382
|
+
}
|
|
4383
|
+
}
|
|
4384
|
+
}
|
|
4385
|
+
return sanitizeEntities(merged);
|
|
4386
|
+
};
|
|
4387
|
+
const OPERATOR_REGISTRY = {
|
|
4388
|
+
replace: {
|
|
4389
|
+
type: "replace",
|
|
4390
|
+
reversibility: "reversible",
|
|
4391
|
+
apply: (_text, _label, placeholder) => placeholder
|
|
4392
|
+
},
|
|
4393
|
+
redact: {
|
|
4394
|
+
type: "redact",
|
|
4395
|
+
reversibility: "irreversible",
|
|
4396
|
+
apply: (_text, _label, _placeholder, redactString) => redactString
|
|
4397
|
+
}
|
|
4398
|
+
};
|
|
4399
|
+
/**
|
|
4400
|
+
* Default operator config: replace for all labels.
|
|
4401
|
+
* Preserves existing pipeline behaviour.
|
|
4402
|
+
*/
|
|
4403
|
+
const DEFAULT_OPERATOR_CONFIG = {
|
|
4404
|
+
operators: {},
|
|
4405
|
+
redactString: "[REDACTED]"
|
|
4406
|
+
};
|
|
4407
|
+
/**
|
|
4408
|
+
* Resolve the operator for a label, falling back to "replace".
|
|
4409
|
+
*/
|
|
4410
|
+
const resolveOperator = (config, label) => config.operators[label] ?? "replace";
|
|
4411
|
+
//#endregion
|
|
4412
|
+
//#region src/redact.ts
|
|
4413
|
+
const WHITESPACE_RE = /\s+/g;
|
|
4414
|
+
const PHONE_NOISE_RE = /[()\s-]/g;
|
|
4415
|
+
const SPACE_DASH_RE = /[\s-]/g;
|
|
4416
|
+
/**
|
|
4417
|
+
* Normalize entity text so that surface-form variations
|
|
4418
|
+
* of the same real-world value map to a single canonical
|
|
4419
|
+
* key. Lowercased emails, stripped phone formatting, etc.
|
|
4420
|
+
*/
|
|
4421
|
+
const normalizeEntityText = (label, text) => {
|
|
4422
|
+
const upper = label.toUpperCase().replace(WHITESPACE_RE, "_");
|
|
4423
|
+
if (upper === "EMAIL_ADDRESS" || upper === "EMAIL") return text.toLowerCase().trim();
|
|
4424
|
+
if (upper === "PHONE_NUMBER" || upper === "PHONE") return text.replace(PHONE_NOISE_RE, "");
|
|
4425
|
+
if (upper === "IBAN" || upper === "BANK_ACCOUNT_NUMBER" || upper === "TAX_IDENTIFICATION_NUMBER" || upper === "REGISTRATION_NUMBER") return text.replace(SPACE_DASH_RE, "").toUpperCase();
|
|
4426
|
+
if (upper === "PERSON" || upper === "ORGANIZATION" || upper === "ADDRESS") return text.replace(WHITESPACE_RE, " ").toLowerCase().trim();
|
|
4427
|
+
return text.trim();
|
|
4428
|
+
};
|
|
4429
|
+
/**
|
|
4430
|
+
* Build a stable mapping from entity text to numbered
|
|
4431
|
+
* placeholders. Same real-world value always maps to the
|
|
4432
|
+
* same placeholder (e.g., "Dr. Muller" and "Dr. Muller"
|
|
4433
|
+
* both become [PERSON_1]).
|
|
4434
|
+
*
|
|
4435
|
+
* Placeholder format: [LABEL_N] where LABEL is uppercase
|
|
4436
|
+
* and N is a 1-based counter per label.
|
|
4437
|
+
*
|
|
4438
|
+
* @param ctx Pipeline context. Must be the same instance
|
|
4439
|
+
* passed to `runPipeline` (or `findCoreferenceSpans`)
|
|
4440
|
+
* so coreference placeholder links are preserved.
|
|
4441
|
+
* Defaults to `defaultContext` for single-tenant usage.
|
|
4442
|
+
*/
|
|
4443
|
+
const buildPlaceholderMap = (entities, ctx = defaultContext) => {
|
|
4444
|
+
const counters = /* @__PURE__ */ new Map();
|
|
4445
|
+
const textLabelToPlaceholder = /* @__PURE__ */ new Map();
|
|
4446
|
+
const normalizedToPlaceholder = /* @__PURE__ */ new Map();
|
|
4447
|
+
const sorted = entities.toSorted((a, b) => a.start - b.start);
|
|
4448
|
+
for (const entity of sorted) {
|
|
4449
|
+
const compositeKey = `${entity.label}\0${entity.text}`;
|
|
4450
|
+
if (textLabelToPlaceholder.has(compositeKey)) continue;
|
|
4451
|
+
const labelKey = entity.label.toUpperCase().replace(WHITESPACE_RE, "_");
|
|
4452
|
+
const sourceText = ctx.corefSourceMap.get(corefKey(entity));
|
|
4453
|
+
if (sourceText !== void 0) {
|
|
4454
|
+
const sourceNormalizedKey = `${labelKey}\0${normalizeEntityText(entity.label, sourceText)}`;
|
|
4455
|
+
const sourceExisting = normalizedToPlaceholder.get(sourceNormalizedKey);
|
|
4456
|
+
if (sourceExisting) {
|
|
4457
|
+
textLabelToPlaceholder.set(compositeKey, sourceExisting);
|
|
4458
|
+
continue;
|
|
4459
|
+
}
|
|
4460
|
+
}
|
|
4461
|
+
const normalizedKey = `${labelKey}\0${normalizeEntityText(entity.label, entity.text)}`;
|
|
4462
|
+
const existing = normalizedToPlaceholder.get(normalizedKey);
|
|
4463
|
+
if (existing) {
|
|
4464
|
+
textLabelToPlaceholder.set(compositeKey, existing);
|
|
4465
|
+
continue;
|
|
4466
|
+
}
|
|
4467
|
+
const count = (counters.get(labelKey) ?? 0) + 1;
|
|
4468
|
+
counters.set(labelKey, count);
|
|
4469
|
+
const placeholder = `[${labelKey}_${count}]`;
|
|
4470
|
+
textLabelToPlaceholder.set(compositeKey, placeholder);
|
|
4471
|
+
normalizedToPlaceholder.set(normalizedKey, placeholder);
|
|
4472
|
+
}
|
|
4473
|
+
return textLabelToPlaceholder;
|
|
4474
|
+
};
|
|
4475
|
+
/**
|
|
4476
|
+
* Apply redactions to the source text, replacing each
|
|
4477
|
+
* confirmed entity span using the configured operator.
|
|
4478
|
+
*
|
|
4479
|
+
* Co-references are consistent: if the same text appears
|
|
4480
|
+
* multiple times, all occurrences get the same placeholder.
|
|
4481
|
+
*
|
|
4482
|
+
* @param ctx Pipeline context. Must be the same instance
|
|
4483
|
+
* passed to `runPipeline` (or `findCoreferenceSpans`)
|
|
4484
|
+
* so coreference placeholder links are preserved.
|
|
4485
|
+
* Defaults to `defaultContext` for single-tenant usage.
|
|
4486
|
+
*/
|
|
4487
|
+
const redactText = (fullText, entities, config = DEFAULT_OPERATOR_CONFIG, ctx = defaultContext) => {
|
|
4488
|
+
if (entities.length === 0) return {
|
|
4489
|
+
redactedText: fullText,
|
|
4490
|
+
redactionMap: /* @__PURE__ */ new Map(),
|
|
4491
|
+
operatorMap: /* @__PURE__ */ new Map(),
|
|
4492
|
+
entityCount: 0
|
|
4493
|
+
};
|
|
4494
|
+
const placeholderMap = buildPlaceholderMap(entities, ctx);
|
|
4495
|
+
const sorted = entities.toSorted((a, b) => a.start - b.start);
|
|
4496
|
+
const nonOverlapping = [];
|
|
4497
|
+
let lastEnd = 0;
|
|
4498
|
+
for (const entity of sorted) if (entity.start >= lastEnd) {
|
|
4499
|
+
nonOverlapping.push(entity);
|
|
4500
|
+
lastEnd = entity.end;
|
|
4501
|
+
}
|
|
4502
|
+
const parts = [];
|
|
4503
|
+
const redactionMap = /* @__PURE__ */ new Map();
|
|
4504
|
+
const operatorMap = /* @__PURE__ */ new Map();
|
|
4505
|
+
let cursor = 0;
|
|
4506
|
+
for (const entity of nonOverlapping) {
|
|
4507
|
+
if (entity.start > cursor) parts.push(fullText.slice(cursor, entity.start));
|
|
4508
|
+
const placeholder = placeholderMap.get(`${entity.label}\0${entity.text}`) ?? `[${entity.label.toUpperCase().replace(/\s+/g, "_")}]`;
|
|
4509
|
+
const opType = resolveOperator(config, entity.label);
|
|
4510
|
+
const operator = OPERATOR_REGISTRY[opType];
|
|
4511
|
+
const replacement = operator.apply(entity.text, entity.label, placeholder, config.redactString);
|
|
4512
|
+
parts.push(replacement);
|
|
4513
|
+
operatorMap.set(placeholder, opType);
|
|
4514
|
+
if (operator.reversibility === "reversible" && !redactionMap.has(placeholder)) redactionMap.set(placeholder, entity.text);
|
|
4515
|
+
cursor = entity.end;
|
|
4516
|
+
}
|
|
4517
|
+
if (cursor < fullText.length) parts.push(fullText.slice(cursor));
|
|
4518
|
+
return {
|
|
4519
|
+
redactedText: parts.join(""),
|
|
4520
|
+
redactionMap,
|
|
4521
|
+
operatorMap,
|
|
4522
|
+
entityCount: nonOverlapping.length
|
|
4523
|
+
};
|
|
4524
|
+
};
|
|
4525
|
+
/**
|
|
4526
|
+
* Serialize the redaction key to JSON for export.
|
|
4527
|
+
* Includes operator metadata so the export is self-describing.
|
|
4528
|
+
*/
|
|
4529
|
+
const exportRedactionKey = (redactionMap, operatorMap) => {
|
|
4530
|
+
const entries = {};
|
|
4531
|
+
for (const [placeholder, value] of redactionMap) entries[placeholder] = {
|
|
4532
|
+
original: value,
|
|
4533
|
+
operator: operatorMap.get(placeholder) ?? "replace"
|
|
4534
|
+
};
|
|
4535
|
+
return JSON.stringify({ entries }, null, 2);
|
|
4536
|
+
};
|
|
4537
|
+
/**
|
|
4538
|
+
* De-anonymise text using a redaction key.
|
|
4539
|
+
* Replaces placeholders back with original values.
|
|
4540
|
+
* Only works for reversible operators (replace).
|
|
4541
|
+
*/
|
|
4542
|
+
const deanonymise = (redactedText, redactionMap) => {
|
|
4543
|
+
let result = redactedText;
|
|
4544
|
+
for (const [placeholder, original] of redactionMap) result = result.replaceAll(placeholder, original);
|
|
4545
|
+
return result;
|
|
4546
|
+
};
|
|
4547
|
+
//#endregion
|
|
4548
|
+
//#region src/gliner/decoder.ts
|
|
4549
|
+
const sigmoid$1 = (x) => 1 / (1 + Math.exp(-x));
|
|
4550
|
+
/** Check if two spans overlap (optionally allowing multi-label). */
|
|
4551
|
+
const hasOverlapping = (a, b, multiLabel) => {
|
|
4552
|
+
const a0 = a[0] ?? 0;
|
|
4553
|
+
const a1 = a[1] ?? 0;
|
|
4554
|
+
const b0 = b[0] ?? 0;
|
|
4555
|
+
const b1 = b[1] ?? 0;
|
|
4556
|
+
if (a0 === b0 && a1 === b1) return !multiLabel;
|
|
4557
|
+
return !(a0 > b1 || b0 > a1);
|
|
4558
|
+
};
|
|
4559
|
+
/**
|
|
4560
|
+
* Greedy non-overlapping span selection.
|
|
4561
|
+
* Sorts by score descending, keeps each span only if it
|
|
4562
|
+
* doesn't overlap with already-selected spans.
|
|
4563
|
+
*/
|
|
4564
|
+
const greedySearch = (spans, flatNer, multiLabel) => {
|
|
4565
|
+
const sorted = spans.toSorted((a, b) => b[4] - a[4]);
|
|
4566
|
+
const selected = [];
|
|
4567
|
+
for (const span of sorted) if (!selected.some((s) => {
|
|
4568
|
+
if (flatNer) return hasOverlapping([span[1], span[2]], [s[1], s[2]], multiLabel);
|
|
4569
|
+
if (span[1] <= s[1] && span[2] >= s[2] || s[1] <= span[1] && s[2] >= span[2]) return false;
|
|
4570
|
+
return hasOverlapping([span[1], span[2]], [s[1], s[2]], multiLabel);
|
|
4571
|
+
})) selected.push(span);
|
|
4572
|
+
return selected.toSorted((a, b) => a[1] - b[1]);
|
|
4573
|
+
};
|
|
4574
|
+
/**
|
|
4575
|
+
* Decode span-level model logits into entity results.
|
|
4576
|
+
*/
|
|
4577
|
+
const decodeSpans = (batchSize, inputLength, maxWidth, numEntities, texts, batchIds, batchWordsStartIdx, batchWordsEndIdx, idToClass, modelOutput, flatNer, threshold, multiLabel) => {
|
|
4578
|
+
const spans = Array.from({ length: batchSize }, () => []);
|
|
4579
|
+
const batchPadding = inputLength * maxWidth * numEntities;
|
|
4580
|
+
const startTokenPadding = maxWidth * numEntities;
|
|
4581
|
+
const endTokenPadding = numEntities;
|
|
4582
|
+
for (let id = 0; id < modelOutput.length; id++) {
|
|
4583
|
+
const batch = Math.floor(id / batchPadding);
|
|
4584
|
+
const startToken = Math.floor(id / startTokenPadding) % inputLength;
|
|
4585
|
+
const endToken = startToken + Math.floor(id / endTokenPadding) % maxWidth;
|
|
4586
|
+
const entity = id % numEntities;
|
|
4587
|
+
const prob = sigmoid$1(modelOutput[id] ?? 0);
|
|
4588
|
+
const batchStarts = batchWordsStartIdx[batch];
|
|
4589
|
+
const batchEnds = batchWordsEndIdx[batch];
|
|
4590
|
+
const batchSpans = spans[batch];
|
|
4591
|
+
if (!batchStarts || !batchEnds || !batchSpans) continue;
|
|
4592
|
+
if (prob >= threshold && startToken < batchStarts.length && endToken < batchEnds.length) {
|
|
4593
|
+
const globalBatch = batchIds[batch] ?? 0;
|
|
4594
|
+
const startIdx = batchStarts[startToken] ?? 0;
|
|
4595
|
+
const endIdx = batchEnds[endToken] ?? 0;
|
|
4596
|
+
const spanText = (texts[globalBatch] ?? "").slice(startIdx, endIdx);
|
|
4597
|
+
batchSpans.push([
|
|
4598
|
+
spanText,
|
|
4599
|
+
startIdx,
|
|
4600
|
+
endIdx,
|
|
4601
|
+
idToClass[entity + 1] ?? "",
|
|
4602
|
+
prob
|
|
4603
|
+
]);
|
|
4604
|
+
}
|
|
4605
|
+
}
|
|
4606
|
+
return spans.map((batchSpans) => greedySearch(batchSpans, flatNer, multiLabel));
|
|
4607
|
+
};
|
|
4608
|
+
//#endregion
|
|
4609
|
+
//#region src/gliner/token-decoder.ts
|
|
4610
|
+
const sigmoid = (x) => 1 / (1 + Math.exp(-x));
|
|
4611
|
+
const B_TAG = 0;
|
|
4612
|
+
const I_TAG = 1;
|
|
4613
|
+
/**
|
|
4614
|
+
* Decode token-level BIO logits into entity spans.
|
|
4615
|
+
*
|
|
4616
|
+
* For each word, checks if the B(egin) logit for any class
|
|
4617
|
+
* exceeds the threshold. If so, extends the span by consuming
|
|
4618
|
+
* subsequent I(nside) tokens of the same class.
|
|
4619
|
+
*/
|
|
4620
|
+
const decodeTokenSpans = (batchSize, numWords, numEntities, texts, batchIds, batchWordsStartIdx, batchWordsEndIdx, idToClass, modelOutput, threshold) => {
|
|
4621
|
+
const results = Array.from({ length: batchSize }, () => []);
|
|
4622
|
+
const wordStride = numEntities * 3;
|
|
4623
|
+
const batchStride = numWords * wordStride;
|
|
4624
|
+
for (let b = 0; b < batchSize; b++) {
|
|
4625
|
+
const batchOffset = b * batchStride;
|
|
4626
|
+
const starts = batchWordsStartIdx[b];
|
|
4627
|
+
const ends = batchWordsEndIdx[b];
|
|
4628
|
+
const text = texts[batchIds[b] ?? 0] ?? "";
|
|
4629
|
+
const batchSpans = results[b];
|
|
4630
|
+
if (!starts || !ends || !batchSpans) continue;
|
|
4631
|
+
const actualWords = starts.length;
|
|
4632
|
+
for (let e = 0; e < numEntities; e++) {
|
|
4633
|
+
let w = 0;
|
|
4634
|
+
while (w < actualWords) {
|
|
4635
|
+
const bScore = sigmoid(modelOutput[batchOffset + w * wordStride + e * 3 + B_TAG] ?? 0);
|
|
4636
|
+
if (bScore < threshold) {
|
|
4637
|
+
w++;
|
|
4638
|
+
continue;
|
|
4639
|
+
}
|
|
4640
|
+
const spanStart = w;
|
|
4641
|
+
let spanEnd = w;
|
|
4642
|
+
let maxScore = bScore;
|
|
4643
|
+
while (spanEnd + 1 < actualWords) {
|
|
4644
|
+
const iScore = sigmoid(modelOutput[batchOffset + (spanEnd + 1) * wordStride + e * 3 + I_TAG] ?? 0);
|
|
4645
|
+
if (iScore < threshold) break;
|
|
4646
|
+
spanEnd++;
|
|
4647
|
+
maxScore = Math.max(maxScore, iScore);
|
|
4648
|
+
}
|
|
4649
|
+
const charStart = starts[spanStart] ?? 0;
|
|
4650
|
+
const charEnd = ends[spanEnd] ?? 0;
|
|
4651
|
+
const spanText = text.slice(charStart, charEnd);
|
|
4652
|
+
const label = idToClass[e + 1] ?? "";
|
|
4653
|
+
if (spanText.trim().length > 0 && label) batchSpans.push([
|
|
4654
|
+
spanText,
|
|
4655
|
+
charStart,
|
|
4656
|
+
charEnd,
|
|
4657
|
+
label,
|
|
4658
|
+
maxScore
|
|
4659
|
+
]);
|
|
4660
|
+
w = spanEnd + 1;
|
|
4661
|
+
}
|
|
4662
|
+
}
|
|
4663
|
+
const selected = [];
|
|
4664
|
+
for (const span of batchSpans.toSorted((x, y) => y[4] - x[4])) if (!selected.some((s) => span[1] < s[2] && span[2] > s[1])) selected.push(span);
|
|
4665
|
+
results[b] = selected.toSorted((x, y) => x[1] - y[1]);
|
|
4666
|
+
}
|
|
4667
|
+
return results;
|
|
4668
|
+
};
|
|
4669
|
+
//#endregion
|
|
4670
|
+
//#region src/gliner/processor.ts
|
|
4671
|
+
const segmenter = new Intl.Segmenter(void 0, { granularity: "word" });
|
|
4672
|
+
/** Tokenize text into words with character offsets. */
|
|
4673
|
+
const tokenizeText = (text) => {
|
|
4674
|
+
const words = [];
|
|
4675
|
+
const starts = [];
|
|
4676
|
+
const ends = [];
|
|
4677
|
+
for (const { segment, index, isWordLike } of segmenter.segment(text)) {
|
|
4678
|
+
if (!isWordLike && !/\d/u.test(segment)) continue;
|
|
4679
|
+
words.push(segment);
|
|
4680
|
+
starts.push(index);
|
|
4681
|
+
ends.push(index + segment.length);
|
|
4682
|
+
}
|
|
4683
|
+
return [
|
|
4684
|
+
words,
|
|
4685
|
+
starts,
|
|
4686
|
+
ends
|
|
4687
|
+
];
|
|
4688
|
+
};
|
|
4689
|
+
/** Build entity label <-> id mappings. */
|
|
4690
|
+
const createMappings = (labels) => {
|
|
4691
|
+
const classToId = {};
|
|
4692
|
+
const idToClass = {};
|
|
4693
|
+
for (let i = 0; i < labels.length; i++) {
|
|
4694
|
+
const label = labels[i];
|
|
4695
|
+
if (label === void 0) continue;
|
|
4696
|
+
const id = i + 1;
|
|
4697
|
+
classToId[label] = id;
|
|
4698
|
+
idToClass[id] = label;
|
|
4699
|
+
}
|
|
4700
|
+
return {
|
|
4701
|
+
classToId,
|
|
4702
|
+
idToClass
|
|
4703
|
+
};
|
|
4704
|
+
};
|
|
4705
|
+
/** Prepend entity prompt tokens to word tokens. */
|
|
4706
|
+
const prepareTextInputs = (batchTokens, entities) => {
|
|
4707
|
+
const inputTexts = [];
|
|
4708
|
+
const promptLengths = [];
|
|
4709
|
+
const textLengths = [];
|
|
4710
|
+
for (const tokens of batchTokens) {
|
|
4711
|
+
textLengths.push(tokens.length);
|
|
4712
|
+
const prompt = [];
|
|
4713
|
+
for (const ent of entities) {
|
|
4714
|
+
prompt.push("<<ENT>>");
|
|
4715
|
+
prompt.push(ent);
|
|
4716
|
+
}
|
|
4717
|
+
prompt.push("<<SEP>>");
|
|
4718
|
+
promptLengths.push(prompt.length);
|
|
4719
|
+
inputTexts.push([...prompt, ...tokens]);
|
|
4720
|
+
}
|
|
4721
|
+
return [
|
|
4722
|
+
inputTexts,
|
|
4723
|
+
textLengths,
|
|
4724
|
+
promptLengths
|
|
4725
|
+
];
|
|
4726
|
+
};
|
|
4727
|
+
/** Encode word sequences into token IDs with masks. */
|
|
4728
|
+
const encodeInputs = (tokenizer, texts, promptLengths) => {
|
|
4729
|
+
const clsTokenId = tokenizer.token_to_id("[CLS]") ?? 1;
|
|
4730
|
+
const sepTokenId = tokenizer.token_to_id("[SEP]") ?? 2;
|
|
4731
|
+
const allInputIds = [];
|
|
4732
|
+
const allAttentionMasks = [];
|
|
4733
|
+
const allWordsMasks = [];
|
|
4734
|
+
for (let idx = 0; idx < texts.length; idx++) {
|
|
4735
|
+
const promptLength = promptLengths[idx] ?? 0;
|
|
4736
|
+
const words = texts[idx];
|
|
4737
|
+
if (!words) continue;
|
|
4738
|
+
const wordsMask = [0];
|
|
4739
|
+
const inputIds = [clsTokenId];
|
|
4740
|
+
const attentionMask = [1];
|
|
4741
|
+
let wordCounter = 1;
|
|
4742
|
+
for (let wordId = 0; wordId < words.length; wordId++) {
|
|
4743
|
+
const word = words[wordId];
|
|
4744
|
+
if (word === void 0) continue;
|
|
4745
|
+
const wordTokens = tokenizer.encode(word).ids.slice(1, -1);
|
|
4746
|
+
for (let tokenId = 0; tokenId < wordTokens.length; tokenId++) {
|
|
4747
|
+
attentionMask.push(1);
|
|
4748
|
+
if (wordId < promptLength) wordsMask.push(0);
|
|
4749
|
+
else if (tokenId === 0) {
|
|
4750
|
+
wordsMask.push(wordCounter);
|
|
4751
|
+
wordCounter++;
|
|
4752
|
+
} else wordsMask.push(0);
|
|
4753
|
+
inputIds.push(wordTokens[tokenId] ?? 0);
|
|
4754
|
+
}
|
|
4755
|
+
}
|
|
4756
|
+
inputIds.push(sepTokenId);
|
|
4757
|
+
wordsMask.push(0);
|
|
4758
|
+
attentionMask.push(1);
|
|
4759
|
+
allInputIds.push(inputIds);
|
|
4760
|
+
allAttentionMasks.push(attentionMask);
|
|
4761
|
+
allWordsMasks.push(wordsMask);
|
|
4762
|
+
}
|
|
4763
|
+
return [
|
|
4764
|
+
allInputIds,
|
|
4765
|
+
allAttentionMasks,
|
|
4766
|
+
allWordsMasks
|
|
4767
|
+
];
|
|
4768
|
+
};
|
|
4769
|
+
/** Build span index pairs and masks for the model. */
|
|
4770
|
+
const prepareSpans = (batchTokens, maxWidth) => {
|
|
4771
|
+
const spanIdxs = [];
|
|
4772
|
+
const spanMasks = [];
|
|
4773
|
+
for (const tokens of batchTokens) {
|
|
4774
|
+
const len = tokens.length;
|
|
4775
|
+
const idx = [];
|
|
4776
|
+
const mask = [];
|
|
4777
|
+
for (let i = 0; i < len; i++) for (let j = 0; j < maxWidth; j++) {
|
|
4778
|
+
const endIdx = Math.min(i + j, len - 1);
|
|
4779
|
+
idx.push([i, endIdx]);
|
|
4780
|
+
mask.push(endIdx < len);
|
|
4781
|
+
}
|
|
4782
|
+
spanIdxs.push(idx);
|
|
4783
|
+
spanMasks.push(mask);
|
|
4784
|
+
}
|
|
4785
|
+
return {
|
|
4786
|
+
spanIdxs,
|
|
4787
|
+
spanMasks
|
|
4788
|
+
};
|
|
4789
|
+
};
|
|
4790
|
+
/** Pad a 2D or 3D array to uniform inner length. */
|
|
4791
|
+
const padArray = (arr, dimensions = 2) => {
|
|
4792
|
+
if (arr.length === 0) return [];
|
|
4793
|
+
const maxLength = Math.max(...arr.map((sub) => sub.length));
|
|
4794
|
+
let finalDim = 0;
|
|
4795
|
+
if (dimensions === 3) {
|
|
4796
|
+
for (const sub of arr) if (sub.length > 0) {
|
|
4797
|
+
finalDim = sub[0].length;
|
|
4798
|
+
break;
|
|
4799
|
+
}
|
|
4800
|
+
}
|
|
4801
|
+
return arr.map((sub) => {
|
|
4802
|
+
const padCount = maxLength - sub.length;
|
|
4803
|
+
const fill = dimensions === 3 ? Array.from({ length: padCount }, () => Array.from({ length: finalDim }).fill(0)) : Array.from({ length: padCount }).fill(0);
|
|
4804
|
+
return [...sub, ...fill];
|
|
4805
|
+
});
|
|
4806
|
+
};
|
|
4807
|
+
/** Prepare a complete batch for ONNX inference. */
|
|
4808
|
+
const prepareBatch = (tokenizer, texts, entities, maxWidth) => {
|
|
4809
|
+
const batchTokens = [];
|
|
4810
|
+
const batchWordsStartIdx = [];
|
|
4811
|
+
const batchWordsEndIdx = [];
|
|
4812
|
+
for (const text of texts) {
|
|
4813
|
+
const [words, starts, ends] = tokenizeText(text);
|
|
4814
|
+
batchTokens.push(words);
|
|
4815
|
+
batchWordsStartIdx.push(starts);
|
|
4816
|
+
batchWordsEndIdx.push(ends);
|
|
4817
|
+
}
|
|
4818
|
+
const { idToClass } = createMappings(entities);
|
|
4819
|
+
const [inputTokens, textLengths, promptLengths] = prepareTextInputs(batchTokens, entities);
|
|
4820
|
+
let [inputsIds, attentionMasks, wordsMasks] = encodeInputs(tokenizer, inputTokens, promptLengths);
|
|
4821
|
+
inputsIds = padArray(inputsIds);
|
|
4822
|
+
attentionMasks = padArray(attentionMasks);
|
|
4823
|
+
wordsMasks = padArray(wordsMasks);
|
|
4824
|
+
let { spanIdxs, spanMasks } = prepareSpans(batchTokens, maxWidth);
|
|
4825
|
+
spanIdxs = padArray(spanIdxs, 3);
|
|
4826
|
+
spanMasks = padArray(spanMasks);
|
|
4827
|
+
return {
|
|
4828
|
+
inputsIds,
|
|
4829
|
+
attentionMasks,
|
|
4830
|
+
wordsMasks,
|
|
4831
|
+
textLengths,
|
|
4832
|
+
spanIdxs,
|
|
4833
|
+
spanMasks,
|
|
4834
|
+
idToClass,
|
|
4835
|
+
batchTokens,
|
|
4836
|
+
batchWordsStartIdx,
|
|
4837
|
+
batchWordsEndIdx
|
|
4838
|
+
};
|
|
4839
|
+
};
|
|
4840
|
+
//#endregion
|
|
4841
|
+
//#region src/util/chunker.ts
|
|
4842
|
+
const MAX_CHUNK_CHARS = 1500;
|
|
4843
|
+
const OVERLAP_CHARS = 50;
|
|
4844
|
+
const MIN_CHUNK_LENGTH = 10;
|
|
4845
|
+
/**
|
|
4846
|
+
* Split text into overlapping chunks for GLiNER's
|
|
4847
|
+
* ~512 token context window. Character-based splitting
|
|
4848
|
+
* (rough approximation of token limits).
|
|
4849
|
+
*
|
|
4850
|
+
* Tries to break at sentence boundaries when possible.
|
|
4851
|
+
*/
|
|
4852
|
+
const chunkText = (text) => {
|
|
4853
|
+
const chunks = [];
|
|
4854
|
+
let offset = 0;
|
|
4855
|
+
while (offset < text.length) {
|
|
4856
|
+
let end = Math.min(offset + MAX_CHUNK_CHARS, text.length);
|
|
4857
|
+
if (end < text.length) {
|
|
4858
|
+
const lastPeriod = text.slice(offset, end).lastIndexOf(". ");
|
|
4859
|
+
if (lastPeriod > MAX_CHUNK_CHARS * .5) end = offset + lastPeriod + 2;
|
|
4860
|
+
}
|
|
4861
|
+
const chunk = text.slice(offset, end);
|
|
4862
|
+
if (chunk.trim().length > MIN_CHUNK_LENGTH) chunks.push(chunk);
|
|
4863
|
+
offset = Math.max(offset + 1, end - OVERLAP_CHARS);
|
|
4864
|
+
}
|
|
4865
|
+
return chunks;
|
|
4866
|
+
};
|
|
4867
|
+
/**
|
|
4868
|
+
* Compute the byte offset of each chunk within the
|
|
4869
|
+
* original document text.
|
|
4870
|
+
*/
|
|
4871
|
+
const computeChunkOffsets = (fullText, chunks) => {
|
|
4872
|
+
const offsets = [];
|
|
4873
|
+
let searchFrom = 0;
|
|
4874
|
+
for (const chunk of chunks) {
|
|
4875
|
+
const idx = fullText.indexOf(chunk, searchFrom);
|
|
4876
|
+
offsets.push(idx !== -1 ? idx : searchFrom);
|
|
4877
|
+
searchFrom = idx !== -1 ? idx + Math.max(1, chunk.length - OVERLAP_CHARS) : searchFrom;
|
|
4878
|
+
}
|
|
4879
|
+
return offsets;
|
|
4880
|
+
};
|
|
4881
|
+
const POSITION_THRESHOLD = 5;
|
|
4882
|
+
/**
|
|
4883
|
+
* Merge entities from overlapping chunks back to
|
|
4884
|
+
* document-level offsets. Deduplicates entities that
|
|
4885
|
+
* appear in overlap regions (keeps highest score).
|
|
4886
|
+
*
|
|
4887
|
+
* Dedup invariant: each incoming entity is compared
|
|
4888
|
+
* against the highest-scored same-label near-dup in
|
|
4889
|
+
* its proximity window. If it loses, it is dropped.
|
|
4890
|
+
* This does NOT guarantee that all pairwise near-dup
|
|
4891
|
+
* relationships in the output are resolved; a lower-
|
|
4892
|
+
* scored entity can survive if the bridging entity
|
|
4893
|
+
* that would have replaced it was itself dropped by
|
|
4894
|
+
* a higher-scored match.
|
|
4895
|
+
*
|
|
4896
|
+
* Uses a reverse-scan over the sorted merged array
|
|
4897
|
+
* so each entity only compares against nearby
|
|
4898
|
+
* predecessors — O(n * w) average where w is the max
|
|
4899
|
+
* entities per POSITION_THRESHOLD window, O(n²) worst
|
|
4900
|
+
* case when replacements dominate (splice is O(n)).
|
|
4901
|
+
*/
|
|
4902
|
+
const mergeChunkEntities = (chunkOffsets, chunkResults) => {
|
|
4903
|
+
const allEntities = [];
|
|
4904
|
+
for (let i = 0; i < chunkResults.length; i++) {
|
|
4905
|
+
const offset = chunkOffsets[i] ?? 0;
|
|
4906
|
+
const entities = chunkResults[i];
|
|
4907
|
+
if (!entities) continue;
|
|
4908
|
+
for (const entity of entities) allEntities.push({
|
|
4909
|
+
...entity,
|
|
4910
|
+
start: entity.start + offset,
|
|
4911
|
+
end: entity.end + offset
|
|
4912
|
+
});
|
|
4913
|
+
}
|
|
4914
|
+
const sorted = allEntities.toSorted((a, b) => a.start - b.start);
|
|
4915
|
+
const merged = [];
|
|
4916
|
+
for (const entity of sorted) {
|
|
4917
|
+
let bestDupIndex = -1;
|
|
4918
|
+
let bestDupScore = -1;
|
|
4919
|
+
for (let j = merged.length - 1; j >= 0; j--) {
|
|
4920
|
+
const existing = merged[j];
|
|
4921
|
+
if (existing === void 0) continue;
|
|
4922
|
+
if (entity.start - existing.start >= POSITION_THRESHOLD) break;
|
|
4923
|
+
if (existing.label === entity.label && Math.abs(existing.end - entity.end) < POSITION_THRESHOLD && existing.score > bestDupScore) {
|
|
4924
|
+
bestDupIndex = j;
|
|
4925
|
+
bestDupScore = existing.score;
|
|
4926
|
+
}
|
|
4927
|
+
}
|
|
4928
|
+
if (bestDupIndex !== -1) {
|
|
4929
|
+
const existing = merged[bestDupIndex];
|
|
4930
|
+
if (existing !== void 0 && entity.score > existing.score) {
|
|
4931
|
+
merged.splice(bestDupIndex, 1);
|
|
4932
|
+
merged.push({ ...entity });
|
|
4933
|
+
}
|
|
4934
|
+
} else merged.push({ ...entity });
|
|
4935
|
+
}
|
|
4936
|
+
return merged;
|
|
4937
|
+
};
|
|
4938
|
+
//#endregion
|
|
4939
|
+
//#region src/util/levenshtein.ts
|
|
4940
|
+
/**
|
|
4941
|
+
* Compute the Levenshtein edit distance between two
|
|
4942
|
+
* strings. O(n*m) time, O(min(n,m)) space using a
|
|
4943
|
+
* single-row DP approach.
|
|
4944
|
+
*/
|
|
4945
|
+
const levenshtein = (rawA, rawB) => {
|
|
4946
|
+
if (rawA === rawB) return 0;
|
|
4947
|
+
if (rawA.length === 0) return rawB.length;
|
|
4948
|
+
if (rawB.length === 0) return rawA.length;
|
|
4949
|
+
const [shorter, longer] = rawA.length <= rawB.length ? [rawA, rawB] : [rawB, rawA];
|
|
4950
|
+
const aLen = shorter.length;
|
|
4951
|
+
const bLen = longer.length;
|
|
4952
|
+
const row = new Uint16Array(aLen + 1);
|
|
4953
|
+
for (let i = 0; i <= aLen; i++) row[i] = i;
|
|
4954
|
+
for (let j = 1; j <= bLen; j++) {
|
|
4955
|
+
let prev = row[0] ?? 0;
|
|
4956
|
+
row[0] = j;
|
|
4957
|
+
for (let i = 1; i <= aLen; i++) {
|
|
4958
|
+
const cost = shorter[i - 1] === longer[j - 1] ? 0 : 1;
|
|
4959
|
+
const temp = row[i] ?? 0;
|
|
4960
|
+
row[i] = Math.min((row[i] ?? 0) + 1, (row[i - 1] ?? 0) + 1, prev + cost);
|
|
4961
|
+
prev = temp;
|
|
4962
|
+
}
|
|
4963
|
+
}
|
|
4964
|
+
return row[aLen] ?? 0;
|
|
4965
|
+
};
|
|
4966
|
+
//#endregion
|
|
4967
|
+
export { CURRENCY_PATTERN_META, DATE_PATTERN_META, DEFAULT_ENTITY_LABELS, DEFAULT_OPERATOR_CONFIG, DETECTION_SOURCES, DETECTOR_PRIORITY, OPERATOR_REGISTRY, OPERATOR_TYPES, REGEX_META, REGEX_PATTERNS, REGIONS, ZONE_SCORE_ADJUSTMENTS, applyHotwordRules, applyZoneAdjustments, boostNearMissEntities, buildDenyList, buildGazetteerPatterns, buildLegalFormPatterns, buildPlaceholderMap, buildStreetTypePatterns, buildTriggerPatterns, buildUnifiedSearch, chunkText, classifyZones, computeChunkOffsets, corefKey, createPipelineContext, deanonymise, decodeSpans, decodeTokenSpans, detectNameCorpus, ensureDenyListData, exportRedactionKey, extractDefinedTerms, filterFalsePositives, findCoreferenceSpans, getCurrencyPatterns, getDatePatterns, initHotwordRules, initNameCorpus, initZoneClassifier, levenshtein, mergeAndDedup, mergeChunkEntities, normalizeForSearch, prepareBatch, processAddressSeeds, processDenyListMatches, processGazetteerMatches, processLegalFormMatches, processRegexMatches, processTriggerMatches, propagateOrgNames, redactText, resolveCountries, resolveOperator, runPipeline, runUnifiedSearch, sanitizeEntities, tokenizeText };
|
|
4968
|
+
|
|
4969
|
+
//# sourceMappingURL=index.js.map
|