@doccloak/core 0.11.0 → 0.12.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +148 -2
- package/README.md +252 -19
- package/dist/doc.d.ts +68 -1
- package/dist/doc.js +447 -62
- package/dist/dom/docx.d.ts +135 -7
- package/dist/dom/docx.js +752 -125
- package/dist/dom/drawingml.d.ts +59 -0
- package/dist/dom/drawingml.js +176 -0
- package/dist/dom/errors.d.ts +35 -0
- package/dist/dom/errors.js +24 -0
- package/dist/dom/index.d.ts +3 -0
- package/dist/dom/index.js +6 -0
- package/dist/dom/office.d.ts +56 -0
- package/dist/dom/office.js +76 -23
- package/dist/dom/opc.d.ts +135 -7
- package/dist/dom/opc.js +407 -14
- package/dist/dom/package-policy.d.ts +137 -0
- package/dist/dom/package-policy.js +355 -0
- package/dist/dom/xlsx.d.ts +64 -6
- package/dist/dom/xlsx.js +130 -20
- package/dist/engine.d.ts +18 -0
- package/dist/engine.js +37 -13
- package/dist/env.d.ts +21 -1
- package/dist/index.d.ts +8 -6
- package/dist/index.js +10 -5
- package/dist/model-loader.d.ts +61 -6
- package/dist/model-loader.js +122 -17
- package/dist/pipeline.d.ts +87 -8
- package/dist/pipeline.js +249 -50
- package/dist/providers/bardsai.d.ts +8 -1
- package/dist/providers/bardsai.js +22 -2
- package/dist/providers/gliner-base.d.ts +8 -1
- package/dist/providers/gliner-base.js +22 -2
- package/dist/providers/gliner.d.ts +8 -1
- package/dist/providers/gliner.js +22 -2
- package/dist/regex/loader.d.ts +2 -0
- package/dist/regex/loader.js +2 -0
- package/dist/regex/rules.data.js +15 -14
- package/dist/regex/types.d.ts +6 -0
- package/dist/session.d.ts +101 -3
- package/dist/session.js +397 -43
- package/dist/surrogates.js +57 -45
- package/dist/worker-protocol.d.ts +13 -1
- package/dist/worker-protocol.js +164 -12
- package/package.json +1 -1
- package/rules/at.json +1 -1
- package/rules/be.json +1 -1
- package/rules/ch.json +1 -1
- package/rules/de.json +1 -1
- package/rules/dk.json +1 -1
- package/rules/es.json +1 -1
- package/rules/fi.json +1 -1
- package/rules/fr.json +1 -1
- package/rules/it.json +1 -1
- package/rules/no.json +1 -1
- package/rules/pl.json +1 -1
- package/rules/pt.json +1 -1
- package/rules/schema.json +4 -0
- package/rules/se.json +1 -1
- package/rules/universal.json +2 -1
package/dist/session.js
CHANGED
|
@@ -1,10 +1,17 @@
|
|
|
1
1
|
import { buildTolerantTokenIndex, resolveMangledToken } from "./restore-tokens.js";
|
|
2
|
-
import { generateSessionSalt,
|
|
2
|
+
import { generateSessionSalt, generateSurrogate } from "./surrogates.js";
|
|
3
3
|
export * from "./restore-tokens.js";
|
|
4
4
|
// T099: credentials never get realistic stand-ins - a same-shape fake key
|
|
5
5
|
// still looks like a live credential to anyone (or any scanner) reading the
|
|
6
6
|
// prompt. Surrogate mode falls back to typed placeholders for these types.
|
|
7
7
|
const PLACEHOLDER_ONLY_TYPES = new Set(['SECRET', 'API_KEY']);
|
|
8
|
+
/**
|
|
9
|
+
* Edge punctuation stripped by personTokens. The run is BOUNDED (R14): an
|
|
10
|
+
* unbounded `[...]+$` backtracks from every position of a long punctuation
|
|
11
|
+
* run, which is quadratic on a hostile 100 KB span; {1,64} keeps it linear
|
|
12
|
+
* and 64 marks is more than any real name carries.
|
|
13
|
+
*/
|
|
14
|
+
const EDGE_PUNCTUATION_RE = /^[.,;:!?()"'\u201E\u201C\u201D]{1,64}|[.,;:!?()"'\u201E\u201C\u201D]{1,64}$/g;
|
|
8
15
|
/**
|
|
9
16
|
* Person-variant tokenization (T057): lowercase word tokens with edge
|
|
10
17
|
* punctuation stripped. EVERY token participates in the subset test -
|
|
@@ -14,7 +21,7 @@ function personTokens(value) {
|
|
|
14
21
|
return value
|
|
15
22
|
.toLowerCase()
|
|
16
23
|
.split(/\s+/)
|
|
17
|
-
.map((t) => t.replace(
|
|
24
|
+
.map((t) => t.replace(EDGE_PUNCTUATION_RE, ''))
|
|
18
25
|
.filter((t) => t.length > 0);
|
|
19
26
|
}
|
|
20
27
|
/**
|
|
@@ -31,8 +38,23 @@ function isPersonVariant(a, b) {
|
|
|
31
38
|
const tb = personTokens(b);
|
|
32
39
|
if (ta.length === 0 || tb.length === 0)
|
|
33
40
|
return false;
|
|
34
|
-
const
|
|
35
|
-
|
|
41
|
+
const fits = (small, big) => small.every((t) => hasToken(big, t)) && small.some((t) => t.length >= 3);
|
|
42
|
+
if (ta.length < tb.length)
|
|
43
|
+
return fits(ta, tb);
|
|
44
|
+
if (tb.length < ta.length)
|
|
45
|
+
return fits(tb, ta);
|
|
46
|
+
// Equal counts: either side may carry the hyphen-extended surname.
|
|
47
|
+
return fits(ta, tb) || fits(tb, ta);
|
|
48
|
+
}
|
|
49
|
+
/**
|
|
50
|
+
* Whether `token` occurs among `tokens`, exactly or as one part of a
|
|
51
|
+
* hyphenated token: "nowak" is found in "Anna Nowak-Kowalska", so the
|
|
52
|
+
* double-surname form still counts as the same person once matching is
|
|
53
|
+
* restricted to group references (R5) and can no longer ride on a
|
|
54
|
+
* shorter variant. A hyphenated token itself only matches exactly.
|
|
55
|
+
*/
|
|
56
|
+
function hasToken(tokens, token) {
|
|
57
|
+
return tokens.some((t) => t === token || (t.includes('-') && t.split('-').includes(token)));
|
|
36
58
|
}
|
|
37
59
|
/**
|
|
38
60
|
* Variant suffix (T171): where a person variant sits inside the reference
|
|
@@ -47,6 +69,11 @@ function variantSuffix(variant, reference) {
|
|
|
47
69
|
const rt = personTokens(reference);
|
|
48
70
|
if (vt.length > rt.length)
|
|
49
71
|
return 'FULL';
|
|
72
|
+
// Same token count but the reference fits inside a longer variant: a
|
|
73
|
+
// double surname extending the reference ("Nowak" -> "Nowak-Kowalska").
|
|
74
|
+
if (vt.length === rt.length && variant.length > reference.length && isTokenSubset(reference, variant)) {
|
|
75
|
+
return 'FULL';
|
|
76
|
+
}
|
|
50
77
|
if (vt.length === rt.length)
|
|
51
78
|
return 'ALT';
|
|
52
79
|
if (vt.length === 1) {
|
|
@@ -63,7 +90,7 @@ function variantSuffix(variant, reference) {
|
|
|
63
90
|
function isTokenSubset(a, b) {
|
|
64
91
|
const tb = personTokens(b);
|
|
65
92
|
const ta = personTokens(a);
|
|
66
|
-
return ta.length > 0 && ta.every((t) => tb
|
|
93
|
+
return ta.length > 0 && ta.every((t) => hasToken(tb, t));
|
|
67
94
|
}
|
|
68
95
|
/**
|
|
69
96
|
* A variant token derived from its base token: the suffix goes inside the
|
|
@@ -99,6 +126,91 @@ const TYPED_PLACEHOLDER_RE = /^\[([A-Z_]+)_(\d+)\]$/;
|
|
|
99
126
|
* safe - replacement still requires an unambiguous index hit.
|
|
100
127
|
*/
|
|
101
128
|
const MANGLED_CANDIDATE_RE = /\[[^[\]\n]+\]|<<[^<>\n]+>>|(?<![\p{L}\p{N}_[<])[A-Z][A-Z0-9]*(?:_[A-Z0-9]+)*_\d+(?![\p{L}\p{N}_])/gu;
|
|
129
|
+
/**
|
|
130
|
+
* Literal typed placeholders already present in USER text (R9): a
|
|
131
|
+
* re-processed document or a template carrying "[PERSON_1]" must not
|
|
132
|
+
* collide with the placeholders this session issues.
|
|
133
|
+
*/
|
|
134
|
+
const LITERAL_PLACEHOLDER_RE = /\[([A-Z_]+)_(\d+)\]/g;
|
|
135
|
+
/** Counter numbers beyond this many digits are ignored (never issued). */
|
|
136
|
+
const MAX_COUNTER_DIGITS = 9;
|
|
137
|
+
/** Upper bound for deserialize/importEntries (R11). */
|
|
138
|
+
const MAX_IMPORT_ENTRIES = 10000;
|
|
139
|
+
/**
|
|
140
|
+
* Surrogate re-derivations tried before the numeric-suffix fallback (R3).
|
|
141
|
+
* Every attempt is a fresh deterministic draw from the locale pools; with
|
|
142
|
+
* 40 x 40 names per locale and gender, 24 draws leave a starved pool (a
|
|
143
|
+
* document naming most of a pool) as the only way to reach the fallback.
|
|
144
|
+
*/
|
|
145
|
+
const MAX_SURROGATE_ATTEMPTS = 24;
|
|
146
|
+
/**
|
|
147
|
+
* Shortest document name token that the surrogate containment rule (R3)
|
|
148
|
+
* considers: a PERSON surrogate must not CONTAIN a document name of this
|
|
149
|
+
* length or more ("Johnson" for a document that names "John", "Marianna"
|
|
150
|
+
* for "Anna"). Same threshold as isPersonVariant's substantive token.
|
|
151
|
+
*/
|
|
152
|
+
const MIN_NAME_TOKEN_LENGTH = 3;
|
|
153
|
+
/** A token made of letters only (names); digits, emails and codes are not indexed for containment. */
|
|
154
|
+
const LETTERS_ONLY_RE = /^\p{L}+$/u;
|
|
155
|
+
/** Only regex syntax characters are escaped: valid under the `u` flag. */
|
|
156
|
+
function escapeRegExp(s) {
|
|
157
|
+
return s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
|
158
|
+
}
|
|
159
|
+
/**
|
|
160
|
+
* Restore keys that are tokens rather than natural text: bracket
|
|
161
|
+
* placeholders (generated or renamed) and legacy angle tokens. They carry
|
|
162
|
+
* their own delimiters, so they match anywhere ("[PERSON_1]'s", glued
|
|
163
|
+
* "[PERSON_1][EMAIL_1]").
|
|
164
|
+
*/
|
|
165
|
+
const TOKEN_SHAPED_KEY_RE = /^(?:\[[^[\]\n]+\]|<<[^<>\n]+>>)$/;
|
|
166
|
+
/**
|
|
167
|
+
* Edge characters that ask for a word boundary on their side of a natural
|
|
168
|
+
* key: digits and letters of space-delimited scripts (Latin, Cyrillic,
|
|
169
|
+
* Greek). CJK keys stay unguarded because those scripts write words
|
|
170
|
+
* without separators (王伟说 must restore 王伟).
|
|
171
|
+
*/
|
|
172
|
+
const BOUNDED_EDGE_RE = /^[\p{N}\p{Script=Latin}\p{Script=Cyrillic}\p{Script=Greek}]$/u;
|
|
173
|
+
/**
|
|
174
|
+
* One alternation branch per restore key (R4). A natural key such as a
|
|
175
|
+
* surrogate name gets `(?<![\p{L}\p{N}])` only when it STARTS with a
|
|
176
|
+
* bounded-edge character and `(?![\p{L}\p{N}])` only when it ENDS with
|
|
177
|
+
* one, so "Bo" never fires inside "Bob" or "Boston" while "+48 601..."
|
|
178
|
+
* (starting with '+') and "$1,234" still match after any character.
|
|
179
|
+
*/
|
|
180
|
+
function restoreKeyPattern(key) {
|
|
181
|
+
const body = escapeRegExp(key);
|
|
182
|
+
if (TOKEN_SHAPED_KEY_RE.test(key))
|
|
183
|
+
return body;
|
|
184
|
+
const chars = [...key];
|
|
185
|
+
const first = chars[0] ?? '';
|
|
186
|
+
const last = chars[chars.length - 1] ?? '';
|
|
187
|
+
const lead = BOUNDED_EDGE_RE.test(first) ? '(?<![\\p{L}\\p{N}])' : '';
|
|
188
|
+
const trail = BOUNDED_EDGE_RE.test(last) ? '(?![\\p{L}\\p{N}])' : '';
|
|
189
|
+
return `${lead}${body}${trail}`;
|
|
190
|
+
}
|
|
191
|
+
/**
|
|
192
|
+
* Shape check for one imported entry (R11): plain object, string fields,
|
|
193
|
+
* a non-blank replacement (an empty or whitespace key would match between
|
|
194
|
+
* every character and explode the restored text) and a non-empty original.
|
|
195
|
+
* Throws a descriptive Error naming the entry index.
|
|
196
|
+
*/
|
|
197
|
+
function validateImportEntry(entry, index) {
|
|
198
|
+
const where = `Invalid session map entry #${index}`;
|
|
199
|
+
if (typeof entry !== 'object' || entry === null || Array.isArray(entry)) {
|
|
200
|
+
throw new Error(`${where}: expected an object`);
|
|
201
|
+
}
|
|
202
|
+
const { original, replacement, entityType } = entry;
|
|
203
|
+
if (typeof original !== 'string' || original.length === 0) {
|
|
204
|
+
throw new Error(`${where}: "original" must be a non-empty string`);
|
|
205
|
+
}
|
|
206
|
+
if (typeof replacement !== 'string' || replacement.trim().length === 0) {
|
|
207
|
+
throw new Error(`${where}: "replacement" must be a non-blank string`);
|
|
208
|
+
}
|
|
209
|
+
if (typeof entityType !== 'string' || entityType.length === 0) {
|
|
210
|
+
throw new Error(`${where}: "entityType" must be a non-empty string`);
|
|
211
|
+
}
|
|
212
|
+
return { original, replacement, entityType: entityType };
|
|
213
|
+
}
|
|
102
214
|
export class AnonymizationSession {
|
|
103
215
|
constructor(options) {
|
|
104
216
|
this.forwardMap = new Map();
|
|
@@ -110,6 +222,34 @@ export class AnonymizationSession {
|
|
|
110
222
|
* the base re-derives its variants and deserialized maps re-link.
|
|
111
223
|
*/
|
|
112
224
|
this.variantOf = new Map();
|
|
225
|
+
/**
|
|
226
|
+
* Compiled exact-restore alternation (R4), rebuilt lazily; null after
|
|
227
|
+
* every reverse-map mutation (see invalidateRestore).
|
|
228
|
+
*/
|
|
229
|
+
this.restoreRe = null;
|
|
230
|
+
/**
|
|
231
|
+
* Every value known to occur in the session's documents (R3), lowercased:
|
|
232
|
+
* whole entity values plus their person tokens (and hyphen parts). Fed by
|
|
233
|
+
* anonymizeText, anonymize, importEntries and registerDocumentValues;
|
|
234
|
+
* cumulative until clear(). A surrogate is never one of these. No
|
|
235
|
+
* capitalised-word heuristic: it would starve the pools in German and
|
|
236
|
+
* Polish prose (plan v2 section 9).
|
|
237
|
+
*/
|
|
238
|
+
this.documentValues = new Set();
|
|
239
|
+
/**
|
|
240
|
+
* Letter-only document tokens of MIN_NAME_TOKEN_LENGTH or more, bucketed
|
|
241
|
+
* by their first MIN_NAME_TOKEN_LENGTH characters, for the containment
|
|
242
|
+
* rule in isDocumentValue: linear in the candidate's length instead of
|
|
243
|
+
* a scan over every document token.
|
|
244
|
+
*/
|
|
245
|
+
this.documentNameIndex = new Map();
|
|
246
|
+
/**
|
|
247
|
+
* PERSON values that matched two or more mapped people equally well and
|
|
248
|
+
* therefore got a fresh identity instead of a guessed one (R5).
|
|
249
|
+
*/
|
|
250
|
+
this.ambiguousValues = new Set();
|
|
251
|
+
/** Whether the most recent anonymize() call refused an ambiguous unification. */
|
|
252
|
+
this.lastAmbiguousFlag = false;
|
|
113
253
|
this.mode = options?.mode ?? 'labeled';
|
|
114
254
|
this.salt = options?.salt ?? generateSessionSalt();
|
|
115
255
|
}
|
|
@@ -147,36 +287,164 @@ export class AnonymizationSession {
|
|
|
147
287
|
* deanonymize restores it via the same exact-literal lookup.
|
|
148
288
|
*/
|
|
149
289
|
nextSurrogate(original, entityType) {
|
|
150
|
-
|
|
290
|
+
// R3: besides the maps, a candidate is refused when it is (or, for a
|
|
291
|
+
// PERSON, contains) a value the documents carry. That rule cannot be
|
|
292
|
+
// satisfied by generateUniqueSurrogate's digit suffix (it only changes
|
|
293
|
+
// the last token), so the session runs its own attempt loop and the
|
|
294
|
+
// suffix fallback keeps the exact checks only.
|
|
295
|
+
const ctx = { salt: this.salt, entries: this.getEntries() };
|
|
296
|
+
const exactlyTaken = (candidate) => candidate === original
|
|
297
|
+
|| this.forwardMap.has(candidate)
|
|
298
|
+
|| this.reverseMap.has(candidate)
|
|
299
|
+
|| this.documentValues.has(candidate.toLowerCase());
|
|
300
|
+
let candidate = '';
|
|
301
|
+
for (let attempt = 0; attempt < MAX_SURROGATE_ATTEMPTS; attempt++) {
|
|
302
|
+
candidate = generateSurrogate(original, entityType, ctx, attempt);
|
|
303
|
+
if (!exactlyTaken(candidate) && !this.containsDocumentName(candidate, entityType)) {
|
|
304
|
+
return candidate;
|
|
305
|
+
}
|
|
306
|
+
}
|
|
307
|
+
for (let n = 2;; n++) {
|
|
308
|
+
const suffixed = `${candidate}${n}`;
|
|
309
|
+
if (!exactlyTaken(suffixed))
|
|
310
|
+
return suffixed;
|
|
311
|
+
}
|
|
151
312
|
}
|
|
152
313
|
/**
|
|
153
|
-
*
|
|
154
|
-
*
|
|
155
|
-
*
|
|
314
|
+
* Whether a PERSON surrogate candidate carries a document name (R3):
|
|
315
|
+
* any of its tokens is a document value, or contains one of at least
|
|
316
|
+
* MIN_NAME_TOKEN_LENGTH letters ("Johnson" with "John" in the document,
|
|
317
|
+
* "Nowakowski" with "Nowak"). Other types are covered by the exact
|
|
318
|
+
* whole-value check alone: their tokens ("+48", "ul.", "GmbH") recur in
|
|
319
|
+
* every surrogate of the same shape and would starve the pools.
|
|
320
|
+
*/
|
|
321
|
+
containsDocumentName(candidate, entityType) {
|
|
322
|
+
if (entityType !== 'PERSON')
|
|
323
|
+
return false;
|
|
324
|
+
for (const token of personTokens(candidate)) {
|
|
325
|
+
if (this.documentValues.has(token))
|
|
326
|
+
return true;
|
|
327
|
+
for (let i = 0; i + MIN_NAME_TOKEN_LENGTH <= token.length; i++) {
|
|
328
|
+
const bucket = this.documentNameIndex.get(token.slice(i, i + MIN_NAME_TOKEN_LENGTH));
|
|
329
|
+
if (bucket && bucket.some((name) => token.startsWith(name, i)))
|
|
330
|
+
return true;
|
|
331
|
+
}
|
|
332
|
+
}
|
|
333
|
+
return false;
|
|
334
|
+
}
|
|
335
|
+
/** Record one document value: whole (lowercased), its tokens and hyphen parts. */
|
|
336
|
+
addDocumentValue(value) {
|
|
337
|
+
const whole = value.toLowerCase();
|
|
338
|
+
if (whole.length === 0)
|
|
339
|
+
return;
|
|
340
|
+
this.documentValues.add(whole);
|
|
341
|
+
for (const token of personTokens(value)) {
|
|
342
|
+
const parts = token.includes('-') ? [token, ...token.split('-').filter(Boolean)] : [token];
|
|
343
|
+
for (const part of parts) {
|
|
344
|
+
this.documentValues.add(part);
|
|
345
|
+
if (part.length >= MIN_NAME_TOKEN_LENGTH && LETTERS_ONLY_RE.test(part)) {
|
|
346
|
+
const key = part.slice(0, MIN_NAME_TOKEN_LENGTH);
|
|
347
|
+
const bucket = this.documentNameIndex.get(key);
|
|
348
|
+
if (bucket === undefined)
|
|
349
|
+
this.documentNameIndex.set(key, [part]);
|
|
350
|
+
else if (!bucket.includes(part))
|
|
351
|
+
bucket.push(part);
|
|
352
|
+
}
|
|
353
|
+
}
|
|
354
|
+
}
|
|
355
|
+
}
|
|
356
|
+
/**
|
|
357
|
+
* Declare values that occur in the session's documents (R3) so no later
|
|
358
|
+
* surrogate equals or, for people, contains one of them. anonymizeText
|
|
359
|
+
* registers its entities itself; hosts that call anonymize() value by
|
|
360
|
+
* value should register the whole entity list first, otherwise a person
|
|
361
|
+
* mapped early can still receive the name of a person mapped later.
|
|
362
|
+
* Accepts plain strings or objects with a `value` (DetectedEntity,
|
|
363
|
+
* ReplacementEntry-like `{ value }` shapes). Cumulative until clear().
|
|
364
|
+
*/
|
|
365
|
+
registerDocumentValues(input) {
|
|
366
|
+
for (const item of input) {
|
|
367
|
+
const value = typeof item === 'string' ? item : item.value;
|
|
368
|
+
if (typeof value === 'string')
|
|
369
|
+
this.addDocumentValue(value);
|
|
370
|
+
}
|
|
371
|
+
}
|
|
372
|
+
/**
|
|
373
|
+
* Whether the most recent anonymize() call met an ambiguous PERSON value
|
|
374
|
+
* (R5): a bare name that fits two or more mapped people equally well.
|
|
375
|
+
* Such a value is NOT unified; it gets its own number or surrogate. For
|
|
376
|
+
* anonymizeText, which maps many values, use getAmbiguousValues().
|
|
377
|
+
*/
|
|
378
|
+
lastAmbiguous() {
|
|
379
|
+
return this.lastAmbiguousFlag;
|
|
380
|
+
}
|
|
381
|
+
/**
|
|
382
|
+
* Every PERSON value this session refused to unify because it matched
|
|
383
|
+
* two or more people equally well (R5), in mapping order. Cumulative
|
|
384
|
+
* until clear(); hosts can flag them for the user to resolve.
|
|
385
|
+
*/
|
|
386
|
+
getAmbiguousValues() {
|
|
387
|
+
return [...this.ambiguousValues];
|
|
388
|
+
}
|
|
389
|
+
/**
|
|
390
|
+
* One reference name per person group (R5): the fullest mapped original
|
|
391
|
+
* of each group (most tokens; the first mapped on ties). Variants never
|
|
392
|
+
* act as match targets on their own, so "Anna" cannot pull "Anna
|
|
393
|
+
* Kowalska" into Anna Nowak's group, while a group whose fuller form
|
|
394
|
+
* arrived later ("John" then "John Smith") still matches "Smith".
|
|
395
|
+
*/
|
|
396
|
+
personReferences() {
|
|
397
|
+
const byGroup = new Map();
|
|
398
|
+
for (const [original, token] of this.forwardMap) {
|
|
399
|
+
if (this.entityTypeMap.get(original) !== 'PERSON')
|
|
400
|
+
continue;
|
|
401
|
+
const base = this.variantOf.get(token)?.base ?? token;
|
|
402
|
+
const size = personTokens(original).length;
|
|
403
|
+
const current = byGroup.get(base);
|
|
404
|
+
if (current === undefined || size > current.size)
|
|
405
|
+
byGroup.set(base, { original, size });
|
|
406
|
+
}
|
|
407
|
+
return [...byGroup.values()].map((group) => group.original);
|
|
408
|
+
}
|
|
409
|
+
/**
|
|
410
|
+
* The mapped person that a new PERSON value is a variant of (T057):
|
|
411
|
+
* only group references are considered (see personReferences); most
|
|
412
|
+
* shared tokens wins. A tie between two or more people is AMBIGUOUS
|
|
413
|
+
* (R5): no guess is made, the value gets a fresh identity and the
|
|
414
|
+
* session records it (lastAmbiguous / getAmbiguousValues).
|
|
415
|
+
* Deterministic; match is null when nothing (or too much) matches.
|
|
156
416
|
*/
|
|
157
417
|
findPersonVariant(value) {
|
|
158
418
|
const tokens = personTokens(value);
|
|
159
419
|
if (tokens.length === 0)
|
|
160
|
-
return null;
|
|
420
|
+
return { match: null, ambiguous: false };
|
|
161
421
|
let best = null;
|
|
162
422
|
let bestShared = 0;
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
if (!isPersonVariant(value, original))
|
|
423
|
+
let tied = false;
|
|
424
|
+
for (const reference of this.personReferences()) {
|
|
425
|
+
if (!isPersonVariant(value, reference))
|
|
167
426
|
continue;
|
|
168
|
-
const shared = personTokens(
|
|
427
|
+
const shared = personTokens(reference).filter((t) => hasToken(tokens, t)).length;
|
|
169
428
|
if (shared > bestShared) {
|
|
170
429
|
bestShared = shared;
|
|
171
|
-
best =
|
|
430
|
+
best = reference;
|
|
431
|
+
tied = false;
|
|
432
|
+
}
|
|
433
|
+
else if (shared === bestShared) {
|
|
434
|
+
tied = true;
|
|
172
435
|
}
|
|
173
436
|
}
|
|
174
|
-
|
|
437
|
+
if (tied)
|
|
438
|
+
return { match: null, ambiguous: true };
|
|
439
|
+
return { match: best, ambiguous: false };
|
|
175
440
|
}
|
|
176
441
|
anonymize(original, entityType) {
|
|
442
|
+
this.lastAmbiguousFlag = false;
|
|
177
443
|
if (this.forwardMap.has(original)) {
|
|
178
444
|
return this.forwardMap.get(original);
|
|
179
445
|
}
|
|
446
|
+
// R3: the value itself is by definition a document value.
|
|
447
|
+
this.addDocumentValue(original);
|
|
180
448
|
// T057 person-variant unification: "John Smith", "John" and "Smith"
|
|
181
449
|
// in one session are the same person and share one identity. T171
|
|
182
450
|
// makes restore exact: every distinct variant gets its OWN token that
|
|
@@ -184,24 +452,27 @@ export class AnonymizationSession {
|
|
|
184
452
|
// surrogate mode, the matching part of the group's surrogate ("Nowak"
|
|
185
453
|
// for "Smith" when "John Smith" became "Adam Nowak"), so the reverse map
|
|
186
454
|
// stays one-to-one and restore writes back exactly what was there.
|
|
187
|
-
|
|
188
|
-
|
|
455
|
+
// Blanked mode has no identities to share (every value is '________'),
|
|
456
|
+
// so it skips the lookup and never reports ambiguity.
|
|
457
|
+
if (entityType === 'PERSON' && this.mode !== 'blanked') {
|
|
458
|
+
const { match, ambiguous } = this.findPersonVariant(original);
|
|
459
|
+
if (ambiguous) {
|
|
460
|
+
this.lastAmbiguousFlag = true;
|
|
461
|
+
this.ambiguousValues.add(original);
|
|
462
|
+
}
|
|
189
463
|
if (match) {
|
|
190
|
-
const token = this.mode === '
|
|
191
|
-
?
|
|
192
|
-
: this.
|
|
193
|
-
? this.variantSurrogate(original, match)
|
|
194
|
-
: this.variantPlaceholder(original, match);
|
|
464
|
+
const token = this.mode === 'surrogate'
|
|
465
|
+
? this.variantSurrogate(original, match)
|
|
466
|
+
: this.variantPlaceholder(original, match);
|
|
195
467
|
this.forwardMap.set(original, token);
|
|
196
468
|
this.entityTypeMap.set(original, entityType);
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
this.reverseMap.set(token, original);
|
|
203
|
-
}
|
|
469
|
+
// Token-mapping can legitimately fall back to reusing the group's
|
|
470
|
+
// surrogate; then the longest original stays canonical (T057).
|
|
471
|
+
const canonical = this.reverseMap.get(token);
|
|
472
|
+
if (canonical === undefined || original.length > canonical.length) {
|
|
473
|
+
this.reverseMap.set(token, original);
|
|
204
474
|
}
|
|
475
|
+
this.invalidateRestore();
|
|
205
476
|
return token;
|
|
206
477
|
}
|
|
207
478
|
}
|
|
@@ -215,10 +486,31 @@ export class AnonymizationSession {
|
|
|
215
486
|
// reversible, so keep them out of the restore map.
|
|
216
487
|
if (this.mode !== 'blanked') {
|
|
217
488
|
this.reverseMap.set(placeholder, original);
|
|
489
|
+
this.invalidateRestore();
|
|
218
490
|
}
|
|
219
491
|
this.entityTypeMap.set(original, entityType);
|
|
220
492
|
return placeholder;
|
|
221
493
|
}
|
|
494
|
+
/** Drop the cached restore regex; called on every reverse-map change. */
|
|
495
|
+
invalidateRestore() {
|
|
496
|
+
this.restoreRe = null;
|
|
497
|
+
}
|
|
498
|
+
/**
|
|
499
|
+
* The exact-restore regex (R4): ONE alternation of every reverse-map
|
|
500
|
+
* key, longest first so a longer key always wins at a given position
|
|
501
|
+
* ("[PERSON_11]" before "[PERSON_1]", "Adam Nowak" before "Adam"), each
|
|
502
|
+
* key escaped and guarded per restoreKeyPattern. Compiled once per map
|
|
503
|
+
* state and reused across replies. Null when nothing is restorable.
|
|
504
|
+
*/
|
|
505
|
+
restoreRegex() {
|
|
506
|
+
if (this.restoreRe)
|
|
507
|
+
return this.restoreRe;
|
|
508
|
+
if (this.reverseMap.size === 0)
|
|
509
|
+
return null;
|
|
510
|
+
const keys = [...this.reverseMap.keys()].sort((a, b) => b.length - a.length || (a < b ? -1 : a > b ? 1 : 0));
|
|
511
|
+
this.restoreRe = new RegExp(keys.map(restoreKeyPattern).join('|'), 'gu');
|
|
512
|
+
return this.restoreRe;
|
|
513
|
+
}
|
|
222
514
|
/**
|
|
223
515
|
* The group a mapped value belongs to (T171): its base token and the
|
|
224
516
|
* canonical original that token restores to.
|
|
@@ -279,12 +571,17 @@ export class AnonymizationSession {
|
|
|
279
571
|
return mapped;
|
|
280
572
|
}
|
|
281
573
|
deanonymize(text) {
|
|
574
|
+
// R4: a SINGLE pass over the input. Every key is matched against the
|
|
575
|
+
// original reply only, never against text an earlier replacement just
|
|
576
|
+
// wrote, so a restored "Anna Kowalska" is never rewritten by a key
|
|
577
|
+
// "Anna" and prose that merely contains a key ("Bob" vs "Bo") is left
|
|
578
|
+
// alone by the boundaries in restoreKeyPattern. The replacer function
|
|
579
|
+
// keeps '$' sequences in the original value inert.
|
|
580
|
+
const re = this.restoreRegex();
|
|
282
581
|
let result = text;
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
// Replacer function keeps '$' sequences in the original value inert
|
|
287
|
-
result = result.replaceAll(placeholder, () => original);
|
|
582
|
+
if (re) {
|
|
583
|
+
re.lastIndex = 0;
|
|
584
|
+
result = text.replace(re, (m) => this.reverseMap.get(m) ?? m);
|
|
288
585
|
}
|
|
289
586
|
// T098: exact-literal replacement above is the unchanged fast path;
|
|
290
587
|
// tokens an LLM mangled (case, spacing, markdown, dropped brackets)
|
|
@@ -325,7 +622,29 @@ export class AnonymizationSession {
|
|
|
325
622
|
return text;
|
|
326
623
|
return out + text.slice(last);
|
|
327
624
|
}
|
|
625
|
+
/**
|
|
626
|
+
* R9: placeholders written LITERALLY in the user's text ("[PERSON_1]" in
|
|
627
|
+
* a template or a previously anonymized document) raise the per-type
|
|
628
|
+
* counter above them, so a freshly issued placeholder never restores to
|
|
629
|
+
* a value the text never carried under that token.
|
|
630
|
+
*/
|
|
631
|
+
reserveLiteralPlaceholders(text) {
|
|
632
|
+
LITERAL_PLACEHOLDER_RE.lastIndex = 0;
|
|
633
|
+
let match;
|
|
634
|
+
while ((match = LITERAL_PLACEHOLDER_RE.exec(text)) !== null) {
|
|
635
|
+
const [, type, digits] = match;
|
|
636
|
+
if (digits.length > MAX_COUNTER_DIGITS)
|
|
637
|
+
continue;
|
|
638
|
+
const current = this.typeCounters.get(type) ?? 0;
|
|
639
|
+
this.typeCounters.set(type, Math.max(current, Number(digits)));
|
|
640
|
+
}
|
|
641
|
+
}
|
|
328
642
|
anonymizeText(text, entities) {
|
|
643
|
+
this.reserveLiteralPlaceholders(text);
|
|
644
|
+
// R3: every entity value is a document value BEFORE the first surrogate
|
|
645
|
+
// is drawn, so "Jan Kowalski" can never become "Adam Nowak" when Adam
|
|
646
|
+
// Nowak is mapped a few entities later.
|
|
647
|
+
this.registerDocumentValues(entities);
|
|
329
648
|
// PERSON pre-pass: map people before anything else so that (T171) each
|
|
330
649
|
// variant group forms around its FULLEST name - "John Smith" is the
|
|
331
650
|
// base and "Smith" / "smith" hang off it, whatever order they appear in
|
|
@@ -380,6 +699,11 @@ export class AnonymizationSession {
|
|
|
380
699
|
const oldLabel = this.forwardMap.get(original);
|
|
381
700
|
if (!oldLabel || oldLabel === newLabel)
|
|
382
701
|
return [];
|
|
702
|
+
// R6: a blank label would match between every character of a reply
|
|
703
|
+
// and a label another value already restores to would hijack it.
|
|
704
|
+
// Both are refused without touching the maps.
|
|
705
|
+
if (newLabel.trim() === '' || this.reverseMap.has(newLabel))
|
|
706
|
+
return [];
|
|
383
707
|
const pairs = [];
|
|
384
708
|
const rename = (from, to) => {
|
|
385
709
|
// Every original sharing the label (legacy many-to-one maps) moves
|
|
@@ -392,6 +716,7 @@ export class AnonymizationSession {
|
|
|
392
716
|
this.reverseMap.delete(from);
|
|
393
717
|
if (canonical !== undefined)
|
|
394
718
|
this.reverseMap.set(to, canonical);
|
|
719
|
+
this.invalidateRestore();
|
|
395
720
|
pairs.push([from, to]);
|
|
396
721
|
};
|
|
397
722
|
if (this.variantOf.has(oldLabel)) {
|
|
@@ -482,15 +807,29 @@ export class AnonymizationSession {
|
|
|
482
807
|
* forward map but excluded from the reverse map, preserving their
|
|
483
808
|
* irreversibility. Entries merge into whatever the session already
|
|
484
809
|
* holds; call clear() first for a fresh seeded session.
|
|
810
|
+
*
|
|
811
|
+
* R11: the input is validated BEFORE anything is applied (an array of
|
|
812
|
+
* at most 10 000 well-formed entries, see validateImportEntry); a
|
|
813
|
+
* violation throws and leaves the session untouched.
|
|
485
814
|
*/
|
|
486
815
|
importEntries(entries) {
|
|
487
|
-
|
|
816
|
+
if (!Array.isArray(entries)) {
|
|
817
|
+
throw new Error('Invalid session map: expected an array of entries');
|
|
818
|
+
}
|
|
819
|
+
if (entries.length > MAX_IMPORT_ENTRIES) {
|
|
820
|
+
throw new Error(`Invalid session map: too many entries (${entries.length} > ${MAX_IMPORT_ENTRIES})`);
|
|
821
|
+
}
|
|
822
|
+
const valid = entries.map((entry, i) => validateImportEntry(entry, i));
|
|
823
|
+
for (const entry of valid) {
|
|
824
|
+
// R3: imported originals came from a document of this matter.
|
|
825
|
+
this.addDocumentValue(entry.original);
|
|
488
826
|
this.forwardMap.set(entry.original, entry.replacement);
|
|
489
827
|
if (entry.replacement !== '________') {
|
|
490
828
|
const canonical = this.reverseMap.get(entry.replacement);
|
|
491
829
|
if (canonical === undefined || entry.original.length > canonical.length) {
|
|
492
830
|
this.reverseMap.set(entry.replacement, entry.original);
|
|
493
831
|
}
|
|
832
|
+
this.invalidateRestore();
|
|
494
833
|
}
|
|
495
834
|
this.entityTypeMap.set(entry.original, entry.entityType);
|
|
496
835
|
// T171 variant tokens ([PERSON_1_LAST], [PERSON_1_LAST_2]) carry no
|
|
@@ -500,11 +839,13 @@ export class AnonymizationSession {
|
|
|
500
839
|
const match = TYPED_PLACEHOLDER_RE.exec(entry.replacement);
|
|
501
840
|
if (match) {
|
|
502
841
|
const [, type, num] = match;
|
|
842
|
+
if (num.length > MAX_COUNTER_DIGITS)
|
|
843
|
+
continue;
|
|
503
844
|
const current = this.typeCounters.get(type) ?? 0;
|
|
504
845
|
this.typeCounters.set(type, Math.max(current, Number(num)));
|
|
505
846
|
}
|
|
506
847
|
}
|
|
507
|
-
for (const entry of
|
|
848
|
+
for (const entry of valid) {
|
|
508
849
|
const variant = VARIANT_TOKEN_RE.exec(entry.replacement);
|
|
509
850
|
if (!variant)
|
|
510
851
|
continue;
|
|
@@ -544,11 +885,19 @@ export class AnonymizationSession {
|
|
|
544
885
|
else {
|
|
545
886
|
throw new Error('Invalid session map JSON: expected a top-level array');
|
|
546
887
|
}
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
888
|
+
if (entries.length > MAX_IMPORT_ENTRIES) {
|
|
889
|
+
throw new Error(`Invalid session map: too many entries (${entries.length} > ${MAX_IMPORT_ENTRIES})`);
|
|
890
|
+
}
|
|
891
|
+
// Wire entries are only re-keyed here (entity_type -> entityType);
|
|
892
|
+
// importEntries validates every field. A non-object entry (e.g. null)
|
|
893
|
+
// is passed through as-is so the validator names its index.
|
|
894
|
+
session.importEntries(entries.map((entry) => typeof entry === 'object' && entry !== null && !Array.isArray(entry)
|
|
895
|
+
? {
|
|
896
|
+
original: entry.original,
|
|
897
|
+
replacement: entry.replacement,
|
|
898
|
+
entityType: entry.entity_type,
|
|
899
|
+
}
|
|
900
|
+
: entry));
|
|
552
901
|
return session;
|
|
553
902
|
}
|
|
554
903
|
clear() {
|
|
@@ -557,5 +906,10 @@ export class AnonymizationSession {
|
|
|
557
906
|
this.entityTypeMap.clear();
|
|
558
907
|
this.typeCounters.clear();
|
|
559
908
|
this.variantOf.clear();
|
|
909
|
+
this.documentValues.clear();
|
|
910
|
+
this.documentNameIndex.clear();
|
|
911
|
+
this.ambiguousValues.clear();
|
|
912
|
+
this.lastAmbiguousFlag = false;
|
|
913
|
+
this.invalidateRestore();
|
|
560
914
|
}
|
|
561
915
|
}
|