@doccloak/core 0.8.0 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +108 -0
- package/dist/index.d.ts +6 -4
- package/dist/index.js +6 -3
- package/dist/model-loader.d.ts +45 -2
- package/dist/model-loader.js +99 -5
- package/dist/providers/bardsai.d.ts +12 -0
- package/dist/providers/bardsai.js +35 -3
- package/dist/providers/gliner.d.ts +12 -0
- package/dist/providers/gliner.js +36 -4
- package/dist/regex/rules.data.js +186 -0
- package/dist/regex/validators.d.ts +7 -0
- package/dist/regex/validators.js +95 -4
- package/dist/restore-tokens.d.ts +67 -0
- package/dist/restore-tokens.js +120 -0
- package/dist/session.d.ts +89 -14
- package/dist/session.js +270 -37
- package/dist/surrogates.d.ts +82 -0
- package/dist/surrogates.js +1510 -0
- package/dist/types.d.ts +1 -1
- package/dist/types.js +4 -0
- package/package.json +7 -1
- package/rules/gb.json +17 -0
- package/rules/schema.json +2 -0
- package/rules/universal.json +169 -0
package/dist/session.d.ts
CHANGED
|
@@ -1,15 +1,65 @@
|
|
|
1
1
|
import type { EntityType, DetectedEntity, ReplacementEntry } from './types.js';
|
|
2
|
-
export
|
|
2
|
+
export * from './restore-tokens.js';
|
|
3
|
+
export type ReplacementMode = 'labeled' | 'blanked' | 'surrogate';
|
|
4
|
+
export interface SessionOptions {
|
|
5
|
+
/** Initial replacement mode; defaults to 'labeled' (typed placeholders). */
|
|
6
|
+
mode?: ReplacementMode;
|
|
7
|
+
/**
|
|
8
|
+
* Per-session surrogate salt; defaults to a fresh random one. Passing an
|
|
9
|
+
* explicit salt makes surrogate generation fully reproducible (tests,
|
|
10
|
+
* deserialization).
|
|
11
|
+
*/
|
|
12
|
+
salt?: string;
|
|
13
|
+
}
|
|
3
14
|
export declare class AnonymizationSession {
|
|
4
15
|
private forwardMap;
|
|
5
16
|
private reverseMap;
|
|
6
17
|
private entityTypeMap;
|
|
7
|
-
private
|
|
18
|
+
private typeCounters;
|
|
8
19
|
private mode;
|
|
20
|
+
/** Surrogate-mode salt (T043); constant for the session's lifetime. */
|
|
21
|
+
private salt;
|
|
22
|
+
constructor(options?: SessionOptions);
|
|
9
23
|
setMode(mode: ReplacementMode): void;
|
|
10
24
|
getMode(): ReplacementMode;
|
|
25
|
+
getSalt(): string;
|
|
26
|
+
/**
|
|
27
|
+
* Issue the next typed placeholder for the given entity type using a
|
|
28
|
+
* per-type counter: [PERSON_1], [PERSON_2], [EMAIL_1], ... Numbers
|
|
29
|
+
* already present in the reverse map (a renamed label or an entry
|
|
30
|
+
* restored via deserialize) are skipped so a fresh placeholder never
|
|
31
|
+
* collides with an existing one.
|
|
32
|
+
*/
|
|
33
|
+
private nextPlaceholder;
|
|
34
|
+
/**
|
|
35
|
+
* Surrogate-mode replacement (T043): deterministic in (salt, type,
|
|
36
|
+
* original), collision-safe against every original value already in the
|
|
37
|
+
* session and every already-issued replacement. The map API is
|
|
38
|
+
* unchanged: the surrogate simply IS the replacement string, so
|
|
39
|
+
* deanonymize restores it via the same exact-literal lookup.
|
|
40
|
+
*/
|
|
41
|
+
private nextSurrogate;
|
|
42
|
+
/**
|
|
43
|
+
* The already-mapped PERSON value that best matches a new variant
|
|
44
|
+
* (T057): most shared tokens wins; ties keep the first-mapped value.
|
|
45
|
+
* Deterministic; returns null when nothing matches.
|
|
46
|
+
*/
|
|
47
|
+
private findPersonVariant;
|
|
11
48
|
anonymize(original: string, entityType: EntityType): string;
|
|
12
49
|
deanonymize(text: string): string;
|
|
50
|
+
/**
|
|
51
|
+
* Tolerant pass (T098) over text the exact pass already handled: scan
|
|
52
|
+
* for remaining token-shaped candidates and restore each one that
|
|
53
|
+
* unambiguously identifies a single map key (see
|
|
54
|
+
* buildTolerantTokenIndex). Anything else - unknown tokens, ambiguous
|
|
55
|
+
* canonical forms, partial tokens the candidate regex cannot even see -
|
|
56
|
+
* stays exactly as written: a missed restore is visible and
|
|
57
|
+
* recoverable, a guessed one silently corrupts the document. Runs after
|
|
58
|
+
* the exact pass, so an exact key never reaches this code; originals
|
|
59
|
+
* restored by the exact pass are plain user text and canonicalize to
|
|
60
|
+
* forms no key owns.
|
|
61
|
+
*/
|
|
62
|
+
private deanonymizeMangledTokens;
|
|
13
63
|
anonymizeText(text: string, entities: DetectedEntity[]): string;
|
|
14
64
|
getEntries(): ReplacementEntry[];
|
|
15
65
|
getForward(original: string): string | undefined;
|
|
@@ -23,15 +73,15 @@ export declare class AnonymizationSession {
|
|
|
23
73
|
* [
|
|
24
74
|
* {
|
|
25
75
|
* "original": string, // the original sensitive value
|
|
26
|
-
* "replacement": string, // its placeholder, e.g. "
|
|
76
|
+
* "replacement": string, // its placeholder, e.g. "[PERSON_1]" or a renamed label
|
|
27
77
|
* "entity_type": string // EntityType value, e.g. "PERSON" (snake_case key, matching Python)
|
|
28
78
|
* },
|
|
29
79
|
* ...
|
|
30
80
|
* ]
|
|
31
81
|
*
|
|
32
82
|
* - Top level is a JSON array; one object per mapping, in insertion order.
|
|
33
|
-
* -
|
|
34
|
-
* derives
|
|
83
|
+
* - Counters are NOT stored as fields. Like Python's load_map, deserialize
|
|
84
|
+
* derives them from the entries themselves.
|
|
35
85
|
* - The mode is NOT stored as a field. Like Python's load_map (which returns
|
|
36
86
|
* `cls()` with the default mode), deserialize yields a 'labeled' session.
|
|
37
87
|
* - Blanked entries are intentionally OMITTED from the output: blanked
|
|
@@ -40,19 +90,44 @@ export declare class AnonymizationSession {
|
|
|
40
90
|
* defeat that. Only reversible entries are serialized.
|
|
41
91
|
* - Output formatting matches Python's json.dumps(entries, indent=2,
|
|
42
92
|
* ensure_ascii=False): 2-space indent, non-ASCII characters kept raw.
|
|
93
|
+
*
|
|
94
|
+
* Surrogate mode (T043) is the one exception to the top-level-array
|
|
95
|
+
* schema: it wraps the same entry objects in
|
|
96
|
+
* `{ "mode": "surrogate", "salt": "...", "entries": [...] }` so
|
|
97
|
+
* deserialize can restore the mode and the salt (new values anonymized
|
|
98
|
+
* after a restore then derive the same surrogates, and the per-session
|
|
99
|
+
* date offset stays stable). Placeholder ('labeled') and blanked
|
|
100
|
+
* sessions keep emitting the plain array byte-identically, so the
|
|
101
|
+
* Python-parity schema is untouched and absent fields mean placeholder
|
|
102
|
+
* mode.
|
|
43
103
|
*/
|
|
44
104
|
serialize(): string;
|
|
105
|
+
/**
|
|
106
|
+
* Seed the live session with previously issued mappings (T106: the
|
|
107
|
+
* matter-persistence interface). Each entry lands in the forward map
|
|
108
|
+
* (value -> replacement, so anonymizing the same value again reuses
|
|
109
|
+
* the stored replacement instead of issuing a new one), the reverse
|
|
110
|
+
* map (replacement -> value, so deanonymize restores text produced in
|
|
111
|
+
* an earlier session; when several entries share a replacement, e.g.
|
|
112
|
+
* T057 person variants, the LONGEST original stays canonical
|
|
113
|
+
* regardless of entry order) and the type map. Per-type counters are
|
|
114
|
+
* raised to the highest N found among '[TYPE_N]' replacements so new
|
|
115
|
+
* entities anonymized afterwards continue numbering without colliding
|
|
116
|
+
* (nextPlaceholder additionally skips any taken number, which also
|
|
117
|
+
* covers renamed labels shaped like '[TYPE_N]'). Legacy
|
|
118
|
+
* '<<REDACTED_N>>' replacements are kept verbatim in both maps; they
|
|
119
|
+
* never collide with newly issued bracket placeholders and therefore
|
|
120
|
+
* need no counter. Blanked entries ('________') are kept in the
|
|
121
|
+
* forward map but excluded from the reverse map, preserving their
|
|
122
|
+
* irreversibility. Entries merge into whatever the session already
|
|
123
|
+
* holds; call clear() first for a fresh seeded session.
|
|
124
|
+
*/
|
|
125
|
+
importEntries(entries: ReplacementEntry[]): void;
|
|
45
126
|
/**
|
|
46
127
|
* Rebuild a session from JSON produced by serialize() or by the Python
|
|
47
|
-
* CLI's save_map. Mirrors Python's load_map:
|
|
48
|
-
*
|
|
49
|
-
*
|
|
50
|
-
* '<<REDACTED_N>>' placeholders (a map serialized with gaps, e.g. after
|
|
51
|
-
* blanked entries were omitted, would otherwise reissue a taken number),
|
|
52
|
-
* so new entities anonymized after restore continue numbering without
|
|
53
|
-
* colliding with restored placeholders. Blanked entries ('________') that a
|
|
54
|
-
* Python-produced map may contain are kept in the forward map but excluded
|
|
55
|
-
* from the reverse map, preserving their irreversibility.
|
|
128
|
+
* CLI's save_map. Mirrors Python's load_map: the per-entry map and
|
|
129
|
+
* counter population is importEntries (see its doc for the exact
|
|
130
|
+
* semantics, including legacy and blanked entries).
|
|
56
131
|
*/
|
|
57
132
|
static deserialize(json: string): AnonymizationSession;
|
|
58
133
|
clear(): void;
|
package/dist/session.js
CHANGED
|
@@ -1,10 +1,59 @@
|
|
|
1
|
+
import { buildTolerantTokenIndex, resolveMangledToken } from "./restore-tokens.js";
|
|
2
|
+
import { generateSessionSalt, generateUniqueSurrogate } from "./surrogates.js";
|
|
3
|
+
export * from "./restore-tokens.js";
|
|
4
|
+
// T099: credentials never get realistic stand-ins - a same-shape fake key
|
|
5
|
+
// still looks like a live credential to anyone (or any scanner) reading the
|
|
6
|
+
// prompt. Surrogate mode falls back to typed placeholders for these types.
|
|
7
|
+
const PLACEHOLDER_ONLY_TYPES = new Set(['SECRET', 'API_KEY']);
|
|
8
|
+
/**
|
|
9
|
+
* Person-variant tokenization (T057): lowercase word tokens with edge
|
|
10
|
+
* punctuation stripped. EVERY token participates in the subset test -
|
|
11
|
+
* dropping short ones would collapse "Person 1" into "Person 11".
|
|
12
|
+
*/
|
|
13
|
+
function personTokens(value) {
|
|
14
|
+
return value
|
|
15
|
+
.toLowerCase()
|
|
16
|
+
.split(/\s+/)
|
|
17
|
+
.map((t) => t.replace(/^[.,;:!?()"'\u201E\u201C\u201D]+|[.,;:!?()"'\u201E\u201C\u201D]+$/g, ''))
|
|
18
|
+
.filter((t) => t.length > 0);
|
|
19
|
+
}
|
|
20
|
+
/**
|
|
21
|
+
* Whether two PERSON values plausibly name the same person (T057): the
|
|
22
|
+
* token set of one must be a subset of the other's, and the shared part
|
|
23
|
+
* must contain at least one substantive token (>= 3 chars) so initials,
|
|
24
|
+
* digits or stray marks alone never unify. "John" and "Smith" both match
|
|
25
|
+
* "John Smith"; "Mr. Smith" does NOT (the 'mr' token has no counterpart,
|
|
26
|
+
* and honorific guessing would be overreach); "Person 1" never matches
|
|
27
|
+
* "Person 11" (the '1' token has no counterpart in the other value).
|
|
28
|
+
*/
|
|
29
|
+
function isPersonVariant(a, b) {
|
|
30
|
+
const ta = personTokens(a);
|
|
31
|
+
const tb = personTokens(b);
|
|
32
|
+
if (ta.length === 0 || tb.length === 0)
|
|
33
|
+
return false;
|
|
34
|
+
const [small, big] = ta.length <= tb.length ? [ta, tb] : [tb, ta];
|
|
35
|
+
return small.every((t) => big.includes(t)) && small.some((t) => t.length >= 3);
|
|
36
|
+
}
|
|
37
|
+
/**
|
|
38
|
+
* Typed placeholder token, e.g. "[PERSON_1]" or "[CREDIT_CARD_2]".
|
|
39
|
+
* EntityType ids are stable uppercase ASCII ([A-Z_]), so generated tokens
|
|
40
|
+
* always match the candidate pattern `\[[A-Z_]+_\d+\]`.
|
|
41
|
+
*/
|
|
42
|
+
const TYPED_PLACEHOLDER_RE = /^\[([A-Z_]+)_(\d+)\]$/;
|
|
43
|
+
/**
|
|
44
|
+
* Candidate scan for the tolerant deanonymize pass: bracketed tokens,
|
|
45
|
+
* legacy angle tokens, or strict bare typed tokens. Broad candidates are
|
|
46
|
+
* safe - replacement still requires an unambiguous index hit.
|
|
47
|
+
*/
|
|
48
|
+
const MANGLED_CANDIDATE_RE = /\[[^[\]\n]+\]|<<[^<>\n]+>>|(?<![\p{L}\p{N}_[<])[A-Z][A-Z0-9]*(?:_[A-Z0-9]+)*_\d+(?![\p{L}\p{N}_])/gu;
|
|
1
49
|
export class AnonymizationSession {
|
|
2
|
-
constructor() {
|
|
50
|
+
constructor(options) {
|
|
3
51
|
this.forwardMap = new Map();
|
|
4
52
|
this.reverseMap = new Map();
|
|
5
53
|
this.entityTypeMap = new Map();
|
|
6
|
-
this.
|
|
7
|
-
this.mode = 'labeled';
|
|
54
|
+
this.typeCounters = new Map();
|
|
55
|
+
this.mode = options?.mode ?? 'labeled';
|
|
56
|
+
this.salt = options?.salt ?? generateSessionSalt();
|
|
8
57
|
}
|
|
9
58
|
setMode(mode) {
|
|
10
59
|
this.mode = mode;
|
|
@@ -12,14 +61,89 @@ export class AnonymizationSession {
|
|
|
12
61
|
getMode() {
|
|
13
62
|
return this.mode;
|
|
14
63
|
}
|
|
64
|
+
getSalt() {
|
|
65
|
+
return this.salt;
|
|
66
|
+
}
|
|
67
|
+
/**
|
|
68
|
+
* Issue the next typed placeholder for the given entity type using a
|
|
69
|
+
* per-type counter: [PERSON_1], [PERSON_2], [EMAIL_1], ... Numbers
|
|
70
|
+
* already present in the reverse map (a renamed label or an entry
|
|
71
|
+
* restored via deserialize) are skipped so a fresh placeholder never
|
|
72
|
+
* collides with an existing one.
|
|
73
|
+
*/
|
|
74
|
+
nextPlaceholder(entityType) {
|
|
75
|
+
let n = this.typeCounters.get(entityType) ?? 0;
|
|
76
|
+
let placeholder;
|
|
77
|
+
do {
|
|
78
|
+
n++;
|
|
79
|
+
placeholder = `[${entityType}_${n}]`;
|
|
80
|
+
} while (this.reverseMap.has(placeholder));
|
|
81
|
+
this.typeCounters.set(entityType, n);
|
|
82
|
+
return placeholder;
|
|
83
|
+
}
|
|
84
|
+
/**
|
|
85
|
+
* Surrogate-mode replacement (T043): deterministic in (salt, type,
|
|
86
|
+
* original), collision-safe against every original value already in the
|
|
87
|
+
* session and every already-issued replacement. The map API is
|
|
88
|
+
* unchanged: the surrogate simply IS the replacement string, so
|
|
89
|
+
* deanonymize restores it via the same exact-literal lookup.
|
|
90
|
+
*/
|
|
91
|
+
nextSurrogate(original, entityType) {
|
|
92
|
+
return generateUniqueSurrogate(original, entityType, { salt: this.salt, entries: this.getEntries() }, (candidate) => this.forwardMap.has(candidate) || this.reverseMap.has(candidate));
|
|
93
|
+
}
|
|
94
|
+
/**
|
|
95
|
+
* The already-mapped PERSON value that best matches a new variant
|
|
96
|
+
* (T057): most shared tokens wins; ties keep the first-mapped value.
|
|
97
|
+
* Deterministic; returns null when nothing matches.
|
|
98
|
+
*/
|
|
99
|
+
findPersonVariant(value) {
|
|
100
|
+
const tokens = personTokens(value);
|
|
101
|
+
if (tokens.length === 0)
|
|
102
|
+
return null;
|
|
103
|
+
let best = null;
|
|
104
|
+
let bestShared = 0;
|
|
105
|
+
for (const [original] of this.forwardMap) {
|
|
106
|
+
if (this.entityTypeMap.get(original) !== 'PERSON')
|
|
107
|
+
continue;
|
|
108
|
+
if (!isPersonVariant(value, original))
|
|
109
|
+
continue;
|
|
110
|
+
const shared = personTokens(original).filter((t) => tokens.includes(t)).length;
|
|
111
|
+
if (shared > bestShared) {
|
|
112
|
+
bestShared = shared;
|
|
113
|
+
best = original;
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
return best;
|
|
117
|
+
}
|
|
15
118
|
anonymize(original, entityType) {
|
|
16
119
|
if (this.forwardMap.has(original)) {
|
|
17
120
|
return this.forwardMap.get(original);
|
|
18
121
|
}
|
|
19
|
-
|
|
122
|
+
// T057 person-variant unification: "John Smith", "John" and "Smith"
|
|
123
|
+
// in one session are the same person and share one replacement (in
|
|
124
|
+
// every mode - the variant simply reuses the matched value's
|
|
125
|
+
// placeholder/surrogate). The reverse map keeps the LONGEST variant,
|
|
126
|
+
// so restore always yields the fullest known form.
|
|
127
|
+
if (entityType === 'PERSON') {
|
|
128
|
+
const match = this.findPersonVariant(original);
|
|
129
|
+
if (match) {
|
|
130
|
+
const placeholder = this.forwardMap.get(match);
|
|
131
|
+
this.forwardMap.set(original, placeholder);
|
|
132
|
+
this.entityTypeMap.set(original, entityType);
|
|
133
|
+
if (this.mode !== 'blanked') {
|
|
134
|
+
const canonical = this.reverseMap.get(placeholder);
|
|
135
|
+
if (canonical === undefined || original.length > canonical.length) {
|
|
136
|
+
this.reverseMap.set(placeholder, original);
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
return placeholder;
|
|
140
|
+
}
|
|
141
|
+
}
|
|
20
142
|
const placeholder = this.mode === 'blanked'
|
|
21
143
|
? '________'
|
|
22
|
-
:
|
|
144
|
+
: this.mode === 'surrogate' && !PLACEHOLDER_ONLY_TYPES.has(entityType)
|
|
145
|
+
? this.nextSurrogate(original, entityType)
|
|
146
|
+
: this.nextPlaceholder(entityType);
|
|
23
147
|
this.forwardMap.set(original, placeholder);
|
|
24
148
|
// Blanked placeholders all collide on the same string; they are not
|
|
25
149
|
// reversible, so keep them out of the restore map.
|
|
@@ -37,9 +161,58 @@ export class AnonymizationSession {
|
|
|
37
161
|
// Replacer function keeps '$' sequences in the original value inert
|
|
38
162
|
result = result.replaceAll(placeholder, () => original);
|
|
39
163
|
}
|
|
40
|
-
|
|
164
|
+
// T098: exact-literal replacement above is the unchanged fast path;
|
|
165
|
+
// tokens an LLM mangled (case, spacing, markdown, dropped brackets)
|
|
166
|
+
// survive it untouched and get one tolerant pass.
|
|
167
|
+
return this.deanonymizeMangledTokens(result);
|
|
168
|
+
}
|
|
169
|
+
/**
|
|
170
|
+
* Tolerant pass (T098) over text the exact pass already handled: scan
|
|
171
|
+
* for remaining token-shaped candidates and restore each one that
|
|
172
|
+
* unambiguously identifies a single map key (see
|
|
173
|
+
* buildTolerantTokenIndex). Anything else - unknown tokens, ambiguous
|
|
174
|
+
* canonical forms, partial tokens the candidate regex cannot even see -
|
|
175
|
+
* stays exactly as written: a missed restore is visible and
|
|
176
|
+
* recoverable, a guessed one silently corrupts the document. Runs after
|
|
177
|
+
* the exact pass, so an exact key never reaches this code; originals
|
|
178
|
+
* restored by the exact pass are plain user text and canonicalize to
|
|
179
|
+
* forms no key owns.
|
|
180
|
+
*/
|
|
181
|
+
deanonymizeMangledTokens(text) {
|
|
182
|
+
const index = buildTolerantTokenIndex(this.reverseMap.keys());
|
|
183
|
+
if (index.byCanonical.size === 0)
|
|
184
|
+
return text;
|
|
185
|
+
MANGLED_CANDIDATE_RE.lastIndex = 0;
|
|
186
|
+
let out = '';
|
|
187
|
+
let last = 0;
|
|
188
|
+
let match;
|
|
189
|
+
while ((match = MANGLED_CANDIDATE_RE.exec(text)) !== null) {
|
|
190
|
+
const key = resolveMangledToken(match[0], index);
|
|
191
|
+
if (key === null)
|
|
192
|
+
continue;
|
|
193
|
+
const original = this.reverseMap.get(key);
|
|
194
|
+
if (original === undefined)
|
|
195
|
+
continue;
|
|
196
|
+
out += text.slice(last, match.index) + original;
|
|
197
|
+
last = match.index + match[0].length;
|
|
198
|
+
}
|
|
199
|
+
if (last === 0)
|
|
200
|
+
return text;
|
|
201
|
+
return out + text.slice(last);
|
|
41
202
|
}
|
|
42
203
|
anonymizeText(text, entities) {
|
|
204
|
+
// Surrogate mode (T043): map PERSON entities first, in reading order,
|
|
205
|
+
// so an email appearing before/after its owner in the text can derive
|
|
206
|
+
// its local part from the person's surrogate (jan.kowalski maps to
|
|
207
|
+
// adam.nowak style). anonymize() is idempotent per value, so the
|
|
208
|
+
// replacement pass below reuses these mappings. Labeled mode keeps its
|
|
209
|
+
// historical end-to-start issuing order (placeholder numbering).
|
|
210
|
+
if (this.mode === 'surrogate') {
|
|
211
|
+
for (const entity of [...entities].sort((a, b) => a.start - b.start)) {
|
|
212
|
+
if (entity.type === 'PERSON')
|
|
213
|
+
this.anonymize(entity.value, entity.type);
|
|
214
|
+
}
|
|
215
|
+
}
|
|
43
216
|
// Sort entities by position (end to start) to preserve indices during replacement
|
|
44
217
|
const sorted = [...entities].sort((a, b) => b.start - a.start);
|
|
45
218
|
let result = text;
|
|
@@ -69,9 +242,15 @@ export class AnonymizationSession {
|
|
|
69
242
|
const oldLabel = this.forwardMap.get(original);
|
|
70
243
|
if (!oldLabel)
|
|
71
244
|
return;
|
|
72
|
-
|
|
245
|
+
// T057: variants share one label; renaming any of them renames the
|
|
246
|
+
// whole group, and the canonical restore value carries over.
|
|
247
|
+
const canonical = this.reverseMap.get(oldLabel) ?? original;
|
|
248
|
+
for (const [orig, label] of this.forwardMap) {
|
|
249
|
+
if (label === oldLabel)
|
|
250
|
+
this.forwardMap.set(orig, newLabel);
|
|
251
|
+
}
|
|
73
252
|
this.reverseMap.delete(oldLabel);
|
|
74
|
-
this.reverseMap.set(newLabel,
|
|
253
|
+
this.reverseMap.set(newLabel, canonical);
|
|
75
254
|
}
|
|
76
255
|
/**
|
|
77
256
|
* Serialize the session map to JSON.
|
|
@@ -82,15 +261,15 @@ export class AnonymizationSession {
|
|
|
82
261
|
* [
|
|
83
262
|
* {
|
|
84
263
|
* "original": string, // the original sensitive value
|
|
85
|
-
* "replacement": string, // its placeholder, e.g. "
|
|
264
|
+
* "replacement": string, // its placeholder, e.g. "[PERSON_1]" or a renamed label
|
|
86
265
|
* "entity_type": string // EntityType value, e.g. "PERSON" (snake_case key, matching Python)
|
|
87
266
|
* },
|
|
88
267
|
* ...
|
|
89
268
|
* ]
|
|
90
269
|
*
|
|
91
270
|
* - Top level is a JSON array; one object per mapping, in insertion order.
|
|
92
|
-
* -
|
|
93
|
-
* derives
|
|
271
|
+
* - Counters are NOT stored as fields. Like Python's load_map, deserialize
|
|
272
|
+
* derives them from the entries themselves.
|
|
94
273
|
* - The mode is NOT stored as a field. Like Python's load_map (which returns
|
|
95
274
|
* `cls()` with the default mode), deserialize yields a 'labeled' session.
|
|
96
275
|
* - Blanked entries are intentionally OMITTED from the output: blanked
|
|
@@ -99,6 +278,16 @@ export class AnonymizationSession {
|
|
|
99
278
|
* defeat that. Only reversible entries are serialized.
|
|
100
279
|
* - Output formatting matches Python's json.dumps(entries, indent=2,
|
|
101
280
|
* ensure_ascii=False): 2-space indent, non-ASCII characters kept raw.
|
|
281
|
+
*
|
|
282
|
+
* Surrogate mode (T043) is the one exception to the top-level-array
|
|
283
|
+
* schema: it wraps the same entry objects in
|
|
284
|
+
* `{ "mode": "surrogate", "salt": "...", "entries": [...] }` so
|
|
285
|
+
* deserialize can restore the mode and the salt (new values anonymized
|
|
286
|
+
* after a restore then derive the same surrogates, and the per-session
|
|
287
|
+
* date offset stays stable). Placeholder ('labeled') and blanked
|
|
288
|
+
* sessions keep emitting the plain array byte-identically, so the
|
|
289
|
+
* Python-parity schema is untouched and absent fields mean placeholder
|
|
290
|
+
* mode.
|
|
102
291
|
*/
|
|
103
292
|
serialize() {
|
|
104
293
|
const entries = this.getEntries()
|
|
@@ -108,46 +297,90 @@ export class AnonymizationSession {
|
|
|
108
297
|
replacement,
|
|
109
298
|
entity_type: entityType,
|
|
110
299
|
}));
|
|
300
|
+
if (this.mode === 'surrogate') {
|
|
301
|
+
return JSON.stringify({ mode: 'surrogate', salt: this.salt, entries }, null, 2);
|
|
302
|
+
}
|
|
111
303
|
return JSON.stringify(entries, null, 2);
|
|
112
304
|
}
|
|
113
305
|
/**
|
|
114
|
-
*
|
|
115
|
-
*
|
|
116
|
-
*
|
|
117
|
-
*
|
|
118
|
-
*
|
|
119
|
-
*
|
|
120
|
-
*
|
|
121
|
-
*
|
|
122
|
-
*
|
|
123
|
-
*
|
|
306
|
+
* Seed the live session with previously issued mappings (T106: the
|
|
307
|
+
* matter-persistence interface). Each entry lands in the forward map
|
|
308
|
+
* (value -> replacement, so anonymizing the same value again reuses
|
|
309
|
+
* the stored replacement instead of issuing a new one), the reverse
|
|
310
|
+
* map (replacement -> value, so deanonymize restores text produced in
|
|
311
|
+
* an earlier session; when several entries share a replacement, e.g.
|
|
312
|
+
* T057 person variants, the LONGEST original stays canonical
|
|
313
|
+
* regardless of entry order) and the type map. Per-type counters are
|
|
314
|
+
* raised to the highest N found among '[TYPE_N]' replacements so new
|
|
315
|
+
* entities anonymized afterwards continue numbering without colliding
|
|
316
|
+
* (nextPlaceholder additionally skips any taken number, which also
|
|
317
|
+
* covers renamed labels shaped like '[TYPE_N]'). Legacy
|
|
318
|
+
* '<<REDACTED_N>>' replacements are kept verbatim in both maps; they
|
|
319
|
+
* never collide with newly issued bracket placeholders and therefore
|
|
320
|
+
* need no counter. Blanked entries ('________') are kept in the
|
|
321
|
+
* forward map but excluded from the reverse map, preserving their
|
|
322
|
+
* irreversibility. Entries merge into whatever the session already
|
|
323
|
+
* holds; call clear() first for a fresh seeded session.
|
|
124
324
|
*/
|
|
125
|
-
|
|
126
|
-
const
|
|
127
|
-
|
|
128
|
-
throw new Error('Invalid session map JSON: expected a top-level array');
|
|
129
|
-
}
|
|
130
|
-
const session = new AnonymizationSession();
|
|
131
|
-
let maxPlaceholderNumber = 0;
|
|
132
|
-
for (const entry of data) {
|
|
133
|
-
session.forwardMap.set(entry.original, entry.replacement);
|
|
325
|
+
importEntries(entries) {
|
|
326
|
+
for (const entry of entries) {
|
|
327
|
+
this.forwardMap.set(entry.original, entry.replacement);
|
|
134
328
|
if (entry.replacement !== '________') {
|
|
135
|
-
|
|
329
|
+
const canonical = this.reverseMap.get(entry.replacement);
|
|
330
|
+
if (canonical === undefined || entry.original.length > canonical.length) {
|
|
331
|
+
this.reverseMap.set(entry.replacement, entry.original);
|
|
332
|
+
}
|
|
136
333
|
}
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
const match = /^<<REDACTED_(\d+)>>$/.exec(entry.replacement);
|
|
334
|
+
this.entityTypeMap.set(entry.original, entry.entityType);
|
|
335
|
+
const match = TYPED_PLACEHOLDER_RE.exec(entry.replacement);
|
|
140
336
|
if (match) {
|
|
141
|
-
|
|
337
|
+
const [, type, num] = match;
|
|
338
|
+
const current = this.typeCounters.get(type) ?? 0;
|
|
339
|
+
this.typeCounters.set(type, Math.max(current, Number(num)));
|
|
142
340
|
}
|
|
143
341
|
}
|
|
144
|
-
|
|
342
|
+
}
|
|
343
|
+
/**
|
|
344
|
+
* Rebuild a session from JSON produced by serialize() or by the Python
|
|
345
|
+
* CLI's save_map. Mirrors Python's load_map: the per-entry map and
|
|
346
|
+
* counter population is importEntries (see its doc for the exact
|
|
347
|
+
* semantics, including legacy and blanked entries).
|
|
348
|
+
*/
|
|
349
|
+
static deserialize(json) {
|
|
350
|
+
const data = JSON.parse(json);
|
|
351
|
+
let entries;
|
|
352
|
+
let session;
|
|
353
|
+
if (Array.isArray(data)) {
|
|
354
|
+
entries = data;
|
|
355
|
+
session = new AnonymizationSession();
|
|
356
|
+
}
|
|
357
|
+
else if (typeof data === 'object' &&
|
|
358
|
+
data !== null &&
|
|
359
|
+
data.mode === 'surrogate' &&
|
|
360
|
+
Array.isArray(data.entries)) {
|
|
361
|
+
// Surrogate-mode wrapper (T043): restore mode + salt so surrogate
|
|
362
|
+
// derivation stays consistent after the round trip.
|
|
363
|
+
const wrapper = data;
|
|
364
|
+
entries = wrapper.entries;
|
|
365
|
+
session = new AnonymizationSession({
|
|
366
|
+
mode: 'surrogate',
|
|
367
|
+
salt: typeof wrapper.salt === 'string' ? wrapper.salt : undefined,
|
|
368
|
+
});
|
|
369
|
+
}
|
|
370
|
+
else {
|
|
371
|
+
throw new Error('Invalid session map JSON: expected a top-level array');
|
|
372
|
+
}
|
|
373
|
+
session.importEntries(entries.map((entry) => ({
|
|
374
|
+
original: entry.original,
|
|
375
|
+
replacement: entry.replacement,
|
|
376
|
+
entityType: entry.entity_type,
|
|
377
|
+
})));
|
|
145
378
|
return session;
|
|
146
379
|
}
|
|
147
380
|
clear() {
|
|
148
381
|
this.forwardMap.clear();
|
|
149
382
|
this.reverseMap.clear();
|
|
150
383
|
this.entityTypeMap.clear();
|
|
151
|
-
this.
|
|
384
|
+
this.typeCounters.clear();
|
|
152
385
|
}
|
|
153
386
|
}
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @doccloak/core - realistic, shape-preserving surrogate generator (T043).
|
|
3
|
+
*
|
|
4
|
+
* Generates fake-but-plausible replacements for detected entities instead
|
|
5
|
+
* of bracket placeholders: locale-aware names, dates shifted while keeping
|
|
6
|
+
* their format, IBANs/phones/IDs that keep their shape (checksum-valid
|
|
7
|
+
* where the checksum is cheap to compute).
|
|
8
|
+
*
|
|
9
|
+
* Determinism contract: every surrogate is a pure function of
|
|
10
|
+
* (session salt, entity type, original value, attempt). No Math.random or
|
|
11
|
+
* Date.now is ever consulted at module or generation scope, so the same
|
|
12
|
+
* original always yields the same surrogate within a session and tests are
|
|
13
|
+
* reproducible. The salt itself is created once per session (see
|
|
14
|
+
* generateSessionSalt, called from the AnonymizationSession constructor)
|
|
15
|
+
* and persisted with the map so a deserialized session keeps producing
|
|
16
|
+
* consistent surrogates.
|
|
17
|
+
*
|
|
18
|
+
* Fake-identifier policy: we never deliberately generate a real person's
|
|
19
|
+
* identifier. Generated PESELs encode a 19th-century birth date (a range
|
|
20
|
+
* the registry never issued numbers for), generated SSN shapes use the
|
|
21
|
+
* 900-999 area (never allocated by the SSA) with a group outside every
|
|
22
|
+
* valid ITIN range (T101: a 9xx area alone could still form a real ITIN),
|
|
23
|
+
* generated EIN shapes use campus prefixes the IRS has never assigned,
|
|
24
|
+
* generated ABA routing shapes are checksum-valid but carry the 99 leading
|
|
25
|
+
* pair (the ABA assigns only 00-12, 21-32, 61-72, and 80), generated NHS
|
|
26
|
+
* numbers sit in the mod-11-valid 999 test range (never issued to
|
|
27
|
+
* patients), and generated NINOs use the QQ prefix HMRC reserves for
|
|
28
|
+
* documentation examples. Random collisions with real identifiers of other
|
|
29
|
+
* kinds are statistically possible but not targetable: nothing about the
|
|
30
|
+
* original value survives into the surrogate beyond its shape.
|
|
31
|
+
*/
|
|
32
|
+
import type { EntityType } from './types.js';
|
|
33
|
+
/**
|
|
34
|
+
* One salt per session, created at session construction time (never at
|
|
35
|
+
* module scope). WebCrypto when available; a constructor-time Math.random
|
|
36
|
+
* fallback otherwise (the salt only needs to be unique-ish per session,
|
|
37
|
+
* not cryptographically strong - surrogates are not secrets).
|
|
38
|
+
*/
|
|
39
|
+
export declare function generateSessionSalt(): string;
|
|
40
|
+
/** Reserved/example domains only (RFC 2606 / RFC 6761): never routable. */
|
|
41
|
+
export declare const FAKE_EMAIL_DOMAINS: readonly ["example.com", "example.org", "example.net", "mail.example", "post.example", "inbox.test", "mail.test", "poczta.test"];
|
|
42
|
+
export interface SurrogateContext {
|
|
43
|
+
/** Per-session salt; the sole source of randomness. */
|
|
44
|
+
salt: string;
|
|
45
|
+
/**
|
|
46
|
+
* Already-issued session mappings, used to derive an email local part
|
|
47
|
+
* from the surrogate of the matching PERSON (jan.kowalski maps to
|
|
48
|
+
* adam.nowak style). Optional; without it emails get an unrelated
|
|
49
|
+
* deterministic fake identity.
|
|
50
|
+
*/
|
|
51
|
+
entries?: ReadonlyArray<{
|
|
52
|
+
original: string;
|
|
53
|
+
replacement: string;
|
|
54
|
+
entityType: EntityType;
|
|
55
|
+
}>;
|
|
56
|
+
}
|
|
57
|
+
/**
|
|
58
|
+
* Per-session day offset in [-30, -1] or [1, 30]: derived from the salt
|
|
59
|
+
* only, so every date in a session shifts by the same amount and relative
|
|
60
|
+
* spacing between dates is preserved. Never 0 (a zero shift would leak the
|
|
61
|
+
* original date unchanged).
|
|
62
|
+
*/
|
|
63
|
+
export declare function sessionDayOffset(salt: string): number;
|
|
64
|
+
/**
|
|
65
|
+
* Test/diagnostic probe (T073): which locale pack a PERSON value would
|
|
66
|
+
* use. Not part of the anonymization data path.
|
|
67
|
+
*/
|
|
68
|
+
export declare function detectSurrogateLocale(value: string): string;
|
|
69
|
+
/**
|
|
70
|
+
* Generate a surrogate for one entity. Pure and deterministic in
|
|
71
|
+
* (ctx.salt, type, original, attempt); bump `attempt` to re-derive after
|
|
72
|
+
* a collision.
|
|
73
|
+
*/
|
|
74
|
+
export declare function generateSurrogate(original: string, type: EntityType, ctx: SurrogateContext, attempt?: number): string;
|
|
75
|
+
/**
|
|
76
|
+
* Collision-safe wrapper: re-derives with a bumped attempt while the
|
|
77
|
+
* candidate equals the original or `isTaken` reports a clash (another
|
|
78
|
+
* original value or an already-issued replacement); after bounded retries
|
|
79
|
+
* it appends digits until unique. Termination is guaranteed because the
|
|
80
|
+
* taken set is finite.
|
|
81
|
+
*/
|
|
82
|
+
export declare function generateUniqueSurrogate(original: string, type: EntityType, ctx: SurrogateContext, isTaken: (candidate: string) => boolean): string;
|