@link-foundation/sensitive-data-sanitizer 0.13.0 → 0.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/README.md +12 -1
- package/bin/sensitive-data-sanitizer.js +5 -0
- package/docs/API.md +64 -24
- package/docs/COVERAGE.md +23 -2
- package/package.json +1 -1
- package/src/context.js +23 -9
- package/src/detection.js +43 -26
- package/src/document-entities.js +87 -2
- package/src/document-vocabulary.js +1 -1
- package/src/engines.js +6 -2
- package/src/entities.js +7 -0
- package/src/fake-entities.js +304 -0
- package/src/fake-names.js +63 -8
- package/src/fake.js +4 -8
- package/src/index.d.ts +8 -0
- package/src/keywords.js +17 -0
- package/src/names.js +1 -1
- package/src/personal-context.js +1 -1
- package/src/personal-variants.js +184 -0
- package/src/personal.js +18 -7
- package/src/placeholder-types.js +33 -0
- package/src/placeholders.js +57 -10
- package/src/projection.js +6 -2
- package/src/propagation.js +17 -1
- package/src/russian-declension.js +167 -0
- package/src/russian-names.js +36 -0
- package/src/sanitizer.js +28 -2
- package/src/transforms.js +21 -4
|
@@ -0,0 +1,304 @@
|
|
|
1
|
+
import { createHmac } from 'node:crypto';
|
|
2
|
+
import { fakeName, fakeWord, wordProfile } from './fake-names.js';
|
|
3
|
+
import { canonicalPerson } from './placeholders.js';
|
|
4
|
+
import {
|
|
5
|
+
isGivenName,
|
|
6
|
+
isPatronymic,
|
|
7
|
+
russianNameRoles,
|
|
8
|
+
} from './russian-names.js';
|
|
9
|
+
import {
|
|
10
|
+
casesOf,
|
|
11
|
+
commonCase,
|
|
12
|
+
formOf,
|
|
13
|
+
governedCases,
|
|
14
|
+
russianNameForms,
|
|
15
|
+
russianNoun,
|
|
16
|
+
} from './russian-declension.js';
|
|
17
|
+
import {
|
|
18
|
+
organizationCore,
|
|
19
|
+
samePersonal,
|
|
20
|
+
wordPattern,
|
|
21
|
+
} from './personal-variants.js';
|
|
22
|
+
|
|
23
|
+
// Fictional but ordinary names. The lists are finite on purpose; callers can
|
|
24
|
+
// replace them with `transformation.pools`.
|
|
25
|
+
const vocabulary = {
|
|
26
|
+
ORGANIZATION: {
|
|
27
|
+
ru: 'Северный ветер|Альфа-Строй|Бета-Сервис|Гамма-Трейд|Меридиан|Горизонт|Гранит|Рассвет|Прогресс|Импульс|Орион|Азимут|Кристалл|Арсенал|Квант|Заря|Радуга|Волна|Орбита|Вершина|Искра|Тихая гавань|Новый горизонт|Белый парус|Золотой колос|Синяя птица',
|
|
28
|
+
latin:
|
|
29
|
+
'Northwind Trading|Bluebird Logistics|Silver Birch Partners|Harbor Lane Studio|Summit Ridge Supply|Clearwater Works|Redwood Systems|Maple Street Foods',
|
|
30
|
+
},
|
|
31
|
+
LOCATION: {
|
|
32
|
+
ru: 'Зареченск|Приморск|Светлогорск|Лесногорск|Березовка|Сосновка|Каменка|Покровка|Ольховка|Заречье|Краснополье',
|
|
33
|
+
latin: 'Riverton|Oakdale|Fairview|Lakewood|Brookfield|Greenville|Ashford',
|
|
34
|
+
},
|
|
35
|
+
EDUCATION: {
|
|
36
|
+
ru: 'Северный государственный университет|Приморский технический институт|Лесногорский гуманитарный университет|Зареченский экономический колледж|Светлогорская городская школа',
|
|
37
|
+
latin:
|
|
38
|
+
'Riverside University|Lakeview College|Oakdale Institute of Technology|Fairview State University',
|
|
39
|
+
},
|
|
40
|
+
};
|
|
41
|
+
const cyrillic = /\p{Script=Cyrillic}/u;
|
|
42
|
+
const surfacesKey = Symbol('fakeSurfaces');
|
|
43
|
+
const textKey = Symbol('fakeText');
|
|
44
|
+
const surfaceLimit = 1000;
|
|
45
|
+
|
|
46
|
+
// Lets each span see the other mentions of its document, as placeholders do.
|
|
47
|
+
export function prepareFakeText(text, findings, options) {
|
|
48
|
+
const surfaces = new Map();
|
|
49
|
+
for (const finding of findings) {
|
|
50
|
+
if (
|
|
51
|
+
['PERSON', 'ORGANIZATION', 'LOCATION', 'EDUCATION'].includes(finding.type)
|
|
52
|
+
) {
|
|
53
|
+
const values = surfaces.get(finding.type) ?? new Set();
|
|
54
|
+
if (values.size < surfaceLimit) {
|
|
55
|
+
values.add(text.slice(finding.start, finding.end));
|
|
56
|
+
}
|
|
57
|
+
surfaces.set(finding.type, values);
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
return { ...options, [surfacesKey]: surfaces, [textKey]: text };
|
|
61
|
+
}
|
|
62
|
+
const wordCount = (value) => value.trim().split(/\s+/u).length;
|
|
63
|
+
// The fullest form this mention belongs to, or the mention itself.
|
|
64
|
+
function entityLemma(value, type, options) {
|
|
65
|
+
const known = [
|
|
66
|
+
...(options.knownPersonal ?? []),
|
|
67
|
+
...(options.confirmedPersonal ?? []),
|
|
68
|
+
]
|
|
69
|
+
.filter((entry) => entry.type === type)
|
|
70
|
+
.map((entry) => entry.value);
|
|
71
|
+
const mentions = [...(options[surfacesKey]?.get(type) ?? [])];
|
|
72
|
+
for (const group of [known, mentions]) {
|
|
73
|
+
const matches = group.filter((other) => samePersonal(value, other, type));
|
|
74
|
+
const longest = Math.max(0, ...matches.map(wordCount));
|
|
75
|
+
const fullest = [
|
|
76
|
+
...new Set(matches.filter((other) => wordCount(other) === longest)),
|
|
77
|
+
];
|
|
78
|
+
// `Гранит` covers its own form `Граните`, so several forms share one lemma.
|
|
79
|
+
const lemma = fullest.filter((other) =>
|
|
80
|
+
fullest.every((form) => samePersonal(form, other, type))
|
|
81
|
+
);
|
|
82
|
+
if (lemma.length === 1) {
|
|
83
|
+
return lemma[0];
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
return type === 'PERSON' ? canonicalPerson(value, mentions) : value;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
const applyStyle = (surface, output) =>
|
|
90
|
+
surface.length > 1 && surface === surface.toUpperCase()
|
|
91
|
+
? output.toUpperCase()
|
|
92
|
+
: surface === surface.toLowerCase()
|
|
93
|
+
? output.toLowerCase()
|
|
94
|
+
: output;
|
|
95
|
+
const capitalize = (word) =>
|
|
96
|
+
word[0].toUpperCase() + word.slice(1).toLowerCase();
|
|
97
|
+
function nameSlots(parts) {
|
|
98
|
+
const roles = russianNameRoles(parts);
|
|
99
|
+
if (roles) {
|
|
100
|
+
return parts.map((_part, i) =>
|
|
101
|
+
i === roles.surname
|
|
102
|
+
? 'lastName'
|
|
103
|
+
: i === roles.patronymic
|
|
104
|
+
? 'middleName'
|
|
105
|
+
: 'firstName'
|
|
106
|
+
);
|
|
107
|
+
}
|
|
108
|
+
if (parts.length === 1) {
|
|
109
|
+
return [wordProfile(parts[0]).role];
|
|
110
|
+
}
|
|
111
|
+
const given = parts.map((part) => isGivenName(part));
|
|
112
|
+
if (parts.length === 2 && given[0] !== given[1]) {
|
|
113
|
+
return given.map((isGiven) => (isGiven ? 'firstName' : 'lastName'));
|
|
114
|
+
}
|
|
115
|
+
return parts.map((_part, i) =>
|
|
116
|
+
i === 0 ? 'firstName' : i === parts.length - 1 ? 'lastName' : 'middleName'
|
|
117
|
+
);
|
|
118
|
+
}
|
|
119
|
+
function nameSex(parts, slots) {
|
|
120
|
+
const at = (slot) => parts[slots.indexOf(slot)];
|
|
121
|
+
const middle = at('middleName');
|
|
122
|
+
if (middle && (isPatronymic(middle) || /(?:вна|чна)$/iu.test(middle))) {
|
|
123
|
+
return /(?:вна|чна)$/iu.test(middle) ? 'female' : 'male';
|
|
124
|
+
}
|
|
125
|
+
const last = at('lastName');
|
|
126
|
+
if (last && /(?:ова|ева|ёва|ина|ына|ая|яя)$/iu.test(last)) {
|
|
127
|
+
return 'female';
|
|
128
|
+
}
|
|
129
|
+
if (last && /(?:ов|ев|ёв|ин|ын|ий|ой|ый)$/iu.test(last)) {
|
|
130
|
+
return 'male';
|
|
131
|
+
}
|
|
132
|
+
return wordProfile(at('firstName') ?? parts[0]).sex;
|
|
133
|
+
}
|
|
134
|
+
const letters = /[\p{L}\p{M}]+/gu;
|
|
135
|
+
// One identity per lemma: each lemma word has one seeded fake, and the matched
|
|
136
|
+
// surface only decides its case, its order and its letter case.
|
|
137
|
+
function russianFakeName(surface, lemma, key, context) {
|
|
138
|
+
const parts = lemma.trim().split(/\s+/u);
|
|
139
|
+
const slots = nameSlots(parts);
|
|
140
|
+
const sex = nameSex(parts, slots);
|
|
141
|
+
const lemmaWords = parts.flatMap((part, i) =>
|
|
142
|
+
part.match(letters).map((word) => {
|
|
143
|
+
const slot = slots[i];
|
|
144
|
+
return {
|
|
145
|
+
slot,
|
|
146
|
+
lemmaForms: russianNameForms(word, slot, sex),
|
|
147
|
+
pattern: new RegExp(`^(?:${wordPattern(word, 'PERSON')})$`, 'iu'),
|
|
148
|
+
first: word[0].toLowerCase().replace('ё', 'е'),
|
|
149
|
+
fake: fakeWord(capitalize(word), key, slot, undefined, sex),
|
|
150
|
+
};
|
|
151
|
+
})
|
|
152
|
+
);
|
|
153
|
+
const matched = surface.match(letters).map((word) => {
|
|
154
|
+
const initial = word.length === 1;
|
|
155
|
+
const entry = lemmaWords.find(
|
|
156
|
+
(candidate) =>
|
|
157
|
+
!candidate.used &&
|
|
158
|
+
(initial
|
|
159
|
+
? candidate.slot !== 'lastName' &&
|
|
160
|
+
candidate.first === word.toLowerCase().replace('ё', 'е')
|
|
161
|
+
: candidate.pattern.test(word))
|
|
162
|
+
);
|
|
163
|
+
if (entry) {
|
|
164
|
+
entry.used = true;
|
|
165
|
+
}
|
|
166
|
+
return { word, entry, initial };
|
|
167
|
+
});
|
|
168
|
+
const index = commonCase(
|
|
169
|
+
matched
|
|
170
|
+
.filter(({ entry, initial }) => entry && !initial)
|
|
171
|
+
.map(({ word, entry }) => casesOf(word, entry.lemmaForms)),
|
|
172
|
+
context
|
|
173
|
+
);
|
|
174
|
+
let position = 0;
|
|
175
|
+
return surface.replace(letters, () => {
|
|
176
|
+
const { word, entry, initial } = matched[position++];
|
|
177
|
+
if (!entry) {
|
|
178
|
+
return fakeWord(word, key);
|
|
179
|
+
}
|
|
180
|
+
if (initial) {
|
|
181
|
+
return entry.fake[0].toUpperCase();
|
|
182
|
+
}
|
|
183
|
+
const forms = russianNameForms(entry.fake, entry.slot, sex);
|
|
184
|
+
return applyStyle(word, capitalize(formOf(forms, index)));
|
|
185
|
+
});
|
|
186
|
+
}
|
|
187
|
+
const contextOf = (finding, options) =>
|
|
188
|
+
governedCases(options[textKey] ?? '', finding.start);
|
|
189
|
+
export function fakePerson(value, finding, transform, options) {
|
|
190
|
+
const lemma = entityLemma(value, 'PERSON', options);
|
|
191
|
+
return cyrillic.test(lemma) &&
|
|
192
|
+
cyrillic.test(value) &&
|
|
193
|
+
!/\p{Script=Latin}/u.test(`${lemma}${value}`)
|
|
194
|
+
? russianFakeName(value, lemma, transform.key, contextOf(finding, options))
|
|
195
|
+
: fakeName(value, transform.key);
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
const assigned = new WeakMap();
|
|
199
|
+
const lemmaKey = (core) => core.toLowerCase().replaceAll('ё', 'е');
|
|
200
|
+
const poolFor = (type, core, transform) =>
|
|
201
|
+
transform.pools?.[type] ??
|
|
202
|
+
vocabulary[type][cyrillic.test(core) ? 'ru' : 'latin'].split('|');
|
|
203
|
+
function isReal(candidate, type, options) {
|
|
204
|
+
return [
|
|
205
|
+
...(options.knownPersonal ?? []),
|
|
206
|
+
...(options.confirmedPersonal ?? []),
|
|
207
|
+
...[...(options[surfacesKey]?.get(type) ?? [])].map((value) => ({
|
|
208
|
+
type,
|
|
209
|
+
value,
|
|
210
|
+
})),
|
|
211
|
+
].some(
|
|
212
|
+
(entry) => entry.type === type && samePersonal(candidate, entry.value, type)
|
|
213
|
+
);
|
|
214
|
+
}
|
|
215
|
+
// A deterministic start, then the next free name so that two entities of one
|
|
216
|
+
// run never share a fake. An exhausted list continues with numbered names.
|
|
217
|
+
function assignFake(core, type, transform, options) {
|
|
218
|
+
let state = assigned.get(transform);
|
|
219
|
+
if (!state) {
|
|
220
|
+
state = new Map();
|
|
221
|
+
assigned.set(transform, state);
|
|
222
|
+
}
|
|
223
|
+
const entityKey = `${type}\0${lemmaKey(core)}`;
|
|
224
|
+
if (state.has(entityKey)) {
|
|
225
|
+
return state.get(entityKey);
|
|
226
|
+
}
|
|
227
|
+
const taken = new Set(state.values());
|
|
228
|
+
const pool = poolFor(type, core, transform);
|
|
229
|
+
const start = createHmac('sha256', transform.key)
|
|
230
|
+
.update(`entity\0${entityKey}`)
|
|
231
|
+
.digest()
|
|
232
|
+
.readUInt32BE(0);
|
|
233
|
+
for (let attempt = 0; ; attempt++) {
|
|
234
|
+
const name = pool[(start + attempt) % pool.length];
|
|
235
|
+
const round = Math.floor(attempt / pool.length);
|
|
236
|
+
const candidate = round ? `${name}-${round + 1}` : name;
|
|
237
|
+
if (!taken.has(candidate) && !isReal(candidate, type, options)) {
|
|
238
|
+
state.set(entityKey, candidate);
|
|
239
|
+
return candidate;
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
}
|
|
243
|
+
const nounOptions = (word, phrase, position) => ({
|
|
244
|
+
animate: false,
|
|
245
|
+
sex: phrase.some((other) => /[ая]я$/u.test(other)) ? 'female' : undefined,
|
|
246
|
+
adjective:
|
|
247
|
+
!word.includes('-') &&
|
|
248
|
+
(/(?:ый|ая|яя)$/u.test(word) ||
|
|
249
|
+
(/(?:ий|ой|ое|ее)$/u.test(word) && position < phrase.length - 1)),
|
|
250
|
+
});
|
|
251
|
+
// Only the last part of a hyphenated name declines: в Альфа-Строе.
|
|
252
|
+
const phraseForms = (phrase) => {
|
|
253
|
+
const words = phrase.toLowerCase().split(/\s+/u);
|
|
254
|
+
return words.map((word, position) => {
|
|
255
|
+
const head = word.slice(0, word.lastIndexOf('-') + 1);
|
|
256
|
+
const tail = word.slice(head.length);
|
|
257
|
+
return {
|
|
258
|
+
head,
|
|
259
|
+
forms: russianNoun(tail, nounOptions(word, words, position)),
|
|
260
|
+
};
|
|
261
|
+
});
|
|
262
|
+
};
|
|
263
|
+
function phraseCase(surface, lemma, context) {
|
|
264
|
+
const words = surface.split(/\s+/u);
|
|
265
|
+
const lemmaForms = phraseForms(lemma);
|
|
266
|
+
if (words.length !== lemmaForms.length) {
|
|
267
|
+
return 0;
|
|
268
|
+
}
|
|
269
|
+
return commonCase(
|
|
270
|
+
words.map((word, i) =>
|
|
271
|
+
casesOf(word.slice(word.lastIndexOf('-') + 1), lemmaForms[i].forms)
|
|
272
|
+
),
|
|
273
|
+
context
|
|
274
|
+
);
|
|
275
|
+
}
|
|
276
|
+
function declinedPhrase(phrase, index) {
|
|
277
|
+
if (!cyrillic.test(phrase)) {
|
|
278
|
+
return phrase;
|
|
279
|
+
}
|
|
280
|
+
const original = phrase.split(/\s+/u);
|
|
281
|
+
return phraseForms(phrase)
|
|
282
|
+
.map(({ head, forms }, i) => {
|
|
283
|
+
const word = head + formOf(forms, index);
|
|
284
|
+
return original[i][0] === original[i][0].toUpperCase()
|
|
285
|
+
? word.replace(
|
|
286
|
+
/(^|-)(\p{L})/gu,
|
|
287
|
+
(_m, gap, letter) => `${gap}${letter.toUpperCase()}`
|
|
288
|
+
)
|
|
289
|
+
: word;
|
|
290
|
+
})
|
|
291
|
+
.join(' ');
|
|
292
|
+
}
|
|
293
|
+
const decoration = /^((?:(?:ООО|АО|ПАО|ИП|ЗАО)[ \t]+)?[«“"]?)(.*?)([»”"]?)$/su;
|
|
294
|
+
// `ООО «Ромашка»` keeps its legal form and quotes; only the name is replaced.
|
|
295
|
+
export function fakeEntity(value, finding, transform, options) {
|
|
296
|
+
const { type } = finding;
|
|
297
|
+
const [, before, core, after] = decoration.exec(value);
|
|
298
|
+
const lemma = organizationCore(entityLemma(value, type, options));
|
|
299
|
+
const fake = assignFake(lemma, type, transform, options);
|
|
300
|
+
const output = cyrillic.test(core)
|
|
301
|
+
? declinedPhrase(fake, phraseCase(core, lemma, contextOf(finding, options)))
|
|
302
|
+
: fake;
|
|
303
|
+
return `${before}${applyStyle(core, output)}${after}`;
|
|
304
|
+
}
|
package/src/fake-names.js
CHANGED
|
@@ -146,6 +146,58 @@ function scriptLocale(value) {
|
|
|
146
146
|
}
|
|
147
147
|
export function fakeName(value, key) {
|
|
148
148
|
indexNames();
|
|
149
|
+
const words = value.match(/[\p{L}\p{M}]+/gu) ?? [];
|
|
150
|
+
const roles = new Map();
|
|
151
|
+
let nameSex;
|
|
152
|
+
if (
|
|
153
|
+
/\p{Script=Cyrillic}/u.test(value) &&
|
|
154
|
+
words.length >= 2 &&
|
|
155
|
+
words.length <= 3
|
|
156
|
+
) {
|
|
157
|
+
const middle = words.findIndex((w) =>
|
|
158
|
+
/(?:ович|евич|ична|овна|евна)$/iu.test(w)
|
|
159
|
+
);
|
|
160
|
+
const surname = words.findIndex(
|
|
161
|
+
(w, i) =>
|
|
162
|
+
i !== middle &&
|
|
163
|
+
/(?:ов|ев|ин|ский|ова|ева|ина|ская)$/iu.test(w) &&
|
|
164
|
+
!given.has(latinName(w))
|
|
165
|
+
);
|
|
166
|
+
const genderWord = words[middle >= 0 ? middle : surname];
|
|
167
|
+
if (genderWord) {
|
|
168
|
+
nameSex = /(?:ова|ева|ина|ская|овна|евна|ична)$/iu.test(genderWord)
|
|
169
|
+
? 'female'
|
|
170
|
+
: 'male';
|
|
171
|
+
}
|
|
172
|
+
if (middle > 0) {
|
|
173
|
+
words.forEach((w, i) =>
|
|
174
|
+
roles.set(
|
|
175
|
+
w,
|
|
176
|
+
i === middle
|
|
177
|
+
? 'middleName'
|
|
178
|
+
: i === middle - 1
|
|
179
|
+
? 'firstName'
|
|
180
|
+
: 'lastName'
|
|
181
|
+
)
|
|
182
|
+
);
|
|
183
|
+
} else if (surname >= 0) {
|
|
184
|
+
words.forEach((w, i) =>
|
|
185
|
+
roles.set(w, i === surname ? 'lastName' : 'firstName')
|
|
186
|
+
);
|
|
187
|
+
} else {
|
|
188
|
+
// Ambiguous spans receive a complete name with the same token count.
|
|
189
|
+
words.forEach((w, i) =>
|
|
190
|
+
roles.set(
|
|
191
|
+
w,
|
|
192
|
+
i === 0
|
|
193
|
+
? 'firstName'
|
|
194
|
+
: i === words.length - 1
|
|
195
|
+
? 'lastName'
|
|
196
|
+
: 'middleName'
|
|
197
|
+
)
|
|
198
|
+
);
|
|
199
|
+
}
|
|
200
|
+
}
|
|
149
201
|
return value.replace(/[\p{L}\p{M}]+/gu, (word) => {
|
|
150
202
|
for (const locale of ['ja', 'zh', 'ko']) {
|
|
151
203
|
const family = firstNames[locale]
|
|
@@ -158,10 +210,11 @@ export function fakeName(value, key) {
|
|
|
158
210
|
);
|
|
159
211
|
}
|
|
160
212
|
}
|
|
161
|
-
return fakeWord(word, key);
|
|
213
|
+
return fakeWord(word, key, roles.get(word), undefined, nameSex);
|
|
162
214
|
});
|
|
163
215
|
}
|
|
164
|
-
function wordProfile(word, forcedRole, forcedLocale) {
|
|
216
|
+
export function wordProfile(word, forcedRole, forcedLocale) {
|
|
217
|
+
indexNames();
|
|
165
218
|
const canonical = latinName(word);
|
|
166
219
|
const patronymic = /(?:vich|vna|ichna|ogly|kyzy)$/.test(canonical);
|
|
167
220
|
const slavic = /(?:ova|eva|ov|ev|in|ina|enko|sky|ski|skaya)$/.test(canonical);
|
|
@@ -188,12 +241,14 @@ function wordProfile(word, forcedRole, forcedLocale) {
|
|
|
188
241
|
: 'lastName');
|
|
189
242
|
return { canonical, locale, sex, role };
|
|
190
243
|
}
|
|
191
|
-
function fakeWord(word, key, forcedRole, forcedLocale) {
|
|
192
|
-
const {
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
244
|
+
export function fakeWord(word, key, forcedRole, forcedLocale, forcedSex) {
|
|
245
|
+
const {
|
|
246
|
+
canonical,
|
|
247
|
+
locale,
|
|
248
|
+
sex: detectedSex,
|
|
249
|
+
role,
|
|
250
|
+
} = wordProfile(word, forcedRole, forcedLocale);
|
|
251
|
+
const sex = forcedSex ?? detectedSex;
|
|
197
252
|
const faker = new Faker({ locale: [locales[locale] ?? en, en] });
|
|
198
253
|
faker.seed(
|
|
199
254
|
createHmac('sha256', key)
|
package/src/fake.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { fakeEntity, fakePerson } from './fake-entities.js';
|
|
1
2
|
import { createHmac } from 'node:crypto';
|
|
2
3
|
import { parsePhoneNumberFromString } from 'libphonenumber-js/max';
|
|
3
4
|
import { mrzCheck } from './identity.js';
|
|
@@ -487,7 +488,7 @@ function fakeTd1(parts, transform) {
|
|
|
487
488
|
.padEnd(30, '<');
|
|
488
489
|
return parts.join('');
|
|
489
490
|
}
|
|
490
|
-
export function realisticFake(value, finding, transform) {
|
|
491
|
+
export function realisticFake(value, finding, transform, options = {}) {
|
|
491
492
|
let output;
|
|
492
493
|
if (finding.category === 'credential') {
|
|
493
494
|
if (!transform.credentials) {
|
|
@@ -500,14 +501,9 @@ export function realisticFake(value, finding, transform) {
|
|
|
500
501
|
.toString('hex')
|
|
501
502
|
.slice(0, 8)}!`;
|
|
502
503
|
} else if (finding.type === 'PERSON') {
|
|
503
|
-
output =
|
|
504
|
+
output = fakePerson(value, finding, transform, options);
|
|
504
505
|
} else if (['ORGANIZATION', 'EDUCATION', 'LOCATION'].includes(finding.type)) {
|
|
505
|
-
|
|
506
|
-
ORGANIZATION: 'Organization',
|
|
507
|
-
EDUCATION: 'School',
|
|
508
|
-
LOCATION: 'Location',
|
|
509
|
-
}[finding.type];
|
|
510
|
-
output = `Example ${label} ${digest(value, transform.key, finding.type).readUInt32BE(0)}`;
|
|
506
|
+
output = fakeEntity(value, finding, transform, options);
|
|
511
507
|
} else if (/DATE|BIRTH/.test(finding.type)) {
|
|
512
508
|
output = fakeDate(value, transform.key);
|
|
513
509
|
} else if (finding.type === 'PASSPORT_MRZ') {
|
package/src/index.d.ts
CHANGED
|
@@ -42,6 +42,8 @@ export interface Detector {
|
|
|
42
42
|
export interface SanitizerOptions {
|
|
43
43
|
/** Preserve all personal/account data; credential findings still redact. */
|
|
44
44
|
credentialsOnly?: boolean;
|
|
45
|
+
/** Replace only knownPersonal/knownSecrets and reused map values; skip automatic detectors. */
|
|
46
|
+
knownOnly?: boolean;
|
|
45
47
|
profile?: 'publication' | 'documents';
|
|
46
48
|
/** Opt in to bounded employer, school, city/region heuristics. */
|
|
47
49
|
documentEntities?: boolean;
|
|
@@ -97,6 +99,8 @@ export interface SanitizedText {
|
|
|
97
99
|
findings: Finding[];
|
|
98
100
|
/** Number of disjoint replacements after overlapping detections are merged. */
|
|
99
101
|
redactions: number;
|
|
102
|
+
/** Placeholder-mode replacements absent from the restore map (e.g. credentials). */
|
|
103
|
+
irreversibleRedactions?: number;
|
|
100
104
|
/** Private personal data. Absent for map:false and non-placeholder modes. */
|
|
101
105
|
map?: RestoreMap;
|
|
102
106
|
}
|
|
@@ -246,6 +250,10 @@ export type Transformation =
|
|
|
246
250
|
credentials?: boolean;
|
|
247
251
|
/** Keep original MRZ issuer/nationality unless Utopia is requested. */
|
|
248
252
|
mrzCountry?: 'UTO';
|
|
253
|
+
/** Caller-reviewed names that replace the generated entity vocabulary. */
|
|
254
|
+
pools?: Partial<
|
|
255
|
+
Record<'ORGANIZATION' | 'LOCATION' | 'EDUCATION', string[]>
|
|
256
|
+
>;
|
|
249
257
|
}
|
|
250
258
|
| { mode: 'date-shift'; days: number }
|
|
251
259
|
| { mode: 'bucket'; size: number };
|
package/src/keywords.js
CHANGED
|
@@ -27,10 +27,27 @@ export function keywordMatcher(words) {
|
|
|
27
27
|
queue.push(child);
|
|
28
28
|
}
|
|
29
29
|
}
|
|
30
|
+
// At the root, characters that cannot start any keyword need no state
|
|
31
|
+
// transitions. A native search skips large padding runs without Map work
|
|
32
|
+
// per character. Escape UTF-16 units so metacharacters retain literal meaning.
|
|
33
|
+
const starters = new RegExp(
|
|
34
|
+
`[${[...nodes[0].next.keys()]
|
|
35
|
+
.map((char) => `\\u${char.charCodeAt(0).toString(16).padStart(4, '0')}`)
|
|
36
|
+
.join('')}]`,
|
|
37
|
+
'g'
|
|
38
|
+
);
|
|
30
39
|
return (text) => {
|
|
31
40
|
const found = new Set();
|
|
32
41
|
let state = 0;
|
|
33
42
|
for (let i = 0; i < text.length; i++) {
|
|
43
|
+
if (!state && !nodes[0].output.length && !nodes[0].next.has(text[i])) {
|
|
44
|
+
starters.lastIndex = i + 1;
|
|
45
|
+
const next = starters.exec(text);
|
|
46
|
+
if (!next) {
|
|
47
|
+
break;
|
|
48
|
+
}
|
|
49
|
+
i = next.index;
|
|
50
|
+
}
|
|
34
51
|
const char = text[i];
|
|
35
52
|
while (state && !nodes[state].next.has(char)) {
|
|
36
53
|
state = nodes[state].fail;
|
package/src/names.js
CHANGED
|
@@ -12,7 +12,7 @@ export const firstNames = {
|
|
|
12
12
|
jaLatn: 'Yamada Sato Suzuki Takahashi Tanaka Ito Watanabe Nakamura Kobayashi',
|
|
13
13
|
zhLatn:
|
|
14
14
|
'Zhang Wang Li Zhao Liu Chen Yang Huang Zhou Wu Xu Sun Hu Zhu Gao Lin',
|
|
15
|
-
ru: 'Иван Пётр Петр Сергей Александр Алексей Дмитрий Михаил Николай Андрей Владимир Павел Виктор Олег Борис Евгений Василий Юрий Марина Анна Мария Елена Ольга Наталья Ирина Светлана Татьяна Екатерина Анастасия Юлия Ивану Ивана Иваном Сергея Сергею Александра Александру Анне Анну Елены',
|
|
15
|
+
ru: 'Данил Максим Алина Иван Пётр Петр Сергей Александр Алексей Дмитрий Михаил Николай Андрей Владимир Павел Виктор Олег Борис Евгений Василий Юрий Марина Анна Мария Елена Ольга Наталья Ирина Светлана Татьяна Екатерина Анастасия Юлия Ивану Ивана Иваном Сергея Сергею Александра Александру Анне Анну Елены',
|
|
16
16
|
ja: '山田 佐藤 鈴木 高橋 田中 伊藤 渡辺 渡邊 中村 小林 加藤 吉田 松本 山本 井上 木村 林 清水 斎藤 斉藤',
|
|
17
17
|
zh: '张 王 李 赵 刘 陈 杨 黄 周 吴 徐 孙 胡 朱 高 林 何 郭 马 罗 張 趙 劉 陳 楊 黃 吳 孫 馬 羅',
|
|
18
18
|
ko: '김 이 박 최 정 강 조 윤 장 임 한 오 서 신 권 황 안 송 전 홍 유 고 문 양 손 배 백',
|
package/src/personal-context.js
CHANGED
|
@@ -2,7 +2,7 @@ import { collectMatches } from './detection.js';
|
|
|
2
2
|
|
|
3
3
|
const birth =
|
|
4
4
|
/(?<![\p{L}\p{N}_])(?:born|birth[ _-]?date|date[ _-]of[ _-]birth|DOB|дата рождения|д\.р\.|родился|родилась|Geburtsdatum|fecha de nacimiento|date de naissance)(?![\p{L}\p{N}_])/giu;
|
|
5
|
-
const months =
|
|
5
|
+
export const months =
|
|
6
6
|
'January February March April May June July August September October November December Jan Feb Mar Apr Jun Jul Aug Sep Sept Oct Nov Dec января февраля марта апреля мая июня июля августа сентября октября ноября декабря январь февраль март апрель май июнь июль август сентябрь октябрь ноябрь декабрь'
|
|
7
7
|
.split(' ')
|
|
8
8
|
.join('|');
|
|
@@ -0,0 +1,184 @@
|
|
|
1
|
+
import { escapePattern, failure } from './detection.js';
|
|
2
|
+
import { russianNameRoles } from './russian-names.js';
|
|
3
|
+
|
|
4
|
+
const boundary = '[\\p{L}\\p{M}\\p{N}_]';
|
|
5
|
+
export function organizationCore(value) {
|
|
6
|
+
return value
|
|
7
|
+
.replace(/^(?:ООО|АО|ПАО|ИП|ЗАО)[ \t]+/u, '')
|
|
8
|
+
.replace(/^[«“"]|[»”"]$/gu, '')
|
|
9
|
+
.trim();
|
|
10
|
+
}
|
|
11
|
+
// `Вектор-1` also identifies its base name and numbered siblings like `Вектор-2`.
|
|
12
|
+
export function organizationFamily(core) {
|
|
13
|
+
const base = /^(.{3,})-\d{1,9}$/u.exec(core)?.[1];
|
|
14
|
+
return base
|
|
15
|
+
? {
|
|
16
|
+
base,
|
|
17
|
+
pattern: new RegExp(
|
|
18
|
+
`(?<!${boundary})${escapePattern(base)}(?:-\\d{1,9})?(?!${boundary}|-\\d)`,
|
|
19
|
+
'giu'
|
|
20
|
+
),
|
|
21
|
+
}
|
|
22
|
+
: undefined;
|
|
23
|
+
}
|
|
24
|
+
// ё and е are one letter in ordinary Russian writing; the i flag covers Ё and Е.
|
|
25
|
+
const foldYo = (pattern) => pattern.replace(/[еёЕЁ]/gu, '[её]');
|
|
26
|
+
const fleeting = { павел: 'Павл', лев: 'Льв' };
|
|
27
|
+
const hardCases = '(?:а|у|ом|е)';
|
|
28
|
+
// Only caller-confirmed Russian words receive finite case alternatives. A
|
|
29
|
+
// minimum three-letter stem avoids turning short names into broad matches.
|
|
30
|
+
function cyrillicPattern(word, type) {
|
|
31
|
+
const endings = [
|
|
32
|
+
[/(?:ова|ева|ина)$/u, 1, '(?:а|ой|ою|у|е|ы)'],
|
|
33
|
+
[/(?:ая)$/u, 2, '(?:ая|ой|ою|ую)'],
|
|
34
|
+
[
|
|
35
|
+
/(?:ий|ый|ой)$/u,
|
|
36
|
+
2,
|
|
37
|
+
'(?:ий|ый|ой|ого|его|ому|ему|им|ым|ом|ем|ия|ию|ием|ии)',
|
|
38
|
+
],
|
|
39
|
+
[/[аеуюя]й$/u, 1, '(?:й|я|ю|ем|е)'],
|
|
40
|
+
[/(?:ов|ев|ин)$/u, 0, '(?:а|у|ым|ом|е|ой)?'],
|
|
41
|
+
[/[жчшщц]$/u, 0, '(?:а|у|ем|ом|е)?'],
|
|
42
|
+
[/[жчшщц]а$/u, 1, '(?:а|и|е|у|ей|ею|ой|ою)'],
|
|
43
|
+
[/а$/u, 1, '(?:а|ы|и|е|у|ой|ою)'],
|
|
44
|
+
[/я$/u, 1, '(?:я|и|е|ю|ей|ею)'],
|
|
45
|
+
[/ь$/u, 1, '(?:ь|я|ю|ем|е|и|ью)'],
|
|
46
|
+
];
|
|
47
|
+
const lower = word.toLowerCase();
|
|
48
|
+
if (fleeting[lower]) {
|
|
49
|
+
return `(?:${word}|${fleeting[lower]}${hardCases})`;
|
|
50
|
+
}
|
|
51
|
+
for (const [ending, remove, suffix] of endings) {
|
|
52
|
+
if (ending.test(lower) && word.length - remove >= 3) {
|
|
53
|
+
return escapePattern(remove ? word.slice(0, -remove) : word) + suffix;
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
if (/^[а-яё]{3,}[бвгджзклмнпрстфхцчшщ]$/u.test(lower)) {
|
|
57
|
+
// Plural forms would turn a given name into a surname: Петр and Петров.
|
|
58
|
+
return `${escapePattern(word)}${type === 'PERSON' ? `${hardCases}?` : '(?:а|у|ом|е|ы|ов|ам|ами|ах)?'}`;
|
|
59
|
+
}
|
|
60
|
+
return escapePattern(word);
|
|
61
|
+
}
|
|
62
|
+
export function wordPattern(word, type) {
|
|
63
|
+
return foldYo(
|
|
64
|
+
word
|
|
65
|
+
.split('-')
|
|
66
|
+
.map((part) =>
|
|
67
|
+
/^[а-яё]+$/iu.test(part)
|
|
68
|
+
? cyrillicPattern(part, type)
|
|
69
|
+
: escapePattern(part)
|
|
70
|
+
)
|
|
71
|
+
.join('-')
|
|
72
|
+
);
|
|
73
|
+
}
|
|
74
|
+
const initialGap = '[ \\t\\u00a0]*';
|
|
75
|
+
const initialPattern = (word) => `${foldYo(escapePattern(word[0]))}\\.`;
|
|
76
|
+
// Everyday orders, partial forms and initials of a Russian full name, longest
|
|
77
|
+
// first so that alternation prefers the complete form at each position.
|
|
78
|
+
function russianForms(words, roles, type) {
|
|
79
|
+
const name = Object.fromEntries(
|
|
80
|
+
Object.entries(roles).map(([role, i]) => [role, words[i]])
|
|
81
|
+
);
|
|
82
|
+
const gap = '[ \\t]+';
|
|
83
|
+
const full = (...parts) =>
|
|
84
|
+
parts.map((part) => wordPattern(part, type)).join(gap);
|
|
85
|
+
if (name.surname === undefined) {
|
|
86
|
+
return [];
|
|
87
|
+
}
|
|
88
|
+
const initial = initialPattern(name.given);
|
|
89
|
+
const last = wordPattern(name.surname, type);
|
|
90
|
+
const short = [
|
|
91
|
+
full(name.surname, name.given),
|
|
92
|
+
full(name.given, name.surname),
|
|
93
|
+
`${last}${gap}${initial}`,
|
|
94
|
+
`${initial}[ \\t\\u00a0]+${last}`,
|
|
95
|
+
];
|
|
96
|
+
if (name.patronymic === undefined) {
|
|
97
|
+
return short;
|
|
98
|
+
}
|
|
99
|
+
const initials = `${initial}${initialGap}${initialPattern(name.patronymic)}`;
|
|
100
|
+
return [
|
|
101
|
+
full(name.surname, name.given, name.patronymic),
|
|
102
|
+
full(name.given, name.patronymic, name.surname),
|
|
103
|
+
`${last}[ \\t\\u00a0]+${initials}`,
|
|
104
|
+
`${initials}[ \\t\\u00a0]+${last}`,
|
|
105
|
+
...short,
|
|
106
|
+
full(name.given, name.patronymic),
|
|
107
|
+
];
|
|
108
|
+
}
|
|
109
|
+
export function personalPatterns(value, type) {
|
|
110
|
+
if (value.length > 256) {
|
|
111
|
+
// Large explicit values still support literal matching without expansion.
|
|
112
|
+
return [escapePattern(value)];
|
|
113
|
+
}
|
|
114
|
+
const values = new Set([value]);
|
|
115
|
+
if (type === 'ORGANIZATION') {
|
|
116
|
+
const core = organizationCore(value);
|
|
117
|
+
if (core.length >= 3) {
|
|
118
|
+
values.add(core);
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
const patterns = new Set();
|
|
122
|
+
for (const surface of values) {
|
|
123
|
+
const parts = surface.split(/([ \t]+)/u);
|
|
124
|
+
const pattern = parts
|
|
125
|
+
.map((part, i) => (i % 2 ? '[ \\t]+' : wordPattern(part, type)))
|
|
126
|
+
.join('');
|
|
127
|
+
patterns.add(pattern);
|
|
128
|
+
if (type === 'PERSON') {
|
|
129
|
+
const words = parts.filter((_part, i) => i % 2 === 0);
|
|
130
|
+
const roles = russianNameRoles(words);
|
|
131
|
+
for (const form of roles ? russianForms(words, roles, type) : []) {
|
|
132
|
+
patterns.add(form);
|
|
133
|
+
}
|
|
134
|
+
if (words.length === 2) {
|
|
135
|
+
patterns.add(
|
|
136
|
+
[wordPattern(words[1], type), wordPattern(words[0], type)].join(
|
|
137
|
+
'[ \\t]+'
|
|
138
|
+
)
|
|
139
|
+
);
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
return [...patterns];
|
|
144
|
+
}
|
|
145
|
+
export function personalRegex(value, type) {
|
|
146
|
+
return new RegExp(
|
|
147
|
+
`(?<!${boundary})(?:${personalPatterns(value, type).join('|')})(?!${boundary})`,
|
|
148
|
+
'giu'
|
|
149
|
+
);
|
|
150
|
+
}
|
|
151
|
+
export function samePersonal(value, original, type) {
|
|
152
|
+
return personalPatterns(original, type).some((pattern) =>
|
|
153
|
+
new RegExp(`^(?:${pattern})$`, 'iu').test(value)
|
|
154
|
+
);
|
|
155
|
+
}
|
|
156
|
+
export function detectKnownPersonal(text, entries, rule, emit, options) {
|
|
157
|
+
let count = 0;
|
|
158
|
+
for (const entry of entries ?? []) {
|
|
159
|
+
const family =
|
|
160
|
+
rule === 'confirmed-personal' &&
|
|
161
|
+
options.documentEntities &&
|
|
162
|
+
!options.knownOnly &&
|
|
163
|
+
entry.type === 'ORGANIZATION' &&
|
|
164
|
+
organizationFamily(organizationCore(entry.value));
|
|
165
|
+
for (const pattern of [
|
|
166
|
+
personalRegex(entry.value, entry.type),
|
|
167
|
+
...(family ? [family.pattern] : []),
|
|
168
|
+
]) {
|
|
169
|
+
for (const match of text.matchAll(pattern)) {
|
|
170
|
+
if (++count > (options.maxFindings ?? 100000)) {
|
|
171
|
+
throw failure('ERR_LIMIT');
|
|
172
|
+
}
|
|
173
|
+
emit({
|
|
174
|
+
start: match.index,
|
|
175
|
+
end: match.index + match[0].length,
|
|
176
|
+
type: entry.type,
|
|
177
|
+
category: 'personal',
|
|
178
|
+
rule,
|
|
179
|
+
confidence: rule === 'known-personal' ? 0.85 : 0.9,
|
|
180
|
+
});
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
}
|