@link-foundation/sensitive-data-sanitizer 0.13.0 → 0.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/README.md +12 -1
- package/bin/sensitive-data-sanitizer.js +5 -0
- package/docs/API.md +64 -24
- package/docs/COVERAGE.md +23 -2
- package/package.json +1 -1
- package/src/context.js +23 -9
- package/src/detection.js +43 -26
- package/src/document-entities.js +87 -2
- package/src/document-vocabulary.js +1 -1
- package/src/engines.js +6 -2
- package/src/entities.js +7 -0
- package/src/fake-entities.js +304 -0
- package/src/fake-names.js +63 -8
- package/src/fake.js +4 -8
- package/src/index.d.ts +8 -0
- package/src/keywords.js +17 -0
- package/src/names.js +1 -1
- package/src/personal-context.js +1 -1
- package/src/personal-variants.js +184 -0
- package/src/personal.js +18 -7
- package/src/placeholder-types.js +33 -0
- package/src/placeholders.js +57 -10
- package/src/projection.js +6 -2
- package/src/propagation.js +17 -1
- package/src/russian-declension.js +167 -0
- package/src/russian-names.js +36 -0
- package/src/sanitizer.js +28 -2
- package/src/transforms.js +21 -4
package/src/personal.js
CHANGED
|
@@ -29,6 +29,9 @@ const ibanLengths = Object.fromEntries(
|
|
|
29
29
|
.map((entry) => [entry.slice(0, 2), Number(entry.slice(2))])
|
|
30
30
|
);
|
|
31
31
|
|
|
32
|
+
// `@handle/jane@example.com` is a mention and an address, not a package path.
|
|
33
|
+
const mentionedEmail =
|
|
34
|
+
/^\/[\p{L}\p{N}_.!#$%&'*+?^`{|}~-]+@[\p{L}\p{N}][\p{L}\p{N}.-]{0,252}\.\p{L}{2,63}(?![\p{L}\p{N}_-])/u;
|
|
32
35
|
export function iban(value) {
|
|
33
36
|
const compact = value.replace(/\s/g, '').toUpperCase();
|
|
34
37
|
if (
|
|
@@ -179,7 +182,12 @@ const russianFull =
|
|
|
179
182
|
/(?<![\p{L}])(?:[А-ЯЁ][а-яё]{2,30}[ \t]+){2}[А-ЯЁ][а-яё]{2,30}(?:вич|вна|ична)|(?<![\p{L}])(?:[А-ЯЁ][а-яё]{2,30}[ \t]+){2}(?:оглы|кызы)(?![\p{L}])/gu;
|
|
180
183
|
export function detectPersonal(text, emit) {
|
|
181
184
|
detectDocumentNames(text, emit);
|
|
182
|
-
for (const pattern of [
|
|
185
|
+
for (const pattern of [
|
|
186
|
+
latinPairs,
|
|
187
|
+
slavicPairs,
|
|
188
|
+
russianFull,
|
|
189
|
+
/(?<![\p{L}])(?:[А-ЯЁ][а-яё]{2,30}[ \t]+)[А-ЯЁ][а-яё]{2,30}(?:вич|вна|ична)[ \t]+[А-ЯЁ][а-яё]{2,30}(?![\p{L}])/gu,
|
|
190
|
+
]) {
|
|
183
191
|
collectMatches(
|
|
184
192
|
text,
|
|
185
193
|
pattern,
|
|
@@ -202,10 +210,11 @@ export function detectPersonal(text, emit) {
|
|
|
202
210
|
}
|
|
203
211
|
collectMatches(
|
|
204
212
|
text,
|
|
205
|
-
/(?:компании|организации|company|organization|société|empresa|Firma)[ \t]+([\p{Lu}][\p{L}\p{M}&'
|
|
213
|
+
/(?:компании|организации|company|organization|société|empresa|Firma)[ \t]+([\p{Lu}][\p{L}\p{M}&']{0,80}(?:-[\p{L}\p{M}\p{N}&']{1,40}){0,4})/gu,
|
|
206
214
|
emit,
|
|
207
215
|
details('ORGANIZATION', 'organization-context', 0.7),
|
|
208
|
-
1
|
|
216
|
+
1,
|
|
217
|
+
(value) => value.length >= 2
|
|
209
218
|
);
|
|
210
219
|
collectMatches(
|
|
211
220
|
text,
|
|
@@ -231,10 +240,12 @@ export function detectPersonal(text, emit) {
|
|
|
231
240
|
emit,
|
|
232
241
|
details('USERNAME', 'messenger-handle', 0.65),
|
|
233
242
|
0,
|
|
234
|
-
(_value, match) =>
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
243
|
+
(_value, match) => {
|
|
244
|
+
const rest = text.slice(match.index + match[0].length);
|
|
245
|
+
return (
|
|
246
|
+
!/^\/[a-z0-9][a-z0-9_.-]*/i.test(rest) || mentionedEmail.test(rest)
|
|
247
|
+
);
|
|
248
|
+
}
|
|
238
249
|
);
|
|
239
250
|
collectMatches(
|
|
240
251
|
text,
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
import { catalogTypes } from './entities.js';
|
|
2
|
+
|
|
3
|
+
// Native unlabelled types supplement the shared vendor-label catalog.
|
|
4
|
+
export const reservedPlaceholderTypes = () => [
|
|
5
|
+
...catalogTypes(),
|
|
6
|
+
'ADDRESS',
|
|
7
|
+
'AU_TFN',
|
|
8
|
+
'BRAZIL_CPF_NUMBER',
|
|
9
|
+
'CREDENTIAL',
|
|
10
|
+
'CREDIT_CARD',
|
|
11
|
+
'DATE',
|
|
12
|
+
'EDUCATION',
|
|
13
|
+
'EMAIL',
|
|
14
|
+
'ENCODED_SENSITIVE',
|
|
15
|
+
'IBAN',
|
|
16
|
+
'ID',
|
|
17
|
+
'IP_ADDRESS',
|
|
18
|
+
'KR_RRN',
|
|
19
|
+
'LOCATION',
|
|
20
|
+
'MAC_ADDRESS',
|
|
21
|
+
'ORGANIZATION',
|
|
22
|
+
'PASSPORT_MRZ',
|
|
23
|
+
'PASSPORT_NUMBER',
|
|
24
|
+
'PERSON',
|
|
25
|
+
'PHONE',
|
|
26
|
+
'PL_PESEL',
|
|
27
|
+
'RU_INN',
|
|
28
|
+
'RU_SNILS',
|
|
29
|
+
'SG_NRIC_FIN',
|
|
30
|
+
'STATE',
|
|
31
|
+
'USERNAME',
|
|
32
|
+
'US_HEALTHCARE_NPI',
|
|
33
|
+
];
|
package/src/placeholders.js
CHANGED
|
@@ -1,4 +1,6 @@
|
|
|
1
1
|
import { failure, REDACTED, escapePattern } from './detection.js';
|
|
2
|
+
import { reservedPlaceholderTypes } from './placeholder-types.js';
|
|
3
|
+
import { samePersonal } from './personal-variants.js';
|
|
2
4
|
import { nameAliases } from './transliteration.js';
|
|
3
5
|
import { createHash } from 'node:crypto';
|
|
4
6
|
import { jsonStringLiterals } from './json-string-runs.js';
|
|
@@ -105,11 +107,29 @@ function aliases(value, type) {
|
|
|
105
107
|
),
|
|
106
108
|
];
|
|
107
109
|
}
|
|
110
|
+
// A bare name is the same entity as its numbered form, including maps saved
|
|
111
|
+
// when the span stopped at the hyphen of `Вектор-99`.
|
|
112
|
+
function numberedOrganization(value, entry) {
|
|
113
|
+
return (entry.values ?? []).some((original) => {
|
|
114
|
+
const base = original.replace(/-\d*$/u, '');
|
|
115
|
+
return base !== original && samePersonal(value, base, entry.type);
|
|
116
|
+
});
|
|
117
|
+
}
|
|
108
118
|
function related(value, entry) {
|
|
119
|
+
if (
|
|
120
|
+
(entry.values ?? []).some((original) =>
|
|
121
|
+
samePersonal(value, original, entry.type)
|
|
122
|
+
)
|
|
123
|
+
) {
|
|
124
|
+
return true;
|
|
125
|
+
}
|
|
109
126
|
const candidates = aliases(value, entry.type);
|
|
110
127
|
if (candidates.some((alias) => entry.aliases.includes(alias))) {
|
|
111
128
|
return true;
|
|
112
129
|
}
|
|
130
|
+
if (entry.type === 'ORGANIZATION') {
|
|
131
|
+
return numberedOrganization(value, entry);
|
|
132
|
+
}
|
|
113
133
|
if (entry.type !== 'PERSON') {
|
|
114
134
|
return false;
|
|
115
135
|
}
|
|
@@ -139,7 +159,7 @@ function related(value, entry) {
|
|
|
139
159
|
return false;
|
|
140
160
|
}
|
|
141
161
|
export function placeholderNameFindings(text, findings, options) {
|
|
142
|
-
if (!hasPlaceholders(options)) {
|
|
162
|
+
if (!hasPlaceholders(options) || options.knownOnly) {
|
|
143
163
|
return findings;
|
|
144
164
|
}
|
|
145
165
|
const people = [
|
|
@@ -199,12 +219,23 @@ export function preparePlaceholderText(text, findings, options) {
|
|
|
199
219
|
if (!state || !hasPlaceholders(options)) {
|
|
200
220
|
return;
|
|
201
221
|
}
|
|
202
|
-
|
|
203
|
-
|
|
222
|
+
const types = new Set([
|
|
223
|
+
...reservedPlaceholderTypes(),
|
|
224
|
+
...findings.map((f) =>
|
|
225
|
+
f.category === 'credential' ? 'CREDENTIAL' : f.type
|
|
226
|
+
),
|
|
227
|
+
...(options.knownPersonal ?? []).map((e) => e.type),
|
|
228
|
+
...state.map.entries.map((e) => e.type),
|
|
229
|
+
]);
|
|
230
|
+
const [before, after] = state.map.format.split('{n}');
|
|
231
|
+
const tokens = [...types].map(
|
|
232
|
+
(type) =>
|
|
233
|
+
`${escapePattern(
|
|
234
|
+
placeholderToken(type, 1, before)
|
|
235
|
+
)}[1-9][0-9]*${escapePattern(placeholderToken(type, 1, after))}`
|
|
236
|
+
);
|
|
204
237
|
const pattern = new RegExp(
|
|
205
|
-
|
|
206
|
-
.replace(/\\\{(?:Type|TYPE|type)\\\}/, '[A-Za-z][A-Za-z0-9_]*')
|
|
207
|
-
.replace('\\{n\\}', '[1-9][0-9]*'),
|
|
238
|
+
`(?<![\\p{L}\\p{M}\\p{N}_])(?:${tokens.join('|')})(?![\\p{L}\\p{M}\\p{N}_])`,
|
|
208
239
|
'u'
|
|
209
240
|
);
|
|
210
241
|
if (namespaceCollision(text, pattern)) {
|
|
@@ -237,7 +268,7 @@ function namespaceCollision(text, pattern) {
|
|
|
237
268
|
return false;
|
|
238
269
|
}
|
|
239
270
|
|
|
240
|
-
function canonicalPerson(value, people) {
|
|
271
|
+
export function canonicalPerson(value, people) {
|
|
241
272
|
// Resolve components only when there is a unique compatible full name.
|
|
242
273
|
const matches = people.filter((person) => {
|
|
243
274
|
const words = person.match(/\p{L}+/gu) ?? [];
|
|
@@ -255,6 +286,18 @@ function canonicalPerson(value, people) {
|
|
|
255
286
|
);
|
|
256
287
|
return distinct.size === 1 ? [...distinct.values()][0] : value;
|
|
257
288
|
}
|
|
289
|
+
function canonicalKnown(value, type, options, state) {
|
|
290
|
+
const known = (options.knownPersonal ?? []).filter(
|
|
291
|
+
(e) => e.type === type && samePersonal(value, e.value, type)
|
|
292
|
+
);
|
|
293
|
+
// `Петров Данил` and `Петров Данил Сергеевич` are one person: the fullest wins.
|
|
294
|
+
const longest = Math.max(0, ...known.map((e) => e.value.length));
|
|
295
|
+
const fullest = known.filter((e) => e.value.length === longest);
|
|
296
|
+
if (fullest.length === 1) {
|
|
297
|
+
return fullest[0].value;
|
|
298
|
+
}
|
|
299
|
+
return type === 'PERSON' ? canonicalPerson(value, state.people ?? []) : value;
|
|
300
|
+
}
|
|
258
301
|
export function placeholderReplacement(value, finding, transform, options) {
|
|
259
302
|
if (finding.category === 'credential' && !transform.credentials) {
|
|
260
303
|
return REDACTED;
|
|
@@ -270,12 +313,16 @@ export function placeholderReplacement(value, finding, transform, options) {
|
|
|
270
313
|
: finding.type === 'DATE_OF_BIRTH'
|
|
271
314
|
? 'DATE'
|
|
272
315
|
: finding.type;
|
|
273
|
-
const canonical =
|
|
274
|
-
type === 'PERSON' ? canonicalPerson(value, state.people ?? []) : value;
|
|
316
|
+
const canonical = canonicalKnown(value, type, options, state);
|
|
275
317
|
const candidates = map.entries.filter(
|
|
276
318
|
(entry) => entry.type === type && related(canonical, entry)
|
|
277
319
|
);
|
|
278
|
-
|
|
320
|
+
// Several related entities are ambiguous, unless one holds this exact value.
|
|
321
|
+
const exact = candidates.filter((entry) =>
|
|
322
|
+
entry.values.some((original) => samePersonal(canonical, original, type))
|
|
323
|
+
);
|
|
324
|
+
const matches = candidates.length === 1 ? candidates : exact;
|
|
325
|
+
let entry = matches.length === 1 ? matches[0] : undefined;
|
|
279
326
|
if (!entry) {
|
|
280
327
|
if (map.entries.length >= (options.maxFindings ?? 100000)) {
|
|
281
328
|
throw failure('ERR_LIMIT');
|
package/src/projection.js
CHANGED
|
@@ -27,8 +27,12 @@ export function projectText(input) {
|
|
|
27
27
|
!/\x1b\[|\\u[\da-fA-F]{4}|[\u200b-\u200f\u202a-\u202e\u2060-\u206f\ufeff]/u.test(
|
|
28
28
|
input
|
|
29
29
|
) &&
|
|
30
|
-
|
|
31
|
-
|
|
30
|
+
// ASCII already has normalized letters and digits. Avoid constructing a
|
|
31
|
+
// second multi-megabyte string just to check its decimal characters.
|
|
32
|
+
// eslint-disable-next-line no-control-regex -- Check the ASCII range.
|
|
33
|
+
(!/[^\x00-\x7f]/.test(input) ||
|
|
34
|
+
(input.normalize('NFKC') === input &&
|
|
35
|
+
!/[^0-9]/.test(input.replace(/[^\p{Nd}]/gu, ''))))
|
|
32
36
|
) {
|
|
33
37
|
return { text: input, changed: false };
|
|
34
38
|
}
|
package/src/propagation.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { personalPattern, maxPropagationLength, failure } from './detection.js';
|
|
2
2
|
import { actionable } from './confidence.js';
|
|
3
|
+
import { organizationCore, organizationFamily } from './personal-variants.js';
|
|
3
4
|
import { nameAliases } from './transliteration.js';
|
|
4
5
|
|
|
5
6
|
function mrzSeeds(value, finding, seeds) {
|
|
@@ -32,11 +33,13 @@ function decodedValue(value) {
|
|
|
32
33
|
|
|
33
34
|
function eligibleSeed(finding, options) {
|
|
34
35
|
return (
|
|
36
|
+
!options.knownOnly &&
|
|
35
37
|
finding.category === 'personal' &&
|
|
36
38
|
finding.type !== 'ENCODED_SENSITIVE' &&
|
|
37
39
|
!['confirmed-personal', 'document-propagation'].includes(finding.rule) &&
|
|
38
40
|
actionable(finding, options) &&
|
|
39
|
-
(
|
|
41
|
+
((options.documentEntities && finding.type === 'ORGANIZATION') ||
|
|
42
|
+
finding.confidence >= 0.85 ||
|
|
40
43
|
[
|
|
41
44
|
'PERSON',
|
|
42
45
|
'PASSPORT_NUMBER',
|
|
@@ -47,6 +50,18 @@ function eligibleSeed(finding, options) {
|
|
|
47
50
|
].includes(finding.type))
|
|
48
51
|
);
|
|
49
52
|
}
|
|
53
|
+
function organizationSeed(text, value, finding, seeds, options) {
|
|
54
|
+
if (options.documentEntities && finding.type === 'ORGANIZATION') {
|
|
55
|
+
const core = organizationCore(value);
|
|
56
|
+
if (core.length >= 3) {
|
|
57
|
+
seeds.set(core, finding);
|
|
58
|
+
}
|
|
59
|
+
const family = organizationFamily(core);
|
|
60
|
+
for (const match of family ? text.matchAll(family.pattern) : []) {
|
|
61
|
+
seeds.set(match[0], finding);
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
}
|
|
50
65
|
// Only confirmed values seed propagation. A low-score numeric coincidence
|
|
51
66
|
// must never turn every log occurrence into a high-score identity.
|
|
52
67
|
export function confirmedSeeds(text, findings, options) {
|
|
@@ -71,6 +86,7 @@ export function confirmedSeeds(text, findings, options) {
|
|
|
71
86
|
mrzSeeds(value, finding, seeds);
|
|
72
87
|
} else if (value.length >= 2) {
|
|
73
88
|
seeds.set(value, finding);
|
|
89
|
+
organizationSeed(text, value, finding, seeds, options);
|
|
74
90
|
if (finding.type === 'PERSON') {
|
|
75
91
|
for (const name of value.match(/\p{L}{2,}/gu) ?? []) {
|
|
76
92
|
seeds.set(name, finding);
|
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
// Finite Russian declension shared by fake names and fake entities. Case order:
|
|
2
|
+
// nominative, genitive, dative, accusative, instrumental, prepositional.
|
|
3
|
+
// `|` lists a rarer spelling, accepted when reading a form, never generated.
|
|
4
|
+
const fleeting = { павел: 'павл', лев: 'льв', ветер: 'ветр' };
|
|
5
|
+
const hush = /[жчшщц]$/u;
|
|
6
|
+
const a = (stem) => [
|
|
7
|
+
'а',
|
|
8
|
+
/[гкхжчшщ]$/u.test(stem) ? 'и' : 'ы',
|
|
9
|
+
'е',
|
|
10
|
+
'у',
|
|
11
|
+
/[жчшщц]$/u.test(stem) ? 'ей|ею' : 'ой|ою',
|
|
12
|
+
'е',
|
|
13
|
+
];
|
|
14
|
+
const adjectives = [
|
|
15
|
+
[
|
|
16
|
+
/[ый]й$|ой$/u,
|
|
17
|
+
2,
|
|
18
|
+
(w) => [w.slice(-2), 'ого', 'ому', w.slice(-2), 'ым', 'ом'],
|
|
19
|
+
],
|
|
20
|
+
[/[кгхжчшщ]ий$/u, 2, () => ['ий', 'ого', 'ому', 'ий', 'им', 'ом']],
|
|
21
|
+
[/ий$/u, 2, () => ['ий', 'его', 'ему', 'ий', 'им', 'ем']],
|
|
22
|
+
[/ая$/u, 2, () => ['ая', 'ой', 'ой', 'ую', 'ой|ою', 'ой']],
|
|
23
|
+
[/яя$/u, 2, () => ['яя', 'ей', 'ей', 'юю', 'ей|ею', 'ей']],
|
|
24
|
+
[/ое$/u, 2, () => ['ое', 'ого', 'ому', 'ое', 'ым', 'ом']],
|
|
25
|
+
[/ее$/u, 2, () => ['ее', 'его', 'ему', 'ее', 'им', 'ем']],
|
|
26
|
+
];
|
|
27
|
+
function noun(word, sex, animate) {
|
|
28
|
+
const accusative = (genitive, nominative = '') =>
|
|
29
|
+
animate ? genitive : nominative;
|
|
30
|
+
if (fleeting[word]) {
|
|
31
|
+
const hard = ['а', 'у', accusative('а'), 'ом', 'е'];
|
|
32
|
+
return [word, ...hard.map((ending) => fleeting[word] + ending)];
|
|
33
|
+
}
|
|
34
|
+
const table = [
|
|
35
|
+
[/ия$/u, 2, ['ия', 'ии', 'ии', 'ию', 'ией|иею', 'ии']],
|
|
36
|
+
[/ье$/u, 2, ['ье', 'ья', 'ью', 'ье', 'ьем', 'ье']],
|
|
37
|
+
[/ие$/u, 2, ['ие', 'ия', 'ию', 'ие', 'ием', 'ии']],
|
|
38
|
+
[/ий$/u, 2, ['ий', 'ия', 'ию', accusative('ия', 'ий'), 'ием', 'ии']],
|
|
39
|
+
[/й$/u, 1, ['й', 'я', 'ю', accusative('я', 'й'), 'ем', 'е']],
|
|
40
|
+
[/я$/u, 1, ['я', 'и', 'е', 'ю', 'ей|ею', 'е']],
|
|
41
|
+
[/а$/u, 1, a(word.slice(0, -1))],
|
|
42
|
+
[/о$/u, 1, ['о', 'а', 'у', 'о', 'ом', 'е']],
|
|
43
|
+
];
|
|
44
|
+
for (const [ending, remove, cases] of table) {
|
|
45
|
+
if (ending.test(word)) {
|
|
46
|
+
return cases.map((suffix) => attach(word.slice(0, -remove), suffix));
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
if (word.endsWith('ь')) {
|
|
50
|
+
const stem = word.slice(0, -1);
|
|
51
|
+
const feminine = ['ь', 'и', 'и', 'ь', 'ью', 'и'];
|
|
52
|
+
const masculine = ['ь', 'я', 'ю', accusative('я', 'ь'), 'ем', 'е'];
|
|
53
|
+
const cases =
|
|
54
|
+
sex === 'female'
|
|
55
|
+
? feminine
|
|
56
|
+
: sex === 'male'
|
|
57
|
+
? masculine
|
|
58
|
+
: masculine.map((suffix, i) =>
|
|
59
|
+
suffix === feminine[i] ? suffix : `${suffix}|${feminine[i]}`
|
|
60
|
+
);
|
|
61
|
+
return cases.map((suffix) => attach(stem, suffix));
|
|
62
|
+
}
|
|
63
|
+
if (!/[бвгджзклмнпрстфхцчшщ]$/u.test(word)) {
|
|
64
|
+
return Array(6).fill(word); // Indeclinable: Шевченко, Сочи.
|
|
65
|
+
}
|
|
66
|
+
return [
|
|
67
|
+
'',
|
|
68
|
+
'а',
|
|
69
|
+
'у',
|
|
70
|
+
accusative('а'),
|
|
71
|
+
hush.test(word) ? 'ем' : 'ом',
|
|
72
|
+
'е',
|
|
73
|
+
].map((suffix) => attach(word, suffix));
|
|
74
|
+
}
|
|
75
|
+
const attach = (stem, suffix) =>
|
|
76
|
+
suffix
|
|
77
|
+
.split('|')
|
|
78
|
+
.map((ending) => stem + ending)
|
|
79
|
+
.join('|');
|
|
80
|
+
function adjective(word) {
|
|
81
|
+
for (const [ending, remove, cases] of adjectives) {
|
|
82
|
+
if (ending.test(word)) {
|
|
83
|
+
return cases(word).map((suffix) =>
|
|
84
|
+
attach(word.slice(0, -remove), suffix)
|
|
85
|
+
);
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
return undefined;
|
|
89
|
+
}
|
|
90
|
+
function surname(word, sex) {
|
|
91
|
+
if (/(?:ов|ев|ёв|ин|ын)$/u.test(word)) {
|
|
92
|
+
return ['', 'а', 'у', 'а', 'ым|ом', 'е'].map((s) => attach(word, s));
|
|
93
|
+
}
|
|
94
|
+
if (/(?:ова|ева|ёва|ина|ына)$/u.test(word)) {
|
|
95
|
+
const stem = word.slice(0, -1);
|
|
96
|
+
return ['а', 'ой', 'ой', 'у', 'ой|ою', 'ой'].map((s) => attach(stem, s));
|
|
97
|
+
}
|
|
98
|
+
const forms = adjective(word);
|
|
99
|
+
if (forms) {
|
|
100
|
+
return forms;
|
|
101
|
+
}
|
|
102
|
+
return sex === 'female' && !/[ая]$/u.test(word)
|
|
103
|
+
? Array(6).fill(word)
|
|
104
|
+
: noun(word, sex, true);
|
|
105
|
+
}
|
|
106
|
+
export const russianNoun = (word, options) =>
|
|
107
|
+
options.adjective && adjective(word)
|
|
108
|
+
? adjective(word)
|
|
109
|
+
: noun(word, options.sex, options.animate);
|
|
110
|
+
// `kind` is the name slot: lastName, firstName or middleName.
|
|
111
|
+
export function russianNameForms(word, kind, sex) {
|
|
112
|
+
const lower = word.toLowerCase();
|
|
113
|
+
if (kind === 'lastName') {
|
|
114
|
+
return surname(lower, sex);
|
|
115
|
+
}
|
|
116
|
+
if (kind === 'middleName' && /ич$/u.test(lower)) {
|
|
117
|
+
return ['', 'а', 'у', 'а', 'ем', 'е'].map((s) => attach(lower, s));
|
|
118
|
+
}
|
|
119
|
+
return noun(lower, sex, true);
|
|
120
|
+
}
|
|
121
|
+
const fold = (word) => word.toLowerCase().replaceAll('ё', 'е');
|
|
122
|
+
// Every case a surface can stand for, in case order.
|
|
123
|
+
export function casesOf(surface, forms) {
|
|
124
|
+
const wanted = fold(surface);
|
|
125
|
+
return forms.flatMap((form, index) =>
|
|
126
|
+
fold(form).split('|').includes(wanted) ? [index] : []
|
|
127
|
+
);
|
|
128
|
+
}
|
|
129
|
+
export const formOf = (forms, index) => forms[index].split('|')[0];
|
|
130
|
+
// Cases governed by a preposition, in order of preference. Bare forms like
|
|
131
|
+
// `Ромашке` (dative or prepositional) cannot be told apart without it.
|
|
132
|
+
const governed = [
|
|
133
|
+
[/^(?:в|во|на|о|об|обо|при)$/u, [5, 3]],
|
|
134
|
+
[/^(?:к|ко|по)$/u, [2]],
|
|
135
|
+
[/^(?:с|со|над|под|перед|за)$/u, [4, 1, 3]],
|
|
136
|
+
[/^(?:у|от|до|из|без|для|около|после|кроме|вокруг|против)$/u, [1]],
|
|
137
|
+
];
|
|
138
|
+
const lastWord = /(?<![\p{L}\p{M}\p{N}_-])(\p{L}{1,7})[ \t\u00a0]+([«“"]?)$/u;
|
|
139
|
+
// `в Лютике и Ромашке`: a conjunct takes the case of the word before it.
|
|
140
|
+
const conjunct =
|
|
141
|
+
/[\p{L}\p{M}-]{1,32}[»”"]?(?:,|[ \t\u00a0]+(?:и|или))[ \t\u00a0]+[«“"]?$/u;
|
|
142
|
+
export function governedCases(text, start) {
|
|
143
|
+
const before = text.slice(Math.max(0, start - 64), start);
|
|
144
|
+
const conjunction = conjunct.exec(before);
|
|
145
|
+
const source = conjunction ? before.slice(0, conjunction.index) : before;
|
|
146
|
+
const match = lastWord.exec(source.slice(-12));
|
|
147
|
+
if (!match) {
|
|
148
|
+
return { cases: [] };
|
|
149
|
+
}
|
|
150
|
+
const word = match[1].toLowerCase();
|
|
151
|
+
const rule = governed.find(([pattern]) => pattern.test(word));
|
|
152
|
+
return { cases: rule?.[1] ?? [], quoted: Boolean(match[2]) };
|
|
153
|
+
}
|
|
154
|
+
// Intersects the candidates of each word; a word that fits no case is ignored.
|
|
155
|
+
// A quoted name in the nominative stays there: в «Вектор».
|
|
156
|
+
export function commonCase(candidates, context = { cases: [] }) {
|
|
157
|
+
const present = candidates.filter((cases) => cases.length);
|
|
158
|
+
const common = [0, 1, 2, 3, 4, 5].filter((index) =>
|
|
159
|
+
present.every((cases) => cases.includes(index))
|
|
160
|
+
);
|
|
161
|
+
if (context.quoted && common.includes(0)) {
|
|
162
|
+
return 0;
|
|
163
|
+
}
|
|
164
|
+
return (
|
|
165
|
+
context.cases.find((index) => common.includes(index)) ?? common[0] ?? 0
|
|
166
|
+
);
|
|
167
|
+
}
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import { firstNames } from './names.js';
|
|
2
|
+
|
|
3
|
+
// Finite Russian name grammar shared by known-value matching and fake mode.
|
|
4
|
+
const patronymic = /(?:ич|овна|евна|ична|vich|ovna|evna)$/iu;
|
|
5
|
+
const surname = /(?:ов|ев|ин|ын|ский|цкий|ова|ева|ина|ына|ская|цкая)$/iu;
|
|
6
|
+
const fold = (word) => word.toLowerCase().replaceAll('ё', 'е');
|
|
7
|
+
const givenNames = new Set(firstNames.ru.split(' ').map(fold));
|
|
8
|
+
|
|
9
|
+
export const isPatronymic = (word) => patronymic.test(word);
|
|
10
|
+
export const isGivenName = (word) => givenNames.has(fold(word));
|
|
11
|
+
|
|
12
|
+
// Returns word indexes by role, or undefined when the roles are not evident.
|
|
13
|
+
// `Имя Отчество Фамилия` is checked first so that surnames in -ич, as in
|
|
14
|
+
// `Шостакович Дмитрий Дмитриевич`, are not mistaken for patronymics.
|
|
15
|
+
export function russianNameRoles(words, known = isGivenName) {
|
|
16
|
+
if (words.length === 3) {
|
|
17
|
+
if (isPatronymic(words[1])) {
|
|
18
|
+
return { given: 0, patronymic: 1, surname: 2 };
|
|
19
|
+
}
|
|
20
|
+
return isPatronymic(words[2])
|
|
21
|
+
? { surname: 0, given: 1, patronymic: 2 }
|
|
22
|
+
: undefined;
|
|
23
|
+
}
|
|
24
|
+
if (words.length !== 2) {
|
|
25
|
+
return undefined;
|
|
26
|
+
}
|
|
27
|
+
if (isPatronymic(words[1])) {
|
|
28
|
+
return { given: 0, patronymic: 1 };
|
|
29
|
+
}
|
|
30
|
+
const candidates = [0, 1].filter(
|
|
31
|
+
(i) => surname.test(words[i]) && !known(words[i])
|
|
32
|
+
);
|
|
33
|
+
return candidates.length === 1
|
|
34
|
+
? { surname: candidates[0], given: 1 - candidates[0] }
|
|
35
|
+
: undefined;
|
|
36
|
+
}
|
package/src/sanitizer.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { generatedValues } from './fake.js';
|
|
2
|
+
import { prepareFakeText } from './fake-entities.js';
|
|
2
3
|
import { alignEscapes } from './escape-spans.js';
|
|
3
4
|
import { actionable } from './confidence.js';
|
|
4
5
|
import { propagate, confirmationOptions } from './propagation.js';
|
|
@@ -45,6 +46,7 @@ const publicTypes = new Set([
|
|
|
45
46
|
]);
|
|
46
47
|
const optionNames = new Set([
|
|
47
48
|
'credentialsOnly',
|
|
49
|
+
'knownOnly',
|
|
48
50
|
'knownSecrets',
|
|
49
51
|
'knownPersonal',
|
|
50
52
|
'confirmedPersonal',
|
|
@@ -166,6 +168,7 @@ export function validateOptions(options = {}) {
|
|
|
166
168
|
for (const key of [
|
|
167
169
|
'paranoid',
|
|
168
170
|
'credentialsOnly',
|
|
171
|
+
'knownOnly',
|
|
169
172
|
'decode',
|
|
170
173
|
'secretlint',
|
|
171
174
|
'publicKnowledge',
|
|
@@ -690,11 +693,12 @@ export function redactResult(
|
|
|
690
693
|
) {
|
|
691
694
|
options = applyProfile(options);
|
|
692
695
|
const spans = validatedSpans(text, findings, options);
|
|
693
|
-
options = initializePlaceholders(options);
|
|
696
|
+
options = prepareFakeText(text, findings, initializePlaceholders(options));
|
|
694
697
|
preparePlaceholderText(text, findings, options);
|
|
695
698
|
const parts = [];
|
|
696
699
|
const faked = [];
|
|
697
700
|
const occurrences = [];
|
|
701
|
+
let irreversibleRedactions = 0;
|
|
698
702
|
let outputLength = 0;
|
|
699
703
|
let cursor = 0;
|
|
700
704
|
for (const span of spans) {
|
|
@@ -716,6 +720,11 @@ export function redactResult(
|
|
|
716
720
|
const transform =
|
|
717
721
|
options.transformations?.[span.type] ?? options.transformation;
|
|
718
722
|
if (retainedReplacement(transform, 'placeholder', span, replacement)) {
|
|
723
|
+
// `Organization-1` followed by `99` would read as a token that does not
|
|
724
|
+
// exist, so a span that ends there fails the run instead.
|
|
725
|
+
if (/\d$/.test(replacement) && /\d/.test(text[span.end] ?? '')) {
|
|
726
|
+
throw failure('ERR_RESIDUAL');
|
|
727
|
+
}
|
|
719
728
|
occurrences.push({
|
|
720
729
|
start: outputLength,
|
|
721
730
|
end: outputLength + replacement.length,
|
|
@@ -723,6 +732,12 @@ export function redactResult(
|
|
|
723
732
|
raw: text.slice(span.start, span.end),
|
|
724
733
|
});
|
|
725
734
|
}
|
|
735
|
+
if (
|
|
736
|
+
hasPlaceholders(options) &&
|
|
737
|
+
!retainedReplacement(transform, 'placeholder', span, replacement)
|
|
738
|
+
) {
|
|
739
|
+
irreversibleRedactions++;
|
|
740
|
+
}
|
|
726
741
|
outputLength += replacement.length;
|
|
727
742
|
if (retainedReplacement(transform, 'fake', span, replacement)) {
|
|
728
743
|
faked.push(span);
|
|
@@ -731,7 +746,11 @@ export function redactResult(
|
|
|
731
746
|
}
|
|
732
747
|
parts.push(text.slice(cursor));
|
|
733
748
|
const result = finishPlaceholders(
|
|
734
|
-
{
|
|
749
|
+
{
|
|
750
|
+
text: parts.join(''),
|
|
751
|
+
redactions: spans.length,
|
|
752
|
+
...(hasPlaceholders(options) ? { irreversibleRedactions } : {}),
|
|
753
|
+
},
|
|
735
754
|
occurrences,
|
|
736
755
|
options
|
|
737
756
|
);
|
|
@@ -861,6 +880,13 @@ export function redactAgain(result, findings, options, round, rerun) {
|
|
|
861
880
|
const merge = (next) => ({
|
|
862
881
|
...next,
|
|
863
882
|
redactions: result.redactions + next.redactions,
|
|
883
|
+
...(hasPlaceholders(options)
|
|
884
|
+
? {
|
|
885
|
+
irreversibleRedactions:
|
|
886
|
+
(result.irreversibleRedactions ?? 0) +
|
|
887
|
+
(next.irreversibleRedactions ?? 0),
|
|
888
|
+
}
|
|
889
|
+
: {}),
|
|
864
890
|
findings: auditFaked(findings, options, result),
|
|
865
891
|
});
|
|
866
892
|
const next = rerun({ ...options, findings: [] }, round + 1);
|
package/src/transforms.js
CHANGED
|
@@ -41,6 +41,22 @@ export function validateTransforms(options) {
|
|
|
41
41
|
function positiveInteger(value) {
|
|
42
42
|
return Number.isSafeInteger(value) && value > 0;
|
|
43
43
|
}
|
|
44
|
+
const poolTypes = new Set(['ORGANIZATION', 'LOCATION', 'EDUCATION']);
|
|
45
|
+
function validPools(pools) {
|
|
46
|
+
return (
|
|
47
|
+
pools === undefined ||
|
|
48
|
+
(Boolean(pools) &&
|
|
49
|
+
typeof pools === 'object' &&
|
|
50
|
+
!Array.isArray(pools) &&
|
|
51
|
+
Object.entries(pools).every(
|
|
52
|
+
([type, names]) =>
|
|
53
|
+
poolTypes.has(type) &&
|
|
54
|
+
Array.isArray(names) &&
|
|
55
|
+
names.length > 0 &&
|
|
56
|
+
names.every((name) => typeof name === 'string' && name.trim())
|
|
57
|
+
))
|
|
58
|
+
);
|
|
59
|
+
}
|
|
44
60
|
function validateTransform(transform) {
|
|
45
61
|
if (!transform || !modes.has(transform.mode)) {
|
|
46
62
|
throw failure('ERR_CONFIG');
|
|
@@ -54,7 +70,8 @@ function validateTransform(transform) {
|
|
|
54
70
|
Buffer.byteLength(transform.key) >= 16 &&
|
|
55
71
|
(transform.credentials === undefined ||
|
|
56
72
|
typeof transform.credentials === 'boolean') &&
|
|
57
|
-
(transform.mrzCountry === undefined || transform.mrzCountry === 'UTO')
|
|
73
|
+
(transform.mrzCountry === undefined || transform.mrzCountry === 'UTO') &&
|
|
74
|
+
validPools(transform.pools),
|
|
58
75
|
pseudonym: () =>
|
|
59
76
|
typeof transform.key === 'string' &&
|
|
60
77
|
Buffer.byteLength(transform.key) >= 16,
|
|
@@ -130,7 +147,7 @@ function masked(value, finding, transform) {
|
|
|
130
147
|
? REDACTED
|
|
131
148
|
: `${chars.slice(0, start).join('')}${transform.marker ?? '*'.repeat(chars.length - start - end)}${end ? chars.slice(-end).join('') : ''}`;
|
|
132
149
|
}
|
|
133
|
-
function transformed(value, finding, transform) {
|
|
150
|
+
function transformed(value, finding, transform, options) {
|
|
134
151
|
if (!transform || transform.mode === 'redact') {
|
|
135
152
|
return REDACTED;
|
|
136
153
|
}
|
|
@@ -141,7 +158,7 @@ function transformed(value, finding, transform) {
|
|
|
141
158
|
: REDACTED;
|
|
142
159
|
}
|
|
143
160
|
if (transform.mode === 'fake') {
|
|
144
|
-
return realisticFake(value, finding, transform);
|
|
161
|
+
return realisticFake(value, finding, transform, options);
|
|
145
162
|
}
|
|
146
163
|
if (finding.category === 'credential') {
|
|
147
164
|
return REDACTED;
|
|
@@ -198,6 +215,6 @@ function renderValue(text, finding, transform, options) {
|
|
|
198
215
|
const result =
|
|
199
216
|
transform?.mode === 'placeholder'
|
|
200
217
|
? placeholderReplacement(raw, finding, transform, options)
|
|
201
|
-
: transformed(raw, finding, transform);
|
|
218
|
+
: transformed(raw, finding, transform, options);
|
|
202
219
|
return jsonQuoted ? JSON.stringify(result).slice(1, -1) : result;
|
|
203
220
|
}
|