@link-foundation/sensitive-data-sanitizer 0.12.2 → 0.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/README.md +44 -0
- package/bin/placeholders.js +115 -0
- package/bin/sensitive-data-sanitizer.js +47 -16
- package/bin/stream.js +2 -1
- package/docs/API.md +130 -21
- package/docs/COVERAGE.md +50 -0
- package/examples/reversible-placeholders.mjs +35 -0
- package/package.json +4 -2
- package/src/context.js +23 -9
- package/src/detection.js +26 -25
- package/src/document-entities.js +110 -0
- package/src/document-vocabulary.js +71 -0
- package/src/engines.js +42 -18
- package/src/entities.js +7 -0
- package/src/fake-names.js +61 -7
- package/src/fake.js +7 -0
- package/src/index.d.ts +55 -2
- package/src/index.js +8 -0
- package/src/json-string-runs.js +18 -0
- package/src/jsonl-stream.js +5 -0
- package/src/keywords.js +17 -0
- package/src/names.js +1 -1
- package/src/personal-variants.js +94 -0
- package/src/personal.js +6 -1
- package/src/placeholder-types.js +33 -0
- package/src/placeholders.js +405 -0
- package/src/projection.js +6 -2
- package/src/propagation.js +13 -1
- package/src/restore-map.js +220 -0
- package/src/sanitizer.js +119 -18
- package/src/stream.js +23 -2
- package/src/structured.js +5 -1
- package/src/transforms.js +12 -1
package/src/json-string-runs.js
CHANGED
|
@@ -40,3 +40,21 @@ export function* jsonStringRuns(text) {
|
|
|
40
40
|
}
|
|
41
41
|
}
|
|
42
42
|
}
|
|
43
|
+
|
|
44
|
+
// Complete JSON literals for callers whose input already has an outer bound.
|
|
45
|
+
// Skip escaped characters once, including escaped quotes, with constant state.
|
|
46
|
+
export function* jsonStringLiterals(text) {
|
|
47
|
+
let start = -1;
|
|
48
|
+
for (let i = 0; i < text.length; i++) {
|
|
49
|
+
if (start < 0) {
|
|
50
|
+
if (text[i] === '"') {
|
|
51
|
+
start = i;
|
|
52
|
+
}
|
|
53
|
+
} else if (text[i] === '\\') {
|
|
54
|
+
i++;
|
|
55
|
+
} else if (text[i] === '"') {
|
|
56
|
+
yield text.slice(start, i + 1);
|
|
57
|
+
start = -1;
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
}
|
package/src/jsonl-stream.js
CHANGED
|
@@ -6,6 +6,7 @@ import { encodingFailure } from './utf8.js';
|
|
|
6
6
|
import { workerExecArgv } from './worker-options.js';
|
|
7
7
|
import { prepareStreamEngine } from './engines.js';
|
|
8
8
|
import { inspect } from './sanitizer.js';
|
|
9
|
+
import { hasPlaceholders } from './placeholders.js';
|
|
9
10
|
|
|
10
11
|
async function* records(source, { maxRecordBytes, maxTotalBytes }) {
|
|
11
12
|
const decoder = new TextDecoder('utf-8', { fatal: true, ignoreBOM: true });
|
|
@@ -123,6 +124,10 @@ function workerSlot(options, heap, timeout) {
|
|
|
123
124
|
};
|
|
124
125
|
}
|
|
125
126
|
function workerCount(options, config) {
|
|
127
|
+
// Stateful numbering must run in the coordinator, in source order.
|
|
128
|
+
if (hasPlaceholders(config.sanitizerOptions)) {
|
|
129
|
+
return 1;
|
|
130
|
+
}
|
|
126
131
|
const callbacks =
|
|
127
132
|
options.sanitizer ||
|
|
128
133
|
config.sanitizerOptions.debug ||
|
package/src/keywords.js
CHANGED
|
@@ -27,10 +27,27 @@ export function keywordMatcher(words) {
|
|
|
27
27
|
queue.push(child);
|
|
28
28
|
}
|
|
29
29
|
}
|
|
30
|
+
// At the root, characters that cannot start any keyword need no state
|
|
31
|
+
// transitions. A native search skips large padding runs without Map work
|
|
32
|
+
// per character. Escape UTF-16 units so metacharacters retain literal meaning.
|
|
33
|
+
const starters = new RegExp(
|
|
34
|
+
`[${[...nodes[0].next.keys()]
|
|
35
|
+
.map((char) => `\\u${char.charCodeAt(0).toString(16).padStart(4, '0')}`)
|
|
36
|
+
.join('')}]`,
|
|
37
|
+
'g'
|
|
38
|
+
);
|
|
30
39
|
return (text) => {
|
|
31
40
|
const found = new Set();
|
|
32
41
|
let state = 0;
|
|
33
42
|
for (let i = 0; i < text.length; i++) {
|
|
43
|
+
if (!state && !nodes[0].output.length && !nodes[0].next.has(text[i])) {
|
|
44
|
+
starters.lastIndex = i + 1;
|
|
45
|
+
const next = starters.exec(text);
|
|
46
|
+
if (!next) {
|
|
47
|
+
break;
|
|
48
|
+
}
|
|
49
|
+
i = next.index;
|
|
50
|
+
}
|
|
34
51
|
const char = text[i];
|
|
35
52
|
while (state && !nodes[state].next.has(char)) {
|
|
36
53
|
state = nodes[state].fail;
|
package/src/names.js
CHANGED
|
@@ -12,7 +12,7 @@ export const firstNames = {
|
|
|
12
12
|
jaLatn: 'Yamada Sato Suzuki Takahashi Tanaka Ito Watanabe Nakamura Kobayashi',
|
|
13
13
|
zhLatn:
|
|
14
14
|
'Zhang Wang Li Zhao Liu Chen Yang Huang Zhou Wu Xu Sun Hu Zhu Gao Lin',
|
|
15
|
-
ru: 'Иван Пётр Петр Сергей Александр Алексей Дмитрий Михаил Николай Андрей Владимир Павел Виктор Олег Борис Евгений Василий Юрий Марина Анна Мария Елена Ольга Наталья Ирина Светлана Татьяна Екатерина Анастасия Юлия Ивану Ивана Иваном Сергея Сергею Александра Александру Анне Анну Елены',
|
|
15
|
+
ru: 'Данил Максим Алина Иван Пётр Петр Сергей Александр Алексей Дмитрий Михаил Николай Андрей Владимир Павел Виктор Олег Борис Евгений Василий Юрий Марина Анна Мария Елена Ольга Наталья Ирина Светлана Татьяна Екатерина Анастасия Юлия Ивану Ивана Иваном Сергея Сергею Александра Александру Анне Анну Елены',
|
|
16
16
|
ja: '山田 佐藤 鈴木 高橋 田中 伊藤 渡辺 渡邊 中村 小林 加藤 吉田 松本 山本 井上 木村 林 清水 斎藤 斉藤',
|
|
17
17
|
zh: '张 王 李 赵 刘 陈 杨 黄 周 吴 徐 孙 胡 朱 高 林 何 郭 马 罗 張 趙 劉 陳 楊 黃 吳 孫 馬 羅',
|
|
18
18
|
ko: '김 이 박 최 정 강 조 윤 장 임 한 오 서 신 권 황 안 송 전 홍 유 고 문 양 손 배 백',
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
import { escapePattern, failure } from './detection.js';
|
|
2
|
+
|
|
3
|
+
const boundary = '[\\p{L}\\p{M}\\p{N}_]';
|
|
4
|
+
export function organizationCore(value) {
|
|
5
|
+
return value
|
|
6
|
+
.replace(/^(?:ООО|АО|ПАО|ИП|ЗАО)[ \t]+/u, '')
|
|
7
|
+
.replace(/^[«“"]|[»”"]$/gu, '')
|
|
8
|
+
.trim();
|
|
9
|
+
}
|
|
10
|
+
// Only caller-confirmed Russian words receive finite case alternatives. A
|
|
11
|
+
// minimum three-letter stem avoids turning short names into broad matches.
|
|
12
|
+
function wordPattern(word) {
|
|
13
|
+
const endings = [
|
|
14
|
+
[/(?:ова|ева|ина)$/u, 1, '(?:а|ой|ою|у|е|ы)'],
|
|
15
|
+
[/(?:ая)$/u, 2, '(?:ая|ой|ою|ую)'],
|
|
16
|
+
[
|
|
17
|
+
/(?:ий|ый|ой)$/u,
|
|
18
|
+
2,
|
|
19
|
+
'(?:ий|ый|ой|ого|его|ому|ему|им|ым|ом|ем|ия|ию|ием|ии)',
|
|
20
|
+
],
|
|
21
|
+
[/(?:ов|ев|ин)$/u, 0, '(?:а|у|ым|ом|е|ой)?'],
|
|
22
|
+
[/а$/u, 1, '(?:а|ы|и|е|у|ой|ою)'],
|
|
23
|
+
[/я$/u, 1, '(?:я|и|е|ю|ей|ею)'],
|
|
24
|
+
[/ь$/u, 1, '(?:ь|я|ю|ем|и)'],
|
|
25
|
+
];
|
|
26
|
+
const lower = word.toLowerCase();
|
|
27
|
+
if (/^[а-яё]+$/u.test(lower)) {
|
|
28
|
+
for (const [ending, remove, suffix] of endings) {
|
|
29
|
+
if (ending.test(lower) && word.length - remove >= 3) {
|
|
30
|
+
return escapePattern(remove ? word.slice(0, -remove) : word) + suffix;
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
if (/^[а-яё]{4,}[бвгджзклмнпрстфхцчшщ]$/u.test(lower)) {
|
|
35
|
+
return `${escapePattern(word)}(?:а|у|ом|е|ы|ов|ам|ами|ах)?`;
|
|
36
|
+
}
|
|
37
|
+
return escapePattern(word);
|
|
38
|
+
}
|
|
39
|
+
export function personalPatterns(value, type) {
|
|
40
|
+
if (value.length > 256) {
|
|
41
|
+
// Large explicit values still support literal matching without expansion.
|
|
42
|
+
return [escapePattern(value)];
|
|
43
|
+
}
|
|
44
|
+
const values = new Set([value]);
|
|
45
|
+
if (type === 'ORGANIZATION') {
|
|
46
|
+
const core = organizationCore(value);
|
|
47
|
+
if (core.length >= 3) {
|
|
48
|
+
values.add(core);
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
const patterns = new Set();
|
|
52
|
+
for (const surface of values) {
|
|
53
|
+
const parts = surface.split(/([ \t]+)/u);
|
|
54
|
+
const pattern = parts
|
|
55
|
+
.map((part, i) => (i % 2 ? '[ \\t]+' : wordPattern(part)))
|
|
56
|
+
.join('');
|
|
57
|
+
patterns.add(pattern);
|
|
58
|
+
if (type === 'PERSON' && parts.length === 3) {
|
|
59
|
+
patterns.add(
|
|
60
|
+
[wordPattern(parts[2]), wordPattern(parts[0])].join('[ \\t]+')
|
|
61
|
+
);
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
return [...patterns];
|
|
65
|
+
}
|
|
66
|
+
export function personalRegex(value, type) {
|
|
67
|
+
return new RegExp(
|
|
68
|
+
`(?<!${boundary})(?:${personalPatterns(value, type).join('|')})(?!${boundary})`,
|
|
69
|
+
'giu'
|
|
70
|
+
);
|
|
71
|
+
}
|
|
72
|
+
export function samePersonal(value, original, type) {
|
|
73
|
+
return personalPatterns(original, type).some((pattern) =>
|
|
74
|
+
new RegExp(`^(?:${pattern})$`, 'iu').test(value)
|
|
75
|
+
);
|
|
76
|
+
}
|
|
77
|
+
export function detectKnownPersonal(text, entries, rule, emit, options) {
|
|
78
|
+
let count = 0;
|
|
79
|
+
for (const entry of entries ?? []) {
|
|
80
|
+
for (const match of text.matchAll(personalRegex(entry.value, entry.type))) {
|
|
81
|
+
if (++count > (options.maxFindings ?? 100000)) {
|
|
82
|
+
throw failure('ERR_LIMIT');
|
|
83
|
+
}
|
|
84
|
+
emit({
|
|
85
|
+
start: match.index,
|
|
86
|
+
end: match.index + match[0].length,
|
|
87
|
+
type: entry.type,
|
|
88
|
+
category: 'personal',
|
|
89
|
+
rule,
|
|
90
|
+
confidence: rule === 'known-personal' ? 0.85 : 0.9,
|
|
91
|
+
});
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
}
|
package/src/personal.js
CHANGED
|
@@ -179,7 +179,12 @@ const russianFull =
|
|
|
179
179
|
/(?<![\p{L}])(?:[А-ЯЁ][а-яё]{2,30}[ \t]+){2}[А-ЯЁ][а-яё]{2,30}(?:вич|вна|ична)|(?<![\p{L}])(?:[А-ЯЁ][а-яё]{2,30}[ \t]+){2}(?:оглы|кызы)(?![\p{L}])/gu;
|
|
180
180
|
export function detectPersonal(text, emit) {
|
|
181
181
|
detectDocumentNames(text, emit);
|
|
182
|
-
for (const pattern of [
|
|
182
|
+
for (const pattern of [
|
|
183
|
+
latinPairs,
|
|
184
|
+
slavicPairs,
|
|
185
|
+
russianFull,
|
|
186
|
+
/(?<![\p{L}])(?:[А-ЯЁ][а-яё]{2,30}[ \t]+)[А-ЯЁ][а-яё]{2,30}(?:вич|вна|ична)[ \t]+[А-ЯЁ][а-яё]{2,30}(?![\p{L}])/gu,
|
|
187
|
+
]) {
|
|
183
188
|
collectMatches(
|
|
184
189
|
text,
|
|
185
190
|
pattern,
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
import { catalogTypes } from './entities.js';
|
|
2
|
+
|
|
3
|
+
// Native unlabelled types supplement the shared vendor-label catalog.
|
|
4
|
+
export const reservedPlaceholderTypes = () => [
|
|
5
|
+
...catalogTypes(),
|
|
6
|
+
'ADDRESS',
|
|
7
|
+
'AU_TFN',
|
|
8
|
+
'BRAZIL_CPF_NUMBER',
|
|
9
|
+
'CREDENTIAL',
|
|
10
|
+
'CREDIT_CARD',
|
|
11
|
+
'DATE',
|
|
12
|
+
'EDUCATION',
|
|
13
|
+
'EMAIL',
|
|
14
|
+
'ENCODED_SENSITIVE',
|
|
15
|
+
'IBAN',
|
|
16
|
+
'ID',
|
|
17
|
+
'IP_ADDRESS',
|
|
18
|
+
'KR_RRN',
|
|
19
|
+
'LOCATION',
|
|
20
|
+
'MAC_ADDRESS',
|
|
21
|
+
'ORGANIZATION',
|
|
22
|
+
'PASSPORT_MRZ',
|
|
23
|
+
'PASSPORT_NUMBER',
|
|
24
|
+
'PERSON',
|
|
25
|
+
'PHONE',
|
|
26
|
+
'PL_PESEL',
|
|
27
|
+
'RU_INN',
|
|
28
|
+
'RU_SNILS',
|
|
29
|
+
'SG_NRIC_FIN',
|
|
30
|
+
'STATE',
|
|
31
|
+
'USERNAME',
|
|
32
|
+
'US_HEALTHCARE_NPI',
|
|
33
|
+
];
|
|
@@ -0,0 +1,405 @@
|
|
|
1
|
+
import { failure, REDACTED, escapePattern } from './detection.js';
|
|
2
|
+
import { reservedPlaceholderTypes } from './placeholder-types.js';
|
|
3
|
+
import { samePersonal } from './personal-variants.js';
|
|
4
|
+
import { nameAliases } from './transliteration.js';
|
|
5
|
+
import { createHash } from 'node:crypto';
|
|
6
|
+
import { jsonStringLiterals } from './json-string-runs.js';
|
|
7
|
+
import {
|
|
8
|
+
createRestoreMap,
|
|
9
|
+
validateRestoreMap,
|
|
10
|
+
placeholderToken,
|
|
11
|
+
documentDigest,
|
|
12
|
+
validatePlaceholderFormat,
|
|
13
|
+
} from './restore-map.js';
|
|
14
|
+
|
|
15
|
+
export const placeholderState = Symbol('placeholder-state');
|
|
16
|
+
const placeholderNested = Symbol('placeholder-nested');
|
|
17
|
+
export const nestedPlaceholderOptions = (options) => ({
|
|
18
|
+
...options,
|
|
19
|
+
[placeholderNested]: true,
|
|
20
|
+
});
|
|
21
|
+
export const hasPlaceholders = (options) =>
|
|
22
|
+
[
|
|
23
|
+
options.transformation,
|
|
24
|
+
...Object.values(options.transformations ?? {}),
|
|
25
|
+
].some((t) => t?.mode === 'placeholder');
|
|
26
|
+
export const placeholderValues = (options) =>
|
|
27
|
+
(options[placeholderState]?.map.entries ?? []).map((entry) => ({
|
|
28
|
+
value: entry.token,
|
|
29
|
+
credential: entry.type === 'CREDENTIAL',
|
|
30
|
+
}));
|
|
31
|
+
const transforms = (options) =>
|
|
32
|
+
[
|
|
33
|
+
options.transformation,
|
|
34
|
+
...Object.values(options.transformations ?? {}),
|
|
35
|
+
].filter((t) => t?.mode === 'placeholder');
|
|
36
|
+
|
|
37
|
+
export function validatePlaceholder(transform) {
|
|
38
|
+
validatePlaceholderFormat(transform.format ?? transform.map?.format);
|
|
39
|
+
if (
|
|
40
|
+
transform.credentials !== undefined &&
|
|
41
|
+
typeof transform.credentials !== 'boolean'
|
|
42
|
+
) {
|
|
43
|
+
throw failure('ERR_CONFIG');
|
|
44
|
+
}
|
|
45
|
+
if (transform.map !== undefined && transform.map !== false) {
|
|
46
|
+
validateRestoreMap(transform.map);
|
|
47
|
+
if (
|
|
48
|
+
transform.format !== undefined &&
|
|
49
|
+
transform.format !== transform.map.format
|
|
50
|
+
) {
|
|
51
|
+
throw failure('ERR_CONFIG');
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
export function initializePlaceholders(options) {
|
|
56
|
+
const selected = transforms(options);
|
|
57
|
+
if (!selected.length || options[placeholderState]) {
|
|
58
|
+
return options;
|
|
59
|
+
}
|
|
60
|
+
const first = selected[0];
|
|
61
|
+
const map =
|
|
62
|
+
first.map === false
|
|
63
|
+
? createRestoreMap({ format: first.format })
|
|
64
|
+
: (first.map ?? createRestoreMap({ format: first.format }));
|
|
65
|
+
if (
|
|
66
|
+
selected.some(
|
|
67
|
+
(t) =>
|
|
68
|
+
(t.map !== undefined && t.map !== first.map) ||
|
|
69
|
+
(t.format !== undefined && t.format !== map.format)
|
|
70
|
+
)
|
|
71
|
+
) {
|
|
72
|
+
throw failure('ERR_CONFIG');
|
|
73
|
+
}
|
|
74
|
+
return { ...options, [placeholderState]: { map, keep: first.map !== false } };
|
|
75
|
+
}
|
|
76
|
+
export function placeholderKnown(options) {
|
|
77
|
+
const map = options[placeholderState]?.map;
|
|
78
|
+
if (!map) {
|
|
79
|
+
return options;
|
|
80
|
+
}
|
|
81
|
+
return {
|
|
82
|
+
...options,
|
|
83
|
+
confirmedPersonal: [
|
|
84
|
+
...(options.confirmedPersonal ?? []),
|
|
85
|
+
...map.entries
|
|
86
|
+
.filter((entry) => entry.type !== 'CREDENTIAL')
|
|
87
|
+
.flatMap((entry) =>
|
|
88
|
+
entry.values
|
|
89
|
+
.filter((value) => value.length <= 256)
|
|
90
|
+
.map((value) => ({ type: entry.type, value }))
|
|
91
|
+
),
|
|
92
|
+
],
|
|
93
|
+
};
|
|
94
|
+
}
|
|
95
|
+
const stem = (word) =>
|
|
96
|
+
word.replace(
|
|
97
|
+
/(?:овым|евым|овой|евой|ова|ева|ову|еву|ове|еве|ов|ев|ovym|evym|ovoi|evoi|ova|eva|ovu|evu|ove|eve|ov|ev)$/u,
|
|
98
|
+
(ending) => (/^(?:ов|ov)/.test(ending) ? 'ov' : 'ev')
|
|
99
|
+
);
|
|
100
|
+
function aliases(value, type) {
|
|
101
|
+
if (type !== 'PERSON') {
|
|
102
|
+
return [value.normalize('NFC').toLowerCase()];
|
|
103
|
+
}
|
|
104
|
+
return [
|
|
105
|
+
...new Set(
|
|
106
|
+
nameAliases(value).map((v) => v.replace(/[\p{L}\p{M}]+/gu, stem))
|
|
107
|
+
),
|
|
108
|
+
];
|
|
109
|
+
}
|
|
110
|
+
function related(value, entry) {
|
|
111
|
+
if (
|
|
112
|
+
(entry.values ?? []).some((original) =>
|
|
113
|
+
samePersonal(value, original, entry.type)
|
|
114
|
+
)
|
|
115
|
+
) {
|
|
116
|
+
return true;
|
|
117
|
+
}
|
|
118
|
+
const candidates = aliases(value, entry.type);
|
|
119
|
+
if (candidates.some((alias) => entry.aliases.includes(alias))) {
|
|
120
|
+
return true;
|
|
121
|
+
}
|
|
122
|
+
if (entry.type !== 'PERSON') {
|
|
123
|
+
return false;
|
|
124
|
+
}
|
|
125
|
+
for (const candidate of candidates) {
|
|
126
|
+
const parts = candidate.match(/[\p{L}\p{M}]+/gu) ?? [];
|
|
127
|
+
for (const alias of entry.aliases) {
|
|
128
|
+
const full = alias.match(/[\p{L}\p{M}]+/gu) ?? [];
|
|
129
|
+
if (
|
|
130
|
+
parts.length === 1 &&
|
|
131
|
+
parts[0].length > 1 &&
|
|
132
|
+
full.includes(parts[0])
|
|
133
|
+
) {
|
|
134
|
+
return true;
|
|
135
|
+
}
|
|
136
|
+
if (
|
|
137
|
+
parts.length === full.length &&
|
|
138
|
+
parts.length > 1 &&
|
|
139
|
+
parts.every(
|
|
140
|
+
(part, i) =>
|
|
141
|
+
part === full[i] || (part.length === 1 && full[i].startsWith(part))
|
|
142
|
+
)
|
|
143
|
+
) {
|
|
144
|
+
return true;
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
return false;
|
|
149
|
+
}
|
|
150
|
+
export function placeholderNameFindings(text, findings, options) {
|
|
151
|
+
if (!hasPlaceholders(options) || options.knownOnly) {
|
|
152
|
+
return findings;
|
|
153
|
+
}
|
|
154
|
+
const people = [
|
|
155
|
+
...findings
|
|
156
|
+
.filter((f) => f.type === 'PERSON')
|
|
157
|
+
.map((f) => text.slice(f.start, f.end)),
|
|
158
|
+
...(options[placeholderState]?.map.entries ?? [])
|
|
159
|
+
.filter((e) => e.type === 'PERSON')
|
|
160
|
+
.flatMap((e) => e.values),
|
|
161
|
+
];
|
|
162
|
+
const patterns = new Set();
|
|
163
|
+
for (const person of people) {
|
|
164
|
+
addNamePatterns(person, patterns);
|
|
165
|
+
}
|
|
166
|
+
const result = [...findings];
|
|
167
|
+
for (const pattern of patterns) {
|
|
168
|
+
for (const match of text.matchAll(
|
|
169
|
+
new RegExp(`(?<![\\p{L}\\p{M}])${pattern}(?![\\p{L}\\p{M}])`, 'giu')
|
|
170
|
+
)) {
|
|
171
|
+
result.push({
|
|
172
|
+
start: match.index,
|
|
173
|
+
end: match.index + match[0].length,
|
|
174
|
+
type: 'PERSON',
|
|
175
|
+
category: 'personal',
|
|
176
|
+
rule: 'name-variant',
|
|
177
|
+
confidence: 0.9,
|
|
178
|
+
});
|
|
179
|
+
if (result.length > (options.maxFindings ?? 100000)) {
|
|
180
|
+
throw failure('ERR_LIMIT');
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
return result;
|
|
185
|
+
}
|
|
186
|
+
function addNamePatterns(person, patterns) {
|
|
187
|
+
const words = person.match(/[\p{L}\p{M}]+/gu) ?? [];
|
|
188
|
+
if (words.length < 2 || words.some((word) => word.length < 2)) {
|
|
189
|
+
return;
|
|
190
|
+
}
|
|
191
|
+
for (const alias of nameAliases(person)) {
|
|
192
|
+
const parts = alias.match(/[\p{L}\p{M}]+/gu) ?? [];
|
|
193
|
+
if (parts.length !== words.length) {
|
|
194
|
+
continue;
|
|
195
|
+
}
|
|
196
|
+
patterns.add(
|
|
197
|
+
`${escapePattern(parts[0][0])}\\.[ \\t]+${parts.slice(1).map(escapePattern).join('[ \\t]+')}`
|
|
198
|
+
);
|
|
199
|
+
for (const word of parts) {
|
|
200
|
+
if (/^[а-яё]+(?:ов|ев)$/u.test(word)) {
|
|
201
|
+
patterns.add(`${escapePattern(word)}(?:ым|ой|а|у|е|ы|ых|ами)?`);
|
|
202
|
+
}
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
}
|
|
206
|
+
export function preparePlaceholderText(text, findings, options) {
|
|
207
|
+
const state = options[placeholderState];
|
|
208
|
+
if (!state || !hasPlaceholders(options)) {
|
|
209
|
+
return;
|
|
210
|
+
}
|
|
211
|
+
const types = new Set([
|
|
212
|
+
...reservedPlaceholderTypes(),
|
|
213
|
+
...findings.map((f) =>
|
|
214
|
+
f.category === 'credential' ? 'CREDENTIAL' : f.type
|
|
215
|
+
),
|
|
216
|
+
...(options.knownPersonal ?? []).map((e) => e.type),
|
|
217
|
+
...state.map.entries.map((e) => e.type),
|
|
218
|
+
]);
|
|
219
|
+
const [before, after] = state.map.format.split('{n}');
|
|
220
|
+
const tokens = [...types].map(
|
|
221
|
+
(type) =>
|
|
222
|
+
`${escapePattern(
|
|
223
|
+
placeholderToken(type, 1, before)
|
|
224
|
+
)}[1-9][0-9]*${escapePattern(placeholderToken(type, 1, after))}`
|
|
225
|
+
);
|
|
226
|
+
const pattern = new RegExp(
|
|
227
|
+
`(?<![\\p{L}\\p{M}\\p{N}_])(?:${tokens.join('|')})(?![\\p{L}\\p{M}\\p{N}_])`,
|
|
228
|
+
'u'
|
|
229
|
+
);
|
|
230
|
+
if (namespaceCollision(text, pattern)) {
|
|
231
|
+
throw failure('ERR_PLACEHOLDER_COLLISION');
|
|
232
|
+
}
|
|
233
|
+
const people = findings
|
|
234
|
+
.filter((f) => f.type === 'PERSON')
|
|
235
|
+
.map((f) => text.slice(f.start, f.end));
|
|
236
|
+
state.people = [
|
|
237
|
+
...people,
|
|
238
|
+
...state.map.entries
|
|
239
|
+
.filter((e) => e.type === 'PERSON')
|
|
240
|
+
.flatMap((e) => e.values),
|
|
241
|
+
];
|
|
242
|
+
}
|
|
243
|
+
function namespaceCollision(text, pattern) {
|
|
244
|
+
if (pattern.test(text)) {
|
|
245
|
+
return true;
|
|
246
|
+
}
|
|
247
|
+
// JSON escape spellings must not conceal existing tokens in keys or values.
|
|
248
|
+
for (const literal of jsonStringLiterals(text)) {
|
|
249
|
+
try {
|
|
250
|
+
if (pattern.test(JSON.parse(literal))) {
|
|
251
|
+
return true;
|
|
252
|
+
}
|
|
253
|
+
} catch {
|
|
254
|
+
// Ordinary free text may include strings that are not JSON literals.
|
|
255
|
+
}
|
|
256
|
+
}
|
|
257
|
+
return false;
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
function canonicalPerson(value, people) {
|
|
261
|
+
// Resolve components only when there is a unique compatible full name.
|
|
262
|
+
const matches = people.filter((person) => {
|
|
263
|
+
const words = person.match(/\p{L}+/gu) ?? [];
|
|
264
|
+
return (
|
|
265
|
+
words.length > 1 &&
|
|
266
|
+
words.every((word) => word.length > 1) &&
|
|
267
|
+
related(value, { type: 'PERSON', aliases: aliases(person, 'PERSON') })
|
|
268
|
+
);
|
|
269
|
+
});
|
|
270
|
+
const distinct = new Map(
|
|
271
|
+
matches.map((person) => [
|
|
272
|
+
aliases(person, 'PERSON').sort().join('\0'),
|
|
273
|
+
person,
|
|
274
|
+
])
|
|
275
|
+
);
|
|
276
|
+
return distinct.size === 1 ? [...distinct.values()][0] : value;
|
|
277
|
+
}
|
|
278
|
+
function canonicalKnown(value, type, options, state) {
|
|
279
|
+
const known = (options.knownPersonal ?? []).filter(
|
|
280
|
+
(e) => e.type === type && samePersonal(value, e.value, type)
|
|
281
|
+
);
|
|
282
|
+
if (known.length === 1) {
|
|
283
|
+
return known[0].value;
|
|
284
|
+
}
|
|
285
|
+
return type === 'PERSON' ? canonicalPerson(value, state.people ?? []) : value;
|
|
286
|
+
}
|
|
287
|
+
export function placeholderReplacement(value, finding, transform, options) {
|
|
288
|
+
if (finding.category === 'credential' && !transform.credentials) {
|
|
289
|
+
return REDACTED;
|
|
290
|
+
}
|
|
291
|
+
const state = options[placeholderState];
|
|
292
|
+
if (!state) {
|
|
293
|
+
throw failure('ERR_CONFIG');
|
|
294
|
+
}
|
|
295
|
+
const { map } = state;
|
|
296
|
+
const type =
|
|
297
|
+
finding.category === 'credential'
|
|
298
|
+
? 'CREDENTIAL'
|
|
299
|
+
: finding.type === 'DATE_OF_BIRTH'
|
|
300
|
+
? 'DATE'
|
|
301
|
+
: finding.type;
|
|
302
|
+
const canonical = canonicalKnown(value, type, options, state);
|
|
303
|
+
const candidates = map.entries.filter(
|
|
304
|
+
(entry) => entry.type === type && related(canonical, entry)
|
|
305
|
+
);
|
|
306
|
+
let entry = candidates.length === 1 ? candidates[0] : undefined;
|
|
307
|
+
if (!entry) {
|
|
308
|
+
if (map.entries.length >= (options.maxFindings ?? 100000)) {
|
|
309
|
+
throw failure('ERR_LIMIT');
|
|
310
|
+
}
|
|
311
|
+
const number =
|
|
312
|
+
1 +
|
|
313
|
+
map.entries.reduce(
|
|
314
|
+
(n, e) => (e.type === type ? Math.max(n, e.number) : n),
|
|
315
|
+
0
|
|
316
|
+
);
|
|
317
|
+
const token = placeholderToken(type, number, map.format);
|
|
318
|
+
if (
|
|
319
|
+
!Number.isSafeInteger(number) ||
|
|
320
|
+
map.entries.some((e) => e.token === token)
|
|
321
|
+
) {
|
|
322
|
+
throw failure('ERR_PLACEHOLDER_COLLISION');
|
|
323
|
+
}
|
|
324
|
+
entry = {
|
|
325
|
+
type,
|
|
326
|
+
number,
|
|
327
|
+
token,
|
|
328
|
+
values: [canonical],
|
|
329
|
+
aliases: aliases(canonical, type),
|
|
330
|
+
};
|
|
331
|
+
map.entries.push(entry);
|
|
332
|
+
}
|
|
333
|
+
if (!entry.values.includes(value)) {
|
|
334
|
+
entry.values.push(value);
|
|
335
|
+
}
|
|
336
|
+
return entry.token;
|
|
337
|
+
}
|
|
338
|
+
export function finishPlaceholders(result, occurrences, options) {
|
|
339
|
+
const state = options[placeholderState];
|
|
340
|
+
if (!state?.keep || !hasPlaceholders(options)) {
|
|
341
|
+
return result;
|
|
342
|
+
}
|
|
343
|
+
if (options[placeholderNested]) {
|
|
344
|
+
return { ...result, map: state.map };
|
|
345
|
+
} else if (state.stream) {
|
|
346
|
+
state.stream.add(result.text, occurrences);
|
|
347
|
+
} else {
|
|
348
|
+
addDocument(state.map, documentDigest(result.text), occurrences);
|
|
349
|
+
}
|
|
350
|
+
return { ...result, map: state.map };
|
|
351
|
+
}
|
|
352
|
+
function addDocument(map, digest, occurrences) {
|
|
353
|
+
const existing = map.documents.find((doc) => doc.digest === digest);
|
|
354
|
+
if (existing) {
|
|
355
|
+
if (JSON.stringify(existing.occurrences) !== JSON.stringify(occurrences)) {
|
|
356
|
+
throw failure('ERR_RESTORE_AMBIGUITY');
|
|
357
|
+
}
|
|
358
|
+
return;
|
|
359
|
+
}
|
|
360
|
+
if (map.documents.length >= 100000) {
|
|
361
|
+
throw failure('ERR_LIMIT');
|
|
362
|
+
}
|
|
363
|
+
map.documents.push({ digest, occurrences });
|
|
364
|
+
}
|
|
365
|
+
// Hash the whole output incrementally; retain only bounded restoration spans.
|
|
366
|
+
export function beginPlaceholderStream(options) {
|
|
367
|
+
const state = options[placeholderState];
|
|
368
|
+
if (!state || state.stream) {
|
|
369
|
+
return () => {};
|
|
370
|
+
}
|
|
371
|
+
const hash = createHash('sha256'),
|
|
372
|
+
occurrences = [];
|
|
373
|
+
let offset = 0;
|
|
374
|
+
state.stream = {
|
|
375
|
+
add(text, spans) {
|
|
376
|
+
hash.update(text);
|
|
377
|
+
for (const span of spans) {
|
|
378
|
+
if (occurrences.length >= (options.maxFindings ?? 100000)) {
|
|
379
|
+
throw failure('ERR_LIMIT');
|
|
380
|
+
}
|
|
381
|
+
occurrences.push({
|
|
382
|
+
...span,
|
|
383
|
+
start: span.start + offset,
|
|
384
|
+
end: span.end + offset,
|
|
385
|
+
});
|
|
386
|
+
}
|
|
387
|
+
offset += text.length;
|
|
388
|
+
},
|
|
389
|
+
};
|
|
390
|
+
return (complete) => {
|
|
391
|
+
state.stream = undefined;
|
|
392
|
+
if (complete && state.keep) {
|
|
393
|
+
addDocument(state.map, hash.digest('hex'), occurrences);
|
|
394
|
+
}
|
|
395
|
+
discardPlaceholderState(options);
|
|
396
|
+
};
|
|
397
|
+
}
|
|
398
|
+
export function discardPlaceholderState(options) {
|
|
399
|
+
const state = options[placeholderState];
|
|
400
|
+
if (state && !state.keep && !state.stream) {
|
|
401
|
+
state.map.entries.length = 0;
|
|
402
|
+
state.map.documents.length = 0;
|
|
403
|
+
state.people = [];
|
|
404
|
+
}
|
|
405
|
+
}
|
package/src/projection.js
CHANGED
|
@@ -27,8 +27,12 @@ export function projectText(input) {
|
|
|
27
27
|
!/\x1b\[|\\u[\da-fA-F]{4}|[\u200b-\u200f\u202a-\u202e\u2060-\u206f\ufeff]/u.test(
|
|
28
28
|
input
|
|
29
29
|
) &&
|
|
30
|
-
|
|
31
|
-
|
|
30
|
+
// ASCII already has normalized letters and digits. Avoid constructing a
|
|
31
|
+
// second multi-megabyte string just to check its decimal characters.
|
|
32
|
+
// eslint-disable-next-line no-control-regex -- Check the ASCII range.
|
|
33
|
+
(!/[^\x00-\x7f]/.test(input) ||
|
|
34
|
+
(input.normalize('NFKC') === input &&
|
|
35
|
+
!/[^0-9]/.test(input.replace(/[^\p{Nd}]/gu, ''))))
|
|
32
36
|
) {
|
|
33
37
|
return { text: input, changed: false };
|
|
34
38
|
}
|
package/src/propagation.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { personalPattern, maxPropagationLength, failure } from './detection.js';
|
|
2
2
|
import { actionable } from './confidence.js';
|
|
3
|
+
import { organizationCore } from './personal-variants.js';
|
|
3
4
|
import { nameAliases } from './transliteration.js';
|
|
4
5
|
|
|
5
6
|
function mrzSeeds(value, finding, seeds) {
|
|
@@ -32,11 +33,13 @@ function decodedValue(value) {
|
|
|
32
33
|
|
|
33
34
|
function eligibleSeed(finding, options) {
|
|
34
35
|
return (
|
|
36
|
+
!options.knownOnly &&
|
|
35
37
|
finding.category === 'personal' &&
|
|
36
38
|
finding.type !== 'ENCODED_SENSITIVE' &&
|
|
37
39
|
!['confirmed-personal', 'document-propagation'].includes(finding.rule) &&
|
|
38
40
|
actionable(finding, options) &&
|
|
39
|
-
(
|
|
41
|
+
((options.documentEntities && finding.type === 'ORGANIZATION') ||
|
|
42
|
+
finding.confidence >= 0.85 ||
|
|
40
43
|
[
|
|
41
44
|
'PERSON',
|
|
42
45
|
'PASSPORT_NUMBER',
|
|
@@ -47,6 +50,14 @@ function eligibleSeed(finding, options) {
|
|
|
47
50
|
].includes(finding.type))
|
|
48
51
|
);
|
|
49
52
|
}
|
|
53
|
+
function organizationSeed(value, finding, seeds, options) {
|
|
54
|
+
if (options.documentEntities && finding.type === 'ORGANIZATION') {
|
|
55
|
+
const core = organizationCore(value);
|
|
56
|
+
if (core.length >= 3) {
|
|
57
|
+
seeds.set(core, finding);
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
}
|
|
50
61
|
// Only confirmed values seed propagation. A low-score numeric coincidence
|
|
51
62
|
// must never turn every log occurrence into a high-score identity.
|
|
52
63
|
export function confirmedSeeds(text, findings, options) {
|
|
@@ -71,6 +82,7 @@ export function confirmedSeeds(text, findings, options) {
|
|
|
71
82
|
mrzSeeds(value, finding, seeds);
|
|
72
83
|
} else if (value.length >= 2) {
|
|
73
84
|
seeds.set(value, finding);
|
|
85
|
+
organizationSeed(value, finding, seeds, options);
|
|
74
86
|
if (finding.type === 'PERSON') {
|
|
75
87
|
for (const name of value.match(/\p{L}{2,}/gu) ?? []) {
|
|
76
88
|
seeds.set(name, finding);
|