@link-foundation/sensitive-data-sanitizer 0.0.0-stage → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +578 -0
- package/LICENSE +24 -0
- package/README.md +184 -2
- package/bin/legacy-example-package-name.js +95 -0
- package/bin/sensitive-data-sanitizer.js +430 -0
- package/bin/stream.js +56 -0
- package/docs/API.md +336 -0
- package/docs/COVERAGE.md +136 -0
- package/docs/HISTORY-REMEDIATION.md +41 -0
- package/examples/combined-scanners.mjs +23 -0
- package/examples/presidio-bridge.py +32 -0
- package/examples/realistic-fakes.mjs +18 -0
- package/examples/sanitize-session.mjs +27 -0
- package/examples/shared-tool-logs.mjs +21 -0
- package/examples/stream-jsonl.mjs +17 -0
- package/package.json +124 -4
- package/src/adapters.js +204 -0
- package/src/arithmetic.js +11 -0
- package/src/confidence.js +65 -0
- package/src/context.js +424 -0
- package/src/credentials.js +54 -0
- package/src/detection.js +293 -0
- package/src/encoded.js +233 -0
- package/src/engines.js +348 -0
- package/src/entities.js +166 -0
- package/src/escape-spans.js +20 -0
- package/src/eslint-plugin.d.ts +10 -0
- package/src/eslint-plugin.js +193 -0
- package/src/external.js +169 -0
- package/src/fake-names.js +222 -0
- package/src/fake.js +524 -0
- package/src/files.js +131 -0
- package/src/gitleaks.js +194 -0
- package/src/history-rewrite.js +339 -0
- package/src/history.js +103 -0
- package/src/identity.js +274 -0
- package/src/index.d.ts +321 -0
- package/src/index.js +40 -0
- package/src/jsonl-stream.js +198 -0
- package/src/keywords.js +45 -0
- package/src/known.js +164 -0
- package/src/legacy.d.ts +3 -0
- package/src/legacy.js +2 -0
- package/src/locales.js +217 -0
- package/src/log-bytes.js +85 -0
- package/src/names.js +65 -0
- package/src/national-checks.js +129 -0
- package/src/national.js +128 -0
- package/src/outbound.js +28 -0
- package/src/personal-context.js +85 -0
- package/src/personal.js +248 -0
- package/src/projection.js +126 -0
- package/src/propagation.js +163 -0
- package/src/public-verifier.js +77 -0
- package/src/public.js +153 -0
- package/src/record-worker.js +20 -0
- package/src/rules.js +210 -0
- package/src/sanitizer.js +794 -0
- package/src/specimens.js +34 -0
- package/src/stream-identity.js +129 -0
- package/src/stream-worker.js +18 -0
- package/src/stream.js +449 -0
- package/src/structured.js +278 -0
- package/src/transforms.js +192 -0
- package/src/transliteration.js +75 -0
- package/src/utf8.js +60 -0
- package/src/vendor/entities/aws.json +42 -0
- package/src/vendor/entities/azure.json +182 -0
- package/src/vendor/entities/google.json +219 -0
- package/src/vendor/gitleaks/LICENSE +21 -0
- package/src/vendor/gitleaks/provenance.json +8 -0
- package/src/vendor/gitleaks/rules-0.json +268 -0
- package/src/vendor/gitleaks/rules-1.json +1794 -0
- package/src/vendor/gitleaks/rules-2.json +292 -0
- package/src/vendor/gitleaks/rules-3.json +263 -0
- package/src/vendor/gitleaks/rules-4.json +277 -0
- package/src/vendor/gitleaks/rules-5.json +154 -0
- package/src/worker-options.js +27 -0
package/src/personal.js
ADDED
|
@@ -0,0 +1,248 @@
|
|
|
1
|
+
import { collectMatches, escapePattern } from './detection.js';
|
|
2
|
+
import { firstNames } from './names.js';
|
|
3
|
+
import { labels } from './rules.js';
|
|
4
|
+
import { transliteratedGivenNames } from './transliteration.js';
|
|
5
|
+
import { serviceAccounts } from './public.js';
|
|
6
|
+
const labelWords = new Set(Object.values(labels).flat());
|
|
7
|
+
|
|
8
|
+
const details = (type, rule, confidence = 0.85) => ({
|
|
9
|
+
type,
|
|
10
|
+
rule,
|
|
11
|
+
category: 'personal',
|
|
12
|
+
confidence,
|
|
13
|
+
});
|
|
14
|
+
const names = Object.entries(firstNames).map(([language, list]) => {
|
|
15
|
+
const words = list.split(' ').map(escapePattern).join('|');
|
|
16
|
+
const surname = ['ar', 'he', 'hi', 'th'].includes(language)
|
|
17
|
+
? '[\\p{L}\\p{M}]{2,32}'
|
|
18
|
+
: "[\\p{Lu}][\\p{L}\\p{M}’'-]{1,32}";
|
|
19
|
+
const pattern = ['ja', 'zh', 'ko'].includes(language)
|
|
20
|
+
? `(?:${words})[\\p{Script=Han}\\p{Script=Hiragana}\\p{Script=Katakana}\\p{Script=Hangul}]{1,4}`
|
|
21
|
+
: `(?:${words})[ \\t]+${surname}${language === 'vi' ? `(?:[ \\t]+${surname})?` : ''}`;
|
|
22
|
+
return new RegExp(`(?<![\\p{L}\\p{M}])${pattern}(?![\\p{L}\\p{M}])`, 'gu');
|
|
23
|
+
});
|
|
24
|
+
|
|
25
|
+
// ISO 13616 national lengths; checksum arithmetic never exceeds 3 digits.
|
|
26
|
+
const ibanLengths = Object.fromEntries(
|
|
27
|
+
'AD24 AE23 AL28 AT20 AZ28 BA20 BE16 BG22 BH22 BI27 BR29 BY28 CH21 CR22 CY28 CZ24 DE22 DJ27 DK18 DO28 EE20 EG29 ES24 FI18 FK18 FO18 FR27 GB22 GE22 GI23 GL18 GR27 GT28 HR21 HU28 IE22 IL23 IQ23 IS26 IT27 JO30 KW30 KZ20 LB28 LC32 LI21 LT20 LU20 LV21 LY25 MC27 MD24 ME22 MK19 MN20 MR27 MT31 MU30 NI28 NL18 NO15 OM23 PK24 PL28 PS29 PT25 QA29 RO24 RS22 RU33 SA24 SC31 SD18 SE24 SI19 SK24 SM27 SO23 ST25 SV28 TL23 TN24 TR26 UA29 VA22 VG24 XK20'
|
|
28
|
+
.split(' ')
|
|
29
|
+
.map((entry) => [entry.slice(0, 2), Number(entry.slice(2))])
|
|
30
|
+
);
|
|
31
|
+
|
|
32
|
+
export function iban(value) {
|
|
33
|
+
const compact = value.replace(/\s/g, '').toUpperCase();
|
|
34
|
+
if (
|
|
35
|
+
!/^[A-Z]{2}\d{2}[A-Z0-9]+$/.test(compact) ||
|
|
36
|
+
compact.length !== ibanLengths[compact.slice(0, 2)]
|
|
37
|
+
) {
|
|
38
|
+
return false;
|
|
39
|
+
}
|
|
40
|
+
const rotated = compact.slice(4) + compact.slice(0, 4);
|
|
41
|
+
let remainder = 0;
|
|
42
|
+
for (const char of rotated) {
|
|
43
|
+
const digits = /\d/.test(char) ? char : String(char.charCodeAt(0) - 55);
|
|
44
|
+
for (const digit of digits) {
|
|
45
|
+
remainder = (remainder * 10 + Number(digit)) % 97;
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
return remainder === 1;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
function detectIban(text, emit) {
|
|
52
|
+
for (const match of text.matchAll(
|
|
53
|
+
/(?<![A-Z0-9])[A-Z]{2}\d{2}(?:[ ]?[A-Z0-9]){8,40}/g
|
|
54
|
+
)) {
|
|
55
|
+
const needed = ibanLengths[match[0].slice(0, 2)];
|
|
56
|
+
if (!needed) {
|
|
57
|
+
continue;
|
|
58
|
+
}
|
|
59
|
+
let end = 0,
|
|
60
|
+
count = 0;
|
|
61
|
+
while (end < match[0].length && count < needed) {
|
|
62
|
+
if (match[0][end] !== ' ') {
|
|
63
|
+
count++;
|
|
64
|
+
}
|
|
65
|
+
end++;
|
|
66
|
+
}
|
|
67
|
+
const value = match[0].slice(0, end);
|
|
68
|
+
if (iban(value)) {
|
|
69
|
+
emit({
|
|
70
|
+
start: match.index,
|
|
71
|
+
end: match.index + end,
|
|
72
|
+
...details('IBAN', 'iban-checksum', 0.99),
|
|
73
|
+
});
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
const latinGiven = Object.entries(firstNames)
|
|
79
|
+
.filter(([locale]) =>
|
|
80
|
+
['en', 'es', 'fr', 'de', 'pt', 'ruLatn', 'tr'].includes(locale)
|
|
81
|
+
)
|
|
82
|
+
.flatMap(([, list]) => list.split(' '))
|
|
83
|
+
.concat(transliteratedGivenNames)
|
|
84
|
+
.map(escapePattern)
|
|
85
|
+
.join('|');
|
|
86
|
+
const slavicSurname =
|
|
87
|
+
'[\\p{L}]{2,28}(?:ova|eva|ov|ev|in|ina|enko|sky|ski|skaya)';
|
|
88
|
+
const latinPairs = new RegExp(
|
|
89
|
+
`(?<![\\p{L}\\p{M}])(?:${latinGiven})[ \\t]+(?:${slavicSurname}|Smith|Jones|Brown|Roe|Doe)(?![\\p{L}\\p{M}])`,
|
|
90
|
+
'giu'
|
|
91
|
+
);
|
|
92
|
+
const shapedName = '[\\p{Lu}][\\p{L}\\p{M}]{1,31}';
|
|
93
|
+
const suffixes =
|
|
94
|
+
'ova eva ov ev in ina enko sky ski skaya ова ева ов ев ин ина енко ский ская'
|
|
95
|
+
.split(' ')
|
|
96
|
+
.map((s) => [...s].map((c) => `[${c}${c.toUpperCase()}]`).join(''))
|
|
97
|
+
.join('|');
|
|
98
|
+
const shapedSurname = `[\\p{Lu}][\\p{L}]{1,28}(?:${suffixes})`;
|
|
99
|
+
const slavicPairs = new RegExp(
|
|
100
|
+
`(?<![\\p{L}\\p{M}])(?:${shapedName}[ \\t]+${shapedSurname}|${shapedSurname}[ \\t]+${shapedName})(?![\\p{L}\\p{M}])`,
|
|
101
|
+
'gu'
|
|
102
|
+
);
|
|
103
|
+
const documentWord =
|
|
104
|
+
/^(?:PASSPORT|VISA|ID|PHOTO|SCAN|ПАСПОРТ|ВИЗА|СКАН|ФОТО)$/iu;
|
|
105
|
+
const placeWord = /^(?:Bay|Sands|Hotel|Street|Road|Avenue)$/iu;
|
|
106
|
+
const givenWords = new Set(
|
|
107
|
+
Object.values(firstNames)
|
|
108
|
+
.flatMap((list) => list.toLowerCase().split(' '))
|
|
109
|
+
.concat(transliteratedGivenNames.map((name) => name.toLowerCase()))
|
|
110
|
+
);
|
|
111
|
+
function nameShaped(candidates) {
|
|
112
|
+
return (
|
|
113
|
+
candidates.some((token) => givenWords.has(token[0].toLowerCase())) ||
|
|
114
|
+
candidates.every((token) =>
|
|
115
|
+
/^(?:\p{Lu}\p{Ll}[\p{L}\p{M}]*|\p{Lu}{2,32})$/u.test(token[0])
|
|
116
|
+
)
|
|
117
|
+
);
|
|
118
|
+
}
|
|
119
|
+
function documentReference(text, match) {
|
|
120
|
+
const following = text.slice(
|
|
121
|
+
match.index + match[0].length,
|
|
122
|
+
match.index + match[0].length + 16
|
|
123
|
+
);
|
|
124
|
+
return (
|
|
125
|
+
/^\.[a-z\d]{1,10}(?![\p{L}\p{N}_])/iu.test(following) ||
|
|
126
|
+
/[/\\]/.test(text[match.index - 1] ?? '')
|
|
127
|
+
);
|
|
128
|
+
}
|
|
129
|
+
function detectDocumentNames(text, emit) {
|
|
130
|
+
for (const match of text.matchAll(
|
|
131
|
+
/(?<![\p{L}\p{N}_-])[\p{L}]{2,32}(?:[_-][\p{L}]{2,32}){1,12}(?![\p{L}\p{N}_-])/gu
|
|
132
|
+
)) {
|
|
133
|
+
if (!documentReference(text, match)) {
|
|
134
|
+
continue;
|
|
135
|
+
}
|
|
136
|
+
// Field names such as NATIONAL_ID identify the label, not its value.
|
|
137
|
+
if (
|
|
138
|
+
/^[ \t]*(?:["'][ \t]*)?[:=:]/.test(
|
|
139
|
+
text.slice(match.index + match[0].length)
|
|
140
|
+
)
|
|
141
|
+
) {
|
|
142
|
+
continue;
|
|
143
|
+
}
|
|
144
|
+
const tokens = [...match[0].matchAll(/\p{L}+/gu)];
|
|
145
|
+
for (let i = 0; i < tokens.length; i++) {
|
|
146
|
+
if (!documentWord.test(tokens[i][0])) {
|
|
147
|
+
continue;
|
|
148
|
+
}
|
|
149
|
+
for (const step of [-1, 1]) {
|
|
150
|
+
const candidates = [];
|
|
151
|
+
for (
|
|
152
|
+
let j = i + step;
|
|
153
|
+
j >= 0 && j < tokens.length && candidates.length < 4;
|
|
154
|
+
j += step
|
|
155
|
+
) {
|
|
156
|
+
if (documentWord.test(tokens[j][0])) {
|
|
157
|
+
break;
|
|
158
|
+
}
|
|
159
|
+
candidates.push(tokens[j]);
|
|
160
|
+
}
|
|
161
|
+
if (candidates.length < 2 || candidates.length > 3) {
|
|
162
|
+
continue;
|
|
163
|
+
}
|
|
164
|
+
candidates.sort((a, b) => a.index - b.index);
|
|
165
|
+
if (!nameShaped(candidates)) {
|
|
166
|
+
continue;
|
|
167
|
+
}
|
|
168
|
+
emit({
|
|
169
|
+
start: match.index + candidates[0].index,
|
|
170
|
+
end:
|
|
171
|
+
match.index + candidates.at(-1).index + candidates.at(-1)[0].length,
|
|
172
|
+
...details('PERSON', 'name-document', 0.9),
|
|
173
|
+
});
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
const russianFull =
|
|
179
|
+
/(?<![\p{L}])(?:[А-ЯЁ][а-яё]{2,30}[ \t]+){2}[А-ЯЁ][а-яё]{2,30}(?:вич|вна|ична)|(?<![\p{L}])(?:[А-ЯЁ][а-яё]{2,30}[ \t]+){2}(?:оглы|кызы)(?![\p{L}])/gu;
|
|
180
|
+
export function detectPersonal(text, emit) {
|
|
181
|
+
detectDocumentNames(text, emit);
|
|
182
|
+
for (const pattern of [latinPairs, slavicPairs, russianFull]) {
|
|
183
|
+
collectMatches(
|
|
184
|
+
text,
|
|
185
|
+
pattern,
|
|
186
|
+
emit,
|
|
187
|
+
details('PERSON', 'name-identity', 0.9),
|
|
188
|
+
0,
|
|
189
|
+
(value) => !placeWord.test(value.split(/[ \t]+/).at(-1))
|
|
190
|
+
);
|
|
191
|
+
}
|
|
192
|
+
for (const pattern of names) {
|
|
193
|
+
collectMatches(
|
|
194
|
+
text,
|
|
195
|
+
pattern,
|
|
196
|
+
emit,
|
|
197
|
+
details('PERSON', 'name-gazetteer', 0.7),
|
|
198
|
+
0,
|
|
199
|
+
(value) =>
|
|
200
|
+
!labelWords.has(value) && !placeWord.test(value.split(/[ \t]+/).at(-1))
|
|
201
|
+
);
|
|
202
|
+
}
|
|
203
|
+
collectMatches(
|
|
204
|
+
text,
|
|
205
|
+
/(?:компании|организации|company|organization|société|empresa|Firma)[ \t]+([\p{Lu}][\p{L}\p{M}&'-]{1,80})/gu,
|
|
206
|
+
emit,
|
|
207
|
+
details('ORGANIZATION', 'organization-context', 0.7),
|
|
208
|
+
1
|
|
209
|
+
);
|
|
210
|
+
collectMatches(
|
|
211
|
+
text,
|
|
212
|
+
/(?<![\d(])(?:[78][ -]?)?\(\d{3}\)[ -]?\d{3}[ -]\d{2}[ -]?\d{2}(?!\d)|(?<!\d)(?:\d{3}[.-]){2}\d{4}(?!\d)/g,
|
|
213
|
+
emit,
|
|
214
|
+
details('PHONE', 'national-phone')
|
|
215
|
+
);
|
|
216
|
+
collectMatches(
|
|
217
|
+
text,
|
|
218
|
+
/(?<![\p{L}\p{N}])\d{1,6}[A-Za-z]?[ \t]+(?:[\p{Lu}][\p{L}.'-]*[ \t]+){1,5}(?:Street|St|Road|Rd|Avenue|Ave|Lane|Ln|Drive|Dr|Boulevard|Blvd|Way|Straße|Strasse|Rue|Calle|Rua)\b(?:,[ \t]+(?:[^\r\n<>]{1,80}?[ \t]+(?:[A-Z]{1,2}\d[A-Z\d]?[ \t]+\d[A-Z]{2}|\d{5}(?:-\d{4})?)\b|(?:[\p{Lu}][\p{Ll}.'-]+[ \t]*){1,4}))?/gu,
|
|
219
|
+
emit,
|
|
220
|
+
details('ADDRESS', 'postal-address')
|
|
221
|
+
);
|
|
222
|
+
collectMatches(
|
|
223
|
+
text,
|
|
224
|
+
/(?<![\da-fA-F:.-])(?:[\da-fA-F]{2}[:-]){5}[\da-fA-F]{2}(?![\da-fA-F:.-])/g,
|
|
225
|
+
emit,
|
|
226
|
+
details('MAC_ADDRESS', 'mac')
|
|
227
|
+
);
|
|
228
|
+
collectMatches(
|
|
229
|
+
text,
|
|
230
|
+
/(?<![\p{L}\p{N}_.\]])@[\p{L}\p{N}_]{2,32}(?![\p{L}\p{N}_.])/gu,
|
|
231
|
+
emit,
|
|
232
|
+
details('USERNAME', 'messenger-handle', 0.65),
|
|
233
|
+
0,
|
|
234
|
+
(_value, match) =>
|
|
235
|
+
!/^\/[a-z0-9][a-z0-9_.-]*/i.test(
|
|
236
|
+
text.slice(match.index + match[0].length)
|
|
237
|
+
)
|
|
238
|
+
);
|
|
239
|
+
collectMatches(
|
|
240
|
+
text,
|
|
241
|
+
/(?:\/Users\/|\/home\/|[A-Za-z]:\\Users\\)([^/\\\s"'<>[\]]+)/g,
|
|
242
|
+
emit,
|
|
243
|
+
details('USERNAME', 'home-path', 0.95),
|
|
244
|
+
1,
|
|
245
|
+
(value) => !serviceAccounts.has(value.toLowerCase())
|
|
246
|
+
);
|
|
247
|
+
detectIban(text, emit);
|
|
248
|
+
}
|
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
// Map a detection view back to the original UTF-16 offsets. Normalization
|
|
2
|
+
// never writes to the original text, and offsets remain valid after emoji.
|
|
3
|
+
import { failure } from './detection.js';
|
|
4
|
+
|
|
5
|
+
// Unicode Nd zero codepoints. NFKC already maps mathematical/fullwidth digits;
|
|
6
|
+
// other numeral systems need explicit conversion with original-source spans.
|
|
7
|
+
const digitZeros = [
|
|
8
|
+
0x30, 0x660, 0x6f0, 0x7c0, 0x966, 0x9e6, 0xa66, 0xae6, 0xb66, 0xbe6, 0xc66,
|
|
9
|
+
0xce6, 0xd66, 0xde6, 0xe50, 0xed0, 0xf20, 0x1040, 0x1090, 0x17e0, 0x1810,
|
|
10
|
+
0x1946, 0x19d0, 0x1a80, 0x1a90, 0x1b50, 0x1bb0, 0x1c40, 0x1c50, 0xa620,
|
|
11
|
+
0xa8d0, 0xa900, 0xa9d0, 0xa9f0, 0xaa50, 0xabf0, 0x104a0, 0x10d30, 0x11066,
|
|
12
|
+
0x110f0, 0x11136, 0x111d0, 0x112f0, 0x11450, 0x114d0, 0x11650, 0x116c0,
|
|
13
|
+
0x11730, 0x118e0, 0x11950, 0x11c50, 0x11d50, 0x11da0, 0x16a60, 0x16ac0,
|
|
14
|
+
0x16b50, 0x1e140, 0x1e2f0, 0x1e950, 0x1fbf0,
|
|
15
|
+
];
|
|
16
|
+
function asciiDigits(value) {
|
|
17
|
+
return value.replace(/\p{Nd}/gu, (digit) => {
|
|
18
|
+
const code = digit.codePointAt(0);
|
|
19
|
+
const zero = digitZeros.find((start) => code >= start && code <= start + 9);
|
|
20
|
+
return zero === undefined ? digit : String(code - zero);
|
|
21
|
+
});
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
export function projectText(input) {
|
|
25
|
+
if (
|
|
26
|
+
// eslint-disable-next-line no-control-regex -- ANSI escape sequences are removed from the detection view.
|
|
27
|
+
!/\x1b\[|\\u[\da-fA-F]{4}|[\u200b-\u200f\u202a-\u202e\u2060-\u206f\ufeff]/u.test(
|
|
28
|
+
input
|
|
29
|
+
) &&
|
|
30
|
+
input.normalize('NFKC') === input &&
|
|
31
|
+
!/[^0-9]/.test(input.replace(/[^\p{Nd}]/gu, ''))
|
|
32
|
+
) {
|
|
33
|
+
return { text: input, changed: false };
|
|
34
|
+
}
|
|
35
|
+
const writer = projectionWriter(input.length);
|
|
36
|
+
// eslint-disable-next-line no-control-regex -- Tokenize ANSI sequences with their original offsets.
|
|
37
|
+
const units = /\x1b\[[0-?]*[ -/]*[@-~]|\\u[\da-fA-F]{4}|[\s\S]\p{M}*/gu;
|
|
38
|
+
let changed = false;
|
|
39
|
+
for (const match of input.matchAll(units)) {
|
|
40
|
+
const raw = match[0];
|
|
41
|
+
let value = raw;
|
|
42
|
+
if (
|
|
43
|
+
raw.startsWith('\x1b[') ||
|
|
44
|
+
/^[\u200b-\u200f\u202a-\u202e\u2060-\u206f\ufeff]$/u.test(raw)
|
|
45
|
+
) {
|
|
46
|
+
changed = true;
|
|
47
|
+
continue;
|
|
48
|
+
}
|
|
49
|
+
if (/^\\u[\da-fA-F]{4}$/.test(raw)) {
|
|
50
|
+
value = String.fromCharCode(parseInt(raw.slice(2), 16));
|
|
51
|
+
}
|
|
52
|
+
if (/^[\u200b-\u200f\u202a-\u202e\u2060-\u206f\ufeff]$/u.test(value)) {
|
|
53
|
+
changed = true;
|
|
54
|
+
continue;
|
|
55
|
+
}
|
|
56
|
+
value = asciiDigits(value.normalize('NFKC'));
|
|
57
|
+
changed ||= value !== raw;
|
|
58
|
+
writer.append(value, match.index, match.index + raw.length);
|
|
59
|
+
}
|
|
60
|
+
return { ...writer.finish(), changed };
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
export function codePointRange(text, start, end) {
|
|
64
|
+
if (
|
|
65
|
+
!Number.isInteger(start) ||
|
|
66
|
+
!Number.isInteger(end) ||
|
|
67
|
+
start < 0 ||
|
|
68
|
+
end <= start
|
|
69
|
+
) {
|
|
70
|
+
throw new Error('Invalid code point range.');
|
|
71
|
+
}
|
|
72
|
+
const offsets = [0];
|
|
73
|
+
let offset = 0;
|
|
74
|
+
for (const char of text) {
|
|
75
|
+
offset += char.length;
|
|
76
|
+
offsets.push(offset);
|
|
77
|
+
}
|
|
78
|
+
if (end >= offsets.length) {
|
|
79
|
+
throw new Error('Invalid code point range.');
|
|
80
|
+
}
|
|
81
|
+
return { start: offsets[start], end: offsets[end] };
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
function projectionWriter(inputLength) {
|
|
85
|
+
// Input length has already been checked by the caller. Unicode NFKC can
|
|
86
|
+
// expand a single scalar to at most 18 scalars; retain a finite bound.
|
|
87
|
+
const maxLength = Math.max(inputLength * 18, 16);
|
|
88
|
+
let capacity = Math.min(Math.max(inputLength, 16), maxLength);
|
|
89
|
+
let starts = new Uint32Array(capacity),
|
|
90
|
+
ends = new Uint32Array(capacity);
|
|
91
|
+
const parts = [];
|
|
92
|
+
let length = 0,
|
|
93
|
+
chunk = '';
|
|
94
|
+
return {
|
|
95
|
+
append(value, start, end) {
|
|
96
|
+
if (length + value.length > maxLength) {
|
|
97
|
+
throw failure('ERR_LIMIT');
|
|
98
|
+
}
|
|
99
|
+
if (length + value.length > capacity) {
|
|
100
|
+
capacity = Math.min(
|
|
101
|
+
Math.max(capacity * 2, length + value.length),
|
|
102
|
+
maxLength
|
|
103
|
+
);
|
|
104
|
+
const nextStarts = new Uint32Array(capacity),
|
|
105
|
+
nextEnds = new Uint32Array(capacity);
|
|
106
|
+
nextStarts.set(starts);
|
|
107
|
+
nextEnds.set(ends);
|
|
108
|
+
starts = nextStarts;
|
|
109
|
+
ends = nextEnds;
|
|
110
|
+
}
|
|
111
|
+
chunk += value;
|
|
112
|
+
if (chunk.length >= 8192) {
|
|
113
|
+
parts.push(chunk);
|
|
114
|
+
chunk = '';
|
|
115
|
+
}
|
|
116
|
+
for (let j = 0; j < value.length; j++) {
|
|
117
|
+
starts[length] = start;
|
|
118
|
+
ends[length++] = end;
|
|
119
|
+
}
|
|
120
|
+
},
|
|
121
|
+
finish() {
|
|
122
|
+
parts.push(chunk);
|
|
123
|
+
return { text: parts.join(''), starts, ends };
|
|
124
|
+
},
|
|
125
|
+
};
|
|
126
|
+
}
|
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
import { personalPattern, maxPropagationLength, failure } from './detection.js';
|
|
2
|
+
import { actionable } from './confidence.js';
|
|
3
|
+
import { nameAliases } from './transliteration.js';
|
|
4
|
+
|
|
5
|
+
function mrzSeeds(value, finding, seeds) {
|
|
6
|
+
for (const match of value.matchAll(
|
|
7
|
+
/(?:[PIACV][A-Z<][A-Z<]{3})?([A-Z]{2,30})<<([A-Z]+(?:<[A-Z]+)*)/g
|
|
8
|
+
)) {
|
|
9
|
+
const surname =
|
|
10
|
+
match.index > 0 && /[PIACV][A-Z<]$/.test(value.slice(0, match.index))
|
|
11
|
+
? match[1].slice(3)
|
|
12
|
+
: match[1];
|
|
13
|
+
for (const name of [surname, ...match[2].split('<')]) {
|
|
14
|
+
seeds.set(name, { ...finding, type: 'PERSON' });
|
|
15
|
+
}
|
|
16
|
+
}
|
|
17
|
+
const number = /(?:^|[\r\n])([A-Z0-9]{9})[\d<][A-Z<]{3}\d{6}/.exec(value);
|
|
18
|
+
if (number) {
|
|
19
|
+
seeds.set(number[1], { ...finding, type: 'PASSPORT_NUMBER' });
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
function decodedValue(value) {
|
|
23
|
+
if (value.includes('\\')) {
|
|
24
|
+
try {
|
|
25
|
+
return JSON.parse(`"${value}"`);
|
|
26
|
+
} catch {
|
|
27
|
+
/* Plain paths are not JSON. */
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
return value;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
function eligibleSeed(finding, options) {
|
|
34
|
+
return (
|
|
35
|
+
finding.category === 'personal' &&
|
|
36
|
+
finding.type !== 'ENCODED_SENSITIVE' &&
|
|
37
|
+
!['confirmed-personal', 'document-propagation'].includes(finding.rule) &&
|
|
38
|
+
actionable(finding, options) &&
|
|
39
|
+
(finding.confidence >= 0.85 ||
|
|
40
|
+
[
|
|
41
|
+
'PERSON',
|
|
42
|
+
'PASSPORT_NUMBER',
|
|
43
|
+
'BOOKING_REFERENCE',
|
|
44
|
+
'VISA_NUMBER',
|
|
45
|
+
'TICKET_NUMBER',
|
|
46
|
+
'PASSPORT_MRZ',
|
|
47
|
+
].includes(finding.type))
|
|
48
|
+
);
|
|
49
|
+
}
|
|
50
|
+
// Only confirmed values seed propagation. A low-score numeric coincidence
|
|
51
|
+
// must never turn every log occurrence into a high-score identity.
|
|
52
|
+
export function confirmedSeeds(text, findings, options) {
|
|
53
|
+
const seeds = new Map();
|
|
54
|
+
for (const finding of findings) {
|
|
55
|
+
if (!eligibleSeed(finding, options)) {
|
|
56
|
+
continue;
|
|
57
|
+
}
|
|
58
|
+
if (
|
|
59
|
+
finding.end - finding.start > maxPropagationLength &&
|
|
60
|
+
finding.type !== 'PASSPORT_MRZ'
|
|
61
|
+
) {
|
|
62
|
+
options.debug?.({
|
|
63
|
+
event: 'propagation-skipped',
|
|
64
|
+
reason: 'length',
|
|
65
|
+
type: finding.type,
|
|
66
|
+
});
|
|
67
|
+
continue;
|
|
68
|
+
}
|
|
69
|
+
const value = decodedValue(text.slice(finding.start, finding.end)).trim();
|
|
70
|
+
if (finding.type === 'PASSPORT_MRZ') {
|
|
71
|
+
mrzSeeds(value, finding, seeds);
|
|
72
|
+
} else if (value.length >= 2) {
|
|
73
|
+
seeds.set(value, finding);
|
|
74
|
+
if (finding.type === 'PERSON') {
|
|
75
|
+
for (const name of value.match(/\p{L}{2,}/gu) ?? []) {
|
|
76
|
+
seeds.set(name, finding);
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
for (const [value, finding] of [...seeds]) {
|
|
82
|
+
if (finding.type === 'PERSON') {
|
|
83
|
+
for (const alias of nameAliases(value)) {
|
|
84
|
+
seeds.set(alias, finding);
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
return seeds;
|
|
89
|
+
}
|
|
90
|
+
export function confirmationOptions(text, findings, options) {
|
|
91
|
+
const values = new Map(
|
|
92
|
+
(options.confirmedPersonal ?? []).map((entry) => [entry.value, entry])
|
|
93
|
+
);
|
|
94
|
+
for (const [value, finding] of confirmedSeeds(text, findings, options)) {
|
|
95
|
+
values.set(value, { value, type: finding.type });
|
|
96
|
+
if (values.size > (options.maxFindings ?? 100000)) {
|
|
97
|
+
throw failure('ERR_LIMIT');
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
return { ...options, confirmedPersonal: [...values.values()] };
|
|
101
|
+
}
|
|
102
|
+
function rangeFilter(ranges) {
|
|
103
|
+
let index = 0;
|
|
104
|
+
return (match) => {
|
|
105
|
+
if (!ranges) {
|
|
106
|
+
return true;
|
|
107
|
+
}
|
|
108
|
+
while (index < ranges.length && ranges[index].end <= match.index) {
|
|
109
|
+
index++;
|
|
110
|
+
}
|
|
111
|
+
const range = ranges[index];
|
|
112
|
+
return (
|
|
113
|
+
!!range &&
|
|
114
|
+
match.index >= range.start &&
|
|
115
|
+
match.index + match[0].length <= range.end
|
|
116
|
+
);
|
|
117
|
+
};
|
|
118
|
+
}
|
|
119
|
+
export function propagate(text, findings, options, ranges) {
|
|
120
|
+
const result = [...findings];
|
|
121
|
+
const seen = new Set(findings.map((f) => `${f.start}:${f.end}:${f.type}`));
|
|
122
|
+
for (const [value, seed] of confirmedSeeds(text, findings, options)) {
|
|
123
|
+
const pattern = new RegExp(personalPattern(value), 'giu');
|
|
124
|
+
const withinRange = rangeFilter(ranges);
|
|
125
|
+
for (const match of text.matchAll(pattern)) {
|
|
126
|
+
if (!withinRange(match)) {
|
|
127
|
+
continue;
|
|
128
|
+
}
|
|
129
|
+
if (
|
|
130
|
+
/[\p{L}\p{N}]/u.test(text[match.index - 1] ?? '') ||
|
|
131
|
+
/[\p{L}\p{N}]/u.test(text[match.index + match[0].length] ?? '')
|
|
132
|
+
) {
|
|
133
|
+
continue;
|
|
134
|
+
}
|
|
135
|
+
// A confirmed full name already protects its components at this location.
|
|
136
|
+
// Keep that span intact for public-name verification and transformations.
|
|
137
|
+
if (
|
|
138
|
+
findings.some(
|
|
139
|
+
(f) =>
|
|
140
|
+
f.type === seed.type &&
|
|
141
|
+
f.start <= match.index &&
|
|
142
|
+
f.end >= match.index + match[0].length
|
|
143
|
+
)
|
|
144
|
+
) {
|
|
145
|
+
continue;
|
|
146
|
+
}
|
|
147
|
+
const id = `${match.index}:${match.index + match[0].length}:${seed.type}`;
|
|
148
|
+
if (!seen.has(id)) {
|
|
149
|
+
seen.add(id);
|
|
150
|
+
result.push({
|
|
151
|
+
...seed,
|
|
152
|
+
start: match.index,
|
|
153
|
+
end: match.index + match[0].length,
|
|
154
|
+
rule: 'document-propagation',
|
|
155
|
+
});
|
|
156
|
+
if (result.length > (options.maxFindings ?? 100000)) {
|
|
157
|
+
throw failure('ERR_LIMIT');
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
return result;
|
|
163
|
+
}
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
import { URL } from 'node:url';
|
|
2
|
+
import { failure } from './detection.js';
|
|
3
|
+
/** Opt-in network lookup: sends only candidate public names, never credentials. */
|
|
4
|
+
export function createWikidataVerifier({
|
|
5
|
+
fetch: request = globalThis.fetch,
|
|
6
|
+
language = 'en',
|
|
7
|
+
timeoutMs = 5000,
|
|
8
|
+
maxEntries = 256,
|
|
9
|
+
} = {}) {
|
|
10
|
+
if (
|
|
11
|
+
typeof request !== 'function' ||
|
|
12
|
+
!/^[a-z]{2,3}(?:-[a-z]+)?$/i.test(language) ||
|
|
13
|
+
!Number.isSafeInteger(timeoutMs) ||
|
|
14
|
+
timeoutMs <= 0 ||
|
|
15
|
+
!Number.isSafeInteger(maxEntries) ||
|
|
16
|
+
maxEntries <= 0
|
|
17
|
+
) {
|
|
18
|
+
throw failure('ERR_CONFIG');
|
|
19
|
+
}
|
|
20
|
+
const cache = new Map();
|
|
21
|
+
return async ({ type, value }) => {
|
|
22
|
+
if (!['PERSON', 'ORGANIZATION'].includes(type)) {
|
|
23
|
+
return undefined;
|
|
24
|
+
}
|
|
25
|
+
const key = `${type}:${value}`;
|
|
26
|
+
if (cache.has(key)) {
|
|
27
|
+
return cache.get(key);
|
|
28
|
+
}
|
|
29
|
+
const url = new URL('https://www.wikidata.org/w/api.php');
|
|
30
|
+
for (const [name, setting] of Object.entries({
|
|
31
|
+
action: 'wbsearchentities',
|
|
32
|
+
format: 'json',
|
|
33
|
+
language,
|
|
34
|
+
search: value,
|
|
35
|
+
limit: '5',
|
|
36
|
+
})) {
|
|
37
|
+
url.searchParams.set(name, setting);
|
|
38
|
+
}
|
|
39
|
+
let payload;
|
|
40
|
+
try {
|
|
41
|
+
const response = await request(url, {
|
|
42
|
+
signal: AbortSignal.timeout(timeoutMs),
|
|
43
|
+
});
|
|
44
|
+
if (!response.ok) {
|
|
45
|
+
throw failure('ERR_PUBLIC_VERIFICATION');
|
|
46
|
+
}
|
|
47
|
+
payload = await response.json();
|
|
48
|
+
if (!Array.isArray(payload.search)) {
|
|
49
|
+
throw failure('ERR_PUBLIC_VERIFICATION');
|
|
50
|
+
}
|
|
51
|
+
} catch {
|
|
52
|
+
throw failure('ERR_PUBLIC_VERIFICATION');
|
|
53
|
+
}
|
|
54
|
+
const match = payload.search.find(
|
|
55
|
+
(entry) =>
|
|
56
|
+
entry.label?.normalize('NFC') === value.normalize('NFC') &&
|
|
57
|
+
/^Q\d+$/.test(entry.id) &&
|
|
58
|
+
(type === 'PERSON'
|
|
59
|
+
? /(?:scientist|politician|writer|actor|singer|historian|physicist|mathematician|president|entrepreneur|philosopher)/i
|
|
60
|
+
: /(?:company|organization|organisation|foundation|agency|university)/i
|
|
61
|
+
).test(entry.description ?? '')
|
|
62
|
+
);
|
|
63
|
+
const result = match
|
|
64
|
+
? {
|
|
65
|
+
type,
|
|
66
|
+
value,
|
|
67
|
+
source: `https://www.wikidata.org/wiki/${match.id}`,
|
|
68
|
+
reviewedAt: new Date().toISOString().slice(0, 10),
|
|
69
|
+
}
|
|
70
|
+
: undefined;
|
|
71
|
+
if (cache.size >= maxEntries) {
|
|
72
|
+
cache.delete(cache.keys().next().value);
|
|
73
|
+
}
|
|
74
|
+
cache.set(key, result);
|
|
75
|
+
return result;
|
|
76
|
+
};
|
|
77
|
+
}
|