@link-foundation/sensitive-data-sanitizer 0.0.0-stage → 0.11.31
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +530 -0
- package/LICENSE +24 -0
- package/README.md +184 -2
- package/bin/legacy-example-package-name.js +95 -0
- package/bin/sensitive-data-sanitizer.js +430 -0
- package/bin/stream.js +56 -0
- package/docs/API.md +336 -0
- package/docs/COVERAGE.md +136 -0
- package/docs/HISTORY-REMEDIATION.md +41 -0
- package/examples/combined-scanners.mjs +23 -0
- package/examples/presidio-bridge.py +32 -0
- package/examples/realistic-fakes.mjs +18 -0
- package/examples/sanitize-session.mjs +27 -0
- package/examples/shared-tool-logs.mjs +21 -0
- package/examples/stream-jsonl.mjs +17 -0
- package/package.json +124 -4
- package/src/adapters.js +204 -0
- package/src/arithmetic.js +11 -0
- package/src/confidence.js +65 -0
- package/src/context.js +424 -0
- package/src/credentials.js +54 -0
- package/src/detection.js +293 -0
- package/src/encoded.js +233 -0
- package/src/engines.js +348 -0
- package/src/entities.js +166 -0
- package/src/escape-spans.js +20 -0
- package/src/eslint-plugin.d.ts +10 -0
- package/src/eslint-plugin.js +193 -0
- package/src/external.js +169 -0
- package/src/fake-names.js +222 -0
- package/src/fake.js +524 -0
- package/src/files.js +131 -0
- package/src/gitleaks.js +194 -0
- package/src/history-rewrite.js +339 -0
- package/src/history.js +103 -0
- package/src/identity.js +274 -0
- package/src/index.d.ts +321 -0
- package/src/index.js +40 -0
- package/src/jsonl-stream.js +198 -0
- package/src/keywords.js +45 -0
- package/src/known.js +164 -0
- package/src/legacy.d.ts +3 -0
- package/src/legacy.js +2 -0
- package/src/locales.js +217 -0
- package/src/log-bytes.js +85 -0
- package/src/names.js +65 -0
- package/src/national-checks.js +129 -0
- package/src/national.js +128 -0
- package/src/outbound.js +28 -0
- package/src/personal-context.js +85 -0
- package/src/personal.js +248 -0
- package/src/projection.js +126 -0
- package/src/propagation.js +163 -0
- package/src/public-verifier.js +77 -0
- package/src/public.js +153 -0
- package/src/record-worker.js +20 -0
- package/src/rules.js +210 -0
- package/src/sanitizer.js +794 -0
- package/src/specimens.js +34 -0
- package/src/stream-identity.js +129 -0
- package/src/stream-worker.js +18 -0
- package/src/stream.js +449 -0
- package/src/structured.js +278 -0
- package/src/transforms.js +192 -0
- package/src/transliteration.js +75 -0
- package/src/utf8.js +60 -0
- package/src/vendor/entities/aws.json +42 -0
- package/src/vendor/entities/azure.json +182 -0
- package/src/vendor/entities/google.json +219 -0
- package/src/vendor/gitleaks/LICENSE +21 -0
- package/src/vendor/gitleaks/provenance.json +8 -0
- package/src/vendor/gitleaks/rules-0.json +268 -0
- package/src/vendor/gitleaks/rules-1.json +1794 -0
- package/src/vendor/gitleaks/rules-2.json +292 -0
- package/src/vendor/gitleaks/rules-3.json +263 -0
- package/src/vendor/gitleaks/rules-4.json +277 -0
- package/src/vendor/gitleaks/rules-5.json +154 -0
- package/src/worker-options.js +27 -0
package/src/identity.js
ADDED
|
@@ -0,0 +1,274 @@
|
|
|
1
|
+
import { collectMatches } from './detection.js';
|
|
2
|
+
|
|
3
|
+
export const identityTypes = new Set([
|
|
4
|
+
'SSN',
|
|
5
|
+
'US_SSN',
|
|
6
|
+
'BRAZIL_CPF_NUMBER',
|
|
7
|
+
'INDIA_PAN_INDIVIDUAL',
|
|
8
|
+
'UK_NATIONAL_INSURANCE_NUMBER',
|
|
9
|
+
'SPAIN_DNI_NUMBER',
|
|
10
|
+
'US_HEALTHCARE_NPI',
|
|
11
|
+
'DRIVER_LICENSE',
|
|
12
|
+
'DRIVERS_LICENSE',
|
|
13
|
+
'NATIONAL_ID',
|
|
14
|
+
'CHINA_RESIDENT_ID_NUMBER',
|
|
15
|
+
'HONG_KONG_ID_NUMBER',
|
|
16
|
+
'SOUTH_AFRICA_ID_NUMBER',
|
|
17
|
+
'TAIWAN_ID_NUMBER',
|
|
18
|
+
'TURKEY_ID_NUMBER',
|
|
19
|
+
'GOVERNMENT_ID',
|
|
20
|
+
'DOD_ID_NUMBER',
|
|
21
|
+
'JAPAN_INDIVIDUAL_NUMBER',
|
|
22
|
+
'INDIA_GST_INDIVIDUAL',
|
|
23
|
+
'US_MEDICARE_BENEFICIARY_ID_NUMBER',
|
|
24
|
+
'ID',
|
|
25
|
+
'PASSPORT_NUMBER',
|
|
26
|
+
'PASSPORT_MRZ',
|
|
27
|
+
'BOOKING_REFERENCE',
|
|
28
|
+
'TICKET_NUMBER',
|
|
29
|
+
'VISA_NUMBER',
|
|
30
|
+
'UK_NHS',
|
|
31
|
+
'KR_RRN',
|
|
32
|
+
'IT_FISCAL_CODE',
|
|
33
|
+
'SG_NRIC_FIN',
|
|
34
|
+
'PL_PESEL',
|
|
35
|
+
'SE_PERSONNUMMER',
|
|
36
|
+
'ZA_ID_NUMBER',
|
|
37
|
+
'AU_TFN',
|
|
38
|
+
'CA_SIN',
|
|
39
|
+
'RU_SNILS',
|
|
40
|
+
'RU_INN',
|
|
41
|
+
]);
|
|
42
|
+
export function isIdentityType(type) {
|
|
43
|
+
return (
|
|
44
|
+
identityTypes.has(type) ||
|
|
45
|
+
/PASSPORT|(?:DRIVERS?|DRIVING)_?LICENSE|IDENTITY|NATIONAL_?(?:ID|IDENTIFICATION)|PERSONAL_?(?:ID|IDENTIFICATION)|UNIQUE_?IDENTIFICATION|SOCIAL_?(?:SECURITY|INSURANCE|IDENTIFICATION)|RESIDENT_?REGISTRATION|(?:TAX|TAXPAYER).*?(?:IDENTIFICATION|NUMBER|REFERENCE)|AADHAAR|FISCAL_CODE|PESEL|RRN|NHS|SNILS|INN_NUMBER/.test(
|
|
46
|
+
type
|
|
47
|
+
)
|
|
48
|
+
);
|
|
49
|
+
}
|
|
50
|
+
const personal = (type, rule, confidence = 0.85) => ({
|
|
51
|
+
type,
|
|
52
|
+
rule,
|
|
53
|
+
category: 'personal',
|
|
54
|
+
confidence,
|
|
55
|
+
});
|
|
56
|
+
const passportContext =
|
|
57
|
+
/(?<![\p{L}\p{N}_])(?:passports?|series and number|document number|паспорт[\p{L}]*|загран[\p{L}]*|серия|номер|pasaporte|Reisepass|passeport|passaporto|护照|パスポート)(?![\p{L}\p{N}_])/iu;
|
|
58
|
+
|
|
59
|
+
export function mrzCheck(value) {
|
|
60
|
+
let sum = 0;
|
|
61
|
+
for (let i = 0; i < value.length; i++) {
|
|
62
|
+
const char = value[i];
|
|
63
|
+
const digit =
|
|
64
|
+
char === '<'
|
|
65
|
+
? 0
|
|
66
|
+
: /\d/.test(char)
|
|
67
|
+
? Number(char)
|
|
68
|
+
: char.charCodeAt(0) - 55;
|
|
69
|
+
sum += digit * [7, 3, 1][i % 3];
|
|
70
|
+
}
|
|
71
|
+
return String(sum % 10);
|
|
72
|
+
}
|
|
73
|
+
function mrzConfidence(value) {
|
|
74
|
+
const compact = value.replace(/[ \t\r\n]/g, '');
|
|
75
|
+
const match =
|
|
76
|
+
/^([A-Z0-9<]{9})([\d<])[A-Z<]{3}(\d{6})(\d)[MF<](\d{6})([\d<])/.exec(
|
|
77
|
+
compact
|
|
78
|
+
);
|
|
79
|
+
if (match) {
|
|
80
|
+
return mrzCheck(match[1]) === match[2] &&
|
|
81
|
+
mrzCheck(match[3]) === match[4] &&
|
|
82
|
+
mrzCheck(match[5]) === match[6]
|
|
83
|
+
? 0.99
|
|
84
|
+
: 0.85;
|
|
85
|
+
}
|
|
86
|
+
const td1Header = /^[IAC][A-Z<][A-Z<]{3}([A-Z0-9<]{9})([\d<])/.exec(compact);
|
|
87
|
+
if (td1Header) {
|
|
88
|
+
return mrzCheck(td1Header[1]) === td1Header[2] ? 0.99 : 0.85;
|
|
89
|
+
}
|
|
90
|
+
const td1Data = /^([\d<]{6})(\d)[MF<]([\d<]{6})(\d)[A-Z<]{3}/.exec(compact);
|
|
91
|
+
return td1Data &&
|
|
92
|
+
mrzCheck(td1Data[1]) === td1Data[2] &&
|
|
93
|
+
mrzCheck(td1Data[3]) === td1Data[4]
|
|
94
|
+
? 0.99
|
|
95
|
+
: 0.85;
|
|
96
|
+
}
|
|
97
|
+
function detectMrz(text, emit) {
|
|
98
|
+
// Each candidate has a finite span. Checks improve confidence, never gate
|
|
99
|
+
// redaction: OCR damage and truncated personal data remain sensitive.
|
|
100
|
+
const patterns = [
|
|
101
|
+
/(?<![A-Za-z0-9_])[PIACV][A-Z<][ \t\r\n]{0,4}[A-Z<]{3}[ \t\r\n]{0,4}[A-Z0-9<][A-Z0-9< \t\r\n]{8,100}/g,
|
|
102
|
+
/(?<![A-Za-z0-9_])[A-Z]{2,30}<<[A-Z< \t\r\n]{2,72}/g,
|
|
103
|
+
/(?<![A-Za-z0-9_])[A-Z0-9<]{9}[\d<][ \t\r\n]{0,4}[A-Z<]{3}[ \t\r\n]{0,4}[\d<]{6}[\d<][ \t\r\n]{0,4}[MF<][ \t\r\n]{0,4}[\d<]{6}[\d<][A-Z0-9< \t\r\n]{0,40}/g,
|
|
104
|
+
/(?<![A-Za-z0-9_])[\d<]{6}[\d<][ \t\r\n]{0,4}[MF<][ \t\r\n]{0,4}[\d<]{6}[\d<][ \t\r\n]{0,4}[A-Z<]{3}[A-Z0-9< \t\r\n]{4,20}/g,
|
|
105
|
+
/(?<![A-Za-z0-9_])[A-Z0-9<]{9}[\d<][ \t\r\n]{0,4}[A-Z<]{3}[ \t\r\n]{0,4}\d{6}(?:\d[ \t\r\n]{0,4}[MF<](?:[ \t\r\n]{0,4}[\d<]{1,7})?)?/g,
|
|
106
|
+
];
|
|
107
|
+
patterns.forEach((pattern, index) => {
|
|
108
|
+
for (const match of text.matchAll(pattern)) {
|
|
109
|
+
const value = (
|
|
110
|
+
/[a-z]/.test(text[match.index + match[0].length] ?? '')
|
|
111
|
+
? match[0].replace(/(?:[ \t\r\n]+)[A-Z]{1,6}$/, '')
|
|
112
|
+
: match[0]
|
|
113
|
+
).trimEnd(),
|
|
114
|
+
compact = value.replace(/[ \t\r\n]/g, '');
|
|
115
|
+
const validShape =
|
|
116
|
+
index === 0
|
|
117
|
+
? /^[PIACV][A-Z<][A-Z<]{3}(?:[A-Z]{2,}<<[A-Z<]+|[A-Z0-9<]{9}[\d<]<{2,})/.test(
|
|
118
|
+
compact
|
|
119
|
+
)
|
|
120
|
+
: index === 1
|
|
121
|
+
? /^[A-Z]{2,}<<[A-Z]+</.test(compact)
|
|
122
|
+
: index === 3
|
|
123
|
+
? /\d/.test(compact.slice(0, 7))
|
|
124
|
+
: /[A-Z0-9]/.test(compact.slice(0, 9));
|
|
125
|
+
if (validShape) {
|
|
126
|
+
emit({
|
|
127
|
+
start: match.index,
|
|
128
|
+
end: match.index + value.length,
|
|
129
|
+
...personal('PASSPORT_MRZ', 'mrz', mrzConfidence(value)),
|
|
130
|
+
});
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
});
|
|
134
|
+
}
|
|
135
|
+
function numericNeighbour(text, start, end) {
|
|
136
|
+
return (
|
|
137
|
+
/\d[ ()+.-]{0,4}$/.test(text.slice(Math.max(0, start - 5), start)) ||
|
|
138
|
+
/^[ ()+.-]{0,4}\d/.test(text.slice(end, end + 5))
|
|
139
|
+
);
|
|
140
|
+
}
|
|
141
|
+
function detectPassports(text, emit) {
|
|
142
|
+
const patterns = [
|
|
143
|
+
/(?<![\p{L}\p{N}_])(?:\d{2}[ -]\d{2}[ -]\d{6}|\d{4}[ -]\d{6}|\d{2}[ -]\d{7})(?![\p{L}\p{N}_])/gu,
|
|
144
|
+
/(?<![\p{L}\p{N}_])(?:\d{9}|[CFGHJK][0-9CFGHJKLMNPRTVWXYZ]{8}|[A-Z]\d{7}|M\d{3}[A-Z]\d{4}|[A-Z]{2}\d{7})(?![\p{L}\p{N}_])/gu,
|
|
145
|
+
/(?<![\p{L}\p{N}_])[A-Z0-9<]{9}\d(?![\p{L}\p{N}_])/gu,
|
|
146
|
+
];
|
|
147
|
+
patterns.forEach((pattern, index) => {
|
|
148
|
+
for (const match of text.matchAll(pattern)) {
|
|
149
|
+
const value = match[0],
|
|
150
|
+
start = match.index,
|
|
151
|
+
end = start + value.length;
|
|
152
|
+
const context = text.slice(
|
|
153
|
+
Math.max(0, start - 80),
|
|
154
|
+
Math.min(text.length, end + 80)
|
|
155
|
+
);
|
|
156
|
+
const checked =
|
|
157
|
+
index === 2 &&
|
|
158
|
+
/[A-Z]/.test(value) &&
|
|
159
|
+
mrzCheck(value.slice(0, 9)) === value[9];
|
|
160
|
+
if ((index === 2 && !checked) || numericNeighbour(text, start, end)) {
|
|
161
|
+
continue;
|
|
162
|
+
}
|
|
163
|
+
if (
|
|
164
|
+
/^\d+$/.test(value) &&
|
|
165
|
+
/(?:timestamp|epoch|version|commit|thread|call)[ :=_-]*$/i.test(
|
|
166
|
+
text.slice(Math.max(0, start - 24), start)
|
|
167
|
+
)
|
|
168
|
+
) {
|
|
169
|
+
continue;
|
|
170
|
+
}
|
|
171
|
+
emit({
|
|
172
|
+
start,
|
|
173
|
+
end,
|
|
174
|
+
...personal(
|
|
175
|
+
'PASSPORT_NUMBER',
|
|
176
|
+
'passport-format',
|
|
177
|
+
checked ? 0.99 : passportContext.test(context) ? 0.9 : 0.4
|
|
178
|
+
),
|
|
179
|
+
});
|
|
180
|
+
}
|
|
181
|
+
});
|
|
182
|
+
}
|
|
183
|
+
const booking =
|
|
184
|
+
/(?<![\p{L}\p{N}_])(?:booking(?:[ _-]+(?:reference|code))?|reservation(?:[ _-]+code)?|confirmation(?:[ _-]+code)?|бронь|record[ _-]+locator|PNR|код брони|номер брони|бронирование)[ \t]*[:=:]?[ \t]*[`"']?([A-Z0-9]{4,8})(?![\p{L}\p{N}_])/giu;
|
|
185
|
+
const ticket =
|
|
186
|
+
/(?<![\p{L}\p{N}_])(?:e[ -]?ticket|ticket[ _-]*(?:number|no)?|номер билета|электронный билет)[ \t]*[:=:]?[ \t]*[`"']?(\d{3}[ -]?\d{10})(?![\p{L}\p{N}_])/giu;
|
|
187
|
+
const visa =
|
|
188
|
+
/(?<![\p{L}\p{N}_])(?:e[ -]?visa|visa|номер визы|виза)(?:[ \t_-]+(?:registration|application|number|no|code|номер|код)){0,3}[ \t]*[:=:]?[ \t]*[`"']?([A-Z0-9][A-Z0-9-]{4,31})(?![\p{L}\p{N}_])/giu;
|
|
189
|
+
const placeholder =
|
|
190
|
+
/^(?:NONE|UNKNOWN|NULL|UNDEFINED|REDACTED|EXAMPLE|SAMPLE|PENDING|HOTEL|BOOKING|REFERENCE|REGISTRATION|APPLICATION|NUMBER|CODE|FAILED|ERROR|HTTP|OK|CANCELLED|CONFIRMED|SUCCESS|READY|ACTIVE|CLOSED)$/i;
|
|
191
|
+
function detectTravel(text, emit) {
|
|
192
|
+
collectMatches(
|
|
193
|
+
text,
|
|
194
|
+
booking,
|
|
195
|
+
emit,
|
|
196
|
+
personal('BOOKING_REFERENCE', 'booking-context'),
|
|
197
|
+
1,
|
|
198
|
+
(v) => !placeholder.test(v) && (v === v.toUpperCase() || /\d/.test(v))
|
|
199
|
+
);
|
|
200
|
+
collectMatches(
|
|
201
|
+
text,
|
|
202
|
+
/(?:AirIndia|Air-?India|AirFrance|BritishAirways|Lufthansa|Emirates|Qatar|Delta|United|Ryanair|Aeroflot|Аэрофлот)[-_]([A-Z0-9]{5,8})(?=[-_.])/g,
|
|
203
|
+
emit,
|
|
204
|
+
personal('BOOKING_REFERENCE', 'booking-filename', 0.9),
|
|
205
|
+
1,
|
|
206
|
+
(v) => !placeholder.test(v)
|
|
207
|
+
);
|
|
208
|
+
collectMatches(
|
|
209
|
+
text,
|
|
210
|
+
ticket,
|
|
211
|
+
emit,
|
|
212
|
+
personal('TICKET_NUMBER', 'ticket-context'),
|
|
213
|
+
1
|
|
214
|
+
);
|
|
215
|
+
collectMatches(
|
|
216
|
+
text,
|
|
217
|
+
visa,
|
|
218
|
+
emit,
|
|
219
|
+
personal('VISA_NUMBER', 'visa-context'),
|
|
220
|
+
1,
|
|
221
|
+
(v) => !placeholder.test(v) && /\d/.test(v)
|
|
222
|
+
);
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
export function detectIdentity(text, emit) {
|
|
226
|
+
detectMrz(text, emit);
|
|
227
|
+
detectPassports(text, emit);
|
|
228
|
+
detectTravel(text, emit);
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
export function fakeIdentityValue(value, type, options) {
|
|
232
|
+
if (!isIdentityType(type)) {
|
|
233
|
+
return false;
|
|
234
|
+
}
|
|
235
|
+
if (options.fakeValues?.includes(value)) {
|
|
236
|
+
return true;
|
|
237
|
+
}
|
|
238
|
+
if (options.fakeIdentity !== 'specimen-and-synthetic') {
|
|
239
|
+
return false;
|
|
240
|
+
}
|
|
241
|
+
const compact = value.replace(/\s/g, '').toUpperCase();
|
|
242
|
+
if (/^(?:SPECIMEN|ОБРАЗЕЦ|MUSTERMANN)$/.test(compact)) {
|
|
243
|
+
return true;
|
|
244
|
+
}
|
|
245
|
+
if (type === 'PASSPORT_MRZ') {
|
|
246
|
+
if (
|
|
247
|
+
/^[PIACV][A-Z<]UTO/.test(compact) ||
|
|
248
|
+
/^(?:UTO)?ERIKSSON<<ANNA<MARIA</.test(compact) ||
|
|
249
|
+
/^L898902C3[\d<]UTO/.test(compact) ||
|
|
250
|
+
/^[\d<]{7}[MF<][\d<]{7}UTO/.test(compact)
|
|
251
|
+
) {
|
|
252
|
+
return true;
|
|
253
|
+
}
|
|
254
|
+
const document = /^[PIACV][A-Z<][A-Z<]{3}/.test(compact)
|
|
255
|
+
? compact.slice(5, 14)
|
|
256
|
+
: compact.slice(0, 9);
|
|
257
|
+
return (
|
|
258
|
+
/^(?:C01X00T47|T22000129)$/.test(document) || syntheticDocument(document)
|
|
259
|
+
);
|
|
260
|
+
}
|
|
261
|
+
return (
|
|
262
|
+
/^(?:L898902C3(?:6)?|D23145890(?:7)?|C01X00T47|T22000129)$/.test(compact) ||
|
|
263
|
+
syntheticDocument(compact)
|
|
264
|
+
);
|
|
265
|
+
}
|
|
266
|
+
function syntheticDocument(value) {
|
|
267
|
+
const digits = value.replace(/[ \t-]/g, '');
|
|
268
|
+
return (
|
|
269
|
+
(digits.length >= 5 && /^(\d)\1+$/.test(digits)) ||
|
|
270
|
+
/012345|123456|234567|345678|456789|987654|876543|765432|654321|543210/.test(
|
|
271
|
+
digits
|
|
272
|
+
)
|
|
273
|
+
);
|
|
274
|
+
}
|
package/src/index.d.ts
ADDED
|
@@ -0,0 +1,321 @@
|
|
|
1
|
+
export type Category = 'credential' | 'personal';
|
|
2
|
+
/** UTF-16 offsets, zero-based start and exclusive end; never contains source values. */
|
|
3
|
+
export interface Finding {
|
|
4
|
+
start: number;
|
|
5
|
+
end: number;
|
|
6
|
+
category: Category;
|
|
7
|
+
type: string;
|
|
8
|
+
rule: string;
|
|
9
|
+
/** Detector confidence, not a cross-engine probability calibration. */
|
|
10
|
+
confidence?: number;
|
|
11
|
+
likelihood?:
|
|
12
|
+
| 'VERY_UNLIKELY'
|
|
13
|
+
| 'UNLIKELY'
|
|
14
|
+
| 'POSSIBLE'
|
|
15
|
+
| 'LIKELY'
|
|
16
|
+
| 'VERY_LIKELY';
|
|
17
|
+
/** One-based line and UTF-16 column, populated by inspect. */
|
|
18
|
+
line?: number;
|
|
19
|
+
column?: number;
|
|
20
|
+
/** Reviewed/generated fake exemption; generated credentials require explicit opt-in. */
|
|
21
|
+
kept?: 'fake';
|
|
22
|
+
/** Returned finding replaced with, or recognized as, a generated fake. */
|
|
23
|
+
faked?: true;
|
|
24
|
+
}
|
|
25
|
+
export interface PersonalValue {
|
|
26
|
+
type: string;
|
|
27
|
+
value: string;
|
|
28
|
+
}
|
|
29
|
+
export interface PublicEntity {
|
|
30
|
+
type: 'PERSON' | 'ORGANIZATION' | 'EMAIL';
|
|
31
|
+
value: string;
|
|
32
|
+
/** Organization-controlled HTTPS evidence, reviewed by the policy author. */
|
|
33
|
+
source: string;
|
|
34
|
+
/** ISO calendar date. The library does not fetch the evidence. */
|
|
35
|
+
reviewedAt: string;
|
|
36
|
+
}
|
|
37
|
+
export interface Detector {
|
|
38
|
+
id?: string;
|
|
39
|
+
/** Required detectors must throw on failure and return offsets for this exact input. */
|
|
40
|
+
detect(text: string): Finding[] | Promise<Finding[]>;
|
|
41
|
+
}
|
|
42
|
+
export interface SanitizerOptions {
|
|
43
|
+
/** Preserve all personal/account data; credential findings still redact. */
|
|
44
|
+
credentialsOnly?: boolean;
|
|
45
|
+
profile?: 'publication';
|
|
46
|
+
confirmedPersonal?: Array<{ value: string; type: string }>;
|
|
47
|
+
knownSecrets?: string[];
|
|
48
|
+
knownPersonal?: PersonalValue[];
|
|
49
|
+
publicEntities?: PublicEntity[];
|
|
50
|
+
/** Built-in exact public entities/resolvers and role mail on known domains. */
|
|
51
|
+
publicKnowledge?: boolean;
|
|
52
|
+
findings?: Finding[];
|
|
53
|
+
transformation?: Transformation;
|
|
54
|
+
transformations?: Record<string, Transformation>;
|
|
55
|
+
preserveEncoding?: boolean;
|
|
56
|
+
/** First/last two characters and a fixed *** marker for identity values >=5. */
|
|
57
|
+
identityMask?: boolean;
|
|
58
|
+
/** Explicit opt-in. Words such as test/fake/example never grant exemptions. */
|
|
59
|
+
fakeIdentity?: false | 'specimen-and-synthetic';
|
|
60
|
+
/** Reviewed exact fake identity values, maintained outside published text. */
|
|
61
|
+
fakeValues?: string[];
|
|
62
|
+
/** Scan JSON string keys/values in bounded batches and preserve parseability. */
|
|
63
|
+
structured?: 'json' | 'jsonl';
|
|
64
|
+
/** Property names whose metadata heuristics may be suppressed. */
|
|
65
|
+
structuralFields?: string[];
|
|
66
|
+
/** Personal replacement threshold, default 0.5. Inspect retains lower scores. */
|
|
67
|
+
minConfidence?: number;
|
|
68
|
+
/** Alias for minConfidence; specifying both requires equal values. */
|
|
69
|
+
threshold?: number;
|
|
70
|
+
/** Explicit network/callback verification; credentials never qualify. */
|
|
71
|
+
verifyPublic?: (candidate: {
|
|
72
|
+
type: PublicEntity['type'];
|
|
73
|
+
value: string;
|
|
74
|
+
}) => Promise<PublicEntity | undefined>;
|
|
75
|
+
|
|
76
|
+
paranoid?: boolean;
|
|
77
|
+
decode?: boolean;
|
|
78
|
+
maxInputLength?: number;
|
|
79
|
+
maxFindings?: number;
|
|
80
|
+
debug?: (event: { event: 'detected'; count: number }) => void;
|
|
81
|
+
/** Used by createSanitizer; synchronous functions use native rules only. */
|
|
82
|
+
detectors?: Detector[];
|
|
83
|
+
/** Required by default in createSanitizer. */
|
|
84
|
+
secretlint?: boolean;
|
|
85
|
+
}
|
|
86
|
+
export interface SanitizedText {
|
|
87
|
+
text: string;
|
|
88
|
+
findings: Finding[];
|
|
89
|
+
/** Number of disjoint replacements after overlapping detections are merged. */
|
|
90
|
+
redactions: number;
|
|
91
|
+
}
|
|
92
|
+
export interface Sanitizer {
|
|
93
|
+
inspect(text: string): Promise<Finding[]>;
|
|
94
|
+
sanitize(text: string): Promise<SanitizedText>;
|
|
95
|
+
}
|
|
96
|
+
export interface JsonlSanitizer extends Sanitizer {
|
|
97
|
+
sanitizeJsonl(text: string): Promise<SanitizedText>;
|
|
98
|
+
}
|
|
99
|
+
export declare const REDACTED: '[REDACTED]';
|
|
100
|
+
export declare function inspect(
|
|
101
|
+
text: string,
|
|
102
|
+
options?: SanitizerOptions
|
|
103
|
+
): Finding[];
|
|
104
|
+
export declare function sanitize(
|
|
105
|
+
text: string,
|
|
106
|
+
options?: SanitizerOptions
|
|
107
|
+
): SanitizedText;
|
|
108
|
+
/** Apply validated findings; use sanitize for automatic residual verification. */
|
|
109
|
+
export declare function redact(
|
|
110
|
+
text: string,
|
|
111
|
+
findings: Finding[],
|
|
112
|
+
options?: SanitizerOptions
|
|
113
|
+
): string;
|
|
114
|
+
export declare function createSanitizer(
|
|
115
|
+
options?: SanitizerOptions
|
|
116
|
+
): JsonlSanitizer;
|
|
117
|
+
export declare function entropy(value: string): number;
|
|
118
|
+
export declare function luhn(value: string): boolean;
|
|
119
|
+
export declare function codePointRange(
|
|
120
|
+
text: string,
|
|
121
|
+
start: number,
|
|
122
|
+
end: number
|
|
123
|
+
): { start: number; end: number };
|
|
124
|
+
export declare function byteRange(
|
|
125
|
+
text: string,
|
|
126
|
+
start: number,
|
|
127
|
+
end: number
|
|
128
|
+
): { start: number; end: number };
|
|
129
|
+
export declare function fromGitleaks(
|
|
130
|
+
text: string,
|
|
131
|
+
report: Array<{ Secret: string; [key: string]: unknown }>
|
|
132
|
+
): Finding[];
|
|
133
|
+
export declare function fromTrufflehog(
|
|
134
|
+
text: string,
|
|
135
|
+
report: Array<{ Raw: string; RawV2?: string; [key: string]: unknown }>
|
|
136
|
+
): Finding[];
|
|
137
|
+
export declare function fromDetectSecrets(
|
|
138
|
+
text: string,
|
|
139
|
+
report: {
|
|
140
|
+
results: Record<
|
|
141
|
+
string,
|
|
142
|
+
Array<{ line_number: number; [key: string]: unknown }>
|
|
143
|
+
>;
|
|
144
|
+
},
|
|
145
|
+
filename: string
|
|
146
|
+
): Finding[];
|
|
147
|
+
export declare function fromPresidio(
|
|
148
|
+
text: string,
|
|
149
|
+
report: Array<{
|
|
150
|
+
start: number;
|
|
151
|
+
end: number;
|
|
152
|
+
entity_type: string;
|
|
153
|
+
[key: string]: unknown;
|
|
154
|
+
}>
|
|
155
|
+
): Finding[];
|
|
156
|
+
export declare function fromOffsetReport(
|
|
157
|
+
text: string,
|
|
158
|
+
entries: Array<{
|
|
159
|
+
start: number;
|
|
160
|
+
end: number;
|
|
161
|
+
category: Category;
|
|
162
|
+
type: string;
|
|
163
|
+
[key: string]: unknown;
|
|
164
|
+
}>,
|
|
165
|
+
options: { unit: 'utf16' | 'codepoint' | 'byte'; rule: string }
|
|
166
|
+
): Finding[];
|
|
167
|
+
export declare function knownSecretsFromEnv(
|
|
168
|
+
env?: Record<string, string | undefined>,
|
|
169
|
+
names?: string[]
|
|
170
|
+
): string[];
|
|
171
|
+
/** Reversible UTF-8 view: invalid bytes use unpaired U+DC80..U+DCFF. */
|
|
172
|
+
export declare function decodeLogBytes(bytes: Uint8Array): string;
|
|
173
|
+
/** Encode sanitized text while restoring surrogate-escaped invalid bytes. */
|
|
174
|
+
export declare function encodeLogText(text: string): Uint8Array;
|
|
175
|
+
export declare function knownSecretsFromGitHubConfig(options?: {
|
|
176
|
+
path?: string;
|
|
177
|
+
}): Promise<string[]>;
|
|
178
|
+
export declare function personalVariants(
|
|
179
|
+
entries: PersonalValue[]
|
|
180
|
+
): PersonalValue[];
|
|
181
|
+
export interface LocalScannerOptions {
|
|
182
|
+
command?: string;
|
|
183
|
+
timeoutMs?: number;
|
|
184
|
+
maxOutputBytes?: number;
|
|
185
|
+
}
|
|
186
|
+
export declare function createGitleaksDetector(
|
|
187
|
+
options?: LocalScannerOptions
|
|
188
|
+
): Detector;
|
|
189
|
+
export declare function createTrufflehogDetector(
|
|
190
|
+
options?: LocalScannerOptions
|
|
191
|
+
): Detector;
|
|
192
|
+
export declare function createPresidioDetector(
|
|
193
|
+
options: LocalScannerOptions & { args: string[] }
|
|
194
|
+
): Detector;
|
|
195
|
+
export interface HistoryAudit {
|
|
196
|
+
scanned: number;
|
|
197
|
+
complete: boolean;
|
|
198
|
+
findings: Array<{
|
|
199
|
+
object: string;
|
|
200
|
+
type: 'blob' | 'commit' | 'tag';
|
|
201
|
+
findings: Finding[];
|
|
202
|
+
}>;
|
|
203
|
+
skipped: Array<{ object: string; reason: string }>;
|
|
204
|
+
}
|
|
205
|
+
export declare function auditGitHistory(
|
|
206
|
+
repository: string,
|
|
207
|
+
options?: {
|
|
208
|
+
sanitizer?: Pick<Sanitizer, 'inspect'>;
|
|
209
|
+
maxObjects?: number;
|
|
210
|
+
maxBytes?: number;
|
|
211
|
+
maxTotalBytes?: number;
|
|
212
|
+
}
|
|
213
|
+
): Promise<HistoryAudit>;
|
|
214
|
+
export type Transformation =
|
|
215
|
+
| { mode: 'redact' | 'hive-mask' }
|
|
216
|
+
| {
|
|
217
|
+
mode: 'mask';
|
|
218
|
+
keepStart?: number;
|
|
219
|
+
keepEnd?: number;
|
|
220
|
+
marker?: string;
|
|
221
|
+
minLength?: number;
|
|
222
|
+
}
|
|
223
|
+
| { mode: 'pseudonym' | 'format-preserving'; key: string }
|
|
224
|
+
| {
|
|
225
|
+
mode: 'fake';
|
|
226
|
+
/** At least 16 UTF-8 bytes; keep this pseudonymisation key private. */
|
|
227
|
+
key: string;
|
|
228
|
+
/** Emit visibly invalid fake credentials; default is full redaction. */
|
|
229
|
+
credentials?: boolean;
|
|
230
|
+
/** Keep original MRZ issuer/nationality unless Utopia is requested. */
|
|
231
|
+
mrzCountry?: 'UTO';
|
|
232
|
+
}
|
|
233
|
+
| { mode: 'date-shift'; days: number }
|
|
234
|
+
| { mode: 'bucket'; size: number };
|
|
235
|
+
export interface StreamOptions extends SanitizerOptions {
|
|
236
|
+
/** Atomic file APIs only: spool input privately, collect, then redact. */
|
|
237
|
+
twoPass?: boolean;
|
|
238
|
+
maxRegistryValues?: number;
|
|
239
|
+
sanitizerOptions?: SanitizerOptions;
|
|
240
|
+
sanitizer?: Sanitizer;
|
|
241
|
+
maxRecordBytes?: number;
|
|
242
|
+
batchBytes?: number;
|
|
243
|
+
maxTotalBytes?: number;
|
|
244
|
+
worker?: boolean;
|
|
245
|
+
/** JSONL record workers, 1–16; default 2, or 1 for callbacks/Deno. */
|
|
246
|
+
workers?: number;
|
|
247
|
+
workerHeapMb?: number;
|
|
248
|
+
workerTimeoutMs?: number;
|
|
249
|
+
replace?: boolean;
|
|
250
|
+
}
|
|
251
|
+
export declare function sanitizeStream(
|
|
252
|
+
source: AsyncIterable<string | Uint8Array> | Iterable<string | Uint8Array>,
|
|
253
|
+
options?: StreamOptions
|
|
254
|
+
): AsyncGenerator<string>;
|
|
255
|
+
export declare function sanitizeStreamToFile(
|
|
256
|
+
source: AsyncIterable<string | Uint8Array> | Iterable<string | Uint8Array>,
|
|
257
|
+
path: string,
|
|
258
|
+
options?: StreamOptions
|
|
259
|
+
): Promise<{ outputBytes: number }>;
|
|
260
|
+
export declare function sanitizeFileToFile(
|
|
261
|
+
source: string,
|
|
262
|
+
target: string,
|
|
263
|
+
options?: StreamOptions
|
|
264
|
+
): Promise<{ inputBytes: number; outputBytes: number }>;
|
|
265
|
+
export declare function sanitizeFileBounded(
|
|
266
|
+
source: string,
|
|
267
|
+
target: string,
|
|
268
|
+
options?: StreamOptions
|
|
269
|
+
): Promise<{ inputBytes: number; outputBytes: number }>;
|
|
270
|
+
export declare const entityCatalogs: Record<
|
|
271
|
+
'google' | 'azure' | 'aws',
|
|
272
|
+
{ source: string; sha256: string; names: string[] }
|
|
273
|
+
>;
|
|
274
|
+
export declare function createWikidataVerifier(options?: {
|
|
275
|
+
fetch?: typeof fetch;
|
|
276
|
+
language?: string;
|
|
277
|
+
timeoutMs?: number;
|
|
278
|
+
maxEntries?: number;
|
|
279
|
+
}): NonNullable<SanitizerOptions['verifyPublic']>;
|
|
280
|
+
export declare function knownSecretsFromGitHubAuth(options?: {
|
|
281
|
+
command?: string;
|
|
282
|
+
hostname?: string;
|
|
283
|
+
timeoutMs?: number;
|
|
284
|
+
}): Promise<string[]>;
|
|
285
|
+
export interface OutboundOptions {
|
|
286
|
+
sanitizer?: Pick<Sanitizer, 'sanitize'>;
|
|
287
|
+
sanitizerOptions?: SanitizerOptions;
|
|
288
|
+
}
|
|
289
|
+
export declare function sanitizePayload<T>(
|
|
290
|
+
value: T,
|
|
291
|
+
options?: OutboundOptions
|
|
292
|
+
): Promise<T>;
|
|
293
|
+
export declare function createSentryBeforeSend<T>(
|
|
294
|
+
options?: OutboundOptions
|
|
295
|
+
): (event: T) => Promise<T | null>;
|
|
296
|
+
export declare function createOutboundSanitizer<T, R>(
|
|
297
|
+
publish: (payload: T) => R | Promise<R>,
|
|
298
|
+
options?: OutboundOptions
|
|
299
|
+
): (payload: T) => Promise<R>;
|
|
300
|
+
export interface HistoryRewrite {
|
|
301
|
+
applied: boolean;
|
|
302
|
+
repository: string;
|
|
303
|
+
before: HistoryAudit;
|
|
304
|
+
after?: HistoryAudit;
|
|
305
|
+
changedPaths: number;
|
|
306
|
+
changedRefs: number;
|
|
307
|
+
}
|
|
308
|
+
export declare function rewriteGitHistory(
|
|
309
|
+
source: string,
|
|
310
|
+
destination: string,
|
|
311
|
+
options?: {
|
|
312
|
+
apply?: boolean;
|
|
313
|
+
sanitizer?: Sanitizer;
|
|
314
|
+
sanitizerOptions?: SanitizerOptions;
|
|
315
|
+
filterRepoCommand?: string;
|
|
316
|
+
maxPaths?: number;
|
|
317
|
+
maxObjects?: number;
|
|
318
|
+
maxBytes?: number;
|
|
319
|
+
maxTotalBytes?: number;
|
|
320
|
+
}
|
|
321
|
+
): Promise<HistoryRewrite>;
|
package/src/index.js
ADDED
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
/** Sensitive text detection and verified publication APIs. */
|
|
2
|
+
|
|
3
|
+
export { inspect, sanitize, redact } from './sanitizer.js';
|
|
4
|
+
export { createSanitizer } from './engines.js';
|
|
5
|
+
export { REDACTED, entropy, luhn } from './detection.js';
|
|
6
|
+
export { codePointRange } from './projection.js';
|
|
7
|
+
export {
|
|
8
|
+
fromGitleaks,
|
|
9
|
+
fromTrufflehog,
|
|
10
|
+
fromDetectSecrets,
|
|
11
|
+
fromPresidio,
|
|
12
|
+
byteRange,
|
|
13
|
+
fromOffsetReport,
|
|
14
|
+
} from './adapters.js';
|
|
15
|
+
export { knownSecretsFromEnv, personalVariants } from './known.js';
|
|
16
|
+
export { auditGitHistory } from './history.js';
|
|
17
|
+
export {
|
|
18
|
+
createGitleaksDetector,
|
|
19
|
+
createTrufflehogDetector,
|
|
20
|
+
createPresidioDetector,
|
|
21
|
+
} from './external.js';
|
|
22
|
+
|
|
23
|
+
export { entityCatalogs } from './entities.js';
|
|
24
|
+
export {
|
|
25
|
+
sanitizeStream,
|
|
26
|
+
sanitizeStreamToFile,
|
|
27
|
+
sanitizeFileToFile,
|
|
28
|
+
sanitizeFileBounded,
|
|
29
|
+
} from './stream.js';
|
|
30
|
+
|
|
31
|
+
export { createWikidataVerifier } from './public-verifier.js';
|
|
32
|
+
export {
|
|
33
|
+
sanitizePayload,
|
|
34
|
+
createSentryBeforeSend,
|
|
35
|
+
createOutboundSanitizer,
|
|
36
|
+
} from './outbound.js';
|
|
37
|
+
export { knownSecretsFromGitHubAuth } from './known.js';
|
|
38
|
+
export { knownSecretsFromGitHubConfig } from './known.js';
|
|
39
|
+
export { decodeLogBytes, encodeLogText } from './log-bytes.js';
|
|
40
|
+
export { rewriteGitHistory } from './history-rewrite.js';
|