@link-foundation/sensitive-data-sanitizer 0.19.3 → 0.21.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/README.md +22 -2
- package/bin/sensitive-data-sanitizer.js +50 -28
- package/docs/API.md +355 -16
- package/docs/COVERAGE.md +19 -9
- package/docs/RULES.md +218 -0
- package/examples/second-pass-placeholders.mjs +28 -0
- package/package.json +12 -3
- package/src/allow-list.js +93 -0
- package/src/confidence.js +24 -9
- package/src/context.js +287 -40
- package/src/credential-spans.js +180 -0
- package/src/credentials-entry.d.ts +23 -0
- package/src/credentials-entry.js +62 -0
- package/src/credentials.js +51 -0
- package/src/crypto-addresses.js +74 -0
- package/src/detection.js +87 -41
- package/src/document-person-cues.js +1 -1
- package/src/encoded.js +137 -51
- package/src/encrypt.js +95 -0
- package/src/engines.js +33 -10
- package/src/entities.js +4 -3
- package/src/exempt.js +72 -0
- package/src/fake-memory.js +32 -0
- package/src/fake.js +33 -29
- package/src/features.js +29 -0
- package/src/files.js +57 -12
- package/src/gitleaks.js +82 -27
- package/src/html-tags.js +8 -0
- package/src/identifier-detector.js +36 -0
- package/src/identity-types.js +63 -0
- package/src/identity.js +3 -47
- package/src/index.d.ts +157 -3
- package/src/index.js +13 -0
- package/src/known.js +33 -15
- package/src/national-checks.js +209 -0
- package/src/national.js +310 -92
- package/src/pattern-detector.js +219 -0
- package/src/personal-context.js +3 -2
- package/src/personal-evidence.js +233 -0
- package/src/personal-features.js +39 -0
- package/src/personal.js +66 -10
- package/src/placeholder-types.js +154 -4
- package/src/placeholders.js +127 -45
- package/src/projection.js +78 -26
- package/src/proper-names.js +48 -5
- package/src/prose-words.js +120 -0
- package/src/record-worker.js +1 -0
- package/src/rule-catalog.js +426 -0
- package/src/rules.js +8 -0
- package/src/sanitizer.js +133 -23
- package/src/sarif.js +127 -0
- package/src/stream-worker.js +1 -0
- package/src/strict-attributes.js +48 -4
- package/src/strict.js +1 -1
- package/src/structured.js +2 -0
- package/src/token-checksum.js +49 -0
- package/src/transforms.js +146 -33
- package/src/vendor/betterleaks/LICENSE +21 -0
- package/src/vendor/betterleaks/provenance.json +11 -0
- package/src/vendor/betterleaks/rules-0.json +463 -0
- package/src/vendor/betterleaks/rules-1.json +454 -0
- package/src/vendor/betterleaks/rules-2.json +461 -0
- package/src/vendor/betterleaks/rules-3.json +455 -0
- package/src/vendor/betterleaks/rules-4.json +483 -0
- package/src/vendor/betterleaks/rules-5.json +35 -0
- package/src/vendor/entities/aws.json +1 -1
- package/src/vendor/entities/azure.json +6 -2
- package/src/vendor/entities/google.json +2 -1
- package/src/vocabulary.js +27 -1
- package/src/worker-options.js +1 -0
package/src/transforms.js
CHANGED
|
@@ -1,10 +1,11 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { feature } from './features.js';
|
|
2
2
|
import { createHmac } from 'node:crypto';
|
|
3
3
|
import { failure, REDACTED } from './detection.js';
|
|
4
4
|
import { encodedValue, encodeRun, decodedRuns } from './encoded.js';
|
|
5
|
-
import { isIdentityType } from './identity.js';
|
|
5
|
+
import { isIdentityType } from './identity-types.js';
|
|
6
6
|
import { validatePlaceholder, placeholderReplacement } from './placeholders.js';
|
|
7
7
|
import { projectText } from './projection.js';
|
|
8
|
+
import { encryptValue } from './encrypt.js';
|
|
8
9
|
|
|
9
10
|
const modes = new Set([
|
|
10
11
|
'redact',
|
|
@@ -16,6 +17,7 @@ const modes = new Set([
|
|
|
16
17
|
'date-shift',
|
|
17
18
|
'bucket',
|
|
18
19
|
'placeholder',
|
|
20
|
+
'encrypt',
|
|
19
21
|
]);
|
|
20
22
|
export function validateTransforms(options) {
|
|
21
23
|
if (
|
|
@@ -86,16 +88,10 @@ function validateTransform(transform) {
|
|
|
86
88
|
Number.isSafeInteger(transform.days) &&
|
|
87
89
|
Math.abs(transform.days) <= 365000,
|
|
88
90
|
bucket: () => positiveInteger(transform.size),
|
|
89
|
-
mask: () =>
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
)
|
|
93
|
-
(transform.minLength === undefined ||
|
|
94
|
-
positiveInteger(transform.minLength)) &&
|
|
95
|
-
(transform.marker === undefined ||
|
|
96
|
-
(typeof transform.marker === 'string' &&
|
|
97
|
-
transform.marker.length > 0 &&
|
|
98
|
-
transform.marker.length <= 64)),
|
|
91
|
+
mask: () => validMask(transform),
|
|
92
|
+
encrypt: () =>
|
|
93
|
+
typeof transform.key === 'string' &&
|
|
94
|
+
Buffer.byteLength(transform.key) >= 16,
|
|
99
95
|
};
|
|
100
96
|
if (checks[transform.mode] && !checks[transform.mode]()) {
|
|
101
97
|
throw failure('ERR_CONFIG');
|
|
@@ -141,16 +137,112 @@ function shiftedDate(value, days) {
|
|
|
141
137
|
? REDACTED
|
|
142
138
|
: date.toISOString().slice(0, 10);
|
|
143
139
|
}
|
|
140
|
+
// Google charactersToIgnore/commonCharactersToIgnore and Presidio-style
|
|
141
|
+
// character classes that a mask may leave visible.
|
|
142
|
+
const maskClasses = {
|
|
143
|
+
digits: /\p{Nd}/u,
|
|
144
|
+
upper: /\p{Lu}/u,
|
|
145
|
+
lower: /\p{Ll}/u,
|
|
146
|
+
punctuation: /[\p{P}\p{S}]/u,
|
|
147
|
+
whitespace: /\s/u,
|
|
148
|
+
};
|
|
149
|
+
const maskTuning = ['char', 'ignore', 'ignoreClasses', 'count', 'fromEnd'];
|
|
150
|
+
const unset = (value, test) => value === undefined || test(value);
|
|
151
|
+
const maskChecks = [
|
|
152
|
+
(t) =>
|
|
153
|
+
[t.keepStart ?? 0, t.keepEnd ?? 0].every(
|
|
154
|
+
(value) => Number.isSafeInteger(value) && value >= 0
|
|
155
|
+
),
|
|
156
|
+
(t) => unset(t.minLength, positiveInteger),
|
|
157
|
+
(t) =>
|
|
158
|
+
unset(
|
|
159
|
+
t.marker,
|
|
160
|
+
(marker) =>
|
|
161
|
+
!maskTuning.some((key) => t[key] !== undefined) &&
|
|
162
|
+
typeof marker === 'string' &&
|
|
163
|
+
marker.length > 0 &&
|
|
164
|
+
marker.length <= 64
|
|
165
|
+
),
|
|
166
|
+
(t) =>
|
|
167
|
+
unset(
|
|
168
|
+
t.char,
|
|
169
|
+
(char) => typeof char === 'string' && Array.from(char).length === 1
|
|
170
|
+
),
|
|
171
|
+
(t) =>
|
|
172
|
+
unset(
|
|
173
|
+
t.ignore,
|
|
174
|
+
(ignore) =>
|
|
175
|
+
typeof ignore === 'string' && ignore.length > 0 && ignore.length <= 256
|
|
176
|
+
),
|
|
177
|
+
(t) =>
|
|
178
|
+
unset(
|
|
179
|
+
t.ignoreClasses,
|
|
180
|
+
(names) =>
|
|
181
|
+
Array.isArray(names) &&
|
|
182
|
+
names.every((name) => Object.hasOwn(maskClasses, name))
|
|
183
|
+
),
|
|
184
|
+
(t) => unset(t.count, positiveInteger),
|
|
185
|
+
(t) => unset(t.fromEnd, (value) => typeof value === 'boolean'),
|
|
186
|
+
];
|
|
187
|
+
function validMask(transform) {
|
|
188
|
+
return maskChecks.every((check) => check(transform));
|
|
189
|
+
}
|
|
190
|
+
// Indexes of the characters a mask replaces, after ignored characters, the
|
|
191
|
+
// direction and the count limit are applied.
|
|
192
|
+
function maskPositions(chars, start, end, transform) {
|
|
193
|
+
const ignored = (char) =>
|
|
194
|
+
Boolean(transform.ignore?.includes(char)) ||
|
|
195
|
+
(transform.ignoreClasses ?? []).some((name) =>
|
|
196
|
+
maskClasses[name].test(char)
|
|
197
|
+
);
|
|
198
|
+
const positions = [];
|
|
199
|
+
for (let i = start; i < chars.length - end; i++) {
|
|
200
|
+
if (!ignored(chars[i])) {
|
|
201
|
+
positions.push(i);
|
|
202
|
+
}
|
|
203
|
+
}
|
|
204
|
+
if (transform.fromEnd) {
|
|
205
|
+
positions.reverse();
|
|
206
|
+
}
|
|
207
|
+
return transform.count === undefined
|
|
208
|
+
? positions
|
|
209
|
+
: positions.slice(0, transform.count);
|
|
210
|
+
}
|
|
144
211
|
function masked(value, finding, transform) {
|
|
145
212
|
const chars = Array.from(value),
|
|
146
213
|
start = transform.keepStart ?? 0,
|
|
147
214
|
end = transform.keepEnd ?? 0;
|
|
148
|
-
|
|
149
|
-
|
|
215
|
+
if (
|
|
216
|
+
chars.length <
|
|
217
|
+
(transform.minLength ?? (isIdentityType(finding.type) ? 5 : 0)) ||
|
|
150
218
|
start + end >= chars.length
|
|
151
|
-
|
|
152
|
-
|
|
219
|
+
) {
|
|
220
|
+
return REDACTED;
|
|
221
|
+
}
|
|
222
|
+
if (transform.marker !== undefined) {
|
|
223
|
+
return `${chars.slice(0, start).join('')}${transform.marker}${end ? chars.slice(-end).join('') : ''}`;
|
|
224
|
+
}
|
|
225
|
+
const positions = maskPositions(chars, start, end, transform);
|
|
226
|
+
if (!positions.length) {
|
|
227
|
+
return REDACTED;
|
|
228
|
+
}
|
|
229
|
+
for (const i of positions) {
|
|
230
|
+
chars[i] = transform.char ?? '*';
|
|
231
|
+
}
|
|
232
|
+
return chars.join('');
|
|
153
233
|
}
|
|
234
|
+
// Modes that apply to personal data only; credentials are always redacted.
|
|
235
|
+
const personalModes = {
|
|
236
|
+
mask: masked,
|
|
237
|
+
encrypt: (value, finding, transform) =>
|
|
238
|
+
encryptValue(value, finding.type, transform.key),
|
|
239
|
+
pseudonym: (value, finding, transform) =>
|
|
240
|
+
`[PSEUDONYM_${keyed(value, finding.type, transform.key).toString('hex').slice(0, 24)}]`,
|
|
241
|
+
'format-preserving': (value, finding, transform) =>
|
|
242
|
+
formatPseudonym(value, finding.type, transform.key),
|
|
243
|
+
'date-shift': (value, finding, transform) =>
|
|
244
|
+
shiftedDate(value, transform.days),
|
|
245
|
+
};
|
|
154
246
|
function transformed(value, finding, transform, options) {
|
|
155
247
|
if (typeof transform === 'function') {
|
|
156
248
|
const result = transform(Object.freeze({ ...finding, value }));
|
|
@@ -170,22 +262,13 @@ function transformed(value, finding, transform, options) {
|
|
|
170
262
|
: REDACTED;
|
|
171
263
|
}
|
|
172
264
|
if (transform.mode === 'fake') {
|
|
173
|
-
return realisticFake(value, finding, transform, options);
|
|
265
|
+
return feature('realisticFake')(value, finding, transform, options);
|
|
174
266
|
}
|
|
175
267
|
if (finding.category === 'credential') {
|
|
176
268
|
return REDACTED;
|
|
177
269
|
}
|
|
178
|
-
if (transform.mode
|
|
179
|
-
return
|
|
180
|
-
}
|
|
181
|
-
if (transform.mode === 'pseudonym') {
|
|
182
|
-
return `[PSEUDONYM_${keyed(value, finding.type, transform.key).toString('hex').slice(0, 24)}]`;
|
|
183
|
-
}
|
|
184
|
-
if (transform.mode === 'format-preserving') {
|
|
185
|
-
return formatPseudonym(value, finding.type, transform.key);
|
|
186
|
-
}
|
|
187
|
-
if (transform.mode === 'date-shift') {
|
|
188
|
-
return shiftedDate(value, transform.days);
|
|
270
|
+
if (Object.hasOwn(personalModes, transform.mode)) {
|
|
271
|
+
return personalModes[transform.mode](value, finding, transform);
|
|
189
272
|
}
|
|
190
273
|
const number = Number(value);
|
|
191
274
|
if (!Number.isSafeInteger(number)) {
|
|
@@ -194,6 +277,32 @@ function transformed(value, finding, transform, options) {
|
|
|
194
277
|
const lower = Math.floor(number / transform.size) * transform.size;
|
|
195
278
|
return `${lower}–${lower + transform.size - 1}`;
|
|
196
279
|
}
|
|
280
|
+
function jsonValueText(raw) {
|
|
281
|
+
try {
|
|
282
|
+
const value = JSON.parse(raw);
|
|
283
|
+
return typeof value === 'string' && value ? value : raw;
|
|
284
|
+
} catch {
|
|
285
|
+
return raw;
|
|
286
|
+
}
|
|
287
|
+
}
|
|
288
|
+
// A whole JSON value becomes one JSON string. Placeholder mode types it by the
|
|
289
|
+
// field it came from, so `{"age":34}` reads `{"age":"Age-1"}`.
|
|
290
|
+
function jsonValueReplacement(text, finding, options) {
|
|
291
|
+
const transform =
|
|
292
|
+
options.transformations?.JSON_VALUE ?? options.transformation;
|
|
293
|
+
if (transform?.mode !== 'placeholder') {
|
|
294
|
+
return JSON.stringify(REDACTED);
|
|
295
|
+
}
|
|
296
|
+
const raw = text.slice(finding.start, finding.end);
|
|
297
|
+
return JSON.stringify(
|
|
298
|
+
placeholderReplacement(
|
|
299
|
+
jsonValueText(raw),
|
|
300
|
+
{ ...finding, type: finding.valueType ?? 'SENSITIVE' },
|
|
301
|
+
transform,
|
|
302
|
+
options
|
|
303
|
+
)
|
|
304
|
+
);
|
|
305
|
+
}
|
|
197
306
|
export function renderReplacement(
|
|
198
307
|
text,
|
|
199
308
|
finding,
|
|
@@ -202,7 +311,7 @@ export function renderReplacement(
|
|
|
202
311
|
jsonRanges
|
|
203
312
|
) {
|
|
204
313
|
if (finding.type === 'JSON_VALUE') {
|
|
205
|
-
return
|
|
314
|
+
return jsonValueReplacement(text, finding, options);
|
|
206
315
|
}
|
|
207
316
|
const transform =
|
|
208
317
|
options.transformations?.[finding.type] ??
|
|
@@ -217,11 +326,15 @@ export function renderReplacement(
|
|
|
217
326
|
const sanitized = encodedSanitize(run.text).text;
|
|
218
327
|
// Mapped escaped fragments can lose their detecting context on decode.
|
|
219
328
|
// Redact the fragment if recursive scanning makes no progress.
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
329
|
+
const encoded = encodeRun(
|
|
330
|
+
run,
|
|
331
|
+
sanitized === run.text ? REDACTED : sanitized
|
|
332
|
+
);
|
|
333
|
+
if (encoded !== undefined) {
|
|
334
|
+
return encoded;
|
|
335
|
+
}
|
|
224
336
|
}
|
|
337
|
+
// Without a recognizable re-encoding, the encoded text itself is masked.
|
|
225
338
|
}
|
|
226
339
|
return renderValue(text, finding, transform, options, jsonRanges);
|
|
227
340
|
}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Zachary Rice
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
{
|
|
2
|
+
"version": "v1.10.0",
|
|
3
|
+
"source": "https://raw.githubusercontent.com/betterleaks/betterleaks/v1.10.0/config/betterleaks.toml",
|
|
4
|
+
"sha256": "3489c5044d3a6e5abfd17a381e9de188b6f4fdee49f7f44bae89e2f8932464fa",
|
|
5
|
+
"rules": 203,
|
|
6
|
+
"files": 6,
|
|
7
|
+
"partialFilters": [
|
|
8
|
+
"box-api-access-token"
|
|
9
|
+
],
|
|
10
|
+
"policy": "Only rule ids absent from the vendored Gitleaks set; original RE2 patterns; entropy and secret-only matchesAny/containsAny filters; no tokenRatio, nearby-line evidence, validation or skipReport components."
|
|
11
|
+
}
|