@link-foundation/sensitive-data-sanitizer 0.20.0 → 0.21.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/README.md +14 -1
- package/bin/sensitive-data-sanitizer.js +1 -0
- package/docs/API.md +179 -16
- package/docs/COVERAGE.md +16 -6
- package/docs/RULES.md +1 -0
- package/examples/second-pass-placeholders.mjs +28 -0
- package/package.json +12 -3
- package/src/context.js +176 -25
- package/src/credential-spans.js +180 -0
- package/src/credentials-entry.d.ts +23 -0
- package/src/credentials-entry.js +62 -0
- package/src/credentials.js +8 -1
- package/src/detection.js +68 -37
- package/src/encoded.js +137 -51
- package/src/engines.js +33 -10
- package/src/exempt.js +72 -0
- package/src/fake-memory.js +32 -0
- package/src/fake.js +4 -28
- package/src/features.js +29 -0
- package/src/html-tags.js +8 -0
- package/src/identity-types.js +63 -0
- package/src/identity.js +3 -61
- package/src/index.d.ts +41 -2
- package/src/index.js +6 -0
- package/src/known.js +33 -15
- package/src/personal-features.js +39 -0
- package/src/personal.js +29 -5
- package/src/placeholder-types.js +164 -4
- package/src/placeholders.js +144 -45
- package/src/projection.js +1 -8
- package/src/proper-names.js +48 -5
- package/src/prose-words.js +116 -0
- package/src/public-technologies.js +32 -0
- package/src/public.js +3 -0
- package/src/record-worker.js +1 -0
- package/src/rule-catalog.js +2 -0
- package/src/rules.js +8 -0
- package/src/sanitizer.js +92 -21
- package/src/stream-worker.js +1 -0
- package/src/strict-attributes.js +60 -4
- package/src/strict.js +1 -1
- package/src/structured.js +2 -0
- package/src/transforms.js +38 -8
- package/src/vocabulary.js +39 -2
- package/src/worker-options.js +1 -0
package/src/context.js
CHANGED
|
@@ -2,7 +2,9 @@ import { labels } from './rules.js';
|
|
|
2
2
|
import { escapePattern, REDACTED } from './detection.js';
|
|
3
3
|
import { credentialPlaceholder, referencePrefix } from './credentials.js';
|
|
4
4
|
import { servicePath } from './public.js';
|
|
5
|
+
import { cookieValues } from './credential-spans.js';
|
|
5
6
|
import { plausiblePerson } from './names.js';
|
|
7
|
+
import { htmlTag } from './html-tags.js';
|
|
6
8
|
|
|
7
9
|
// Bounded vocabularies are explicit so the short PASS/PW forms do not match
|
|
8
10
|
// unrelated words such as passport. Verb forms cover the supported locales.
|
|
@@ -54,12 +56,19 @@ const label = labels.credential
|
|
|
54
56
|
// are fixed so that ordinary words ending in "pass" stay prose.
|
|
55
57
|
const glued =
|
|
56
58
|
'(?:db|user|root|admin|smtp|mail|ftp|sftp|ssh|redis|ldap|bind|proxy|tls|ssl|keystore|truststore|store|master|sys|app|login|account|acct|wifi|wpa)(?:password|passwd|pass|pwd|pw|secret|token)';
|
|
57
|
-
|
|
59
|
+
// Plural credential names (`--access-tokens`, `API_KEYS`) hold credentials;
|
|
60
|
+
// other plurals (`--max-tokens`, `tokens: 1234`) are accounting.
|
|
61
|
+
const plural =
|
|
62
|
+
'(?:access|refresh|auth|session|api|bearer|client)[_.-]?(?:tokens|keys|secrets)';
|
|
63
|
+
const key = `(?:[\\p{L}\\p{N}]+[_.-])*(?:auth|authorization|PGPASSWORD|${glued}|${plural}|${label}|pass|pw)(?:[_.-][\\p{L}\\p{N}]+)*`;
|
|
64
|
+
// Ruby/Perl/PHP hash rockets (`'password' => 'x'`, `:token => "x"`) take a
|
|
65
|
+
// quoted literal; an arrow function (`token => token.trim()`) does not.
|
|
66
|
+
const rocket = `=>(?=[ \\t]*\\\\*["'])`;
|
|
58
67
|
// A qualifier such as "for staging" stays on one logical line: it never
|
|
59
68
|
// crosses an escaped line break, an escape sequence or an enclosing quote.
|
|
60
69
|
// Verbs are whole words, so "request" does not end in the verb "est".
|
|
61
70
|
const credentialPrefix = new RegExp(
|
|
62
|
-
`(?<![\\p{L}\\p{N}_])(?:\\\\*["'])?(${key})(?![\\p{L}\\p{N}_])(?:\\\\*["'])?(?:[ \\t]+(?:от|для|к|for|de|du|del)[ \\t]+[^:=\\r\\n"\\\\]{1,64})?[ \\t]*([:=:]|(?<![\\p{L}\\p{N}_])(?:${verbs})(?![\\p{L}\\p{N}_]))[ \\t]*`,
|
|
71
|
+
`(?<![\\p{L}\\p{N}_])(?:\\\\*["'])?(${key})(?![\\p{L}\\p{N}_])(?:\\\\*["'])?(?:[ \\t]+(?:от|для|к|for|de|du|del)[ \\t]+[^:=\\r\\n"\\\\]{1,64})?[ \\t]*(${rocket}|[:=:]|(?<![\\p{L}\\p{N}_])(?:${verbs})(?![\\p{L}\\p{N}_]))[ \\t]*`,
|
|
63
72
|
'giu'
|
|
64
73
|
);
|
|
65
74
|
// camelCase and PascalCase keys (`clientSecret`, `secretKey`, `dbPassword`,
|
|
@@ -68,7 +77,7 @@ const credentialPrefix = new RegExp(
|
|
|
68
77
|
const camelKey =
|
|
69
78
|
'[\\p{L}][\\p{Ll}\\p{N}]*(?:\\p{Lu}[\\p{Ll}\\p{N}]*)*?(?:Secret|Password|Passwd|Passphrase|Pass|Pwd|Token|Credentials?|(?:Api|Private|Access|Secret|Signing|Encryption|Master|Client|App|Subscription|Account|Storage)Key)(?:Key|Value)?|(?:[Ss]ecret|[Pp]assword|[Pp]asswd|[Tt]oken|[Pp]rivate|[Aa]ccess|[Ss]igning|[Ee]ncryption)(?:Key|Value|Secret|Token)';
|
|
70
79
|
const camelPrefix = new RegExp(
|
|
71
|
-
`(?<![\\p{L}\\p{N}_$])(?:\\\\*["'])?(${camelKey})(?![\\p{L}\\p{N}_])(?:\\\\*["'])?[ \\t]*([:=])[ \\t]*`,
|
|
80
|
+
`(?<![\\p{L}\\p{N}_$])(?:\\\\*["'])?(${camelKey})(?![\\p{L}\\p{N}_])(?:\\\\*["'])?[ \\t]*(${rocket}|[:=])[ \\t]*`,
|
|
72
81
|
'gu'
|
|
73
82
|
);
|
|
74
83
|
// Bare generic labels are common English nouns. After a verb they need a
|
|
@@ -120,8 +129,20 @@ function emitValue(text, start, end, emit, details, keyName = '') {
|
|
|
120
129
|
emit({ start, end, ...details });
|
|
121
130
|
}
|
|
122
131
|
|
|
132
|
+
// `HIVE_MIND_ROUTER_AUTH_GUARD=0`, `GIT_AUTH_TRANSPORT=1`, `AUTH_MODE=2`:
|
|
133
|
+
// a number or a boolean under a configuration name is a setting. Credential
|
|
134
|
+
// names (`AUTH_TOKEN=123456`, `AUTH_PASSWORD=1`) do not end in one of these.
|
|
135
|
+
const configurationSuffix =
|
|
136
|
+
/[_.-](?:guard|transport|mode|method|type|kind|flow|strategy|scheme|protocol|port|level|state|status|enabled?|disabled?|required|strict|debug|verbose|timeout|ttl|interval|retries|retry|attempts|count|limit|max|min|size|length|version|flag)$/i;
|
|
137
|
+
const configurationValue =
|
|
138
|
+
/^(?:[+-]?\d{1,9}(?:\.\d+)?|true|false|yes|no|on|off|enabled|disabled|none)$/i;
|
|
139
|
+
export function configurationSetting(keyName, value) {
|
|
140
|
+
return configurationSuffix.test(keyName) && configurationValue.test(value);
|
|
141
|
+
}
|
|
142
|
+
|
|
123
143
|
function credentialMetadata(keyName, value) {
|
|
124
144
|
return (
|
|
145
|
+
configurationSetting(keyName, value) ||
|
|
125
146
|
(/(?:^|[_.-])auth[_.-](?:mode|method|type)$/i.test(keyName) &&
|
|
126
147
|
/^(?:chatgpt|oauth2?|apikey|api_key|bearer|basic|none)$/i.test(value)) ||
|
|
127
148
|
(keyName === 'id-token' && /^(?:write|read|none)$/.test(value)) ||
|
|
@@ -167,15 +188,27 @@ export function valueRange(text, start, mode = 'token', maxLength = Infinity) {
|
|
|
167
188
|
}
|
|
168
189
|
// Header credentials occupy a scheme and one token even inside shell/JSON
|
|
169
190
|
// strings. A closing quote or escape belongs to the surrounding log record.
|
|
170
|
-
|
|
191
|
+
// The scheme (`token`, `Bearer`, `Digest`) stays readable; only the
|
|
192
|
+
// credential is returned. Auth-param lists (`Digest username="a", nonce=…`,
|
|
193
|
+
// AWS SigV4) return each secret parameter value.
|
|
194
|
+
export function authorizationRanges(text, start) {
|
|
171
195
|
const opening = /^(\\*)(["'])/.exec(text.slice(start));
|
|
172
196
|
start += opening?.[0].length ?? 0;
|
|
173
197
|
const reference = referencePrefix(text.slice(start, start + 512));
|
|
174
198
|
if (reference) {
|
|
175
|
-
return { start, end: start + reference[0].length };
|
|
199
|
+
return [{ start, end: start + reference[0].length }];
|
|
176
200
|
}
|
|
177
201
|
const scheme = /^[a-z][a-z0-9_-]*[ \t]+/i.exec(text.slice(start));
|
|
178
202
|
const tokenStart = start + (scheme?.[0].length ?? 0);
|
|
203
|
+
const params = scheme ? authParams(text, tokenStart) : undefined;
|
|
204
|
+
if (params) {
|
|
205
|
+
return params;
|
|
206
|
+
}
|
|
207
|
+
const end = authorizationEnd(text, tokenStart);
|
|
208
|
+
const token = text.slice(tokenStart, end);
|
|
209
|
+
return usableAuthorization(token) ? [{ start: tokenStart, end }] : [];
|
|
210
|
+
}
|
|
211
|
+
function authorizationEnd(text, tokenStart) {
|
|
179
212
|
const brackets = [];
|
|
180
213
|
let end = tokenStart;
|
|
181
214
|
for (; end < text.length; end++) {
|
|
@@ -195,8 +228,69 @@ export function authorizationRange(text, start) {
|
|
|
195
228
|
break;
|
|
196
229
|
}
|
|
197
230
|
}
|
|
198
|
-
|
|
199
|
-
|
|
231
|
+
return end;
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
// RFC 7616/7235 parameters that name the request, not a secret.
|
|
235
|
+
const publicAuthParam =
|
|
236
|
+
/^(?:username|realm|uri|qop|algorithm|nc|charset|userhash|domain|stale|SignedHeaders|oauth_signature_method|oauth_version|oauth_timestamp|oauth_callback|keyId|headers|created|expires)$/i;
|
|
237
|
+
const authParam = /^([A-Za-z][\w-]*)[ \t]*=[ \t]*/;
|
|
238
|
+
function authParams(text, start) {
|
|
239
|
+
const ranges = [];
|
|
240
|
+
let cursor = start,
|
|
241
|
+
count = 0,
|
|
242
|
+
quoted = false;
|
|
243
|
+
for (;;) {
|
|
244
|
+
const name = authParam.exec(text.slice(cursor, cursor + 128));
|
|
245
|
+
if (!name) {
|
|
246
|
+
break;
|
|
247
|
+
}
|
|
248
|
+
const valueStart = cursor + name[0].length;
|
|
249
|
+
const value = authParamValue(text, valueStart);
|
|
250
|
+
if (!value) {
|
|
251
|
+
break;
|
|
252
|
+
}
|
|
253
|
+
count++;
|
|
254
|
+
quoted ||= value.quoted;
|
|
255
|
+
if (!publicAuthParam.test(name[1]) && value.end > value.start) {
|
|
256
|
+
ranges.push({ start: value.start, end: value.end });
|
|
257
|
+
}
|
|
258
|
+
const separator = /^[ \t]*,[ \t]*/.exec(
|
|
259
|
+
text.slice(value.after, value.after + 64)
|
|
260
|
+
);
|
|
261
|
+
if (!separator) {
|
|
262
|
+
break;
|
|
263
|
+
}
|
|
264
|
+
cursor = value.after + separator[0].length;
|
|
265
|
+
}
|
|
266
|
+
// `Basic dXNlcg==` is one token68 credential, not a parameter list.
|
|
267
|
+
return count > 1 || (count === 1 && quoted) ? ranges : undefined;
|
|
268
|
+
}
|
|
269
|
+
function authParamValue(text, start) {
|
|
270
|
+
const quote = /^(\\*)"/.exec(text.slice(start, start + 64));
|
|
271
|
+
if (quote) {
|
|
272
|
+
// Parameter values are short; a quote further away is not their end.
|
|
273
|
+
const window = text.slice(start, start + 4096);
|
|
274
|
+
const offset = window.indexOf(quote[0], quote[0].length);
|
|
275
|
+
const close = start + offset;
|
|
276
|
+
if (offset < 0 || /[\r\n]/.test(window.slice(0, offset))) {
|
|
277
|
+
return undefined;
|
|
278
|
+
}
|
|
279
|
+
const after = close + quote[0].length;
|
|
280
|
+
// A quoted value ends the header or continues the list.
|
|
281
|
+
if (
|
|
282
|
+
!/^(?:[ \t]*(?:,|$|[\r\n])|\\*["'\r\n])/.test(
|
|
283
|
+
text.slice(after, after + 64)
|
|
284
|
+
)
|
|
285
|
+
) {
|
|
286
|
+
return undefined;
|
|
287
|
+
}
|
|
288
|
+
return { start: start + quote[0].length, end: close, after, quoted: true };
|
|
289
|
+
}
|
|
290
|
+
const token = /^[^\s,"'\\]{1,4096}/.exec(text.slice(start, start + 4096));
|
|
291
|
+
return token
|
|
292
|
+
? { start, end: start + token[0].length, after: start + token[0].length }
|
|
293
|
+
: undefined;
|
|
200
294
|
}
|
|
201
295
|
function usableAuthorization(token) {
|
|
202
296
|
return Boolean(token) && !credentialPlaceholder(token);
|
|
@@ -275,6 +369,25 @@ function operationalLocationPath(value) {
|
|
|
275
369
|
servicePath(value) || /^(?:\.?\.?\/)?[\w.-]+(?:\/[\w.-]+)+\/?$/.test(value)
|
|
276
370
|
);
|
|
277
371
|
}
|
|
372
|
+
// Types with a fixed value shape decide on the value alone; `undefined` leaves
|
|
373
|
+
// the decision to the general checks.
|
|
374
|
+
function shapedValueAllowed(value, type) {
|
|
375
|
+
if (type === 'PHONE') {
|
|
376
|
+
return (
|
|
377
|
+
/^\+?[\d () .-]{7,40}$/.test(value) &&
|
|
378
|
+
/\d{3}/.test(value.replace(/\D/g, ''))
|
|
379
|
+
);
|
|
380
|
+
}
|
|
381
|
+
if (type === 'EMAIL') {
|
|
382
|
+
return /^[^\s@]+@[^\s@]+\.[\p{L}]{2,63}$/u.test(value);
|
|
383
|
+
}
|
|
384
|
+
// A URL holds no whitespace, so prose under a `url` key is left to the
|
|
385
|
+
// detectors of what the prose contains.
|
|
386
|
+
if (type === 'URL' && /\s/u.test(value)) {
|
|
387
|
+
return false;
|
|
388
|
+
}
|
|
389
|
+
return undefined;
|
|
390
|
+
}
|
|
278
391
|
// Shared by native labels and the vendor catalogs. Code expressions and
|
|
279
392
|
// line escapes are not personal field values, even when a key is familiar.
|
|
280
393
|
export function personalValueAllowed(text, range, type, labelStart) {
|
|
@@ -288,14 +401,9 @@ export function personalValueAllowed(text, range, type, labelStart) {
|
|
|
288
401
|
) {
|
|
289
402
|
return false;
|
|
290
403
|
}
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
/\d{3}/.test(value.replace(/\D/g, ''))
|
|
295
|
-
);
|
|
296
|
-
}
|
|
297
|
-
if (type === 'EMAIL') {
|
|
298
|
-
return /^[^\s@]+@[^\s@]+\.[\p{L}]{2,63}$/u.test(value);
|
|
404
|
+
const shaped = shapedValueAllowed(value, type);
|
|
405
|
+
if (shaped !== undefined) {
|
|
406
|
+
return shaped;
|
|
299
407
|
}
|
|
300
408
|
if (/^(?:\\[nrt]|[\r\n])/.test(value)) {
|
|
301
409
|
return false;
|
|
@@ -391,12 +499,38 @@ function credentialMode(text, match) {
|
|
|
391
499
|
) {
|
|
392
500
|
return 'query';
|
|
393
501
|
}
|
|
394
|
-
if (match[2]
|
|
502
|
+
if (/^=>?$/.test(match[2]) || /["']\s*[::]/.test(match[0])) {
|
|
395
503
|
return 'token';
|
|
396
504
|
}
|
|
397
505
|
return /[::]/.test(match[2]) ? 'line' : 'prose';
|
|
398
506
|
}
|
|
399
507
|
|
|
508
|
+
// `token: <script>alert(1)</script>`: markup is the value. A bare
|
|
509
|
+
// `<placeholder>` that ends the value stays a placeholder.
|
|
510
|
+
const angleEnd = {
|
|
511
|
+
line: /[\r\n"]|\\+[nrt"']/,
|
|
512
|
+
token: /[\s,;}\]"']|\\+[nrt"']/,
|
|
513
|
+
};
|
|
514
|
+
// Formatting tags around a value (`token=<em>…</em>`) are dropped from the
|
|
515
|
+
// detection view instead.
|
|
516
|
+
const formattingTag = new RegExp(`^${htmlTag}`, 'i');
|
|
517
|
+
function angleValue(text, start, mode) {
|
|
518
|
+
const tail = text.slice(start, start + 4096);
|
|
519
|
+
if (text[start] !== '<' || !angleEnd[mode] || formattingTag.test(tail)) {
|
|
520
|
+
return undefined;
|
|
521
|
+
}
|
|
522
|
+
const placeholder = /^<[^<>\r\n]{1,64}>/.exec(tail)?.[0];
|
|
523
|
+
if (
|
|
524
|
+
placeholder &&
|
|
525
|
+
credentialPlaceholder(placeholder) &&
|
|
526
|
+
angleEnd[mode].test(tail[placeholder.length] ?? '\n')
|
|
527
|
+
) {
|
|
528
|
+
return { start, end: start + placeholder.length };
|
|
529
|
+
}
|
|
530
|
+
const stop = tail.search(angleEnd[mode]);
|
|
531
|
+
return { start, end: start + (stop < 0 ? tail.length : stop) };
|
|
532
|
+
}
|
|
533
|
+
|
|
400
534
|
function credentialRange(text, match, camel) {
|
|
401
535
|
const start = match.index + match[0].length;
|
|
402
536
|
if (
|
|
@@ -405,13 +539,25 @@ function credentialRange(text, match, camel) {
|
|
|
405
539
|
) {
|
|
406
540
|
return undefined;
|
|
407
541
|
}
|
|
542
|
+
if (/^(?:proxy-)?authorization$/i.test(match[1])) {
|
|
543
|
+
return authorizationRanges(text, start);
|
|
544
|
+
}
|
|
408
545
|
const mode = credentialMode(text, match);
|
|
409
|
-
const range =
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
546
|
+
const range = angleValue(text, start, mode) ?? valueRange(text, start, mode);
|
|
547
|
+
const value = range && mode === 'prose' ? generatedWord(text, range) : range;
|
|
548
|
+
if (value && /^(?:set-)?cookie$/i.test(match[1])) {
|
|
549
|
+
return cookieValues(text, value.start, value.end);
|
|
550
|
+
}
|
|
551
|
+
return [value].filter(Boolean);
|
|
413
552
|
}
|
|
414
553
|
|
|
554
|
+
// `<password>…</password>`, `<clientSecret attr="1">…</clientSecret>`:
|
|
555
|
+
// attributes in the opening tag stay readable.
|
|
556
|
+
const element = (name) =>
|
|
557
|
+
`<(${name})(?:[ \\t\\r\\n]+[^<>]{0,512})?>([^<]+)</\\1[ \\t]*>`;
|
|
558
|
+
const xmlElement = new RegExp(element(key), 'giu');
|
|
559
|
+
const camelElement = new RegExp(element(camelKey), 'gu');
|
|
560
|
+
|
|
415
561
|
function detectCredentials(text, emit) {
|
|
416
562
|
const details = { category: 'credential', type: 'SECRET', rule: 'context' };
|
|
417
563
|
const matches = [
|
|
@@ -419,13 +565,18 @@ function detectCredentials(text, emit) {
|
|
|
419
565
|
...[...text.matchAll(camelPrefix)].map((match) => [match, true]),
|
|
420
566
|
];
|
|
421
567
|
for (const [match, camel] of matches) {
|
|
422
|
-
|
|
423
|
-
if (
|
|
424
|
-
|
|
568
|
+
// `https://x-access-token:…@host` is URL user info (`url-userinfo`).
|
|
569
|
+
if (/:\/\/$/.test(text.slice(Math.max(0, match.index - 3), match.index))) {
|
|
570
|
+
continue;
|
|
571
|
+
}
|
|
572
|
+
for (const value of credentialRange(text, match, camel) ?? []) {
|
|
573
|
+
emitValue(text, value.start, value.end, emit, details, match[1]);
|
|
425
574
|
}
|
|
426
575
|
}
|
|
427
|
-
const
|
|
428
|
-
|
|
576
|
+
for (const match of [
|
|
577
|
+
...text.matchAll(xmlElement),
|
|
578
|
+
...text.matchAll(camelElement),
|
|
579
|
+
]) {
|
|
429
580
|
const start = match.index + match[0].indexOf('>') + 1;
|
|
430
581
|
emitValue(text, start, start + match[2].length, emit, details, match[1]);
|
|
431
582
|
}
|
|
@@ -0,0 +1,180 @@
|
|
|
1
|
+
import { maskedValue } from './credentials.js';
|
|
2
|
+
|
|
3
|
+
// Credential findings cover the secret, not the readable context around it:
|
|
4
|
+
// PEM markers, URL hosts and user names, cookie names and the rest of a
|
|
5
|
+
// sentence stay visible, so a masked log still shows what was masked.
|
|
6
|
+
|
|
7
|
+
// Context rules that read "to the end of the value" without knowing the
|
|
8
|
+
// value's format. A structured vendor token that starts the same value
|
|
9
|
+
// bounds them.
|
|
10
|
+
const openEnded = new Set([
|
|
11
|
+
'context',
|
|
12
|
+
'cookie',
|
|
13
|
+
'named-value',
|
|
14
|
+
'cli-password',
|
|
15
|
+
'quoted-entropy',
|
|
16
|
+
'entropy',
|
|
17
|
+
'gitleaks/generic-api-key',
|
|
18
|
+
'betterleaks/generic-api-key',
|
|
19
|
+
]);
|
|
20
|
+
const notBounding = new Set([...openEnded, 'known-secret', 'authorization']);
|
|
21
|
+
const lineBreak = '(?:\\r?\\n|\\\\+[nr])';
|
|
22
|
+
const pemMarker = new RegExp(
|
|
23
|
+
`-----(BEGIN|END) [A-Z0-9 _-]{0,100}-----(?:[ \\t]*${lineBreak})?`,
|
|
24
|
+
'g'
|
|
25
|
+
);
|
|
26
|
+
|
|
27
|
+
// Body spans of every BEGIN/END block inside [start, end). Text between two
|
|
28
|
+
// blocks never belongs to a key; an unterminated block runs to `end`.
|
|
29
|
+
export function pemBodies(text, start, end) {
|
|
30
|
+
const segment = text.slice(start, end);
|
|
31
|
+
if (!/^(?:\\*["'])?-----BEGIN /.test(segment)) {
|
|
32
|
+
return undefined;
|
|
33
|
+
}
|
|
34
|
+
const bodies = [];
|
|
35
|
+
let open;
|
|
36
|
+
for (const marker of segment.matchAll(pemMarker)) {
|
|
37
|
+
if (marker[1] === 'BEGIN') {
|
|
38
|
+
open = marker.index + marker[0].length;
|
|
39
|
+
} else if (open !== undefined) {
|
|
40
|
+
bodies.push(trimmed(text, start + open, start + marker.index));
|
|
41
|
+
open = undefined;
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
if (open !== undefined) {
|
|
45
|
+
bodies.push(trimmed(text, start + open, end));
|
|
46
|
+
}
|
|
47
|
+
return bodies.filter((body) => body.end > body.start);
|
|
48
|
+
}
|
|
49
|
+
function trimmed(text, start, end) {
|
|
50
|
+
const body = text.slice(start, end);
|
|
51
|
+
const lead = /^(?:\s|\\+[nrt])+/.exec(body)?.[0].length ?? 0;
|
|
52
|
+
const tail = /(?:\s|\\+[nrt])+$/.exec(body.slice(lead))?.[0].length ?? 0;
|
|
53
|
+
return { start: start + lead, end: end - tail };
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
// `scheme://user:password@host`: only the password is the credential.
|
|
57
|
+
const userinfo = /^[a-z][a-z0-9+.-]*:\/\/([^\s/@:]+):([^\s/@]+)@/i;
|
|
58
|
+
// A URL secret keeps its scheme, host and fixed path words
|
|
59
|
+
// (`https://hooks.slack.com/services/`), so a reader still sees which
|
|
60
|
+
// endpoint it was.
|
|
61
|
+
const serviceUser =
|
|
62
|
+
/^(?:x-access-token|x-oauth-basic|x-token-auth|oauth2?|git|gitlab-ci-token|token|pat|apikey|api)$/i;
|
|
63
|
+
const endpoint = /^(?:https?:\/\/)?[a-z0-9-]+(?:\.[a-z0-9-]+)+\//i;
|
|
64
|
+
const pathWord = /^(?:[a-z]{2,20}(?:\d{1,2})?\/)+/i;
|
|
65
|
+
|
|
66
|
+
function urlSecret(text, finding) {
|
|
67
|
+
const value = text.slice(finding.start, finding.end);
|
|
68
|
+
// `url-userinfo` already covers only `user:password`.
|
|
69
|
+
const own = finding.rule === 'url-userinfo';
|
|
70
|
+
const credentials = own
|
|
71
|
+
? /^([^:]+):(.+)$/s.exec(value)
|
|
72
|
+
: userinfo.exec(value);
|
|
73
|
+
if (credentials) {
|
|
74
|
+
const end = finding.start + credentials[0].length - (own ? 0 : 1);
|
|
75
|
+
const start = end - credentials[2].length;
|
|
76
|
+
const user = start - 1 - credentials[1].length;
|
|
77
|
+
return [
|
|
78
|
+
{ start, end },
|
|
79
|
+
// Outside credentialsOnly the user name is still personal data.
|
|
80
|
+
...(serviceUser.test(credentials[1]) || maskedValue(credentials[1])
|
|
81
|
+
? []
|
|
82
|
+
: [
|
|
83
|
+
{
|
|
84
|
+
start: user,
|
|
85
|
+
end: user + credentials[1].length,
|
|
86
|
+
type: 'USERNAME',
|
|
87
|
+
category: 'personal',
|
|
88
|
+
},
|
|
89
|
+
]),
|
|
90
|
+
];
|
|
91
|
+
}
|
|
92
|
+
const host = endpoint.exec(value);
|
|
93
|
+
if (!host || finding.rule === 'url-userinfo') {
|
|
94
|
+
return undefined;
|
|
95
|
+
}
|
|
96
|
+
const words = pathWord.exec(value.slice(host[0].length))?.[0] ?? '';
|
|
97
|
+
const start = finding.start + host[0].length + words.length;
|
|
98
|
+
return finding.end - start >= 8 ? [{ start, end: finding.end }] : undefined;
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
function narrowed(text, finding) {
|
|
102
|
+
if (finding.category !== 'credential') {
|
|
103
|
+
return [finding];
|
|
104
|
+
}
|
|
105
|
+
const parts =
|
|
106
|
+
pemBodies(text, finding.start, finding.end) ?? urlSecret(text, finding);
|
|
107
|
+
return parts ? parts.map((part) => ({ ...finding, ...part })) : [finding];
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
// `token: ghp_… . Next sentence` — a context value that a vendor rule ends
|
|
111
|
+
// earlier stops at that end when only punctuation or a space follows.
|
|
112
|
+
function bounded(text, finding, findings) {
|
|
113
|
+
if (!openEnded.has(finding.rule) || finding.category !== 'credential') {
|
|
114
|
+
return finding;
|
|
115
|
+
}
|
|
116
|
+
let end = finding.end;
|
|
117
|
+
for (const other of findings) {
|
|
118
|
+
if (
|
|
119
|
+
other.category === 'credential' &&
|
|
120
|
+
!notBounding.has(other.rule) &&
|
|
121
|
+
other.start === finding.start &&
|
|
122
|
+
other.end < end &&
|
|
123
|
+
other.end > other.start &&
|
|
124
|
+
/^[.,;:!?]*(?:\s|$)/.test(text.slice(other.end, finding.end))
|
|
125
|
+
) {
|
|
126
|
+
end = other.end;
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
return end === finding.end ? finding : { ...finding, end };
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
// PEM armor (`-----BEGIN RSA PRIVATE KEY-----`) stays readable, and its
|
|
133
|
+
// upper-case words are not a person or an organization.
|
|
134
|
+
function pemMarkers(text, findings) {
|
|
135
|
+
const markers = [];
|
|
136
|
+
for (const finding of findings) {
|
|
137
|
+
const segment = text.slice(finding.start, finding.end);
|
|
138
|
+
if (finding.category === 'credential' && /-----BEGIN /.test(segment)) {
|
|
139
|
+
for (const marker of segment.matchAll(pemMarker)) {
|
|
140
|
+
const start = finding.start + marker.index;
|
|
141
|
+
markers.push({ start, end: start + marker[0].length });
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
return markers;
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
export function refineCredentialSpans(text, findings) {
|
|
149
|
+
const markers = pemMarkers(text, findings);
|
|
150
|
+
const narrow = findings
|
|
151
|
+
.filter(
|
|
152
|
+
(finding) =>
|
|
153
|
+
finding.category === 'credential' ||
|
|
154
|
+
!markers.some(
|
|
155
|
+
(marker) => finding.start < marker.end && marker.start < finding.end
|
|
156
|
+
)
|
|
157
|
+
)
|
|
158
|
+
.flatMap((finding) => narrowed(text, finding));
|
|
159
|
+
return narrow.map((finding) => bounded(text, finding, narrow));
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
// `Cookie: a=1; b=2` and `Set-Cookie: id=…; Path=/; HttpOnly`: each cookie
|
|
163
|
+
// value is a credential; names, separators and attributes stay readable.
|
|
164
|
+
const cookieAttribute =
|
|
165
|
+
/^(?:path|domain|expires|max-age|samesite|secure|httponly|priority|partitioned|version|comment)$/i;
|
|
166
|
+
export function cookieValues(text, start, end) {
|
|
167
|
+
const values = [];
|
|
168
|
+
for (const pair of text.slice(start, end).matchAll(/[^;]+/g)) {
|
|
169
|
+
const part = /^[ \t]*(?:([^=\s]+)[ \t]*=[ \t]*)?(.*?)[ \t]*$/s.exec(
|
|
170
|
+
pair[0]
|
|
171
|
+
);
|
|
172
|
+
const [, name, value] = part;
|
|
173
|
+
if (!value || cookieAttribute.test(name ?? value)) {
|
|
174
|
+
continue;
|
|
175
|
+
}
|
|
176
|
+
const valueEnd = start + pair.index + pair[0].replace(/[ \t]+$/, '').length;
|
|
177
|
+
values.push({ start: valueEnd - value.length, end: valueEnd });
|
|
178
|
+
}
|
|
179
|
+
return values;
|
|
180
|
+
}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Credentials-only entry point. It has the same API as the main entry for the
|
|
3
|
+
* functions it exports, always runs with `credentialsOnly: true`, and rejects
|
|
4
|
+
* `credentialsOnly: false` and `mode: 'fake'` with `ERR_CONFIG`.
|
|
5
|
+
*/
|
|
6
|
+
export type * from './index.js';
|
|
7
|
+
export {
|
|
8
|
+
inspect,
|
|
9
|
+
sanitize,
|
|
10
|
+
redact,
|
|
11
|
+
createSanitizer,
|
|
12
|
+
createSyncSanitizer,
|
|
13
|
+
REDACTED,
|
|
14
|
+
entropy,
|
|
15
|
+
builtInRules,
|
|
16
|
+
tokenChecksum,
|
|
17
|
+
tokenWithChecksum,
|
|
18
|
+
knownSecretsFromEnv,
|
|
19
|
+
knownSecretsFromGitHubAuth,
|
|
20
|
+
knownSecretsFromGitHubConfig,
|
|
21
|
+
reservedPlaceholderTypes,
|
|
22
|
+
isPlaceholderToken,
|
|
23
|
+
} from './index.js';
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Credentials-only entry point (`@link-foundation/sensitive-data-sanitizer/credentials`).
|
|
3
|
+
*
|
|
4
|
+
* It loads the credential detectors without the personal-data detectors,
|
|
5
|
+
* faker or libphonenumber, so it starts faster and suits synchronous loggers.
|
|
6
|
+
* Every function here runs with `credentialsOnly: true` (issue 137).
|
|
7
|
+
*/
|
|
8
|
+
import {
|
|
9
|
+
inspect as inspectText,
|
|
10
|
+
sanitize as sanitizeText,
|
|
11
|
+
redact as redactText,
|
|
12
|
+
} from './sanitizer.js';
|
|
13
|
+
import { failure } from './detection.js';
|
|
14
|
+
import { createSanitizer as createAsyncSanitizer } from './engines.js';
|
|
15
|
+
import { createSyncSanitizer as createSync } from './sync-sanitizer.js';
|
|
16
|
+
|
|
17
|
+
export { REDACTED, entropy } from './detection.js';
|
|
18
|
+
export { builtInRules } from './rule-catalog.js';
|
|
19
|
+
export { tokenChecksum, tokenWithChecksum } from './token-checksum.js';
|
|
20
|
+
export {
|
|
21
|
+
reservedPlaceholderTypes,
|
|
22
|
+
isPlaceholderToken,
|
|
23
|
+
} from './placeholder-types.js';
|
|
24
|
+
export {
|
|
25
|
+
knownSecretsFromEnv,
|
|
26
|
+
knownSecretsFromGitHubAuth,
|
|
27
|
+
knownSecretsFromGitHubConfig,
|
|
28
|
+
} from './known.js';
|
|
29
|
+
|
|
30
|
+
function credentialOptions(options = {}) {
|
|
31
|
+
const transforms = [
|
|
32
|
+
options.transformation,
|
|
33
|
+
...Object.values(options.transformations ?? {}),
|
|
34
|
+
];
|
|
35
|
+
// The feature registry is shared by both entries in one process, so these
|
|
36
|
+
// options are rejected explicitly.
|
|
37
|
+
if (
|
|
38
|
+
options.credentialsOnly === false ||
|
|
39
|
+
options.properNames !== undefined ||
|
|
40
|
+
options.scripts !== undefined ||
|
|
41
|
+
transforms.some((transform) => transform?.mode === 'fake')
|
|
42
|
+
) {
|
|
43
|
+
throw failure('ERR_CONFIG');
|
|
44
|
+
}
|
|
45
|
+
return { ...options, credentialsOnly: true };
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
export function inspect(text, options) {
|
|
49
|
+
return inspectText(text, credentialOptions(options));
|
|
50
|
+
}
|
|
51
|
+
export function sanitize(text, options) {
|
|
52
|
+
return sanitizeText(text, credentialOptions(options));
|
|
53
|
+
}
|
|
54
|
+
export function redact(text, findings, options) {
|
|
55
|
+
return redactText(text, findings, credentialOptions(options));
|
|
56
|
+
}
|
|
57
|
+
export function createSanitizer(options) {
|
|
58
|
+
return createAsyncSanitizer(credentialOptions(options));
|
|
59
|
+
}
|
|
60
|
+
export function createSyncSanitizer(options) {
|
|
61
|
+
return createSync(credentialOptions(options));
|
|
62
|
+
}
|
package/src/credentials.js
CHANGED
|
@@ -10,8 +10,15 @@ export function credentialPlaceholder(value) {
|
|
|
10
10
|
// One template or mask token: ${NAME}, $${NAME}, $NAME, $1, <name>, [NAME],
|
|
11
11
|
// ****, XXXX or an ellipsis. Values made only of these (optionally split by
|
|
12
12
|
// a user:password colon) carry no credential material.
|
|
13
|
+
// `hive-mask` and `mask` output (`ghp…b4N`, `ab***yz`; NFKC projection turns
|
|
14
|
+
// `…` into `...`) keeps at most a few characters around the marker, so a
|
|
15
|
+
// second pass leaves it unchanged.
|
|
13
16
|
const templateToken =
|
|
14
|
-
/^(?:\$?\$\{[A-Za-z_][\w.-]*\}|\$[A-Za-z_]\w*|\$\d|<[^<>\r\n]{1,64}>|\[[A-Z][A-Z0-9_ ]{0,63}\]|\*{3,}|[xX]{4,}
|
|
17
|
+
/^(?:\$?\$\{[A-Za-z_][\w.-]*\}|\$[A-Za-z_]\w*|\$\d|<[^<>\r\n]{1,64}>|\[[A-Z][A-Z0-9_ ]{0,63}\]|[^\s…*.]{0,4}(?:…|\.{3}|\*{3,})[^\s…*.]{0,4}|[xX]{4,})$/u;
|
|
18
|
+
// A whole value that is already a template, a mask or `[REDACTED]`.
|
|
19
|
+
export function maskedValue(value) {
|
|
20
|
+
return credentialPlaceholder(value) || templateToken.test(value);
|
|
21
|
+
}
|
|
15
22
|
const unquote = (part) => part.replace(/^["']|["']$/g, '');
|
|
16
23
|
|
|
17
24
|
function templatedValue(value) {
|