@link-foundation/sensitive-data-sanitizer 0.19.3 → 0.21.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/README.md +22 -2
- package/bin/sensitive-data-sanitizer.js +50 -28
- package/docs/API.md +355 -16
- package/docs/COVERAGE.md +19 -9
- package/docs/RULES.md +218 -0
- package/examples/second-pass-placeholders.mjs +28 -0
- package/package.json +12 -3
- package/src/allow-list.js +93 -0
- package/src/confidence.js +24 -9
- package/src/context.js +287 -40
- package/src/credential-spans.js +180 -0
- package/src/credentials-entry.d.ts +23 -0
- package/src/credentials-entry.js +62 -0
- package/src/credentials.js +51 -0
- package/src/crypto-addresses.js +74 -0
- package/src/detection.js +87 -41
- package/src/document-person-cues.js +1 -1
- package/src/encoded.js +137 -51
- package/src/encrypt.js +95 -0
- package/src/engines.js +33 -10
- package/src/entities.js +4 -3
- package/src/exempt.js +72 -0
- package/src/fake-memory.js +32 -0
- package/src/fake.js +33 -29
- package/src/features.js +29 -0
- package/src/files.js +57 -12
- package/src/gitleaks.js +82 -27
- package/src/html-tags.js +8 -0
- package/src/identifier-detector.js +36 -0
- package/src/identity-types.js +63 -0
- package/src/identity.js +3 -47
- package/src/index.d.ts +157 -3
- package/src/index.js +13 -0
- package/src/known.js +33 -15
- package/src/national-checks.js +209 -0
- package/src/national.js +310 -92
- package/src/pattern-detector.js +219 -0
- package/src/personal-context.js +3 -2
- package/src/personal-evidence.js +233 -0
- package/src/personal-features.js +39 -0
- package/src/personal.js +66 -10
- package/src/placeholder-types.js +154 -4
- package/src/placeholders.js +127 -45
- package/src/projection.js +78 -26
- package/src/proper-names.js +48 -5
- package/src/prose-words.js +120 -0
- package/src/record-worker.js +1 -0
- package/src/rule-catalog.js +426 -0
- package/src/rules.js +8 -0
- package/src/sanitizer.js +133 -23
- package/src/sarif.js +127 -0
- package/src/stream-worker.js +1 -0
- package/src/strict-attributes.js +48 -4
- package/src/strict.js +1 -1
- package/src/structured.js +2 -0
- package/src/token-checksum.js +49 -0
- package/src/transforms.js +146 -33
- package/src/vendor/betterleaks/LICENSE +21 -0
- package/src/vendor/betterleaks/provenance.json +11 -0
- package/src/vendor/betterleaks/rules-0.json +463 -0
- package/src/vendor/betterleaks/rules-1.json +454 -0
- package/src/vendor/betterleaks/rules-2.json +461 -0
- package/src/vendor/betterleaks/rules-3.json +455 -0
- package/src/vendor/betterleaks/rules-4.json +483 -0
- package/src/vendor/betterleaks/rules-5.json +35 -0
- package/src/vendor/entities/aws.json +1 -1
- package/src/vendor/entities/azure.json +6 -2
- package/src/vendor/entities/google.json +2 -1
- package/src/vocabulary.js +27 -1
- package/src/worker-options.js +1 -0
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Credentials-only entry point (`@link-foundation/sensitive-data-sanitizer/credentials`).
|
|
3
|
+
*
|
|
4
|
+
* It loads the credential detectors without the personal-data detectors,
|
|
5
|
+
* faker or libphonenumber, so it starts faster and suits synchronous loggers.
|
|
6
|
+
* Every function here runs with `credentialsOnly: true` (issue 137).
|
|
7
|
+
*/
|
|
8
|
+
import {
|
|
9
|
+
inspect as inspectText,
|
|
10
|
+
sanitize as sanitizeText,
|
|
11
|
+
redact as redactText,
|
|
12
|
+
} from './sanitizer.js';
|
|
13
|
+
import { failure } from './detection.js';
|
|
14
|
+
import { createSanitizer as createAsyncSanitizer } from './engines.js';
|
|
15
|
+
import { createSyncSanitizer as createSync } from './sync-sanitizer.js';
|
|
16
|
+
|
|
17
|
+
export { REDACTED, entropy } from './detection.js';
|
|
18
|
+
export { builtInRules } from './rule-catalog.js';
|
|
19
|
+
export { tokenChecksum, tokenWithChecksum } from './token-checksum.js';
|
|
20
|
+
export {
|
|
21
|
+
reservedPlaceholderTypes,
|
|
22
|
+
isPlaceholderToken,
|
|
23
|
+
} from './placeholder-types.js';
|
|
24
|
+
export {
|
|
25
|
+
knownSecretsFromEnv,
|
|
26
|
+
knownSecretsFromGitHubAuth,
|
|
27
|
+
knownSecretsFromGitHubConfig,
|
|
28
|
+
} from './known.js';
|
|
29
|
+
|
|
30
|
+
function credentialOptions(options = {}) {
|
|
31
|
+
const transforms = [
|
|
32
|
+
options.transformation,
|
|
33
|
+
...Object.values(options.transformations ?? {}),
|
|
34
|
+
];
|
|
35
|
+
// The feature registry is shared by both entries in one process, so these
|
|
36
|
+
// options are rejected explicitly.
|
|
37
|
+
if (
|
|
38
|
+
options.credentialsOnly === false ||
|
|
39
|
+
options.properNames !== undefined ||
|
|
40
|
+
options.scripts !== undefined ||
|
|
41
|
+
transforms.some((transform) => transform?.mode === 'fake')
|
|
42
|
+
) {
|
|
43
|
+
throw failure('ERR_CONFIG');
|
|
44
|
+
}
|
|
45
|
+
return { ...options, credentialsOnly: true };
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
export function inspect(text, options) {
|
|
49
|
+
return inspectText(text, credentialOptions(options));
|
|
50
|
+
}
|
|
51
|
+
export function sanitize(text, options) {
|
|
52
|
+
return sanitizeText(text, credentialOptions(options));
|
|
53
|
+
}
|
|
54
|
+
export function redact(text, findings, options) {
|
|
55
|
+
return redactText(text, findings, credentialOptions(options));
|
|
56
|
+
}
|
|
57
|
+
export function createSanitizer(options) {
|
|
58
|
+
return createAsyncSanitizer(credentialOptions(options));
|
|
59
|
+
}
|
|
60
|
+
export function createSyncSanitizer(options) {
|
|
61
|
+
return createSync(credentialOptions(options));
|
|
62
|
+
}
|
package/src/credentials.js
CHANGED
|
@@ -7,6 +7,57 @@ export function credentialPlaceholder(value) {
|
|
|
7
7
|
) || referencePrefix(value)?.[0] === value
|
|
8
8
|
);
|
|
9
9
|
}
|
|
10
|
+
// One template or mask token: ${NAME}, $${NAME}, $NAME, $1, <name>, [NAME],
|
|
11
|
+
// ****, XXXX or an ellipsis. Values made only of these (optionally split by
|
|
12
|
+
// a user:password colon) carry no credential material.
|
|
13
|
+
// `hive-mask` and `mask` output (`ghp…b4N`, `ab***yz`; NFKC projection turns
|
|
14
|
+
// `…` into `...`) keeps at most a few characters around the marker, so a
|
|
15
|
+
// second pass leaves it unchanged.
|
|
16
|
+
const templateToken =
|
|
17
|
+
/^(?:\$?\$\{[A-Za-z_][\w.-]*\}|\$[A-Za-z_]\w*|\$\d|<[^<>\r\n]{1,64}>|\[[A-Z][A-Z0-9_ ]{0,63}\]|[^\s…*.]{0,4}(?:…|\.{3}|\*{3,})[^\s…*.]{0,4}|[xX]{4,})$/u;
|
|
18
|
+
// A whole value that is already a template, a mask or `[REDACTED]`.
|
|
19
|
+
export function maskedValue(value) {
|
|
20
|
+
return credentialPlaceholder(value) || templateToken.test(value);
|
|
21
|
+
}
|
|
22
|
+
const unquote = (part) => part.replace(/^["']|["']$/g, '');
|
|
23
|
+
|
|
24
|
+
function templatedValue(value) {
|
|
25
|
+
const body = value.replace(/^(?:token|Bearer|Basic)\s+/i, '');
|
|
26
|
+
const colon = body.indexOf(':');
|
|
27
|
+
const parts = (
|
|
28
|
+
colon < 0 ? [body] : [body.slice(0, colon), body.slice(colon + 1)]
|
|
29
|
+
).map((part) => unquote(part.trim()));
|
|
30
|
+
return (
|
|
31
|
+
parts.some(Boolean) &&
|
|
32
|
+
parts.every((part) => !part || templateToken.test(part))
|
|
33
|
+
);
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
// Detectors stop at whitespace or brackets, so `<API token>` or `[PASSWORD]`
|
|
37
|
+
// can arrive truncated; complete them from the following text first.
|
|
38
|
+
export function templatedCredential(text, finding) {
|
|
39
|
+
if (finding.category !== 'credential' || finding.rule === 'known-secret') {
|
|
40
|
+
return false;
|
|
41
|
+
}
|
|
42
|
+
let value = text.slice(finding.start, finding.end);
|
|
43
|
+
const closer =
|
|
44
|
+
value.lastIndexOf('<') > value.lastIndexOf('>')
|
|
45
|
+
? '>'
|
|
46
|
+
: value.lastIndexOf('[') > value.lastIndexOf(']')
|
|
47
|
+
? ']'
|
|
48
|
+
: '';
|
|
49
|
+
if (closer) {
|
|
50
|
+
const rest = text.slice(finding.end, finding.end + 64);
|
|
51
|
+
const index = rest.indexOf(closer);
|
|
52
|
+
if (index < 0 || /[\r\n]/.test(rest.slice(0, index))) {
|
|
53
|
+
return false;
|
|
54
|
+
}
|
|
55
|
+
value += rest.slice(0, index + 1);
|
|
56
|
+
}
|
|
57
|
+
// A template glued to literal characters (`<x>abc`, `[A]x9`) is not one.
|
|
58
|
+
const after = text[finding.start + value.length] ?? '';
|
|
59
|
+
return !/[\w$%+/=.-]/.test(after) && templatedValue(value);
|
|
60
|
+
}
|
|
10
61
|
// Only variable references and the documented gh lookup are exempt. Arbitrary
|
|
11
62
|
// shell commands and Actions expressions can contain literal credentials.
|
|
12
63
|
export function referencePrefix(value) {
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
import { createHash } from 'node:crypto';
|
|
2
|
+
|
|
3
|
+
// Offline checksum validation for Bitcoin addresses (Presidio parity): legacy
|
|
4
|
+
// Base58Check (P2PKH/P2SH) and SegWit Bech32/Bech32m. Ethereum addresses carry
|
|
5
|
+
// no case-insensitive checksum and stay format-only.
|
|
6
|
+
const base58 = '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz';
|
|
7
|
+
const sha256 = (bytes) => createHash('sha256').update(bytes).digest();
|
|
8
|
+
|
|
9
|
+
export function base58Check(value) {
|
|
10
|
+
let number = 0n;
|
|
11
|
+
for (const char of value) {
|
|
12
|
+
const digit = base58.indexOf(char);
|
|
13
|
+
if (digit < 0) {
|
|
14
|
+
return false;
|
|
15
|
+
}
|
|
16
|
+
number = number * 58n + BigInt(digit);
|
|
17
|
+
}
|
|
18
|
+
const hex = number.toString(16);
|
|
19
|
+
const body = Buffer.from(hex.length % 2 ? `0${hex}` : hex, 'hex');
|
|
20
|
+
const zeros = /^1*/.exec(value)[0].length;
|
|
21
|
+
const bytes = Buffer.concat([Buffer.alloc(zeros), body]);
|
|
22
|
+
if (bytes.length !== 25) {
|
|
23
|
+
return false;
|
|
24
|
+
}
|
|
25
|
+
const check = sha256(sha256(bytes.subarray(0, 21))).subarray(0, 4);
|
|
26
|
+
return check.equals(bytes.subarray(21));
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
const bech32Charset = 'qpzry9x8gf2tvdw0s3jn54khce6mua7l';
|
|
30
|
+
function polymod(values) {
|
|
31
|
+
const generator = [
|
|
32
|
+
0x3b6a57b2, 0x26508e6d, 0x1ea119fa, 0x3d4233dd, 0x2a1462b3,
|
|
33
|
+
];
|
|
34
|
+
let checksum = 1;
|
|
35
|
+
for (const value of values) {
|
|
36
|
+
const top = checksum >>> 25;
|
|
37
|
+
checksum = ((checksum & 0x1ffffff) << 5) ^ value;
|
|
38
|
+
for (let i = 0; i < 5; i++) {
|
|
39
|
+
if ((top >>> i) & 1) {
|
|
40
|
+
checksum ^= generator[i];
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
return checksum >>> 0;
|
|
45
|
+
}
|
|
46
|
+
export function bech32Address(value) {
|
|
47
|
+
if (value !== value.toLowerCase() && value !== value.toUpperCase()) {
|
|
48
|
+
return false;
|
|
49
|
+
}
|
|
50
|
+
const lower = value.toLowerCase();
|
|
51
|
+
const separator = lower.lastIndexOf('1');
|
|
52
|
+
const prefix = lower.slice(0, separator);
|
|
53
|
+
const data = [...lower.slice(separator + 1)].map((char) =>
|
|
54
|
+
bech32Charset.indexOf(char)
|
|
55
|
+
);
|
|
56
|
+
if (!['bc', 'tb'].includes(prefix) || data.length < 7 || data.includes(-1)) {
|
|
57
|
+
return false;
|
|
58
|
+
}
|
|
59
|
+
const expanded = [
|
|
60
|
+
...[...prefix].map((char) => char.charCodeAt(0) >> 5),
|
|
61
|
+
0,
|
|
62
|
+
...[...prefix].map((char) => char.charCodeAt(0) & 31),
|
|
63
|
+
...data,
|
|
64
|
+
];
|
|
65
|
+
const constant = polymod(expanded);
|
|
66
|
+
return constant === 1 || constant === 0x2bc830a3;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
export function cryptoAddress(value) {
|
|
70
|
+
if (/^0x[a-fA-F0-9]{40}$/.test(value)) {
|
|
71
|
+
return true;
|
|
72
|
+
}
|
|
73
|
+
return /^(?:bc|tb)1/i.test(value) ? bech32Address(value) : base58Check(value);
|
|
74
|
+
}
|
package/src/detection.js
CHANGED
|
@@ -1,23 +1,25 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import {
|
|
2
|
+
cardNetwork,
|
|
3
|
+
cardEvidence,
|
|
4
|
+
cardContext,
|
|
5
|
+
uatpCard,
|
|
6
|
+
} from './confidence.js';
|
|
2
7
|
import { isIP } from 'node:net';
|
|
8
|
+
import { Buffer } from 'node:buffer';
|
|
3
9
|
import { serviceRules } from './rules.js';
|
|
4
|
-
import { detectContext, valueRange,
|
|
5
|
-
import { detectPersonal } from './personal.js';
|
|
10
|
+
import { detectContext, valueRange, authorizationRanges } from './context.js';
|
|
6
11
|
import { detectGitleaks } from './gitleaks.js';
|
|
7
|
-
import { detectCatalog } from './entities.js';
|
|
8
|
-
import { detectIdentity } from './identity.js';
|
|
9
|
-
import { detectNational } from './national.js';
|
|
10
|
-
import { detectPersonalContext } from './personal-context.js';
|
|
11
12
|
import {
|
|
12
13
|
detectCredentialFormats,
|
|
13
14
|
credentialPlaceholder,
|
|
15
|
+
maskedValue,
|
|
14
16
|
} from './credentials.js';
|
|
15
17
|
import { localAddress } from './public.js';
|
|
16
18
|
import { detectKnownPersonal } from './personal-variants.js';
|
|
17
|
-
import { detectDocumentEntities } from './document-entities.js';
|
|
18
|
-
import { detectProseAndDocuments } from './proper-names.js';
|
|
19
19
|
import { builtInEnabled } from './policy.js';
|
|
20
|
-
import {
|
|
20
|
+
import { feature } from './features.js';
|
|
21
|
+
import { tokenChecksum } from './token-checksum.js';
|
|
22
|
+
import { refineCredentialSpans, cookieValues } from './credential-spans.js';
|
|
21
23
|
|
|
22
24
|
export const REDACTED = '[REDACTED]';
|
|
23
25
|
export const failure = (code = 'ERR_SANITIZATION', location = {}) =>
|
|
@@ -49,6 +51,30 @@ export function personalPattern(value) {
|
|
|
49
51
|
.map(escapePattern)
|
|
50
52
|
.join('[ \\t_-]+');
|
|
51
53
|
}
|
|
54
|
+
// Hex and base64 spellings of a known secret, at any length. The generic
|
|
55
|
+
// decoders need 16 or more encoded characters, so they cannot see short ones.
|
|
56
|
+
const encodedForms = new Map();
|
|
57
|
+
function knownEncodedForms(value) {
|
|
58
|
+
let pattern = encodedForms.get(value);
|
|
59
|
+
if (!pattern) {
|
|
60
|
+
const bytes = Buffer.from(value, 'utf8');
|
|
61
|
+
const hex = bytes.toString('hex');
|
|
62
|
+
const base64 = bytes.toString('base64').replace(/=+$/, '');
|
|
63
|
+
const url = bytes.toString('base64url');
|
|
64
|
+
const alternatives = [...new Set([hex, hex.toUpperCase()])].join('|');
|
|
65
|
+
const base64Forms = [...new Set([base64, url])].map(escapePattern);
|
|
66
|
+
pattern = new RegExp(
|
|
67
|
+
`(?<![\\da-fA-F])(?:${alternatives})(?![\\da-fA-F])|(?<![\\w+/-])(?:${base64Forms.join('|')})(?:={0,2}(?![\\w+/=-]))`,
|
|
68
|
+
'g'
|
|
69
|
+
);
|
|
70
|
+
if (encodedForms.size > 1024) {
|
|
71
|
+
encodedForms.clear();
|
|
72
|
+
}
|
|
73
|
+
encodedForms.set(value, pattern);
|
|
74
|
+
}
|
|
75
|
+
return pattern;
|
|
76
|
+
}
|
|
77
|
+
|
|
52
78
|
export function escapePattern(text) {
|
|
53
79
|
return text.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
|
54
80
|
}
|
|
@@ -123,34 +149,42 @@ export function luhn(value) {
|
|
|
123
149
|
function detectCookies(text, emit) {
|
|
124
150
|
for (const match of text.matchAll(/(?:Set-)?Cookie[ \t]*:[ \t]*/gi)) {
|
|
125
151
|
const range = valueRange(text, match.index + match[0].length, 'line');
|
|
126
|
-
|
|
127
|
-
range
|
|
128
|
-
|
|
129
|
-
text.slice(
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
}
|
|
152
|
+
for (const value of range
|
|
153
|
+
? cookieValues(text, range.start, range.end)
|
|
154
|
+
: []) {
|
|
155
|
+
if (!maskedValue(text.slice(value.start, value.end))) {
|
|
156
|
+
emit({
|
|
157
|
+
...value,
|
|
158
|
+
type: 'SECRET',
|
|
159
|
+
category: 'credential',
|
|
160
|
+
rule: 'cookie',
|
|
161
|
+
});
|
|
162
|
+
}
|
|
137
163
|
}
|
|
138
164
|
}
|
|
139
165
|
}
|
|
140
166
|
|
|
141
167
|
export function nativeDetect(text, options, emit) {
|
|
142
|
-
const
|
|
143
|
-
|
|
168
|
+
const found = [];
|
|
169
|
+
detectNative(text, options, (finding) => found.push(finding));
|
|
170
|
+
for (const finding of refineCredentialSpans(text, found)) {
|
|
144
171
|
if (builtInEnabled(finding, options)) {
|
|
145
|
-
|
|
172
|
+
emit(finding);
|
|
146
173
|
}
|
|
147
|
-
}
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
function detectNative(text, options, emit) {
|
|
148
178
|
const credential = { type: 'SECRET', category: 'credential' };
|
|
149
179
|
for (const value of options.knownSecrets ?? []) {
|
|
150
180
|
collectMatches(text, new RegExp(escapePattern(value), 'g'), emit, {
|
|
151
181
|
...credential,
|
|
152
182
|
rule: 'known-secret',
|
|
153
183
|
});
|
|
184
|
+
collectMatches(text, knownEncodedForms(value), emit, {
|
|
185
|
+
...credential,
|
|
186
|
+
rule: 'known-secret',
|
|
187
|
+
});
|
|
154
188
|
}
|
|
155
189
|
detectKnownPersonal(
|
|
156
190
|
text,
|
|
@@ -176,18 +210,18 @@ export function nativeDetect(text, options, emit) {
|
|
|
176
210
|
? (details) => options.debug({ event: 'gitleaks-rule', ...details })
|
|
177
211
|
: undefined
|
|
178
212
|
);
|
|
213
|
+
const checked = (finding) =>
|
|
214
|
+
emit(
|
|
215
|
+
tokenChecksum(text.slice(finding.start, finding.end))
|
|
216
|
+
? { ...finding, confidence: 0.99 }
|
|
217
|
+
: finding
|
|
218
|
+
);
|
|
179
219
|
for (const [rule, pattern] of serviceRules) {
|
|
180
|
-
collectMatches(text, pattern,
|
|
220
|
+
collectMatches(text, pattern, checked, { ...credential, rule });
|
|
181
221
|
}
|
|
182
222
|
detectContext(text, emit, options.credentialsOnly);
|
|
183
223
|
if (!options.credentialsOnly) {
|
|
184
|
-
|
|
185
|
-
detectProseAndDocuments(text, emit, options, detectDocumentEntities);
|
|
186
|
-
detectCatalog(text, emit);
|
|
187
|
-
detectIdentity(text, emit);
|
|
188
|
-
detectNational(text, emit);
|
|
189
|
-
detectPersonalContext(text, emit);
|
|
190
|
-
detectStrict(text, emit, options);
|
|
224
|
+
feature('detectPersonalData')(text, emit, options);
|
|
191
225
|
}
|
|
192
226
|
detectCredentialFormats(text, emit);
|
|
193
227
|
// detect-secrets-style quoted entropy detection; bare commit hashes and
|
|
@@ -248,9 +282,11 @@ function detectAuthorization(text, emit, credential) {
|
|
|
248
282
|
for (const match of text.matchAll(
|
|
249
283
|
/(?:Proxy-)?Authorization[ \t]*:[ \t]*/gi
|
|
250
284
|
)) {
|
|
251
|
-
const
|
|
252
|
-
|
|
253
|
-
|
|
285
|
+
const start = match.index + match[0].length;
|
|
286
|
+
for (const range of authorizationRanges(text, start)) {
|
|
287
|
+
if (!credentialPlaceholder(text.slice(range.start, range.end))) {
|
|
288
|
+
emit({ ...range, ...credential, rule: 'authorization' });
|
|
289
|
+
}
|
|
254
290
|
}
|
|
255
291
|
}
|
|
256
292
|
}
|
|
@@ -277,7 +313,16 @@ function detectPersonalFormats(text, options, emit) {
|
|
|
277
313
|
collectMatches(
|
|
278
314
|
text,
|
|
279
315
|
/(?<![\p{L}\p{N}_.!#$%&'*+/?^`{|}~-])[\p{L}\p{N}_.!#$%&'*+/?^`{|}~-]+@[\p{L}\p{N}](?:[\p{L}\p{N}.-]{0,252})\.[\p{L}]{2,63}(?![\p{L}\p{N}_-])/gu,
|
|
280
|
-
(finding) =>
|
|
316
|
+
(finding) => {
|
|
317
|
+
// `https://user:password@host` is URL user info, not an address.
|
|
318
|
+
if (
|
|
319
|
+
!/[a-z][a-z0-9+.-]*:\/\/[^\s/@]*:$/i.test(
|
|
320
|
+
text.slice(Math.max(0, finding.start - 256), finding.start)
|
|
321
|
+
)
|
|
322
|
+
) {
|
|
323
|
+
emit(emailAfterMention(text, finding));
|
|
324
|
+
}
|
|
325
|
+
},
|
|
281
326
|
{ type: 'EMAIL', category: 'personal', rule: 'email' }
|
|
282
327
|
);
|
|
283
328
|
collectMatches(
|
|
@@ -311,20 +356,21 @@ function detectPersonalFormats(text, options, emit) {
|
|
|
311
356
|
0,
|
|
312
357
|
(value, match) =>
|
|
313
358
|
luhn(value) &&
|
|
314
|
-
cardNetwork(value)
|
|
359
|
+
(cardNetwork(value) ||
|
|
360
|
+
(uatpCard(value) && cardContext(text, match.index, value))) &&
|
|
315
361
|
cardEvidence(text, match.index, value)
|
|
316
362
|
);
|
|
317
363
|
collectMatches(
|
|
318
364
|
text,
|
|
319
|
-
|
|
365
|
+
/(?<![\w.-])\d{3}([-.])\d{2}\1\d{4}(?![\w-]|\.\d)/g,
|
|
320
366
|
emit,
|
|
321
367
|
{ type: 'ID', category: 'personal', rule: 'us-ssn' },
|
|
322
368
|
0,
|
|
323
|
-
(v) => !/^(?:000|666|9\d\d)
|
|
369
|
+
(v) => !/^(?:000|666|9\d\d)\D|^\d{3}\D00\D|0000$/.test(v)
|
|
324
370
|
);
|
|
325
371
|
collectMatches(
|
|
326
372
|
text,
|
|
327
|
-
/(?<![\w.])(?:\d{1,3}\.){3}\d{1,3}(
|
|
373
|
+
/(?<![\w.])(?:\d{1,3}\.){3}\d{1,3}(?!\w|\.\d)/g,
|
|
328
374
|
emit,
|
|
329
375
|
{ type: 'IP_ADDRESS', category: 'personal', rule: 'ipv4' },
|
|
330
376
|
0,
|
|
@@ -8,7 +8,7 @@ const name = "[A-Z][a-z]{1,47}(?:['’-][A-Z]?[a-z]{1,47})?";
|
|
|
8
8
|
// unit in logs and lowercase `dr` is usually a word or path fragment. A
|
|
9
9
|
// capitalized `Ms` right after a number is still a unit (`took 30 Ms Retry`).
|
|
10
10
|
const cue =
|
|
11
|
-
'(?:[Cc]andidate|[Rr]ecruiter|[Aa]pplicant|[Dd]ear|[Hh]i|Mr\\.?|(?<!\\d[ \\t]*)
|
|
11
|
+
'(?:[Cc]andidate|[Rr]ecruiter|[Aa]pplicant|[Dd]ear|[Hh]i|Mr\\.?|Ms(?<!\\d[ \\t]*Ms)\\.?|Dr\\.?|[Mm]essage[ \\t]+from)';
|
|
12
12
|
const pattern = new RegExp(
|
|
13
13
|
`(?<!${boundary})${cue}[ \\t]+(${name})(?!${boundary})`,
|
|
14
14
|
'gu'
|
package/src/encoded.js
CHANGED
|
@@ -1,68 +1,101 @@
|
|
|
1
1
|
import { TextDecoder } from 'node:util';
|
|
2
2
|
import { Buffer } from 'node:buffer';
|
|
3
|
-
import { failure } from './detection.js';
|
|
4
3
|
import { jsonStringRuns } from './json-string-runs.js';
|
|
5
4
|
|
|
5
|
+
// Decoded candidates are re-scanned by every detector, so their number is
|
|
6
|
+
// bounded per window of the input; a denser window falls back to masking the
|
|
7
|
+
// remaining candidates as `encoded-limit` runs, without blocking the text.
|
|
8
|
+
const WINDOW = 65536;
|
|
9
|
+
const WINDOW_CANDIDATES = 4096;
|
|
10
|
+
|
|
11
|
+
const WRAPPED =
|
|
12
|
+
/(?<![A-Za-z0-9_+/-])[A-Za-z0-9_+/-]{16,128}(?:\r?\n[A-Za-z0-9_+/-]{1,128})+={0,2}(?![A-Za-z0-9_+/-])/g;
|
|
13
|
+
|
|
6
14
|
export function* decodedRuns(text, depth = 0) {
|
|
7
15
|
if (depth >= 2) {
|
|
8
16
|
return;
|
|
9
17
|
}
|
|
18
|
+
// Wrapped runs come first: a line inside a decoded wrapped run is covered
|
|
19
|
+
// by the run's own finding and is not decoded again.
|
|
10
20
|
const patterns = [
|
|
21
|
+
WRAPPED,
|
|
11
22
|
/(?<![A-Za-z0-9_+/-])[A-Za-z0-9_+/-]{16}[A-Za-z0-9_+/-]*={0,2}(?![A-Za-z0-9_+/-])/g,
|
|
12
23
|
/(?:[^\s"'<>?=&:/]{0,128}%[\da-fA-F]{2})+(?:[^\s"'<>?=&:/]{0,128})/g,
|
|
13
|
-
/(?<![A-Za-z0-9_+/-])[A-Za-z0-9_+/-]{16,128}(?:\r?\n[A-Za-z0-9_+/-]{1,128})+={0,2}(?![A-Za-z0-9_+/-])/g,
|
|
14
24
|
/(?:&#(?:x[\da-fA-F]{1,6}|\d{1,7});|&(?:amp|lt|gt|quot|apos|nbsp);){2,2048}/g,
|
|
15
25
|
/(?:\\x[\da-fA-F]{2}){2,2048}/g,
|
|
16
26
|
undefined, // JSON candidates use a linear scanner below.
|
|
17
27
|
];
|
|
18
|
-
const
|
|
19
|
-
let count = 0;
|
|
28
|
+
const scan = { depth, seen: new Set(), counts: new Map(), covered: [] };
|
|
20
29
|
for (const [index, pattern] of patterns.entries()) {
|
|
21
|
-
const marker = ['', '
|
|
30
|
+
const marker = ['\n', '', '%', '&', '\\x', '"'][index];
|
|
22
31
|
if (marker && !text.includes(marker)) {
|
|
23
32
|
continue;
|
|
24
33
|
}
|
|
34
|
+
const inWrappedRun = coverage(scan.covered);
|
|
25
35
|
for (const match of candidateMatches(text, pattern)) {
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
if (isLimitedTextRun(raw)) {
|
|
29
|
-
yield {
|
|
30
|
-
start: match.index,
|
|
31
|
-
end: match.index + raw.length,
|
|
32
|
-
limited: true,
|
|
33
|
-
depth,
|
|
34
|
-
};
|
|
35
|
-
}
|
|
36
|
-
continue;
|
|
37
|
-
}
|
|
38
|
-
const candidates = decodeCandidates(raw);
|
|
39
|
-
for (const candidate of candidates) {
|
|
40
|
-
const { text: decoded, encoding } = candidate;
|
|
41
|
-
const key = `${match.index}:${raw.length}:${decoded}`;
|
|
42
|
-
if (seen.has(key) || !decodedText(raw, decoded)) {
|
|
43
|
-
continue;
|
|
44
|
-
}
|
|
45
|
-
seen.add(key);
|
|
46
|
-
if (++count > 4096) {
|
|
47
|
-
throw failure('ERR_LIMIT');
|
|
48
|
-
}
|
|
49
|
-
const inset = encoding === 'json-content' ? 1 : 0;
|
|
50
|
-
const run = {
|
|
51
|
-
start: match.index + inset,
|
|
52
|
-
end: match.index + raw.length - inset,
|
|
53
|
-
text: decoded,
|
|
54
|
-
encoding,
|
|
55
|
-
depth,
|
|
56
|
-
};
|
|
57
|
-
yield run;
|
|
58
|
-
for (const inner of decodedRuns(decoded, depth + 1)) {
|
|
59
|
-
yield { ...run, text: inner.text, depth: inner.depth };
|
|
60
|
-
}
|
|
36
|
+
if (index !== 1 || !inWrappedRun(match.index)) {
|
|
37
|
+
yield* matchRuns(match, index === 0, scan);
|
|
61
38
|
}
|
|
62
39
|
}
|
|
63
40
|
}
|
|
64
41
|
}
|
|
65
42
|
|
|
43
|
+
// Tests ascending offsets against the sorted, disjoint wrapped runs.
|
|
44
|
+
function coverage(covered) {
|
|
45
|
+
let cursor = 0;
|
|
46
|
+
return (start) => {
|
|
47
|
+
while (cursor < covered.length && covered[cursor].end <= start) {
|
|
48
|
+
cursor++;
|
|
49
|
+
}
|
|
50
|
+
return cursor < covered.length && covered[cursor].start <= start;
|
|
51
|
+
};
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
function* matchRuns(match, wrapped, scan) {
|
|
55
|
+
const raw = match[0];
|
|
56
|
+
const start = match.index;
|
|
57
|
+
const end = start + raw.length;
|
|
58
|
+
const limited = { start, end, limited: true, depth: scan.depth };
|
|
59
|
+
if (raw.length > 8192) {
|
|
60
|
+
if (isLimitedTextRun(raw, wrapped)) {
|
|
61
|
+
scan.covered.push(limited);
|
|
62
|
+
yield limited;
|
|
63
|
+
}
|
|
64
|
+
return;
|
|
65
|
+
}
|
|
66
|
+
const window = Math.floor(start / WINDOW);
|
|
67
|
+
for (const candidate of decodeCandidates(raw)) {
|
|
68
|
+
const { text: decoded, encoding } = candidate;
|
|
69
|
+
const key = `${start}:${raw.length}:${decoded}`;
|
|
70
|
+
if (scan.seen.has(key) || !decodedText(raw, decoded)) {
|
|
71
|
+
continue;
|
|
72
|
+
}
|
|
73
|
+
scan.seen.add(key);
|
|
74
|
+
const count = (scan.counts.get(window) ?? 0) + 1;
|
|
75
|
+
scan.counts.set(window, count);
|
|
76
|
+
if (count > WINDOW_CANDIDATES) {
|
|
77
|
+
yield limited;
|
|
78
|
+
return;
|
|
79
|
+
}
|
|
80
|
+
if (wrapped && scan.covered.at(-1)?.start !== start) {
|
|
81
|
+
scan.covered.push({ start, end });
|
|
82
|
+
}
|
|
83
|
+
const inset = encoding === 'json-content' ? 1 : 0;
|
|
84
|
+
const run = {
|
|
85
|
+
start: start + inset,
|
|
86
|
+
end: end - inset,
|
|
87
|
+
text: decoded,
|
|
88
|
+
encoding,
|
|
89
|
+
depth: scan.depth,
|
|
90
|
+
...(candidate.layout && { layout: candidate.layout }),
|
|
91
|
+
};
|
|
92
|
+
yield run;
|
|
93
|
+
for (const inner of decodedRuns(decoded, scan.depth + 1)) {
|
|
94
|
+
yield { ...run, text: inner.text, depth: inner.depth };
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
|
|
66
99
|
function candidateMatches(text, pattern) {
|
|
67
100
|
return pattern ? text.matchAll(pattern) : jsonStringRuns(text);
|
|
68
101
|
}
|
|
@@ -76,20 +109,59 @@ function decodedText(raw, decoded) {
|
|
|
76
109
|
);
|
|
77
110
|
}
|
|
78
111
|
|
|
79
|
-
function isLimitedTextRun(raw) {
|
|
112
|
+
function isLimitedTextRun(raw, wrapped) {
|
|
80
113
|
// Bare opaque identifiers may share the base64 alphabet. Require a
|
|
81
|
-
// textual decode,
|
|
82
|
-
|
|
114
|
+
// textual decode, an explicit percent encoding, or the uniform layout of
|
|
115
|
+
// wrapped base64 (`base64 -w`, PEM bodies). Lists of identifiers or commit
|
|
116
|
+
// hashes are ordinary lines and are scanned one by one.
|
|
117
|
+
if (raw.includes('%')) {
|
|
118
|
+
return true;
|
|
119
|
+
}
|
|
120
|
+
const head = wrapped
|
|
121
|
+
? raw.slice(0, 8192).replace(/\s/g, '').slice(0, 4096)
|
|
122
|
+
: raw.slice(0, 4096);
|
|
123
|
+
const probe = decodeCandidates(head, true);
|
|
83
124
|
return (
|
|
84
|
-
raw.includes('%') ||
|
|
85
|
-
raw.includes('\n') ||
|
|
86
125
|
probe.some(
|
|
87
126
|
// eslint-disable-next-line no-control-regex -- Exclude binary controls from the bounded UTF-8 text probe.
|
|
88
127
|
(c) => c.text && !/[\x00-\x08\x0e-\x1f\x7f]/.test(c.text)
|
|
128
|
+
) ||
|
|
129
|
+
(wrapped && isWrappedBinary(raw))
|
|
130
|
+
);
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
// Wrapped binary base64 uses the standard alphabet, lines of one width that
|
|
134
|
+
// is a multiple of four and is not plain hexadecimal.
|
|
135
|
+
function isWrappedBinary(raw) {
|
|
136
|
+
// Trim the padding without a regular expression: `/=+$/` backtracks
|
|
137
|
+
// quadratically over long `=` runs that are not at the end.
|
|
138
|
+
let end = raw.length;
|
|
139
|
+
while (end > 0 && raw.charCodeAt(end - 1) === 61) {
|
|
140
|
+
end -= 1;
|
|
141
|
+
}
|
|
142
|
+
const lines = raw.slice(0, end).split(/\r?\n/);
|
|
143
|
+
const width = lines[0].length;
|
|
144
|
+
return (
|
|
145
|
+
width % 4 === 0 &&
|
|
146
|
+
!/[-_]/.test(raw) &&
|
|
147
|
+
!/^[\da-fA-F\s]+$/.test(raw) &&
|
|
148
|
+
lines.every((line, i) =>
|
|
149
|
+
i === lines.length - 1 ? line.length <= width : line.length === width
|
|
89
150
|
)
|
|
90
151
|
);
|
|
91
152
|
}
|
|
92
153
|
|
|
154
|
+
// Line width and separator of a wrapped run, re-applied when re-encoding.
|
|
155
|
+
function wrapLayout(raw) {
|
|
156
|
+
if (!raw.includes('\n')) {
|
|
157
|
+
return undefined;
|
|
158
|
+
}
|
|
159
|
+
return {
|
|
160
|
+
width: raw.split(/\r?\n/, 1)[0].length,
|
|
161
|
+
newline: raw.includes('\r\n') ? '\r\n' : '\n',
|
|
162
|
+
};
|
|
163
|
+
}
|
|
164
|
+
|
|
93
165
|
function decodeCandidates(raw, partial) {
|
|
94
166
|
const candidates = [];
|
|
95
167
|
if (raw.startsWith('"')) {
|
|
@@ -153,8 +225,7 @@ function decodeCandidates(raw, partial) {
|
|
|
153
225
|
text: new TextDecoder('utf-8', { fatal: true }).decode(bytes, {
|
|
154
226
|
stream: partial,
|
|
155
227
|
}),
|
|
156
|
-
|
|
157
|
-
padded: raw.endsWith('='),
|
|
228
|
+
layout: wrapLayout(raw),
|
|
158
229
|
});
|
|
159
230
|
} catch {
|
|
160
231
|
/* Binary content. */
|
|
@@ -167,6 +238,7 @@ function decodeCandidates(raw, partial) {
|
|
|
167
238
|
text: new TextDecoder('utf-8', { fatal: true }).decode(
|
|
168
239
|
Buffer.from(compact, 'hex')
|
|
169
240
|
),
|
|
241
|
+
layout: wrapLayout(raw),
|
|
170
242
|
});
|
|
171
243
|
} catch {
|
|
172
244
|
/* Binary content. */
|
|
@@ -198,16 +270,30 @@ export function encodeRun(run, text) {
|
|
|
198
270
|
: `&#${c.codePointAt(0)};`
|
|
199
271
|
).join('');
|
|
200
272
|
} else {
|
|
201
|
-
encoded = bytes.toString(run.encoding);
|
|
273
|
+
encoded = wrap(bytes.toString(run.encoding), run.layout);
|
|
202
274
|
}
|
|
203
275
|
const decoded =
|
|
204
276
|
run.encoding === 'json-content'
|
|
205
277
|
? { text: JSON.parse(`"${encoded}"`) }
|
|
206
278
|
: decodeCandidates(encoded).find((c) => c.encoding === run.encoding);
|
|
207
|
-
|
|
208
|
-
|
|
279
|
+
// A short payload may re-encode below the length a decoder recognizes
|
|
280
|
+
// (hex needs 32 digits); callers then mask the encoded text as it stands.
|
|
281
|
+
return decoded?.text === text ? encoded : undefined;
|
|
282
|
+
}
|
|
283
|
+
|
|
284
|
+
function wrap(encoded, layout) {
|
|
285
|
+
if (!layout?.width || encoded.length <= layout.width) {
|
|
286
|
+
return encoded;
|
|
287
|
+
}
|
|
288
|
+
const lines = [];
|
|
289
|
+
for (let i = 0; i < encoded.length; i += layout.width) {
|
|
290
|
+
lines.push(encoded.slice(i, i + layout.width));
|
|
291
|
+
}
|
|
292
|
+
// Padding never starts a line of its own.
|
|
293
|
+
if (/^=+$/.test(lines.at(-1)) && lines.length > 1) {
|
|
294
|
+
lines[lines.length - 2] += lines.pop();
|
|
209
295
|
}
|
|
210
|
-
return
|
|
296
|
+
return lines.join(layout.newline);
|
|
211
297
|
}
|
|
212
298
|
|
|
213
299
|
export function encodedValue(raw) {
|