@link-foundation/sensitive-data-sanitizer 0.19.2 → 0.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/README.md +8 -1
- package/bin/sensitive-data-sanitizer.js +49 -28
- package/docs/API.md +193 -2
- package/docs/COVERAGE.md +22 -4
- package/docs/RULES.md +217 -0
- package/package.json +1 -1
- package/src/allow-list.js +93 -0
- package/src/confidence.js +24 -9
- package/src/context.js +121 -25
- package/src/credentials.js +44 -0
- package/src/crypto-addresses.js +74 -0
- package/src/detection.js +21 -6
- package/src/document-person-cues.js +1 -1
- package/src/encrypt.js +95 -0
- package/src/entities.js +4 -3
- package/src/fake.js +29 -1
- package/src/files.js +57 -12
- package/src/gitleaks.js +82 -27
- package/src/identifier-detector.js +36 -0
- package/src/identity.js +14 -0
- package/src/index.d.ts +117 -1
- package/src/index.js +7 -0
- package/src/national-checks.js +209 -0
- package/src/national.js +310 -92
- package/src/pattern-detector.js +219 -0
- package/src/personal-context.js +3 -2
- package/src/personal-evidence.js +233 -0
- package/src/personal.js +42 -10
- package/src/projection.js +85 -26
- package/src/rule-catalog.js +424 -0
- package/src/sanitizer.js +48 -3
- package/src/sarif.js +127 -0
- package/src/strict-attributes.js +71 -0
- package/src/strict.js +2 -12
- package/src/token-checksum.js +49 -0
- package/src/transforms.js +108 -25
- package/src/vendor/betterleaks/LICENSE +21 -0
- package/src/vendor/betterleaks/provenance.json +11 -0
- package/src/vendor/betterleaks/rules-0.json +463 -0
- package/src/vendor/betterleaks/rules-1.json +454 -0
- package/src/vendor/betterleaks/rules-2.json +461 -0
- package/src/vendor/betterleaks/rules-3.json +455 -0
- package/src/vendor/betterleaks/rules-4.json +483 -0
- package/src/vendor/betterleaks/rules-5.json +35 -0
- package/src/vendor/entities/aws.json +1 -1
- package/src/vendor/entities/azure.json +6 -2
- package/src/vendor/entities/google.json +2 -1
package/src/sarif.js
ADDED
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
// SARIF 2.1.0 export for code-scanning dashboards (GitHub, GitLab, Azure
|
|
2
|
+
// DevOps), matching the reporting formats of Gitleaks/Betterleaks, TruffleHog,
|
|
3
|
+
// detect-secrets and Kingfisher. Only metadata is emitted: rule, type,
|
|
4
|
+
// category, confidence and location. Matched values are never included.
|
|
5
|
+
const schema = 'https://json.schemastore.org/sarif-2.1.0.json';
|
|
6
|
+
const levels = { credential: 'error', personal: 'warning' };
|
|
7
|
+
|
|
8
|
+
function artifactUri(path) {
|
|
9
|
+
if (path === '-' || path === undefined) {
|
|
10
|
+
return 'stdin';
|
|
11
|
+
}
|
|
12
|
+
return encodeURI(String(path).replaceAll('\\', '/'));
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
function region(finding) {
|
|
16
|
+
const result = {
|
|
17
|
+
charOffset: finding.start,
|
|
18
|
+
charLength: finding.end - finding.start,
|
|
19
|
+
};
|
|
20
|
+
if (Number.isSafeInteger(finding.line)) {
|
|
21
|
+
result.startLine = finding.line;
|
|
22
|
+
}
|
|
23
|
+
if (Number.isSafeInteger(finding.column)) {
|
|
24
|
+
result.startColumn = finding.column;
|
|
25
|
+
}
|
|
26
|
+
return result;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
function sarifResult(finding, uri, ruleIndex) {
|
|
30
|
+
return {
|
|
31
|
+
ruleId: finding.rule,
|
|
32
|
+
ruleIndex,
|
|
33
|
+
level: levels[finding.category] ?? 'note',
|
|
34
|
+
message: {
|
|
35
|
+
text: `${finding.type} ${finding.category === 'credential' ? 'credential' : 'personal data'} detected`,
|
|
36
|
+
},
|
|
37
|
+
locations: [
|
|
38
|
+
{
|
|
39
|
+
physicalLocation: {
|
|
40
|
+
artifactLocation: { uri },
|
|
41
|
+
region: region(finding),
|
|
42
|
+
},
|
|
43
|
+
},
|
|
44
|
+
],
|
|
45
|
+
properties: {
|
|
46
|
+
type: finding.type,
|
|
47
|
+
category: finding.category,
|
|
48
|
+
confidence: finding.confidence,
|
|
49
|
+
},
|
|
50
|
+
};
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
/**
|
|
54
|
+
* Converts scan results to a SARIF 2.1.0 log. Accepts the CLI scan report
|
|
55
|
+
* (`{complete, results: [{path, findings}], skipped}`), an array of
|
|
56
|
+
* `{path, findings}` entries, or a findings array for one artifact. Values
|
|
57
|
+
* are never copied into the log; paths should already be sanitized.
|
|
58
|
+
*/
|
|
59
|
+
function entriesOf(report, path) {
|
|
60
|
+
if (!Array.isArray(report)) {
|
|
61
|
+
return report?.results ?? [];
|
|
62
|
+
}
|
|
63
|
+
return report.every((item) => Array.isArray(item?.findings))
|
|
64
|
+
? report
|
|
65
|
+
: [{ path, findings: report }];
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
export function toSarif(report, { version, path = '-' } = {}) {
|
|
69
|
+
const entries = entriesOf(report, path);
|
|
70
|
+
const rules = [],
|
|
71
|
+
indexes = new Map(),
|
|
72
|
+
results = [];
|
|
73
|
+
for (const entry of entries) {
|
|
74
|
+
const uri = artifactUri(entry.path);
|
|
75
|
+
for (const finding of entry.findings) {
|
|
76
|
+
if (!indexes.has(finding.rule)) {
|
|
77
|
+
indexes.set(finding.rule, rules.length);
|
|
78
|
+
rules.push({
|
|
79
|
+
id: finding.rule,
|
|
80
|
+
name: finding.rule,
|
|
81
|
+
shortDescription: { text: `${finding.type} (${finding.category})` },
|
|
82
|
+
defaultConfiguration: {
|
|
83
|
+
level: levels[finding.category] ?? 'note',
|
|
84
|
+
},
|
|
85
|
+
properties: { tags: [finding.category, finding.type] },
|
|
86
|
+
});
|
|
87
|
+
}
|
|
88
|
+
results.push(sarifResult(finding, uri, indexes.get(finding.rule)));
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
const skipped = report?.skipped ?? [];
|
|
92
|
+
return {
|
|
93
|
+
$schema: schema,
|
|
94
|
+
version: '2.1.0',
|
|
95
|
+
runs: [
|
|
96
|
+
{
|
|
97
|
+
tool: {
|
|
98
|
+
driver: {
|
|
99
|
+
name: 'sensitive-data-sanitizer',
|
|
100
|
+
informationUri:
|
|
101
|
+
'https://github.com/link-foundation/sensitive-data-sanitizer',
|
|
102
|
+
...(version ? { version } : {}),
|
|
103
|
+
rules,
|
|
104
|
+
},
|
|
105
|
+
},
|
|
106
|
+
invocations: [
|
|
107
|
+
{
|
|
108
|
+
executionSuccessful: report?.complete ?? true,
|
|
109
|
+
toolExecutionNotifications: skipped.map((item) => ({
|
|
110
|
+
level: 'warning',
|
|
111
|
+
message: { text: `Skipped: ${item.reason}` },
|
|
112
|
+
locations: [
|
|
113
|
+
{
|
|
114
|
+
physicalLocation: {
|
|
115
|
+
artifactLocation: { uri: artifactUri(item.path) },
|
|
116
|
+
},
|
|
117
|
+
},
|
|
118
|
+
],
|
|
119
|
+
})),
|
|
120
|
+
},
|
|
121
|
+
],
|
|
122
|
+
columnKind: 'utf16CodeUnits',
|
|
123
|
+
results,
|
|
124
|
+
},
|
|
125
|
+
],
|
|
126
|
+
};
|
|
127
|
+
}
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
import { collectMatches } from './detection.js';
|
|
2
|
+
|
|
3
|
+
const attributes =
|
|
4
|
+
/(?<![\p{L}\p{N}_])(?:женщин[аыуеой]{1,2}|мужчин[аыуеой]{1,2}|замужем|женат(?:а|ы)?|(?:один|одна|двое|трое|четверо|пятеро|\d{1,2})[ \t]+(?:детей|реб[её]нок|реб[её]нка)|русск(?:ая|ий|ого|ой|ие)|татар(?:ин|ка|ы)|православн(?:ый|ая|ого|ой)|мусульман(?:ин|ка)|инвалидност[ьи](?:[ \t]+\d[ \t]+группы)?|беременн(?:а|ая|ость)|судимость(?:[ \t]+погашена)?|член[ \t]+партии|woman|women|male|female|married|divorced|pregnant|pregnancy|disability|christian|muslim|jewish|orthodox|hispanic|criminal[ \t]+record|party[ \t]+member|(?:one|two|three|\d{1,2})[ \t]+children)(?![\p{L}\p{N}_])/giu;
|
|
5
|
+
const additionalAttributes =
|
|
6
|
+
/(?<![\p{L}\p{N}_])(?:диабет(?:ом|а|у|е)?|ВИЧ(?:-положительн(?:ый|ая|ого|ой))?|болен|больна|больны|католик(?:ом|а|и)?|католичк(?:а|ой|и)|буддист(?:ка|кой|ом|а|ы)?|атеист(?:ка|кой|ом|а|ы)?|украин(?:ец|ка|цы|цем|кой)|белорус(?:ка|кой|ом|а|ы)?|несудим(?:а|ы)?|судим(?:а|ы)?|член[ \t]+политической[ \t]+партии|холост(?:а|ы)?|развед[её]н(?:а|ы)?|в[ \t]+разводе|diabetes|diabetic|HIV(?:[ \t-]+positive)?|cancer|buddhist|catholic|atheist|ukrainian|belarusian|previous[ \t]+conviction|member[ \t]+of[ \t]+(?:a[ \t]+)?(?:political[ \t]+)?party|unmarried|widowed)(?![\p{L}\p{N}_])/giu;
|
|
7
|
+
|
|
8
|
+
// Bounded suffixes cover Russian cases, plurals and adjectival forms without
|
|
9
|
+
// turning a stem into an unbounded substring search. Category cues below cover
|
|
10
|
+
// unfamiliar diagnoses, nationalities and religions outside this lexicon.
|
|
11
|
+
const categoryForms =
|
|
12
|
+
/(?<![\p{L}\p{N}_])(?:(?:диабет|католик|католич|буддист|атеист|белорус|православн|беременн|судим)[\p{L}]{0,8}|татар(?:ин(?:а|у|ом|е)?|к[аиуе]|кой|ы|ам|ами|ах)?|мусульман(?:ин(?:а|у|ом|е)?|к[аиуе]|кой|е|ам|ами|ах)?|украин(?:ец|ца|цу|цем|це|цы|цев|цам|цами|цах|к[аиуе]|кой)|инвалид(?:а|у|ом|е|ы|ов|ам|ами|ах|к[аиу]|кой)?|армян(?:ин|ина|ину|ином|ине|е|ам|ами|ах|ка|ки|ке|ку|кой)|евре(?:й|я|ю|ем|и|ев|ям|ями|ях|йк[аиуе]|йкой)|казах(?:а|у|ом|е|и|ов|ам|ами|ах|к[аиуе]|кой)?|христиан(?:ин|ина|ину|ином|ине|е|ам|ами|ах|к[аиуе]|кой)|иуде(?:й|я|ю|ем|и|ев|ям|ями|ях|йк[аиуе]|йкой)|ислам[\p{L}]{0,5}|эпилепси[\p{L}]{1,5}|астм[\p{L}]{1,5}|онкологи[\p{L}]{1,5}|туберкул[её]з[\p{L}]{0,5}|психиатр[\p{L}]{0,6}|депресси[\p{L}]{1,5}|гепатит[\p{L}]{0,5}|гомосексуал[\p{L}]{0,7}|лесбиян[\p{L}]{1,5}|ге(?:й|я|ю|ем|и|ев|ям|ями|ях)|нетрадиционн[\p{L}]{2,3}[ \t]+ориентаци[\p{L}]{1,3}|отбыва(?:л[аи]?|ет|ют|вш[\p{L}]{2,3})[ \t]+наказани[\p{L}]{1,3}|ВИЧ-инфицирован[\p{L}]{0,4}|epilepsy|asthma|tuberculosis|hepatitis|depression|gay|lesbian|homosexual|bisexual|armenian)(?![\p{L}\p{N}_])/giu;
|
|
13
|
+
const categoryCue =
|
|
14
|
+
/(?<![\p{L}\p{N}_])(?:по[ \t]+национальности|национальность|этничность|вероисповедание|религия|диагноз|исповеду(?:ет|ют|ю|ешь)|боле(?:ет|ют|ю|ешь)|страда(?:ет|ют|ю|ешь)[ \t]+от|на[ \t]+уч[её]те[ \t]+у|diagnosed[ \t]+with|suffers[ \t]+from|nationality|ethnicity|religion|diagnosis)(?:["']?[ \t]*[:=][ \t]*|[ \t]+)["']?([\p{L}][\p{L}\p{M}-]{0,63})(?![\p{L}\p{M}\p{N}_])/giu;
|
|
15
|
+
const ambiguous =
|
|
16
|
+
/(?<![\p{L}\p{N}_])(?:disabled|single|man|men|russian|asian)(?![\p{L}\p{N}_])/giu;
|
|
17
|
+
|
|
18
|
+
function personalContext(text, match) {
|
|
19
|
+
const prefix = text.slice(Math.max(0, match.index - 128), match.index);
|
|
20
|
+
const suffix = text.slice(
|
|
21
|
+
match.index + match[0].length,
|
|
22
|
+
match.index + match[0].length + 64
|
|
23
|
+
);
|
|
24
|
+
// These continuations remain technical even after a pronoun ("it is a
|
|
25
|
+
// single-flight lock"). No document-wide cue can authorize another clause.
|
|
26
|
+
if (
|
|
27
|
+
/^(?:-flight\b|[ \t]+(?:page|pages|run|worker|thread|lock|region|endpoint|locale|module|feature|flag)\b)/iu.test(
|
|
28
|
+
suffix
|
|
29
|
+
)
|
|
30
|
+
) {
|
|
31
|
+
return false;
|
|
32
|
+
}
|
|
33
|
+
return (
|
|
34
|
+
/(?<![\p{L}\p{N}_])(?:i|he|she|we|they|candidate|applicant|patient|person|employee)(?:[ \t]+(?:am|is|are|was|were|identifies[ \t]+as))?(?:[ \t]+(?:a|an))?[ \t]+$/iu.test(
|
|
35
|
+
prefix
|
|
36
|
+
) ||
|
|
37
|
+
/(?<![\p{L}\p{N}_])(?:gender|sex|nationality|ethnicity|marital[ \t_-]*status|disability)(?:["'][ \t]*)?[ \t]*[:=][ \t]*["']?$/iu.test(
|
|
38
|
+
prefix
|
|
39
|
+
) ||
|
|
40
|
+
/^[ \t]+(?:candidate|applicant|patient|person|people|employee)(?![\p{L}\p{N}_])/iu.test(
|
|
41
|
+
suffix
|
|
42
|
+
)
|
|
43
|
+
);
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
export function detectStrictAttributes(text, emit, options) {
|
|
47
|
+
let detected = 0,
|
|
48
|
+
suppressed = 0;
|
|
49
|
+
const details = {
|
|
50
|
+
type: 'PERSONAL_ATTRIBUTE',
|
|
51
|
+
rule: 'strict-attribute',
|
|
52
|
+
category: 'personal',
|
|
53
|
+
confidence: 0.9,
|
|
54
|
+
};
|
|
55
|
+
const report = (finding) => {
|
|
56
|
+
detected++;
|
|
57
|
+
emit(finding);
|
|
58
|
+
};
|
|
59
|
+
for (const pattern of [attributes, additionalAttributes, categoryForms]) {
|
|
60
|
+
collectMatches(text, pattern, report, details);
|
|
61
|
+
}
|
|
62
|
+
collectMatches(text, categoryCue, report, details, 1);
|
|
63
|
+
collectMatches(text, ambiguous, report, details, 0, (_value, match) => {
|
|
64
|
+
const accepted = personalContext(text, match);
|
|
65
|
+
if (!accepted) {
|
|
66
|
+
suppressed++;
|
|
67
|
+
}
|
|
68
|
+
return accepted;
|
|
69
|
+
});
|
|
70
|
+
options.debug?.({ event: 'strict-attributes', detected, suppressed });
|
|
71
|
+
}
|
package/src/strict.js
CHANGED
|
@@ -3,12 +3,9 @@ import { months } from './personal-context.js';
|
|
|
3
3
|
import { isDocumentTechnology } from './document-experience.js';
|
|
4
4
|
import { placeholderValues } from './placeholders.js';
|
|
5
5
|
import { generatedValues } from './fake.js';
|
|
6
|
+
import { detectStrictAttributes } from './strict-attributes.js';
|
|
6
7
|
|
|
7
8
|
const boundary = '[\\p{L}\\p{N}_]';
|
|
8
|
-
const attributes =
|
|
9
|
-
/(?<![\p{L}\p{N}_])(?:женщин[аыуеой]{1,2}|мужчин[аыуеой]{1,2}|замужем|женат(?:а|ы)?|(?:один|одна|двое|трое|четверо|пятеро|\d{1,2})[ \t]+(?:детей|реб[её]нок|реб[её]нка)|русск(?:ая|ий|ого|ой|ие)|татар(?:ин|ка|ы)|православн(?:ый|ая|ого|ой)|мусульман(?:ин|ка)|инвалидност[ьи](?:[ \t]+\d[ \t]+группы)?|беременн(?:а|ая|ость)|судимость(?:[ \t]+погашена)?|член[ \t]+партии|woman|women|man|men|male|female|married|divorced|pregnant|pregnancy|disabled|disability|christian|muslim|jewish|orthodox|russian|hispanic|asian|criminal[ \t]+record|party[ \t]+member|(?:one|two|three|\d{1,2})[ \t]+children)(?![\p{L}\p{N}_])/giu;
|
|
10
|
-
const additionalAttributes =
|
|
11
|
-
/(?<![\p{L}\p{N}_])(?:диабет(?:ом|а|у|е)?|ВИЧ(?:-положительн(?:ый|ая|ого|ой))?|болен|больна|больны|католик(?:ом|а|и)?|католичк(?:а|ой|и)|буддист(?:ка|кой|ом|а|ы)?|атеист(?:ка|кой|ом|а|ы)?|украин(?:ец|ка|цы|цем|кой)|белорус(?:ка|кой|ом|а|ы)?|несудим(?:а|ы)?|судим(?:а|ы)?|член[ \t]+политической[ \t]+партии|холост(?:а|ы)?|развед[её]н(?:а|ы)?|в[ \t]+разводе|diabetes|diabetic|HIV(?:[ \t-]+positive)?|cancer|buddhist|catholic|atheist|ukrainian|belarusian|previous[ \t]+conviction|member[ \t]+of[ \t]+(?:a[ \t]+)?(?:political[ \t]+)?party|single|unmarried|widowed)(?![\p{L}\p{N}_])/giu;
|
|
12
9
|
|
|
13
10
|
function isDuration(text, match, replacements) {
|
|
14
11
|
// Keep cues on the same clause; an age on the next line is independent.
|
|
@@ -57,14 +54,7 @@ export function detectStrict(text, emit, options) {
|
|
|
57
54
|
if (!options.strict) {
|
|
58
55
|
return;
|
|
59
56
|
}
|
|
60
|
-
|
|
61
|
-
collectMatches(text, pattern, emit, {
|
|
62
|
-
type: 'PERSONAL_ATTRIBUTE',
|
|
63
|
-
rule: 'strict-attribute',
|
|
64
|
-
category: 'personal',
|
|
65
|
-
confidence: 0.9,
|
|
66
|
-
});
|
|
67
|
-
}
|
|
57
|
+
detectStrictAttributes(text, emit, options);
|
|
68
58
|
const date = new RegExp(
|
|
69
59
|
`(?<!${boundary})(?:\\d{4}-\\d{2}-\\d{2}|\\d{1,2}[./]\\d{1,2}[./]\\d{4}|\\d{1,2}[ \\t]+(?:${months})[ \\t]+\\d{4})(?!${boundary}|[Tt]\\d)`,
|
|
70
60
|
'giu'
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
// Offline structural check for prefixed tokens that end in a checksum.
|
|
2
|
+
// GitHub (ghp_, gho_, ghu_, ghs_, ghr_) and npm (npm_) tokens append six
|
|
3
|
+
// base62 characters: CRC32 (IEEE) of the characters after the first `_`.
|
|
4
|
+
// A valid checksum raises confidence; an invalid one never removes a finding
|
|
5
|
+
// because truncated or retyped tokens still need redaction.
|
|
6
|
+
const BASE62 = '0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz';
|
|
7
|
+
const CHECKSUM_TOKEN = /^(?:gh[opsur]|npm)_([A-Za-z0-9]{30})([A-Za-z0-9]{6})$/;
|
|
8
|
+
|
|
9
|
+
let table;
|
|
10
|
+
function crcTable() {
|
|
11
|
+
table ??= Array.from({ length: 256 }, (_, n) => {
|
|
12
|
+
let c = n;
|
|
13
|
+
for (let k = 0; k < 8; k++) {
|
|
14
|
+
c = c & 1 ? 0xedb88320 ^ (c >>> 1) : c >>> 1;
|
|
15
|
+
}
|
|
16
|
+
return c >>> 0;
|
|
17
|
+
});
|
|
18
|
+
return table;
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
export function crc32(text) {
|
|
22
|
+
let crc = 0xffffffff;
|
|
23
|
+
for (let i = 0; i < text.length; i++) {
|
|
24
|
+
crc = crcTable()[(crc ^ text.charCodeAt(i)) & 0xff] ^ (crc >>> 8);
|
|
25
|
+
}
|
|
26
|
+
return (crc ^ 0xffffffff) >>> 0;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
export function base62(value, width = 6) {
|
|
30
|
+
let out = '';
|
|
31
|
+
for (let i = 0; i < width; i++) {
|
|
32
|
+
out = BASE62[value % 62] + out;
|
|
33
|
+
value = Math.floor(value / 62);
|
|
34
|
+
}
|
|
35
|
+
return out;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
// Builds a checksum-valid token from a prefix (including `_`) and a
|
|
39
|
+
// 30-character body, so tests and examples need no stored token literals.
|
|
40
|
+
export function tokenWithChecksum(prefix, body) {
|
|
41
|
+
return `${prefix}${body}${base62(crc32(body))}`;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
// true: checksum matches; false: format matches but checksum does not;
|
|
45
|
+
// undefined: the value is not a checksummed token format.
|
|
46
|
+
export function tokenChecksum(value) {
|
|
47
|
+
const match = CHECKSUM_TOKEN.exec(value);
|
|
48
|
+
return match ? base62(crc32(match[1])) === match[2] : undefined;
|
|
49
|
+
}
|
package/src/transforms.js
CHANGED
|
@@ -5,6 +5,7 @@ import { encodedValue, encodeRun, decodedRuns } from './encoded.js';
|
|
|
5
5
|
import { isIdentityType } from './identity.js';
|
|
6
6
|
import { validatePlaceholder, placeholderReplacement } from './placeholders.js';
|
|
7
7
|
import { projectText } from './projection.js';
|
|
8
|
+
import { encryptValue } from './encrypt.js';
|
|
8
9
|
|
|
9
10
|
const modes = new Set([
|
|
10
11
|
'redact',
|
|
@@ -16,6 +17,7 @@ const modes = new Set([
|
|
|
16
17
|
'date-shift',
|
|
17
18
|
'bucket',
|
|
18
19
|
'placeholder',
|
|
20
|
+
'encrypt',
|
|
19
21
|
]);
|
|
20
22
|
export function validateTransforms(options) {
|
|
21
23
|
if (
|
|
@@ -86,16 +88,10 @@ function validateTransform(transform) {
|
|
|
86
88
|
Number.isSafeInteger(transform.days) &&
|
|
87
89
|
Math.abs(transform.days) <= 365000,
|
|
88
90
|
bucket: () => positiveInteger(transform.size),
|
|
89
|
-
mask: () =>
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
)
|
|
93
|
-
(transform.minLength === undefined ||
|
|
94
|
-
positiveInteger(transform.minLength)) &&
|
|
95
|
-
(transform.marker === undefined ||
|
|
96
|
-
(typeof transform.marker === 'string' &&
|
|
97
|
-
transform.marker.length > 0 &&
|
|
98
|
-
transform.marker.length <= 64)),
|
|
91
|
+
mask: () => validMask(transform),
|
|
92
|
+
encrypt: () =>
|
|
93
|
+
typeof transform.key === 'string' &&
|
|
94
|
+
Buffer.byteLength(transform.key) >= 16,
|
|
99
95
|
};
|
|
100
96
|
if (checks[transform.mode] && !checks[transform.mode]()) {
|
|
101
97
|
throw failure('ERR_CONFIG');
|
|
@@ -141,16 +137,112 @@ function shiftedDate(value, days) {
|
|
|
141
137
|
? REDACTED
|
|
142
138
|
: date.toISOString().slice(0, 10);
|
|
143
139
|
}
|
|
140
|
+
// Google charactersToIgnore/commonCharactersToIgnore and Presidio-style
|
|
141
|
+
// character classes that a mask may leave visible.
|
|
142
|
+
const maskClasses = {
|
|
143
|
+
digits: /\p{Nd}/u,
|
|
144
|
+
upper: /\p{Lu}/u,
|
|
145
|
+
lower: /\p{Ll}/u,
|
|
146
|
+
punctuation: /[\p{P}\p{S}]/u,
|
|
147
|
+
whitespace: /\s/u,
|
|
148
|
+
};
|
|
149
|
+
const maskTuning = ['char', 'ignore', 'ignoreClasses', 'count', 'fromEnd'];
|
|
150
|
+
const unset = (value, test) => value === undefined || test(value);
|
|
151
|
+
const maskChecks = [
|
|
152
|
+
(t) =>
|
|
153
|
+
[t.keepStart ?? 0, t.keepEnd ?? 0].every(
|
|
154
|
+
(value) => Number.isSafeInteger(value) && value >= 0
|
|
155
|
+
),
|
|
156
|
+
(t) => unset(t.minLength, positiveInteger),
|
|
157
|
+
(t) =>
|
|
158
|
+
unset(
|
|
159
|
+
t.marker,
|
|
160
|
+
(marker) =>
|
|
161
|
+
!maskTuning.some((key) => t[key] !== undefined) &&
|
|
162
|
+
typeof marker === 'string' &&
|
|
163
|
+
marker.length > 0 &&
|
|
164
|
+
marker.length <= 64
|
|
165
|
+
),
|
|
166
|
+
(t) =>
|
|
167
|
+
unset(
|
|
168
|
+
t.char,
|
|
169
|
+
(char) => typeof char === 'string' && Array.from(char).length === 1
|
|
170
|
+
),
|
|
171
|
+
(t) =>
|
|
172
|
+
unset(
|
|
173
|
+
t.ignore,
|
|
174
|
+
(ignore) =>
|
|
175
|
+
typeof ignore === 'string' && ignore.length > 0 && ignore.length <= 256
|
|
176
|
+
),
|
|
177
|
+
(t) =>
|
|
178
|
+
unset(
|
|
179
|
+
t.ignoreClasses,
|
|
180
|
+
(names) =>
|
|
181
|
+
Array.isArray(names) &&
|
|
182
|
+
names.every((name) => Object.hasOwn(maskClasses, name))
|
|
183
|
+
),
|
|
184
|
+
(t) => unset(t.count, positiveInteger),
|
|
185
|
+
(t) => unset(t.fromEnd, (value) => typeof value === 'boolean'),
|
|
186
|
+
];
|
|
187
|
+
function validMask(transform) {
|
|
188
|
+
return maskChecks.every((check) => check(transform));
|
|
189
|
+
}
|
|
190
|
+
// Indexes of the characters a mask replaces, after ignored characters, the
|
|
191
|
+
// direction and the count limit are applied.
|
|
192
|
+
function maskPositions(chars, start, end, transform) {
|
|
193
|
+
const ignored = (char) =>
|
|
194
|
+
Boolean(transform.ignore?.includes(char)) ||
|
|
195
|
+
(transform.ignoreClasses ?? []).some((name) =>
|
|
196
|
+
maskClasses[name].test(char)
|
|
197
|
+
);
|
|
198
|
+
const positions = [];
|
|
199
|
+
for (let i = start; i < chars.length - end; i++) {
|
|
200
|
+
if (!ignored(chars[i])) {
|
|
201
|
+
positions.push(i);
|
|
202
|
+
}
|
|
203
|
+
}
|
|
204
|
+
if (transform.fromEnd) {
|
|
205
|
+
positions.reverse();
|
|
206
|
+
}
|
|
207
|
+
return transform.count === undefined
|
|
208
|
+
? positions
|
|
209
|
+
: positions.slice(0, transform.count);
|
|
210
|
+
}
|
|
144
211
|
function masked(value, finding, transform) {
|
|
145
212
|
const chars = Array.from(value),
|
|
146
213
|
start = transform.keepStart ?? 0,
|
|
147
214
|
end = transform.keepEnd ?? 0;
|
|
148
|
-
|
|
149
|
-
|
|
215
|
+
if (
|
|
216
|
+
chars.length <
|
|
217
|
+
(transform.minLength ?? (isIdentityType(finding.type) ? 5 : 0)) ||
|
|
150
218
|
start + end >= chars.length
|
|
151
|
-
|
|
152
|
-
|
|
219
|
+
) {
|
|
220
|
+
return REDACTED;
|
|
221
|
+
}
|
|
222
|
+
if (transform.marker !== undefined) {
|
|
223
|
+
return `${chars.slice(0, start).join('')}${transform.marker}${end ? chars.slice(-end).join('') : ''}`;
|
|
224
|
+
}
|
|
225
|
+
const positions = maskPositions(chars, start, end, transform);
|
|
226
|
+
if (!positions.length) {
|
|
227
|
+
return REDACTED;
|
|
228
|
+
}
|
|
229
|
+
for (const i of positions) {
|
|
230
|
+
chars[i] = transform.char ?? '*';
|
|
231
|
+
}
|
|
232
|
+
return chars.join('');
|
|
153
233
|
}
|
|
234
|
+
// Modes that apply to personal data only; credentials are always redacted.
|
|
235
|
+
const personalModes = {
|
|
236
|
+
mask: masked,
|
|
237
|
+
encrypt: (value, finding, transform) =>
|
|
238
|
+
encryptValue(value, finding.type, transform.key),
|
|
239
|
+
pseudonym: (value, finding, transform) =>
|
|
240
|
+
`[PSEUDONYM_${keyed(value, finding.type, transform.key).toString('hex').slice(0, 24)}]`,
|
|
241
|
+
'format-preserving': (value, finding, transform) =>
|
|
242
|
+
formatPseudonym(value, finding.type, transform.key),
|
|
243
|
+
'date-shift': (value, finding, transform) =>
|
|
244
|
+
shiftedDate(value, transform.days),
|
|
245
|
+
};
|
|
154
246
|
function transformed(value, finding, transform, options) {
|
|
155
247
|
if (typeof transform === 'function') {
|
|
156
248
|
const result = transform(Object.freeze({ ...finding, value }));
|
|
@@ -175,17 +267,8 @@ function transformed(value, finding, transform, options) {
|
|
|
175
267
|
if (finding.category === 'credential') {
|
|
176
268
|
return REDACTED;
|
|
177
269
|
}
|
|
178
|
-
if (transform.mode
|
|
179
|
-
return
|
|
180
|
-
}
|
|
181
|
-
if (transform.mode === 'pseudonym') {
|
|
182
|
-
return `[PSEUDONYM_${keyed(value, finding.type, transform.key).toString('hex').slice(0, 24)}]`;
|
|
183
|
-
}
|
|
184
|
-
if (transform.mode === 'format-preserving') {
|
|
185
|
-
return formatPseudonym(value, finding.type, transform.key);
|
|
186
|
-
}
|
|
187
|
-
if (transform.mode === 'date-shift') {
|
|
188
|
-
return shiftedDate(value, transform.days);
|
|
270
|
+
if (Object.hasOwn(personalModes, transform.mode)) {
|
|
271
|
+
return personalModes[transform.mode](value, finding, transform);
|
|
189
272
|
}
|
|
190
273
|
const number = Number(value);
|
|
191
274
|
if (!Number.isSafeInteger(number)) {
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Zachary Rice
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
{
|
|
2
|
+
"version": "v1.10.0",
|
|
3
|
+
"source": "https://raw.githubusercontent.com/betterleaks/betterleaks/v1.10.0/config/betterleaks.toml",
|
|
4
|
+
"sha256": "3489c5044d3a6e5abfd17a381e9de188b6f4fdee49f7f44bae89e2f8932464fa",
|
|
5
|
+
"rules": 203,
|
|
6
|
+
"files": 6,
|
|
7
|
+
"partialFilters": [
|
|
8
|
+
"box-api-access-token"
|
|
9
|
+
],
|
|
10
|
+
"policy": "Only rule ids absent from the vendored Gitleaks set; original RE2 patterns; entropy and secret-only matchesAny/containsAny filters; no tokenRatio, nearby-line evidence, validation or skipReport components."
|
|
11
|
+
}
|