@link-foundation/sensitive-data-sanitizer 0.19.3 → 0.21.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (71) hide show
  1. package/CHANGELOG.md +12 -0
  2. package/README.md +22 -2
  3. package/bin/sensitive-data-sanitizer.js +50 -28
  4. package/docs/API.md +355 -16
  5. package/docs/COVERAGE.md +19 -9
  6. package/docs/RULES.md +218 -0
  7. package/examples/second-pass-placeholders.mjs +28 -0
  8. package/package.json +12 -3
  9. package/src/allow-list.js +93 -0
  10. package/src/confidence.js +24 -9
  11. package/src/context.js +287 -40
  12. package/src/credential-spans.js +180 -0
  13. package/src/credentials-entry.d.ts +23 -0
  14. package/src/credentials-entry.js +62 -0
  15. package/src/credentials.js +51 -0
  16. package/src/crypto-addresses.js +74 -0
  17. package/src/detection.js +87 -41
  18. package/src/document-person-cues.js +1 -1
  19. package/src/encoded.js +137 -51
  20. package/src/encrypt.js +95 -0
  21. package/src/engines.js +33 -10
  22. package/src/entities.js +4 -3
  23. package/src/exempt.js +72 -0
  24. package/src/fake-memory.js +32 -0
  25. package/src/fake.js +33 -29
  26. package/src/features.js +29 -0
  27. package/src/files.js +57 -12
  28. package/src/gitleaks.js +82 -27
  29. package/src/html-tags.js +8 -0
  30. package/src/identifier-detector.js +36 -0
  31. package/src/identity-types.js +63 -0
  32. package/src/identity.js +3 -47
  33. package/src/index.d.ts +157 -3
  34. package/src/index.js +13 -0
  35. package/src/known.js +33 -15
  36. package/src/national-checks.js +209 -0
  37. package/src/national.js +310 -92
  38. package/src/pattern-detector.js +219 -0
  39. package/src/personal-context.js +3 -2
  40. package/src/personal-evidence.js +233 -0
  41. package/src/personal-features.js +39 -0
  42. package/src/personal.js +66 -10
  43. package/src/placeholder-types.js +154 -4
  44. package/src/placeholders.js +127 -45
  45. package/src/projection.js +78 -26
  46. package/src/proper-names.js +48 -5
  47. package/src/prose-words.js +120 -0
  48. package/src/record-worker.js +1 -0
  49. package/src/rule-catalog.js +426 -0
  50. package/src/rules.js +8 -0
  51. package/src/sanitizer.js +133 -23
  52. package/src/sarif.js +127 -0
  53. package/src/stream-worker.js +1 -0
  54. package/src/strict-attributes.js +48 -4
  55. package/src/strict.js +1 -1
  56. package/src/structured.js +2 -0
  57. package/src/token-checksum.js +49 -0
  58. package/src/transforms.js +146 -33
  59. package/src/vendor/betterleaks/LICENSE +21 -0
  60. package/src/vendor/betterleaks/provenance.json +11 -0
  61. package/src/vendor/betterleaks/rules-0.json +463 -0
  62. package/src/vendor/betterleaks/rules-1.json +454 -0
  63. package/src/vendor/betterleaks/rules-2.json +461 -0
  64. package/src/vendor/betterleaks/rules-3.json +455 -0
  65. package/src/vendor/betterleaks/rules-4.json +483 -0
  66. package/src/vendor/betterleaks/rules-5.json +35 -0
  67. package/src/vendor/entities/aws.json +1 -1
  68. package/src/vendor/entities/azure.json +6 -2
  69. package/src/vendor/entities/google.json +2 -1
  70. package/src/vocabulary.js +27 -1
  71. package/src/worker-options.js +1 -0
package/src/transforms.js CHANGED
@@ -1,10 +1,11 @@
1
- import { realisticFake } from './fake.js';
1
+ import { feature } from './features.js';
2
2
  import { createHmac } from 'node:crypto';
3
3
  import { failure, REDACTED } from './detection.js';
4
4
  import { encodedValue, encodeRun, decodedRuns } from './encoded.js';
5
- import { isIdentityType } from './identity.js';
5
+ import { isIdentityType } from './identity-types.js';
6
6
  import { validatePlaceholder, placeholderReplacement } from './placeholders.js';
7
7
  import { projectText } from './projection.js';
8
+ import { encryptValue } from './encrypt.js';
8
9
 
9
10
  const modes = new Set([
10
11
  'redact',
@@ -16,6 +17,7 @@ const modes = new Set([
16
17
  'date-shift',
17
18
  'bucket',
18
19
  'placeholder',
20
+ 'encrypt',
19
21
  ]);
20
22
  export function validateTransforms(options) {
21
23
  if (
@@ -86,16 +88,10 @@ function validateTransform(transform) {
86
88
  Number.isSafeInteger(transform.days) &&
87
89
  Math.abs(transform.days) <= 365000,
88
90
  bucket: () => positiveInteger(transform.size),
89
- mask: () =>
90
- [transform.keepStart ?? 0, transform.keepEnd ?? 0].every(
91
- (value) => Number.isSafeInteger(value) && value >= 0
92
- ) &&
93
- (transform.minLength === undefined ||
94
- positiveInteger(transform.minLength)) &&
95
- (transform.marker === undefined ||
96
- (typeof transform.marker === 'string' &&
97
- transform.marker.length > 0 &&
98
- transform.marker.length <= 64)),
91
+ mask: () => validMask(transform),
92
+ encrypt: () =>
93
+ typeof transform.key === 'string' &&
94
+ Buffer.byteLength(transform.key) >= 16,
99
95
  };
100
96
  if (checks[transform.mode] && !checks[transform.mode]()) {
101
97
  throw failure('ERR_CONFIG');
@@ -141,16 +137,112 @@ function shiftedDate(value, days) {
141
137
  ? REDACTED
142
138
  : date.toISOString().slice(0, 10);
143
139
  }
140
+ // Google charactersToIgnore/commonCharactersToIgnore and Presidio-style
141
+ // character classes that a mask may leave visible.
142
+ const maskClasses = {
143
+ digits: /\p{Nd}/u,
144
+ upper: /\p{Lu}/u,
145
+ lower: /\p{Ll}/u,
146
+ punctuation: /[\p{P}\p{S}]/u,
147
+ whitespace: /\s/u,
148
+ };
149
+ const maskTuning = ['char', 'ignore', 'ignoreClasses', 'count', 'fromEnd'];
150
+ const unset = (value, test) => value === undefined || test(value);
151
+ const maskChecks = [
152
+ (t) =>
153
+ [t.keepStart ?? 0, t.keepEnd ?? 0].every(
154
+ (value) => Number.isSafeInteger(value) && value >= 0
155
+ ),
156
+ (t) => unset(t.minLength, positiveInteger),
157
+ (t) =>
158
+ unset(
159
+ t.marker,
160
+ (marker) =>
161
+ !maskTuning.some((key) => t[key] !== undefined) &&
162
+ typeof marker === 'string' &&
163
+ marker.length > 0 &&
164
+ marker.length <= 64
165
+ ),
166
+ (t) =>
167
+ unset(
168
+ t.char,
169
+ (char) => typeof char === 'string' && Array.from(char).length === 1
170
+ ),
171
+ (t) =>
172
+ unset(
173
+ t.ignore,
174
+ (ignore) =>
175
+ typeof ignore === 'string' && ignore.length > 0 && ignore.length <= 256
176
+ ),
177
+ (t) =>
178
+ unset(
179
+ t.ignoreClasses,
180
+ (names) =>
181
+ Array.isArray(names) &&
182
+ names.every((name) => Object.hasOwn(maskClasses, name))
183
+ ),
184
+ (t) => unset(t.count, positiveInteger),
185
+ (t) => unset(t.fromEnd, (value) => typeof value === 'boolean'),
186
+ ];
187
+ function validMask(transform) {
188
+ return maskChecks.every((check) => check(transform));
189
+ }
190
+ // Indexes of the characters a mask replaces, after ignored characters, the
191
+ // direction and the count limit are applied.
192
+ function maskPositions(chars, start, end, transform) {
193
+ const ignored = (char) =>
194
+ Boolean(transform.ignore?.includes(char)) ||
195
+ (transform.ignoreClasses ?? []).some((name) =>
196
+ maskClasses[name].test(char)
197
+ );
198
+ const positions = [];
199
+ for (let i = start; i < chars.length - end; i++) {
200
+ if (!ignored(chars[i])) {
201
+ positions.push(i);
202
+ }
203
+ }
204
+ if (transform.fromEnd) {
205
+ positions.reverse();
206
+ }
207
+ return transform.count === undefined
208
+ ? positions
209
+ : positions.slice(0, transform.count);
210
+ }
144
211
  function masked(value, finding, transform) {
145
212
  const chars = Array.from(value),
146
213
  start = transform.keepStart ?? 0,
147
214
  end = transform.keepEnd ?? 0;
148
- return chars.length <
149
- (transform.minLength ?? (isIdentityType(finding.type) ? 5 : 0)) ||
215
+ if (
216
+ chars.length <
217
+ (transform.minLength ?? (isIdentityType(finding.type) ? 5 : 0)) ||
150
218
  start + end >= chars.length
151
- ? REDACTED
152
- : `${chars.slice(0, start).join('')}${transform.marker ?? '*'.repeat(chars.length - start - end)}${end ? chars.slice(-end).join('') : ''}`;
219
+ ) {
220
+ return REDACTED;
221
+ }
222
+ if (transform.marker !== undefined) {
223
+ return `${chars.slice(0, start).join('')}${transform.marker}${end ? chars.slice(-end).join('') : ''}`;
224
+ }
225
+ const positions = maskPositions(chars, start, end, transform);
226
+ if (!positions.length) {
227
+ return REDACTED;
228
+ }
229
+ for (const i of positions) {
230
+ chars[i] = transform.char ?? '*';
231
+ }
232
+ return chars.join('');
153
233
  }
234
+ // Modes that apply to personal data only; credentials are always redacted.
235
+ const personalModes = {
236
+ mask: masked,
237
+ encrypt: (value, finding, transform) =>
238
+ encryptValue(value, finding.type, transform.key),
239
+ pseudonym: (value, finding, transform) =>
240
+ `[PSEUDONYM_${keyed(value, finding.type, transform.key).toString('hex').slice(0, 24)}]`,
241
+ 'format-preserving': (value, finding, transform) =>
242
+ formatPseudonym(value, finding.type, transform.key),
243
+ 'date-shift': (value, finding, transform) =>
244
+ shiftedDate(value, transform.days),
245
+ };
154
246
  function transformed(value, finding, transform, options) {
155
247
  if (typeof transform === 'function') {
156
248
  const result = transform(Object.freeze({ ...finding, value }));
@@ -170,22 +262,13 @@ function transformed(value, finding, transform, options) {
170
262
  : REDACTED;
171
263
  }
172
264
  if (transform.mode === 'fake') {
173
- return realisticFake(value, finding, transform, options);
265
+ return feature('realisticFake')(value, finding, transform, options);
174
266
  }
175
267
  if (finding.category === 'credential') {
176
268
  return REDACTED;
177
269
  }
178
- if (transform.mode === 'mask') {
179
- return masked(value, finding, transform);
180
- }
181
- if (transform.mode === 'pseudonym') {
182
- return `[PSEUDONYM_${keyed(value, finding.type, transform.key).toString('hex').slice(0, 24)}]`;
183
- }
184
- if (transform.mode === 'format-preserving') {
185
- return formatPseudonym(value, finding.type, transform.key);
186
- }
187
- if (transform.mode === 'date-shift') {
188
- return shiftedDate(value, transform.days);
270
+ if (Object.hasOwn(personalModes, transform.mode)) {
271
+ return personalModes[transform.mode](value, finding, transform);
189
272
  }
190
273
  const number = Number(value);
191
274
  if (!Number.isSafeInteger(number)) {
@@ -194,6 +277,32 @@ function transformed(value, finding, transform, options) {
194
277
  const lower = Math.floor(number / transform.size) * transform.size;
195
278
  return `${lower}–${lower + transform.size - 1}`;
196
279
  }
280
+ function jsonValueText(raw) {
281
+ try {
282
+ const value = JSON.parse(raw);
283
+ return typeof value === 'string' && value ? value : raw;
284
+ } catch {
285
+ return raw;
286
+ }
287
+ }
288
+ // A whole JSON value becomes one JSON string. Placeholder mode types it by the
289
+ // field it came from, so `{"age":34}` reads `{"age":"Age-1"}`.
290
+ function jsonValueReplacement(text, finding, options) {
291
+ const transform =
292
+ options.transformations?.JSON_VALUE ?? options.transformation;
293
+ if (transform?.mode !== 'placeholder') {
294
+ return JSON.stringify(REDACTED);
295
+ }
296
+ const raw = text.slice(finding.start, finding.end);
297
+ return JSON.stringify(
298
+ placeholderReplacement(
299
+ jsonValueText(raw),
300
+ { ...finding, type: finding.valueType ?? 'SENSITIVE' },
301
+ transform,
302
+ options
303
+ )
304
+ );
305
+ }
197
306
  export function renderReplacement(
198
307
  text,
199
308
  finding,
@@ -202,7 +311,7 @@ export function renderReplacement(
202
311
  jsonRanges
203
312
  ) {
204
313
  if (finding.type === 'JSON_VALUE') {
205
- return JSON.stringify(REDACTED);
314
+ return jsonValueReplacement(text, finding, options);
206
315
  }
207
316
  const transform =
208
317
  options.transformations?.[finding.type] ??
@@ -217,11 +326,15 @@ export function renderReplacement(
217
326
  const sanitized = encodedSanitize(run.text).text;
218
327
  // Mapped escaped fragments can lose their detecting context on decode.
219
328
  // Redact the fragment if recursive scanning makes no progress.
220
- return encodeRun(run, sanitized === run.text ? REDACTED : sanitized);
221
- }
222
- if (options.preserveEncoding) {
223
- throw failure('ERR_ENCODING');
329
+ const encoded = encodeRun(
330
+ run,
331
+ sanitized === run.text ? REDACTED : sanitized
332
+ );
333
+ if (encoded !== undefined) {
334
+ return encoded;
335
+ }
224
336
  }
337
+ // Without a recognizable re-encoding, the encoded text itself is masked.
225
338
  }
226
339
  return renderValue(text, finding, transform, options, jsonRanges);
227
340
  }
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Zachary Rice
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,11 @@
1
+ {
2
+ "version": "v1.10.0",
3
+ "source": "https://raw.githubusercontent.com/betterleaks/betterleaks/v1.10.0/config/betterleaks.toml",
4
+ "sha256": "3489c5044d3a6e5abfd17a381e9de188b6f4fdee49f7f44bae89e2f8932464fa",
5
+ "rules": 203,
6
+ "files": 6,
7
+ "partialFilters": [
8
+ "box-api-access-token"
9
+ ],
10
+ "policy": "Only rule ids absent from the vendored Gitleaks set; original RE2 patterns; entropy and secret-only matchesAny/containsAny filters; no tokenRatio, nearby-line evidence, validation or skipReport components."
11
+ }