@link-foundation/sensitive-data-sanitizer 0.12.2 → 0.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/README.md +44 -0
- package/bin/placeholders.js +115 -0
- package/bin/sensitive-data-sanitizer.js +47 -16
- package/bin/stream.js +2 -1
- package/docs/API.md +130 -21
- package/docs/COVERAGE.md +50 -0
- package/examples/reversible-placeholders.mjs +35 -0
- package/package.json +4 -2
- package/src/context.js +23 -9
- package/src/detection.js +26 -25
- package/src/document-entities.js +110 -0
- package/src/document-vocabulary.js +71 -0
- package/src/engines.js +42 -18
- package/src/entities.js +7 -0
- package/src/fake-names.js +61 -7
- package/src/fake.js +7 -0
- package/src/index.d.ts +55 -2
- package/src/index.js +8 -0
- package/src/json-string-runs.js +18 -0
- package/src/jsonl-stream.js +5 -0
- package/src/keywords.js +17 -0
- package/src/names.js +1 -1
- package/src/personal-variants.js +94 -0
- package/src/personal.js +6 -1
- package/src/placeholder-types.js +33 -0
- package/src/placeholders.js +405 -0
- package/src/projection.js +6 -2
- package/src/propagation.js +13 -1
- package/src/restore-map.js +220 -0
- package/src/sanitizer.js +119 -18
- package/src/stream.js +23 -2
- package/src/structured.js +5 -1
- package/src/transforms.js +12 -1
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,17 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 0.14.0
|
|
4
|
+
|
|
5
|
+
### Minor Changes
|
|
6
|
+
|
|
7
|
+
- d5b2aac: Add known-only reversible sanitization, Unicode known-value boundaries and bounded Russian entity case matching. Correct placeholder namespace collisions, Russian document entities and fake-name roles, and credential prose false positives. Report irreversible replacements in placeholder results.
|
|
8
|
+
|
|
9
|
+
## 0.13.0
|
|
10
|
+
|
|
11
|
+
### Minor Changes
|
|
12
|
+
|
|
13
|
+
- 818f198: Add reversible typed placeholders with private JSON and Links Notation maps, map reuse and restoration, a no-map anonymization mode, and opt-in multilingual document entity detection for employers, schools, cities and regions.
|
|
14
|
+
|
|
3
15
|
## 0.12.2
|
|
4
16
|
|
|
5
17
|
### Patch Changes
|
package/README.md
CHANGED
|
@@ -22,6 +22,50 @@ sensitive-data-sanitizer redact session.jsonl --stream --jsonl --two-pass --outp
|
|
|
22
22
|
|
|
23
23
|
## Quick Start
|
|
24
24
|
|
|
25
|
+
### Readable placeholders and document profiles
|
|
26
|
+
|
|
27
|
+
Use the opt-in `documents` profile for CVs and similar prose. It adds bounded
|
|
28
|
+
employer, school, city and region detection across the existing 17 locales.
|
|
29
|
+
Default and publication log profiles keep their existing detection behavior.
|
|
30
|
+
|
|
31
|
+
```js
|
|
32
|
+
import { sanitize, restore } from '@link-foundation/sensitive-data-sanitizer';
|
|
33
|
+
|
|
34
|
+
const result = sanitize('Jane Doe worked at Acme Ltd in Springfield.', {
|
|
35
|
+
profile: 'documents',
|
|
36
|
+
transformation: { mode: 'placeholder' },
|
|
37
|
+
});
|
|
38
|
+
// result.text: Person-1 worked at Organization-1 in Location-1.
|
|
39
|
+
// Keep result.map private. Share only result.text.
|
|
40
|
+
restore(result.text, result.map); // exact original document
|
|
41
|
+
restore('Ask Person-1 about Organization-1.', result.map); // derived answer
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
The restore map **is personal data**. It contains original spellings and private
|
|
45
|
+
occurrence metadata; never commit it, send it to a model, or publish the whole
|
|
46
|
+
result object. The CLI writes maps separately with mode `0600`, and publication
|
|
47
|
+
checks reject serialized restore maps. Reuse `result.map` in the next
|
|
48
|
+
`transformation.map` to keep IDs stable across files. JSON and Links Notation
|
|
49
|
+
serializations carry the same versioned map. Credentials stay fully redacted
|
|
50
|
+
and out of the map unless explicitly opted in.
|
|
51
|
+
|
|
52
|
+
```sh
|
|
53
|
+
sensitive-data-sanitizer redact cv.txt --profile documents --placeholder --map private.lino --output safe.txt
|
|
54
|
+
sensitive-data-sanitizer restore safe.txt --map private.lino --output restored.txt
|
|
55
|
+
sensitive-data-sanitizer redact cv.txt --profile documents --placeholder --no-map
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
`knownOnly: true` (CLI `--known-only`) limits replacement to `knownPersonal`,
|
|
59
|
+
`knownSecrets` and reused map identities, preserving fictional examples.
|
|
60
|
+
Placeholder results report `irreversibleRedactions`; a nonzero count means some
|
|
61
|
+
replacements, such as credentials excluded from the map, cannot be restored.
|
|
62
|
+
|
|
63
|
+
For irreversible output, choose `transformation: { mode: 'placeholder', map:
|
|
64
|
+
false }`: no restore map is returned or written. Detection limits still apply
|
|
65
|
+
in either mode. See the [placeholder API](docs/API.md#reversible-placeholders),
|
|
66
|
+
[coverage](docs/COVERAGE.md#document-entities), and
|
|
67
|
+
[runnable example](examples/reversible-placeholders.mjs).
|
|
68
|
+
|
|
25
69
|
Sanitizer APIs require Node.js 22.13 or later. Template arithmetic utilities are available under `@link-foundation/sensitive-data-sanitizer/legacy`; the existing universal example uses that browser-compatible entry. From this checkout:
|
|
26
70
|
|
|
27
71
|
```sh
|
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
import { resolve } from 'node:path';
|
|
2
|
+
import { failure } from '../src/detection.js';
|
|
3
|
+
import {
|
|
4
|
+
createRestoreMap,
|
|
5
|
+
readRestoreMap,
|
|
6
|
+
writeRestoreMap,
|
|
7
|
+
restore,
|
|
8
|
+
} from '../src/restore-map.js';
|
|
9
|
+
import { readTextFile, atomicWrite } from '../src/files.js';
|
|
10
|
+
|
|
11
|
+
export async function placeholderOptions(config, options) {
|
|
12
|
+
if (config.command === 'restore') {
|
|
13
|
+
return;
|
|
14
|
+
}
|
|
15
|
+
if (config.placeholder) {
|
|
16
|
+
options.transformation = { mode: 'placeholder' };
|
|
17
|
+
}
|
|
18
|
+
const transform = options.transformation;
|
|
19
|
+
validatePlaceholderFlags(config, transform);
|
|
20
|
+
if (transform?.mode !== 'placeholder') {
|
|
21
|
+
return;
|
|
22
|
+
}
|
|
23
|
+
validateMapArgs(config);
|
|
24
|
+
if (config.placeholderFormat) {
|
|
25
|
+
transform.format = config.placeholderFormat;
|
|
26
|
+
}
|
|
27
|
+
if (config.mapCredentials) {
|
|
28
|
+
transform.credentials = true;
|
|
29
|
+
}
|
|
30
|
+
if (config.noMap) {
|
|
31
|
+
transform.map = false;
|
|
32
|
+
}
|
|
33
|
+
if (config.map) {
|
|
34
|
+
try {
|
|
35
|
+
const { lstat } = await import('node:fs/promises');
|
|
36
|
+
await lstat(config.map);
|
|
37
|
+
transform.map = await readRestoreMap(config.map);
|
|
38
|
+
config.replaceMap = true;
|
|
39
|
+
} catch (error) {
|
|
40
|
+
if (error.code !== 'ENOENT') {
|
|
41
|
+
throw error;
|
|
42
|
+
}
|
|
43
|
+
transform.map = createRestoreMap({ format: transform.format });
|
|
44
|
+
}
|
|
45
|
+
} else if (transform.map !== false) {
|
|
46
|
+
// The CLI cannot return a private map alongside stdout safely.
|
|
47
|
+
throw failure('ERR_ARGUMENT');
|
|
48
|
+
}
|
|
49
|
+
config.restoreMap = transform.map;
|
|
50
|
+
}
|
|
51
|
+
function validatePlaceholderFlags(config, transform) {
|
|
52
|
+
if (
|
|
53
|
+
(config.map ||
|
|
54
|
+
config.noMap ||
|
|
55
|
+
config.placeholderFormat ||
|
|
56
|
+
config.mapCredentials ||
|
|
57
|
+
config.mapFormat) &&
|
|
58
|
+
transform?.mode !== 'placeholder'
|
|
59
|
+
) {
|
|
60
|
+
throw failure('ERR_ARGUMENT');
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
function validateMapArgs(config) {
|
|
64
|
+
if (config.map && config.noMap) {
|
|
65
|
+
throw failure('ERR_ARGUMENT');
|
|
66
|
+
}
|
|
67
|
+
if (config.mapFormat && !['json', 'lino'].includes(config.mapFormat)) {
|
|
68
|
+
throw failure('ERR_ARGUMENT');
|
|
69
|
+
}
|
|
70
|
+
if (
|
|
71
|
+
config.map &&
|
|
72
|
+
[config.path, config.output, config.config]
|
|
73
|
+
.filter(Boolean)
|
|
74
|
+
.some((path) => resolve(path) === resolve(config.map))
|
|
75
|
+
) {
|
|
76
|
+
throw failure('ERR_OUTPUT');
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
export async function saveMap(config) {
|
|
81
|
+
if (config.map) {
|
|
82
|
+
await writeRestoreMap(config.map, config.restoreMap, {
|
|
83
|
+
replace: Boolean(config.replaceMap),
|
|
84
|
+
...(config.mapFormat ? { format: config.mapFormat } : {}),
|
|
85
|
+
});
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
export async function restoreCommand(config, { stdout }) {
|
|
89
|
+
if (
|
|
90
|
+
!config.map ||
|
|
91
|
+
config.path === '-' ||
|
|
92
|
+
config.stream ||
|
|
93
|
+
config.noMap ||
|
|
94
|
+
config.placeholder ||
|
|
95
|
+
config.mapCredentials ||
|
|
96
|
+
config.json ||
|
|
97
|
+
config.jsonl ||
|
|
98
|
+
config.inPlace
|
|
99
|
+
) {
|
|
100
|
+
throw failure('ERR_ARGUMENT');
|
|
101
|
+
}
|
|
102
|
+
if (config.output && resolve(config.output) === resolve(config.map)) {
|
|
103
|
+
throw failure('ERR_OUTPUT');
|
|
104
|
+
}
|
|
105
|
+
const text = restore(
|
|
106
|
+
await readTextFile(config.path, config.maxBytes),
|
|
107
|
+
await readRestoreMap(config.map)
|
|
108
|
+
);
|
|
109
|
+
if (config.output) {
|
|
110
|
+
await atomicWrite(config.output, text);
|
|
111
|
+
} else {
|
|
112
|
+
stdout.write(text);
|
|
113
|
+
}
|
|
114
|
+
return 0;
|
|
115
|
+
}
|
|
@@ -21,16 +21,18 @@ import { encodeLogText } from '../src/log-bytes.js';
|
|
|
21
21
|
import { createWikidataVerifier } from '../src/public-verifier.js';
|
|
22
22
|
import { streamCommand } from './stream.js';
|
|
23
23
|
import { sessionStructuralFields } from '../src/structured.js';
|
|
24
|
+
import { placeholderOptions, saveMap, restoreCommand } from './placeholders.js';
|
|
24
25
|
import {
|
|
25
26
|
createGitleaksDetector,
|
|
26
27
|
createTrufflehogDetector,
|
|
27
28
|
createPresidioDetector,
|
|
28
29
|
} from '../src/external.js';
|
|
29
30
|
|
|
30
|
-
const help = `Usage: sensitive-data-sanitizer <redact|scan|history> [path|-] [options]
|
|
31
|
+
const help = `Usage: sensitive-data-sanitizer <redact|scan|restore|history> [path|-] [options]
|
|
31
32
|
|
|
32
33
|
redact Sanitize one text log or stdin; publish only after verification
|
|
33
34
|
scan Report metadata for a file, directory, or stdin (never source text)
|
|
35
|
+
restore Restore placeholders in a file using --map PATH (private output)
|
|
34
36
|
history Audit local reachable Git blobs, commit/tag metadata, and reflogs
|
|
35
37
|
history rewrite SOURCE --output DIRECTORY [--apply]
|
|
36
38
|
Preview a fresh mirror clone; --apply rewrites only that clone
|
|
@@ -40,6 +42,7 @@ history rewrite SOURCE --output DIRECTORY [--apply]
|
|
|
40
42
|
--in-place Atomically replace the input file (redact only)
|
|
41
43
|
--native-only Disable Secretlint; selected local detectors remain required
|
|
42
44
|
--credentials-only Preserve personal/account details; redact credentials only
|
|
45
|
+
--known-only Replace only knownPersonal/knownSecrets and reused map values
|
|
43
46
|
--env-secrets Include values of all credential-named environment variables
|
|
44
47
|
--gh-config Include tokens from the local gh hosts.yml credential file
|
|
45
48
|
--paranoid Add entropy detection (can redact innocent identifiers)
|
|
@@ -56,7 +59,13 @@ history rewrite SOURCE --output DIRECTORY [--apply]
|
|
|
56
59
|
--json Sanitize JSON strings and keys; preserve session metadata
|
|
57
60
|
--jsonl Sanitize JSONL records; combine with --stream for sessions
|
|
58
61
|
--identity-mask First/last 2 identity mask (publication profile default)
|
|
59
|
-
--profile NAME
|
|
62
|
+
--profile NAME publication (redact default) or documents (CV/entity rules)
|
|
63
|
+
--placeholder Typed numbered placeholders; requires --map or --no-map
|
|
64
|
+
--map PATH Private JSON/.lino restore map; reuse it across files/runs
|
|
65
|
+
--no-map Discard placeholder mapping after full anonymization
|
|
66
|
+
--placeholder-format TEMPLATE Default {Type}-{n}; also [{TYPE}_{n}]
|
|
67
|
+
--map-format FORMAT json or lino (otherwise inferred from .lino extension)
|
|
68
|
+
--map-credentials Explicitly keep credentials in the private restore map
|
|
60
69
|
--max-record-bytes N Maximum record bytes (text: total limit; JSONL: 8388608)
|
|
61
70
|
--hive-mask Opt-in first/last 3 mask for values longer than 12 characters
|
|
62
71
|
--preserve-encoding Re-encode sanitized encoded payloads with round-trip checks
|
|
@@ -80,6 +89,7 @@ function parseArgs(argv) {
|
|
|
80
89
|
['--in-place', 'inPlace'],
|
|
81
90
|
['--native-only', 'nativeOnly'],
|
|
82
91
|
['--credentials-only', 'credentialsOnly'],
|
|
92
|
+
['--known-only', 'knownOnly'],
|
|
83
93
|
['--env-secrets', 'envSecrets'],
|
|
84
94
|
['--gh-config', 'ghConfig'],
|
|
85
95
|
['--paranoid', 'paranoid'],
|
|
@@ -96,6 +106,9 @@ function parseArgs(argv) {
|
|
|
96
106
|
['--gh-auth', 'ghAuth'],
|
|
97
107
|
['--verify-public', 'verifyPublic'],
|
|
98
108
|
['--apply', 'apply'],
|
|
109
|
+
['--placeholder', 'placeholder'],
|
|
110
|
+
['--no-map', 'noMap'],
|
|
111
|
+
['--map-credentials', 'mapCredentials'],
|
|
99
112
|
]);
|
|
100
113
|
const valued = new Map([
|
|
101
114
|
['--config', 'config'],
|
|
@@ -109,6 +122,9 @@ function parseArgs(argv) {
|
|
|
109
122
|
['--worker-timeout-ms', 'workerTimeoutMs'],
|
|
110
123
|
['--filter-repo', 'filterRepo'],
|
|
111
124
|
['--profile', 'profile'],
|
|
125
|
+
['--map', 'map'],
|
|
126
|
+
['--map-format', 'mapFormat'],
|
|
127
|
+
['--placeholder-format', 'placeholderFormat'],
|
|
112
128
|
]);
|
|
113
129
|
let hasPath = false;
|
|
114
130
|
for (let i = rewrite ? 2 : 1; i < argv.length; i++) {
|
|
@@ -134,7 +150,7 @@ function parseArgs(argv) {
|
|
|
134
150
|
function validateArgs(config) {
|
|
135
151
|
config.maxBytes = Number(config.maxBytes ?? defaultByteLimit(config));
|
|
136
152
|
if (
|
|
137
|
-
!['redact', 'scan', 'history'].includes(config.command) ||
|
|
153
|
+
!['redact', 'scan', 'restore', 'history'].includes(config.command) ||
|
|
138
154
|
!Number.isSafeInteger(config.maxBytes) ||
|
|
139
155
|
config.maxBytes <= 0
|
|
140
156
|
) {
|
|
@@ -143,7 +159,7 @@ function validateArgs(config) {
|
|
|
143
159
|
if (
|
|
144
160
|
(config.output && config.inPlace) ||
|
|
145
161
|
((config.output || config.inPlace) &&
|
|
146
|
-
config.command
|
|
162
|
+
!['redact', 'restore'].includes(config.command) &&
|
|
147
163
|
!config.rewrite) ||
|
|
148
164
|
(config.inPlace && config.path === '-') ||
|
|
149
165
|
(config.command === 'history' && config.path === '-')
|
|
@@ -252,19 +268,8 @@ export async function runCli(
|
|
|
252
268
|
return 0;
|
|
253
269
|
}
|
|
254
270
|
const config = parseArgs(argv);
|
|
255
|
-
const options = await loadOptions(config);
|
|
256
|
-
const engine = createSanitizer(options);
|
|
257
271
|
const context = { stdin, stdout };
|
|
258
|
-
|
|
259
|
-
return await historyCommand(config, engine, context);
|
|
260
|
-
}
|
|
261
|
-
if (config.stream) {
|
|
262
|
-
return await streamCommand(config, engine, options, context);
|
|
263
|
-
}
|
|
264
|
-
if (config.command === 'redact') {
|
|
265
|
-
return await redactCommand(config, engine, context);
|
|
266
|
-
}
|
|
267
|
-
return await scanCommand(config, engine, context);
|
|
272
|
+
return await dispatchCommand(config, context);
|
|
268
273
|
} catch (error) {
|
|
269
274
|
const location =
|
|
270
275
|
Number.isSafeInteger(error.record) && Number.isSafeInteger(error.offset)
|
|
@@ -277,6 +282,27 @@ export async function runCli(
|
|
|
277
282
|
}
|
|
278
283
|
}
|
|
279
284
|
|
|
285
|
+
async function dispatchCommand(config, context) {
|
|
286
|
+
if (config.command === 'restore') {
|
|
287
|
+
return await restoreCommand(config, context);
|
|
288
|
+
}
|
|
289
|
+
const options = await loadOptions(config);
|
|
290
|
+
const engine = createSanitizer(options);
|
|
291
|
+
if (config.command === 'history') {
|
|
292
|
+
return await historyCommand(config, engine, context);
|
|
293
|
+
}
|
|
294
|
+
if (config.stream) {
|
|
295
|
+
return await streamCommand(config, engine, options, {
|
|
296
|
+
...context,
|
|
297
|
+
beforePublish: () => saveMap(config),
|
|
298
|
+
});
|
|
299
|
+
}
|
|
300
|
+
if (config.command === 'redact') {
|
|
301
|
+
return await redactCommand(config, engine, context);
|
|
302
|
+
}
|
|
303
|
+
return await scanCommand(config, engine, context);
|
|
304
|
+
}
|
|
305
|
+
|
|
280
306
|
async function loadOptions(config) {
|
|
281
307
|
let options = {};
|
|
282
308
|
if (config.config) {
|
|
@@ -300,6 +326,7 @@ async function loadOptions(config) {
|
|
|
300
326
|
options.paranoid = true;
|
|
301
327
|
}
|
|
302
328
|
await featureOptions(config, options);
|
|
329
|
+
await placeholderOptions(config, options);
|
|
303
330
|
options.detectors = selectedDetectors(config, options.detectors);
|
|
304
331
|
options.maxInputLength = Math.min(
|
|
305
332
|
options.maxInputLength ?? config.maxBytes,
|
|
@@ -314,6 +341,9 @@ async function loadOptions(config) {
|
|
|
314
341
|
|
|
315
342
|
async function featureOptions(config, options) {
|
|
316
343
|
await localCredentials(config, options);
|
|
344
|
+
if (config.knownOnly) {
|
|
345
|
+
options.knownOnly = true;
|
|
346
|
+
}
|
|
317
347
|
if (config.credentialsOnly) {
|
|
318
348
|
options.credentialsOnly = true;
|
|
319
349
|
}
|
|
@@ -391,6 +421,7 @@ async function redactCommand(config, engine, { stdin, stdout }) {
|
|
|
391
421
|
const result = await engine.sanitize(
|
|
392
422
|
await inputText(config.path, config, stdin)
|
|
393
423
|
);
|
|
424
|
+
await saveMap(config);
|
|
394
425
|
if (config.output || config.inPlace) {
|
|
395
426
|
await atomicWrite(
|
|
396
427
|
config.output ?? config.path,
|
package/bin/stream.js
CHANGED
|
@@ -9,7 +9,7 @@ export async function streamCommand(
|
|
|
9
9
|
config,
|
|
10
10
|
engine,
|
|
11
11
|
options,
|
|
12
|
-
{ stdin, stdout }
|
|
12
|
+
{ stdin, stdout, beforePublish }
|
|
13
13
|
) {
|
|
14
14
|
let spool;
|
|
15
15
|
try {
|
|
@@ -42,6 +42,7 @@ export async function streamCommand(
|
|
|
42
42
|
: { sanitizer: engine, sanitizerOptions: options }),
|
|
43
43
|
});
|
|
44
44
|
}
|
|
45
|
+
await beforePublish?.();
|
|
45
46
|
if (spool) {
|
|
46
47
|
for await (const chunk of createReadStream(target)) {
|
|
47
48
|
if (stdout.write(chunk) === false) {
|
package/docs/API.md
CHANGED
|
@@ -19,27 +19,136 @@ Import the ESM root entry; complete declarations are in `src/index.d.ts`. Inputs
|
|
|
19
19
|
|
|
20
20
|
## Options
|
|
21
21
|
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
22
|
+
### Replace only known values
|
|
23
|
+
|
|
24
|
+
Use `knownOnly: true` to preserve fictional examples while replacing selected
|
|
25
|
+
private values. This applies to sync/async, JSON/JSONL, streams and fake mode.
|
|
26
|
+
It skips automatic credential detection too; supply `knownSecrets` explicitly
|
|
27
|
+
for credentials that must be removed. `credentialsOnly` can further exclude
|
|
28
|
+
personal values. CLI: `--known-only` with a `--config` containing the known lists.
|
|
29
|
+
|
|
30
|
+
```js
|
|
31
|
+
const result = sanitize('Анна Смирнова; example Jane Doe', {
|
|
32
|
+
knownOnly: true,
|
|
33
|
+
knownPersonal: [{ type: 'PERSON', value: 'Анна Смирнова' }],
|
|
34
|
+
transformation: { mode: 'placeholder' },
|
|
35
|
+
});
|
|
36
|
+
// result.text: 'Person-1; example Jane Doe'
|
|
37
|
+
restore(result.text, result.map); // exact original
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
### Reversible placeholders
|
|
41
|
+
|
|
42
|
+
`transformation: {mode: 'placeholder'}` emits typed tokens such as `Person-1`,
|
|
43
|
+
`Email-1`, `Phone-1`, `Organization-1`, `Education-1`, `Location-1` and `Date-1`.
|
|
44
|
+
Numbers start at one per type and follow the first occurrence in the input.
|
|
45
|
+
`DATE_OF_BIRTH` uses the `Date` label; other detector types retain their type
|
|
46
|
+
name in title case. Per-type `transformations` also support placeholder mode;
|
|
47
|
+
all placeholder transformations in an operation must share one map and format.
|
|
48
|
+
|
|
49
|
+
The result has a private `map` alongside `text`, `findings` and `redactions`.
|
|
50
|
+
The map holds token entries, every original surface form, name aliases, and
|
|
51
|
+
replacement positions indexed by the SHA-256 digest of the sanitized document.
|
|
52
|
+
It does not store the original full document. `restore(text, map)` restores
|
|
53
|
+
the exact bytes represented by the original JavaScript string, including JSON
|
|
54
|
+
escaping and whitespace, when the document digest matches. A derived response
|
|
55
|
+
that quotes tokens uses the entry's canonical original spelling instead.
|
|
56
|
+
Restoration uses a single pass so restored originals are never interpreted as
|
|
57
|
+
more tokens. Unknown tokens and larger words containing token-like fragments
|
|
58
|
+
are left alone.
|
|
59
|
+
|
|
60
|
+
`format` is a template with exactly one type field (`{Type}`, `{TYPE}` or
|
|
61
|
+
`{type}`) and one `{n}` field. Examples: `{Type}-{n}`, `[{TYPE}_{n}]`,
|
|
62
|
+
`{{{type}_{n}}}`. Sanitization refuses input containing any token in the chosen
|
|
63
|
+
namespace for built-in and supplied type labels, including numbers not allocated yet.
|
|
64
|
+
Unicode letters, marks, digits and underscores bound those tokens. Ordinary
|
|
65
|
+
technical names such as GPT-4, ISO-9001 and Windows-10 are not reserved. The map's format cannot change
|
|
66
|
+
on reuse. A malformed map or ambiguous exact restoration blocks output.
|
|
67
|
+
|
|
68
|
+
`createRestoreMap({format?})` creates an empty reusable map. Pass a map as
|
|
69
|
+
`transformation.map` to continue a corpus; the operation updates that same
|
|
70
|
+
object. A `createSanitizer()` engine also keeps its registry across calls.
|
|
71
|
+
Callers must serialize access to a shared engine/map; parallel independent
|
|
72
|
+
registries cannot assign consistent first-appearance numbers. For streams,
|
|
73
|
+
pass an explicit map object and keep it outside the output stream. Placeholder
|
|
74
|
+
JSONL uses the coordinator in source order, even if several workers were
|
|
75
|
+
requested. File APIs also keep this registry in the caller process.
|
|
76
|
+
|
|
77
|
+
`serializeRestoreMap(map, 'json' | 'lino')` and `parseRestoreMap(text)` round
|
|
78
|
+
trip the same schema. Lino uses the upstream `links-notation` parser and
|
|
79
|
+
formatter: a named `sensitive-data-restore-map` link carries the escaped JSON
|
|
80
|
+
schema as a reference, preserving all metadata and unusual original strings.
|
|
81
|
+
`readRestoreMap(path)` and `writeRestoreMap(path, map, {format?, replace?})`
|
|
82
|
+
use bounded file reads and the existing atomic `0600` writer. The default
|
|
83
|
+
serialization is JSON, or Lino for a `.lino` output path. Existing files require
|
|
84
|
+
`replace: true`; symlinks are refused. `inspect`, `sanitize`, scanner union and
|
|
85
|
+
CLI publication paths reject marked map content with `ERR_PRIVATE_MAP`.
|
|
86
|
+
Keep maps out of Git, logs, model requests and publication directories. The map
|
|
87
|
+
is plaintext personal data, not encrypted storage; protect it as the original.
|
|
88
|
+
|
|
89
|
+
`credentials: true` explicitly opts credentials into `Credential-n` entries.
|
|
90
|
+
Otherwise they remain `[REDACTED]` and cannot be recovered, including through
|
|
91
|
+
occurrence metadata. Placeholder results include `irreversibleRedactions`, the
|
|
92
|
+
number of replacements excluded from the restore map (including mixed policies).
|
|
93
|
+
Check that this is zero before relying on exact restoration. A positive count
|
|
94
|
+
means `restore()` cannot reconstruct all of the input. `map: false` returns only the anonymized text and ordinary
|
|
95
|
+
value-free finding metadata. The temporary numbering registry is discarded
|
|
96
|
+
after an operation (or the complete stream); no map file is created. Residual verification
|
|
97
|
+
still runs, including every required engine for `createSanitizer`.
|
|
98
|
+
|
|
99
|
+
Name components, unique initial-plus-surname forms, existing transliteration
|
|
100
|
+
aliases and selected Russian `-ов`/`-ев` inflections share an entity token.
|
|
101
|
+
Explicit `knownPersonal` and reused map entries also match bounded Russian noun,
|
|
102
|
+
adjective, feminine surname and given-name case endings, two-part person names
|
|
103
|
+
in reverse order, and organization cores without legal forms/quotes. Inflected
|
|
104
|
+
forms consume whole Unicode words; minimum stem lengths limit overmatching.
|
|
105
|
+
This is conservative heuristic identity linking, not general coreference:
|
|
106
|
+
ambiguous surnames/initials and unrelated people are not intentionally merged.
|
|
107
|
+
Every replaced occurrence retains its exact original surface privately. With
|
|
108
|
+
one identical token, a derived answer cannot encode the original grammatical
|
|
109
|
+
case; it receives the canonical spelling. Two distinct originals that produce
|
|
110
|
+
the same entire sanitized document cannot both be distinguished by that text
|
|
111
|
+
and a shared map; such conflicting occurrence metadata is refused. Choose
|
|
112
|
+
separate maps for those documents if exact independent restoration is needed.
|
|
113
|
+
|
|
114
|
+
JSON/JSONL keys and values participate equally in placeholder mode, including
|
|
115
|
+
ordinary identifier keys and declared structural fields. Parseability and
|
|
116
|
+
duplicate-key checks remain enforced. Plain redact/fake policies retain their
|
|
117
|
+
existing metadata/key behavior. Use `structured` for valid structured input.
|
|
118
|
+
The normal encoding limits and credential redaction policy continue to apply.
|
|
119
|
+
|
|
120
|
+
CLI: `redact FILE --placeholder --map private.json` creates or continues a map;
|
|
121
|
+
`--no-map` explicitly discards it. A CLI placeholder operation requires one of
|
|
122
|
+
these choices, to avoid losing a map silently. `--placeholder-format`,
|
|
123
|
+
`--map-format json|lino` and `--map-credentials` configure the transformation.
|
|
124
|
+
`restore safe.txt --map private.lino [--output restored.txt]` also works on a
|
|
125
|
+
derived model answer. Input, output and map paths must be separate. Maps are
|
|
126
|
+
saved separately before stdout publication. `--profile documents` enables the
|
|
127
|
+
new document recognizers; `documentEntities: false` turns them off explicitly.
|
|
128
|
+
|
|
129
|
+
| Option | Default / meaning |
|
|
130
|
+
| ------------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
|
131
|
+
| `documentEntities` | Off; bounded organization, education and location rules; enabled by the documents profile |
|
|
132
|
+
| `profile` | Off; `'publication'` sets a 0.3 threshold and identity masking; `'documents'` enables document entities |
|
|
133
|
+
| `knownOnly` | `false`; disable all automatic detection (including credentials, Secretlint and custom detectors); use only explicit known values and reused map identities |
|
|
134
|
+
| `knownSecrets` | `[]`; exact literal credential values, regardless of length or format |
|
|
135
|
+
| `knownPersonal` | `[]`; `{type, value}` values with whole-word Unicode case-insensitive matching and bounded Russian case forms |
|
|
136
|
+
| `publicEntities` | `[]`; exact PERSON/ORGANIZATION/EDUCATION/LOCATION/EMAIL exception with HTTPS evidence and review date |
|
|
137
|
+
| `findings` | `[]`; validated detections for the original input, supplied by the caller |
|
|
138
|
+
| `detectors` | `[]`; additional required callbacks in the asynchronous engine |
|
|
139
|
+
| `secretlint` | `true` in the asynchronous engine |
|
|
140
|
+
| `decode` | `true`; bounded base64/base64url, hex and percent decoding |
|
|
141
|
+
| `paranoid` | `false`; entropy-based supplemental credential heuristic |
|
|
142
|
+
| `maxInputLength` | 33,554,432 UTF-16 units |
|
|
143
|
+
| `maxFindings` | 100,000 detector hits before deduplication |
|
|
144
|
+
| `minConfidence` | 0.5; personal findings below this score remain visible to inspect but are not replaced |
|
|
145
|
+
| `threshold` | Alias for minConfidence; both options must agree when supplied |
|
|
146
|
+
| `debug` | Optional trusted callback for value-free counts, timing and event metadata |
|
|
147
|
+
| `identityMask` | `false`; identity types keep 2+2 Unicode characters with fixed `***`, or redact below five |
|
|
148
|
+
| `fakeIdentity` | Off; `'specimen-and-synthetic'` retains explicitly fake identity values |
|
|
149
|
+
| `fakeValues` | `[]`; exact caller-reviewed identity values to retain, regardless of fakeIdentity |
|
|
150
|
+
| `structured` | Off; `'json'` or `'jsonl'` validates and sanitizes string keys/values separately |
|
|
151
|
+
| `structuralFields` | `[]`; declared property names suppress weak name/ID/entropy heuristics in their string values |
|
|
43
152
|
|
|
44
153
|
Unknown options and invalid limits are errors. `knownSecretsFromEnv(env, names?)` discovers credential-named variables (excluding auth configuration names) when names are omitted, or uses an explicit selection, deduplicating nonempty values. It never invokes shell commands or reads auth files. `personalVariants(entries)` generates numeric and ISO/day-first date spellings; name case variants are already handled by detection. Neither helper invents transliterations.
|
|
45
154
|
|
package/docs/COVERAGE.md
CHANGED
|
@@ -14,6 +14,56 @@ Broader unquoted entropy scanning is opt-in (`paranoid: true`), uses Shannon ent
|
|
|
14
14
|
|
|
15
15
|
## Personal data and languages
|
|
16
16
|
|
|
17
|
+
### Document entities
|
|
18
|
+
|
|
19
|
+
`profile: 'documents'` (or `documentEntities: true`) opts into the following
|
|
20
|
+
additional rules. They are off under the existing default and publication
|
|
21
|
+
profiles, preserving credential/log fixture behavior and avoiding extra company
|
|
22
|
+
name redactions in stack traces. Existing contextual organization detection
|
|
23
|
+
remains available under those profiles.
|
|
24
|
+
|
|
25
|
+
| Type | Coverage | Confidence |
|
|
26
|
+
| -------------- | ------------------------------------------------------------------------------------------------------------------------------------- | ---------- |
|
|
27
|
+
| `ORGANIZATION` | Proper names next to legal forms in either order, optionally quoted: Inc, LLC, Ltd, GmbH, AG, S.A., ООО, АО, ПАО, ИП and locale forms | 0.70 |
|
|
28
|
+
| `EDUCATION` | Proper names next to University, College, Institute, school and locale equivalents, in either order | 0.65 |
|
|
29
|
+
| `LOCATION` | Exact city/region entries from a bounded offline vocabulary | 0.60 |
|
|
30
|
+
| `LOCATION` | Bounded proper names after location labels, including based in, located in, relocated from, г., город and locale equivalents | 0.75 |
|
|
31
|
+
|
|
32
|
+
Russian document rules also accept legal-form quoted names with lowercase words,
|
|
33
|
+
digits and hyphens (up to 160 characters), organization Group/Групп/Holding/Холдинг
|
|
34
|
+
markers, adjectival institution names with lowercase connectors and optional
|
|
35
|
+
`имени` tails, education acronyms in context, residence labels and city/country
|
|
36
|
+
phrasing. A known given name after `Меня зовут` or `С уважением` scores 0.70.
|
|
37
|
+
Detected organization cores propagate within a document and across confirmed
|
|
38
|
+
stream records. Russian regional cities include Ярославль and Кострома; the
|
|
39
|
+
vocabulary remains finite. Default profiles do not enable these document rules.
|
|
40
|
+
|
|
41
|
+
These confidence values indicate heuristic strength, not calibrated
|
|
42
|
+
probabilities. `minConfidence` still controls replacement. Findings retain
|
|
43
|
+
their confidence even below threshold. `publicEntities` supports exact reviewed
|
|
44
|
+
exceptions for all three types; `knownPersonal` always forces removal of a
|
|
45
|
+
listed private value. All types support placeholder mode; fake mode uses stable,
|
|
46
|
+
visibly synthetic organization/school/location labels.
|
|
47
|
+
|
|
48
|
+
`src/document-vocabulary.js` is the complete authored vocabulary: English,
|
|
49
|
+
Spanish, French, German, Portuguese, Russian, Japanese, Simplified/Traditional
|
|
50
|
+
Chinese, Korean, Arabic, Hebrew, Hindi, Turkish, Indonesian, Vietnamese and
|
|
51
|
+
Thai each have list-form employer, school and city fixtures. Proper-name
|
|
52
|
+
rules are finite (up to six capitalized words, or four words in scripts without
|
|
53
|
+
case); they do not implement grammatical parsing. Other lowercase connectors,
|
|
54
|
+
unusual typography, unsupported legal forms and unfamiliar institutions can be
|
|
55
|
+
missed. Legal-form words and uncased scripts can also yield false positives.
|
|
56
|
+
|
|
57
|
+
The gazetteer has only the cities/regions explicitly listed in that module,
|
|
58
|
+
including Springfield and Riverton. It is not a worldwide geocoder. Location
|
|
59
|
+
labels can find unlisted proper names; unlabelled places outside the vocabulary
|
|
60
|
+
need `knownPersonal` or an additional detector such as the Presidio bridge.
|
|
61
|
+
Name linking is also heuristic, especially for shared surnames, transliterations
|
|
62
|
+
and inflected forms. Employers, schools, locations, dates and other
|
|
63
|
+
quasi-identifiers can re-identify a person when detection misses them. Neither
|
|
64
|
+
placeholder mode nor no-map mode promises universal anonymity. A restore map
|
|
65
|
+
contains personal data and must remain private; see [API.md](API.md#reversible-placeholders).
|
|
66
|
+
|
|
17
67
|
Native rules cover international/Unicode emails, international and selected national-format phones, Luhn-valid payment cards, US SSNs, IPv4/IPv6, checksum-validated IBAN/CPF/Spanish DNI/US NPI, Indian PAN, UK NI, wallet shapes, MACs, messenger handles, home usernames, labelled PII and selected postal address grammars. A bundled 17-locale name gazetteer supplies default free-text detection, including selected inflections/transliterations. Each locale has credential prose and eight labelled PII fixtures. This finite dictionary/grammar is not unrestricted statistical NER; unfamiliar names and unsupported national formats require known values or additional recognizers.
|
|
18
68
|
|
|
19
69
|
Credential labels have fixtures for 17 language variants:
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import {
|
|
2
|
+
sanitize,
|
|
3
|
+
restore,
|
|
4
|
+
createRestoreMap,
|
|
5
|
+
serializeRestoreMap,
|
|
6
|
+
} from '../src/index.js';
|
|
7
|
+
|
|
8
|
+
// Keep this map private. Publish only result.text, never the map or result.
|
|
9
|
+
const map = createRestoreMap();
|
|
10
|
+
const options = {
|
|
11
|
+
profile: 'documents',
|
|
12
|
+
transformation: { mode: 'placeholder', map },
|
|
13
|
+
};
|
|
14
|
+
const text = 'Jane Doe worked at Acme Ltd in Springfield.';
|
|
15
|
+
const result = sanitize(text, options);
|
|
16
|
+
console.log(result.text);
|
|
17
|
+
console.log('Exact private round trip:', restore(result.text, map) === text);
|
|
18
|
+
console.log(
|
|
19
|
+
'Derived answer restored privately:',
|
|
20
|
+
restore('Ask Person-1 about Organization-1.', map)
|
|
21
|
+
);
|
|
22
|
+
console.log(
|
|
23
|
+
'Private map serialization available:',
|
|
24
|
+
typeof serializeRestoreMap(map, 'lino') === 'string'
|
|
25
|
+
);
|
|
26
|
+
const anonymous = sanitize(text, {
|
|
27
|
+
...options,
|
|
28
|
+
transformation: { mode: 'placeholder', map: false },
|
|
29
|
+
});
|
|
30
|
+
console.log(
|
|
31
|
+
'No-map output:',
|
|
32
|
+
anonymous.text,
|
|
33
|
+
'Map returned:',
|
|
34
|
+
Object.hasOwn(anonymous, 'map')
|
|
35
|
+
);
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@link-foundation/sensitive-data-sanitizer",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.14.0",
|
|
4
4
|
"description": "Offline sensitive-text sanitization with credential, multilingual personal-data and scanner union detection",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "src/index.js",
|
|
@@ -23,7 +23,8 @@
|
|
|
23
23
|
"CHANGELOG.md",
|
|
24
24
|
"LICENSE",
|
|
25
25
|
"README.md",
|
|
26
|
-
"examples/shared-tool-logs.mjs"
|
|
26
|
+
"examples/shared-tool-logs.mjs",
|
|
27
|
+
"examples/reversible-placeholders.mjs"
|
|
27
28
|
],
|
|
28
29
|
"exports": {
|
|
29
30
|
".": {
|
|
@@ -119,6 +120,7 @@
|
|
|
119
120
|
"@secretlint/profiler": "13.0.7",
|
|
120
121
|
"@secretlint/secretlint-rule-preset-recommend": "13.0.7",
|
|
121
122
|
"libphonenumber-js": "1.13.14",
|
|
123
|
+
"links-notation": "0.30.0",
|
|
122
124
|
"re2-wasm": "1.0.2",
|
|
123
125
|
"re2js": "^2.9.0",
|
|
124
126
|
"yaml": "^2.9.1"
|