@link-foundation/sensitive-data-sanitizer 0.12.2 → 0.14.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGELOG.md CHANGED
@@ -1,5 +1,17 @@
1
1
  # Changelog
2
2
 
3
+ ## 0.14.0
4
+
5
+ ### Minor Changes
6
+
7
+ - d5b2aac: Add known-only reversible sanitization, Unicode known-value boundaries and bounded Russian entity case matching. Correct placeholder namespace collisions, Russian document entities and fake-name roles, and credential prose false positives. Report irreversible replacements in placeholder results.
8
+
9
+ ## 0.13.0
10
+
11
+ ### Minor Changes
12
+
13
+ - 818f198: Add reversible typed placeholders with private JSON and Links Notation maps, map reuse and restoration, a no-map anonymization mode, and opt-in multilingual document entity detection for employers, schools, cities and regions.
14
+
3
15
  ## 0.12.2
4
16
 
5
17
  ### Patch Changes
package/README.md CHANGED
@@ -22,6 +22,50 @@ sensitive-data-sanitizer redact session.jsonl --stream --jsonl --two-pass --outp
22
22
 
23
23
  ## Quick Start
24
24
 
25
+ ### Readable placeholders and document profiles
26
+
27
+ Use the opt-in `documents` profile for CVs and similar prose. It adds bounded
28
+ employer, school, city and region detection across the existing 17 locales.
29
+ Default and publication log profiles keep their existing detection behavior.
30
+
31
+ ```js
32
+ import { sanitize, restore } from '@link-foundation/sensitive-data-sanitizer';
33
+
34
+ const result = sanitize('Jane Doe worked at Acme Ltd in Springfield.', {
35
+ profile: 'documents',
36
+ transformation: { mode: 'placeholder' },
37
+ });
38
+ // result.text: Person-1 worked at Organization-1 in Location-1.
39
+ // Keep result.map private. Share only result.text.
40
+ restore(result.text, result.map); // exact original document
41
+ restore('Ask Person-1 about Organization-1.', result.map); // derived answer
42
+ ```
43
+
44
+ The restore map **is personal data**. It contains original spellings and private
45
+ occurrence metadata; never commit it, send it to a model, or publish the whole
46
+ result object. The CLI writes maps separately with mode `0600`, and publication
47
+ checks reject serialized restore maps. Reuse `result.map` in the next
48
+ `transformation.map` to keep IDs stable across files. JSON and Links Notation
49
+ serializations carry the same versioned map. Credentials stay fully redacted
50
+ and out of the map unless explicitly opted in.
51
+
52
+ ```sh
53
+ sensitive-data-sanitizer redact cv.txt --profile documents --placeholder --map private.lino --output safe.txt
54
+ sensitive-data-sanitizer restore safe.txt --map private.lino --output restored.txt
55
+ sensitive-data-sanitizer redact cv.txt --profile documents --placeholder --no-map
56
+ ```
57
+
58
+ `knownOnly: true` (CLI `--known-only`) limits replacement to `knownPersonal`,
59
+ `knownSecrets` and reused map identities, preserving fictional examples.
60
+ Placeholder results report `irreversibleRedactions`; a nonzero count means some
61
+ replacements, such as credentials excluded from the map, cannot be restored.
62
+
63
+ For irreversible output, choose `transformation: { mode: 'placeholder', map:
64
+ false }`: no restore map is returned or written. Detection limits still apply
65
+ in either mode. See the [placeholder API](docs/API.md#reversible-placeholders),
66
+ [coverage](docs/COVERAGE.md#document-entities), and
67
+ [runnable example](examples/reversible-placeholders.mjs).
68
+
25
69
  Sanitizer APIs require Node.js 22.13 or later. Template arithmetic utilities are available under `@link-foundation/sensitive-data-sanitizer/legacy`; the existing universal example uses that browser-compatible entry. From this checkout:
26
70
 
27
71
  ```sh
@@ -0,0 +1,115 @@
1
+ import { resolve } from 'node:path';
2
+ import { failure } from '../src/detection.js';
3
+ import {
4
+ createRestoreMap,
5
+ readRestoreMap,
6
+ writeRestoreMap,
7
+ restore,
8
+ } from '../src/restore-map.js';
9
+ import { readTextFile, atomicWrite } from '../src/files.js';
10
+
11
+ export async function placeholderOptions(config, options) {
12
+ if (config.command === 'restore') {
13
+ return;
14
+ }
15
+ if (config.placeholder) {
16
+ options.transformation = { mode: 'placeholder' };
17
+ }
18
+ const transform = options.transformation;
19
+ validatePlaceholderFlags(config, transform);
20
+ if (transform?.mode !== 'placeholder') {
21
+ return;
22
+ }
23
+ validateMapArgs(config);
24
+ if (config.placeholderFormat) {
25
+ transform.format = config.placeholderFormat;
26
+ }
27
+ if (config.mapCredentials) {
28
+ transform.credentials = true;
29
+ }
30
+ if (config.noMap) {
31
+ transform.map = false;
32
+ }
33
+ if (config.map) {
34
+ try {
35
+ const { lstat } = await import('node:fs/promises');
36
+ await lstat(config.map);
37
+ transform.map = await readRestoreMap(config.map);
38
+ config.replaceMap = true;
39
+ } catch (error) {
40
+ if (error.code !== 'ENOENT') {
41
+ throw error;
42
+ }
43
+ transform.map = createRestoreMap({ format: transform.format });
44
+ }
45
+ } else if (transform.map !== false) {
46
+ // The CLI cannot return a private map alongside stdout safely.
47
+ throw failure('ERR_ARGUMENT');
48
+ }
49
+ config.restoreMap = transform.map;
50
+ }
51
+ function validatePlaceholderFlags(config, transform) {
52
+ if (
53
+ (config.map ||
54
+ config.noMap ||
55
+ config.placeholderFormat ||
56
+ config.mapCredentials ||
57
+ config.mapFormat) &&
58
+ transform?.mode !== 'placeholder'
59
+ ) {
60
+ throw failure('ERR_ARGUMENT');
61
+ }
62
+ }
63
+ function validateMapArgs(config) {
64
+ if (config.map && config.noMap) {
65
+ throw failure('ERR_ARGUMENT');
66
+ }
67
+ if (config.mapFormat && !['json', 'lino'].includes(config.mapFormat)) {
68
+ throw failure('ERR_ARGUMENT');
69
+ }
70
+ if (
71
+ config.map &&
72
+ [config.path, config.output, config.config]
73
+ .filter(Boolean)
74
+ .some((path) => resolve(path) === resolve(config.map))
75
+ ) {
76
+ throw failure('ERR_OUTPUT');
77
+ }
78
+ }
79
+
80
+ export async function saveMap(config) {
81
+ if (config.map) {
82
+ await writeRestoreMap(config.map, config.restoreMap, {
83
+ replace: Boolean(config.replaceMap),
84
+ ...(config.mapFormat ? { format: config.mapFormat } : {}),
85
+ });
86
+ }
87
+ }
88
+ export async function restoreCommand(config, { stdout }) {
89
+ if (
90
+ !config.map ||
91
+ config.path === '-' ||
92
+ config.stream ||
93
+ config.noMap ||
94
+ config.placeholder ||
95
+ config.mapCredentials ||
96
+ config.json ||
97
+ config.jsonl ||
98
+ config.inPlace
99
+ ) {
100
+ throw failure('ERR_ARGUMENT');
101
+ }
102
+ if (config.output && resolve(config.output) === resolve(config.map)) {
103
+ throw failure('ERR_OUTPUT');
104
+ }
105
+ const text = restore(
106
+ await readTextFile(config.path, config.maxBytes),
107
+ await readRestoreMap(config.map)
108
+ );
109
+ if (config.output) {
110
+ await atomicWrite(config.output, text);
111
+ } else {
112
+ stdout.write(text);
113
+ }
114
+ return 0;
115
+ }
@@ -21,16 +21,18 @@ import { encodeLogText } from '../src/log-bytes.js';
21
21
  import { createWikidataVerifier } from '../src/public-verifier.js';
22
22
  import { streamCommand } from './stream.js';
23
23
  import { sessionStructuralFields } from '../src/structured.js';
24
+ import { placeholderOptions, saveMap, restoreCommand } from './placeholders.js';
24
25
  import {
25
26
  createGitleaksDetector,
26
27
  createTrufflehogDetector,
27
28
  createPresidioDetector,
28
29
  } from '../src/external.js';
29
30
 
30
- const help = `Usage: sensitive-data-sanitizer <redact|scan|history> [path|-] [options]
31
+ const help = `Usage: sensitive-data-sanitizer <redact|scan|restore|history> [path|-] [options]
31
32
 
32
33
  redact Sanitize one text log or stdin; publish only after verification
33
34
  scan Report metadata for a file, directory, or stdin (never source text)
35
+ restore Restore placeholders in a file using --map PATH (private output)
34
36
  history Audit local reachable Git blobs, commit/tag metadata, and reflogs
35
37
  history rewrite SOURCE --output DIRECTORY [--apply]
36
38
  Preview a fresh mirror clone; --apply rewrites only that clone
@@ -40,6 +42,7 @@ history rewrite SOURCE --output DIRECTORY [--apply]
40
42
  --in-place Atomically replace the input file (redact only)
41
43
  --native-only Disable Secretlint; selected local detectors remain required
42
44
  --credentials-only Preserve personal/account details; redact credentials only
45
+ --known-only Replace only knownPersonal/knownSecrets and reused map values
43
46
  --env-secrets Include values of all credential-named environment variables
44
47
  --gh-config Include tokens from the local gh hosts.yml credential file
45
48
  --paranoid Add entropy detection (can redact innocent identifiers)
@@ -56,7 +59,13 @@ history rewrite SOURCE --output DIRECTORY [--apply]
56
59
  --json Sanitize JSON strings and keys; preserve session metadata
57
60
  --jsonl Sanitize JSONL records; combine with --stream for sessions
58
61
  --identity-mask First/last 2 identity mask (publication profile default)
59
- --profile NAME Publication profile (redact default; config can override)
62
+ --profile NAME publication (redact default) or documents (CV/entity rules)
63
+ --placeholder Typed numbered placeholders; requires --map or --no-map
64
+ --map PATH Private JSON/.lino restore map; reuse it across files/runs
65
+ --no-map Discard placeholder mapping after full anonymization
66
+ --placeholder-format TEMPLATE Default {Type}-{n}; also [{TYPE}_{n}]
67
+ --map-format FORMAT json or lino (otherwise inferred from .lino extension)
68
+ --map-credentials Explicitly keep credentials in the private restore map
60
69
  --max-record-bytes N Maximum record bytes (text: total limit; JSONL: 8388608)
61
70
  --hive-mask Opt-in first/last 3 mask for values longer than 12 characters
62
71
  --preserve-encoding Re-encode sanitized encoded payloads with round-trip checks
@@ -80,6 +89,7 @@ function parseArgs(argv) {
80
89
  ['--in-place', 'inPlace'],
81
90
  ['--native-only', 'nativeOnly'],
82
91
  ['--credentials-only', 'credentialsOnly'],
92
+ ['--known-only', 'knownOnly'],
83
93
  ['--env-secrets', 'envSecrets'],
84
94
  ['--gh-config', 'ghConfig'],
85
95
  ['--paranoid', 'paranoid'],
@@ -96,6 +106,9 @@ function parseArgs(argv) {
96
106
  ['--gh-auth', 'ghAuth'],
97
107
  ['--verify-public', 'verifyPublic'],
98
108
  ['--apply', 'apply'],
109
+ ['--placeholder', 'placeholder'],
110
+ ['--no-map', 'noMap'],
111
+ ['--map-credentials', 'mapCredentials'],
99
112
  ]);
100
113
  const valued = new Map([
101
114
  ['--config', 'config'],
@@ -109,6 +122,9 @@ function parseArgs(argv) {
109
122
  ['--worker-timeout-ms', 'workerTimeoutMs'],
110
123
  ['--filter-repo', 'filterRepo'],
111
124
  ['--profile', 'profile'],
125
+ ['--map', 'map'],
126
+ ['--map-format', 'mapFormat'],
127
+ ['--placeholder-format', 'placeholderFormat'],
112
128
  ]);
113
129
  let hasPath = false;
114
130
  for (let i = rewrite ? 2 : 1; i < argv.length; i++) {
@@ -134,7 +150,7 @@ function parseArgs(argv) {
134
150
  function validateArgs(config) {
135
151
  config.maxBytes = Number(config.maxBytes ?? defaultByteLimit(config));
136
152
  if (
137
- !['redact', 'scan', 'history'].includes(config.command) ||
153
+ !['redact', 'scan', 'restore', 'history'].includes(config.command) ||
138
154
  !Number.isSafeInteger(config.maxBytes) ||
139
155
  config.maxBytes <= 0
140
156
  ) {
@@ -143,7 +159,7 @@ function validateArgs(config) {
143
159
  if (
144
160
  (config.output && config.inPlace) ||
145
161
  ((config.output || config.inPlace) &&
146
- config.command !== 'redact' &&
162
+ !['redact', 'restore'].includes(config.command) &&
147
163
  !config.rewrite) ||
148
164
  (config.inPlace && config.path === '-') ||
149
165
  (config.command === 'history' && config.path === '-')
@@ -252,19 +268,8 @@ export async function runCli(
252
268
  return 0;
253
269
  }
254
270
  const config = parseArgs(argv);
255
- const options = await loadOptions(config);
256
- const engine = createSanitizer(options);
257
271
  const context = { stdin, stdout };
258
- if (config.command === 'history') {
259
- return await historyCommand(config, engine, context);
260
- }
261
- if (config.stream) {
262
- return await streamCommand(config, engine, options, context);
263
- }
264
- if (config.command === 'redact') {
265
- return await redactCommand(config, engine, context);
266
- }
267
- return await scanCommand(config, engine, context);
272
+ return await dispatchCommand(config, context);
268
273
  } catch (error) {
269
274
  const location =
270
275
  Number.isSafeInteger(error.record) && Number.isSafeInteger(error.offset)
@@ -277,6 +282,27 @@ export async function runCli(
277
282
  }
278
283
  }
279
284
 
285
+ async function dispatchCommand(config, context) {
286
+ if (config.command === 'restore') {
287
+ return await restoreCommand(config, context);
288
+ }
289
+ const options = await loadOptions(config);
290
+ const engine = createSanitizer(options);
291
+ if (config.command === 'history') {
292
+ return await historyCommand(config, engine, context);
293
+ }
294
+ if (config.stream) {
295
+ return await streamCommand(config, engine, options, {
296
+ ...context,
297
+ beforePublish: () => saveMap(config),
298
+ });
299
+ }
300
+ if (config.command === 'redact') {
301
+ return await redactCommand(config, engine, context);
302
+ }
303
+ return await scanCommand(config, engine, context);
304
+ }
305
+
280
306
  async function loadOptions(config) {
281
307
  let options = {};
282
308
  if (config.config) {
@@ -300,6 +326,7 @@ async function loadOptions(config) {
300
326
  options.paranoid = true;
301
327
  }
302
328
  await featureOptions(config, options);
329
+ await placeholderOptions(config, options);
303
330
  options.detectors = selectedDetectors(config, options.detectors);
304
331
  options.maxInputLength = Math.min(
305
332
  options.maxInputLength ?? config.maxBytes,
@@ -314,6 +341,9 @@ async function loadOptions(config) {
314
341
 
315
342
  async function featureOptions(config, options) {
316
343
  await localCredentials(config, options);
344
+ if (config.knownOnly) {
345
+ options.knownOnly = true;
346
+ }
317
347
  if (config.credentialsOnly) {
318
348
  options.credentialsOnly = true;
319
349
  }
@@ -391,6 +421,7 @@ async function redactCommand(config, engine, { stdin, stdout }) {
391
421
  const result = await engine.sanitize(
392
422
  await inputText(config.path, config, stdin)
393
423
  );
424
+ await saveMap(config);
394
425
  if (config.output || config.inPlace) {
395
426
  await atomicWrite(
396
427
  config.output ?? config.path,
package/bin/stream.js CHANGED
@@ -9,7 +9,7 @@ export async function streamCommand(
9
9
  config,
10
10
  engine,
11
11
  options,
12
- { stdin, stdout }
12
+ { stdin, stdout, beforePublish }
13
13
  ) {
14
14
  let spool;
15
15
  try {
@@ -42,6 +42,7 @@ export async function streamCommand(
42
42
  : { sanitizer: engine, sanitizerOptions: options }),
43
43
  });
44
44
  }
45
+ await beforePublish?.();
45
46
  if (spool) {
46
47
  for await (const chunk of createReadStream(target)) {
47
48
  if (stdout.write(chunk) === false) {
package/docs/API.md CHANGED
@@ -19,27 +19,136 @@ Import the ESM root entry; complete declarations are in `src/index.d.ts`. Inputs
19
19
 
20
20
  ## Options
21
21
 
22
- | Option | Default / meaning |
23
- | ------------------ | --------------------------------------------------------------------------------------------- |
24
- | `profile` | Off; `'publication'` sets a 0.3 personal threshold, identity masking and strict fake policy |
25
- | `knownSecrets` | `[]`; exact literal credential values, regardless of length or format |
26
- | `knownPersonal` | `[]`; `{type, value}` literals with Unicode case-insensitive matching |
27
- | `publicEntities` | `[]`; exact PERSON/ORGANIZATION/EMAIL exception with HTTPS source and review date |
28
- | `findings` | `[]`; validated detections for the original input, supplied by the caller |
29
- | `detectors` | `[]`; additional required callbacks in the asynchronous engine |
30
- | `secretlint` | `true` in the asynchronous engine |
31
- | `decode` | `true`; bounded base64/base64url, hex and percent decoding |
32
- | `paranoid` | `false`; entropy-based supplemental credential heuristic |
33
- | `maxInputLength` | 33,554,432 UTF-16 units |
34
- | `maxFindings` | 100,000 detector hits before deduplication |
35
- | `minConfidence` | 0.5; personal findings below this score remain visible to inspect but are not replaced |
36
- | `threshold` | Alias for minConfidence; both options must agree when supplied |
37
- | `debug` | Optional trusted callback for value-free counts, timing and event metadata |
38
- | `identityMask` | `false`; identity types keep 2+2 Unicode characters with fixed `***`, or redact below five |
39
- | `fakeIdentity` | Off; `'specimen-and-synthetic'` retains explicitly fake identity values |
40
- | `fakeValues` | `[]`; exact caller-reviewed identity values to retain, regardless of fakeIdentity |
41
- | `structured` | Off; `'json'` or `'jsonl'` validates and sanitizes string keys/values separately |
42
- | `structuralFields` | `[]`; declared property names suppress weak name/ID/entropy heuristics in their string values |
22
+ ### Replace only known values
23
+
24
+ Use `knownOnly: true` to preserve fictional examples while replacing selected
25
+ private values. This applies to sync/async, JSON/JSONL, streams and fake mode.
26
+ It skips automatic credential detection too; supply `knownSecrets` explicitly
27
+ for credentials that must be removed. `credentialsOnly` can further exclude
28
+ personal values. CLI: `--known-only` with a `--config` containing the known lists.
29
+
30
+ ```js
31
+ const result = sanitize('Анна Смирнова; example Jane Doe', {
32
+ knownOnly: true,
33
+ knownPersonal: [{ type: 'PERSON', value: 'Анна Смирнова' }],
34
+ transformation: { mode: 'placeholder' },
35
+ });
36
+ // result.text: 'Person-1; example Jane Doe'
37
+ restore(result.text, result.map); // exact original
38
+ ```
39
+
40
+ ### Reversible placeholders
41
+
42
+ `transformation: {mode: 'placeholder'}` emits typed tokens such as `Person-1`,
43
+ `Email-1`, `Phone-1`, `Organization-1`, `Education-1`, `Location-1` and `Date-1`.
44
+ Numbers start at one per type and follow the first occurrence in the input.
45
+ `DATE_OF_BIRTH` uses the `Date` label; other detector types retain their type
46
+ name in title case. Per-type `transformations` also support placeholder mode;
47
+ all placeholder transformations in an operation must share one map and format.
48
+
49
+ The result has a private `map` alongside `text`, `findings` and `redactions`.
50
+ The map holds token entries, every original surface form, name aliases, and
51
+ replacement positions indexed by the SHA-256 digest of the sanitized document.
52
+ It does not store the original full document. `restore(text, map)` restores
53
+ the exact bytes represented by the original JavaScript string, including JSON
54
+ escaping and whitespace, when the document digest matches. A derived response
55
+ that quotes tokens uses the entry's canonical original spelling instead.
56
+ Restoration uses a single pass so restored originals are never interpreted as
57
+ more tokens. Unknown tokens and larger words containing token-like fragments
58
+ are left alone.
59
+
60
+ `format` is a template with exactly one type field (`{Type}`, `{TYPE}` or
61
+ `{type}`) and one `{n}` field. Examples: `{Type}-{n}`, `[{TYPE}_{n}]`,
62
+ `{{{type}_{n}}}`. Sanitization refuses input containing any token in the chosen
63
+ namespace for built-in and supplied type labels, including numbers not allocated yet.
64
+ Unicode letters, marks, digits and underscores bound those tokens. Ordinary
65
+ technical names such as GPT-4, ISO-9001 and Windows-10 are not reserved. The map's format cannot change
66
+ on reuse. A malformed map or ambiguous exact restoration blocks output.
67
+
68
+ `createRestoreMap({format?})` creates an empty reusable map. Pass a map as
69
+ `transformation.map` to continue a corpus; the operation updates that same
70
+ object. A `createSanitizer()` engine also keeps its registry across calls.
71
+ Callers must serialize access to a shared engine/map; parallel independent
72
+ registries cannot assign consistent first-appearance numbers. For streams,
73
+ pass an explicit map object and keep it outside the output stream. Placeholder
74
+ JSONL uses the coordinator in source order, even if several workers were
75
+ requested. File APIs also keep this registry in the caller process.
76
+
77
+ `serializeRestoreMap(map, 'json' | 'lino')` and `parseRestoreMap(text)` round
78
+ trip the same schema. Lino uses the upstream `links-notation` parser and
79
+ formatter: a named `sensitive-data-restore-map` link carries the escaped JSON
80
+ schema as a reference, preserving all metadata and unusual original strings.
81
+ `readRestoreMap(path)` and `writeRestoreMap(path, map, {format?, replace?})`
82
+ use bounded file reads and the existing atomic `0600` writer. The default
83
+ serialization is JSON, or Lino for a `.lino` output path. Existing files require
84
+ `replace: true`; symlinks are refused. `inspect`, `sanitize`, scanner union and
85
+ CLI publication paths reject marked map content with `ERR_PRIVATE_MAP`.
86
+ Keep maps out of Git, logs, model requests and publication directories. The map
87
+ is plaintext personal data, not encrypted storage; protect it as the original.
88
+
89
+ `credentials: true` explicitly opts credentials into `Credential-n` entries.
90
+ Otherwise they remain `[REDACTED]` and cannot be recovered, including through
91
+ occurrence metadata. Placeholder results include `irreversibleRedactions`, the
92
+ number of replacements excluded from the restore map (including mixed policies).
93
+ Check that this is zero before relying on exact restoration. A positive count
94
+ means `restore()` cannot reconstruct all of the input. `map: false` returns only the anonymized text and ordinary
95
+ value-free finding metadata. The temporary numbering registry is discarded
96
+ after an operation (or the complete stream); no map file is created. Residual verification
97
+ still runs, including every required engine for `createSanitizer`.
98
+
99
+ Name components, unique initial-plus-surname forms, existing transliteration
100
+ aliases and selected Russian `-ов`/`-ев` inflections share an entity token.
101
+ Explicit `knownPersonal` and reused map entries also match bounded Russian noun,
102
+ adjective, feminine surname and given-name case endings, two-part person names
103
+ in reverse order, and organization cores without legal forms/quotes. Inflected
104
+ forms consume whole Unicode words; minimum stem lengths limit overmatching.
105
+ This is conservative heuristic identity linking, not general coreference:
106
+ ambiguous surnames/initials and unrelated people are not intentionally merged.
107
+ Every replaced occurrence retains its exact original surface privately. With
108
+ one identical token, a derived answer cannot encode the original grammatical
109
+ case; it receives the canonical spelling. Two distinct originals that produce
110
+ the same entire sanitized document cannot both be distinguished by that text
111
+ and a shared map; such conflicting occurrence metadata is refused. Choose
112
+ separate maps for those documents if exact independent restoration is needed.
113
+
114
+ JSON/JSONL keys and values participate equally in placeholder mode, including
115
+ ordinary identifier keys and declared structural fields. Parseability and
116
+ duplicate-key checks remain enforced. Plain redact/fake policies retain their
117
+ existing metadata/key behavior. Use `structured` for valid structured input.
118
+ The normal encoding limits and credential redaction policy continue to apply.
119
+
120
+ CLI: `redact FILE --placeholder --map private.json` creates or continues a map;
121
+ `--no-map` explicitly discards it. A CLI placeholder operation requires one of
122
+ these choices, to avoid losing a map silently. `--placeholder-format`,
123
+ `--map-format json|lino` and `--map-credentials` configure the transformation.
124
+ `restore safe.txt --map private.lino [--output restored.txt]` also works on a
125
+ derived model answer. Input, output and map paths must be separate. Maps are
126
+ saved separately before stdout publication. `--profile documents` enables the
127
+ new document recognizers; `documentEntities: false` turns them off explicitly.
128
+
129
+ | Option | Default / meaning |
130
+ | ------------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------- |
131
+ | `documentEntities` | Off; bounded organization, education and location rules; enabled by the documents profile |
132
+ | `profile` | Off; `'publication'` sets a 0.3 threshold and identity masking; `'documents'` enables document entities |
133
+ | `knownOnly` | `false`; disable all automatic detection (including credentials, Secretlint and custom detectors); use only explicit known values and reused map identities |
134
+ | `knownSecrets` | `[]`; exact literal credential values, regardless of length or format |
135
+ | `knownPersonal` | `[]`; `{type, value}` values with whole-word Unicode case-insensitive matching and bounded Russian case forms |
136
+ | `publicEntities` | `[]`; exact PERSON/ORGANIZATION/EDUCATION/LOCATION/EMAIL exception with HTTPS evidence and review date |
137
+ | `findings` | `[]`; validated detections for the original input, supplied by the caller |
138
+ | `detectors` | `[]`; additional required callbacks in the asynchronous engine |
139
+ | `secretlint` | `true` in the asynchronous engine |
140
+ | `decode` | `true`; bounded base64/base64url, hex and percent decoding |
141
+ | `paranoid` | `false`; entropy-based supplemental credential heuristic |
142
+ | `maxInputLength` | 33,554,432 UTF-16 units |
143
+ | `maxFindings` | 100,000 detector hits before deduplication |
144
+ | `minConfidence` | 0.5; personal findings below this score remain visible to inspect but are not replaced |
145
+ | `threshold` | Alias for minConfidence; both options must agree when supplied |
146
+ | `debug` | Optional trusted callback for value-free counts, timing and event metadata |
147
+ | `identityMask` | `false`; identity types keep 2+2 Unicode characters with fixed `***`, or redact below five |
148
+ | `fakeIdentity` | Off; `'specimen-and-synthetic'` retains explicitly fake identity values |
149
+ | `fakeValues` | `[]`; exact caller-reviewed identity values to retain, regardless of fakeIdentity |
150
+ | `structured` | Off; `'json'` or `'jsonl'` validates and sanitizes string keys/values separately |
151
+ | `structuralFields` | `[]`; declared property names suppress weak name/ID/entropy heuristics in their string values |
43
152
 
44
153
  Unknown options and invalid limits are errors. `knownSecretsFromEnv(env, names?)` discovers credential-named variables (excluding auth configuration names) when names are omitted, or uses an explicit selection, deduplicating nonempty values. It never invokes shell commands or reads auth files. `personalVariants(entries)` generates numeric and ISO/day-first date spellings; name case variants are already handled by detection. Neither helper invents transliterations.
45
154
 
package/docs/COVERAGE.md CHANGED
@@ -14,6 +14,56 @@ Broader unquoted entropy scanning is opt-in (`paranoid: true`), uses Shannon ent
14
14
 
15
15
  ## Personal data and languages
16
16
 
17
+ ### Document entities
18
+
19
+ `profile: 'documents'` (or `documentEntities: true`) opts into the following
20
+ additional rules. They are off under the existing default and publication
21
+ profiles, preserving credential/log fixture behavior and avoiding extra company
22
+ name redactions in stack traces. Existing contextual organization detection
23
+ remains available under those profiles.
24
+
25
+ | Type | Coverage | Confidence |
26
+ | -------------- | ------------------------------------------------------------------------------------------------------------------------------------- | ---------- |
27
+ | `ORGANIZATION` | Proper names next to legal forms in either order, optionally quoted: Inc, LLC, Ltd, GmbH, AG, S.A., ООО, АО, ПАО, ИП and locale forms | 0.70 |
28
+ | `EDUCATION` | Proper names next to University, College, Institute, school and locale equivalents, in either order | 0.65 |
29
+ | `LOCATION` | Exact city/region entries from a bounded offline vocabulary | 0.60 |
30
+ | `LOCATION` | Bounded proper names after location labels, including based in, located in, relocated from, г., город and locale equivalents | 0.75 |
31
+
32
+ Russian document rules also accept legal-form quoted names with lowercase words,
33
+ digits and hyphens (up to 160 characters), organization Group/Групп/Holding/Холдинг
34
+ markers, adjectival institution names with lowercase connectors and optional
35
+ `имени` tails, education acronyms in context, residence labels and city/country
36
+ phrasing. A known given name after `Меня зовут` or `С уважением` scores 0.70.
37
+ Detected organization cores propagate within a document and across confirmed
38
+ stream records. Russian regional cities include Ярославль and Кострома; the
39
+ vocabulary remains finite. Default profiles do not enable these document rules.
40
+
41
+ These confidence values indicate heuristic strength, not calibrated
42
+ probabilities. `minConfidence` still controls replacement. Findings retain
43
+ their confidence even below threshold. `publicEntities` supports exact reviewed
44
+ exceptions for all three types; `knownPersonal` always forces removal of a
45
+ listed private value. All types support placeholder mode; fake mode uses stable,
46
+ visibly synthetic organization/school/location labels.
47
+
48
+ `src/document-vocabulary.js` is the complete authored vocabulary: English,
49
+ Spanish, French, German, Portuguese, Russian, Japanese, Simplified/Traditional
50
+ Chinese, Korean, Arabic, Hebrew, Hindi, Turkish, Indonesian, Vietnamese and
51
+ Thai each have list-form employer, school and city fixtures. Proper-name
52
+ rules are finite (up to six capitalized words, or four words in scripts without
53
+ case); they do not implement grammatical parsing. Other lowercase connectors,
54
+ unusual typography, unsupported legal forms and unfamiliar institutions can be
55
+ missed. Legal-form words and uncased scripts can also yield false positives.
56
+
57
+ The gazetteer has only the cities/regions explicitly listed in that module,
58
+ including Springfield and Riverton. It is not a worldwide geocoder. Location
59
+ labels can find unlisted proper names; unlabelled places outside the vocabulary
60
+ need `knownPersonal` or an additional detector such as the Presidio bridge.
61
+ Name linking is also heuristic, especially for shared surnames, transliterations
62
+ and inflected forms. Employers, schools, locations, dates and other
63
+ quasi-identifiers can re-identify a person when detection misses them. Neither
64
+ placeholder mode nor no-map mode promises universal anonymity. A restore map
65
+ contains personal data and must remain private; see [API.md](API.md#reversible-placeholders).
66
+
17
67
  Native rules cover international/Unicode emails, international and selected national-format phones, Luhn-valid payment cards, US SSNs, IPv4/IPv6, checksum-validated IBAN/CPF/Spanish DNI/US NPI, Indian PAN, UK NI, wallet shapes, MACs, messenger handles, home usernames, labelled PII and selected postal address grammars. A bundled 17-locale name gazetteer supplies default free-text detection, including selected inflections/transliterations. Each locale has credential prose and eight labelled PII fixtures. This finite dictionary/grammar is not unrestricted statistical NER; unfamiliar names and unsupported national formats require known values or additional recognizers.
18
68
 
19
69
  Credential labels have fixtures for 17 language variants:
@@ -0,0 +1,35 @@
1
+ import {
2
+ sanitize,
3
+ restore,
4
+ createRestoreMap,
5
+ serializeRestoreMap,
6
+ } from '../src/index.js';
7
+
8
+ // Keep this map private. Publish only result.text, never the map or result.
9
+ const map = createRestoreMap();
10
+ const options = {
11
+ profile: 'documents',
12
+ transformation: { mode: 'placeholder', map },
13
+ };
14
+ const text = 'Jane Doe worked at Acme Ltd in Springfield.';
15
+ const result = sanitize(text, options);
16
+ console.log(result.text);
17
+ console.log('Exact private round trip:', restore(result.text, map) === text);
18
+ console.log(
19
+ 'Derived answer restored privately:',
20
+ restore('Ask Person-1 about Organization-1.', map)
21
+ );
22
+ console.log(
23
+ 'Private map serialization available:',
24
+ typeof serializeRestoreMap(map, 'lino') === 'string'
25
+ );
26
+ const anonymous = sanitize(text, {
27
+ ...options,
28
+ transformation: { mode: 'placeholder', map: false },
29
+ });
30
+ console.log(
31
+ 'No-map output:',
32
+ anonymous.text,
33
+ 'Map returned:',
34
+ Object.hasOwn(anonymous, 'map')
35
+ );
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@link-foundation/sensitive-data-sanitizer",
3
- "version": "0.12.2",
3
+ "version": "0.14.0",
4
4
  "description": "Offline sensitive-text sanitization with credential, multilingual personal-data and scanner union detection",
5
5
  "type": "module",
6
6
  "main": "src/index.js",
@@ -23,7 +23,8 @@
23
23
  "CHANGELOG.md",
24
24
  "LICENSE",
25
25
  "README.md",
26
- "examples/shared-tool-logs.mjs"
26
+ "examples/shared-tool-logs.mjs",
27
+ "examples/reversible-placeholders.mjs"
27
28
  ],
28
29
  "exports": {
29
30
  ".": {
@@ -119,6 +120,7 @@
119
120
  "@secretlint/profiler": "13.0.7",
120
121
  "@secretlint/secretlint-rule-preset-recommend": "13.0.7",
121
122
  "libphonenumber-js": "1.13.14",
123
+ "links-notation": "0.30.0",
122
124
  "re2-wasm": "1.0.2",
123
125
  "re2js": "^2.9.0",
124
126
  "yaml": "^2.9.1"