@link-foundation/sensitive-data-sanitizer 0.12.2 → 0.14.0

package/src/restore-map.js

@@ -0,0 +1,220 @@
+import { createHash } from 'node:crypto';
+import { Parser, Link } from 'links-notation';
+import { failure, escapePattern } from './detection.js';
+import { atomicWrite, readTextFile } from './files.js';
+
+export const restoreMapKind = 'sensitive-data-restore-map';
+export const documentDigest = (text) =>
+  createHash('sha256').update(text).digest('hex');
+const defaultFormat = '{Type}-{n}';
+export function validatePlaceholderFormat(format = defaultFormat) {
+  if (
+    typeof format !== 'string' ||
+    format.length > 128 ||
+    !/\{(?:Type|TYPE|type)\}/.test(format) ||
+    !format.includes('{n}') ||
+    /[\r\n\0]/.test(format) ||
+    (format.match(/\{n\}/g) ?? []).length !== 1 ||
+    (format.match(/\{(?:Type|TYPE|type)\}/g) ?? []).length !== 1 ||
+    /[{}]/.test(
+      format
+        .replace(/\{(?:Type|TYPE|type|n)\}/g, '')
+        .replaceAll('{{', '')
+        .replaceAll('}}', '')
+    )
+  ) {
+    throw failure('ERR_CONFIG');
+  }
+  return format;
+}
+export function placeholderToken(type, number, format = defaultFormat) {
+  const title = type
+    .toLowerCase()
+    .split('_')
+    .filter(Boolean)
+    .map((word) => word[0].toUpperCase() + word.slice(1))
+    .join('');
+  return format.replace(
+    /\{(Type|TYPE|type|n)\}/g,
+    (_, key) =>
+      ({
+        Type: title,
+        TYPE: type,
+        type: type.toLowerCase(),
+        n: String(number),
+      })[key]
+  );
+}
+export function createRestoreMap({ format = defaultFormat } = {}) {
+  return {
+    kind: restoreMapKind,
+    version: 1,
+    format: validatePlaceholderFormat(format),
+    entries: [],
+    documents: [],
+  };
+}
+export function validateRestoreMap(map) {
+  if (
+    !map ||
+    map.kind !== restoreMapKind ||
+    map.version !== 1 ||
+    !Array.isArray(map.entries) ||
+    !Array.isArray(map.documents) ||
+    map.entries.length > 100000 ||
+    map.documents.length > 100000
+  ) {
+    throw failure('ERR_MAP');
+  }
+  validatePlaceholderFormat(map.format);
+  validateEntries(map);
+  for (const doc of map.documents) {
+    validateDocument(doc);
+  }
+  return map;
+}
+function validateEntries(map) {
+  const tokens = new Set();
+  for (const entry of map.entries) {
+    if (
+      !entry ||
+      !/^[A-Z][A-Z0-9_]{0,63}$/.test(entry.type) ||
+      !Number.isSafeInteger(entry.number) ||
+      entry.number < 1 ||
+      entry.token !== placeholderToken(entry.type, entry.number, map.format) ||
+      tokens.has(entry.token) ||
+      !nonemptyStrings(entry.values) ||
+      !Array.isArray(entry.aliases) ||
+      entry.aliases.some((v) => typeof v !== 'string' || !v)
+    ) {
+      throw failure('ERR_MAP');
+    }
+    tokens.add(entry.token);
+  }
+}
+function nonemptyStrings(values) {
+  return (
+    Array.isArray(values) &&
+    values.length > 0 &&
+    values.every((v) => typeof v === 'string' && v.length > 0)
+  );
+}
+function validateDocument(doc) {
+  if (
+    !doc ||
+    !/^[a-f0-9]{64}$/.test(doc.digest) ||
+    !Array.isArray(doc.occurrences)
+  ) {
+    throw failure('ERR_MAP');
+  }
+  let end = 0;
+  for (const occurrence of doc.occurrences) {
+    if (
+      !occurrence ||
+      !Number.isSafeInteger(occurrence.start) ||
+      occurrence.start < end ||
+      !Number.isSafeInteger(occurrence.end) ||
+      occurrence.end <= occurrence.start ||
+      typeof occurrence.raw !== 'string' ||
+      typeof occurrence.token !== 'string' ||
+      !occurrence.token ||
+      occurrence.end - occurrence.start !== occurrence.token.length
+    ) {
+      throw failure('ERR_MAP');
+    }
+    end = occurrence.end;
+  }
+}
+
+export function serializeRestoreMap(map, format = 'json') {
+  validateRestoreMap(map);
+  if (format === 'json') {
+    return `${JSON.stringify(map, null, 2)}\n`;
+  }
+  if (format !== 'lino') {
+    throw failure('ERR_CONFIG');
+  }
+  // Let the upstream formatter handle quotes, slashes, newlines and Unicode.
+  return `${new Link(restoreMapKind, [new Link(JSON.stringify(map))]).toString()}\n`;
+}
+export function parseRestoreMap(text) {
+  try {
+    if (
+      typeof text !== 'string' ||
+      Buffer.byteLength(text) > 32 * 1024 * 1024
+    ) {
+      throw failure('ERR_MAP');
+    }
+    if (text.trimStart().startsWith('{')) {
+      return validateRestoreMap(JSON.parse(text));
+    }
+    const links = new Parser({
+      maxInputSize: 32 * 1024 * 1024,
+      maxDepth: 8,
+    }).parse(text);
+    if (
+      links.length !== 1 ||
+      links[0].id !== restoreMapKind ||
+      links[0].values.length !== 1 ||
+      links[0].values[0].values.length
+    ) {
+      throw failure('ERR_MAP');
+    }
+    return validateRestoreMap(JSON.parse(links[0].values[0].id));
+  } catch {
+    throw failure('ERR_MAP');
+  }
+}
+export const readRestoreMap = async (path) =>
+  parseRestoreMap(await readTextFile(path, 32 * 1024 * 1024));
+export async function writeRestoreMap(
+  path,
+  map,
+  { format = path.endsWith('.lino') ? 'lino' : 'json', replace = false } = {}
+) {
+  await atomicWrite(path, serializeRestoreMap(map, format), { replace });
+}
+export function rejectRestoreMap(text) {
+  if (
+    /"kind"\s*:\s*"sensitive-data-restore-map"|\(["\x27`]?sensitive-data-restore-map["\x27`]?\s*:/.test(
+      text
+    )
+  ) {
+    throw failure('ERR_PRIVATE_MAP');
+  }
+}
+export function restore(text, map) {
+  if (typeof text !== 'string') {
+    throw failure('ERR_INPUT');
+  }
+  validateRestoreMap(map);
+  const document = map.documents.find(
+    (doc) => doc.digest === documentDigest(text)
+  );
+  if (document) {
+    let cursor = 0;
+    const parts = [];
+    for (const occurrence of document.occurrences) {
+      if (text.slice(occurrence.start, occurrence.end) !== occurrence.token) {
+        throw failure('ERR_MAP');
+      }
+      parts.push(text.slice(cursor, occurrence.start), occurrence.raw);
+      cursor = occurrence.end;
+    }
+    return parts.join('') + text.slice(cursor);
+  }
+  const entries = new Map(
+    map.entries.map((entry) => [entry.token, entry.values[0]])
+  );
+  if (!entries.size) {
+    return text;
+  }
+  const pattern = new RegExp(
+    `(?<![\\p{L}\\p{N}_])(?:${[...entries.keys()]
+      .sort((a, b) => b.length - a.length)
+      .map(escapePattern)
+      .join('|')})(?![\\p{L}\\p{N}_-])`,
+    'gu'
+  );
+  return text.replace(pattern, (token) => entries.get(token));
+}

package/src/sanitizer.js

@@ -16,6 +16,17 @@ import { URL } from 'node:url';
 import { isPublic } from './public.js';
 import { validateTransforms, renderReplacement } from './transforms.js';
 import { fakeIdentityValue, isIdentityType } from './identity.js';
+import { rejectRestoreMap } from './restore-map.js';
+import {
+  initializePlaceholders,
+  placeholderKnown,
+  preparePlaceholderText,
+  finishPlaceholders,
+  hasPlaceholders,
+  placeholderValues,
+  placeholderNameFindings,
+  nestedPlaceholderOptions,
+} from './placeholders.js';
 import {
   structuredBatches,
   mapStructuredFindings,
@@ -25,9 +36,16 @@ import {
 } from './structured.js';
 
 const fakedResults = new WeakMap();
-const publicTypes = new Set(['PERSON', 'ORGANIZATION', 'EMAIL']);
+const publicTypes = new Set([
+  'PERSON',
+  'ORGANIZATION',
+  'EDUCATION',
+  'LOCATION',
+  'EMAIL',
+]);
 const optionNames = new Set([
   'credentialsOnly',
+  'knownOnly',
   'knownSecrets',
   'knownPersonal',
   'confirmedPersonal',
@@ -53,9 +71,13 @@ const optionNames = new Set([
   'minConfidence',
   'threshold',
   'profile',
+  'documentEntities',
 ]);
 
 export function applyProfile(options = {}) {
+  if (options?.profile === 'documents') {
+    return { ...options, documentEntities: options.documentEntities ?? true };
+  }
   return options?.profile === 'publication'
     ? {
         ...options,
@@ -145,10 +167,12 @@ export function validateOptions(options = {}) {
   for (const key of [
     'paranoid',
     'credentialsOnly',
+    'knownOnly',
     'decode',
     'secretlint',
     'publicKnowledge',
     'identityMask',
+    'documentEntities',
   ]) {
     if (options[key] !== undefined && typeof options[key] !== 'boolean') {
       throw failure('ERR_CONFIG');
@@ -196,7 +220,7 @@ function validatePolicy(options) {
   for (const [key, values] of [
     ['fakeIdentity', [false, 'specimen-and-synthetic']],
     ['structured', ['json', 'jsonl']],
-    ['profile', ['publication']],
+    ['profile', ['publication', 'documents']],
   ]) {
     if (options[key] !== undefined && !values.includes(options[key])) {
       throw failure('ERR_CONFIG');
@@ -212,6 +236,7 @@ export function validateInput(text, options) {
   if (typeof text !== 'string') {
     throw failure('ERR_INPUT');
   }
+  rejectRestoreMap(text);
   if (text.length > (options.maxInputLength ?? 32 * 1024 * 1024)) {
     throw failure('ERR_LIMIT');
   }
@@ -423,15 +448,16 @@ function inspectText(text, options = {}) {
 
 // Reapply policy to the native/required-engine union without rescanning text.
 export function reviewFindings(text, findings, options) {
+  findings = placeholderNameFindings(text, findings, options);
   if (findings.length > (options.maxFindings ?? 100000)) {
     throw failure('ERR_LIMIT');
   }
   const unique = new Map();
+  const exempt = findings.filter((f) =>
+    allowed(text, f, options.publicEntities ?? [])
+  );
   for (const f of findings) {
-    if (
-      !allowed(text, f, options.publicEntities ?? []) &&
-      !isPublic(text, f, options)
-    ) {
+    if (!coveredByPublic(f, exempt) && !isPublic(text, f, options)) {
       unique.set(`${f.start}:${f.end}:${f.category}:${f.type}:${f.rule}`, f);
     }
   }
@@ -449,6 +475,18 @@ export function reviewFindings(text, findings, options) {
   options.debug?.({ event: 'detected', count: result.length });
   return result;
 }
+function coveredByPublic(finding, exempt) {
+  return (
+    finding.category === 'personal' &&
+    !['known-personal', 'confirmed-personal'].includes(finding.rule) &&
+    exempt.some(
+      (span) =>
+        span.type === finding.type &&
+        span.start <= finding.start &&
+        span.end >= finding.end
+    )
+  );
+}
 
 export function inspectFindingsOnly(text, findings, options) {
   const all = [...findings, ...(options.findings ?? [])].map((f) =>
@@ -484,7 +522,10 @@ function selectedFindings(findings, credentialsOnly) {
 
 export function markFake(text, findings, options) {
   findings = selectedFindings(findings, options.credentialsOnly);
-  const generated = generatedValues(options);
+  const generated = [
+    ...generatedValues(options),
+    ...placeholderValues(options),
+  ];
   if (
     !options.fakeIdentity &&
     !options.fakeValues?.length &&
@@ -651,37 +692,72 @@ export function redactResult(
 ) {
   options = applyProfile(options);
   const spans = validatedSpans(text, findings, options);
+  options = initializePlaceholders(options);
+  preparePlaceholderText(text, findings, options);
   const parts = [];
   const faked = [];
+  const occurrences = [];
+  let irreversibleRedactions = 0;
+  let outputLength = 0;
   let cursor = 0;
   for (const span of spans) {
     const replacement =
       replacements.get(`${span.start}:${span.end}`) ??
       renderReplacement(text, span, options, (decoded) =>
-        sanitize(decoded, {
-          ...confirmationOptions(text, findings, options),
-          findings: [],
-          preserveEncoding: true,
-          structured: undefined,
-        })
+        sanitize(
+          decoded,
+          nestedPlaceholderOptions({
+            ...confirmationOptions(text, findings, options),
+            findings: [],
+            preserveEncoding: true,
+            structured: undefined,
+          })
+        )
       );
     parts.push(text.slice(cursor, span.start), replacement);
+    outputLength += span.start - cursor;
     const transform =
       options.transformations?.[span.type] ?? options.transformation;
+    if (retainedReplacement(transform, 'placeholder', span, replacement)) {
+      occurrences.push({
+        start: outputLength,
+        end: outputLength + replacement.length,
+        token: replacement,
+        raw: text.slice(span.start, span.end),
+      });
+    }
     if (
-      transform?.mode === 'fake' &&
-      replacement !== REDACTED &&
-      (span.category === 'personal' || transform.credentials)
+      hasPlaceholders(options) &&
+      !retainedReplacement(transform, 'placeholder', span, replacement)
     ) {
+      irreversibleRedactions++;
+    }
+    outputLength += replacement.length;
+    if (retainedReplacement(transform, 'fake', span, replacement)) {
       faked.push(span);
     }
     cursor = span.end;
   }
   parts.push(text.slice(cursor));
-  const result = { text: parts.join(''), redactions: spans.length };
+  const result = finishPlaceholders(
+    {
+      text: parts.join(''),
+      redactions: spans.length,
+      ...(hasPlaceholders(options) ? { irreversibleRedactions } : {}),
+    },
+    occurrences,
+    options
+  );
   fakedResults.set(result, faked);
   return result;
 }
+function retainedReplacement(transform, mode, span, replacement) {
+  return (
+    transform?.mode === mode &&
+    replacement !== REDACTED &&
+    (span.category === 'personal' || transform.credentials)
+  );
+}
 
 export async function redactResultAsync(
   text,
@@ -689,10 +765,18 @@ export async function redactResultAsync(
   options,
   sanitizeDecoded
 ) {
+  options = initializePlaceholders(options);
+  preparePlaceholderText(text, findings, options);
   const replacements = new Map();
   {
     for (const span of validatedSpans(text, findings, options)) {
       if (span.type !== 'ENCODED_SENSITIVE') {
+        if (hasPlaceholders(options)) {
+          replacements.set(
+            `${span.start}:${span.end}`,
+            renderReplacement(text, span, options)
+          );
+        }
         continue;
       }
       const run = encodedValue(text.slice(span.start, span.end));
@@ -707,7 +791,7 @@ export async function redactResultAsync(
       }
       const result = await sanitizeDecoded(
         run.text,
-        confirmationOptions(text, findings, options)
+        nestedPlaceholderOptions(confirmationOptions(text, findings, options))
       );
       replacements.set(
         `${span.start}:${span.end}`,
@@ -731,6 +815,8 @@ export function sanitize(text, options = {}) {
 }
 function sanitizeText(text, options = {}, round = 0) {
   options = applyProfile(options);
+  validateOptions(options);
+  options = placeholderKnown(initializePlaceholders(options));
   const findings = inspect(text, options);
   const result = redactResult(text, findings, options);
   const sanitized = redactResult(text, findings, {
@@ -752,6 +838,14 @@ function sanitizeText(text, options = {}, round = 0) {
   if (options.structured) {
     verifyStructured(result.text, options);
   }
+  if (
+    hasPlaceholders(options) &&
+    inspect(result.text, { ...options, findings: [] }).some((f) =>
+      actionable(f, options)
+    )
+  ) {
+    throw failure('ERR_RESIDUAL');
+  }
   return { ...result, findings: auditFaked(findings, options, result) };
 }
 
@@ -780,6 +874,13 @@ export function redactAgain(result, findings, options, round, rerun) {
   const merge = (next) => ({
     ...next,
     redactions: result.redactions + next.redactions,
+    ...(hasPlaceholders(options)
+      ? {
+          irreversibleRedactions:
+            (result.irreversibleRedactions ?? 0) +
+            (next.irreversibleRedactions ?? 0),
+        }
+      : {}),
     findings: auditFaked(findings, options, result),
   });
   const next = rerun({ ...options, findings: [] }, round + 1);

package/src/stream.js

@@ -23,6 +23,11 @@ import { sanitizeJsonlStream } from './jsonl-stream.js';
 import { workerExecArgv } from './worker-options.js';
 import { streamIdentity } from './stream-identity.js';
 import { workerResult } from './file-worker-result.js';
+import {
+  hasPlaceholders,
+  initializePlaceholders,
+  beginPlaceholderStream,
+} from './placeholders.js';
 
 const streamKeys = new Set([
   'sanitizer',
@@ -63,7 +68,7 @@ function configuration(options, collecting = false, complete) {
     collecting,
     complete
   );
-  const sanitizerOptions = identity.options;
+  const sanitizerOptions = initializePlaceholders(identity.options);
   const engine = createSanitizer(sanitizerOptions);
   return {
     maxRecordBytes,
@@ -215,6 +220,18 @@ export async function* sanitizeStream(source, options = {}) {
   }
 }
 async function* streamWithConfiguration(source, options, config) {
+  const finish = config.collecting
+    ? () => {}
+    : beginPlaceholderStream(config.sanitizerOptions);
+  let complete = false;
+  try {
+    yield* streamBatches(source, options, config);
+    complete = true;
+  } finally {
+    finish(complete);
+  }
+}
+async function* streamBatches(source, options, config) {
   if (config.jsonl) {
     yield* sanitizeJsonlStream(source, options, config);
     return;
@@ -422,7 +439,11 @@ export async function sanitizeFileToFile(
   }
 }
 export async function sanitizeFileBounded(source, target, options = {}) {
-  if (options.worker === false || options.sanitizer) {
+  if (
+    options.worker === false ||
+    options.sanitizer ||
+    hasPlaceholders(options.sanitizerOptions ?? options)
+  ) {
     return sanitizeFileToFile(source, target, options);
   }
   const workerHeapMb = options.workerHeapMb ?? 256,

package/src/structured.js

@@ -1,4 +1,5 @@
 import { failure } from './detection.js';
+import { hasPlaceholders } from './placeholders.js';
 
 // Session metadata is opt-in at the CLI; library callers choose their own
 // fields. These names do not grant exemptions to provider/known credentials.
@@ -164,7 +165,7 @@ function ordinaryPersonalKey(entry, finding) {
 // Propagation runs again at source offsets. Apply the same key policy after it
 // so a name confirmed in a value cannot rename an ordinary identifier key.
 export function preserveIdentifierKeys(text, findings, options) {
-  if (!options.structured) {
+  if (!options.structured || hasPlaceholders(options)) {
     return findings;
   }
   const keys = structuredBatches(text, options).flatMap((batch) =>
@@ -193,6 +194,9 @@ export function preserveIdentifierKeys(text, findings, options) {
   });
 }
 function preserveStructuredEntry(entry, finding, text, options) {
+  if (hasPlaceholders(options)) {
+    return false;
+  }
   return (
     (entry.key &&
       ['confirmed-personal', 'document-propagation'].includes(finding.rule)) ||

package/src/transforms.js

@@ -3,6 +3,7 @@ import { createHmac } from 'node:crypto';
 import { failure, REDACTED } from './detection.js';
 import { encodedValue, encodeRun } from './encoded.js';
 import { isIdentityType } from './identity.js';
+import { validatePlaceholder, placeholderReplacement } from './placeholders.js';
 
 const modes = new Set([
   'redact',
@@ -13,6 +14,7 @@ const modes = new Set([
   'format-preserving',
   'date-shift',
   'bucket',
+  'placeholder',
 ]);
 export function validateTransforms(options) {
   if (
@@ -43,6 +45,9 @@ function validateTransform(transform) {
   if (!transform || !modes.has(transform.mode)) {
     throw failure('ERR_CONFIG');
   }
+  if (transform.mode === 'placeholder') {
+    validatePlaceholder(transform);
+  }
   const checks = {
     fake: () =>
       typeof transform.key === 'string' &&
@@ -177,6 +182,9 @@ export function renderReplacement(text, finding, options, encodedSanitize) {
       throw failure('ERR_ENCODING');
     }
   }
+  return renderValue(text, finding, transform, options);
+}
+function renderValue(text, finding, transform, options) {
   let raw = text.slice(finding.start, finding.end);
   const jsonQuoted =
     text[finding.start - 1] === '"' && text[finding.end] === '"';
@@ -187,6 +195,9 @@ export function renderReplacement(text, finding, options, encodedSanitize) {
       return REDACTED;
     }
   }
-  const result = transformed(raw, finding, transform);
+  const result =
+    transform?.mode === 'placeholder'
+      ? placeholderReplacement(raw, finding, transform, options)
+      : transformed(raw, finding, transform);
   return jsonQuoted ? JSON.stringify(result).slice(1, -1) : result;
 }