@link-foundation/sensitive-data-sanitizer 0.19.3 → 0.21.0

package/src/encrypt.js

@@ -0,0 +1,95 @@
+import {
+  createCipheriv,
+  createDecipheriv,
+  createHmac,
+  hkdfSync,
+} from 'node:crypto';
+import { failure } from './detection.js';
+
+// Deterministic authenticated encryption (synthetic IV), comparable to
+// Google's CryptoDeterministicConfig (AES-SIV) and Presidio's encrypt
+// operator. Equal values of one type produce equal tokens, so the output
+// still supports joins and counts, and decrypt() recovers the value. The
+// derived IV is an HMAC of the type and value, so it never repeats for
+// different plaintexts under one key.
+const ivBytes = 12,
+  tagBytes = 16;
+const token = /\[ENCRYPTED:([A-Z][A-Z0-9_]{0,63}):([A-Za-z0-9_-]{38,10000})\]/g;
+const keys = new Map();
+
+function derived(key) {
+  if (typeof key !== 'string' || Buffer.byteLength(key) < 16) {
+    throw failure('ERR_CONFIG');
+  }
+  let entry = keys.get(key);
+  if (!entry) {
+    const material = Buffer.from(
+      hkdfSync(
+        'sha256',
+        key,
+        'sensitive-data-sanitizer',
+        'deterministic-encryption-v1',
+        64
+      )
+    );
+    entry = { cipher: material.subarray(0, 32), mac: material.subarray(32) };
+    if (keys.size >= 16) {
+      keys.clear();
+    }
+    keys.set(key, entry);
+  }
+  return entry;
+}
+
+export function encryptValue(value, type, key) {
+  const { cipher, mac } = derived(key);
+  const iv = createHmac('sha256', mac)
+    .update(`${type}\0${value}`)
+    .digest()
+    .subarray(0, ivBytes);
+  const engine = createCipheriv('aes-256-gcm', cipher, iv);
+  engine.setAAD(Buffer.from(type));
+  const body = Buffer.concat([engine.update(value, 'utf8'), engine.final()]);
+  const payload = Buffer.concat([iv, body, engine.getAuthTag()]);
+  return `[ENCRYPTED:${type}:${payload.toString('base64url')}]`;
+}
+
+function decryptToken(type, encoded, key) {
+  const { cipher } = derived(key);
+  const payload = Buffer.from(encoded, 'base64url');
+  if (payload.length < ivBytes + tagBytes) {
+    throw failure('ERR_MAP');
+  }
+  try {
+    const engine = createDecipheriv(
+      'aes-256-gcm',
+      cipher,
+      payload.subarray(0, ivBytes)
+    );
+    engine.setAAD(Buffer.from(type));
+    engine.setAuthTag(payload.subarray(-tagBytes));
+    return Buffer.concat([
+      engine.update(payload.subarray(ivBytes, -tagBytes)),
+      engine.final(),
+    ]).toString('utf8');
+  } catch {
+    throw failure('ERR_MAP');
+  }
+}
+
+/**
+ * Restores `[ENCRYPTED:TYPE:…]` tokens produced by `{mode: 'encrypt', key}`.
+ * A token that fails authentication under `key` throws ERR_MAP; nothing is
+ * partially restored. With `json: true` restored values are JSON-escaped for
+ * tokens inside JSON string literals.
+ */
+export function decrypt(text, key, { json = false } = {}) {
+  if (typeof text !== 'string') {
+    throw failure('ERR_INPUT');
+  }
+  derived(key);
+  return text.replace(token, (_, type, encoded) => {
+    const value = decryptToken(type, encoded, key);
+    return json ? JSON.stringify(value).slice(1, -1) : value;
+  });
+}

package/src/engines.js

@@ -18,6 +18,7 @@ import { projectText } from './projection.js';
 import { decodedRuns } from './encoded.js';
 import { hasCustomTransforms } from './transforms.js';
 import { builtInEnabled } from './policy.js';
+import { exemptFindingsAsync } from './exempt.js';
 import {
   initializePlaceholders,
   placeholderKnown,
@@ -220,10 +221,7 @@ async function analyzeStructured(
   );
 }
 
-export function createSanitizer(options = {}) {
-  options = applyProfile(options);
-  validateOptions(options);
-  options = initializePlaceholders(options);
+function configuredEngines(options) {
   for (const detector of options.detectors ?? []) {
     if (!detector || typeof detector.detect !== 'function') {
       throw failure('ERR_CONFIG');
@@ -234,6 +232,14 @@ export function createSanitizer(options = {}) {
   if (!options.knownOnly) {
     engines.push(...(options.detectors ?? []));
   }
+  return engines;
+}
+
+export function createSanitizer(options = {}) {
+  options = applyProfile(options);
+  validateOptions(options);
+  options = initializePlaceholders(options);
+  const engines = configuredEngines(options);
   let prepared;
   async function analyze(
     text,
@@ -266,6 +272,7 @@ export function createSanitizer(options = {}) {
             ...options,
             structured: undefined,
             detectors: [],
+            exempt: undefined,
             findings,
           })
         : reviewFindings(text, [...nativeFindings, ...findings], options);
@@ -274,15 +281,31 @@ export function createSanitizer(options = {}) {
       : detected;
   }
 
+  // The caller's `exempt` hook screens the findings of the text it sees,
+  // including the verification passes over the published output.
+  async function screened(
+    text,
+    initial,
+    plain,
+    scopedOptions = options,
+    nativeFindings
+  ) {
+    return exemptFindingsAsync(
+      text,
+      await analyze(text, initial, plain, scopedOptions, nativeFindings),
+      scopedOptions.exempt
+    );
+  }
+
   const sanitizer = {
-    inspect: guarded(analyze),
+    inspect: guarded(screened),
     sanitize: guarded(sanitizeText, () => discardPlaceholderState(options)),
     sanitizeJsonl: (text) =>
       createSanitizer({ ...options, structured: 'jsonl' }).sanitize(text),
   };
   configurations.set(sanitizer, options);
   streamPreparers.set(sanitizer, async (text, nativeFindings) => {
-    const findings = await analyze(
+    const findings = await screened(
       text,
       nativeFindings === undefined ? (options.findings ?? []) : [],
       false,
@@ -305,7 +328,7 @@ export function createSanitizer(options = {}) {
     const findings =
       depth === 0 && prepared?.text === text
         ? prepared.findings
-        : await analyze(text, options.findings ?? [], plain, options);
+        : await screened(text, options.findings ?? [], plain, options);
     if (depth === 0) {
       prepared = undefined;
     }
@@ -329,7 +352,7 @@ export function createSanitizer(options = {}) {
           recursiveOptions
         )
     );
-    if (await customResidual(result.text, options, plain, analyze)) {
+    if (await customResidual(result.text, options, plain, screened)) {
       throw failure('ERR_RESIDUAL');
     }
     const sanitized = fullRedaction
@@ -351,7 +374,7 @@ export function createSanitizer(options = {}) {
         ).text;
     // Verification at the publication boundary runs every enabled engine.
     if (
-      (await analyze(sanitized, [], plain, options)).some((f) =>
+      (await screened(sanitized, [], plain, options)).some((f) =>
         actionable(f, options)
       )
     ) {
@@ -360,7 +383,7 @@ export function createSanitizer(options = {}) {
       );
     }
     verifyStructuredResult(result.text, options, plain);
-    if (await placeholderResidual(result.text, options, plain, analyze)) {
+    if (await placeholderResidual(result.text, options, plain, screened)) {
       throw failure('ERR_RESIDUAL');
     }
     return { ...result, findings: auditFaked(findings, options, result) };

package/src/entities.js

@@ -3,6 +3,7 @@ import azure from './vendor/entities/azure.json' with { type: 'json' };
 import aws from './vendor/entities/aws.json' with { type: 'json' };
 import { escapePattern, collectMatches } from './detection.js';
 import { numericNoise, epochNumber } from './confidence.js';
+import { cryptoAddress } from './crypto-addresses.js';
 import { valueRange, personalValueAllowed } from './context.js';
 export const entityCatalogs = { google, azure, aws };
 const canonical = (name) =>
@@ -161,13 +162,13 @@ const formatRules = [
   ],
   [
     'SPAIN_DNI_NUMBER',
-    /\b\d{8}[A-Z]\b/g,
+    /\b\d{8}-?[A-Z]\b/g,
     (v) => 'TRWAGMYFPDXBNJZSQVHLCKE'[Number(v.slice(0, 8)) % 23] === v.at(-1),
   ],
   [
     'CRYPTO_WALLET',
-    /\b(?:0x[a-fA-F0-9]{40}|[13][a-km-zA-HJ-NP-Z1-9]{25,34}|bc1[ac-hj-np-z02-9]{25,62})\b/g,
-    () => true,
+    /\b(?:0x[a-fA-F0-9]{40}|[13][a-km-zA-HJ-NP-Z1-9]{25,34}|bc1[ac-hj-np-z02-9]{25,62}|BC1[AC-HJ-NP-Z02-9]{25,62})\b/g,
+    cryptoAddress,
   ],
   ['US_HEALTHCARE_NPI', /\b[12]\d{9}\b/g, npi],
 ];

package/src/exempt.js

@@ -0,0 +1,72 @@
+import { failure } from './detection.js';
+
+// `exempt(finding, text) => boolean` keeps findings the caller's policy
+// knows are not sensitive (a configuration flag, a counter, a value another
+// tool already masked). It runs after detection and before transformation.
+// The hook decides per span: verification of the published text consults it
+// again for each finding in the output, so the same value in another place
+// is still caught unless the hook keeps that span too.
+// Values the caller declared (`knownSecrets`, `knownPersonal`,
+// `confirmedPersonal`) and structured `JSON_VALUE` denials are never offered.
+const declared = new Set([
+  'known-secret',
+  'known-personal',
+  'confirmed-personal',
+]);
+const offered = (finding) =>
+  !declared.has(finding.rule) && finding.type !== 'JSON_VALUE';
+
+function view(text, finding) {
+  return Object.freeze({
+    ...finding,
+    value: text.slice(finding.start, finding.end),
+  });
+}
+
+export function validateExempt(exempt) {
+  if (exempt !== undefined && typeof exempt !== 'function') {
+    throw failure('ERR_CONFIG');
+  }
+}
+
+function decision(kept) {
+  if (typeof kept !== 'boolean') {
+    // `undefined` from a forgotten return must not silently keep a secret.
+    throw failure('ERR_CONFIG');
+  }
+  return kept;
+}
+
+export function exemptFindings(text, findings, exempt) {
+  if (!exempt) {
+    return findings;
+  }
+  return findings.filter((finding) => {
+    if (!offered(finding)) {
+      return true;
+    }
+    const kept = exempt(view(text, finding), text);
+    if (typeof kept?.then === 'function') {
+      // An async hook needs the async engine (`createSanitizer`).
+      kept.then(undefined, () => {});
+      throw failure('ERR_CONFIG');
+    }
+    return !decision(kept);
+  });
+}
+
+export async function exemptFindingsAsync(text, findings, exempt) {
+  if (!exempt) {
+    return findings;
+  }
+  const retained = [];
+  for (const finding of findings) {
+    if (
+      !offered(finding) ||
+      !decision(await exempt(view(text, finding), text))
+    ) {
+      retained.push(finding);
+    }
+  }
+  return retained;
+}

package/src/fake-memory.js

@@ -0,0 +1,32 @@
+import { failure } from './detection.js';
+
+// Values that `mode: 'fake'` generated, per transformation object, so that a
+// later pass recognizes them as fictional. Kept apart from fake.js so that the
+// credentials entry can read the (empty) memory without loading faker.
+const generated = new WeakMap();
+export function generatedValues(options) {
+  const transforms = [
+    options.transformation,
+    ...Object.values(options.transformations ?? {}),
+  ];
+  return transforms.flatMap((t) =>
+    t?.mode === 'fake'
+      ? [...(generated.get(t) ?? [])].map(([value, credential]) => ({
+          value,
+          credential,
+        }))
+      : []
+  );
+}
+export function remember(value, transform, credential) {
+  let values = generated.get(transform);
+  if (!values) {
+    values = new Map();
+    generated.set(transform, values);
+  }
+  if (values.size >= 100000 && !values.has(value)) {
+    throw failure('ERR_LIMIT');
+  }
+  values.set(value, credential);
+  return value;
+}

package/src/fake.js

@@ -4,35 +4,11 @@ import { parsePhoneNumberFromString } from 'libphonenumber-js/max';
 import { mrzCheck } from './identity.js';
 import { fakeName } from './fake-names.js';
 import * as checks from './national-checks.js';
-import { REDACTED, failure } from './detection.js';
+import { REDACTED } from './detection.js';
+import { remember } from './fake-memory.js';
+
+export { generatedValues } from './fake-memory.js';
 
-const generated = new WeakMap();
-export function generatedValues(options) {
-  const transforms = [
-    options.transformation,
-    ...Object.values(options.transformations ?? {}),
-  ];
-  return transforms.flatMap((t) =>
-    t?.mode === 'fake'
-      ? [...(generated.get(t) ?? [])].map(([value, credential]) => ({
-          value,
-          credential,
-        }))
-      : []
-  );
-}
-function remember(value, transform, credential) {
-  let values = generated.get(transform);
-  if (!values) {
-    values = new Map();
-    generated.set(transform, values);
-  }
-  if (values.size >= 100000 && !values.has(value)) {
-    throw failure('ERR_LIMIT');
-  }
-  values.set(value, credential);
-  return value;
-}
 function digest(value, key, purpose = 'value') {
   return createHmac('sha256', key).update(`${purpose}\0${value}`).digest();
 }
@@ -116,6 +92,18 @@ const validators = {
   SPAIN_DNI_NUMBER: (v) =>
     'TRWAGMYFPDXBNJZSQVHLCKE'[Number(v.slice(0, 8)) % 23] === v.at(-1),
   BRAZIL_CPF_NUMBER: cpf,
+  ABA_ROUTING_NUMBER: checks.abaRouting,
+  ES_NIE: checks.spanishNie,
+  IT_VAT_CODE: checks.shortLuhn,
+  AU_ACN: checks.acn,
+  IN_AADHAAR: checks.aadhaar,
+  TH_TNIN: checks.thaiId,
+  MEDICAL_LICENSE: checks.dea,
+  IMEI: checks.shortLuhn,
+  DE_TAX_ID: checks.germanTaxId,
+  AE_EMIRATES_ID: checks.emiratesId,
+  SA_NATIONAL_ID: checks.saudiId,
+  IL_ID_NUMBER: checks.israeliId,
 };
 function cpf(d) {
   for (const n of [9, 10]) {
@@ -226,9 +214,24 @@ function idShape(compact, candidate, type) {
     candidate =
       candidate.slice(0, 10) + compact.slice(10, 12) + candidate.slice(12);
   }
-  if (type === 'SG_NRIC_FIN') {
+  if (
+    [
+      'SG_NRIC_FIN',
+      'ES_NIE',
+      'IN_AADHAAR',
+      'TH_TNIN',
+      'DE_TAX_ID',
+      'SA_NATIONAL_ID',
+    ].includes(type)
+  ) {
     candidate = compact[0] + candidate.slice(1);
   }
+  if (['ABA_ROUTING_NUMBER', 'MEDICAL_LICENSE'].includes(type)) {
+    candidate = compact.slice(0, 2) + candidate.slice(2);
+  }
+  if (type === 'AE_EMIRATES_ID') {
+    candidate = compact.slice(0, 7) + candidate.slice(7);
+  }
   if (type === 'CREDIT_CARD') {
     candidate = compact.slice(0, 6) + candidate.slice(6);
   }
@@ -255,6 +258,7 @@ function idChecks(compact, candidate, type) {
       'SG_NRIC_FIN',
       'IT_FISCAL_CODE',
       'SPAIN_DNI_NUMBER',
+      'ES_NIE',
     ].includes(type);
     const count =
       (type === 'RU_INN' && compact.length === 12) ||

package/src/features.js

@@ -0,0 +1,29 @@
+// Personal-data features that the main entry registers through
+// `personal-features.js`. The `./credentials` entry never imports that module,
+// so it does not load the personal detectors, faker or libphonenumber
+// (issue 137). This module must stay free of imports.
+const registered = {};
+
+export function registerFeatures(entries) {
+  Object.assign(registered, entries);
+}
+
+/** Runs the registered implementation, or returns `fallback` without one. */
+export function withFeature(name, fallback, ...args) {
+  const value = registered[name];
+  return value ? value(...args) : fallback;
+}
+
+/** The registered implementation; fails closed when it is unavailable. */
+export function feature(name) {
+  const value = registered[name];
+  if (!value) {
+    throw Object.assign(
+      new Error(
+        'This option needs the personal-data modules of the main entry.'
+      ),
+      { code: 'ERR_CONFIG' }
+    );
+  }
+  return value;
+}

package/src/files.js

@@ -4,31 +4,76 @@ import { dirname, join, resolve } from 'node:path';
 import { randomUUID } from 'node:crypto';
 import { TextDecoder } from 'node:util';
 import { failure } from './detection.js';
-import { decodeLogBytes } from './log-bytes.js';
+import { decodeLogBytes, encodeLogText } from './log-bytes.js';
 
-export function decodeText(
+// UTF-16 text (Windows logs, PowerShell transcripts) is recognised only by its
+// byte order mark; BOM-less UTF-16 remains binary input.
+export function textEncoding(bytes) {
+  if (bytes.length >= 2 && bytes.length % 2 === 0) {
+    if (bytes[0] === 0xff && bytes[1] === 0xfe) {
+      return 'utf-16le';
+    }
+    if (bytes[0] === 0xfe && bytes[1] === 0xff) {
+      return 'utf-16be';
+    }
+  }
+  return 'utf-8';
+}
+
+function decodeWith(encoding, bytes) {
+  try {
+    return new TextDecoder(encoding, { fatal: true, ignoreBOM: true }).decode(
+      bytes
+    );
+  } catch {
+    throw failure('ERR_ENCODING');
+  }
+}
+
+// Returns the decoded text and the encoding needed to write it back.
+export function decodeTextInput(
   bytes,
   { lossless = false, allowBinary = false } = {}
 ) {
   if (typeof allowBinary !== 'boolean') {
     throw failure('ERR_CONFIG');
   }
+  const encoding = textEncoding(bytes);
+  if (encoding !== 'utf-8') {
+    return { text: decodeWith(encoding, bytes), encoding };
+  }
   if (!allowBinary && bytes.includes(0)) {
     throw failure('ERR_BINARY');
   }
-  if (lossless) {
-    return decodeLogBytes(bytes);
-  }
-  try {
-    return new TextDecoder('utf-8', { fatal: true, ignoreBOM: true }).decode(
-      bytes
-    );
-  } catch {
-    throw failure('ERR_ENCODING');
+  return {
+    text: lossless ? decodeLogBytes(bytes) : decodeWith('utf-8', bytes),
+    encoding,
+  };
+}
+
+export function decodeText(bytes, options) {
+  return decodeTextInput(bytes, options).text;
+}
+
+// Encode output in the input encoding; UTF-16 keeps its byte order mark
+// because the decoded text retains it.
+export function encodeTextOutput(text, encoding = 'utf-8') {
+  if (encoding === 'utf-8') {
+    return encodeLogText(text);
   }
+  const bytes = Buffer.from(text, 'utf16le');
+  return encoding === 'utf-16be' ? bytes.swap16() : bytes;
 }
 
 export async function readTextFile(path, maxBytes = 10 * 1024 * 1024, options) {
+  return (await readTextInput(path, maxBytes, options)).text;
+}
+
+export async function readTextInput(
+  path,
+  maxBytes = 10 * 1024 * 1024,
+  options
+) {
   let handle;
   try {
     const info = await lstat(path);
@@ -52,7 +97,7 @@ export async function readTextFile(path, maxBytes = 10 * 1024 * 1024, options) {
     if (read > maxBytes || read > opened.size) {
       throw failure('ERR_LIMIT');
     }
-    return decodeText(bytes.subarray(0, read), options);
+    return decodeTextInput(bytes.subarray(0, read), options);
   } catch (error) {
     if (error.code?.startsWith('ERR_')) {
       throw error;