@link-foundation/sensitive-data-sanitizer 0.18.0 → 0.19.0

package/src/sanitizer.js

@@ -1,4 +1,8 @@
 import { generatedValues } from './fake.js';
+import { isCompiledKnown, projectedKnown } from './compiled-known.js';
+import { detectSync, validateDetectors } from './sync-detectors.js';
+import { validatePolicyControls, builtInEnabled } from './policy.js';
+import { compiledVocabulary } from './vocabulary.js';
 import { validateProperNames } from './proper-names.js';
 import { prepareFakeText } from './fake-entities.js';
 import { alignEscapes } from './escape-spans.js';
@@ -16,7 +20,12 @@ import { projectText } from './projection.js';
 import { decodedRuns, encodeRun, encodedValue } from './encoded.js';
 import { URL } from 'node:url';
 import { isPublic } from './public.js';
-import { validateTransforms, renderReplacement } from './transforms.js';
+import {
+  validateTransforms,
+  renderReplacement,
+  hasCustomTransforms,
+  renderCallbackAsync,
+} from './transforms.js';
 import { fakeIdentityValue, isIdentityType } from './identity.js';
 import { rejectRestoreMap } from './restore-map.js';
 import {
@@ -38,6 +47,7 @@ import {
   mapStructuredFindings,
   preserveIdentifierKeys,
   structuredValueRanges,
+  structuredStringRanges,
   verifyStructured,
 } from './structured.js';
 
@@ -81,6 +91,14 @@ const optionNames = new Set([
   'documentEntities',
   'properNames',
   'scripts',
+  'rules',
+  'types',
+  'fields',
+  'allowCredentialDisable',
+  'vocabulary',
+  'strict',
+  'exactFields',
+  'allowedFields',
 ]);
 
 export function applyProfile(options = {}) {
@@ -108,6 +126,7 @@ function validateCollections(options) {
     'fakeValues',
     'structuralFields',
     'sensitiveFields',
+    'allowedFields',
   ]) {
     if (options[key] !== undefined && !Array.isArray(options[key])) {
       throw failure('ERR_CONFIG');
@@ -118,6 +137,7 @@ function validateCollections(options) {
     'fakeValues',
     'structuralFields',
     'sensitiveFields',
+    'allowedFields',
   ]) {
     if (
       (options[key] ?? []).some((value) => typeof value !== 'string' || !value)
@@ -126,7 +146,9 @@ function validateCollections(options) {
     }
   }
   for (const entry of [
-    ...(options.knownPersonal ?? []),
+    ...(isCompiledKnown(options.knownPersonal)
+      ? []
+      : (options.knownPersonal ?? [])),
     ...(options.confirmedPersonal ?? []),
   ]) {
     if (
@@ -203,6 +225,9 @@ export function validateOptions(options = {}) {
     throw failure('ERR_CONFIG');
   }
   validateCollections(options);
+  validateDetectors(options.detectors);
+  validatePolicyControls(options);
+  compiledVocabulary(options.vocabulary);
   validateTransforms(options);
   validateLimits(options);
   for (const entry of options.publicEntities ?? []) {
@@ -351,7 +376,12 @@ function encodedDetect(text, options, emit) {
     }
     const found = [];
     const view = projectText(run.text).text;
-    nativeDetect(view, options, (f) => found.push(f));
+    nativeDetect(view, options, (f) =>
+      found.push(sanitizeFinding(f, view.length))
+    );
+    detectSync(view, options, (f) =>
+      found.push(sanitizeFinding(f, view.length))
+    );
     const sensitive = reviewFindings(view, found, options).filter((f) =>
       actionable(f, options)
     );
@@ -414,6 +444,7 @@ function inspectText(text, options = {}) {
           ...options,
           structured: undefined,
           findings: [],
+          detectors: [],
           properNames: undefined,
           documentEntities: options.properNames
             ? false
@@ -449,10 +480,12 @@ function inspectText(text, options = {}) {
     knownSecrets: (options.knownSecrets ?? []).map(
       (value) => projectText(value).text
     ),
-    knownPersonal: (options.knownPersonal ?? []).map((entry) => ({
-      ...entry,
-      value: projectText(entry.value).text,
-    })),
+    knownPersonal:
+      projectedKnown(options.knownPersonal) ??
+      (options.knownPersonal ?? []).map((entry) => ({
+        ...entry,
+        value: projectText(entry.value).text,
+      })),
     confirmedPersonal: (options.confirmedPersonal ?? []).map((entry) => ({
       ...entry,
       value: projectText(entry.value).text,
@@ -470,9 +503,15 @@ function inspectText(text, options = {}) {
     );
   if (projection.changed) {
     nativeDetect(text, options, emit);
+    detectSync(text, options, emit);
   }
   nativeDetect(projection.text, nativeOptions, mapped);
-  encodedDetect(projection.text, nativeOptions, mapped);
+  detectSync(projection.text, nativeOptions, mapped);
+  encodedDetect(projection.text, nativeOptions, (finding) => {
+    if (builtInEnabled(finding, options)) {
+      mapped(finding);
+    }
+  });
   for (const f of options.findings ?? []) {
     emit(f);
   }
@@ -745,22 +784,28 @@ export function redactResult(
   const parts = [];
   const faked = [];
   const occurrences = [];
+  const jsonRanges = callbackStringRanges(text, options);
   let irreversibleRedactions = 0;
   let outputLength = 0;
   let cursor = 0;
   for (const span of spans) {
     const replacement =
       replacements.get(`${span.start}:${span.end}`) ??
-      renderReplacement(text, span, options, (decoded) =>
-        sanitize(
-          decoded,
-          nestedPlaceholderOptions({
-            ...confirmationOptions(text, findings, options),
-            findings: [],
-            preserveEncoding: true,
-            structured: undefined,
-          })
-        )
+      renderReplacement(
+        text,
+        span,
+        options,
+        (decoded) =>
+          sanitize(
+            decoded,
+            nestedPlaceholderOptions({
+              ...confirmationOptions(text, findings, options),
+              findings: [],
+              preserveEncoding: true,
+              structured: undefined,
+            })
+          ),
+        jsonRanges
       );
     parts.push(text.slice(cursor, span.start), replacement);
     outputLength += span.start - cursor;
@@ -812,6 +857,20 @@ function retainedReplacement(transform, mode, span, replacement) {
   );
 }
 
+function callbackStringRanges(text, options) {
+  return options.structured && hasCustomTransforms(options)
+    ? structuredStringRanges(text)
+    : undefined;
+}
+function isCallbackSpan(callback, span, run, options) {
+  return (
+    typeof callback === 'function' &&
+    span.type !== 'JSON_VALUE' &&
+    (span.type !== 'ENCODED_SENSITIVE' ||
+      (!options.preserveEncoding && run?.encoding !== 'json-content'))
+  );
+}
+
 export async function redactResultAsync(
   text,
   findings,
@@ -821,8 +880,22 @@ export async function redactResultAsync(
   options = initializePlaceholders(options);
   preparePlaceholderText(text, findings, options);
   const replacements = new Map();
+  const jsonRanges = callbackStringRanges(text, options);
   {
     for (const span of validatedSpans(text, findings, options)) {
+      const callback =
+        options.transformations?.[span.type] ?? options.transformation;
+      const run =
+        span.type === 'ENCODED_SENSITIVE'
+          ? encodedValue(text.slice(span.start, span.end))
+          : undefined;
+      if (isCallbackSpan(callback, span, run, options)) {
+        replacements.set(
+          `${span.start}:${span.end}`,
+          await renderCallbackAsync(text, span, callback, jsonRanges)
+        );
+        continue;
+      }
       if (span.type !== 'ENCODED_SENSITIVE') {
         if (hasPlaceholders(options)) {
           replacements.set(
@@ -832,7 +905,6 @@ export async function redactResultAsync(
         }
         continue;
       }
-      const run = encodedValue(text.slice(span.start, span.end));
       if (!run) {
         if (options.preserveEncoding) {
           throw failure('ERR_ENCODING');
@@ -872,6 +944,14 @@ function sanitizeText(text, options = {}, round = 0) {
   options = placeholderKnown(initializePlaceholders(options));
   const findings = inspect(text, options);
   const result = redactResult(text, findings, options);
+  if (
+    hasCustomTransforms(options) &&
+    inspect(result.text, { ...options, findings: [] }).some((f) =>
+      actionable(f, options)
+    )
+  ) {
+    throw failure('ERR_RESIDUAL');
+  }
   const sanitized = redactResult(text, findings, {
     ...options,
     transformation: undefined,

package/src/stream-worker.js

@@ -1,6 +1,11 @@
 import { parentPort, workerData } from 'node:worker_threads';
 import { sanitizeFileToFile } from './stream.js';
+import { compileKnown } from './compiled-known.js';
 try {
+  const options = workerData.options.sanitizerOptions ?? workerData.options;
+  if (options.knownPersonal) {
+    options.knownPersonal = compileKnown(options.knownPersonal);
+  }
   parentPort.postMessage({
     value: await sanitizeFileToFile(
       workerData.source,

package/src/stream.js

@@ -20,7 +20,7 @@ import {
 } from './engines.js';
 import { failure, internalFailure } from './detection.js';
 import { sanitizeJsonlStream } from './jsonl-stream.js';
-import { workerExecArgv } from './worker-options.js';
+import { workerExecArgv, hasWorkerCallbacks } from './worker-options.js';
 import { streamIdentity } from './stream-identity.js';
 import { workerResult } from './file-worker-result.js';
 import {
@@ -439,11 +439,7 @@ export async function sanitizeFileToFile(
   }
 }
 export async function sanitizeFileBounded(source, target, options = {}) {
-  if (
-    options.worker === false ||
-    options.sanitizer ||
-    hasPlaceholders(options.sanitizerOptions ?? options)
-  ) {
+  if (useCoordinator(options)) {
     return sanitizeFileToFile(source, target, options);
   }
   const workerHeapMb = options.workerHeapMb ?? 256,
@@ -484,3 +480,12 @@ export async function sanitizeFileBounded(source, target, options = {}) {
     await rm(staging, { recursive: true, force: true });
   }
 }
+function useCoordinator(options) {
+  const policy = options.sanitizerOptions ?? options;
+  return (
+    options.worker === false ||
+    options.sanitizer ||
+    hasWorkerCallbacks(policy) ||
+    hasPlaceholders(policy)
+  );
+}

package/src/strict.js

@@ -0,0 +1,66 @@
+import { collectMatches } from './detection.js';
+import { months } from './personal-context.js';
+
+const boundary = '[\\p{L}\\p{N}_]';
+const attributes =
+  /(?<![\p{L}\p{N}_])(?:женщин[аыуеой]{1,2}|мужчин[аыуеой]{1,2}|замужем|женат(?:а|ы)?|(?:один|одна|двое|трое|четверо|пятеро|\d{1,2})[ \t]+(?:детей|реб[её]нок|реб[её]нка)|русск(?:ая|ий|ого|ой|ие)|татар(?:ин|ка|ы)|православн(?:ый|ая|ого|ой)|мусульман(?:ин|ка)|инвалидност[ьи](?:[ \t]+\d[ \t]+группы)?|беременн(?:а|ая|ость)|судимость(?:[ \t]+погашена)?|член[ \t]+партии|woman|women|man|men|male|female|married|divorced|pregnant|pregnancy|disabled|disability|christian|muslim|jewish|orthodox|russian|hispanic|asian|criminal[ \t]+record|party[ \t]+member|(?:one|two|three|\d{1,2})[ \t]+children)(?![\p{L}\p{N}_])/giu;
+const additionalAttributes =
+  /(?<![\p{L}\p{N}_])(?:диабет(?:ом|а|у|е)?|ВИЧ(?:-положительн(?:ый|ая|ого|ой))?|болен|больна|больны|католик(?:ом|а|и)?|католичк(?:а|ой|и)|буддист(?:ка|кой|ом|а|ы)?|атеист(?:ка|кой|ом|а|ы)?|украин(?:ец|ка|цы|цем|кой)|белорус(?:ка|кой|ом|а|ы)?|несудим(?:а|ы)?|судим(?:а|ы)?|член[ \t]+политической[ \t]+партии|холост(?:а|ы)?|развед[её]н(?:а|ы)?|в[ \t]+разводе|diabetes|diabetic|HIV(?:[ \t-]+positive)?|cancer|buddhist|catholic|atheist|ukrainian|belarusian|previous[ \t]+conviction|member[ \t]+of[ \t]+(?:a[ \t]+)?(?:political[ \t]+)?party|single|unmarried|widowed)(?![\p{L}\p{N}_])/giu;
+
+export function detectStrict(text, emit, options) {
+  if (!options.strict) {
+    return;
+  }
+  for (const pattern of [attributes, additionalAttributes]) {
+    collectMatches(text, pattern, emit, {
+      type: 'PERSONAL_ATTRIBUTE',
+      rule: 'strict-attribute',
+      category: 'personal',
+      confidence: 0.9,
+    });
+  }
+  const date = new RegExp(
+    `(?<!${boundary})(?:\\d{4}-\\d{2}-\\d{2}|\\d{1,2}[./]\\d{1,2}[./]\\d{4}|\\d{1,2}[ \\t]+(?:${months})[ \\t]+\\d{4})(?!${boundary}|[Tt]\\d)`,
+    'giu'
+  );
+  collectMatches(text, date, emit, {
+    type: 'DATE',
+    rule: 'strict-date',
+    category: 'personal',
+    confidence: 0.9,
+  });
+  collectMatches(
+    text,
+    /(?<![\p{L}\p{N}_])\d{1,3}[ \t]+(?:год|года|лет|years?[ \t]+old)(?![\p{L}\p{N}_])/giu,
+    emit,
+    { type: 'AGE', rule: 'strict-age', category: 'personal', confidence: 0.9 },
+    0,
+    (_value, match) =>
+      !/(?:опыт(?:[ \t]+работы)?|стаж|experience|duration|за|через|спустя)[ \t]*$/iu.test(
+        text.slice(Math.max(0, match.index - 64), match.index)
+      )
+  );
+  for (const [type, pattern] of [
+    [
+      'PASSPORT_NUMBER',
+      /(?<![\p{L}\p{N}_])(?:\d{2}[ \t]\d{2}[ \t]\d{6}|\d{10})(?![\p{L}\p{N}_])/gu,
+    ],
+    [
+      'PHONE',
+      /(?<![\p{L}\p{N}_])(?:[78][ \t-]?)?9\d{2}[ \t-]?\d{3}[ \t-]?\d{2}[ \t-]?\d{2}(?![\p{L}\p{N}_])/gu,
+    ],
+    ['POSTAL_CODE', /(?<![\p{L}\p{N}_])\d{6}(?![\p{L}\p{N}_])/gu],
+  ]) {
+    collectMatches(
+      text,
+      pattern,
+      emit,
+      { type, rule: 'strict-number', category: 'personal', confidence: 0.9 },
+      0,
+      (value, match) =>
+        !/^[ \t]*(?:ns|ms|µs|us|seconds?|bytes?|kb|mb|gb)\b/iu.test(
+          text.slice(match.index + value.length)
+        )
+    );
+  }
+}

package/src/structured.js

@@ -1,6 +1,8 @@
 import { failure } from './detection.js';
-import { documentFieldType, normalizeFieldName } from './document-fields.js';
+import { documentFieldType } from './document-fields.js';
 import { hasPlaceholders } from './placeholders.js';
+import { scanField, fieldMatches, builtInEnabled } from './policy.js';
+import { vocabularyField } from './vocabulary.js';
 
 // Session metadata is opt-in at the CLI; library callers choose their own
 // fields. These names do not grant exemptions to provider/known credentials.
@@ -35,28 +37,45 @@ function* lexemes(text) {
   }
 }
 
+const fieldPath = (frame) => [
+  ...frame.path,
+  frame.array ? String(frame.index) : frame.field,
+];
+function stringToken(match, frame, offset) {
+  const key = match.key,
+    value = JSON.parse(match.raw);
+  if (key) {
+    frame.field = value;
+  }
+  return {
+    raw: match.raw,
+    start: offset + match.index,
+    field: key ? '' : frame.field,
+    property: key ? value : undefined,
+    path: fieldPath(frame),
+    key,
+  };
+}
 function tokens(text, offset = 0) {
   const strings = [];
-  const stack = [{ field: '' }];
+  const stack = [{ field: '', path: [], index: 0, array: false }];
   for (const match of lexemes(text)) {
     const raw = match.raw,
       frame = stack.at(-1);
     if (raw[0] === '"') {
-      const key = match.key;
-      const value = JSON.parse(raw);
-      if (key) {
-        frame.field = value;
-      }
-      strings.push({
-        raw,
-        start: offset + match.index,
-        field: key ? '' : frame.field,
-        key,
-      });
+      strings.push(stringToken(match, frame, offset));
     } else if (raw === '{' || raw === '[') {
-      stack.push({ field: frame.field });
+      const path = stack.length === 1 ? [] : fieldPath(frame);
+      stack.push({
+        field: raw === '[' ? frame.field : '',
+        path,
+        array: raw === '[',
+        index: 0,
+      });
     } else if (raw === '}' || raw === ']') {
       stack.pop();
+    } else if (raw === ',' && frame.array) {
+      frame.index++;
     }
   }
   return strings;
@@ -116,29 +135,47 @@ export function structuredBatches(text, options) {
 }
 
 // Schema declarations take priority over metadata/public/fake heuristics.
+function configuredFieldType(field, options) {
+  if (!options.documentEntities || scanField(field, options)) {
+    return undefined;
+  }
+  const personal = vocabularyField(field, options),
+    builtin = documentFieldType(field);
+  if (personal === true) {
+    return 'PERSON';
+  }
+  return personal === false && builtin === 'PERSON' ? undefined : builtin;
+}
 export function structuredFieldFindings(batch, options) {
   return batch.entries.flatMap((entry) => {
-    const type = options.documentEntities
-      ? documentFieldType(entry.field)
-      : undefined;
+    const type = configuredFieldType(entry.field, options);
+    const denied =
+      options.allowedFields !== undefined &&
+      !entry.key &&
+      !fieldMatches(
+        options.allowedFields,
+        entry.field,
+        entry.path,
+        options.exactFields
+      );
     if (
       entry.key ||
-      !type ||
-      entry.raw.length <= 2 ||
+      (!type && !denied) ||
+      (!denied && entry.raw.length <= 2) ||
       JSON.parse(entry.raw) === '[REDACTED]'
     ) {
       return [];
     }
     return [
       {
-        start: entry.start + 1,
-        end: entry.start + entry.raw.length - 1,
-        type,
+        start: entry.start + (denied ? 0 : 1),
+        end: entry.start + entry.raw.length - (denied ? 0 : 1),
+        type: denied ? 'JSON_VALUE' : type,
         category: 'personal',
         rule: 'known-personal',
         confidence: 1,
       },
-    ];
+    ].filter((finding) => denied || builtInEnabled(finding, options));
   });
 }
 
@@ -152,8 +189,19 @@ export function structuredValueRanges(text, options) {
       }))
   );
 }
+export function structuredStringRanges(text) {
+  return [...lexemes(text)]
+    .filter((entry) => entry.raw[0] === '"')
+    .map((entry) => ({
+      start: entry.index + 1,
+      end: entry.index + entry.raw.length - 1,
+    }));
+}
 
 function weakStructuralFinding(finding, entry, batchText) {
+  if (finding.rule.startsWith('strict-')) {
+    return true;
+  }
   if (
     ['known-secret', 'known-personal', 'confirmed-personal'].includes(
       finding.rule
@@ -227,6 +275,13 @@ export function preserveIdentifierKeys(text, findings, options) {
   });
 }
 function preserveStructuredEntry(entry, finding, text, options) {
+  if (
+    !entry.key &&
+    scanField(entry.field, options) &&
+    (finding.rule === 'label' || finding.rule.startsWith('catalog/'))
+  ) {
+    return finding.category === 'personal';
+  }
   if (options.properNames && ordinaryPersonalKey(entry, finding, options)) {
     return true;
   }
@@ -407,32 +462,41 @@ export function declaredFieldFindings(text, options) {
   if (!options.sensitiveFields?.length) {
     return [];
   }
-  const fields = new Set(options.sensitiveFields.map(normalizeFieldName));
   const findings = [];
-  for (const token of lexemes(text)) {
-    if (
-      token.index < (findings.at(-1)?.end ?? 0) ||
-      !token.key ||
-      !fields.has(normalizeFieldName(JSON.parse(token.raw)))
-    ) {
-      continue;
-    }
-    let start = token.index + token.raw.length;
-    while (/[\s:]/.test(text[start] ?? '') && start < text.length) {
-      start++;
-    }
-    const end = jsonValueEnd(text, start);
-    if (text.slice(start, end) === '"[REDACTED]"') {
-      continue;
+  const records = options.structured === 'jsonl' ? text.split('\n') : [text];
+  let offset = 0;
+  for (const record of records) {
+    for (const token of tokens(record, offset)) {
+      if (
+        token.start < (findings.at(-1)?.end ?? 0) ||
+        !token.key ||
+        !fieldMatches(
+          options.sensitiveFields,
+          token.property,
+          [...token.path.slice(0, -1), token.property],
+          options.exactFields
+        )
+      ) {
+        continue;
+      }
+      let start = token.start + token.raw.length;
+      while (/[\s:]/.test(text[start] ?? '') && start < text.length) {
+        start++;
+      }
+      const end = jsonValueEnd(text, start);
+      if (text.slice(start, end) === '"[REDACTED]"') {
+        continue;
+      }
+      findings.push({
+        start,
+        end,
+        type: 'JSON_VALUE',
+        category: 'personal',
+        rule: 'known-personal',
+        confidence: 1,
+      });
     }
-    findings.push({
-      start,
-      end,
-      type: 'JSON_VALUE',
-      category: 'personal',
-      rule: 'known-personal',
-      confidence: 1,
-    });
+    offset += record.length + 1;
   }
   return findings;
 }

package/src/sync-detectors.js

@@ -0,0 +1,38 @@
+import { failure } from './detection.js';
+
+export function validateDetectors(detectors = []) {
+  if (
+    !Array.isArray(detectors) ||
+    detectors.some((d) => !d || typeof d.detect !== 'function')
+  ) {
+    throw failure('ERR_CONFIG');
+  }
+}
+
+export function detectSync(text, options, emit) {
+  if (options.knownOnly) {
+    return;
+  }
+  for (const detector of options.detectors ?? []) {
+    let findings;
+    try {
+      findings = detector.detect(text);
+    } catch {
+      throw failure('ERR_ENGINE');
+    }
+    if (findings && typeof findings.then === 'function') {
+      // Observe rejections without allowing async work to publish output.
+      Promise.resolve(findings).catch(() => {});
+      throw failure('ERR_CONFIG');
+    }
+    if (!Array.isArray(findings)) {
+      throw failure('ERR_ENGINE');
+    }
+    if (findings.length > (options.maxFindings ?? 100000)) {
+      throw failure('ERR_LIMIT');
+    }
+    for (const finding of findings) {
+      emit(finding);
+    }
+  }
+}

package/src/sync-sanitizer.js

@@ -0,0 +1,16 @@
+import {
+  inspect,
+  sanitize,
+  validateOptions,
+  applyProfile,
+} from './sanitizer.js';
+
+/** A reusable synchronous engine with the same verified publication boundary. */
+export function createSyncSanitizer(options = {}) {
+  options = applyProfile(options);
+  validateOptions(options);
+  return Object.freeze({
+    inspect: (text) => inspect(text, options),
+    sanitize: (text) => sanitize(text, options),
+  });
+}