@link-foundation/sensitive-data-sanitizer 0.18.0 → 0.19.0

package/src/transforms.js

@@ -1,9 +1,10 @@
 import { realisticFake } from './fake.js';
 import { createHmac } from 'node:crypto';
 import { failure, REDACTED } from './detection.js';
-import { encodedValue, encodeRun } from './encoded.js';
+import { encodedValue, encodeRun, decodedRuns } from './encoded.js';
 import { isIdentityType } from './identity.js';
 import { validatePlaceholder, placeholderReplacement } from './placeholders.js';
+import { projectText } from './projection.js';
 
 const modes = new Set([
   'redact',
@@ -58,6 +59,9 @@ function validPools(pools) {
   );
 }
 function validateTransform(transform) {
+  if (typeof transform === 'function') {
+    return;
+  }
   if (!transform || !modes.has(transform.mode)) {
     throw failure('ERR_CONFIG');
   }
@@ -148,6 +152,14 @@ function masked(value, finding, transform) {
     : `${chars.slice(0, start).join('')}${transform.marker ?? '*'.repeat(chars.length - start - end)}${end ? chars.slice(-end).join('') : ''}`;
 }
 function transformed(value, finding, transform, options) {
+  if (typeof transform === 'function') {
+    const result = transform(Object.freeze({ ...finding, value }));
+    if (result && typeof result.then === 'function') {
+      Promise.resolve(result).catch(() => {});
+      throw failure('ERR_CONFIG');
+    }
+    return verifiedCallback(value, result);
+  }
   if (!transform || transform.mode === 'redact') {
     return REDACTED;
   }
@@ -182,7 +194,13 @@ function transformed(value, finding, transform, options) {
   const lower = Math.floor(number / transform.size) * transform.size;
   return `${lower}–${lower + transform.size - 1}`;
 }
-export function renderReplacement(text, finding, options, encodedSanitize) {
+export function renderReplacement(
+  text,
+  finding,
+  options,
+  encodedSanitize,
+  jsonRanges
+) {
   if (finding.type === 'JSON_VALUE') {
     return JSON.stringify(REDACTED);
   }
@@ -205,22 +223,82 @@ export function renderReplacement(text, finding, options, encodedSanitize) {
       throw failure('ERR_ENCODING');
     }
   }
-  return renderValue(text, finding, transform, options);
+  return renderValue(text, finding, transform, options, jsonRanges);
 }
-function renderValue(text, finding, transform, options) {
+function replacementInput(
+  text,
+  finding,
+  jsonRanges = [],
+  decodeQuoted = false
+) {
   let raw = text.slice(finding.start, finding.end);
+  const inJsonString = jsonRanges.some(
+    (range) => range.start <= finding.start && range.end >= finding.end
+  );
   const jsonQuoted =
-    text[finding.start - 1] === '"' && text[finding.end] === '"';
-  if (jsonQuoted) {
+    (text[finding.start - 1] === '"' && text[finding.end] === '"') ||
+    inJsonString;
+  // Quotes can also surround ordinary decoded prose containing literal tabs.
+  // Decode escapes only when the structured walker confirmed a JSON string.
+  if (inJsonString || (decodeQuoted && jsonQuoted)) {
     try {
       raw = JSON.parse(`"${raw}"`);
     } catch {
-      return REDACTED;
+      if (decodeQuoted) {
+        return { invalid: true };
+      }
+      throw failure('ERR_JSON');
     }
   }
+  return { raw, jsonQuoted };
+}
+function renderValue(text, finding, transform, options, jsonRanges) {
+  const { raw, jsonQuoted, invalid } = replacementInput(
+    text,
+    finding,
+    typeof transform === 'function' ? jsonRanges : undefined,
+    typeof transform !== 'function'
+  );
+  if (invalid) {
+    return REDACTED;
+  }
   const result =
     transform?.mode === 'placeholder'
       ? placeholderReplacement(raw, finding, transform, options)
       : transformed(raw, finding, transform, options);
   return jsonQuoted ? JSON.stringify(result).slice(1, -1) : result;
 }
+
+export const hasCustomTransforms = (options) =>
+  [
+    options.transformation,
+    ...Object.values(options.transformations ?? {}),
+  ].some((t) => typeof t === 'function');
+function verifiedCallback(value, result) {
+  if (typeof result !== 'string') {
+    throw failure('ERR_CONFIG');
+  }
+  const normalize = (s) => projectText(s).text.toLowerCase();
+  const originals = [
+    value,
+    ...Array.from(decodedRuns(value), (run) => run.text),
+  ]
+    .map(normalize)
+    .filter(Boolean);
+  if (
+    [result, ...Array.from(decodedRuns(result), (run) => run.text)].some(
+      (view) => originals.some((original) => normalize(view).includes(original))
+    )
+  ) {
+    throw failure('ERR_RESIDUAL');
+  }
+  return result;
+}
+export async function renderCallbackAsync(text, finding, callback, jsonRanges) {
+  const { raw, jsonQuoted } = replacementInput(text, finding, jsonRanges);
+  const result = verifiedCallback(
+    raw,
+    await callback(Object.freeze({ ...finding, value: raw }))
+  );
+  return jsonQuoted ? JSON.stringify(result).slice(1, -1) : result;
+}

package/src/vocabulary.js

@@ -0,0 +1,217 @@
+import { failure, collectMatches } from './detection.js';
+import { wordPattern } from './personal-variants.js';
+import { normalizeFieldName } from './document-fields.js';
+
+export const defaultVocabulary = Object.freeze(
+  Object.fromEntries(
+    Object.entries({
+      roles: [
+        'менеджер',
+        'директор',
+        'руководитель',
+        'инженер',
+        'аналитик',
+        'разработчик',
+        'manager',
+        'director',
+        'engineer',
+        'developer',
+        'senior',
+        'qa',
+      ],
+      organizationCues: [
+        'компания',
+        'компании',
+        'холдинг',
+        'банк',
+        'группа компаний',
+        'стартап',
+      ],
+      educationCues: [
+        'образование',
+        'вуз',
+        'университет',
+        'школа',
+        'обучение',
+        'колледж',
+        'курсы',
+        'диплом',
+        'бакалавр',
+        'graduated from',
+      ],
+      personalLabels: ['соискатель', 'сотрудник', 'кандидат'],
+      personalFields: ['firstname', 'lastname', 'middlename', 'patronymic'],
+    }).map(([key, values]) => [key, Object.freeze(values)])
+  )
+);
+const cache = new Map();
+const boundary = '[\\p{L}\\p{N}_]';
+const name =
+  '[\\p{Lu}][\\p{L}\\p{M}.-]{1,47}(?:[ \\t]+[\\p{Lu}][\\p{L}\\p{M}.-]{1,47}){0,3}';
+const casedName = new RegExp(`^${name}`, 'u');
+function normalizedLists(vocabulary) {
+  if (vocabulary === undefined) {
+    return undefined;
+  }
+  if (
+    !vocabulary ||
+    typeof vocabulary !== 'object' ||
+    Array.isArray(vocabulary) ||
+    Object.keys(vocabulary).some(
+      (key) => !Object.hasOwn(defaultVocabulary, key)
+    )
+  ) {
+    throw failure('ERR_CONFIG');
+  }
+  const lists = {};
+  for (const [key, defaults] of Object.entries(defaultVocabulary)) {
+    const input = vocabulary[key];
+    if (input === undefined) {
+      lists[key] = defaults;
+      continue;
+    }
+    lists[key] = normalizedList(input, defaults);
+  }
+  return lists;
+}
+function normalizedList(input, defaults) {
+  const values = Array.isArray(input) ? input : input?.values;
+  const invalid =
+    !Array.isArray(input) &&
+    (input?.replace !== true ||
+      Object.keys(input).some((k) => !['values', 'replace'].includes(k)));
+  if (
+    !Array.isArray(values) ||
+    values.length > 256 ||
+    invalid ||
+    values.some(
+      (s) =>
+        typeof s !== 'string' ||
+        !s.trim() ||
+        s.length > 128 ||
+        /[\r\n\0]/.test(s)
+    )
+  ) {
+    throw failure('ERR_CONFIG');
+  }
+  return [
+    ...new Set(
+      [...(input.replace ? [] : defaults), ...values].map((s) =>
+        s.normalize('NFC').trim().toLowerCase()
+      )
+    ),
+  ];
+}
+export function compiledVocabulary(vocabulary) {
+  if (vocabulary === undefined) {
+    return undefined;
+  }
+  const lists = normalizedLists(vocabulary);
+  const key = JSON.stringify(lists);
+  if (cache.has(key)) {
+    return cache.get(key);
+  }
+  const alternatives = (values) =>
+    values.length
+      ? values
+          .map((s) =>
+            s
+              .split(/\s+/u)
+              .map((w) => wordPattern(w, 'ORGANIZATION'))
+              .join('[ \\t]+')
+          )
+          .sort((a, b) => b.length - a.length)
+          .join('|')
+      : '(?!)';
+  const result = { lists, patterns: [] };
+  for (const [list, type] of [
+    ['organizationCues', 'ORGANIZATION'],
+    ['educationCues', 'EDUCATION'],
+    ['personalLabels', 'PERSON'],
+  ]) {
+    const cue = alternatives(lists[list]);
+    // Fold cues here and validate the captured name's casing at emission.
+    const pattern = new RegExp(
+      `(?<!${boundary})(${cue})[ \\t]*(?::|[-–—])?[ \\t]+(${name})(?!${boundary})`,
+      'giu'
+    );
+    result.patterns.push({ type, pattern });
+  }
+  result.roles = new RegExp(
+    `^(?:${alternatives(lists.roles)})(?!\\p{L})`,
+    'iu'
+  );
+  result.rolePattern = new RegExp(
+    `(?<!${boundary})(?:${alternatives(lists.roles)})[ \\t]+(?:в[ \\t]+)?(${name})(?!${boundary})`,
+    'giu'
+  );
+  result.reverseRolePattern = new RegExp(
+    `(?<!${boundary})(${name})[ \\t]*[:,–—-][ \\t]*(?:${alternatives(lists.roles)})(?!${boundary})`,
+    'giu'
+  );
+  result.fields = new Set(lists.personalFields.map(normalizeFieldName));
+  if (cache.size >= 64) {
+    cache.clear();
+  }
+  cache.set(key, result);
+  return result;
+}
+export function vocabularyField(field, options) {
+  const vocabulary = compiledVocabulary(options.vocabulary);
+  if (!vocabulary) {
+    return undefined;
+  }
+  return vocabulary.fields.has(normalizeFieldName(field))
+    ? true
+    : options.vocabulary.personalFields?.replace
+      ? false
+      : undefined;
+}
+export function detectVocabulary(text, emit, options) {
+  const vocabulary = compiledVocabulary(options.vocabulary);
+  if (!vocabulary) {
+    return;
+  }
+  const emitName = (finding) => {
+    const value = text.slice(finding.start, finding.end).match(casedName)?.[0];
+    if (value && value !== 'REDACTED') {
+      emit({ ...finding, end: finding.start + value.length });
+    }
+  };
+  for (const { type, pattern } of vocabulary.patterns) {
+    collectMatches(
+      text,
+      pattern,
+      emitName,
+      { type, category: 'personal', rule: 'vocabulary-cue', confidence: 0.85 },
+      2,
+      (value) => /^\p{Lu}/u.test(value) && value !== 'REDACTED'
+    );
+  }
+  collectMatches(
+    text,
+    vocabulary.rolePattern,
+    emitName,
+    {
+      type: 'ORGANIZATION',
+      category: 'personal',
+      rule: 'vocabulary-role',
+      confidence: 0.8,
+    },
+    1,
+    (value) => /^\p{Lu}/u.test(value) && value !== 'REDACTED'
+  );
+  collectMatches(
+    text,
+    vocabulary.reverseRolePattern,
+    emitName,
+    {
+      type: 'ORGANIZATION',
+      category: 'personal',
+      rule: 'vocabulary-role',
+      confidence: 0.8,
+    },
+    1,
+    (value) => /^\p{Lu}/u.test(value) && value !== 'REDACTED'
+  );
+}

package/src/worker-options.js

@@ -1,4 +1,16 @@
 // Node normally inherits flags while excluding process/V8-only options.
+export function hasWorkerCallbacks(options) {
+  return Boolean(
+    options.debug ||
+    options.verifyPublic ||
+    options.detectors?.length ||
+    [
+      options.transformation,
+      ...Object.values(options.transformations ?? {}),
+    ].some((value) => typeof value === 'function')
+  );
+}
+
 // An explicit execArgv bypasses that filtering, so only override it when
 // parent stdin parsing would reject the file worker's module URL.
 export function workerExecArgv() {