@link-foundation/sensitive-data-sanitizer 0.18.0 → 0.19.0

package/src/engines.js

@@ -16,6 +16,8 @@ import {
 import { failure, internalFailure } from './detection.js';
 import { projectText } from './projection.js';
 import { decodedRuns } from './encoded.js';
+import { hasCustomTransforms } from './transforms.js';
+import { builtInEnabled } from './policy.js';
 import {
   initializePlaceholders,
   placeholderKnown,
@@ -70,7 +72,8 @@ async function secretlintDetect(text) {
   }));
 }
 
-async function detect(engine, text, limit) {
+const secretlintEngine = { detect: secretlintDetect };
+async function detect(engine, text, limit, options) {
   try {
     const found = await engine.detect(text);
     if (!Array.isArray(found)) {
@@ -79,7 +82,12 @@ async function detect(engine, text, limit) {
     if (found.length > limit) {
       throw failure('ERR_LIMIT');
     }
-    return found.map((finding) => sanitizeFinding(finding, text.length));
+    return found
+      .map((finding) => sanitizeFinding(finding, text.length))
+      .filter(
+        (finding) =>
+          engine !== secretlintEngine || builtInEnabled(finding, options)
+      );
   } catch {
     throw failure('ERR_ENGINE');
   }
@@ -88,7 +96,7 @@ async function detect(engine, text, limit) {
 async function engineFindings(engine, projection, options, emit) {
   const limit = options.maxFindings ?? 100000;
   if (projection.changed) {
-    for (const f of await detect(engine, projection.text, limit)) {
+    for (const f of await detect(engine, projection.text, limit, options)) {
       emit({
         ...f,
         start: projection.starts[f.start],
@@ -106,7 +114,7 @@ async function engineFindings(engine, projection, options, emit) {
     const view = projectText(run.text).text;
     const decoded = markFake(
       view,
-      await detect(engine, view, limit),
+      await detect(engine, view, limit, options),
       options
     ).filter((f) => actionable(f, options));
     if (!decoded.length) {
@@ -124,6 +132,28 @@ async function engineFindings(engine, projection, options, emit) {
   }
 }
 
+async function collectEngineFindings(text, engines, options, emit) {
+  const projection = engines.length ? projectText(text) : undefined;
+  for (const engine of engines) {
+    if (engine !== secretlintEngine && !options.detectors?.includes(engine)) {
+      continue;
+    }
+    for (const finding of await detect(
+      engine,
+      text,
+      options.maxFindings ?? 100000,
+      options
+    )) {
+      emit(finding);
+    }
+    await engineFindings(engine, projection, options, (finding) => {
+      if (builtInEnabled(finding, options)) {
+        emit(finding);
+      }
+    });
+  }
+}
+
 async function analyzeStructured(
   text,
   initial,
@@ -142,6 +172,7 @@ async function analyzeStructured(
         true,
         {
           ...options,
+          detectors: [],
           properNames: undefined,
           documentEntities: options.properNames
             ? false
@@ -199,9 +230,7 @@ export function createSanitizer(options = {}) {
     }
   }
   const engines =
-    options.secretlint === false || options.knownOnly
-      ? []
-      : [{ detect: secretlintDetect }];
+    options.secretlint === false || options.knownOnly ? [] : [secretlintEngine];
   if (!options.knownOnly) {
     engines.push(...(options.detectors ?? []));
   }
@@ -230,20 +259,15 @@ export function createSanitizer(options = {}) {
     }
     // Native inspection performs its own normalization. Additional views are
     // needed here only when a required external engine is enabled.
-    const projection = engines.length ? projectText(text) : undefined;
-    for (const engine of engines) {
-      for (const finding of await detect(
-        engine,
-        text,
-        options.maxFindings ?? 100000
-      )) {
-        emit(finding);
-      }
-      await engineFindings(engine, projection, options, emit);
-    }
+    await collectEngineFindings(text, engines, options, emit);
     const detected =
       nativeFindings === undefined
-        ? inspect(text, { ...options, structured: undefined, findings })
+        ? inspect(text, {
+            ...options,
+            structured: undefined,
+            detectors: [],
+            findings,
+          })
         : reviewFindings(text, [...nativeFindings, ...findings], options);
     return options.verifyPublic
       ? verifyFindings(text, detected, options.verifyPublic)
@@ -297,16 +321,17 @@ export function createSanitizer(options = {}) {
       findings,
       fullRedaction ? fullOptions : options,
       (decoded, recursiveOptions) =>
-        depth < 2
-          ? sanitizeText(
-              decoded,
-              depth + 1,
-              fullRedaction,
-              true,
-              recursiveOptions
-            )
-          : Promise.resolve({ text: '[REDACTED]' })
+        sanitizeNested(
+          sanitizeText,
+          decoded,
+          depth,
+          fullRedaction,
+          recursiveOptions
+        )
     );
+    if (await customResidual(result.text, options, plain, analyze)) {
+      throw failure('ERR_RESIDUAL');
+    }
     const sanitized = fullRedaction
       ? result.text
       : (
@@ -315,9 +340,13 @@ export function createSanitizer(options = {}) {
             findings,
             fullOptions,
             (decoded, recursiveOptions) =>
-              depth < 2
-                ? sanitizeText(decoded, depth + 1, true, true, recursiveOptions)
-                : Promise.resolve({ text: '[REDACTED]' })
+              sanitizeNested(
+                sanitizeText,
+                decoded,
+                depth,
+                true,
+                recursiveOptions
+              )
           )
         ).text;
     // Verification at the publication boundary runs every enabled engine.
@@ -330,15 +359,21 @@ export function createSanitizer(options = {}) {
         sanitizeText(result.text, depth, fullRedaction, plain, next, nextRound)
       );
     }
-    if (options.structured && !plain) {
-      verifyStructured(result.text, options);
-    }
+    verifyStructuredResult(result.text, options, plain);
     if (await placeholderResidual(result.text, options, plain, analyze)) {
       throw failure('ERR_RESIDUAL');
     }
     return { ...result, findings: auditFaked(findings, options, result) };
   }
 }
+async function customResidual(text, options, plain, analyze) {
+  return (
+    hasCustomTransforms(options) &&
+    (await analyze(text, [], plain, options)).some((f) =>
+      actionable(f, options)
+    )
+  );
+}
 
 function guarded(callback, cleanup = () => {}) {
   return async (...args) => {
@@ -409,3 +444,15 @@ async function verifyFindings(text, findings, verifier) {
   }
   return retained;
 }
+
+function sanitizeNested(sanitizeText, decoded, depth, fullRedaction, options) {
+  return depth < 2
+    ? sanitizeText(decoded, depth + 1, fullRedaction, true, options)
+    : Promise.resolve({ text: '[REDACTED]' });
+}
+
+function verifyStructuredResult(text, options, plain) {
+  if (options.structured && !plain) {
+    verifyStructured(text, options);
+  }
+}

package/src/index.d.ts

@@ -21,7 +21,20 @@ export interface Finding {
 export interface PersonalValue {
   type: string;
   value: string;
+  properCase?: boolean;
 }
+export type VocabularyList =
+  readonly string[] | { values: readonly string[]; replace: true };
+export type Vocabulary = Partial<
+  Record<
+    | 'roles'
+    | 'organizationCues'
+    | 'educationCues'
+    | 'personalLabels'
+    | 'personalFields',
+    VocabularyList
+  >
+>;
 export interface PublicEntity {
   type: 'PERSON' | 'ORGANIZATION' | 'EDUCATION' | 'LOCATION' | 'EMAIL';
   value: string;
@@ -38,18 +51,28 @@ export interface Detector {
 export interface SanitizerOptions {
   /** Preserve all personal/account data; credential findings still redact. */
   credentialsOnly?: boolean;
-  /** Replace only knownPersonal/knownSecrets and reused map values; skip automatic detectors. */
+  /** Use explicit findings, knownPersonal/knownSecrets and reused map values; skip automatic detectors. */
   knownOnly?: boolean;
   profile?: 'publication' | 'documents';
   /** Opt in to bounded employer, school, city/region heuristics. */
   documentEntities?: boolean;
+  /** Opt-in aggressive stored-log attributes, ages, dates and numeric shapes. */
+  strict?: boolean;
+  vocabulary?: Vocabulary;
+  /** Stable ids from builtInRules. Unknown ids fail with ERR_CONFIG. */
+  rules?: Record<string, boolean>;
+  types?: Record<string, boolean>;
+  /** Required to disable any built-in credential rule or type. */
+  allowCredentialDisable?: boolean;
+  /** Treat a built-in property as ordinary scanned text. */
+  fields?: Record<string, 'scan'>;
   /** Aggressive structural proper names in prose values; opt-in, off by default. */
   properNames?: 'capitalized';
   /** Unicode Script aliases that activate prose detection; default ['Cyrillic']. */
   scripts?: string[];
   confirmedPersonal?: Array<{ value: string; type: string }>;
   knownSecrets?: string[];
-  knownPersonal?: PersonalValue[];
+  knownPersonal?: readonly PersonalValue[];
   publicEntities?: PublicEntity[];
   /** Built-in exact public entities/resolvers and role mail on known domains. */
   publicKnowledge?: boolean;
@@ -67,8 +90,12 @@ export interface SanitizerOptions {
   structured?: 'json' | 'jsonl';
   /** Property names whose metadata heuristics may be suppressed. */
   structuralFields?: string[];
-  /** Property names normalized across case/underscores/hyphens; entire JSON/JSONL values redact. */
+  /** Normalized property names or dotted paths with *; entire JSON/JSONL values redact. */
   sensitiveFields?: string[];
+  /** Match sensitiveFields/allowedFields with exact case and punctuation. */
+  exactFields?: boolean;
+  /** Only these string leaves remain eligible to keep; listed strings are still scanned. */
+  allowedFields?: string[];
   /** Personal replacement threshold, default 0.5. Inspect retains lower scores. */
   minConfidence?: number;
   /** Alias for minConfidence; specifying both requires equal values. */
@@ -84,7 +111,7 @@ export interface SanitizerOptions {
   maxInputLength?: number;
   maxFindings?: number;
   debug?: (event: DebugEvent) => void;
-  /** Used by createSanitizer; synchronous functions use native rules only. */
+  /** Required callbacks; synchronous functions reject Promise-returning detectors. */
   detectors?: Detector[];
   /** Required by default in createSanitizer. */
   secretlint?: boolean;
@@ -111,6 +138,22 @@ export interface Sanitizer {
   inspect(text: string): Promise<Finding[]>;
   sanitize(text: string): Promise<SanitizedText>;
 }
+export interface SyncSanitizer {
+  inspect(text: string): Finding[];
+  sanitize(text: string): SanitizedText;
+}
+export declare function createSyncSanitizer(
+  options?: SanitizerOptions
+): SyncSanitizer;
+/** Copies and freezes caller values, merging lists and caching matching/projection. */
+export declare function compileKnown(
+  ...lists: Array<readonly PersonalValue[]>
+): readonly Readonly<PersonalValue>[];
+export declare const builtInRules: readonly string[];
+export declare const defaultVocabulary: Readonly<
+  Record<keyof Vocabulary, readonly string[]>
+>;
+export declare const documentVocabulary: readonly (readonly string[])[];
 export interface JsonlSanitizer extends Sanitizer {
   sanitizeJsonl(text: string): Promise<SanitizedText>;
 }
@@ -229,7 +272,12 @@ export declare function auditGitHistory(
     maxTotalBytes?: number;
   }
 ): Promise<HistoryAudit>;
+/** Callback receives private matched data. Sync APIs require an immediate string. */
+export type TransformationCallback = (
+  finding: Readonly<Finding & { value: string }>
+) => string | Promise<string>;
 export type Transformation =
+  | TransformationCallback
   | { mode: 'redact' | 'hive-mask' }
   | {
       mode: 'placeholder';

package/src/index.js

@@ -2,6 +2,11 @@
 
 export { inspect, sanitize, redact } from './sanitizer.js';
 export { createSanitizer } from './engines.js';
+export { compileKnown } from './compiled-known.js';
+export { builtInRules } from './rule-catalog.js';
+export { defaultVocabulary } from './vocabulary.js';
+export { documentVocabulary } from './document-vocabulary.js';
+export { createSyncSanitizer } from './sync-sanitizer.js';
 export { REDACTED, entropy, luhn } from './detection.js';
 export { codePointRange } from './projection.js';
 export {

package/src/jsonl-stream.js

@@ -3,7 +3,7 @@ import { Worker } from 'node:worker_threads';
 import { URL } from 'node:url';
 import { failure } from './detection.js';
 import { encodingFailure } from './utf8.js';
-import { workerExecArgv } from './worker-options.js';
+import { workerExecArgv, hasWorkerCallbacks } from './worker-options.js';
 import { prepareStreamEngine } from './engines.js';
 import { inspect } from './sanitizer.js';
 import { hasPlaceholders } from './placeholders.js';
@@ -129,10 +129,7 @@ function workerCount(options, config) {
     return 1;
   }
   const callbacks =
-    options.sanitizer ||
-    config.sanitizerOptions.debug ||
-    config.sanitizerOptions.verifyPublic ||
-    config.sanitizerOptions.detectors?.length;
+    options.sanitizer || hasWorkerCallbacks(config.sanitizerOptions);
   const count =
     options.workers ?? (callbacks || typeof Deno !== 'undefined' ? 1 : 2);
   if (

package/src/ner.d.ts

@@ -0,0 +1,5 @@
+import type { Finding } from './index.d.ts';
+export declare function createNerDetector(options?: {
+  languages?: Array<'ru' | 'en'>;
+  debug?: (event: { event: 'offline-ner'; count: number }) => void;
+}): Promise<{ id: string; detect(text: string): Finding[] }>;

package/src/ner.js

@@ -0,0 +1,96 @@
+// Optional entry: no core module imports NLP.js or this recognizer.
+import { firstNames } from './names.js';
+import { documentVocabulary } from './document-vocabulary.js';
+
+const boundary = '[\\p{L}\\p{M}\\p{N}_]';
+const proper = "\\p{Lu}[\\p{L}\\p{M}’'-]{1,47}";
+const name = `${proper}(?:[ \\t]+${proper}){0,3}`;
+const escape = (s) => s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+const blocked =
+  /^(?:Kubernetes|PostgreSQL|Python|JavaScript|TypeScript|Node|Docker|Redis|Linux|Monday|Tuesday|Wednesday|Thursday|Friday|Saturday|Sunday|REDACTED)$/iu;
+const configError = () =>
+  Object.assign(new Error('Invalid offline recognizer configuration.'), {
+    code: 'ERR_CONFIG',
+  });
+
+function rulesFor(language) {
+  const names = firstNames[language]
+    .split(' ')
+    .filter((s) => !/^(?:May|Will|Mark|Rose)$/u.test(s));
+  const places = documentVocabulary
+    .filter((row) => row[0] === language)
+    .flatMap((row) => row[3].split('|'));
+  const rules = [
+    ['PERSON', `(?:${names.map(escape).join('|')})(?:[ \\t]+${proper})?`],
+    ['LOCATION', `(?:${places.map(escape).join('|') || '(?!)'})`],
+  ];
+  if (language === 'ru') {
+    rules.push(
+      ['ORGANIZATION', `(?<=(?:[Рр]аботал[аи]?|[Сс]лужил[аи]?) в )${name}`],
+      [
+        'ORGANIZATION',
+        `${name}(?=:[ \\t]*(?:менеджер|директор|разработчик|инженер|аналитик|маркетолог)(?!\\p{L}))`,
+      ],
+      ['EDUCATION', '(?:МГУ|СПбГУ|МГТУ|РГГУ|ВШЭ|МФТИ|ИТМО)'],
+      ['EDUCATION', `${name}[ \\t]+(?:университет|институт|академия)`],
+      ['LOCATION', `(?<=(?:[Жж]иву|[Пп]роживаю) в )${name}`]
+    );
+  } else {
+    rules.push(
+      ['ORGANIZATION', `(?<=(?:[Aa]t|[Jj]oined|[Ee]mployer:) )${name}`],
+      ['EDUCATION', `(?<=(?:[Gg]raduated from|[Ss]tudied at) )${name}`],
+      [
+        'EDUCATION',
+        '(?:MIT|MSU|UCLA|UCL|NYU|Harvard|Stanford|Oxford|Cambridge)',
+      ],
+      ['EDUCATION', `${name}[ \\t]+(?:University|College|Institute)`],
+      ['LOCATION', `(?<=(?:[Ll]ive in|[Ll]ocated in|[Bb]ased in) )${name}`]
+    );
+  }
+  return rules.map(([type, source]) => ({
+    name: type,
+    type: 'regex',
+    rules: [new RegExp(`(?<!${boundary})(?:${source})(?!${boundary})`, 'gu')],
+  }));
+}
+
+/** Prepare an offline recognizer with synchronous detection. */
+export async function createNerDetector(options = {}) {
+  const languages = options.languages ?? ['ru', 'en'];
+  if (
+    !Array.isArray(languages) ||
+    !languages.length ||
+    languages.length > 2 ||
+    languages.some((s) => !['ru', 'en'].includes(s)) ||
+    Object.keys(options).some((key) => !['languages', 'debug'].includes(key)) ||
+    (options.debug !== undefined && typeof options.debug !== 'function')
+  ) {
+    throw configError();
+  }
+  const { ExtractorRegex } = await import('@nlpjs/ner');
+  const extractor = new ExtractorRegex();
+  const nerRules = [...new Set(languages)].flatMap(rulesFor);
+  return Object.freeze({
+    id: 'offline-ner',
+    detect(text) {
+      if (typeof text !== 'string' || text.length > 1024 * 1024) {
+        throw Object.assign(new Error('Offline recognizer input limit.'), {
+          code: 'ERR_LIMIT',
+        });
+      }
+      const findings = extractor
+        .extract({ text, nerRules })
+        .edges.filter((edge) => !blocked.test(edge.sourceText))
+        .map((edge) => ({
+          start: edge.start,
+          end: edge.end + 1,
+          type: edge.entity,
+          category: 'personal',
+          rule: 'ner/offline',
+          confidence: 0.8,
+        }));
+      options.debug?.({ event: 'offline-ner', count: findings.length });
+      return findings;
+    },
+  });
+}

package/src/personal-variants.js

@@ -1,5 +1,6 @@
 import { escapePattern, failure } from './detection.js';
 import { russianNameRoles } from './russian-names.js';
+import { compiledPatterns } from './compiled-known.js';
 
 const boundary = '[\\p{L}\\p{M}\\p{N}_]';
 export function organizationCore(value) {
@@ -143,6 +144,22 @@ export function personalPatterns(value, type) {
   return [...patterns];
 }
 export function personalRegex(value, type) {
+  if (type === 'URL') {
+    let core = value.replace(/^https?:\/\//iu, '');
+    const suffix = core.search(/[?#]/u);
+    if (suffix >= 0) {
+      core = core.slice(0, suffix);
+    }
+    let end = core.length;
+    while (end > 0 && core[end - 1] === '/') {
+      end--;
+    }
+    core = core.slice(0, end);
+    return new RegExp(
+      `(?<![\\p{L}\\p{N}_./-])(?:https?:\\/\\/)?${escapePattern(core)}(?:\\/)?(?:[?#][^\\s"<>\\\\]*)?(?![\\p{L}\\p{N}_-])`,
+      'giu'
+    );
+  }
   return new RegExp(
     `(?<!${boundary})(?:${personalPatterns(value, type).join('|')})(?!${boundary})`,
     'giu'
@@ -153,17 +170,21 @@ export function samePersonal(value, original, type) {
     new RegExp(`^(?:${pattern})$`, 'iu').test(value)
   );
 }
+function confirmedFamily(entry, rule, options) {
+  return rule === 'confirmed-personal' &&
+    options.documentEntities &&
+    !options.knownOnly &&
+    entry.type === 'ORGANIZATION'
+    ? organizationFamily(organizationCore(entry.value))
+    : undefined;
+}
 export function detectKnownPersonal(text, entries, rule, emit, options) {
   let count = 0;
-  for (const entry of entries ?? []) {
-    const family =
-      rule === 'confirmed-personal' &&
-      options.documentEntities &&
-      !options.knownOnly &&
-      entry.type === 'ORGANIZATION' &&
-      organizationFamily(organizationCore(entry.value));
+  for (const [index, entry] of (entries ?? []).entries()) {
+    const family = confirmedFamily(entry, rule, options);
     for (const pattern of [
-      personalRegex(entry.value, entry.type),
+      compiledPatterns(entries)?.[index] ??
+        personalRegex(entry.value, entry.type),
       ...(family ? [family.pattern] : []),
     ]) {
       for (const match of text.matchAll(pattern)) {

package/src/policy.js

@@ -0,0 +1,107 @@
+import { failure } from './detection.js';
+import { builtInRules, credentialRules } from './rule-catalog.js';
+import { normalizeFieldName } from './document-fields.js';
+
+const ruleIds = new Set(builtInRules);
+const credentialType = (type) =>
+  /SECRET|TOKEN|PASSWORD|AUTH|CREDENTIAL|PRIVATE_KEY|ACCESS_KEY|^PIN$|CVV|CVC|CONNECTION.*STRING/.test(
+    type
+  );
+const record = (value) =>
+  value && typeof value === 'object' && !Array.isArray(value);
+// These recognizers consume the replaceable cue lists. Independent evidence
+// (legal forms, dates, full names, identifiers) remains available.
+const vocabularyRules = {
+  roles: ['document-reversed-role', 'document-role-employer'],
+  organizationCues: [
+    'organization-context',
+    'employer-label',
+    'employer-prose',
+    'organization-cue',
+    'organization-group-cue',
+    'document-employer-prefix',
+    'document-employer-dash',
+  ],
+  educationCues: [
+    'school-acronym-context',
+    'education-label',
+    'education-training',
+    'document-school-context',
+    'numbered-school',
+    'education-type',
+    'education-russian',
+  ],
+  personalLabels: ['document-role'],
+};
+export function validatePolicyControls(options) {
+  for (const key of ['allowCredentialDisable', 'strict', 'exactFields']) {
+    if (options[key] !== undefined && typeof options[key] !== 'boolean') {
+      throw failure('ERR_CONFIG');
+    }
+  }
+  for (const key of ['rules', 'types', 'fields']) {
+    if (options[key] === undefined) {
+      continue;
+    }
+    if (!record(options[key])) {
+      throw failure('ERR_CONFIG');
+    }
+    for (const [id, value] of Object.entries(options[key])) {
+      validateControl(key, id, value, options);
+    }
+  }
+}
+function validateControl(key, id, value, options) {
+  const validId =
+    key === 'rules'
+      ? ruleIds.has(id)
+      : key === 'types'
+        ? /^[A-Z][A-Z0-9_]{0,63}$/.test(id)
+        : id.length > 0;
+  if (
+    !validId ||
+    (key === 'fields' ? value !== 'scan' : typeof value !== 'boolean')
+  ) {
+    throw failure('ERR_CONFIG');
+  }
+  const credential =
+    key === 'rules' ? credentialRules.has(id) : credentialType(id);
+  if (value === false && !options.allowCredentialDisable && credential) {
+    throw failure('ERR_CONFIG');
+  }
+}
+export function builtInEnabled(finding, options) {
+  if (finding.category === 'credential' && !options.allowCredentialDisable) {
+    return true;
+  }
+  for (const [list, rules] of Object.entries(vocabularyRules)) {
+    if (
+      options.vocabulary?.[list]?.replace === true &&
+      rules.includes(finding.rule)
+    ) {
+      return false;
+    }
+  }
+  return (
+    options.rules?.[finding.rule] !== false &&
+    options.types?.[finding.type] !== false
+  );
+}
+export function scanField(field, options) {
+  return Object.keys(options.fields ?? {}).some(
+    (name) => normalizeFieldName(name) === normalizeFieldName(field)
+  );
+}
+export function fieldMatches(patterns, field, path, exact = false) {
+  const normalize = exact ? (s) => s : normalizeFieldName;
+  return (patterns ?? []).some((pattern) => {
+    const parts = pattern.split('.');
+    const values = parts.length === 1 ? [field] : path;
+    return (
+      parts.length === values.length &&
+      parts.every(
+        (part, i) => part === '*' || normalize(part) === normalize(values[i])
+      )
+    );
+  });
+}

package/src/record-worker.js

@@ -1,5 +1,11 @@
 import { parentPort, workerData } from 'node:worker_threads';
 import { createSanitizer, prepareStreamEngine } from './engines.js';
+import { compileKnown } from './compiled-known.js';
+
+// Structured cloning discards the caller's weak compilation metadata.
+if (workerData.knownPersonal) {
+  workerData.knownPersonal = compileKnown(workerData.knownPersonal);
+}
 
 parentPort.on(
   'message',