@link-foundation/sensitive-data-sanitizer 0.17.1 → 0.19.0

package/src/index.d.ts

@@ -21,7 +21,20 @@ export interface Finding {
 export interface PersonalValue {
   type: string;
   value: string;
+  properCase?: boolean;
 }
+export type VocabularyList =
+  readonly string[] | { values: readonly string[]; replace: true };
+export type Vocabulary = Partial<
+  Record<
+    | 'roles'
+    | 'organizationCues'
+    | 'educationCues'
+    | 'personalLabels'
+    | 'personalFields',
+    VocabularyList
+  >
+>;
 export interface PublicEntity {
   type: 'PERSON' | 'ORGANIZATION' | 'EDUCATION' | 'LOCATION' | 'EMAIL';
   value: string;
@@ -38,14 +51,28 @@ export interface Detector {
 export interface SanitizerOptions {
   /** Preserve all personal/account data; credential findings still redact. */
   credentialsOnly?: boolean;
-  /** Replace only knownPersonal/knownSecrets and reused map values; skip automatic detectors. */
+  /** Use explicit findings, knownPersonal/knownSecrets and reused map values; skip automatic detectors. */
   knownOnly?: boolean;
   profile?: 'publication' | 'documents';
   /** Opt in to bounded employer, school, city/region heuristics. */
   documentEntities?: boolean;
+  /** Opt-in aggressive stored-log attributes, ages, dates and numeric shapes. */
+  strict?: boolean;
+  vocabulary?: Vocabulary;
+  /** Stable ids from builtInRules. Unknown ids fail with ERR_CONFIG. */
+  rules?: Record<string, boolean>;
+  types?: Record<string, boolean>;
+  /** Required to disable any built-in credential rule or type. */
+  allowCredentialDisable?: boolean;
+  /** Treat a built-in property as ordinary scanned text. */
+  fields?: Record<string, 'scan'>;
+  /** Aggressive structural proper names in prose values; opt-in, off by default. */
+  properNames?: 'capitalized';
+  /** Unicode Script aliases that activate prose detection; default ['Cyrillic']. */
+  scripts?: string[];
   confirmedPersonal?: Array<{ value: string; type: string }>;
   knownSecrets?: string[];
-  knownPersonal?: PersonalValue[];
+  knownPersonal?: readonly PersonalValue[];
   publicEntities?: PublicEntity[];
   /** Built-in exact public entities/resolvers and role mail on known domains. */
   publicKnowledge?: boolean;
@@ -63,8 +90,12 @@ export interface SanitizerOptions {
   structured?: 'json' | 'jsonl';
   /** Property names whose metadata heuristics may be suppressed. */
   structuralFields?: string[];
-  /** Exact, case-sensitive property names whose entire values are redacted in JSON/JSONL. */
+  /** Normalized property names or dotted paths with *; entire JSON/JSONL values redact. */
   sensitiveFields?: string[];
+  /** Match sensitiveFields/allowedFields with exact case and punctuation. */
+  exactFields?: boolean;
+  /** Only these string leaves remain eligible to keep; listed strings are still scanned. */
+  allowedFields?: string[];
   /** Personal replacement threshold, default 0.5. Inspect retains lower scores. */
   minConfidence?: number;
   /** Alias for minConfidence; specifying both requires equal values. */
@@ -80,7 +111,7 @@ export interface SanitizerOptions {
   maxInputLength?: number;
   maxFindings?: number;
   debug?: (event: DebugEvent) => void;
-  /** Used by createSanitizer; synchronous functions use native rules only. */
+  /** Required callbacks; synchronous functions reject Promise-returning detectors. */
   detectors?: Detector[];
   /** Required by default in createSanitizer. */
   secretlint?: boolean;
@@ -88,6 +119,7 @@ export interface SanitizerOptions {
 /** Value-free diagnostics; enabled only when a debug callback is supplied. */
 export type DebugEvent =
   | { event: 'detected'; count: number }
+  | { event: 'document-experience' | 'proper-name-prose'; count: number }
   | { event: 'residual-redaction'; round: number }
   | { event: 'two-pass-confirmation'; type: string }
   | { event: 'propagation-skipped'; reason: 'length'; type: string }
@@ -106,6 +138,22 @@ export interface Sanitizer {
   inspect(text: string): Promise<Finding[]>;
   sanitize(text: string): Promise<SanitizedText>;
 }
+export interface SyncSanitizer {
+  inspect(text: string): Finding[];
+  sanitize(text: string): SanitizedText;
+}
+export declare function createSyncSanitizer(
+  options?: SanitizerOptions
+): SyncSanitizer;
+/** Copies and freezes caller values, merging lists and caching matching/projection. */
+export declare function compileKnown(
+  ...lists: Array<readonly PersonalValue[]>
+): readonly Readonly<PersonalValue>[];
+export declare const builtInRules: readonly string[];
+export declare const defaultVocabulary: Readonly<
+  Record<keyof Vocabulary, readonly string[]>
+>;
+export declare const documentVocabulary: readonly (readonly string[])[];
 export interface JsonlSanitizer extends Sanitizer {
   sanitizeJsonl(text: string): Promise<SanitizedText>;
 }
@@ -224,7 +272,12 @@ export declare function auditGitHistory(
     maxTotalBytes?: number;
   }
 ): Promise<HistoryAudit>;
+/** Callback receives private matched data. Sync APIs require an immediate string. */
+export type TransformationCallback = (
+  finding: Readonly<Finding & { value: string }>
+) => string | Promise<string>;
 export type Transformation =
+  | TransformationCallback
   | { mode: 'redact' | 'hive-mask' }
   | {
       mode: 'placeholder';

package/src/index.js

@@ -2,6 +2,11 @@
 
 export { inspect, sanitize, redact } from './sanitizer.js';
 export { createSanitizer } from './engines.js';
+export { compileKnown } from './compiled-known.js';
+export { builtInRules } from './rule-catalog.js';
+export { defaultVocabulary } from './vocabulary.js';
+export { documentVocabulary } from './document-vocabulary.js';
+export { createSyncSanitizer } from './sync-sanitizer.js';
 export { REDACTED, entropy, luhn } from './detection.js';
 export { codePointRange } from './projection.js';
 export {

package/src/jsonl-stream.js

@@ -3,7 +3,7 @@ import { Worker } from 'node:worker_threads';
 import { URL } from 'node:url';
 import { failure } from './detection.js';
 import { encodingFailure } from './utf8.js';
-import { workerExecArgv } from './worker-options.js';
+import { workerExecArgv, hasWorkerCallbacks } from './worker-options.js';
 import { prepareStreamEngine } from './engines.js';
 import { inspect } from './sanitizer.js';
 import { hasPlaceholders } from './placeholders.js';
@@ -129,10 +129,7 @@ function workerCount(options, config) {
     return 1;
   }
   const callbacks =
-    options.sanitizer ||
-    config.sanitizerOptions.debug ||
-    config.sanitizerOptions.verifyPublic ||
-    config.sanitizerOptions.detectors?.length;
+    options.sanitizer || hasWorkerCallbacks(config.sanitizerOptions);
   const count =
     options.workers ?? (callbacks || typeof Deno !== 'undefined' ? 1 : 2);
   if (

package/src/ner.d.ts

@@ -0,0 +1,5 @@
+import type { Finding } from './index.d.ts';
+export declare function createNerDetector(options?: {
+  languages?: Array<'ru' | 'en'>;
+  debug?: (event: { event: 'offline-ner'; count: number }) => void;
+}): Promise<{ id: string; detect(text: string): Finding[] }>;

package/src/ner.js

@@ -0,0 +1,96 @@
+// Optional entry: no core module imports NLP.js or this recognizer.
+import { firstNames } from './names.js';
+import { documentVocabulary } from './document-vocabulary.js';
+
+const boundary = '[\\p{L}\\p{M}\\p{N}_]';
+const proper = "\\p{Lu}[\\p{L}\\p{M}’'-]{1,47}";
+const name = `${proper}(?:[ \\t]+${proper}){0,3}`;
+const escape = (s) => s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+const blocked =
+  /^(?:Kubernetes|PostgreSQL|Python|JavaScript|TypeScript|Node|Docker|Redis|Linux|Monday|Tuesday|Wednesday|Thursday|Friday|Saturday|Sunday|REDACTED)$/iu;
+const configError = () =>
+  Object.assign(new Error('Invalid offline recognizer configuration.'), {
+    code: 'ERR_CONFIG',
+  });
+
+function rulesFor(language) {
+  const names = firstNames[language]
+    .split(' ')
+    .filter((s) => !/^(?:May|Will|Mark|Rose)$/u.test(s));
+  const places = documentVocabulary
+    .filter((row) => row[0] === language)
+    .flatMap((row) => row[3].split('|'));
+  const rules = [
+    ['PERSON', `(?:${names.map(escape).join('|')})(?:[ \\t]+${proper})?`],
+    ['LOCATION', `(?:${places.map(escape).join('|') || '(?!)'})`],
+  ];
+  if (language === 'ru') {
+    rules.push(
+      ['ORGANIZATION', `(?<=(?:[Рр]аботал[аи]?|[Сс]лужил[аи]?) в )${name}`],
+      [
+        'ORGANIZATION',
+        `${name}(?=:[ \\t]*(?:менеджер|директор|разработчик|инженер|аналитик|маркетолог)(?!\\p{L}))`,
+      ],
+      ['EDUCATION', '(?:МГУ|СПбГУ|МГТУ|РГГУ|ВШЭ|МФТИ|ИТМО)'],
+      ['EDUCATION', `${name}[ \\t]+(?:университет|институт|академия)`],
+      ['LOCATION', `(?<=(?:[Жж]иву|[Пп]роживаю) в )${name}`]
+    );
+  } else {
+    rules.push(
+      ['ORGANIZATION', `(?<=(?:[Aa]t|[Jj]oined|[Ee]mployer:) )${name}`],
+      ['EDUCATION', `(?<=(?:[Gg]raduated from|[Ss]tudied at) )${name}`],
+      [
+        'EDUCATION',
+        '(?:MIT|MSU|UCLA|UCL|NYU|Harvard|Stanford|Oxford|Cambridge)',
+      ],
+      ['EDUCATION', `${name}[ \\t]+(?:University|College|Institute)`],
+      ['LOCATION', `(?<=(?:[Ll]ive in|[Ll]ocated in|[Bb]ased in) )${name}`]
+    );
+  }
+  return rules.map(([type, source]) => ({
+    name: type,
+    type: 'regex',
+    rules: [new RegExp(`(?<!${boundary})(?:${source})(?!${boundary})`, 'gu')],
+  }));
+}
+
+/** Prepare an offline recognizer with synchronous detection. */
+export async function createNerDetector(options = {}) {
+  const languages = options.languages ?? ['ru', 'en'];
+  if (
+    !Array.isArray(languages) ||
+    !languages.length ||
+    languages.length > 2 ||
+    languages.some((s) => !['ru', 'en'].includes(s)) ||
+    Object.keys(options).some((key) => !['languages', 'debug'].includes(key)) ||
+    (options.debug !== undefined && typeof options.debug !== 'function')
+  ) {
+    throw configError();
+  }
+  const { ExtractorRegex } = await import('@nlpjs/ner');
+  const extractor = new ExtractorRegex();
+  const nerRules = [...new Set(languages)].flatMap(rulesFor);
+  return Object.freeze({
+    id: 'offline-ner',
+    detect(text) {
+      if (typeof text !== 'string' || text.length > 1024 * 1024) {
+        throw Object.assign(new Error('Offline recognizer input limit.'), {
+          code: 'ERR_LIMIT',
+        });
+      }
+      const findings = extractor
+        .extract({ text, nerRules })
+        .edges.filter((edge) => !blocked.test(edge.sourceText))
+        .map((edge) => ({
+          start: edge.start,
+          end: edge.end + 1,
+          type: edge.entity,
+          category: 'personal',
+          rule: 'ner/offline',
+          confidence: 0.8,
+        }));
+      options.debug?.({ event: 'offline-ner', count: findings.length });
+      return findings;
+    },
+  });
+}

package/src/personal-variants.js

@@ -1,5 +1,6 @@
 import { escapePattern, failure } from './detection.js';
 import { russianNameRoles } from './russian-names.js';
+import { compiledPatterns } from './compiled-known.js';
 
 const boundary = '[\\p{L}\\p{M}\\p{N}_]';
 export function organizationCore(value) {
@@ -143,6 +144,22 @@ export function personalPatterns(value, type) {
   return [...patterns];
 }
 export function personalRegex(value, type) {
+  if (type === 'URL') {
+    let core = value.replace(/^https?:\/\//iu, '');
+    const suffix = core.search(/[?#]/u);
+    if (suffix >= 0) {
+      core = core.slice(0, suffix);
+    }
+    let end = core.length;
+    while (end > 0 && core[end - 1] === '/') {
+      end--;
+    }
+    core = core.slice(0, end);
+    return new RegExp(
+      `(?<![\\p{L}\\p{N}_./-])(?:https?:\\/\\/)?${escapePattern(core)}(?:\\/)?(?:[?#][^\\s"<>\\\\]*)?(?![\\p{L}\\p{N}_-])`,
+      'giu'
+    );
+  }
   return new RegExp(
     `(?<!${boundary})(?:${personalPatterns(value, type).join('|')})(?!${boundary})`,
     'giu'
@@ -153,17 +170,21 @@ export function samePersonal(value, original, type) {
     new RegExp(`^(?:${pattern})$`, 'iu').test(value)
   );
 }
+function confirmedFamily(entry, rule, options) {
+  return rule === 'confirmed-personal' &&
+    options.documentEntities &&
+    !options.knownOnly &&
+    entry.type === 'ORGANIZATION'
+    ? organizationFamily(organizationCore(entry.value))
+    : undefined;
+}
 export function detectKnownPersonal(text, entries, rule, emit, options) {
   let count = 0;
-  for (const entry of entries ?? []) {
-    const family =
-      rule === 'confirmed-personal' &&
-      options.documentEntities &&
-      !options.knownOnly &&
-      entry.type === 'ORGANIZATION' &&
-      organizationFamily(organizationCore(entry.value));
+  for (const [index, entry] of (entries ?? []).entries()) {
+    const family = confirmedFamily(entry, rule, options);
     for (const pattern of [
-      personalRegex(entry.value, entry.type),
+      compiledPatterns(entries)?.[index] ??
+        personalRegex(entry.value, entry.type),
       ...(family ? [family.pattern] : []),
     ]) {
       for (const match of text.matchAll(pattern)) {

package/src/placeholder-types.js

@@ -22,6 +22,7 @@ export const reservedPlaceholderTypes = () => [
   'PASSPORT_MRZ',
   'PASSPORT_NUMBER',
   'PERSON',
+  'PROPER_NAME',
   'PHONE',
   'PL_PESEL',
   'RU_INN',

package/src/policy.js

@@ -0,0 +1,107 @@
+import { failure } from './detection.js';
+import { builtInRules, credentialRules } from './rule-catalog.js';
+import { normalizeFieldName } from './document-fields.js';
+
+const ruleIds = new Set(builtInRules);
+const credentialType = (type) =>
+  /SECRET|TOKEN|PASSWORD|AUTH|CREDENTIAL|PRIVATE_KEY|ACCESS_KEY|^PIN$|CVV|CVC|CONNECTION.*STRING/.test(
+    type
+  );
+const record = (value) =>
+  value && typeof value === 'object' && !Array.isArray(value);
+// These recognizers consume the replaceable cue lists. Independent evidence
+// (legal forms, dates, full names, identifiers) remains available.
+const vocabularyRules = {
+  roles: ['document-reversed-role', 'document-role-employer'],
+  organizationCues: [
+    'organization-context',
+    'employer-label',
+    'employer-prose',
+    'organization-cue',
+    'organization-group-cue',
+    'document-employer-prefix',
+    'document-employer-dash',
+  ],
+  educationCues: [
+    'school-acronym-context',
+    'education-label',
+    'education-training',
+    'document-school-context',
+    'numbered-school',
+    'education-type',
+    'education-russian',
+  ],
+  personalLabels: ['document-role'],
+};
+export function validatePolicyControls(options) {
+  for (const key of ['allowCredentialDisable', 'strict', 'exactFields']) {
+    if (options[key] !== undefined && typeof options[key] !== 'boolean') {
+      throw failure('ERR_CONFIG');
+    }
+  }
+  for (const key of ['rules', 'types', 'fields']) {
+    if (options[key] === undefined) {
+      continue;
+    }
+    if (!record(options[key])) {
+      throw failure('ERR_CONFIG');
+    }
+    for (const [id, value] of Object.entries(options[key])) {
+      validateControl(key, id, value, options);
+    }
+  }
+}
+function validateControl(key, id, value, options) {
+  const validId =
+    key === 'rules'
+      ? ruleIds.has(id)
+      : key === 'types'
+        ? /^[A-Z][A-Z0-9_]{0,63}$/.test(id)
+        : id.length > 0;
+  if (
+    !validId ||
+    (key === 'fields' ? value !== 'scan' : typeof value !== 'boolean')
+  ) {
+    throw failure('ERR_CONFIG');
+  }
+  const credential =
+    key === 'rules' ? credentialRules.has(id) : credentialType(id);
+  if (value === false && !options.allowCredentialDisable && credential) {
+    throw failure('ERR_CONFIG');
+  }
+}
+export function builtInEnabled(finding, options) {
+  if (finding.category === 'credential' && !options.allowCredentialDisable) {
+    return true;
+  }
+  for (const [list, rules] of Object.entries(vocabularyRules)) {
+    if (
+      options.vocabulary?.[list]?.replace === true &&
+      rules.includes(finding.rule)
+    ) {
+      return false;
+    }
+  }
+  return (
+    options.rules?.[finding.rule] !== false &&
+    options.types?.[finding.type] !== false
+  );
+}
+export function scanField(field, options) {
+  return Object.keys(options.fields ?? {}).some(
+    (name) => normalizeFieldName(name) === normalizeFieldName(field)
+  );
+}
+export function fieldMatches(patterns, field, path, exact = false) {
+  const normalize = exact ? (s) => s : normalizeFieldName;
+  return (patterns ?? []).some((pattern) => {
+    const parts = pattern.split('.');
+    const values = parts.length === 1 ? [field] : path;
+    return (
+      parts.length === values.length &&
+      parts.every(
+        (part, i) => part === '*' || normalize(part) === normalize(values[i])
+      )
+    );
+  });
+}

package/src/propagation.js

@@ -36,7 +36,11 @@ function eligibleSeed(finding, options) {
     !options.knownOnly &&
     finding.category === 'personal' &&
     finding.type !== 'ENCODED_SENSITIVE' &&
-    !['confirmed-personal', 'document-propagation'].includes(finding.rule) &&
+    ![
+      'confirmed-personal',
+      'document-propagation',
+      'proper-name-prose',
+    ].includes(finding.rule) &&
     actionable(finding, options) &&
     ((options.documentEntities && finding.type === 'ORGANIZATION') ||
       finding.confidence >= 0.85 ||

package/src/proper-names.js

@@ -0,0 +1,222 @@
+import { failure } from './detection.js';
+import { documentDateRange } from './document-dates.js';
+
+const scriptPatterns = new Map();
+const dateRange = new RegExp(`\\(${documentDateRange}\\)`, 'u');
+const commaMobile = /,[ \t]*[78]9\d{9}(?![\p{L}\p{N}_])/u;
+const mobile = /^[78]9\d{9}$/u;
+const tokenPattern = /[\p{L}\p{M}\p{N}_./:\\?&#%=@+'’~+-]+/gu;
+const upper = /[\p{Lu}\p{Lt}]/u;
+const quotePairs = new Map([
+  ['«', '»'],
+  ['“', '”'],
+  ['„', '“'],
+  ['‘', '’'],
+  ['「', '」'],
+  ['『', '』'],
+]);
+
+function scriptPattern(scripts = ['Cyrillic']) {
+  if (
+    scripts.some(
+      (script) => typeof script !== 'string' || !/^[A-Za-z_]+$/.test(script)
+    )
+  ) {
+    throw failure('ERR_CONFIG');
+  }
+  const key = scripts.join('|');
+  let pattern = scriptPatterns.get(key);
+  if (!pattern) {
+    // The alias grammar prevents regexp injection; constructing the Unicode
+    // property also validates aliases supported by this runtime.
+    try {
+      pattern = new RegExp(
+        scripts.length
+          ? `(?=\\p{L})(?:${scripts.map((s) => `\\p{Script=${s}}`).join('|')})`
+          : '(?!)',
+        'u'
+      );
+    } catch {
+      throw failure('ERR_CONFIG');
+    }
+    // Bound cache growth when callers submit many different configurations.
+    if (scriptPatterns.size >= 64) {
+      scriptPatterns.clear();
+    }
+    scriptPatterns.set(key, pattern);
+  }
+  return pattern;
+}
+export function validateProperNames(options) {
+  if (
+    (options.properNames !== undefined &&
+      options.properNames !== 'capitalized') ||
+    (options.scripts !== undefined && !Array.isArray(options.scripts))
+  ) {
+    throw failure('ERR_CONFIG');
+  }
+  scriptPattern(options.scripts);
+}
+
+export function isProseLink(token) {
+  return (
+    /^https?:\/\/[^/]+/iu.test(token) ||
+    /^[\p{L}\p{N}](?:[\p{L}\p{N}-]*\.)+[\p{L}]{2,63}(?:[/:?#]|$)/u.test(token)
+  );
+}
+export function isDocumentShareLink(token) {
+  return isProseLink(token) && /\/d\/[^/?#]+/u.test(token);
+}
+function proseData(text, options) {
+  if (
+    scriptPattern(options.scripts).test(text) ||
+    dateRange.test(text) ||
+    commaMobile.test(text)
+  ) {
+    return true;
+  }
+  for (const match of text.matchAll(tokenPattern)) {
+    if (isDocumentShareLink(match[0])) {
+      return true;
+    }
+  }
+  return false;
+}
+function quotedSpans(text, emit) {
+  let start = -1,
+    closing;
+  for (let i = 0; i < text.length; i++) {
+    if (start >= 0) {
+      if (text[i] === closing) {
+        emit(start, i + 1, 'PROPER_NAME');
+        start = -1;
+      }
+    } else if (quotePairs.has(text[i])) {
+      start = i;
+      closing = quotePairs.get(text[i]);
+    }
+  }
+}
+function pathToken(value) {
+  return value.includes('/') || value.includes('\\');
+}
+// Keep existing document coverage outside the new replacements. Two naturally
+// ordered runs (quotes, then tokens) provide an interval index without sorting.
+// Overlapping broad document spans must not merge separate prose replacements.
+export function detectProseAndDocuments(text, emit, options, detectDocuments) {
+  if (options.properNames !== 'capitalized') {
+    detectDocuments(text, emit, options);
+    return;
+  }
+  const runs = [[]];
+  detectProperNames(
+    text,
+    (finding) => {
+      if (runs.at(-1).at(-1)?.end > finding.start) {
+        runs.push([]);
+      }
+      runs.at(-1).push(finding);
+      emit(finding);
+    },
+    options
+  );
+  detectDocuments(
+    text,
+    (finding) => {
+      if (!runs.some((run) => overlapsRun(run, finding))) {
+        emit(finding);
+      }
+    },
+    options
+  );
+}
+function overlapsRun(run, finding) {
+  let low = 0,
+    high = run.length;
+  while (low < high) {
+    const mid = (low + high) >>> 1;
+    if (run[mid].end <= finding.start) {
+      low = mid + 1;
+    } else {
+      high = mid;
+    }
+  }
+  return low < run.length && run[low].start < finding.end;
+}
+export function detectProperNames(text, emit, options) {
+  if (options.properNames !== 'capitalized' || !proseData(text, options)) {
+    return false;
+  }
+  let count = 0;
+  const found = (start, end, type) => {
+    emit({
+      start,
+      end,
+      type,
+      rule: 'proper-name-prose',
+      category: 'personal',
+      confidence: 0.85,
+    });
+    count++;
+  };
+  quotedSpans(text, found);
+  const ranges = text.matchAll(new RegExp(dateRange, 'gu'));
+  let range = ranges.next().value;
+  for (const match of text.matchAll(tokenPattern)) {
+    const raw = match[0];
+    // Leave sentence punctuation outside the replacement. Classification runs
+    // once on a maximal token, including hostile tokens without spaces.
+    let length = raw.length;
+    while (length && (raw[length - 1] === '.' || raw[length - 1] === ':')) {
+      length--;
+    }
+    const value = raw.slice(0, length);
+    const start = match.index,
+      end = start + value.length;
+    while (range && range.index + range[0].length <= start) {
+      range = ranges.next().value;
+    }
+    if (preserveToken(text, raw, value, start, end, range)) {
+      continue;
+    }
+    const type = tokenType(value);
+    if (type) {
+      found(start, end, type);
+    }
+  }
+  options.debug?.({ event: 'proper-name-prose', count });
+  return true;
+}
+
+function preserveToken(text, raw, value, start, end, range) {
+  let next = start + raw.length;
+  while (text[next] === ' ' || text[next] === '\t') {
+    next++;
+  }
+  return (
+    !value ||
+    (range && start >= range.index && end <= range.index + range[0].length) ||
+    (value === 'REDACTED' && text[start - 1] === '[' && text[end] === ']') ||
+    raw.endsWith(':') ||
+    text[next] === ':'
+  );
+}
+function tokenType(value) {
+  if (isProseLink(value)) {
+    return 'URL';
+  }
+  if (pathToken(value)) {
+    return undefined;
+  }
+  if (mobile.test(value)) {
+    return 'PHONE';
+  }
+  if (
+    upper.test(value) &&
+    /\p{L}/u.test(value) &&
+    !/^\d{4}-\d{2}-\d{2}/u.test(value)
+  ) {
+    return 'PROPER_NAME';
+  }
+  return undefined;
+}

package/src/record-worker.js

@@ -1,5 +1,11 @@
 import { parentPort, workerData } from 'node:worker_threads';
 import { createSanitizer, prepareStreamEngine } from './engines.js';
+import { compileKnown } from './compiled-known.js';
+
+// Structured cloning discards the caller's weak compilation metadata.
+if (workerData.knownPersonal) {
+  workerData.knownPersonal = compileKnown(workerData.knownPersonal);
+}
 
 parentPort.on(
   'message',