@link-foundation/sensitive-data-sanitizer 0.17.1 → 0.19.0

package/src/compiled-known.js

@@ -0,0 +1,55 @@
+import { failure } from './detection.js';
+import { personalRegex } from './personal-variants.js';
+import { projectText } from './projection.js';
+
+// Caller-owned frozen arrays remain compatible with maps, streams and fakes.
+// Weak keys allow the caller to release all values and compiled expressions.
+const compiled = new WeakMap();
+export const isCompiledKnown = (entries) => compiled.has(entries);
+export const compiledPatterns = (entries) => compiled.get(entries)?.patterns;
+export const projectedKnown = (entries) => compiled.get(entries)?.projected;
+
+export function compileKnown(...lists) {
+  if (lists.some((list) => !Array.isArray(list))) {
+    throw failure('ERR_CONFIG');
+  }
+  const values = new Map();
+  for (const entry of lists.flat()) {
+    if (
+      !entry ||
+      typeof entry.value !== 'string' ||
+      !entry.value ||
+      !/^[A-Z][A-Z0-9_]{0,63}$/.test(entry.type) ||
+      (entry.properCase !== undefined && typeof entry.properCase !== 'boolean')
+    ) {
+      throw failure('ERR_CONFIG');
+    }
+    const copy = Object.freeze({
+      type: entry.type,
+      value: entry.value,
+      ...(entry.properCase === undefined
+        ? {}
+        : { properCase: entry.properCase }),
+    });
+    values.set(
+      `${entry.type}\0${entry.value}\0${entry.properCase ?? false}`,
+      copy
+    );
+  }
+  const entries = Object.freeze([...values.values()]);
+  const patterns = entries.map((entry) =>
+    personalRegex(entry.value, entry.type)
+  );
+  const projected = entries.map((entry) => ({
+    ...entry,
+    value: projectText(entry.value).text,
+  }));
+  const changed = projected.some(
+    (entry, i) => entry.value !== entries[i].value
+  );
+  compiled.set(entries, {
+    patterns,
+    projected: changed ? compileKnown(projected) : entries,
+  });
+  return entries;
+}

package/src/context.js

@@ -257,6 +257,11 @@ function operationalValue(value, type) {
   );
 }
 
+function operationalLocationPath(value) {
+  return (
+    servicePath(value) || /^(?:\.?\.?\/)?[\w.-]+(?:\/[\w.-]+)+\/?$/.test(value)
+  );
+}
 // Shared by native labels and the vendor catalogs. Code expressions and
 // line escapes are not personal field values, even when a key is familiar.
 export function personalValueAllowed(text, range, type, labelStart) {
@@ -264,7 +269,10 @@ export function personalValueAllowed(text, range, type, labelStart) {
   if (!plausibleLabel(value, type)) {
     return false;
   }
-  if (['LOCATION', 'ADDRESS'].includes(type) && servicePath(value)) {
+  if (
+    ['LOCATION', 'ADDRESS'].includes(type) &&
+    operationalLocationPath(value)
+  ) {
     return false;
   }
   if (type === 'PHONE') {

package/src/detection.js

@@ -15,6 +15,9 @@ import {
 import { localAddress } from './public.js';
 import { detectKnownPersonal } from './personal-variants.js';
 import { detectDocumentEntities } from './document-entities.js';
+import { detectProseAndDocuments } from './proper-names.js';
+import { builtInEnabled } from './policy.js';
+import { detectStrict } from './strict.js';
 
 export const REDACTED = '[REDACTED]';
 export const failure = (code = 'ERR_SANITIZATION', location = {}) =>
@@ -136,6 +139,12 @@ function detectCookies(text, emit) {
 }
 
 export function nativeDetect(text, options, emit) {
+  const output = emit;
+  emit = (finding) => {
+    if (builtInEnabled(finding, options)) {
+      output(finding);
+    }
+  };
   const credential = { type: 'SECRET', category: 'credential' };
   for (const value of options.knownSecrets ?? []) {
     collectMatches(text, new RegExp(escapePattern(value), 'g'), emit, {
@@ -173,11 +182,12 @@ export function nativeDetect(text, options, emit) {
   detectContext(text, emit, options.credentialsOnly);
   if (!options.credentialsOnly) {
     detectPersonal(text, emit);
-    detectDocumentEntities(text, emit, options);
+    detectProseAndDocuments(text, emit, options, detectDocumentEntities);
     detectCatalog(text, emit);
     detectIdentity(text, emit);
     detectNational(text, emit);
     detectPersonalContext(text, emit);
+    detectStrict(text, emit, options);
   }
   detectCredentialFormats(text, emit);
   // detect-secrets-style quoted entropy detection; bare commit hashes and

package/src/document-catalog.js

@@ -0,0 +1,246 @@
+import { collectMatches } from './detection.js';
+import { isDocumentTechnology } from './document-experience.js';
+
+const b = '[\\p{L}\\p{N}_]';
+const proper = '[\\p{Lu}][\\p{L}\\p{M}.-]{1,47}';
+const rules = [
+  [
+    'PHONE',
+    'document-local-phone',
+    /(?<![\p{L}\p{N}_])(?:тел\.?|телефон|phone)[ \t]*:?[ \t]*(\d{7})(?!\d)/giu,
+    1,
+  ],
+  [
+    'PHONE',
+    'document-spaced-mobile',
+    /(?<![\p{L}\p{N}_])8[ \t]+9(?:[ \t]+\d){9}(?!\d)/gu,
+    0,
+  ],
+  [
+    'EMAIL',
+    'document-email-obfuscation',
+    /(?<![\p{L}\p{N}_])[\p{L}\p{N}_.+-]{1,64}\(собака\)[\p{L}\p{N}-]{1,63}(?:\(точка\)[\p{L}\p{N}-]{1,63}){1,6}(?![\p{L}\p{N}_])/giu,
+    0,
+  ],
+  [
+    'EMAIL',
+    'document-email-spaces',
+    /(?<![\p{L}\p{N}_])(?:[a-z][ \t]+){1,63}[a-z][ \t]*@[ \t]*[a-z\d-]{1,63}[ \t]*\.[ \t]*[a-z]{2,20}(?![\p{L}\p{N}_])/giu,
+    0,
+  ],
+  [
+    'USERNAME',
+    'document-messenger',
+    /(?<![\p{L}\p{N}_])(?:discord|max)[ \t]*:[ \t]*([a-z][a-z\d_.-]{1,63}(?:#\d{4})?)(?![\p{L}\p{N}_])/giu,
+    1,
+  ],
+  [
+    'URL',
+    'document-social-link',
+    /(?<![\p{L}\p{N}_./-])(?:https?:\/\/)?(?:www\.)?(?:ok\.ru\/profile|gitlab\.com|habr\.com\/ru\/users|facebook\.com)\/[\p{L}\p{N}_.-]{2,100}(?:\/)?(?:[?#][^\s"<>\\]*)?/giu,
+    0,
+  ],
+  [
+    'URL',
+    'document-resume-link',
+    /https?:\/\/[^\s/"<>\\]+\/(?:cv\.pdf(?:[?#][^\s"<>\\]*)?|cv(?:\/[^\s"<>\\]*)?|~[\p{L}\p{N}_.-]{1,100}\/?)/giu,
+    0,
+  ],
+  [
+    'DATE_OF_BIRTH',
+    'document-birthday',
+    /(?<![\p{L}\p{N}_])(?:д\.р\.|день рождения)[ \t]*:?[ \t]*(\d{1,2}(?:[/.]\d{1,2}[/.]\d{2,4}|[ \t]+(?:января|февраля|марта|апреля|мая|июня|июля|августа|сентября|октября|ноября|декабря)(?:[ \t]+\d{4})?))(?!\d)/giu,
+    1,
+  ],
+  [
+    'AGE',
+    'document-age-unit',
+    /(?<![\p{L}\p{N}_])\d{1,3}[ \t]*(?:y\.o\.|лет[ \t]+от[ \t]+роду)(?![\p{L}\p{N}_])/giu,
+    0,
+  ],
+  [
+    'ID',
+    'document-military-id',
+    /(?<![\p{L}\p{N}_])военный билет[ \t]+([А-ЯЁ]{2}[ \t]+\d{7})(?!\d)/giu,
+    1,
+  ],
+  [
+    'VIN',
+    'document-vin',
+    /(?<![\p{L}\p{N}_])VIN[ \t]*:?[ \t]*([A-HJ-NPR-Z\d]{17})(?![\p{L}\p{N}_])/giu,
+    1,
+  ],
+  [
+    'ORGANIZATION',
+    'document-legal-lowercase',
+    /(?<![\p{L}\p{N}_])(?:ооо|зао|пао|ао)[ \t]+[а-яё][а-яё-]{1,47}(?![\p{L}\p{N}_])/gu,
+    0,
+  ],
+  [
+    'ORGANIZATION',
+    'document-employer-dash',
+    new RegExp(
+      `(?<!${b})(?:Компания|Место работы)[ \\t]+[-–—][ \\t]+(${proper})(?!${b})`,
+      'gu'
+    ),
+    1,
+  ],
+  [
+    'ORGANIZATION',
+    'document-employer-prefix',
+    new RegExp(`(?<!${b})(?:Группа|стартап)[ \\t]+(${proper})(?!${b})`, 'gu'),
+    1,
+  ],
+  [
+    'ORGANIZATION',
+    'document-bank-suffix',
+    new RegExp(`(?<!${b})${proper}[ \\t]+Банк(?!${b})`, 'gu'),
+    0,
+  ],
+  [
+    'ORGANIZATION',
+    'document-industry',
+    new RegExp(
+      `(?<!${b})(${proper})(?=[ \\t]*\\((?:ритейл|retail|финансы|finance|IT)\\))`,
+      'gu'
+    ),
+    1,
+  ],
+  [
+    'ORGANIZATION',
+    'document-reversed-role',
+    new RegExp(
+      `(?<!${b})(${proper})[ \\t]*(?:,|[-–—])[ \\t]*(?:менеджер|manager|директор|developer|инженер)(?!${b})`,
+      'gu'
+    ),
+    1,
+  ],
+  [
+    'ORGANIZATION',
+    'document-role-employer',
+    new RegExp(
+      `(?<!${b})(?:менеджер|директор|инженер)[ \\t]+(?:в[ \\t]+)?(${proper})(?!${b})`,
+      'gu'
+    ),
+    1,
+  ],
+  [
+    'ORGANIZATION',
+    'document-at-employer',
+    /(?<![\p{L}\p{N}_])at[ \t]+([A-Z][a-z][\p{L}.-]{0,45})(?![\p{L}\p{N}_])/gu,
+    1,
+  ],
+  [
+    'EDUCATION',
+    'document-school-context',
+    /(?<![\p{L}\p{N}_])(?:graduated from|бакалавр,?|диплом|колледж|[Кк]урсы|курс на)[ \t]+([\p{Lu}][\p{L}.-]{1,47})(?![\p{L}\p{N}_])/gu,
+    1,
+  ],
+  [
+    'EDUCATION',
+    'document-school-year',
+    /(?<![\p{L}\p{N}_])[А-ЯЁA-Z]{2,12}(?=,[ \t]*(?:19|20)\d{2}(?!\d))/gu,
+    0,
+  ],
+  [
+    'EDUCATION',
+    'document-msu',
+    /(?<![\p{L}\p{N}_])МГУ(?:[ \t]+им\.[ \t]+[А-ЯЁ][а-яё-]{1,40})?(?![\p{L}\p{N}_])/gu,
+    0,
+  ],
+  [
+    'PERSON',
+    'document-patronymic',
+    /(?<![\p{L}\p{N}_])[А-ЯЁ][а-яё]{2,32}(?:овна|евна|ична|ович|евич)(?![\p{L}\p{N}_])/gu,
+    0,
+  ],
+  [
+    'PERSON',
+    'document-uppercase-initials',
+    /(?<![\p{L}\p{N}_])[А-ЯЁ]{2,40}[ \t]+(?:[А-ЯЁA-Z]\.[ \t]*){1,2}(?![\p{L}\p{N}_])/gu,
+    0,
+  ],
+  [
+    'PERSON',
+    'document-unfamiliar-name',
+    /(?<![\p{L}\p{N}_])(?:[А-ЯЁ][а-яё]{1,30}[ \t]+(?:Ким|[А-ЯЁ][а-яё]{1,25}(?:чук|енко|швили|дзе))|Ли[ \t]+[А-ЯЁ][а-яё]{1,20})(?![\p{L}\p{N}_])/gu,
+    0,
+  ],
+  [
+    'ADDRESS',
+    'document-street-complete',
+    /(?<![\p{L}\p{N}_])(?:(?:ул\.?|улица|пр-т)[ \t]+[А-ЯЁ][а-яё-]{1,40},?[ \t]+(?:д\.[ \t]*)?\d{1,5}(?:-\d{1,5})?|[А-ЯЁ][а-яё-]{2,40}[ \t]+шоссе,[ \t]*\d{1,5})(?:,[ \t]*кв\.?[ \t]*\d{1,5})?(?!\d)/gu,
+    0,
+  ],
+  [
+    'LOCATION',
+    'document-region',
+    /(?<![\p{L}\p{N}_])[А-ЯЁ][а-яё-]{2,40}[ \t]+(?:область|район)(?![\p{L}\p{N}_])/gu,
+    0,
+  ],
+  [
+    'PERSONAL_ATTRIBUTE',
+    'document-citizenship',
+    /(?<![\p{L}\p{N}_])(?:гражданство|гражданин|гражданка)[ \t]+([А-ЯЁ][А-ЯЁа-яё-]{1,40})(?![\p{L}\p{N}_])/gu,
+    1,
+  ],
+  [
+    'LICENSE_PLATE',
+    'document-vehicle-plate',
+    /(?<![\p{L}\p{N}_])[АВЕКМНОРСТУХABEKMHOPCTYX]\d{3}[АВЕКМНОРСТУХABEKMHOPCTYX]{2}[ \t]*\d{2,3}(?:[ \t]+RUS)?(?![\p{L}\p{N}_])/gu,
+    0,
+  ],
+];
+
+export function detectDocumentCatalog(text, emit) {
+  for (const [type, rule, pattern, group] of rules) {
+    collectMatches(
+      text,
+      pattern,
+      emit,
+      { type, rule, category: 'personal', confidence: 0.85 },
+      group,
+      (value) => type !== 'ORGANIZATION' || !isDocumentTechnology(value)
+    );
+  }
+  // Lookalike substitution preserves UTF-16 offsets and is limited to words
+  // containing Cyrillic plus Latin letters. Ordinary English is untouched.
+  const lookalikes = {
+    A: 'А',
+    a: 'а',
+    B: 'В',
+    C: 'С',
+    c: 'с',
+    E: 'Е',
+    e: 'е',
+    H: 'Н',
+    K: 'К',
+    M: 'М',
+    O: 'О',
+    o: 'о',
+    P: 'Р',
+    p: 'р',
+    T: 'Т',
+    X: 'Х',
+    x: 'х',
+    y: 'у',
+  };
+  const folded = text.replace(/[\p{L}]+/gu, (word) =>
+    /[А-Яа-яЁё]/u.test(word)
+      ? word.replace(/[AaBCcEeHKMOoPpTXxy]/g, (c) => lookalikes[c])
+      : word
+  );
+  if (folded !== text) {
+    collectMatches(
+      folded,
+      /(?<![\p{L}\p{N}_])[А-ЯЁ][а-яё]{1,30}[ \t]+[А-ЯЁ][а-яё]{1,28}(?:ова|ева|ина|ов|ев|ин|ский|ская)(?![\p{L}\p{N}_])/gu,
+      emit,
+      {
+        type: 'PERSON',
+        rule: 'document-mixed-name',
+        category: 'personal',
+        confidence: 0.9,
+      }
+    );
+  }
+}

package/src/document-dates.js

@@ -0,0 +1,15 @@
+import { escapePattern } from './detection.js';
+import { months } from './personal-context.js';
+
+const boundary = '[\\p{L}\\p{M}\\p{N}_]';
+const capitalized = (word) => word[0].toUpperCase() + word.slice(1);
+const month = [
+  ...new Set(months.split('|').flatMap((word) => [word, capitalized(word)])),
+].join('|');
+const rangeStart = `(?:(?:19|20)\\d{2}(?:-\\d{2}){0,2}|\\d{2}[./](?:\\d{2}\\.)?\\d{4}|(?:${month})\\.?[ \\t]+(?:19|20)\\d{2})`;
+const rangeOpen = ['по настоящее время', 'настоящее время', 'сейчас', 'по н.в.']
+  .concat(['н.в.', 'по н/в', 'н/в', 'present', 'now', 'current'])
+  .flatMap((word) => [word, capitalized(word)])
+  .map(escapePattern)
+  .join('|');
+export const documentDateRange = `${rangeStart}(?:[ \\t]*[-–—][ \\t]*|[ \\t]+(?:по|to)[ \\t]+)(?:${rangeStart}|${rangeOpen}|null|(?=\\)))(?!${boundary})`;

package/src/document-entities.js

@@ -2,9 +2,12 @@ import { collectMatches, escapePattern } from './detection.js';
 import { firstNames } from './names.js';
 import { organizationCore, personalRegex } from './personal-variants.js';
 import { documentVocabulary } from './document-vocabulary.js';
-import { months } from './personal-context.js';
+import { documentDateRange as range } from './document-dates.js';
 import { detectDocumentExtras } from './document-extras.js';
 import { detectDocumentPatterns } from './document-patterns.js';
+import { isDocumentTechnology } from './document-experience.js';
+import { detectVocabulary } from './vocabulary.js';
+import { detectDocumentCatalog } from './document-catalog.js';
 
 const words = (column) =>
   [...new Set(documentVocabulary.flatMap((row) => row[column].split('|')))]
@@ -58,17 +61,6 @@ const groupOrganization = new RegExp(
 // CV experience lines: the date range anchors the employer slot, so prose with
 // a comma is untouched. Titles are not required to be lowercase because English
 // ones are often title case (`Senior QA Engineer`).
-const capitalized = (word) => word[0].toUpperCase() + word.slice(1);
-const month = [
-  ...new Set(months.split('|').flatMap((word) => [word, capitalized(word)])),
-].join('|');
-const rangeStart = `(?:(?:19|20)\\d{2}(?:-\\d{2}){0,2}|\\d{2}[./](?:\\d{2}\\.)?\\d{4}|(?:${month})\\.?[ \\t]+(?:19|20)\\d{2})`;
-const rangeOpen = ['по настоящее время', 'настоящее время', 'сейчас', 'по н.в.']
-  .concat(['н.в.', 'по н/в', 'н/в', 'present', 'now', 'current'])
-  .flatMap((word) => [word, capitalized(word)])
-  .map(escapePattern)
-  .join('|');
-const range = `${rangeStart}(?:[ \\t]*[-–—][ \\t]*|[ \\t]+(?:по|to)[ \\t]+)(?:${rangeStart}|${rangeOpen}|null|(?=\\)))(?!${boundary})`;
 const titleWord = '[\\p{L}\\p{N}][\\p{L}\\p{N}/+#.&-]{0,39}';
 // A segment starts a line or follows `|`. `Опыт:` or `Опыт -` before the title
 // reads like a list marker: the date range remains the anchor of the rule.
@@ -89,8 +81,26 @@ export function detectDocumentEntities(text, emit, options) {
   if (!options.documentEntities) {
     return;
   }
+  detectVocabulary(text, emit, options);
+  detectDocumentCatalog(text, emit);
   detectDocumentPatterns(text, emit);
-  detectDocumentExtras(text, emit, range);
+  detectDocumentExtras(text, emit, range, options.debug, options);
+  collectMatches(
+    text,
+    new RegExp(
+      `(?:^|[\\r\\n])[ \\t]*[^|\\r\\n]{1,120}\\|[ \\t]*(${name})[ \\t]*\\|[ \\t]*${range}`,
+      'gu'
+    ),
+    emit,
+    {
+      type: 'ORGANIZATION',
+      rule: 'organization-experience',
+      category: 'personal',
+      confidence: 0.85,
+    },
+    1,
+    (value) => !isDocumentTechnology(value)
+  );
   const found = [];
   const capture = (finding) => {
     found.push(finding);
@@ -113,7 +123,10 @@ export function detectDocumentEntities(text, emit, options) {
       pattern,
       capture,
       { type, rule, category: 'personal', confidence },
-      group
+      group,
+      (value) =>
+        type !== 'ORGANIZATION' ||
+        (!isDocumentTechnology(value) && value !== 'ОГРНИП')
     );
   }
   for (const match of text.matchAll(ruResidence)) {

package/src/document-experience.js

@@ -0,0 +1,115 @@
+import { compiledVocabulary } from './vocabulary.js';
+
+const technology = new Set(
+  [
+    'Python',
+    'Go',
+    'SQL',
+    'Excel',
+    'Power BI',
+    'Kubernetes',
+    'PostgreSQL',
+    'Telegram',
+    'Scrum',
+  ].map((s) => s.toLowerCase())
+);
+export const isDocumentTechnology = (value) =>
+  technology.has(value.trim().toLowerCase());
+
+// Used only to disambiguate the reversed `Company, manager` layout. Forward
+// entries deliberately accept any title, without a title vocabulary.
+const reversedRole =
+  /^(?:менеджер|директор|руководитель|инженер|аналитик|разработчик|manager|director|engineer|developer|senior|qa)(?!\p{L})/iu;
+
+function commaParts(text) {
+  const parts = [];
+  let start = 0,
+    depth = 0,
+    quote;
+  for (let i = 0; i < text.length; i++) {
+    const char = text[i];
+    if (quote) {
+      if (char === quote) {
+        quote = undefined;
+      }
+    } else if ('«“"'.includes(char)) {
+      quote = { '«': '»', '“': '”', '"': '"' }[char];
+    } else if (char === '(') {
+      depth++;
+    } else if (char === ')') {
+      depth = Math.max(0, depth - 1);
+    } else if (char === ',' && !depth) {
+      parts.push({ start, text: text.slice(start, i).trim() });
+      start = i + 1;
+    }
+  }
+  parts.push({ start, text: text.slice(start).trim() });
+  return parts;
+}
+
+function employerSpan(prefix, roles) {
+  // An industry suffix belongs to the entry, but is not an employer. A pipe
+  // before the title remains a section separator, as in `Опыт | Менеджер, ...`.
+  const comma = prefix.indexOf(',');
+  const pipe = comma < 0 ? -1 : prefix.indexOf('|', comma);
+  const content = pipe < 0 ? prefix : prefix.slice(0, pipe);
+  const parts = commaParts(content);
+  let index = 1;
+  if (parts.length === 1) {
+    return undefined;
+  }
+  if (/^[=\s\d]/u.test(parts[0].text)) {
+    index++;
+  }
+  if (parts.length === 2 && roles.test(parts[1].text)) {
+    index = 0;
+  } else {
+    while (index < parts.length - 1 && /^\p{Ll}/u.test(parts[index].text)) {
+      index++;
+    }
+  }
+  const part = parts[index];
+  if (!part || !/^(?:[\p{Lu}«“"]|\d{1,4}-?\p{L})/u.test(part.text)) {
+    return undefined;
+  }
+  const end = index === 0 ? content.indexOf(',') : content.trimEnd().length;
+  const start = part.start + content.slice(part.start).search(/\S/u);
+  return isDocumentTechnology(content.slice(start, end))
+    ? undefined
+    : { start, end };
+}
+
+export function detectDatedEmployers(text, emit, range, debug, options = {}) {
+  let cursor = 0,
+    count = 0;
+  // Match only the anchor, then visit each preceding segment once. No lazy
+  // prefix search restarts at every position on a long nonmatching line.
+  for (const match of text.matchAll(new RegExp(`\\(${range}\\)`, 'gu'))) {
+    const segment = text.slice(cursor, match.index);
+    const boundary =
+      Math.max(
+        segment.lastIndexOf('\n'),
+        segment.lastIndexOf('\r'),
+        segment.lastIndexOf(';')
+      ) + 1;
+    const prefix = segment.slice(boundary).replace(/^[., \t]+/u, '');
+    const span = employerSpan(
+      prefix,
+      compiledVocabulary(options.vocabulary)?.roles ?? reversedRole
+    );
+    if (span) {
+      const offset = match.index - prefix.length;
+      emit({
+        start: offset + span.start,
+        end: offset + span.end,
+        type: 'ORGANIZATION',
+        category: 'personal',
+        rule: 'organization-experience',
+        confidence: 0.85,
+      });
+      count++;
+    }
+    cursor = match.index + match[0].length;
+  }
+  debug?.({ event: 'document-experience', count });
+}