@link-foundation/sensitive-data-sanitizer 0.17.1 → 0.19.0

package/src/document-extras.js

@@ -1,8 +1,14 @@
 import { collectMatches } from './detection.js';
+import {
+  detectDatedEmployers,
+  isDocumentTechnology,
+} from './document-experience.js';
+import { isDocumentShareLink } from './proper-names.js';
 
 const boundary = '[\\p{L}\\p{N}_]';
 const proper = '[\\p{Lu}][\\p{L}\\p{M}\\p{N}.-]{0,46}[\\p{L}\\p{M}\\p{N}]';
 const name = `${proper}(?:[ \\t]+${proper}){0,5}`;
+const employerName = `(?:«[^«»\\r\\n]{1,160}»|“[^“”\\r\\n]{1,160}”|"[^"\\r\\n]{1,160}"|${name})`;
 const orgList = `(?:«[^«»\\r\\n]{1,160}»|${name})(?:[ \\t]*(?:,|и|and)[ \\t]+${name}){0,5}`;
 const cues = new RegExp(
   `(?<!${boundary})(?:компании|опыт[ \\t]+в|проекты[ \\t]+для|[Рр]оль[ \\t]+в)[ \\t]+(${orgList})(?!${boundary})`,
@@ -12,13 +18,42 @@ const duration = new RegExp(
   `(?<!${boundary})(${name})(?=[ \\t]*\\(\\d{1,2}[ \\t]+(?:год|года|лет|years?)\\))`,
   'gu'
 );
-// JavaScript's \b is ASCII-only; Cyrillic titles use a Unicode lookahead.
-const role =
-  /^(?:менеджер|директор|руководитель|инженер|аналитик|разработчик)(?![\p{L}])|^(?:manager|director|engineer|developer|senior|qa)\b/iu;
-
-export function detectDocumentExtras(text, emit, range) {
+export function detectDocumentExtras(text, emit, range, debug, options) {
   for (const [type, rule, pattern, group] of [
     ['PHONE', 'document-mobile-bare', /^[78]9\d{9}$/gm, 0],
+    [
+      'PHONE',
+      'document-mobile-comma',
+      /,[ \t]*([78]9\d{9})(?![\p{L}\p{N}_])/gu,
+      1,
+    ],
+    [
+      'ORGANIZATION',
+      'organization-group-cue',
+      new RegExp(
+        `(?<!${boundary})(?:ГК|[Гг]руппа компаний)[ \\t]+${employerName}`,
+        'gu'
+      ),
+      0,
+    ],
+    [
+      'EDUCATION',
+      'education-bracket',
+      /(?<=\()(МГУ|СПбГУ|МГТУ|РГГУ|ВШЭ)(?=[,)])/gu,
+      1,
+    ],
+    [
+      'EDUCATION',
+      'education-named-acronym',
+      /(?<![\p{L}\p{N}_])(?:МГУ|СПбГУ|МГТУ|РГГУ|ВШЭ)[ \t]+им\.[ \t]+[А-ЯЁ][а-яё-]{2,40}/gu,
+      0,
+    ],
+    [
+      'EDUCATION',
+      'education-training',
+      /(?<![\p{L}\p{N}_])(?:Обучение|Training)[ \t]*:[ \t]*([\p{Lu}][\p{L}\p{M}.-]{1,47})/gu,
+      1,
+    ],
     [
       'PHONE',
       'document-mobile-context',
@@ -56,7 +91,8 @@ export function detectDocumentExtras(text, emit, range) {
       pattern,
       emit,
       { type, rule, category: 'personal', confidence: 0.85 },
-      group
+      group,
+      (value) => type !== 'ORGANIZATION' || !isDocumentTechnology(value)
     );
   }
   for (const match of text.matchAll(cues)) {
@@ -64,6 +100,9 @@ export function detectDocumentExtras(text, emit, range) {
     for (const company of match[1].matchAll(
       new RegExp(`«[^«»\\r\\n]{1,160}»|${name}`, 'gu')
     )) {
+      if (isDocumentTechnology(company[0])) {
+        continue;
+      }
       emit({
         start: offset + company.index,
         end: offset + company.index + company[0].length,
@@ -75,7 +114,57 @@ export function detectDocumentExtras(text, emit, range) {
     }
   }
   detectQueryContacts(text, emit);
-  detectExperience(text, emit, range);
+  collectMatches(
+    text,
+    /https?:\/\/[^\s"<>\\]+/gu,
+    emit,
+    {
+      type: 'URL',
+      category: 'personal',
+      rule: 'document-share-path',
+      confidence: 0.85,
+    },
+    0,
+    isDocumentShareLink
+  );
+  detectEmployerContinuation(text, emit);
+  detectDatedEmployers(text, emit, range, debug, options);
+}
+
+function detectEmployerContinuation(text, emit) {
+  const continuation = new RegExp(
+    `,[ \\t]*(?:потом|затем)[ \\t]+в[ \\t]+(${employerName})`,
+    'uy'
+  );
+  for (const match of text.matchAll(
+    new RegExp(
+      `(?<!${boundary})[Рр]аботал[аи]?[ \\t]+в[ \\t]+(${employerName})`,
+      'gu'
+    )
+  )) {
+    continuation.lastIndex = match.index + match[0].length;
+    let next = continuation.exec(text);
+    if (next) {
+      emitEmployer(match);
+    }
+    while (next) {
+      emitEmployer(next);
+      next = continuation.exec(text);
+    }
+  }
+  function emitEmployer(match) {
+    const end = match.index + match[0].length;
+    if (!isDocumentTechnology(match[1])) {
+      emit({
+        start: end - match[1].length,
+        end,
+        type: 'ORGANIZATION',
+        category: 'personal',
+        rule: 'organization-continuation',
+        confidence: 0.85,
+      });
+    }
+  }
 }
 
 function detectQueryContacts(text, emit) {
@@ -111,60 +200,3 @@ function detectQueryContacts(text, emit) {
     });
   }
 }
-
-function detectExperience(text, emit, range) {
-  // Date anchors bound each entry; do not guess at undated employer/title fragments.
-  collectMatches(
-    text,
-    new RegExp(
-      `(?:^|[\\r\\n])[ \\t]*[^|\\r\\n]{1,120}\\|[ \\t]*(${name})[ \\t]*\\|[ \\t]*${range}`,
-      'gu'
-    ),
-    emit,
-    {
-      type: 'ORGANIZATION',
-      rule: 'organization-experience',
-      category: 'personal',
-      confidence: 0.85,
-    },
-    1
-  );
-  const dated = new RegExp(`([^\\r\\n;|]{1,320}?)\\((${range})\\)`, 'gu');
-  for (const match of text.matchAll(dated)) {
-    const prefix = match[1];
-    const content = prefix
-      .replace(/^.*\(\d[^)]*\)[, \t]*/, '')
-      .replace(/^[, \t]+/, '');
-    const chunks = [...content.matchAll(/[^,]+/g)];
-    const parts = chunks.map((part) => part[0].trim());
-    let value;
-    if (parts.length > 1 && role.test(parts[0])) {
-      let index = 1;
-      while (index < parts.length - 1 && /^[\p{Ll}]/u.test(parts[index])) {
-        index++;
-      }
-      value = content.slice(chunks[index].index).trim();
-    } else if (parts.length === 2 && role.test(parts[1])) {
-      value = parts[0].trim();
-    }
-    if (
-      !value ||
-      !/^[\p{Lu}«“"]/u.test(value) ||
-      value.includes('[REDACTED]')
-    ) {
-      continue;
-    }
-    const start = match.index + prefix.lastIndexOf(value);
-    if (start < match.index) {
-      continue;
-    }
-    emit({
-      start,
-      end: start + value.length,
-      type: 'ORGANIZATION',
-      category: 'personal',
-      rule: 'organization-experience',
-      confidence: 0.85,
-    });
-  }
-}

package/src/document-fields.js

@@ -15,5 +15,7 @@ const types = new Map(
   )
 );
 export function documentFieldType(field) {
-  return types.get(field.replace(/[_-]/g, '').toLowerCase());
+  return types.get(normalizeFieldName(field));
 }
+export const normalizeFieldName = (field) =>
+  field.replace(/[_-]/g, '').toLowerCase();

package/src/document-vocabulary.js

@@ -1,71 +1,83 @@
 // Authored, deliberately finite vocabulary; no network or model downloads.
 // Each row follows localeVocabulary: locale, legal forms, school types, places.
-export const documentVocabulary = [
+export const documentVocabulary = Object.freeze(
   [
-    'en',
-    'Inc|LLC|Ltd|PLC|Corporation',
-    'University|College|Institute|School',
-    'Springfield|Riverton|London|New York|California',
-  ],
-  [
-    'es',
-    'S.A.|S.L.|S.L.U.',
-    'Universidad|Colegio|Instituto|Escuela',
-    'Madrid|Barcelona|Andalucía',
-  ],
-  [
-    'fr',
-    'SARL|SAS|SA|S.A.',
-    'Université|Collège|Institut|École',
-    'Paris|Lyon|Bretagne',
-  ],
-  [
-    'de',
-    'GmbH|AG|KG|UG',
-    'Universität|Hochschule|Institut|Schule',
-    'Berlin|München|Bayern',
-  ],
-  [
-    'pt',
-    'Ltda|S.A.|Unipessoal',
-    'Universidade|Faculdade|Instituto|Escola',
-    'Lisboa|São Paulo|Algarve',
-  ],
-  [
-    'ru',
-    'ООО|АО|ПАО|ИП|ЗАО',
-    'Университет|университет|Институт|институт|Школа|школа',
-    'Москва|Санкт-Петербург|Казань|Татарстан|Ярославль|Кострома|Новосибирск|Екатеринбург|Нижний Новгород|Самара|Омск|Челябинск|Ростов-на-Дону|Уфа|Красноярск|Пермь|Воронеж|Волгоград|Саратов|Тюмень|Тольятти|Ижевск|Барнаул|Иркутск|Хабаровск|Владивосток|Махачкала|Томск|Оренбург|Кемерово|Рязань|Астрахань|Пенза|Липецк|Киров|Чебоксары|Тула|Калининград|Курск|Улан-Удэ|Ставрополь|Сочи|Тверь|Брянск|Белгород|Владимир|Архангельск|Калуга|Смоленск|Вологда|Саранск|Орёл',
-  ],
-  ['ja', '株式会社|有限会社|合同会社', '大学|学院|学校', '東京|大阪|北海道'],
-  ['zh', '有限公司|股份有限公司', '大学|学院|学校', '北京|上海|广东'],
-  ['zh-Hant', '有限公司|股份有限公司', '大學|學院|學校', '臺北|台北|高雄|臺灣'],
-  ['ko', '주식회사|유한회사', '대학교|대학|학교', '서울|부산|경기도'],
-  ['ar', 'ذ.م.م|شركة', 'جامعة|معهد|مدرسة', 'القاهرة|دبي|الرياض'],
-  [
-    'he',
-    'בע״מ|בע"מ|חברה',
-    'אוניברסיטת|אוניברסיטה|מכללת|מכון',
-    'ירושלים|תל אביב|חיפה',
-  ],
-  [
-    'hi',
-    'प्राइवेट लिमिटेड|लिमिटेड',
-    'विश्वविद्यालय|महाविद्यालय|संस्थान',
-    'दिल्ली|मुंबई|महाराष्ट्र',
-  ],
-  [
-    'tr',
-    'A.Ş.|Ltd. Şti.',
-    'Üniversitesi|Üniversite|Enstitüsü|Okulu',
-    'İstanbul|Ankara|İzmir',
-  ],
-  ['id', 'PT|CV|Tbk', 'Universitas|Institut|Sekolah', 'Jakarta|Bandung|Bali'],
-  ['vi', 'TNHH|CTCP', 'Đại học|Trường|Học viện', 'Hà Nội|Hồ Chí Minh|Đà Nẵng'],
-  [
-    'th',
-    'บริษัท|จำกัด',
-    'มหาวิทยาลัย|วิทยาลัย|โรงเรียน',
-    'กรุงเทพมหานคร|เชียงใหม่|ภูเก็ต',
-  ],
-];
+    [
+      'en',
+      'Inc|LLC|Ltd|PLC|Corporation',
+      'University|College|Institute|School',
+      'Springfield|Riverton|London|New York|California',
+    ],
+    [
+      'es',
+      'S.A.|S.L.|S.L.U.',
+      'Universidad|Colegio|Instituto|Escuela',
+      'Madrid|Barcelona|Andalucía',
+    ],
+    [
+      'fr',
+      'SARL|SAS|SA|S.A.',
+      'Université|Collège|Institut|École',
+      'Paris|Lyon|Bretagne',
+    ],
+    [
+      'de',
+      'GmbH|AG|KG|UG',
+      'Universität|Hochschule|Institut|Schule',
+      'Berlin|München|Bayern',
+    ],
+    [
+      'pt',
+      'Ltda|S.A.|Unipessoal',
+      'Universidade|Faculdade|Instituto|Escola',
+      'Lisboa|São Paulo|Algarve',
+    ],
+    [
+      'ru',
+      'ООО|АО|ПАО|ИП|ЗАО',
+      'Университет|университет|Институт|институт|Школа|школа',
+      'Москва|Санкт-Петербург|Казань|Татарстан|Ярославль|Кострома|Новосибирск|Екатеринбург|Нижний Новгород|Самара|Омск|Челябинск|Ростов-на-Дону|Уфа|Красноярск|Пермь|Воронеж|Волгоград|Саратов|Тюмень|Тольятти|Ижевск|Барнаул|Иркутск|Хабаровск|Владивосток|Махачкала|Томск|Оренбург|Кемерово|Рязань|Астрахань|Пенза|Липецк|Киров|Чебоксары|Тула|Калининград|Курск|Улан-Удэ|Ставрополь|Сочи|Тверь|Брянск|Белгород|Владимир|Архангельск|Калуга|Смоленск|Вологда|Саранск|Орёл',
+    ],
+    ['ja', '株式会社|有限会社|合同会社', '大学|学院|学校', '東京|大阪|北海道'],
+    ['zh', '有限公司|股份有限公司', '大学|学院|学校', '北京|上海|广东'],
+    [
+      'zh-Hant',
+      '有限公司|股份有限公司',
+      '大學|學院|學校',
+      '臺北|台北|高雄|臺灣',
+    ],
+    ['ko', '주식회사|유한회사', '대학교|대학|학교', '서울|부산|경기도'],
+    ['ar', 'ذ.م.م|شركة', 'جامعة|معهد|مدرسة', 'القاهرة|دبي|الرياض'],
+    [
+      'he',
+      'בע״מ|בע"מ|חברה',
+      'אוניברסיטת|אוניברסיטה|מכללת|מכון',
+      'ירושלים|תל אביב|חיפה',
+    ],
+    [
+      'hi',
+      'प्राइवेट लिमिटेड|लिमिटेड',
+      'विश्वविद्यालय|महाविद्यालय|संस्थान',
+      'दिल्ली|मुंबई|महाराष्ट्र',
+    ],
+    [
+      'tr',
+      'A.Ş.|Ltd. Şti.',
+      'Üniversitesi|Üniversite|Enstitüsü|Okulu',
+      'İstanbul|Ankara|İzmir',
+    ],
+    ['id', 'PT|CV|Tbk', 'Universitas|Institut|Sekolah', 'Jakarta|Bandung|Bali'],
+    [
+      'vi',
+      'TNHH|CTCP',
+      'Đại học|Trường|Học viện',
+      'Hà Nội|Hồ Chí Minh|Đà Nẵng',
+    ],
+    [
+      'th',
+      'บริษัท|จำกัด',
+      'มหาวิทยาลัย|วิทยาลัย|โรงเรียน',
+      'กรุงเทพมหานคร|เชียงใหม่|ภูเก็ต',
+    ],
+  ].map((row) => Object.freeze(row))
+);

package/src/engines.js

@@ -16,6 +16,8 @@ import {
 import { failure, internalFailure } from './detection.js';
 import { projectText } from './projection.js';
 import { decodedRuns } from './encoded.js';
+import { hasCustomTransforms } from './transforms.js';
+import { builtInEnabled } from './policy.js';
 import {
   initializePlaceholders,
   placeholderKnown,
@@ -70,7 +72,8 @@ async function secretlintDetect(text) {
   }));
 }
 
-async function detect(engine, text, limit) {
+const secretlintEngine = { detect: secretlintDetect };
+async function detect(engine, text, limit, options) {
   try {
     const found = await engine.detect(text);
     if (!Array.isArray(found)) {
@@ -79,7 +82,12 @@ async function detect(engine, text, limit) {
     if (found.length > limit) {
       throw failure('ERR_LIMIT');
     }
-    return found.map((finding) => sanitizeFinding(finding, text.length));
+    return found
+      .map((finding) => sanitizeFinding(finding, text.length))
+      .filter(
+        (finding) =>
+          engine !== secretlintEngine || builtInEnabled(finding, options)
+      );
   } catch {
     throw failure('ERR_ENGINE');
   }
@@ -88,7 +96,7 @@ async function detect(engine, text, limit) {
 async function engineFindings(engine, projection, options, emit) {
   const limit = options.maxFindings ?? 100000;
   if (projection.changed) {
-    for (const f of await detect(engine, projection.text, limit)) {
+    for (const f of await detect(engine, projection.text, limit, options)) {
       emit({
         ...f,
         start: projection.starts[f.start],
@@ -106,7 +114,7 @@ async function engineFindings(engine, projection, options, emit) {
     const view = projectText(run.text).text;
     const decoded = markFake(
       view,
-      await detect(engine, view, limit),
+      await detect(engine, view, limit, options),
       options
     ).filter((f) => actionable(f, options));
     if (!decoded.length) {
@@ -124,6 +132,28 @@ async function engineFindings(engine, projection, options, emit) {
   }
 }
 
+async function collectEngineFindings(text, engines, options, emit) {
+  const projection = engines.length ? projectText(text) : undefined;
+  for (const engine of engines) {
+    if (engine !== secretlintEngine && !options.detectors?.includes(engine)) {
+      continue;
+    }
+    for (const finding of await detect(
+      engine,
+      text,
+      options.maxFindings ?? 100000,
+      options
+    )) {
+      emit(finding);
+    }
+    await engineFindings(engine, projection, options, (finding) => {
+      if (builtInEnabled(finding, options)) {
+        emit(finding);
+      }
+    });
+  }
+}
+
 async function analyzeStructured(
   text,
   initial,
@@ -140,7 +170,14 @@ async function analyzeStructured(
         batch.text,
         [],
         true,
-        options,
+        {
+          ...options,
+          detectors: [],
+          properNames: undefined,
+          documentEntities: options.properNames
+            ? false
+            : options.documentEntities,
+        },
         nativeFindings === undefined ? undefined : []
       ),
       options
@@ -152,7 +189,11 @@ async function analyzeStructured(
         view.text,
         [],
         true,
-        { ...options, structured: undefined },
+        {
+          ...options,
+          structured: undefined,
+          properNames: view.prose ? options.properNames : undefined,
+        },
         nativeFindings === undefined ? undefined : []
       );
       for (const finding of detected) {
@@ -189,9 +230,7 @@ export function createSanitizer(options = {}) {
     }
   }
   const engines =
-    options.secretlint === false || options.knownOnly
-      ? []
-      : [{ detect: secretlintDetect }];
+    options.secretlint === false || options.knownOnly ? [] : [secretlintEngine];
   if (!options.knownOnly) {
     engines.push(...(options.detectors ?? []));
   }
@@ -220,20 +259,15 @@ export function createSanitizer(options = {}) {
     }
     // Native inspection performs its own normalization. Additional views are
     // needed here only when a required external engine is enabled.
-    const projection = engines.length ? projectText(text) : undefined;
-    for (const engine of engines) {
-      for (const finding of await detect(
-        engine,
-        text,
-        options.maxFindings ?? 100000
-      )) {
-        emit(finding);
-      }
-      await engineFindings(engine, projection, options, emit);
-    }
+    await collectEngineFindings(text, engines, options, emit);
     const detected =
       nativeFindings === undefined
-        ? inspect(text, { ...options, structured: undefined, findings })
+        ? inspect(text, {
+            ...options,
+            structured: undefined,
+            detectors: [],
+            findings,
+          })
         : reviewFindings(text, [...nativeFindings, ...findings], options);
     return options.verifyPublic
       ? verifyFindings(text, detected, options.verifyPublic)
@@ -287,16 +321,17 @@ export function createSanitizer(options = {}) {
       findings,
       fullRedaction ? fullOptions : options,
       (decoded, recursiveOptions) =>
-        depth < 2
-          ? sanitizeText(
-              decoded,
-              depth + 1,
-              fullRedaction,
-              true,
-              recursiveOptions
-            )
-          : Promise.resolve({ text: '[REDACTED]' })
+        sanitizeNested(
+          sanitizeText,
+          decoded,
+          depth,
+          fullRedaction,
+          recursiveOptions
+        )
     );
+    if (await customResidual(result.text, options, plain, analyze)) {
+      throw failure('ERR_RESIDUAL');
+    }
     const sanitized = fullRedaction
       ? result.text
       : (
@@ -305,9 +340,13 @@ export function createSanitizer(options = {}) {
             findings,
             fullOptions,
             (decoded, recursiveOptions) =>
-              depth < 2
-                ? sanitizeText(decoded, depth + 1, true, true, recursiveOptions)
-                : Promise.resolve({ text: '[REDACTED]' })
+              sanitizeNested(
+                sanitizeText,
+                decoded,
+                depth,
+                true,
+                recursiveOptions
+              )
           )
         ).text;
     // Verification at the publication boundary runs every enabled engine.
@@ -320,15 +359,21 @@ export function createSanitizer(options = {}) {
         sanitizeText(result.text, depth, fullRedaction, plain, next, nextRound)
       );
     }
-    if (options.structured && !plain) {
-      verifyStructured(result.text, options);
-    }
+    verifyStructuredResult(result.text, options, plain);
     if (await placeholderResidual(result.text, options, plain, analyze)) {
       throw failure('ERR_RESIDUAL');
     }
     return { ...result, findings: auditFaked(findings, options, result) };
   }
 }
+async function customResidual(text, options, plain, analyze) {
+  return (
+    hasCustomTransforms(options) &&
+    (await analyze(text, [], plain, options)).some((f) =>
+      actionable(f, options)
+    )
+  );
+}
 
 function guarded(callback, cleanup = () => {}) {
   return async (...args) => {
@@ -399,3 +444,15 @@ async function verifyFindings(text, findings, verifier) {
   }
   return retained;
 }
+
+function sanitizeNested(sanitizeText, decoded, depth, fullRedaction, options) {
+  return depth < 2
+    ? sanitizeText(decoded, depth + 1, fullRedaction, true, options)
+    : Promise.resolve({ text: '[REDACTED]' });
+}
+
+function verifyStructuredResult(text, options, plain) {
+  if (options.structured && !plain) {
+    verifyStructured(text, options);
+  }
+}