@link-foundation/sensitive-data-sanitizer 0.18.0 → 0.19.0

package/src/document-catalog.js

@@ -0,0 +1,246 @@
+import { collectMatches } from './detection.js';
+import { isDocumentTechnology } from './document-experience.js';
+
+const b = '[\\p{L}\\p{N}_]';
+const proper = '[\\p{Lu}][\\p{L}\\p{M}.-]{1,47}';
+const rules = [
+  [
+    'PHONE',
+    'document-local-phone',
+    /(?<![\p{L}\p{N}_])(?:тел\.?|телефон|phone)[ \t]*:?[ \t]*(\d{7})(?!\d)/giu,
+    1,
+  ],
+  [
+    'PHONE',
+    'document-spaced-mobile',
+    /(?<![\p{L}\p{N}_])8[ \t]+9(?:[ \t]+\d){9}(?!\d)/gu,
+    0,
+  ],
+  [
+    'EMAIL',
+    'document-email-obfuscation',
+    /(?<![\p{L}\p{N}_])[\p{L}\p{N}_.+-]{1,64}\(собака\)[\p{L}\p{N}-]{1,63}(?:\(точка\)[\p{L}\p{N}-]{1,63}){1,6}(?![\p{L}\p{N}_])/giu,
+    0,
+  ],
+  [
+    'EMAIL',
+    'document-email-spaces',
+    /(?<![\p{L}\p{N}_])(?:[a-z][ \t]+){1,63}[a-z][ \t]*@[ \t]*[a-z\d-]{1,63}[ \t]*\.[ \t]*[a-z]{2,20}(?![\p{L}\p{N}_])/giu,
+    0,
+  ],
+  [
+    'USERNAME',
+    'document-messenger',
+    /(?<![\p{L}\p{N}_])(?:discord|max)[ \t]*:[ \t]*([a-z][a-z\d_.-]{1,63}(?:#\d{4})?)(?![\p{L}\p{N}_])/giu,
+    1,
+  ],
+  [
+    'URL',
+    'document-social-link',
+    /(?<![\p{L}\p{N}_./-])(?:https?:\/\/)?(?:www\.)?(?:ok\.ru\/profile|gitlab\.com|habr\.com\/ru\/users|facebook\.com)\/[\p{L}\p{N}_.-]{2,100}(?:\/)?(?:[?#][^\s"<>\\]*)?/giu,
+    0,
+  ],
+  [
+    'URL',
+    'document-resume-link',
+    /https?:\/\/[^\s/"<>\\]+\/(?:cv\.pdf(?:[?#][^\s"<>\\]*)?|cv(?:\/[^\s"<>\\]*)?|~[\p{L}\p{N}_.-]{1,100}\/?)/giu,
+    0,
+  ],
+  [
+    'DATE_OF_BIRTH',
+    'document-birthday',
+    /(?<![\p{L}\p{N}_])(?:д\.р\.|день рождения)[ \t]*:?[ \t]*(\d{1,2}(?:[/.]\d{1,2}[/.]\d{2,4}|[ \t]+(?:января|февраля|марта|апреля|мая|июня|июля|августа|сентября|октября|ноября|декабря)(?:[ \t]+\d{4})?))(?!\d)/giu,
+    1,
+  ],
+  [
+    'AGE',
+    'document-age-unit',
+    /(?<![\p{L}\p{N}_])\d{1,3}[ \t]*(?:y\.o\.|лет[ \t]+от[ \t]+роду)(?![\p{L}\p{N}_])/giu,
+    0,
+  ],
+  [
+    'ID',
+    'document-military-id',
+    /(?<![\p{L}\p{N}_])военный билет[ \t]+([А-ЯЁ]{2}[ \t]+\d{7})(?!\d)/giu,
+    1,
+  ],
+  [
+    'VIN',
+    'document-vin',
+    /(?<![\p{L}\p{N}_])VIN[ \t]*:?[ \t]*([A-HJ-NPR-Z\d]{17})(?![\p{L}\p{N}_])/giu,
+    1,
+  ],
+  [
+    'ORGANIZATION',
+    'document-legal-lowercase',
+    /(?<![\p{L}\p{N}_])(?:ооо|зао|пао|ао)[ \t]+[а-яё][а-яё-]{1,47}(?![\p{L}\p{N}_])/gu,
+    0,
+  ],
+  [
+    'ORGANIZATION',
+    'document-employer-dash',
+    new RegExp(
+      `(?<!${b})(?:Компания|Место работы)[ \\t]+[-–—][ \\t]+(${proper})(?!${b})`,
+      'gu'
+    ),
+    1,
+  ],
+  [
+    'ORGANIZATION',
+    'document-employer-prefix',
+    new RegExp(`(?<!${b})(?:Группа|стартап)[ \\t]+(${proper})(?!${b})`, 'gu'),
+    1,
+  ],
+  [
+    'ORGANIZATION',
+    'document-bank-suffix',
+    new RegExp(`(?<!${b})${proper}[ \\t]+Банк(?!${b})`, 'gu'),
+    0,
+  ],
+  [
+    'ORGANIZATION',
+    'document-industry',
+    new RegExp(
+      `(?<!${b})(${proper})(?=[ \\t]*\\((?:ритейл|retail|финансы|finance|IT)\\))`,
+      'gu'
+    ),
+    1,
+  ],
+  [
+    'ORGANIZATION',
+    'document-reversed-role',
+    new RegExp(
+      `(?<!${b})(${proper})[ \\t]*(?:,|[-–—])[ \\t]*(?:менеджер|manager|директор|developer|инженер)(?!${b})`,
+      'gu'
+    ),
+    1,
+  ],
+  [
+    'ORGANIZATION',
+    'document-role-employer',
+    new RegExp(
+      `(?<!${b})(?:менеджер|директор|инженер)[ \\t]+(?:в[ \\t]+)?(${proper})(?!${b})`,
+      'gu'
+    ),
+    1,
+  ],
+  [
+    'ORGANIZATION',
+    'document-at-employer',
+    /(?<![\p{L}\p{N}_])at[ \t]+([A-Z][a-z][\p{L}.-]{0,45})(?![\p{L}\p{N}_])/gu,
+    1,
+  ],
+  [
+    'EDUCATION',
+    'document-school-context',
+    /(?<![\p{L}\p{N}_])(?:graduated from|бакалавр,?|диплом|колледж|[Кк]урсы|курс на)[ \t]+([\p{Lu}][\p{L}.-]{1,47})(?![\p{L}\p{N}_])/gu,
+    1,
+  ],
+  [
+    'EDUCATION',
+    'document-school-year',
+    /(?<![\p{L}\p{N}_])[А-ЯЁA-Z]{2,12}(?=,[ \t]*(?:19|20)\d{2}(?!\d))/gu,
+    0,
+  ],
+  [
+    'EDUCATION',
+    'document-msu',
+    /(?<![\p{L}\p{N}_])МГУ(?:[ \t]+им\.[ \t]+[А-ЯЁ][а-яё-]{1,40})?(?![\p{L}\p{N}_])/gu,
+    0,
+  ],
+  [
+    'PERSON',
+    'document-patronymic',
+    /(?<![\p{L}\p{N}_])[А-ЯЁ][а-яё]{2,32}(?:овна|евна|ична|ович|евич)(?![\p{L}\p{N}_])/gu,
+    0,
+  ],
+  [
+    'PERSON',
+    'document-uppercase-initials',
+    /(?<![\p{L}\p{N}_])[А-ЯЁ]{2,40}[ \t]+(?:[А-ЯЁA-Z]\.[ \t]*){1,2}(?![\p{L}\p{N}_])/gu,
+    0,
+  ],
+  [
+    'PERSON',
+    'document-unfamiliar-name',
+    /(?<![\p{L}\p{N}_])(?:[А-ЯЁ][а-яё]{1,30}[ \t]+(?:Ким|[А-ЯЁ][а-яё]{1,25}(?:чук|енко|швили|дзе))|Ли[ \t]+[А-ЯЁ][а-яё]{1,20})(?![\p{L}\p{N}_])/gu,
+    0,
+  ],
+  [
+    'ADDRESS',
+    'document-street-complete',
+    /(?<![\p{L}\p{N}_])(?:(?:ул\.?|улица|пр-т)[ \t]+[А-ЯЁ][а-яё-]{1,40},?[ \t]+(?:д\.[ \t]*)?\d{1,5}(?:-\d{1,5})?|[А-ЯЁ][а-яё-]{2,40}[ \t]+шоссе,[ \t]*\d{1,5})(?:,[ \t]*кв\.?[ \t]*\d{1,5})?(?!\d)/gu,
+    0,
+  ],
+  [
+    'LOCATION',
+    'document-region',
+    /(?<![\p{L}\p{N}_])[А-ЯЁ][а-яё-]{2,40}[ \t]+(?:область|район)(?![\p{L}\p{N}_])/gu,
+    0,
+  ],
+  [
+    'PERSONAL_ATTRIBUTE',
+    'document-citizenship',
+    /(?<![\p{L}\p{N}_])(?:гражданство|гражданин|гражданка)[ \t]+([А-ЯЁ][А-ЯЁа-яё-]{1,40})(?![\p{L}\p{N}_])/gu,
+    1,
+  ],
+  [
+    'LICENSE_PLATE',
+    'document-vehicle-plate',
+    /(?<![\p{L}\p{N}_])[АВЕКМНОРСТУХABEKMHOPCTYX]\d{3}[АВЕКМНОРСТУХABEKMHOPCTYX]{2}[ \t]*\d{2,3}(?:[ \t]+RUS)?(?![\p{L}\p{N}_])/gu,
+    0,
+  ],
+];
+
+export function detectDocumentCatalog(text, emit) {
+  for (const [type, rule, pattern, group] of rules) {
+    collectMatches(
+      text,
+      pattern,
+      emit,
+      { type, rule, category: 'personal', confidence: 0.85 },
+      group,
+      (value) => type !== 'ORGANIZATION' || !isDocumentTechnology(value)
+    );
+  }
+  // Lookalike substitution preserves UTF-16 offsets and is limited to words
+  // containing Cyrillic plus Latin letters. Ordinary English is untouched.
+  const lookalikes = {
+    A: 'А',
+    a: 'а',
+    B: 'В',
+    C: 'С',
+    c: 'с',
+    E: 'Е',
+    e: 'е',
+    H: 'Н',
+    K: 'К',
+    M: 'М',
+    O: 'О',
+    o: 'о',
+    P: 'Р',
+    p: 'р',
+    T: 'Т',
+    X: 'Х',
+    x: 'х',
+    y: 'у',
+  };
+  const folded = text.replace(/[\p{L}]+/gu, (word) =>
+    /[А-Яа-яЁё]/u.test(word)
+      ? word.replace(/[AaBCcEeHKMOoPpTXxy]/g, (c) => lookalikes[c])
+      : word
+  );
+  if (folded !== text) {
+    collectMatches(
+      folded,
+      /(?<![\p{L}\p{N}_])[А-ЯЁ][а-яё]{1,30}[ \t]+[А-ЯЁ][а-яё]{1,28}(?:ова|ева|ина|ов|ев|ин|ский|ская)(?![\p{L}\p{N}_])/gu,
+      emit,
+      {
+        type: 'PERSON',
+        rule: 'document-mixed-name',
+        category: 'personal',
+        confidence: 0.9,
+      }
+    );
+  }
+}

package/src/document-entities.js

@@ -6,6 +6,8 @@ import { documentDateRange as range } from './document-dates.js';
 import { detectDocumentExtras } from './document-extras.js';
 import { detectDocumentPatterns } from './document-patterns.js';
 import { isDocumentTechnology } from './document-experience.js';
+import { detectVocabulary } from './vocabulary.js';
+import { detectDocumentCatalog } from './document-catalog.js';
 
 const words = (column) =>
   [...new Set(documentVocabulary.flatMap((row) => row[column].split('|')))]
@@ -79,8 +81,10 @@ export function detectDocumentEntities(text, emit, options) {
   if (!options.documentEntities) {
     return;
   }
+  detectVocabulary(text, emit, options);
+  detectDocumentCatalog(text, emit);
   detectDocumentPatterns(text, emit);
-  detectDocumentExtras(text, emit, range, options.debug);
+  detectDocumentExtras(text, emit, range, options.debug, options);
   collectMatches(
     text,
     new RegExp(
@@ -120,7 +124,9 @@ export function detectDocumentEntities(text, emit, options) {
       capture,
       { type, rule, category: 'personal', confidence },
       group,
-      (value) => type !== 'ORGANIZATION' || !isDocumentTechnology(value)
+      (value) =>
+        type !== 'ORGANIZATION' ||
+        (!isDocumentTechnology(value) && value !== 'ОГРНИП')
     );
   }
   for (const match of text.matchAll(ruResidence)) {

package/src/document-experience.js

@@ -1,3 +1,5 @@
+import { compiledVocabulary } from './vocabulary.js';
+
 const technology = new Set(
   [
     'Python',
@@ -45,7 +47,7 @@ function commaParts(text) {
   return parts;
 }
 
-function employerSpan(prefix) {
+function employerSpan(prefix, roles) {
   // An industry suffix belongs to the entry, but is not an employer. A pipe
   // before the title remains a section separator, as in `Опыт | Менеджер, ...`.
   const comma = prefix.indexOf(',');
@@ -59,7 +61,7 @@ function employerSpan(prefix) {
   if (/^[=\s\d]/u.test(parts[0].text)) {
     index++;
   }
-  if (parts.length === 2 && reversedRole.test(parts[1].text)) {
+  if (parts.length === 2 && roles.test(parts[1].text)) {
     index = 0;
   } else {
     while (index < parts.length - 1 && /^\p{Ll}/u.test(parts[index].text)) {
@@ -77,7 +79,7 @@ function employerSpan(prefix) {
     : { start, end };
 }
 
-export function detectDatedEmployers(text, emit, range, debug) {
+export function detectDatedEmployers(text, emit, range, debug, options = {}) {
   let cursor = 0,
     count = 0;
   // Match only the anchor, then visit each preceding segment once. No lazy
@@ -91,7 +93,10 @@ export function detectDatedEmployers(text, emit, range, debug) {
         segment.lastIndexOf(';')
       ) + 1;
     const prefix = segment.slice(boundary).replace(/^[., \t]+/u, '');
-    const span = employerSpan(prefix);
+    const span = employerSpan(
+      prefix,
+      compiledVocabulary(options.vocabulary)?.roles ?? reversedRole
+    );
     if (span) {
       const offset = match.index - prefix.length;
       emit({

package/src/document-extras.js

@@ -18,7 +18,7 @@ const duration = new RegExp(
   `(?<!${boundary})(${name})(?=[ \\t]*\\(\\d{1,2}[ \\t]+(?:год|года|лет|years?)\\))`,
   'gu'
 );
-export function detectDocumentExtras(text, emit, range, debug) {
+export function detectDocumentExtras(text, emit, range, debug, options) {
   for (const [type, rule, pattern, group] of [
     ['PHONE', 'document-mobile-bare', /^[78]9\d{9}$/gm, 0],
     [
@@ -128,7 +128,7 @@ export function detectDocumentExtras(text, emit, range, debug) {
     isDocumentShareLink
   );
   detectEmployerContinuation(text, emit);
-  detectDatedEmployers(text, emit, range, debug);
+  detectDatedEmployers(text, emit, range, debug, options);
 }
 
 function detectEmployerContinuation(text, emit) {

package/src/document-vocabulary.js

@@ -1,71 +1,83 @@
 // Authored, deliberately finite vocabulary; no network or model downloads.
 // Each row follows localeVocabulary: locale, legal forms, school types, places.
-export const documentVocabulary = [
+export const documentVocabulary = Object.freeze(
   [
-    'en',
-    'Inc|LLC|Ltd|PLC|Corporation',
-    'University|College|Institute|School',
-    'Springfield|Riverton|London|New York|California',
-  ],
-  [
-    'es',
-    'S.A.|S.L.|S.L.U.',
-    'Universidad|Colegio|Instituto|Escuela',
-    'Madrid|Barcelona|Andalucía',
-  ],
-  [
-    'fr',
-    'SARL|SAS|SA|S.A.',
-    'Université|Collège|Institut|École',
-    'Paris|Lyon|Bretagne',
-  ],
-  [
-    'de',
-    'GmbH|AG|KG|UG',
-    'Universität|Hochschule|Institut|Schule',
-    'Berlin|München|Bayern',
-  ],
-  [
-    'pt',
-    'Ltda|S.A.|Unipessoal',
-    'Universidade|Faculdade|Instituto|Escola',
-    'Lisboa|São Paulo|Algarve',
-  ],
-  [
-    'ru',
-    'ООО|АО|ПАО|ИП|ЗАО',
-    'Университет|университет|Институт|институт|Школа|школа',
-    'Москва|Санкт-Петербург|Казань|Татарстан|Ярославль|Кострома|Новосибирск|Екатеринбург|Нижний Новгород|Самара|Омск|Челябинск|Ростов-на-Дону|Уфа|Красноярск|Пермь|Воронеж|Волгоград|Саратов|Тюмень|Тольятти|Ижевск|Барнаул|Иркутск|Хабаровск|Владивосток|Махачкала|Томск|Оренбург|Кемерово|Рязань|Астрахань|Пенза|Липецк|Киров|Чебоксары|Тула|Калининград|Курск|Улан-Удэ|Ставрополь|Сочи|Тверь|Брянск|Белгород|Владимир|Архангельск|Калуга|Смоленск|Вологда|Саранск|Орёл',
-  ],
-  ['ja', '株式会社|有限会社|合同会社', '大学|学院|学校', '東京|大阪|北海道'],
-  ['zh', '有限公司|股份有限公司', '大学|学院|学校', '北京|上海|广东'],
-  ['zh-Hant', '有限公司|股份有限公司', '大學|學院|學校', '臺北|台北|高雄|臺灣'],
-  ['ko', '주식회사|유한회사', '대학교|대학|학교', '서울|부산|경기도'],
-  ['ar', 'ذ.م.م|شركة', 'جامعة|معهد|مدرسة', 'القاهرة|دبي|الرياض'],
-  [
-    'he',
-    'בע״מ|בע"מ|חברה',
-    'אוניברסיטת|אוניברסיטה|מכללת|מכון',
-    'ירושלים|תל אביב|חיפה',
-  ],
-  [
-    'hi',
-    'प्राइवेट लिमिटेड|लिमिटेड',
-    'विश्वविद्यालय|महाविद्यालय|संस्थान',
-    'दिल्ली|मुंबई|महाराष्ट्र',
-  ],
-  [
-    'tr',
-    'A.Ş.|Ltd. Şti.',
-    'Üniversitesi|Üniversite|Enstitüsü|Okulu',
-    'İstanbul|Ankara|İzmir',
-  ],
-  ['id', 'PT|CV|Tbk', 'Universitas|Institut|Sekolah', 'Jakarta|Bandung|Bali'],
-  ['vi', 'TNHH|CTCP', 'Đại học|Trường|Học viện', 'Hà Nội|Hồ Chí Minh|Đà Nẵng'],
-  [
-    'th',
-    'บริษัท|จำกัด',
-    'มหาวิทยาลัย|วิทยาลัย|โรงเรียน',
-    'กรุงเทพมหานคร|เชียงใหม่|ภูเก็ต',
-  ],
-];
+    [
+      'en',
+      'Inc|LLC|Ltd|PLC|Corporation',
+      'University|College|Institute|School',
+      'Springfield|Riverton|London|New York|California',
+    ],
+    [
+      'es',
+      'S.A.|S.L.|S.L.U.',
+      'Universidad|Colegio|Instituto|Escuela',
+      'Madrid|Barcelona|Andalucía',
+    ],
+    [
+      'fr',
+      'SARL|SAS|SA|S.A.',
+      'Université|Collège|Institut|École',
+      'Paris|Lyon|Bretagne',
+    ],
+    [
+      'de',
+      'GmbH|AG|KG|UG',
+      'Universität|Hochschule|Institut|Schule',
+      'Berlin|München|Bayern',
+    ],
+    [
+      'pt',
+      'Ltda|S.A.|Unipessoal',
+      'Universidade|Faculdade|Instituto|Escola',
+      'Lisboa|São Paulo|Algarve',
+    ],
+    [
+      'ru',
+      'ООО|АО|ПАО|ИП|ЗАО',
+      'Университет|университет|Институт|институт|Школа|школа',
+      'Москва|Санкт-Петербург|Казань|Татарстан|Ярославль|Кострома|Новосибирск|Екатеринбург|Нижний Новгород|Самара|Омск|Челябинск|Ростов-на-Дону|Уфа|Красноярск|Пермь|Воронеж|Волгоград|Саратов|Тюмень|Тольятти|Ижевск|Барнаул|Иркутск|Хабаровск|Владивосток|Махачкала|Томск|Оренбург|Кемерово|Рязань|Астрахань|Пенза|Липецк|Киров|Чебоксары|Тула|Калининград|Курск|Улан-Удэ|Ставрополь|Сочи|Тверь|Брянск|Белгород|Владимир|Архангельск|Калуга|Смоленск|Вологда|Саранск|Орёл',
+    ],
+    ['ja', '株式会社|有限会社|合同会社', '大学|学院|学校', '東京|大阪|北海道'],
+    ['zh', '有限公司|股份有限公司', '大学|学院|学校', '北京|上海|广东'],
+    [
+      'zh-Hant',
+      '有限公司|股份有限公司',
+      '大學|學院|學校',
+      '臺北|台北|高雄|臺灣',
+    ],
+    ['ko', '주식회사|유한회사', '대학교|대학|학교', '서울|부산|경기도'],
+    ['ar', 'ذ.م.م|شركة', 'جامعة|معهد|مدرسة', 'القاهرة|دبي|الرياض'],
+    [
+      'he',
+      'בע״מ|בע"מ|חברה',
+      'אוניברסיטת|אוניברסיטה|מכללת|מכון',
+      'ירושלים|תל אביב|חיפה',
+    ],
+    [
+      'hi',
+      'प्राइवेट लिमिटेड|लिमिटेड',
+      'विश्वविद्यालय|महाविद्यालय|संस्थान',
+      'दिल्ली|मुंबई|महाराष्ट्र',
+    ],
+    [
+      'tr',
+      'A.Ş.|Ltd. Şti.',
+      'Üniversitesi|Üniversite|Enstitüsü|Okulu',
+      'İstanbul|Ankara|İzmir',
+    ],
+    ['id', 'PT|CV|Tbk', 'Universitas|Institut|Sekolah', 'Jakarta|Bandung|Bali'],
+    [
+      'vi',
+      'TNHH|CTCP',
+      'Đại học|Trường|Học viện',
+      'Hà Nội|Hồ Chí Minh|Đà Nẵng',
+    ],
+    [
+      'th',
+      'บริษัท|จำกัด',
+      'มหาวิทยาลัย|วิทยาลัย|โรงเรียน',
+      'กรุงเทพมหานคร|เชียงใหม่|ภูเก็ต',
+    ],
+  ].map((row) => Object.freeze(row))
+);