@link-foundation/sensitive-data-sanitizer 0.14.0 → 0.16.0

package/src/russian-declension.js

@@ -0,0 +1,167 @@
+// Finite Russian declension shared by fake names and fake entities. Case order:
+// nominative, genitive, dative, accusative, instrumental, prepositional.
+// `|` lists a rarer spelling, accepted when reading a form, never generated.
+const fleeting = { павел: 'павл', лев: 'льв', ветер: 'ветр' };
+const hush = /[жчшщц]$/u;
+const a = (stem) => [
+  'а',
+  /[гкхжчшщ]$/u.test(stem) ? 'и' : 'ы',
+  'е',
+  'у',
+  /[жчшщц]$/u.test(stem) ? 'ей|ею' : 'ой|ою',
+  'е',
+];
+const adjectives = [
+  [
+    /[ый]й$|ой$/u,
+    2,
+    (w) => [w.slice(-2), 'ого', 'ому', w.slice(-2), 'ым', 'ом'],
+  ],
+  [/[кгхжчшщ]ий$/u, 2, () => ['ий', 'ого', 'ому', 'ий', 'им', 'ом']],
+  [/ий$/u, 2, () => ['ий', 'его', 'ему', 'ий', 'им', 'ем']],
+  [/ая$/u, 2, () => ['ая', 'ой', 'ой', 'ую', 'ой|ою', 'ой']],
+  [/яя$/u, 2, () => ['яя', 'ей', 'ей', 'юю', 'ей|ею', 'ей']],
+  [/ое$/u, 2, () => ['ое', 'ого', 'ому', 'ое', 'ым', 'ом']],
+  [/ее$/u, 2, () => ['ее', 'его', 'ему', 'ее', 'им', 'ем']],
+];
+function noun(word, sex, animate) {
+  const accusative = (genitive, nominative = '') =>
+    animate ? genitive : nominative;
+  if (fleeting[word]) {
+    const hard = ['а', 'у', accusative('а'), 'ом', 'е'];
+    return [word, ...hard.map((ending) => fleeting[word] + ending)];
+  }
+  const table = [
+    [/ия$/u, 2, ['ия', 'ии', 'ии', 'ию', 'ией|иею', 'ии']],
+    [/ье$/u, 2, ['ье', 'ья', 'ью', 'ье', 'ьем', 'ье']],
+    [/ие$/u, 2, ['ие', 'ия', 'ию', 'ие', 'ием', 'ии']],
+    [/ий$/u, 2, ['ий', 'ия', 'ию', accusative('ия', 'ий'), 'ием', 'ии']],
+    [/й$/u, 1, ['й', 'я', 'ю', accusative('я', 'й'), 'ем', 'е']],
+    [/я$/u, 1, ['я', 'и', 'е', 'ю', 'ей|ею', 'е']],
+    [/а$/u, 1, a(word.slice(0, -1))],
+    [/о$/u, 1, ['о', 'а', 'у', 'о', 'ом', 'е']],
+  ];
+  for (const [ending, remove, cases] of table) {
+    if (ending.test(word)) {
+      return cases.map((suffix) => attach(word.slice(0, -remove), suffix));
+    }
+  }
+  if (word.endsWith('ь')) {
+    const stem = word.slice(0, -1);
+    const feminine = ['ь', 'и', 'и', 'ь', 'ью', 'и'];
+    const masculine = ['ь', 'я', 'ю', accusative('я', 'ь'), 'ем', 'е'];
+    const cases =
+      sex === 'female'
+        ? feminine
+        : sex === 'male'
+          ? masculine
+          : masculine.map((suffix, i) =>
+              suffix === feminine[i] ? suffix : `${suffix}|${feminine[i]}`
+            );
+    return cases.map((suffix) => attach(stem, suffix));
+  }
+  if (!/[бвгджзклмнпрстфхцчшщ]$/u.test(word)) {
+    return Array(6).fill(word); // Indeclinable: Шевченко, Сочи.
+  }
+  return [
+    '',
+    'а',
+    'у',
+    accusative('а'),
+    hush.test(word) ? 'ем' : 'ом',
+    'е',
+  ].map((suffix) => attach(word, suffix));
+}
+const attach = (stem, suffix) =>
+  suffix
+    .split('|')
+    .map((ending) => stem + ending)
+    .join('|');
+function adjective(word) {
+  for (const [ending, remove, cases] of adjectives) {
+    if (ending.test(word)) {
+      return cases(word).map((suffix) =>
+        attach(word.slice(0, -remove), suffix)
+      );
+    }
+  }
+  return undefined;
+}
+function surname(word, sex) {
+  if (/(?:ов|ев|ёв|ин|ын)$/u.test(word)) {
+    return ['', 'а', 'у', 'а', 'ым|ом', 'е'].map((s) => attach(word, s));
+  }
+  if (/(?:ова|ева|ёва|ина|ына)$/u.test(word)) {
+    const stem = word.slice(0, -1);
+    return ['а', 'ой', 'ой', 'у', 'ой|ою', 'ой'].map((s) => attach(stem, s));
+  }
+  const forms = adjective(word);
+  if (forms) {
+    return forms;
+  }
+  return sex === 'female' && !/[ая]$/u.test(word)
+    ? Array(6).fill(word)
+    : noun(word, sex, true);
+}
+export const russianNoun = (word, options) =>
+  options.adjective && adjective(word)
+    ? adjective(word)
+    : noun(word, options.sex, options.animate);
+// `kind` is the name slot: lastName, firstName or middleName.
+export function russianNameForms(word, kind, sex) {
+  const lower = word.toLowerCase();
+  if (kind === 'lastName') {
+    return surname(lower, sex);
+  }
+  if (kind === 'middleName' && /ич$/u.test(lower)) {
+    return ['', 'а', 'у', 'а', 'ем', 'е'].map((s) => attach(lower, s));
+  }
+  return noun(lower, sex, true);
+}
+const fold = (word) => word.toLowerCase().replaceAll('ё', 'е');
+// Every case a surface can stand for, in case order.
+export function casesOf(surface, forms) {
+  const wanted = fold(surface);
+  return forms.flatMap((form, index) =>
+    fold(form).split('|').includes(wanted) ? [index] : []
+  );
+}
+export const formOf = (forms, index) => forms[index].split('|')[0];
+// Cases governed by a preposition, in order of preference. Bare forms like
+// `Ромашке` (dative or prepositional) cannot be told apart without it.
+const governed = [
+  [/^(?:в|во|на|о|об|обо|при)$/u, [5, 3]],
+  [/^(?:к|ко|по)$/u, [2]],
+  [/^(?:с|со|над|под|перед|за)$/u, [4, 1, 3]],
+  [/^(?:у|от|до|из|без|для|около|после|кроме|вокруг|против)$/u, [1]],
+];
+const lastWord = /(?<![\p{L}\p{M}\p{N}_-])(\p{L}{1,7})[ \t\u00a0]+([«“"]?)$/u;
+// `в Лютике и Ромашке`: a conjunct takes the case of the word before it.
+const conjunct =
+  /[\p{L}\p{M}-]{1,32}[»”"]?(?:,|[ \t\u00a0]+(?:и|или))[ \t\u00a0]+[«“"]?$/u;
+export function governedCases(text, start) {
+  const before = text.slice(Math.max(0, start - 64), start);
+  const conjunction = conjunct.exec(before);
+  const source = conjunction ? before.slice(0, conjunction.index) : before;
+  const match = lastWord.exec(source.slice(-12));
+  if (!match) {
+    return { cases: [] };
+  }
+  const word = match[1].toLowerCase();
+  const rule = governed.find(([pattern]) => pattern.test(word));
+  return { cases: rule?.[1] ?? [], quoted: Boolean(match[2]) };
+}
+// Intersects the candidates of each word; a word that fits no case is ignored.
+// A quoted name in the nominative stays there: в «Вектор».
+export function commonCase(candidates, context = { cases: [] }) {
+  const present = candidates.filter((cases) => cases.length);
+  const common = [0, 1, 2, 3, 4, 5].filter((index) =>
+    present.every((cases) => cases.includes(index))
+  );
+  if (context.quoted && common.includes(0)) {
+    return 0;
+  }
+  return (
+    context.cases.find((index) => common.includes(index)) ?? common[0] ?? 0
+  );
+}

package/src/russian-names.js

@@ -0,0 +1,64 @@
+import { firstNames } from './names.js';
+
+// Finite Russian name grammar shared by known-value matching and fake mode.
+const patronymic = /(?:ич|овна|евна|ична|vich|ovna|evna)$/iu;
+const surname = /(?:ов|ев|ин|ын|ский|цкий|ова|ева|ина|ына|ская|цкая)$/iu;
+const fold = (word) => word.toLowerCase().replaceAll('ё', 'е');
+const givenNames = new Set(firstNames.ru.split(' ').map(fold));
+
+export const isPatronymic = (word) => patronymic.test(word);
+export const isGivenName = (word) => givenNames.has(fold(word));
+
+// Only decides whether a word beside a known person belongs to that mention, so
+// it stays apart from the detector's list, which would match these names anywhere.
+const mentionGivenNames =
+  'Роман Антон Илья Артём Никита Егор Глеб Денис Игорь Константин Кирилл Леонид Матвей Тимофей Фёдор Руслан Станислав Вячеслав Вадим Валерий Геннадий Григорий Степан Тимур Ярослав Юлия Дарья Полина Ксения Виктория Анастасия Вероника Валерия Алёна Оксана Людмила Галина Лариса Надежда Вера Любовь Кристина Софья Диана Яна Лилия Альбина Жанна Регина Инна Карина Милана Арина Варвара Елизавета Маргарита';
+const anyGivenName = new Set(
+  [...Object.values(firstNames), mentionGivenNames]
+    .flatMap((list) => list.split(' '))
+    .map(fold)
+);
+const inflectedPatronymic =
+  /(?:[оеь]вич|[оеь]ич|ич)(?:а|у|ем|е)?$|(?:овн|евн|ичн)(?:а|ы|е|у|ой|ою)$/iu;
+const nameCase = /(?:ом|ем|ой|ей|ою|ею|а|я|у|ю|е|ы|и)$/u;
+// A capitalized given name or patronymic in any case, for deciding whether a
+// neighbouring word belongs to the same mention. Lemmas are recovered by a
+// finite ending list, so this stays a lookup and never parses the sentence.
+export function isNameForm(word) {
+  const lower = fold(word);
+  if (anyGivenName.has(lower) || inflectedPatronymic.test(lower)) {
+    return true;
+  }
+  const stem = lower.replace(nameCase, '');
+  return (
+    stem !== lower &&
+    stem.length >= 2 &&
+    ['', 'а', 'я', 'й', 'ь'].some((ending) => anyGivenName.has(stem + ending))
+  );
+}
+
+// Returns word indexes by role, or undefined when the roles are not evident.
+// `Имя Отчество Фамилия` is checked first so that surnames in -ич, as in
+// `Шостакович Дмитрий Дмитриевич`, are not mistaken for patronymics.
+export function russianNameRoles(words, known = isGivenName) {
+  if (words.length === 3) {
+    if (isPatronymic(words[1])) {
+      return { given: 0, patronymic: 1, surname: 2 };
+    }
+    return isPatronymic(words[2])
+      ? { surname: 0, given: 1, patronymic: 2 }
+      : undefined;
+  }
+  if (words.length !== 2) {
+    return undefined;
+  }
+  if (isPatronymic(words[1])) {
+    return { given: 0, patronymic: 1 };
+  }
+  const candidates = [0, 1].filter(
+    (i) => surname.test(words[i]) && !known(words[i])
+  );
+  return candidates.length === 1
+    ? { surname: candidates[0], given: 1 - candidates[0] }
+    : undefined;
+}

package/src/sanitizer.js

@@ -1,4 +1,5 @@
 import { generatedValues } from './fake.js';
+import { prepareFakeText } from './fake-entities.js';
 import { alignEscapes } from './escape-spans.js';
 import { actionable } from './confidence.js';
 import { propagate, confirmationOptions } from './propagation.js';
@@ -25,6 +26,7 @@ import {
   hasPlaceholders,
   placeholderValues,
   placeholderNameFindings,
+  coverAdjacentNames,
   nestedPlaceholderOptions,
 } from './placeholders.js';
 import {
@@ -465,7 +467,11 @@ export function reviewFindings(text, findings, options) {
     text,
     markFake(
       text,
-      propagate(text, markFake(text, [...unique.values()], options), options),
+      coverAdjacentNames(
+        text,
+        propagate(text, markFake(text, [...unique.values()], options), options),
+        options
+      ),
       options
     ).sort(
       (a, b) =>
@@ -692,7 +698,7 @@ export function redactResult(
 ) {
   options = applyProfile(options);
   const spans = validatedSpans(text, findings, options);
-  options = initializePlaceholders(options);
+  options = prepareFakeText(text, findings, initializePlaceholders(options));
   preparePlaceholderText(text, findings, options);
   const parts = [];
   const faked = [];
@@ -719,6 +725,11 @@ export function redactResult(
     const transform =
       options.transformations?.[span.type] ?? options.transformation;
     if (retainedReplacement(transform, 'placeholder', span, replacement)) {
+      // `Organization-1` followed by `99` would read as a token that does not
+      // exist, so a span that ends there fails the run instead.
+      if (/\d$/.test(replacement) && /\d/.test(text[span.end] ?? '')) {
+        throw failure('ERR_RESIDUAL');
+      }
       occurrences.push({
         start: outputLength,
         end: outputLength + replacement.length,

package/src/transforms.js

@@ -41,6 +41,22 @@ export function validateTransforms(options) {
 function positiveInteger(value) {
   return Number.isSafeInteger(value) && value > 0;
 }
+const poolTypes = new Set(['ORGANIZATION', 'LOCATION', 'EDUCATION']);
+function validPools(pools) {
+  return (
+    pools === undefined ||
+    (Boolean(pools) &&
+      typeof pools === 'object' &&
+      !Array.isArray(pools) &&
+      Object.entries(pools).every(
+        ([type, names]) =>
+          poolTypes.has(type) &&
+          Array.isArray(names) &&
+          names.length > 0 &&
+          names.every((name) => typeof name === 'string' && name.trim())
+      ))
+  );
+}
 function validateTransform(transform) {
   if (!transform || !modes.has(transform.mode)) {
     throw failure('ERR_CONFIG');
@@ -54,7 +70,8 @@ function validateTransform(transform) {
       Buffer.byteLength(transform.key) >= 16 &&
       (transform.credentials === undefined ||
         typeof transform.credentials === 'boolean') &&
-      (transform.mrzCountry === undefined || transform.mrzCountry === 'UTO'),
+      (transform.mrzCountry === undefined || transform.mrzCountry === 'UTO') &&
+      validPools(transform.pools),
     pseudonym: () =>
       typeof transform.key === 'string' &&
       Buffer.byteLength(transform.key) >= 16,
@@ -130,7 +147,7 @@ function masked(value, finding, transform) {
     ? REDACTED
     : `${chars.slice(0, start).join('')}${transform.marker ?? '*'.repeat(chars.length - start - end)}${end ? chars.slice(-end).join('') : ''}`;
 }
-function transformed(value, finding, transform) {
+function transformed(value, finding, transform, options) {
   if (!transform || transform.mode === 'redact') {
     return REDACTED;
   }
@@ -141,7 +158,7 @@ function transformed(value, finding, transform) {
       : REDACTED;
   }
   if (transform.mode === 'fake') {
-    return realisticFake(value, finding, transform);
+    return realisticFake(value, finding, transform, options);
   }
   if (finding.category === 'credential') {
     return REDACTED;
@@ -198,6 +215,6 @@ function renderValue(text, finding, transform, options) {
   const result =
     transform?.mode === 'placeholder'
       ? placeholderReplacement(raw, finding, transform, options)
-      : transformed(raw, finding, transform);
+      : transformed(raw, finding, transform, options);
   return jsonQuoted ? JSON.stringify(result).slice(1, -1) : result;
 }