@link-foundation/sensitive-data-sanitizer 0.0.0-stage → 0.12.0

package/src/log-bytes.js

@@ -0,0 +1,85 @@
+import { TextDecoder } from 'node:util';
+import { sequenceWidth, validSequence } from './utf8.js';
+
+// Surrogate escape: invalid bytes use unpaired U+DC80..U+DCFF in the internal
+// text view. Valid UTF-8 (including non-BMP characters) keeps its Unicode form.
+// encodeLogText restores those bytes after sanitization; no lossy replacement.
+export function decodeLogBytes(input) {
+  const bytes = Buffer.from(input);
+  try {
+    return new TextDecoder('utf-8', { fatal: true, ignoreBOM: true }).decode(
+      bytes
+    );
+  } catch {
+    const parts = [];
+    let start = 0;
+    for (let i = 0; i < bytes.length; ) {
+      const width = sequenceWidth(bytes[i]);
+      if (validSequence(bytes, i, width)) {
+        i += width;
+        continue;
+      }
+      parts.push(
+        bytes.toString('utf8', start, i),
+        String.fromCharCode(0xdc00 + bytes[i])
+      );
+      start = ++i;
+    }
+    parts.push(bytes.toString('utf8', start));
+    return parts.join('');
+  }
+}
+
+export function encodeLogText(text) {
+  const parts = [];
+  let start = 0;
+  for (let i = 0; i < text.length; i++) {
+    const code = text.charCodeAt(i);
+    if (
+      code >= 0xd800 &&
+      code <= 0xdbff &&
+      text.charCodeAt(i + 1) >= 0xdc00 &&
+      text.charCodeAt(i + 1) <= 0xdfff
+    ) {
+      i++;
+    } else if (code >= 0xdc80 && code <= 0xdcff) {
+      parts.push(
+        Buffer.from(text.slice(start, i)),
+        Buffer.from([code - 0xdc00])
+      );
+      start = i + 1;
+    }
+  }
+  if (!parts.length) {
+    return Buffer.from(text);
+  }
+  parts.push(Buffer.from(text.slice(start)));
+  return Buffer.concat(parts);
+}
+
+export class LogDecoder {
+  pending = Buffer.alloc(0);
+  decode(input = Buffer.alloc(0), { stream = false } = {}) {
+    const bytes = this.pending.length
+      ? Buffer.concat([this.pending, input])
+      : Buffer.from(input);
+    let end = bytes.length;
+    if (stream && end) {
+      let start = end - 1;
+      while (
+        start > 0 &&
+        end - start < 4 &&
+        bytes[start] >= 0x80 &&
+        bytes[start] <= 0xbf
+      ) {
+        start--;
+      }
+      const width = sequenceWidth(bytes[start]);
+      if (width > end - start) {
+        end = start;
+      }
+    }
+    this.pending = Buffer.from(bytes.subarray(end));
+    return decodeLogBytes(bytes.subarray(0, end));
+  }
+}

package/src/names.js

@@ -0,0 +1,65 @@
+// A small offline gazetteer plus script-specific name grammar. This is a
+// transparent baseline, not a statistical NER model. Applications can union
+// their own model with it through the required detector interface.
+export const firstNames = {
+  en: 'John James Robert Michael William David Joseph Thomas Charles Mary Patricia Jennifer Linda Elizabeth Barbara Susan Jessica Sarah Alice Jane Peter Paul Mark Daniel George Emma Olivia Noah Liam Jack Henry Benjamin Anthony Andrew Steven Edward Richard Helen Margaret Nancy Carol Anna Martin Albert Bill Elon',
+  es: 'José Juan Carlos Luis Jorge Pedro Miguel Antonio Francisco Jesús Manuel María Carmen Ana Laura Isabel Elena Marta Lucía Sofía Javier Alejandro Diego Raúl Pablo Teresa Rosa',
+  fr: 'Jean Pierre Michel André Philippe Jacques François Louis Nicolas Antoine Julien Gabriel Étienne Marie Jeanne Françoise Catherine Isabelle Monique Sylvie Nathalie Sophie Camille Émilie Chloé Zoë',
+  de: 'Hans Klaus Wolfgang Jürgen Günter Dieter Karl Heinrich Wilhelm Friedrich Johann Alexander Lukas Leon Julia Hannah Emilia Lena Mia Sabine Ursula Ingrid Petra Monika Helga',
+  pt: 'João José Antônio Francisco Paulo Pedro Lucas Gabriel Rafael Bruno Carlos Maria Ana Beatriz Camila Larissa Mariana Fernanda Juliana Aline Bruna',
+  ruLatn:
+    'Ivan Petr Pyotr Sergey Sergei Aleksandr Alexander Alexey Aleksey Dmitry Dmitri Mikhail Nikolai Andrey Vladimir Pavel Viktor Oleg Boris Evgeny Vasily Yuri Marina Anna Mariya Elena Olga Natalya Irina Svetlana Tatyana Ekaterina Anastasia Yulia',
+  jaLatn: 'Yamada Sato Suzuki Takahashi Tanaka Ito Watanabe Nakamura Kobayashi',
+  zhLatn:
+    'Zhang Wang Li Zhao Liu Chen Yang Huang Zhou Wu Xu Sun Hu Zhu Gao Lin',
+  ru: 'Иван Пётр Петр Сергей Александр Алексей Дмитрий Михаил Николай Андрей Владимир Павел Виктор Олег Борис Евгений Василий Юрий Марина Анна Мария Елена Ольга Наталья Ирина Светлана Татьяна Екатерина Анастасия Юлия Ивану Ивана Иваном Сергея Сергею Александра Александру Анне Анну Елены',
+  ja: '山田 佐藤 鈴木 高橋 田中 伊藤 渡辺 渡邊 中村 小林 加藤 吉田 松本 山本 井上 木村 林 清水 斎藤 斉藤',
+  zh: '张 王 李 赵 刘 陈 杨 黄 周 吴 徐 孙 胡 朱 高 林 何 郭 马 罗 張 趙 劉 陳 楊 黃 吳 孫 馬 羅',
+  ko: '김 이 박 최 정 강 조 윤 장 임 한 오 서 신 권 황 안 송 전 홍 유 고 문 양 손 배 백',
+  ar: 'محمد أحمد احمد علي حسن حسين عمر خالد يوسف إبراهيم ابراهيم عبدالله عبد الرحمن محمود مصطفى سامي سعيد فاطمة مريم عائشة زينب نور ليلى سارة',
+  he: 'דוד משה יוסף אברהם יצחק יעקב דניאל חיים שלמה אהרון ישראל שרה רחל לאה מרים חנה רבקה נועה תמר יעל',
+  hi: 'आरव विवान आदित्य अर्जुन कृष्ण राहुल रोहित अमित विजय सुरेश रमेश राजेश संजय अनिल सुनील प्रिया पूजा नेहा अंजली आशा मीरा आरती लक्ष्मी सीता',
+  tr: 'Mehmet Mustafa Ahmet Ali Hüseyin Hasan İbrahim Murat Yusuf Emre Ömer Ayşe Fatma Emine Hatice Zeynep Elif Merve Esra Selin',
+  id: 'Budi Ahmad Muhammad Agus Bambang Dedi Eko Hadi Joko Rudi Siti Sri Dewi Putri Ayu Wati Lestari Ratna Indah',
+  vi: 'Nguyễn Trần Lê Phạm Hoàng Huỳnh Phan Vũ Võ Đặng Bùi Đỗ Hồ Ngô Dương Lý',
+  th: 'สมชาย สมศักดิ์ สุชาติ วิชัย ประเสริฐ สมพร สุรชัย อนุชา กิตติ วัฒนา สมหญิง สุภา จันทร์เพ็ญ มาลี สุดา อรทัย',
+};
+
+export const nameExamples = [
+  ['en', 'John Smith'],
+  ['es', 'Juan García'],
+  ['fr', 'Jean Dupont'],
+  ['de', 'Hans Müller'],
+  ['pt', 'João Silva'],
+  ['ru', 'Иван Петров'],
+  ['ja', '山田太郎'],
+  ['zh', '张三'],
+  ['zh-Hant', '張三'],
+  ['ko', '김민수'],
+  ['ar', 'محمد علي'],
+  ['he', 'דוד כהן'],
+  ['hi', 'आरव शर्मा'],
+  ['tr', 'Mehmet Yılmaz'],
+  ['id', 'Budi Santoso'],
+  ['vi', 'Nguyễn Văn An'],
+  ['th', 'สมชาย ใจดี'],
+];
+
+const givenNames = new Set(
+  Object.values(firstNames).flatMap((list) => list.toLowerCase().split(' '))
+);
+// A familiar label is evidence only when its value also has a name shape.
+// Single unfamiliar Latin words are commonly tool names or code identifiers.
+export function plausiblePerson(value) {
+  if (!/^[\p{L}\p{M}][\p{L}\p{M}’' -]{1,127}$/u.test(value)) {
+    return false;
+  }
+  const words = value.split(/[ \t]+/);
+  if (words.some((word) => givenNames.has(word.toLowerCase()))) {
+    return true;
+  }
+  if (words.length >= 2 && words.length <= 4) {
+    return words.every((word) => /^\p{Lu}[\p{L}\p{M}’'-]+$/u.test(word));
+  }
+  return /^[\p{L}\p{M}]{2,32}$/u.test(value) && !/[\p{Lu}\p{Ll}]/u.test(value);
+}

package/src/national-checks.js

@@ -0,0 +1,129 @@
+// Format arithmetic; these checks are confidence signals, not verification of
+// issuance. Short-number Luhn is separate from the public credit-card helper.
+export const digitsOf = (value) => value.replace(/\D/g, '');
+const sum = (digits, weights) =>
+  weights.reduce((total, weight, i) => total + Number(digits[i]) * weight, 0);
+export function shortLuhn(value) {
+  const digits = digitsOf(value);
+  let total = 0;
+  for (
+    let i = digits.length - 1, double = false;
+    i >= 0;
+    i--, double = !double
+  ) {
+    let digit = Number(digits[i]);
+    if (double) {
+      digit = digit * 2 > 9 ? digit * 2 - 9 : digit * 2;
+    }
+    total += digit;
+  }
+  return digits.length > 0 && !/^(\d)\1+$/.test(digits) && total % 10 === 0;
+}
+export function datePart(digits, monthOffset = 0, dayOffset = 0) {
+  const month = Number(digits.slice(2, 4)) - monthOffset;
+  const day = Number(digits.slice(4, 6)) - dayOffset;
+  const year = 2000 + Number(digits.slice(0, 2));
+  const date = new Date(Date.UTC(year, month - 1, day));
+  return (
+    date.getUTCFullYear() === year &&
+    date.getUTCMonth() === month - 1 &&
+    date.getUTCDate() === day
+  );
+}
+export function nhs(value) {
+  const d = digitsOf(value),
+    check = (11 - (sum(d, [10, 9, 8, 7, 6, 5, 4, 3, 2]) % 11)) % 11;
+  return (
+    d.length === 10 &&
+    !/^(\d)\1+$/.test(d) &&
+    check < 10 &&
+    check === Number(d[9])
+  );
+}
+export function snils(value) {
+  const d = digitsOf(value),
+    check = sum(d, [9, 8, 7, 6, 5, 4, 3, 2, 1]) % 101;
+  return (
+    d.length === 11 &&
+    !/^(\d)\1+$/.test(d) &&
+    (check === 100 ? 0 : check) === Number(d.slice(9))
+  );
+}
+export function inn(value) {
+  const d = digitsOf(value),
+    check = (weights) => (sum(d, weights) % 11) % 10;
+  if (/^(\d)\1+$/.test(d)) {
+    return false;
+  }
+  return d.length === 10
+    ? check([2, 4, 10, 3, 5, 9, 4, 6, 8]) === Number(d[9])
+    : d.length === 12 &&
+        check([7, 2, 4, 10, 3, 5, 9, 4, 6, 8]) === Number(d[10]) &&
+        check([3, 7, 2, 4, 10, 3, 5, 9, 4, 6, 8]) === Number(d[11]);
+}
+export function pesel(value) {
+  const d = digitsOf(value),
+    month = Number(d.slice(2, 4));
+  return (
+    d.length === 11 &&
+    datePart(d, Math.floor(month / 20) * 20) &&
+    (10 - (sum(d, [1, 3, 7, 9, 1, 3, 7, 9, 1, 3]) % 10)) % 10 === Number(d[10])
+  );
+}
+export function rrn(value) {
+  const d = digitsOf(value);
+  return (
+    d.length === 13 &&
+    datePart(d) &&
+    (11 - (sum(d, [2, 3, 4, 5, 6, 7, 8, 9, 2, 3, 4, 5]) % 11)) % 10 ===
+      Number(d[12])
+  );
+}
+export function tfn(value) {
+  const d = digitsOf(value);
+  return (
+    d.length === 9 &&
+    !/^(\d)\1+$/.test(d) &&
+    sum(d, [1, 4, 3, 7, 5, 8, 6, 9, 10]) % 11 === 0
+  );
+}
+export function nric(value) {
+  const v = value.toUpperCase(),
+    prefix = v[0];
+  const offset = { S: 0, T: 4, F: 0, G: 4, M: 3 }[prefix];
+  const alphabet = /[ST]/.test(prefix)
+    ? 'JZIHGFEDCBA'
+    : prefix === 'M'
+      ? 'KLJNPQRTUWX'
+      : 'XWUTRQPNMLK';
+  const index = (sum(v.slice(1, 8), [2, 7, 6, 5, 4, 3, 2]) + offset) % 11;
+  return (
+    /^[STFGM]\d{7}[A-Z]$/.test(v) &&
+    alphabet[prefix === 'M' ? 10 - index : index] === v[8]
+  );
+}
+export function fiscalCode(value) {
+  const odd = [
+    1, 0, 5, 7, 9, 13, 15, 17, 19, 21, 2, 4, 18, 20, 11, 3, 6, 8, 12, 14, 16,
+    10, 22, 25, 24, 23,
+  ];
+  let total = 0;
+  for (let i = 0; i < 15; i++) {
+    const char = value[i],
+      position = /\d/.test(char) ? Number(char) : char.charCodeAt(0) - 65;
+    total += i % 2 === 0 ? odd[position] : position;
+  }
+  return String.fromCharCode(65 + (total % 26)) === value[15];
+}
+export function personnummer(value) {
+  const d = digitsOf(value).slice(-10),
+    day = Number(d.slice(4, 6));
+  return d.length === 10 && datePart(d, 0, day > 60 ? 60 : 0) && shortLuhn(d);
+}
+export function southAfricanId(value) {
+  return (
+    datePart(value) &&
+    /^[0-2][89]$/.test(value.slice(10, 12)) &&
+    shortLuhn(value)
+  );
+}

package/src/national.js

@@ -0,0 +1,128 @@
+import { numericNoise, epochNumber } from './confidence.js';
+import {
+  digitsOf,
+  datePart,
+  nhs,
+  snils,
+  inn,
+  pesel,
+  rrn,
+  tfn,
+  nric,
+  fiscalCode,
+  personnummer,
+  southAfricanId,
+  shortLuhn,
+} from './national-checks.js';
+
+const rules = [
+  [
+    'UK_NHS',
+    /\d{3}[ -]?\d{3}[ -]?\d{4}/g,
+    /\b(?:NHS|patient|healthcare|provider)\b/i,
+    nhs,
+  ],
+  [
+    'KR_RRN',
+    /\d{6}[ -]?[1-8]\d{6}/g,
+    /\bRRN\b|주민등록/iu,
+    rrn,
+    (v) => datePart(digitsOf(v)),
+  ],
+  [
+    'IT_FISCAL_CODE',
+    /[A-Z]{6}[0-9LMNPQRSTUV]{2}[ABCDEHLMPRST][0-9LMNPQRSTUV]{2}[A-Z][0-9LMNPQRSTUV]{3}[A-Z]/g,
+    /\bCF\b|codice fiscale/iu,
+    fiscalCode,
+    () => true,
+  ],
+  ['SG_NRIC_FIN', /[STFGM]\d{7}[A-Z]/g, /\b(?:NRIC|FIN)\b/iu, nric, () => true],
+  ['PL_PESEL', /\d{11}/g, /\bPESEL\b/iu, pesel],
+  [
+    'SE_PERSONNUMMER',
+    /(?:\d{2})?\d{6}[-+]\d{4}/g,
+    /personnummer/iu,
+    personnummer,
+  ],
+  [
+    'ZA_ID_NUMBER',
+    /\d{10}[0-2][89]\d/g,
+    /\b(?:SA ID|RSA ID)\b|south african id/iu,
+    southAfricanId,
+  ],
+  ['AU_TFN', /\d{3}[ -]?\d{3}[ -]?\d{3}/g, /\bTFN\b|tax file number/iu, tfn],
+  [
+    'CA_SIN',
+    /\d{3}[ -]?\d{3}[ -]?\d{3}/g,
+    /\bSIN\b|social insurance/iu,
+    shortLuhn,
+  ],
+  [
+    'RU_SNILS',
+    /\d{3}[- ]\d{3}[- ]\d{3}[ -]\d{2}|\d{11}/g,
+    /\bSNILS\b|снилс|пенсионное|страховое свидетельство|страховой номер/iu,
+    snils,
+  ],
+  ['RU_INN', /\d{12}|\d{10}/g, /\bINN\b|инн/iu, inn],
+];
+function distinctiveShape(type, value) {
+  return (
+    (type === 'RU_SNILS' && /^\d{3}([- ])\d{3}\1\d{3} \d{2}$/.test(value)) ||
+    (type === 'RU_INN' && value.length === 12)
+  );
+}
+export function detectNational(text, emit) {
+  for (const [type, pattern, context, checksum, plausible] of rules) {
+    for (const match of text.matchAll(pattern)) {
+      const start = match.index,
+        end = start + match[0].length;
+      if (!nationalBoundary(text, start, end)) {
+        continue;
+      }
+      const hasContext = context.test(
+        text.slice(Math.max(0, start - 64), end + 32)
+      );
+      if (
+        !hasContext &&
+        (numericNoise(text, start, end) || epochNumber(match[0]))
+      ) {
+        continue;
+      }
+      const checked = checksum(match[0]);
+      if (checked || hasContext || plausible?.(match[0])) {
+        emit({
+          start,
+          end,
+          type,
+          category: 'personal',
+          rule: 'national-format',
+          confidence: nationalConfidence(
+            hasContext,
+            checked,
+            plausible?.(match[0]) ||
+              (checked && distinctiveShape(type, match[0]))
+          ),
+        });
+      }
+    }
+  }
+}
+function nationalBoundary(text, start, end) {
+  return !(
+    /[\p{L}\p{N}_.+-]/u.test(text[start - 1] ?? '') ||
+    /[\p{L}\p{N}_]/u.test(text[end] ?? '') ||
+    /\d[ ()-]{0,3}$/.test(text.slice(Math.max(0, start - 4), start)) ||
+    /^[ ()-]{0,3}\d/.test(text.slice(end, end + 4))
+  );
+}
+function nationalConfidence(context, checked, plausible) {
+  return context
+    ? checked
+      ? 0.99
+      : 0.9
+    : plausible
+      ? checked
+        ? 0.99
+        : 0.4
+      : 0.35;
+}

package/src/outbound.js

@@ -0,0 +1,28 @@
+import { createSanitizer } from './engines.js';
+import { failure } from './detection.js';
+/** Sanitizes one serialized payload so contextual keys survive to detection. */
+export async function sanitizePayload(value, options = {}) {
+  const engine = options.sanitizer ?? createSanitizer(options.sanitizerOptions);
+  try {
+    const serialized = JSON.stringify(value);
+    const result = await engine.sanitize(serialized);
+    return JSON.parse(result.text);
+  } catch {
+    throw failure('ERR_PUBLICATION');
+  }
+}
+export function createSentryBeforeSend(options = {}) {
+  return async (event) => {
+    try {
+      return await sanitizePayload(event, options);
+    } catch {
+      return null;
+    }
+  };
+}
+export function createOutboundSanitizer(publish, options = {}) {
+  if (typeof publish !== 'function') {
+    throw failure('ERR_CONFIG');
+  }
+  return async (payload) => publish(await sanitizePayload(payload, options));
+}

package/src/personal-context.js

@@ -0,0 +1,85 @@
+import { collectMatches } from './detection.js';
+
+const birth =
+  /(?<![\p{L}\p{N}_])(?:born|birth[ _-]?date|date[ _-]of[ _-]birth|DOB|дата рождения|д\.р\.|родился|родилась|Geburtsdatum|fecha de nacimiento|date de naissance)(?![\p{L}\p{N}_])/giu;
+const months =
+  'January February March April May June July August September October November December Jan Feb Mar Apr Jun Jul Aug Sep Sept Oct Nov Dec января февраля марта апреля мая июня июля августа сентября октября ноября декабря январь февраль март апрель май июнь июль август сентябрь октябрь ноябрь декабрь'
+    .split(' ')
+    .join('|');
+const dates = new RegExp(
+  `(?<![\\p{L}\\p{N}_])(?:\\d{4}-\\d{1,2}-\\d{1,2}|\\d{1,2}[/.]\\d{1,2}[/.]\\d{4}|\\d{1,2}[ \\t]+(?:${months})[ \\t]+\\d{4}|(?:${months})[ \\t]+\\d{1,2},?[ \\t]+\\d{4})(?![\\p{L}\\p{N}_])`,
+  'giu'
+);
+const phoneContext =
+  /(?<![\p{L}\p{N}_])(?:WhatsApp|Zalo|Viber|Telegram|Signal|WeChat|LINE|KakaoTalk|звоните|пишите|звонить|номер телефона|телефон|phone|fax|call|telp|telepon|HP|hubungi|โทร|เบอร์|gọi|SĐT|ĐT|điện thoại)(?![\p{L}\p{N}_])/iu;
+const nationalPhone =
+  /(?<![\p{L}\p{N}_+])(?:\(?08\d{2}[ )-]{0,3}\d{3,4}[ -]{0,2}\d{3,4}|8[ ()-]{0,3}9\d{2}[ )-]{0,3}\d{3}[ -]{0,2}\d{2}[ -]{0,2}\d{2}|\(?0[ ()-]{0,2}[35789]\d[ )-]{0,3}\d{3}[ -]{0,2}\d{4}|\(?0[ ()-]{0,2}\d[ )-]{0,3}\d{3}[ -]{0,2}\d{4}|\(?0[ ()-]{0,2}8\d{2}[ )-]{0,3}\d{4}[ -]{0,2}\d{4})(?![\p{L}\p{N}_])/gu;
+export function detectPersonalContext(text, emit) {
+  const phone = {
+    type: 'PHONE',
+    category: 'personal',
+    rule: 'phone-context',
+    confidence: 0.9,
+  };
+  collectMatches(
+    text,
+    /(?<![\p{L}\p{N}_])tel:[ \t]*(\+?\d[\d(). -]{0,38}\d|\d)(?![\p{L}\p{N}_])/giu,
+    emit,
+    { ...phone, rule: 'tel-uri' },
+    1
+  );
+  collectMatches(
+    text,
+    /(?<![\p{L}\p{N}_])(?:phone|fax|ĐT|điện thoại)(?:[ \t]*[:：][ \t]*|[ \t]+)(\+?\(?\d[\d() .-]{5,38}\d)(?![\p{L}\p{N}_])/giu,
+    emit,
+    phone,
+    1,
+    (value) => {
+      const digits = value.replace(/\D/g, '').length;
+      return digits >= 7 && digits <= 15;
+    }
+  );
+  for (const label of text.matchAll(birth)) {
+    const start = label.index + label[0].length;
+    const window = text.slice(start, start + 80).split(/[\r\n|;]/)[0];
+    const match = new RegExp(dates.source, dates.flags).exec(window);
+    if (match) {
+      emit({
+        start: start + match.index,
+        end: start + match.index + match[0].length,
+        type: 'DATE_OF_BIRTH',
+        category: 'personal',
+        rule: 'birth-context',
+        confidence: 0.9,
+      });
+    }
+  }
+  collectMatches(
+    text,
+    nationalPhone,
+    emit,
+    {
+      type: 'PHONE',
+      category: 'personal',
+      rule: 'phone-context',
+      confidence: 0.9,
+    },
+    0,
+    (value, match) => {
+      const prefix = text
+        .slice(Math.max(0, match.index - 80), match.index)
+        .split(/[\r\n|;]/)
+        .at(-1);
+      return (
+        (phoneContext.test(prefix) ||
+          /^\(?08/.test(value) ||
+          (/^0[689]/.test(value.replace(/\D/g, '')) &&
+            value.replace(/\D/g, '').length === 10)) &&
+        !/\d[ ()-]*$/.test(prefix) &&
+        !/^[ ()-]*\d/.test(
+          text.slice(match.index + value.length, match.index + value.length + 4)
+        )
+      );
+    }
+  );
+}