@link-foundation/sensitive-data-sanitizer 0.0.0-stage → 0.11.31

package/src/public.js

@@ -0,0 +1,153 @@
+import { isIP } from 'node:net';
+
+// Exact public knowledge and narrowly scoped role-mail heuristics. A role name
+// alone is never an exemption on an unknown/personal domain. Credentials and
+// caller-specified private values always outrank these automatic exceptions.
+export const publicKnowledge = {
+  PERSON: [
+    ['Albert Einstein', 'Q937'],
+    ['Marie Curie', 'Q7186'],
+    ['Isaac Newton', 'Q935'],
+    ['Charles Darwin', 'Q1035'],
+    ['Alan Turing', 'Q7251'],
+    ['Ada Lovelace', 'Q7259'],
+    ['Bill Gates', 'Q5284'],
+    ['Elon Musk', 'Q317521'],
+    ['Linus Torvalds', 'Q34253'],
+    ['Satya Nadella', 'Q7426870'],
+    ['Tim Cook', 'Q265852'],
+    ['Grace Hopper', 'Q11641'],
+    ['Sergey Brin', 'Q92764'],
+    ['Pavel Durov', 'Q149067'],
+    ['Vladimir Putin', 'Q7747'],
+    ['Barack Obama', 'Q76'],
+    ['Nelson Mandela', 'Q8023'],
+    ['Mahatma Gandhi', 'Q1001'],
+    ['Martin Luther King', 'Q8027'],
+    ['Альберт Эйнштейн', 'Q937'],
+    ['Мария Кюри', 'Q7186'],
+    ['Исаак Ньютон', 'Q935'],
+    ['李白', 'Q7071'],
+    ['夏目漱石', 'Q48443'],
+  ].map(([value, id]) => ({
+    value,
+    source: `https://www.wikidata.org/wiki/${id}`,
+  })),
+  ORGANIZATION: [
+    ['Microsoft', 'Q2283'],
+    ['Google', 'Q95'],
+    ['Apple', 'Q312'],
+    ['GitHub', 'Q364'],
+    ['Wikimedia Foundation', 'Q180'],
+    ['United Nations', 'Q1065'],
+    ['NASA', 'Q23548'],
+    ['UNESCO', 'Q7809'],
+  ].map(([value, id]) => ({
+    value,
+    source: `https://www.wikidata.org/wiki/${id}`,
+  })),
+  IP_ADDRESS: [
+    ...[
+      '8.8.8.8',
+      '8.8.4.4',
+      '2001:4860:4860::8888',
+      '2001:4860:4860::8844',
+    ].map((value) => ({
+      value,
+      source: 'https://developers.google.com/speed/public-dns/docs/using',
+    })),
+    ...[
+      '1.1.1.1',
+      '1.0.0.1',
+      '2606:4700:4700::1111',
+      '2606:4700:4700::1001',
+    ].map((value) => ({
+      value,
+      source: 'https://developers.cloudflare.com/1.1.1.1/ip-addresses/',
+    })),
+  ],
+};
+export const organizationDomains = new Set([
+  'microsoft.com',
+  'google.com',
+  'apple.com',
+  'github.com',
+  'wikimedia.org',
+  'wikipedia.org',
+  'mozilla.org',
+  'cloudflare.com',
+]);
+export const serviceAccounts = new Set([
+  'runner',
+  'ubuntu',
+  'vsts',
+  'circleci',
+  'node',
+  'root',
+]);
+export function servicePath(value) {
+  const account = /^\/home\/([^/]+)\//.exec(value)?.[1];
+  return (
+    serviceAccounts.has(account) ||
+    /^\/github\/(?:workspace|runner_temp)\//.test(value)
+  );
+}
+const roles =
+  /^(?:press|media|support|info|contact|security|abuse|privacy|sales|noreply|no-reply)$/i;
+
+export function isPublic(text, finding, options) {
+  if (
+    options.publicKnowledge === false ||
+    finding.category === 'credential' ||
+    ['known-personal', 'confirmed-personal'].includes(finding.rule)
+  ) {
+    return false;
+  }
+  const value = text.slice(finding.start, finding.end).normalize('NFC');
+  // Sensitive relationship context wins over a common public/private namesake.
+  if (
+    finding.type === 'PERSON' &&
+    /(?:patient|customer|employee|my name|пациент|клиент)\s*[:=]?\s*$/iu.test(
+      text.slice(Math.max(0, finding.start - 64), finding.start)
+    )
+  ) {
+    return false;
+  }
+  if (
+    (publicKnowledge[finding.type] ?? []).some((entry) => entry.value === value)
+  ) {
+    return true;
+  }
+  if (finding.type === 'EMAIL') {
+    const [local, domain] = value.toLowerCase().split('@');
+    return roles.test(local) && organizationDomains.has(domain);
+  }
+  if (finding.type === 'IP_ADDRESS') {
+    return (
+      localAddress(value) ||
+      /^(?:192\.0\.2\.|198\.51\.100\.|203\.0\.113\.)\d{1,3}$/.test(value) ||
+      /^2001:db8:/i.test(value)
+    );
+  }
+  return false;
+}
+
+export function localAddress(value) {
+  if (isIP(value) === 6) {
+    return (
+      /^(?:::1|0:0:0:0:0:0:0:1)$/i.test(value) ||
+      /^(?:f[cd][\da-f]{2}:|fe[89ab][\da-f]:)/i.test(value)
+    );
+  }
+  if (isIP(value) !== 4) {
+    return false;
+  }
+  const [a, b] = value.split('.').map(Number);
+  return (
+    a === 127 ||
+    a === 10 ||
+    (a === 172 && b >= 16 && b <= 31) ||
+    (a === 192 && b === 168) ||
+    (a === 169 && b === 254)
+  );
+}

package/src/record-worker.js

@@ -0,0 +1,20 @@
+import { parentPort, workerData } from 'node:worker_threads';
+import { createSanitizer, prepareStreamEngine } from './engines.js';
+
+parentPort.on(
+  'message',
+  async ({ text, confirmedPersonal, nativeFindings }) => {
+    try {
+      const sanitizer = createSanitizer({ ...workerData, confirmedPersonal });
+      await prepareStreamEngine(sanitizer, text, nativeFindings);
+      parentPort.postMessage({ value: (await sanitizer.sanitize(text)).text });
+    } catch (error) {
+      parentPort.postMessage({
+        error: /^ERR_[A-Z_]+$/.test(error.code) ? error.code : 'ERR_WORKER',
+        record: error.record,
+        offset: error.offset,
+        recordOffset: error.recordOffset,
+      });
+    }
+  }
+);

package/src/rules.js

@@ -0,0 +1,210 @@
+import { localeVocabulary, personalFieldTypes } from './locales.js';
+
+// Supplemental formats, independently maintained from upstream Secretlint.
+// Lengths are intentionally conservative: suspicious prefixes are redacted
+// even when a pasted credential is truncated. No online verification occurs.
+export const serviceRules = [
+  ['gitlab-runner', /GR1348941[A-Za-z0-9_-]{16}[A-Za-z0-9_-]*/g],
+  [
+    'firebase',
+    /(?<![A-Za-z0-9_-])AAAA[A-Za-z0-9_-]{6}[A-Za-z0-9_-]*:APA91b[A-Za-z0-9_-]{32}[A-Za-z0-9_-]*/g,
+  ],
+  ['github', /(?:github_pat_|gh[pousr]_)[A-Za-z0-9_]{16}[A-Za-z0-9_]*/g],
+  [
+    'gitlab',
+    /(?:glpat|gloas|gldt|glrt|glrtr|glcbt|glptt|glft|glimt|glagent|glwt|glsoat|glffct)-[A-Za-z0-9_-]{16}[A-Za-z0-9_-]*/g,
+  ],
+  [
+    'openai',
+    /(?<![A-Za-z0-9_])sk-(?:proj-|svcacct-|admin-)?[A-Za-z0-9_-]{16}[A-Za-z0-9_-]*/g,
+  ],
+  [
+    'anthropic',
+    /(?<![A-Za-z0-9_])sk-ant-(?:api\d{2}-)?[A-Za-z0-9_-]{16}[A-Za-z0-9_-]*/g,
+  ],
+  ['aws', /(?:AKIA|ASIA|A3T[A-Z0-9])[A-Z0-9]{16}/g],
+  [
+    'google',
+    /(?:AIza[A-Za-z0-9_-]{35}|GOCSPX-[A-Za-z0-9_-]{16}[A-Za-z0-9_-]*|ya29\.[A-Za-z0-9_-]{20}[A-Za-z0-9_-]*)/g,
+  ],
+  ['slack', /(?:xox[a-z]|xapp|xwfp)-[A-Za-z0-9-]{16}[A-Za-z0-9-]*/g],
+  [
+    'stripe',
+    /(?:[sr]k_(?:live|test)_[A-Za-z0-9]{16}[A-Za-z0-9]*|whsec_[A-Za-z0-9]{16}[A-Za-z0-9]*)/g,
+  ],
+  [
+    'sendgrid',
+    /SG\.[A-Za-z0-9_-]{10}[A-Za-z0-9_-]*\.[A-Za-z0-9_-]{20}[A-Za-z0-9_-]*/g,
+  ],
+  ['twilio', /SK[a-fA-F0-9]{32}/g],
+  ['npm', /npm_[A-Za-z0-9]{20}[A-Za-z0-9]*/g],
+  ['pypi', /pypi-[A-Za-z0-9_-]{30}[A-Za-z0-9_-]*/g],
+  ['telegram', /\d{8,12}:[A-Za-z0-9_-]{30}[A-Za-z0-9_-]*/g],
+  [
+    'discord',
+    /[MN][A-Za-z0-9_-]{23}[A-Za-z0-9_-]*\.[A-Za-z0-9_-]{6}\.[A-Za-z0-9_-]{20}[A-Za-z0-9_-]*/g,
+  ],
+  ['huggingface', /hf_[A-Za-z0-9]{20}[A-Za-z0-9]*/g],
+  ['shopify', /shp(?:at|ss|ca|pa)_[a-fA-F0-9]{32}/g],
+  ['databricks', /dapi[a-fA-F0-9]{32}/g],
+  ['square', /sq0(?:atp|csp)-[A-Za-z0-9_-]{20}[A-Za-z0-9_-]*/g],
+  ['mailchimp', /(?<![a-fA-F0-9])[a-fA-F0-9]{32}-us\d{1,2}/g],
+  ['digitalocean', /do(?:pat|o|r)_v1_[a-fA-F0-9]{64}/g],
+  ['vault', /(?:hvs|hvb|hvr)\.[A-Za-z0-9_-]{16}[A-Za-z0-9_-]*/g],
+  ['grafana', /(?:glc_|glsa_)[A-Za-z0-9_-]{20}[A-Za-z0-9_-]*/g],
+  ['linear', /lin_api_[A-Za-z0-9]{20}[A-Za-z0-9]*/g],
+  ['notion', /(?:secret_|ntn_)[A-Za-z0-9]{32}[A-Za-z0-9]*/g],
+  ['vercel', /vercel_[A-Za-z0-9_-]{20}[A-Za-z0-9_-]*/g],
+  [
+    'jwt',
+    /eyJ[A-Za-z0-9_-]{8}[A-Za-z0-9_-]*\.eyJ[A-Za-z0-9_-]{8}[A-Za-z0-9_-]*\.[A-Za-z0-9_-]{8}[A-Za-z0-9_-]*/g,
+  ],
+];
+
+// Label recognition is deliberately separate from general free-text NER.
+export const labels = {
+  credential: [
+    'password',
+    'passwd',
+    'pwd',
+    'passphrase',
+    'credentials',
+    'api key',
+    'mot de passe',
+    'kata sandi',
+    'mật khẩu',
+    'пароля',
+    'паролем',
+    'паролю',
+    'contraseñas',
+    'senhas',
+    'şifresi',
+    'पासवर्ड है',
+    'secret',
+    'token',
+    'api_key',
+    'apikey',
+    'api-key',
+    'authorization',
+    'auth',
+    'cookie',
+    'accountkey',
+    'client_secret',
+    'private_key',
+    'access_key',
+    'access_token',
+    'refresh_token',
+    'session_key',
+    'connectionstring',
+    'sas',
+    'sig',
+    'senha',
+    'contraseña',
+    'clave',
+    'mot_de_passe',
+    'passwort',
+    'пароль',
+    'секрет',
+    'токен',
+    '密码',
+    '密碼',
+    '口令',
+    'パスワード',
+    '비밀번호',
+    'كلمة المرور',
+    'رمز الدخول',
+    'סיסמה',
+    'पासवर्ड',
+    'şifre',
+    'kata_sandi',
+    'mật_khẩu',
+    'รหัสผ่าน',
+  ],
+  PERSON: [
+    'Applicant',
+    'Passenger',
+    'Traveller',
+    'Заявитель',
+    'name',
+    'full_name',
+    'fullname',
+    'first_name',
+    'last_name',
+    'givenname',
+    'surname',
+    'nom',
+    'nombre',
+    'nome',
+    'имя',
+    'фио',
+    'фамилия',
+    '姓名',
+    '氏名',
+    '이름',
+    'الاسم',
+    'שם',
+    'नाम',
+    'ad_soyad',
+    'nama',
+    'họ_tên',
+    'ชื่อ',
+  ],
+  ADDRESS: [
+    'address',
+    'street_address',
+    'adresse',
+    'dirección',
+    'endereço',
+    'адрес',
+    '地址',
+    '住所',
+    '주소',
+    'العنوان',
+    'כתובת',
+    'पता',
+    'adres',
+    'alamat',
+    'địa_chỉ',
+  ],
+  DATE_OF_BIRTH: [
+    'date_of_birth',
+    'dob',
+    'birthdate',
+    'fecha_de_nacimiento',
+    'дата рождения',
+    '出生日期',
+    '生年月日',
+    'تاريخ الميلاد',
+  ],
+  ID: [
+    'passport',
+    'passport_number',
+    'ssn',
+    'national_id',
+    'tax_id',
+    'inn',
+    'инн',
+    'social_security',
+    'cpf',
+    'aadhaar',
+    'snils',
+    'паспорт',
+    'снилс',
+    'инн',
+    '身份证',
+    '护照',
+    '身分証',
+    'جواز السفر',
+  ],
+};
+
+for (const [, credential, ...fields] of localeVocabulary) {
+  labels.credential.push(credential);
+  fields.forEach((word, i) => {
+    labels[personalFieldTypes[i]] ??= [];
+    labels[personalFieldTypes[i]].push(word);
+  });
+}
+for (const [type, words] of Object.entries(labels)) {
+  labels[type] = [...new Set(words)].sort((a, b) => b.length - a.length);
+}