@link-foundation/sensitive-data-sanitizer 0.19.3 → 0.20.0

package/src/crypto-addresses.js

@@ -0,0 +1,74 @@
+import { createHash } from 'node:crypto';
+
+// Offline checksum validation for Bitcoin addresses (Presidio parity): legacy
+// Base58Check (P2PKH/P2SH) and SegWit Bech32/Bech32m. Ethereum addresses carry
+// no case-insensitive checksum and stay format-only.
+const base58 = '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz';
+const sha256 = (bytes) => createHash('sha256').update(bytes).digest();
+
+export function base58Check(value) {
+  let number = 0n;
+  for (const char of value) {
+    const digit = base58.indexOf(char);
+    if (digit < 0) {
+      return false;
+    }
+    number = number * 58n + BigInt(digit);
+  }
+  const hex = number.toString(16);
+  const body = Buffer.from(hex.length % 2 ? `0${hex}` : hex, 'hex');
+  const zeros = /^1*/.exec(value)[0].length;
+  const bytes = Buffer.concat([Buffer.alloc(zeros), body]);
+  if (bytes.length !== 25) {
+    return false;
+  }
+  const check = sha256(sha256(bytes.subarray(0, 21))).subarray(0, 4);
+  return check.equals(bytes.subarray(21));
+}
+
+const bech32Charset = 'qpzry9x8gf2tvdw0s3jn54khce6mua7l';
+function polymod(values) {
+  const generator = [
+    0x3b6a57b2, 0x26508e6d, 0x1ea119fa, 0x3d4233dd, 0x2a1462b3,
+  ];
+  let checksum = 1;
+  for (const value of values) {
+    const top = checksum >>> 25;
+    checksum = ((checksum & 0x1ffffff) << 5) ^ value;
+    for (let i = 0; i < 5; i++) {
+      if ((top >>> i) & 1) {
+        checksum ^= generator[i];
+      }
+    }
+  }
+  return checksum >>> 0;
+}
+export function bech32Address(value) {
+  if (value !== value.toLowerCase() && value !== value.toUpperCase()) {
+    return false;
+  }
+  const lower = value.toLowerCase();
+  const separator = lower.lastIndexOf('1');
+  const prefix = lower.slice(0, separator);
+  const data = [...lower.slice(separator + 1)].map((char) =>
+    bech32Charset.indexOf(char)
+  );
+  if (!['bc', 'tb'].includes(prefix) || data.length < 7 || data.includes(-1)) {
+    return false;
+  }
+  const expanded = [
+    ...[...prefix].map((char) => char.charCodeAt(0) >> 5),
+    0,
+    ...[...prefix].map((char) => char.charCodeAt(0) & 31),
+    ...data,
+  ];
+  const constant = polymod(expanded);
+  return constant === 1 || constant === 0x2bc830a3;
+}
+
+export function cryptoAddress(value) {
+  if (/^0x[a-fA-F0-9]{40}$/.test(value)) {
+    return true;
+  }
+  return /^(?:bc|tb)1/i.test(value) ? bech32Address(value) : base58Check(value);
+}

package/src/detection.js

@@ -1,4 +1,9 @@
-import { cardNetwork, cardEvidence } from './confidence.js';
+import {
+  cardNetwork,
+  cardEvidence,
+  cardContext,
+  uatpCard,
+} from './confidence.js';
 import { isIP } from 'node:net';
 import { serviceRules } from './rules.js';
 import { detectContext, valueRange, authorizationRange } from './context.js';
@@ -8,6 +13,7 @@ import { detectCatalog } from './entities.js';
 import { detectIdentity } from './identity.js';
 import { detectNational } from './national.js';
 import { detectPersonalContext } from './personal-context.js';
+import { detectPersonalEvidence } from './personal-evidence.js';
 import {
   detectCredentialFormats,
   credentialPlaceholder,
@@ -18,6 +24,7 @@ import { detectDocumentEntities } from './document-entities.js';
 import { detectProseAndDocuments } from './proper-names.js';
 import { builtInEnabled } from './policy.js';
 import { detectStrict } from './strict.js';
+import { tokenChecksum } from './token-checksum.js';
 
 export const REDACTED = '[REDACTED]';
 export const failure = (code = 'ERR_SANITIZATION', location = {}) =>
@@ -176,8 +183,14 @@ export function nativeDetect(text, options, emit) {
       ? (details) => options.debug({ event: 'gitleaks-rule', ...details })
       : undefined
   );
+  const checked = (finding) =>
+    emit(
+      tokenChecksum(text.slice(finding.start, finding.end))
+        ? { ...finding, confidence: 0.99 }
+        : finding
+    );
   for (const [rule, pattern] of serviceRules) {
-    collectMatches(text, pattern, emit, { ...credential, rule });
+    collectMatches(text, pattern, checked, { ...credential, rule });
   }
   detectContext(text, emit, options.credentialsOnly);
   if (!options.credentialsOnly) {
@@ -187,6 +200,7 @@ export function nativeDetect(text, options, emit) {
     detectIdentity(text, emit);
     detectNational(text, emit);
     detectPersonalContext(text, emit);
+    detectPersonalEvidence(text, emit);
     detectStrict(text, emit, options);
   }
   detectCredentialFormats(text, emit);
@@ -311,20 +325,21 @@ function detectPersonalFormats(text, options, emit) {
     0,
     (value, match) =>
       luhn(value) &&
-      cardNetwork(value) &&
+      (cardNetwork(value) ||
+        (uatpCard(value) && cardContext(text, match.index, value))) &&
       cardEvidence(text, match.index, value)
   );
   collectMatches(
     text,
-    /\b\d{3}-\d{2}-\d{4}\b/g,
+    /(?<![\w.-])\d{3}([-.])\d{2}\1\d{4}(?![\w-]|\.\d)/g,
     emit,
     { type: 'ID', category: 'personal', rule: 'us-ssn' },
     0,
-    (v) => !/^(?:000|666|9\d\d)-|^\d{3}-00-|0000$/.test(v)
+    (v) => !/^(?:000|666|9\d\d)\D|^\d{3}\D00\D|0000$/.test(v)
   );
   collectMatches(
     text,
-    /(?<![\w.])(?:\d{1,3}\.){3}\d{1,3}(?![\w.])/g,
+    /(?<![\w.])(?:\d{1,3}\.){3}\d{1,3}(?!\w|\.\d)/g,
     emit,
     { type: 'IP_ADDRESS', category: 'personal', rule: 'ipv4' },
     0,

package/src/document-person-cues.js

@@ -8,7 +8,7 @@ const name = "[A-Z][a-z]{1,47}(?:['’-][A-Z]?[a-z]{1,47})?";
 // unit in logs and lowercase `dr` is usually a word or path fragment. A
 // capitalized `Ms` right after a number is still a unit (`took 30 Ms Retry`).
 const cue =
-  '(?:[Cc]andidate|[Rr]ecruiter|[Aa]pplicant|[Dd]ear|[Hh]i|Mr\\.?|(?<!\\d[ \\t]*)Ms\\.?|Dr\\.?|[Mm]essage[ \\t]+from)';
+  '(?:[Cc]andidate|[Rr]ecruiter|[Aa]pplicant|[Dd]ear|[Hh]i|Mr\\.?|Ms(?<!\\d[ \\t]*Ms)\\.?|Dr\\.?|[Mm]essage[ \\t]+from)';
 const pattern = new RegExp(
   `(?<!${boundary})${cue}[ \\t]+(${name})(?!${boundary})`,
   'gu'

package/src/encrypt.js

@@ -0,0 +1,95 @@
+import {
+  createCipheriv,
+  createDecipheriv,
+  createHmac,
+  hkdfSync,
+} from 'node:crypto';
+import { failure } from './detection.js';
+
+// Deterministic authenticated encryption (synthetic IV), comparable to
+// Google's CryptoDeterministicConfig (AES-SIV) and Presidio's encrypt
+// operator. Equal values of one type produce equal tokens, so the output
+// still supports joins and counts, and decrypt() recovers the value. The
+// derived IV is an HMAC of the type and value, so it never repeats for
+// different plaintexts under one key.
+const ivBytes = 12,
+  tagBytes = 16;
+const token = /\[ENCRYPTED:([A-Z][A-Z0-9_]{0,63}):([A-Za-z0-9_-]{38,10000})\]/g;
+const keys = new Map();
+
+function derived(key) {
+  if (typeof key !== 'string' || Buffer.byteLength(key) < 16) {
+    throw failure('ERR_CONFIG');
+  }
+  let entry = keys.get(key);
+  if (!entry) {
+    const material = Buffer.from(
+      hkdfSync(
+        'sha256',
+        key,
+        'sensitive-data-sanitizer',
+        'deterministic-encryption-v1',
+        64
+      )
+    );
+    entry = { cipher: material.subarray(0, 32), mac: material.subarray(32) };
+    if (keys.size >= 16) {
+      keys.clear();
+    }
+    keys.set(key, entry);
+  }
+  return entry;
+}
+
+export function encryptValue(value, type, key) {
+  const { cipher, mac } = derived(key);
+  const iv = createHmac('sha256', mac)
+    .update(`${type}\0${value}`)
+    .digest()
+    .subarray(0, ivBytes);
+  const engine = createCipheriv('aes-256-gcm', cipher, iv);
+  engine.setAAD(Buffer.from(type));
+  const body = Buffer.concat([engine.update(value, 'utf8'), engine.final()]);
+  const payload = Buffer.concat([iv, body, engine.getAuthTag()]);
+  return `[ENCRYPTED:${type}:${payload.toString('base64url')}]`;
+}
+
+function decryptToken(type, encoded, key) {
+  const { cipher } = derived(key);
+  const payload = Buffer.from(encoded, 'base64url');
+  if (payload.length < ivBytes + tagBytes) {
+    throw failure('ERR_MAP');
+  }
+  try {
+    const engine = createDecipheriv(
+      'aes-256-gcm',
+      cipher,
+      payload.subarray(0, ivBytes)
+    );
+    engine.setAAD(Buffer.from(type));
+    engine.setAuthTag(payload.subarray(-tagBytes));
+    return Buffer.concat([
+      engine.update(payload.subarray(ivBytes, -tagBytes)),
+      engine.final(),
+    ]).toString('utf8');
+  } catch {
+    throw failure('ERR_MAP');
+  }
+}
+
+/**
+ * Restores `[ENCRYPTED:TYPE:…]` tokens produced by `{mode: 'encrypt', key}`.
+ * A token that fails authentication under `key` throws ERR_MAP; nothing is
+ * partially restored. With `json: true` restored values are JSON-escaped for
+ * tokens inside JSON string literals.
+ */
+export function decrypt(text, key, { json = false } = {}) {
+  if (typeof text !== 'string') {
+    throw failure('ERR_INPUT');
+  }
+  derived(key);
+  return text.replace(token, (_, type, encoded) => {
+    const value = decryptToken(type, encoded, key);
+    return json ? JSON.stringify(value).slice(1, -1) : value;
+  });
+}

package/src/entities.js

@@ -3,6 +3,7 @@ import azure from './vendor/entities/azure.json' with { type: 'json' };
 import aws from './vendor/entities/aws.json' with { type: 'json' };
 import { escapePattern, collectMatches } from './detection.js';
 import { numericNoise, epochNumber } from './confidence.js';
+import { cryptoAddress } from './crypto-addresses.js';
 import { valueRange, personalValueAllowed } from './context.js';
 export const entityCatalogs = { google, azure, aws };
 const canonical = (name) =>
@@ -161,13 +162,13 @@ const formatRules = [
   ],
   [
     'SPAIN_DNI_NUMBER',
-    /\b\d{8}[A-Z]\b/g,
+    /\b\d{8}-?[A-Z]\b/g,
     (v) => 'TRWAGMYFPDXBNJZSQVHLCKE'[Number(v.slice(0, 8)) % 23] === v.at(-1),
   ],
   [
     'CRYPTO_WALLET',
-    /\b(?:0x[a-fA-F0-9]{40}|[13][a-km-zA-HJ-NP-Z1-9]{25,34}|bc1[ac-hj-np-z02-9]{25,62})\b/g,
-    () => true,
+    /\b(?:0x[a-fA-F0-9]{40}|[13][a-km-zA-HJ-NP-Z1-9]{25,34}|bc1[ac-hj-np-z02-9]{25,62}|BC1[AC-HJ-NP-Z02-9]{25,62})\b/g,
+    cryptoAddress,
   ],
   ['US_HEALTHCARE_NPI', /\b[12]\d{9}\b/g, npi],
 ];

package/src/fake.js

@@ -116,6 +116,18 @@ const validators = {
   SPAIN_DNI_NUMBER: (v) =>
     'TRWAGMYFPDXBNJZSQVHLCKE'[Number(v.slice(0, 8)) % 23] === v.at(-1),
   BRAZIL_CPF_NUMBER: cpf,
+  ABA_ROUTING_NUMBER: checks.abaRouting,
+  ES_NIE: checks.spanishNie,
+  IT_VAT_CODE: checks.shortLuhn,
+  AU_ACN: checks.acn,
+  IN_AADHAAR: checks.aadhaar,
+  TH_TNIN: checks.thaiId,
+  MEDICAL_LICENSE: checks.dea,
+  IMEI: checks.shortLuhn,
+  DE_TAX_ID: checks.germanTaxId,
+  AE_EMIRATES_ID: checks.emiratesId,
+  SA_NATIONAL_ID: checks.saudiId,
+  IL_ID_NUMBER: checks.israeliId,
 };
 function cpf(d) {
   for (const n of [9, 10]) {
@@ -226,9 +238,24 @@ function idShape(compact, candidate, type) {
     candidate =
       candidate.slice(0, 10) + compact.slice(10, 12) + candidate.slice(12);
   }
-  if (type === 'SG_NRIC_FIN') {
+  if (
+    [
+      'SG_NRIC_FIN',
+      'ES_NIE',
+      'IN_AADHAAR',
+      'TH_TNIN',
+      'DE_TAX_ID',
+      'SA_NATIONAL_ID',
+    ].includes(type)
+  ) {
     candidate = compact[0] + candidate.slice(1);
   }
+  if (['ABA_ROUTING_NUMBER', 'MEDICAL_LICENSE'].includes(type)) {
+    candidate = compact.slice(0, 2) + candidate.slice(2);
+  }
+  if (type === 'AE_EMIRATES_ID') {
+    candidate = compact.slice(0, 7) + candidate.slice(7);
+  }
   if (type === 'CREDIT_CARD') {
     candidate = compact.slice(0, 6) + candidate.slice(6);
   }
@@ -255,6 +282,7 @@ function idChecks(compact, candidate, type) {
       'SG_NRIC_FIN',
       'IT_FISCAL_CODE',
       'SPAIN_DNI_NUMBER',
+      'ES_NIE',
     ].includes(type);
     const count =
       (type === 'RU_INN' && compact.length === 12) ||

package/src/files.js

@@ -4,31 +4,76 @@ import { dirname, join, resolve } from 'node:path';
 import { randomUUID } from 'node:crypto';
 import { TextDecoder } from 'node:util';
 import { failure } from './detection.js';
-import { decodeLogBytes } from './log-bytes.js';
+import { decodeLogBytes, encodeLogText } from './log-bytes.js';
 
-export function decodeText(
+// UTF-16 text (Windows logs, PowerShell transcripts) is recognised only by its
+// byte order mark; BOM-less UTF-16 remains binary input.
+export function textEncoding(bytes) {
+  if (bytes.length >= 2 && bytes.length % 2 === 0) {
+    if (bytes[0] === 0xff && bytes[1] === 0xfe) {
+      return 'utf-16le';
+    }
+    if (bytes[0] === 0xfe && bytes[1] === 0xff) {
+      return 'utf-16be';
+    }
+  }
+  return 'utf-8';
+}
+
+function decodeWith(encoding, bytes) {
+  try {
+    return new TextDecoder(encoding, { fatal: true, ignoreBOM: true }).decode(
+      bytes
+    );
+  } catch {
+    throw failure('ERR_ENCODING');
+  }
+}
+
+// Returns the decoded text and the encoding needed to write it back.
+export function decodeTextInput(
   bytes,
   { lossless = false, allowBinary = false } = {}
 ) {
   if (typeof allowBinary !== 'boolean') {
     throw failure('ERR_CONFIG');
   }
+  const encoding = textEncoding(bytes);
+  if (encoding !== 'utf-8') {
+    return { text: decodeWith(encoding, bytes), encoding };
+  }
   if (!allowBinary && bytes.includes(0)) {
     throw failure('ERR_BINARY');
   }
-  if (lossless) {
-    return decodeLogBytes(bytes);
-  }
-  try {
-    return new TextDecoder('utf-8', { fatal: true, ignoreBOM: true }).decode(
-      bytes
-    );
-  } catch {
-    throw failure('ERR_ENCODING');
+  return {
+    text: lossless ? decodeLogBytes(bytes) : decodeWith('utf-8', bytes),
+    encoding,
+  };
+}
+
+export function decodeText(bytes, options) {
+  return decodeTextInput(bytes, options).text;
+}
+
+// Encode output in the input encoding; UTF-16 keeps its byte order mark
+// because the decoded text retains it.
+export function encodeTextOutput(text, encoding = 'utf-8') {
+  if (encoding === 'utf-8') {
+    return encodeLogText(text);
   }
+  const bytes = Buffer.from(text, 'utf16le');
+  return encoding === 'utf-16be' ? bytes.swap16() : bytes;
 }
 
 export async function readTextFile(path, maxBytes = 10 * 1024 * 1024, options) {
+  return (await readTextInput(path, maxBytes, options)).text;
+}
+
+export async function readTextInput(
+  path,
+  maxBytes = 10 * 1024 * 1024,
+  options
+) {
   let handle;
   try {
     const info = await lstat(path);
@@ -52,7 +97,7 @@ export async function readTextFile(path, maxBytes = 10 * 1024 * 1024, options) {
     if (read > maxBytes || read > opened.size) {
       throw failure('ERR_LIMIT');
     }
-    return decodeText(bytes.subarray(0, read), options);
+    return decodeTextInput(bytes.subarray(0, read), options);
   } catch (error) {
     if (error.code?.startsWith('ERR_')) {
       throw error;

package/src/gitleaks.js

@@ -7,22 +7,46 @@ import r2 from './vendor/gitleaks/rules-2.json' with { type: 'json' };
 import r3 from './vendor/gitleaks/rules-3.json' with { type: 'json' };
 import r4 from './vendor/gitleaks/rules-4.json' with { type: 'json' };
 import r5 from './vendor/gitleaks/rules-5.json' with { type: 'json' };
+import b0 from './vendor/betterleaks/rules-0.json' with { type: 'json' };
+import b1 from './vendor/betterleaks/rules-1.json' with { type: 'json' };
+import b2 from './vendor/betterleaks/rules-2.json' with { type: 'json' };
+import b3 from './vendor/betterleaks/rules-3.json' with { type: 'json' };
+import b4 from './vendor/betterleaks/rules-4.json' with { type: 'json' };
+import b5 from './vendor/betterleaks/rules-5.json' with { type: 'json' };
 import { entropy, REDACTED } from './detection.js';
 import { keywordMatcher } from './keywords.js';
+import { tokenChecksum } from './token-checksum.js';
 
 export const gitleaksRules = [...r0, ...r1, ...r2, ...r3, ...r4, ...r5];
+// Betterleaks rules whose ids are absent from the frozen Gitleaks set; see
+// scripts/sync-betterleaks.py for the translated filter subset.
+export const betterleaksRules = [...b0, ...b1, ...b2, ...b3, ...b4, ...b5];
+const upstreamRules = [
+  ...gitleaksRules.map((rule) => ({ rule, name: `gitleaks/${rule.id}` })),
+  ...betterleaksRules.map((rule) => ({
+    rule,
+    name: `betterleaks/${rule.id}`,
+  })),
+];
 const compiled = new Map();
 const linearPatterns = new Map();
 const keywordIndex = keywordMatcher(
-  gitleaksRules.flatMap((r) =>
-    (r.keywords ?? []).map((word) => word.toLowerCase())
+  upstreamRules.flatMap(({ rule }) =>
+    (rule.keywords ?? []).map((word) => word.toLowerCase())
   )
 );
+const confidence = { high: 0.95, medium: 0.85, low: 0.75 };
+// re2-wasm has a fixed 16 MiB heap shared by every compiled rule and every
+// subject copy, and an allocation failure aborts the module for the rest of
+// the process. All bundled rules compiled take about 7 MiB, and each exec
+// copies its subject, so keep subjects well below the remaining space and
+// scan larger texts with RE2JS (experiments/issue-128/re2-big-input.mjs).
+export const WASM_SUBJECT_LIMIT = 1024 * 1024;
 
 export function detectGitleaks(text, emit, trace) {
   const keywords = keywordIndex(text.toLowerCase());
-  const large = Buffer.byteLength(text) > 4 * 1024 * 1024;
-  for (const rule of gitleaksRules) {
+  const large = Buffer.byteLength(text) > WASM_SUBJECT_LIMIT;
+  for (const { rule, name } of upstreamRules) {
     if (
       !rule.regex ||
       (rule.keywords?.length &&
@@ -32,10 +56,10 @@ export function detectGitleaks(text, emit, trace) {
     }
     const started = trace ? performance.now() : 0;
     for (const view of ruleViews(text, rule)) {
-      scanRule(view.text, view.start, rule, emit, large);
+      scanRule(view.text, view.start, { rule, name }, emit, large);
     }
     trace?.({
-      rule: rule.id,
+      rule: name.startsWith('gitleaks/') ? rule.id : name,
       length: text.length,
       ms: performance.now() - started,
     });
@@ -118,8 +142,8 @@ function* boundedViews(text, prefix, id) {
   }
 }
 
-function scanRule(text, base, rule, emit, large) {
-  for (const match of ruleMatches(text, rule, large)) {
+function scanRule(text, base, { rule, name }, emit, large) {
+  for (const match of ruleMatches(text, name, rule, large)) {
     const secret = capturedSecret(rule, match);
     if (!secret) {
       continue;
@@ -133,17 +157,25 @@ function scanRule(text, base, rule, emit, large) {
       end: base + match.index + last + secret.length,
       type: 'SECRET',
       category: 'credential',
-      rule: `gitleaks/${rule.id}`,
-      confidence: rule.entropy ? 0.85 : 0.95,
+      rule: name,
+      confidence: ruleConfidence(rule, secret),
     });
   }
 }
 
-function* ruleMatches(text, rule, large) {
+// A token whose embedded CRC32 checksum matches is structurally valid offline.
+function ruleConfidence(rule, secret) {
+  if (tokenChecksum(secret)) {
+    return 0.99;
+  }
+  return confidence[rule.confidence] ?? (rule.entropy ? 0.85 : 0.95);
+}
+
+function* ruleMatches(text, name, rule, large) {
   if (large) {
-    // re2-wasm has a fixed 16 MiB heap. RE2JS implements the same linear-time
-    // syntax without copying the input into WASM. Its indices are UTF-16.
-    yield* linearMatches(text, rule);
+    // RE2JS implements the same linear-time syntax without copying the input
+    // into WASM. Its indices are UTF-16.
+    yield* linearMatches(text, name, rule.regex);
     return;
   }
   // re2-wasm 1.0.2 returns codepoint indices despite its RegExp-like API.
@@ -157,11 +189,7 @@ function* ruleMatches(text, rule, large) {
       offsets.push(offset);
     }
   }
-  let pattern = compiled.get(rule.id);
-  if (!pattern) {
-    pattern = new RE2(rule.regex, 'gu');
-    compiled.set(rule.id, pattern);
-  }
+  const pattern = compiledRule(name, rule.regex);
   pattern.lastIndex = 0;
   for (let attempts = 0; attempts < 8; attempts++) {
     const match = pattern.exec(text);
@@ -177,16 +205,32 @@ function* ruleMatches(text, rule, large) {
   // Keep the original subject: slicing would change anchors and boundaries.
   const resume = offsets ? offsets[pattern.lastIndex] : pattern.lastIndex;
   if (resume !== undefined && resume <= text.length) {
-    yield* linearMatches(text, rule, resume);
+    yield* linearMatches(text, name, rule.regex, resume);
+  }
+}
+
+// One cached re2-wasm object per rule name. re2-wasm objects are never freed,
+// so every caller, tests included, shares this cache.
+export function compiledRule(name, source) {
+  let pattern = compiled.get(name);
+  if (!pattern) {
+    pattern = new RE2(source, 'gu');
+    compiled.set(name, pattern);
   }
+  return pattern;
 }
 
-function* linearMatches(text, rule, resume = 0) {
-  let linear = linearPatterns.get(rule.id);
+function linearPattern(key, source) {
+  let linear = linearPatterns.get(key);
   if (!linear) {
-    linear = RE2JS.compile(rule.regex);
-    linearPatterns.set(rule.id, linear);
+    linear = RE2JS.compile(source);
+    linearPatterns.set(key, linear);
   }
+  return linear;
+}
+
+function* linearMatches(text, name, source, resume = 0) {
+  const linear = linearPattern(name, source);
   if (linear.test(text)) {
     for (const match of linear.matchAll(text)) {
       if (match.index >= resume) {
@@ -200,9 +244,20 @@ function capturedSecret(rule, match) {
   const secret = rule.secretGroup
     ? match[rule.secretGroup]
     : (match.slice(1).find(Boolean) ?? match[0]);
-  return !secret ||
-    secret === REDACTED ||
-    (rule.entropy && entropy(secret) < rule.entropy)
+  return !secret || secret === REDACTED || filtered(rule, secret)
     ? undefined
     : secret;
 }
+
+// Secret-only Betterleaks filter clauses, translated by the sync script.
+function filtered(rule, secret) {
+  const score = rule.entropy ? entropy(secret) : Infinity;
+  const test = (source) =>
+    linearPattern(`filter:${source}`, source).test(secret);
+  return (
+    score < rule.entropy ||
+    (rule.entropyInclusive && score === rule.entropy) ||
+    (rule.reject ?? []).some(test) ||
+    !(rule.require ?? []).every(test)
+  );
+}

package/src/identifier-detector.js

@@ -0,0 +1,36 @@
+import { failure } from './detection.js';
+import { detectNational } from './national.js';
+
+const validRule = (rule) =>
+  rule &&
+  typeof rule.type === 'string' &&
+  /^[A-Z][A-Z0-9_]{0,63}$/.test(rule.type) &&
+  rule.pattern instanceof RegExp &&
+  rule.pattern.global &&
+  rule.context instanceof RegExp &&
+  !rule.context.global &&
+  !rule.context.sticky &&
+  typeof rule.checksum === 'function' &&
+  ['plausible', 'valid'].every(
+    (key) => rule[key] === undefined || typeof rule[key] === 'function'
+  );
+
+// Adapts identifier rules (the `nationalRules` shape) to the Detector port, so
+// callers can add or replace identifiers without forking the built-in table.
+export function createIdentifierDetector(
+  rules,
+  { id = 'identifier-rules' } = {}
+) {
+  if (!Array.isArray(rules) || rules.length === 0 || !rules.every(validRule)) {
+    throw failure('ERR_CONFIG');
+  }
+  const frozen = rules.map((rule) => Object.freeze({ ...rule }));
+  return {
+    id,
+    detect(text) {
+      const findings = [];
+      detectNational(text, (finding) => findings.push(finding), frozen);
+      return findings;
+    },
+  };
+}

package/src/identity.js

@@ -38,6 +38,20 @@ export const identityTypes = new Set([
   'CA_SIN',
   'RU_SNILS',
   'RU_INN',
+  'US_ITIN',
+  'US_DRIVER_LICENSE',
+  'UK_DRIVING_LICENCE',
+  'ES_NIE',
+  'IN_AADHAAR',
+  'IN_VOTER',
+  'IN_PASSPORT',
+  'FI_PERSONAL_IDENTITY_CODE',
+  'TH_TNIN',
+  'AU_MEDICARE',
+  'DE_TAX_ID',
+  'AE_EMIRATES_ID',
+  'SA_NATIONAL_ID',
+  'IL_ID_NUMBER',
 ]);
 export function isIdentityType(type) {
   return (