euredact 0.2.0 → 0.3.0

package/dist/esm/rules/countries/index.js

@@ -1,4 +1,24 @@
 import { EntityType } from "../../types.js";
+import { IBAN_LENGTHS } from "../validators.js";
+/**
+ * Build a country-independent IBAN pattern from the ISO 13616 length table.
+ *
+ * An IBAN carries its own country code and a mod-97 checksum, so it is
+ * self-identifying and must never depend on the caller's `countries` setting —
+ * a Belgian account in an Austrian file is still an account.
+ *
+ * Each alternative pins the *exact* character count for its country, which
+ * stops the match absorbing a following word: without it, greedy matching
+ * turns `BE68 5390 0754 7034 KBC` into one over-long candidate that fails
+ * mod-97 and is lost entirely.
+ */
+function buildIbanPattern() {
+    const alternatives = Object.entries(IBAN_LENGTHS)
+        .sort(([a], [b]) => a.localeCompare(b))
+        .map(([code, length]) => `${code}\\d{2}(?:[ \\-]?[A-Z0-9]){${length - 4}}`);
+    return String.raw `\b(?:` + alternatives.join("|") + String.raw `)\b`;
+}
+const GENERIC_IBAN = buildIbanPattern();
 /**
  * Helper to create a PatternDef with sensible defaults.
  */
@@ -6,6 +26,45 @@ function p(entityType, pattern, validator = null, description = "", contextKeywo
     return { entityType, pattern, validator, description, contextKeywords, requiresContext };
 }
 // ---------------------------------------------------------------------------
+// Secret context keywords (consolidated, reused across SECRET patterns)
+// ---------------------------------------------------------------------------
+const SECRET_CONTEXT = [
+    // English
+    "key", "token", "secret", "password", "credential",
+    "api_key", "apikey", "api-key", "auth", "bearer",
+    "aws_secret_access_key", "access_key", "private_key",
+    "AES_KEY", "ENCRYPTION_KEY", "SIGNING_KEY",
+    "ACCOUNT_SID", "AUTH_TOKEN", "TWILIO",
+    "RAILS_SECRET", "SECRET_KEY_BASE", "DATABASE_URL",
+    "NPM_TOKEN", "DIGITALOCEAN", "GOOGLE_API",
+    "DB_PASS", "DB_PASSWORD", "DATASOURCE_PASSWORD",
+    "JWT_SIGNING", "passwd", "DISCORD_TOKEN",
+    // Dutch
+    "wachtwoord", "sleutel", "geheim",
+    // French
+    "mot de passe", "mot_de_passe", "clé", "jeton",
+    // German
+    "Passwort", "Kennwort", "Schlüssel", "Geheimnis",
+    // Italian
+    "chiave", "segreto",
+    // Spanish
+    "contraseña", "clave",
+    // Portuguese
+    "senha", "chave",
+    // Polish
+    "hasło", "klucz",
+    // Czech/Slovak
+    "heslo", "kľúč",
+    // Hungarian
+    "jelszó", "kulcs",
+    // Romanian
+    "parola",
+    // Nordic
+    "lösenord", "nøkkel", "nyckel", "avain", "lykill",
+    // Greek
+    "κωδικός",
+];
+// ---------------------------------------------------------------------------
 // DOB context keywords (reused across SHARED patterns)
 // ---------------------------------------------------------------------------
 const DOB_CONTEXT = [
@@ -33,8 +92,28 @@ const SHARED = {
     code: "SHARED",
     name: "Shared EU Patterns",
     patterns: [
-        p(EntityType.EMAIL, String.raw `\b[\w._%+\-]+@[\w.\-]+\.[a-zA-Z]{2,}\b`),
+        // JavaScript's \w and \b are ASCII-only, so the Python engine's \w (which
+        // is Unicode-aware) matched local parts like "petras_ž@..." while this one
+        // did not. \p{L}\p{N} with the `u` flag restores parity; the leading guard
+        // is a Unicode-safe word boundary.
+        p(EntityType.EMAIL, String.raw `(?<![\p{L}\p{N}._%+\-])[\p{L}\p{N}._%+\-]+@(?:[\p{L}\p{N}\-]+\.)+\p{L}{2,}\b`),
+        // IBAN (any country) — structure + mod-97 only. Country-specific IBAN
+        // patterns still exist and win on identical spans; this one guarantees an
+        // IBAN is never missed just because its country was not requested.
+        p(EntityType.BANK_ACCOUNT, GENERIC_IBAN, "iban", "IBAN — any country (ISO 13616, mod-97 validated)"),
+        // International phone (E.164, any country). Runs alongside the per-country
+        // patterns, which each hard-code a single grouping and miss the rest:
+        // "+43 664 8213 907" was invisible to the Austrian pattern, which expects
+        // an unbroken subscriber number. A leading "+" is self-identifying, so
+        // this is deliberately not country-gated.
+        p(EntityType.PHONE, String.raw `\+\d{1,3}(?:[\s\-/]?\(?\d{1,4}\)?){2,7}`, "e164", "International phone number (E.164) — any country"),
         p(EntityType.BIC, String.raw `\b[A-Z]{4}[A-Z]{2}[A-Z0-9]{2}(?:[A-Z0-9]{3})?\b`, "bic"),
+        // Space-separated form: "RZBA AT WW", "NICA BE BB". Written as a separate
+        // pattern requiring a space between *every* group, rather than making the
+        // spaces optional above — optional spaces would let "GEBABEBB KBC" match
+        // as one 11-character code and claim the following word. Only a literal
+        // space is allowed, so a match cannot span a line break.
+        p(EntityType.BIC, String.raw `\b[A-Z]{4} [A-Z]{2} [A-Z0-9]{2}(?: [A-Z0-9]{3})?\b`, "bic", "BIC/SWIFT code (space-separated groups)"),
         p(EntityType.CREDIT_CARD, String.raw `\b(?:4[0-9]{3}[\s\-]?[0-9]{4}[\s\-]?[0-9]{4}[\s\-]?[0-9]{4}|5[1-5][0-9]{2}[\s\-]?[0-9]{4}[\s\-]?[0-9]{4}[\s\-]?[0-9]{4}|3[47][0-9]{2}[\s\-]?[0-9]{6}[\s\-]?[0-9]{5})\b`, "luhn"),
         p(EntityType.VIN, String.raw `\b[A-HJ-NPR-Z0-9]{17}\b`, "vin"),
         p(EntityType.IP_ADDRESS, String.raw `\b(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\b`),
@@ -45,7 +124,7 @@ const SHARED = {
         p(EntityType.IMEI, String.raw `\b\d{2}[\-\s]\d{6}[\-\s]\d{6}[\-\s]\d\b`, "imei", "formatted"),
         p(EntityType.GPS_COORDINATES, String.raw `-?(?:[1-8]?\d(?:\.\d{4,})|90(?:\.0{4,}))\s*[,;/]\s*-?(?:1[0-7]\d(?:\.\d{4,})|180(?:\.0{4,})|\d{1,2}(?:\.\d{4,}))`),
         p(EntityType.UUID, String.raw `\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}\b`),
-        p(EntityType.SOCIAL_HANDLE, String.raw `(?<!\w)@[a-zA-Z][a-zA-Z0-9_.]{1,29}\b`),
+        p(EntityType.SOCIAL_HANDLE, String.raw `(?<![\p{L}\p{N}_])@\p{L}[\p{L}\p{N}_.]{1,29}\b`),
         // --- Secret / API Key (known prefixes — always active) ---
         p(EntityType.SECRET, String.raw `\bAKIA[A-Z0-9]{16}\b`, null, "AWS Access Key ID"),
         p(EntityType.SECRET, String.raw `\bgh[poas]_[a-zA-Z0-9]{36,}\b`, null, "GitHub token"),
@@ -67,35 +146,18 @@ const SHARED = {
         p(EntityType.SECRET, String.raw `\bxapp-\d[\d\-]{20,}`, null, "Slack app numeric token"),
         p(EntityType.SECRET, String.raw `\bxapp-[a-zA-Z0-9\-]{20,}`, null, "Slack app token"),
         p(EntityType.SECRET, String.raw `\bdapi[a-f0-9]{32,}\b`, null, "Databricks PAT"),
+        // Discord bot token (3-segment base64 with numeric first segment)
+        p(EntityType.SECRET, String.raw `\b[MN][A-Za-z0-9]{23,}\.[A-Za-z0-9_\-]{6}\.[A-Za-z0-9_\-]{27,}\b`, null, "Discord bot token"),
+        // PEM private key headers
+        p(EntityType.SECRET, String.raw `-----BEGIN\s(?:RSA\s)?PRIVATE\sKEY-----[\s\S]{0,16384}?-----END\s(?:RSA\s)?PRIVATE\sKEY-----`, null, "PEM private key"),
+        // AWS Secret Access Key (40 chars, base64 with +/ after an AKIA line)
+        p(EntityType.SECRET, String.raw `(?<=aws_secret_access_key\s{0,4}[:=]\s{0,4})[A-Za-z0-9+/]{40}`, null, "AWS Secret Access Key"),
         // --- Secret / API Key (connection strings with embedded credentials) ---
-        p(EntityType.SECRET, String.raw `(?:mongodb|mysql|postgres(?:ql)?|redis|amqp|rabbitmq):\/\/[^\s:]+:[^\s@]+@[^\s]+`, null, "Connection string with credentials"),
+        p(EntityType.SECRET, String.raw `(?:mongodb|mysql|postgres(?:ql)?|redis|amqp|rabbitmq):\/\/[^\s:]{1,256}:[^\s@]{1,256}@[^\s]+`, null, "Connection string with credentials"),
         // --- Secret / API Key (assignment-based: KEY=value or KEY: value) ---
-        p(EntityType.SECRET, String.raw `(?<=[:=] )\S{8,}|(?<=[:=])\S{8,}`, "high_entropy", "Assigned secret value", [
-            "key", "token", "secret", "password", "credential",
-            "api_key", "apikey", "api-key", "auth", "bearer",
-            "aws_secret_access_key", "access_key", "private_key",
-            "wachtwoord", "mot de passe", "Passwort", "Schlüssel",
-            "clé", "sleutel", "lösenord", "hasło", "heslo",
-            "jelszó", "contraseña", "senha", "parola",
-            "AES_KEY", "ENCRYPTION_KEY", "SIGNING_KEY",
-            "ACCOUNT_SID", "AUTH_TOKEN", "TWILIO",
-            "RAILS_SECRET", "SECRET_KEY_BASE", "DATABASE_URL",
-            "NPM_TOKEN", "DIGITALOCEAN", "GOOGLE_API",
-            "kľúč", "nøkkel", "nyckel", "avain", "lykill",
-            "DB_PASS", "DB_PASSWORD", "DATASOURCE_PASSWORD",
-            "JWT_SIGNING", "SIGNING_KEY",
-            "passwd", "Kennwort",
-        ], true),
+        p(EntityType.SECRET, String.raw `(?<=[:=] )\S{8,}|(?<=[:=])\S{8,}`, "high_entropy", "Assigned secret value", SECRET_CONTEXT, true),
         // --- Secret / API Key (entropy-based fallback for longer tokens) ---
-        p(EntityType.SECRET, String.raw `\b[A-Za-z0-9_\-+/]{24,}[A-Za-z0-9_\-+/=]*\b`, "high_entropy", "High-entropy token", [
-            "key", "token", "secret", "password", "credential",
-            "api_key", "apikey", "api-key", "auth", "bearer",
-            "aws_secret_access_key", "access_key", "private_key",
-            "wachtwoord", "mot de passe", "Passwort", "Schlüssel",
-            "clé", "sleutel", "lösenord", "hasło", "heslo",
-            "jelszó", "contraseña", "senha", "parola",
-            "kľúč", "nøkkel", "nyckel", "avain", "lykill",
-        ], true),
+        p(EntityType.SECRET, String.raw `\b[A-Za-z0-9_\-+/]{24,}[A-Za-z0-9_\-+/=]*\b`, "high_entropy", "High-entropy token", SECRET_CONTEXT, true),
         p(EntityType.DOB, String.raw `\b(?:0[1-9]|[12][0-9]|3[01])[/.\-](?:0[1-9]|1[0-2])[/.\-](?:19|20)\d{2}\b`, null, "DD/MM/YYYY", DOB_CONTEXT, true),
         p(EntityType.DATE_OF_DEATH, String.raw `\b(?:0[1-9]|[12][0-9]|3[01])[/.\-](?:0[1-9]|1[0-2])[/.\-](?:19|20)\d{2}\b`, null, "", DATE_OF_DEATH_CONTEXT, true),
         p(EntityType.DOB, String.raw `\b(?:19|20)\d{2}[/.\-](?:0[1-9]|1[0-2])[/.\-](?:0[1-9]|[12][0-9]|3[01])\b`, null, "ISO YYYY-MM-DD", DOB_ISO_CONTEXT, true),
@@ -169,6 +231,11 @@ const AT = {
         p(EntityType.IBAN, String.raw `\bAT\d{2}\s?\d{4}\s?\d{4}\s?\d{4}\s?\d{4}\b`, "iban"),
         p(EntityType.VAT, String.raw `\bATU\d{8}\b`),
         p(EntityType.PHONE, String.raw `\b0\d{3,4}[\s\-]?\d{5,8}\b`),
+        // Short area codes (Vienna is "01") split across three groups:
+        // "01 53460 2215". The pattern above requires 3-4 digits after the trunk
+        // 0, so it never matched these. Both separators are mandatory here, which
+        // keeps "01 2025" (a date fragment) out.
+        p(EntityType.PHONE, String.raw `\b0\d{1,4}[\s\-/]\d{3,7}[\s\-/]\d{2,6}\b`, null, "Austrian national phone — short area code, grouped"),
         p(EntityType.PHONE, String.raw `\+43\s?\d{3,4}[\s\-]?\d{5,8}`),
         p(EntityType.LICENSE_PLATE, String.raw `\b[A-ZÄÖÜ]{1,2}\s?\d{1,5}\s?[A-Z]{1,2}\b`, null, "", ["Kennzeichen", "Nummernschild", "Kfz-Kennzeichen"], true),
         p(EntityType.POSTAL_CODE, String.raw `\b[1-9]\d{3}\b`, null, "", ["PLZ", "Postleitzahl", "Adresse", "Anschrift", "Straße", "Str.", "Gasse", "Weg", "Platz", "Postal:", "Wohnort"], true),

package/dist/esm/rules/engine.d.ts

@@ -1,4 +1,12 @@
 import { type Detection } from "../types.js";
+/**
+ * The country modules use the EU/VAT convention (UK, EL) while the public
+ * contract is ISO 3166-1 alpha-2 (GB, GR). Accept both spellings so a caller
+ * passing correct ISO codes is not punished for it.
+ */
+export declare const COUNTRY_CODE_ALIASES: Record<string, string>;
+/** Map a caller-supplied code to a registered one, or null if unknown. */
+export declare function resolveCountryCode(countryCode: string): string | null;
 export declare class RuleEngine {
     private matcher;
     private loadedCountries;

package/dist/esm/rules/engine.js

@@ -1,12 +1,62 @@
-import { DetectionSource } from "../types.js";
+import { DetectionSource, EntityType } from "../types.js";
 import { MultiPatternMatcher } from "./matchers.js";
 import { shouldSuppress } from "./suppressors.js";
 import { detectStructuralDob } from "./structural.js";
 import { COUNTRY_CONFIGS } from "./countries/index.js";
+/**
+ * Heuristic to detect nested quantifiers that cause catastrophic backtracking.
+ * After stripping escaped chars and character classes, look for an unbounded
+ * quantifier (+/*) closing a group that is itself quantified (+/*).
+ */
+const NESTED_QUANTIFIER_RE = /[+*]\)[+*]/;
+function validateCustomPattern(pattern) {
+    try {
+        new RegExp(pattern, "gu");
+    }
+    catch (e) {
+        throw new Error(`Invalid regex pattern: ${e instanceof Error ? e.message : e}`);
+    }
+    const stripped = pattern
+        .replace(/\\./g, "X")
+        .replace(/\[(?:[^\]\\]|\\.)*\]/g, "X");
+    if (NESTED_QUANTIFIER_RE.test(stripped)) {
+        throw new Error("Pattern contains nested quantifiers (e.g. (a+)+) which can cause " +
+            "catastrophic backtracking (ReDoS). Restructure the pattern to " +
+            "avoid quantifiers inside quantified groups.");
+    }
+}
+/**
+ * The country modules use the EU/VAT convention (UK, EL) while the public
+ * contract is ISO 3166-1 alpha-2 (GB, GR). Accept both spellings so a caller
+ * passing correct ISO codes is not punished for it.
+ */
+export const COUNTRY_CODE_ALIASES = {
+    GB: "UK",
+    GR: "EL",
+    UK: "UK",
+    EL: "EL",
+};
+/** Map a caller-supplied code to a registered one, or null if unknown. */
+export function resolveCountryCode(countryCode) {
+    let code = countryCode.trim().toUpperCase();
+    code = COUNTRY_CODE_ALIASES[code] ?? code;
+    return code in COUNTRY_CONFIGS && code !== "SHARED" ? code : null;
+}
+const warnedCountries = new Set();
+function warnUnknownCountry(code) {
+    if (warnedCountries.has(code))
+        return;
+    warnedCountries.add(code);
+    const available = Object.keys(COUNTRY_CONFIGS).filter(c => c !== "SHARED").sort();
+    console.warn(`[euredact] Unknown country code: ${JSON.stringify(code)}. Continuing with ` +
+        `shared country-independent patterns only (email, IBAN, international ` +
+        `phone, credit card, ...). Available: ${available.join(", ")}`);
+}
 export class RuleEngine {
     matcher = new MultiPatternMatcher();
     loadedCountries = new Set();
     addCustomPattern(name, pattern) {
+        validateCustomPattern(pattern);
         const pdef = {
             entityType: name, pattern, validator: null,
             description: "", contextKeywords: [], requiresContext: false,
@@ -22,7 +72,15 @@ export class RuleEngine {
         else {
             configs = [];
             for (const c of countryCodes) {
-                const code = c.toUpperCase();
+                const code = resolveCountryCode(c);
+                if (code === null) {
+                    // Detection continues with the shared, country-independent
+                    // patterns. Throwing on an unknown locale invites callers to wrap
+                    // the call in try/catch and skip redaction — failing open, with
+                    // unredacted PII in the output.
+                    warnUnknownCountry(c);
+                    continue;
+                }
                 const config = COUNTRY_CONFIGS[code];
                 if (config)
                     configs.push(config);
@@ -44,38 +102,87 @@ export class RuleEngine {
     }
     detect(text, countryCodes) {
         this.loadCountries(countryCodes);
-        let rawMatches = this.matcher.scan(text);
+        // Scope the scan to requested countries for efficiency
+        // Resolve through the alias table so `countries: ["GB"]` actually scans
+        // the patterns registered under "UK".
+        let scanCodes = null;
         if (countryCodes !== null) {
-            const codesUpper = new Set(countryCodes.map(c => c.toUpperCase()));
-            rawMatches = rawMatches.filter(m => codesUpper.has(m.countryCode) || m.countryCode === "SHARED" || m.countryCode === "CUSTOM");
+            scanCodes = new Set();
+            for (const c of countryCodes) {
+                const code = resolveCountryCode(c);
+                if (code !== null)
+                    scanCodes.add(code);
+            }
+            scanCodes.add("SHARED");
+            scanCodes.add("CUSTOM");
         }
+        let rawMatches = this.matcher.scan(text, scanCodes);
         // Matches that have a validator but fail it create suppression zones:
         // the span is recognisably a specific entity (e.g. IBAN-shaped) so
         // overlapping regex-only matches (license plate, phone) are false
         // positives and must be suppressed.
         const validated = [];
-        const suppressionZones = [];
+        const rawZones = [];
         for (const m of rawMatches) {
-            if (this.matcher.validate(m)) {
-                validated.push(m);
+            const isValid = this.matcher.validate(m);
+            if (isValid) {
+                validated.push({ match: m, hasValidValidator: m.patternDef.validator !== null });
             }
             else if (m.patternDef.validator !== null && !m.patternDef.requiresContext) {
-                suppressionZones.push([m.start, m.end]);
+                rawZones.push([m.start, m.end]);
+            }
+        }
+        // Merge overlapping suppression zones for O(log n) containment checks
+        let zoneStarts = [];
+        let zoneEnds = [];
+        if (rawZones.length > 0) {
+            rawZones.sort((a, b) => a[0] - b[0] || a[1] - b[1]);
+            let [ms, me] = rawZones[0];
+            for (let i = 1; i < rawZones.length; i++) {
+                const [zs, ze] = rawZones[i];
+                if (zs <= me) {
+                    me = Math.max(me, ze);
+                }
+                else {
+                    zoneStarts.push(ms);
+                    zoneEnds.push(me);
+                    ms = zs;
+                    me = ze;
+                }
             }
+            zoneStarts.push(ms);
+            zoneEnds.push(me);
         }
         // Build candidates with priority: validated (3) > custom (2) > regex-only (1)
         const candidates = [];
-        for (const match of validated) {
+        for (const { match, hasValidValidator } of validated) {
             if (shouldSuppress(text, match))
                 continue;
-            if (match.patternDef.validator === null && suppressionZones.length > 0) {
-                if (suppressionZones.some(([zStart, zEnd]) => match.start >= zStart && match.end <= zEnd)) {
-                    continue;
+            // O(log n) binary-search containment check on merged zones
+            if (match.patternDef.validator === null && zoneStarts.length > 0) {
+                let lo = 0, hi = zoneStarts.length;
+                while (lo < hi) {
+                    const mid = (lo + hi) >>> 1;
+                    if (zoneStarts[mid] <= match.start)
+                        lo = mid + 1;
+                    else
+                        hi = mid;
                 }
+                const idx = lo - 1;
+                if (idx >= 0 && zoneEnds[idx] >= match.end)
+                    continue;
             }
-            const hasValidator = match.patternDef.validator !== null;
-            const isValid = this.matcher.validate(match);
-            const priority = (hasValidator && isValid) ? 3 : match.countryCode === "CUSTOM" ? 2 : 1;
+            // A bare digit run is the weakest evidence in the engine and must never
+            // re-cut a span a structured detector claims (PHONE, SSN, NATIONAL_ID,
+            // BANK_ACCOUNT, VAT...). Postal codes take unclaimed spans only,
+            // whatever the span lengths.
+            const priority = hasValidValidator
+                ? 3
+                : match.countryCode === "CUSTOM"
+                    ? 2
+                    : match.patternDef.entityType === EntityType.POSTAL_CODE
+                        ? 0
+                        : 1;
             const spanLength = match.end - match.start;
             const score = priority * 1_000_000 + spanLength;
             candidates.push({
@@ -106,16 +213,20 @@ export class RuleEngine {
             return [];
         const sorted = [...candidates].sort((a, b) => b.score - a.score);
         const result = [];
+        const occupied = new Set();
         for (const { det } of sorted) {
             let overlaps = false;
-            for (const kept of result) {
-                if (det.start < kept.end && det.end > kept.start) {
+            for (let p = det.start; p < det.end; p++) {
+                if (occupied.has(p)) {
                     overlaps = true;
                     break;
                 }
             }
-            if (!overlaps)
+            if (!overlaps) {
+                for (let p = det.start; p < det.end; p++)
+                    occupied.add(p);
                 result.push(det);
+            }
         }
         return result;
     }

package/dist/esm/rules/matchers.d.ts

@@ -7,6 +7,6 @@ export declare class MultiPatternMatcher {
     addPattern(patternDef: PatternDef, countryCode: string): void;
     addCountry(config: CountryConfig): void;
     compile(): void;
-    scan(text: string): RawMatch[];
+    scan(text: string, countryCodes?: Set<string> | null): RawMatch[];
     validate(match: RawMatch): boolean;
 }

package/dist/esm/rules/matchers.js

@@ -19,11 +19,13 @@ export class MultiPatternMatcher {
     compile() {
         this.compiled = true;
     }
-    scan(text) {
+    scan(text, countryCodes = null) {
         if (!this.compiled)
             this.compile();
         const matches = [];
         for (const { regex, patternDef, countryCode } of this.patterns) {
+            if (countryCodes !== null && !countryCodes.has(countryCode))
+                continue;
             regex.lastIndex = 0;
             let m;
             while ((m = regex.exec(text)) !== null) {