euredact 0.1.0 → 0.3.0

package/dist/esm/rules/validators.js

@@ -1,21 +1,45 @@
 function clean(candidate, chars = /[\s.\-]/g) {
     return candidate.replace(chars, "");
 }
+/**
+ * Total IBAN length per country (ISO 13616 registry). Also used to build the
+ * country-independent IBAN pattern in rules/countries, so an IBAN is detected
+ * on its own structure rather than on the caller's `countries` setting.
+ */
+export const IBAN_LENGTHS = {
+    AL: 28, AD: 24, AT: 20, AZ: 28, BH: 22, BY: 28, BE: 16, BA: 20, BR: 29, BG: 22, CR: 22, HR: 21,
+    CY: 28, CZ: 24, DK: 18, DO: 28, TL: 23, EE: 20, FO: 18, FI: 18, FR: 27, GE: 22, DE: 22, GI: 23,
+    GR: 27, GL: 18, GT: 28, HU: 28, IS: 26, IQ: 23, IE: 22, IL: 23, IT: 27, JO: 30, KZ: 20, XK: 20,
+    KW: 30, LV: 21, LB: 28, LI: 21, LT: 20, LU: 20, MT: 31, MR: 27, MU: 30, MC: 27, MD: 24, ME: 22,
+    NL: 18, MK: 19, NO: 15, PK: 24, PS: 29, PL: 28, PT: 25, QA: 29, RO: 24, LC: 32, SM: 27, SA: 24,
+    RS: 22, SC: 31, SK: 24, SI: 19, ES: 24, SE: 24, CH: 21, TN: 24, TR: 26, UA: 29, AE: 23, GB: 22,
+    VA: 22, VG: 24,
+};
+/** ISO 3166-1 alpha-2 codes, plus XK (Kosovo, user-assigned but in banking use). */
+export const ISO_3166_ALPHA2 = new Set([
+    "AD", "AE", "AF", "AG", "AI", "AL", "AM", "AO", "AQ", "AR", "AS", "AT", "AU", "AW", "AX", "AZ",
+    "BA", "BB", "BD", "BE", "BF", "BG", "BH", "BI", "BJ", "BL", "BM", "BN", "BO", "BQ", "BR", "BS",
+    "BT", "BV", "BW", "BY", "BZ", "CA", "CC", "CD", "CF", "CG", "CH", "CI", "CK", "CL", "CM", "CN",
+    "CO", "CR", "CU", "CV", "CW", "CX", "CY", "CZ", "DE", "DJ", "DK", "DM", "DO", "DZ", "EC", "EE",
+    "EG", "EH", "ER", "ES", "ET", "FI", "FJ", "FK", "FM", "FO", "FR", "GA", "GB", "GD", "GE", "GF",
+    "GG", "GH", "GI", "GL", "GM", "GN", "GP", "GQ", "GR", "GS", "GT", "GU", "GW", "GY", "HK", "HM",
+    "HN", "HR", "HT", "HU", "ID", "IE", "IL", "IM", "IN", "IO", "IQ", "IR", "IS", "IT", "JE", "JM",
+    "JO", "JP", "KE", "KG", "KH", "KI", "KM", "KN", "KP", "KR", "KW", "KY", "KZ", "LA", "LB", "LC",
+    "LI", "LK", "LR", "LS", "LT", "LU", "LV", "LY", "MA", "MC", "MD", "ME", "MF", "MG", "MH", "MK",
+    "ML", "MM", "MN", "MO", "MP", "MQ", "MR", "MS", "MT", "MU", "MV", "MW", "MX", "MY", "MZ", "NA",
+    "NC", "NE", "NF", "NG", "NI", "NL", "NO", "NP", "NR", "NU", "NZ", "OM", "PA", "PE", "PF", "PG",
+    "PH", "PK", "PL", "PM", "PN", "PR", "PS", "PT", "PW", "PY", "QA", "RE", "RO", "RS", "RU", "RW",
+    "SA", "SB", "SC", "SD", "SE", "SG", "SH", "SI", "SJ", "SK", "SL", "SM", "SN", "SO", "SR", "SS",
+    "ST", "SV", "SX", "SY", "SZ", "TC", "TD", "TF", "TG", "TH", "TJ", "TK", "TL", "TM", "TN", "TO",
+    "TR", "TT", "TV", "TW", "TZ", "UA", "UG", "UM", "US", "UY", "UZ", "VA", "VC", "VE", "VG", "VI",
+    "VN", "VU", "WF", "WS", "XK", "YE", "YT", "ZA", "ZM", "ZW",
+]);
 export function validateIban(candidate) {
     const c = candidate.replace(/[\s\-]/g, "").toUpperCase();
     if (c.length < 5 || !/^[A-Z]{2}/.test(c) || !/^\d{2}$/.test(c.slice(2, 4)))
         return false;
-    const ibanLengths = {
-        AL: 28, AD: 24, AT: 20, AZ: 28, BH: 22, BY: 28, BE: 16, BA: 20, BR: 29, BG: 22, CR: 22, HR: 21,
-        CY: 28, CZ: 24, DK: 18, DO: 28, TL: 23, EE: 20, FO: 18, FI: 18, FR: 27, GE: 22, DE: 22, GI: 23,
-        GR: 27, GL: 18, GT: 28, HU: 28, IS: 26, IQ: 23, IE: 22, IL: 23, IT: 27, JO: 30, KZ: 20, XK: 20,
-        KW: 30, LV: 21, LB: 28, LI: 21, LT: 20, LU: 20, MT: 31, MR: 27, MU: 30, MC: 27, MD: 24, ME: 22,
-        NL: 18, MK: 19, NO: 15, PK: 24, PS: 29, PL: 28, PT: 25, QA: 29, RO: 24, LC: 32, SM: 27, SA: 24,
-        RS: 22, SC: 31, SK: 24, SI: 19, ES: 24, SE: 24, CH: 21, TN: 24, TR: 26, UA: 29, AE: 23, GB: 22,
-        VA: 22, VG: 24,
-    };
     const country = c.slice(0, 2);
-    const expected = ibanLengths[country];
+    const expected = IBAN_LENGTHS[country];
     if (expected !== undefined && c.length !== expected)
         return false;
     const rearranged = c.slice(4) + c.slice(0, 4);
@@ -161,13 +185,22 @@ export function validateVin(candidate) {
         return false;
     return !/[IOQ]/.test(c);
 }
+/**
+ * BIC/SWIFT structural validation (ISO 9362): BBBB CC LL (PPP).
+ *
+ * Necessary but **not sufficient** — BIC has no check digit, and characters
+ * 5-6 of ordinary uppercase words are frequently valid country codes
+ * (`DRINGEND` -> `GE`, `HOSPITAL` -> `IT`). Emission additionally requires a
+ * registry hit or banking context; see `suppressBicWithoutEvidence`.
+ */
 export function validateBic(candidate) {
     const c = candidate.replace(/ /g, "").toUpperCase();
     if (c.length !== 8 && c.length !== 11)
         return false;
     if (!/^[A-Z]{4}/.test(c))
         return false;
-    if (!/^[A-Z]{2}$/.test(c.slice(4, 6)))
+    // Country code must be a real ISO 3166-1 alpha-2 code
+    if (!ISO_3166_ALPHA2.has(c.slice(4, 6)))
         return false;
     if (!/^[A-Z0-9]{2}$/.test(c.slice(6, 8)))
         return false;
@@ -175,6 +208,21 @@ export function validateBic(candidate) {
         return false;
     return true;
 }
+/**
+ * International phone number in E.164 form: `+` then 8-15 digits.
+ *
+ * A `+` prefix makes a phone number self-identifying, so this backs a
+ * country-independent pattern: the per-country patterns each hard-code one
+ * grouping and miss the others (`+43 664 8213 907` was invisible because the
+ * Austrian pattern expects an unbroken subscriber number).
+ */
+export function validateE164(candidate) {
+    if (!candidate.trimStart().startsWith("+"))
+        return false;
+    // A trunk prefix written as "(0)" is not part of the E.164 digit count.
+    const digits = candidate.replace(/\(\s*0\s*\)/g, "").replace(/\D/g, "");
+    return digits.length >= 8 && digits.length <= 15;
+}
 export function validateKvk(candidate) {
     const c = clean(candidate);
     return c.length === 8 && /^\d{8}$/.test(c);
@@ -578,6 +626,35 @@ export function validateGreekAmka(candidate) {
     }
     return total % 10 === 0;
 }
+function validateHighEntropy(candidate) {
+    const clean = candidate.trim();
+    if (clean.length < 8)
+        return false;
+    // Short passwords (< 24 chars): require at least 1 letter, 1 digit, 1 special char
+    if (clean.length < 24) {
+        const hasLetter = /[a-zA-Z]/.test(clean);
+        const hasDigit = /\d/.test(clean);
+        const hasSpecial = /[^a-zA-Z0-9]/.test(clean);
+        if (hasLetter && hasDigit && hasSpecial)
+            return true;
+        // For short tokens without special chars, require entropy check
+        if (!hasSpecial && clean.length < 16)
+            return false;
+    }
+    const freq = new Map();
+    for (const ch of clean)
+        freq.set(ch, (freq.get(ch) ?? 0) + 1);
+    const len = clean.length;
+    let entropy = 0;
+    for (const c of freq.values()) {
+        const p = c / len;
+        entropy -= p * Math.log2(p);
+    }
+    // Hex-only strings have max ~4.0 bits entropy, use lower threshold
+    const isHex = /^[a-fA-F0-9]+$/.test(clean);
+    const threshold = isHex ? 3.0 : clean.length < 24 ? 3.0 : 3.5;
+    return entropy >= threshold;
+}
 export const VALIDATORS = {
     iban: validateIban,
     bsn: validateBsn,
@@ -592,6 +669,7 @@ export const VALIDATORS = {
     french_nir: validateFrenchNir,
     vin: validateVin,
     bic: validateBic,
+    e164: validateE164,
     kvk: validateKvk,
     swedish_pnr: validateSwedishPnr,
     norwegian_fnr: validateNorwegianFnr,
@@ -621,4 +699,5 @@ export const VALIDATORS = {
     uk_nhs: validateUkNhs,
     greek_afm: validateGreekAfm,
     greek_amka: validateGreekAmka,
+    high_entropy: validateHighEntropy,
 };

package/dist/esm/sdk.d.ts

@@ -2,14 +2,22 @@ import { type RedactResult } from "./types.js";
 export interface RedactOptions {
     countries?: string[] | null;
     mode?: string;
-    pseudonymize?: boolean;
+    referentialIntegrity?: boolean;
     detectDates?: boolean;
     cache?: boolean;
 }
 export declare class EuRedact {
     private engine;
     private cache;
-    private pseudonymMapper;
+    private referentialMapper;
+    private maxInputLength;
+    constructor(options?: {
+        maxInputLength?: number;
+    });
+    addCustomPattern(name: string, pattern: string): void;
+    /** Clear the result cache and referential integrity mappings.
+     *  Call this in long-running processes to free PII from memory. */
+    clear(): void;
     redact(text: string, options?: RedactOptions): RedactResult;
     redactBatch(texts: string[], options?: RedactOptions): RedactResult[];
 }

package/dist/esm/sdk.js

@@ -3,10 +3,10 @@ import { normalize, mapOffsetToOriginal } from "./normalizer.js";
 import { RuleEngine } from "./rules/engine.js";
 import { EntityType } from "./types.js";
 const DATE_TYPES = new Set([EntityType.DOB, EntityType.DATE_OF_DEATH]);
-class PseudonymMapper {
+class ReferentialMapper {
     counters = new Map();
     mapping = new Map();
-    getPseudonym(text, entityType) {
+    getLabel(text, entityType) {
         if (!this.mapping.has(text)) {
             const count = (this.counters.get(entityType) ?? 0) + 1;
             this.counters.set(entityType, count);
@@ -14,13 +14,37 @@ class PseudonymMapper {
         }
         return this.mapping.get(text);
     }
+    clear() {
+        this.counters.clear();
+        this.mapping.clear();
+    }
 }
+const DEFAULT_MAX_INPUT_LENGTH = 10_485_760; // ~10 MB of text
 export class EuRedact {
     engine = new RuleEngine();
     cache = new ResultCache();
-    pseudonymMapper = new PseudonymMapper();
+    referentialMapper = new ReferentialMapper();
+    maxInputLength;
+    constructor(options) {
+        this.maxInputLength = options?.maxInputLength ?? DEFAULT_MAX_INPUT_LENGTH;
+    }
+    addCustomPattern(name, pattern) {
+        this.engine.addCustomPattern(name, pattern);
+        this.cache.clear();
+    }
+    /** Clear the result cache and referential integrity mappings.
+     *  Call this in long-running processes to free PII from memory. */
+    clear() {
+        this.cache.clear();
+        this.referentialMapper.clear();
+    }
     redact(text, options = {}) {
-        const { countries = null, mode = "rules", pseudonymize = false, detectDates = false, cache = true, } = options;
+        if (text.length > this.maxInputLength) {
+            throw new Error(`Input text length (${text.length.toLocaleString()} chars) exceeds the maximum ` +
+                `(${this.maxInputLength.toLocaleString()} chars). Split the input or ` +
+                `increase maxInputLength when constructing EuRedact.`);
+        }
+        const { countries = null, mode = "rules", referentialIntegrity = false, detectDates = false, cache = true, } = options;
         const [normalizedText, offsetMapping] = normalize(text);
         const countriesTuple = countries
             ? countries.map(c => c.toUpperCase()).sort()
@@ -48,8 +72,8 @@ export class EuRedact {
         let redacted = text;
         for (let i = detections.length - 1; i >= 0; i--) {
             const det = detections[i];
-            const replacement = pseudonymize
-                ? this.pseudonymMapper.getPseudonym(det.text, det.entityType)
+            const replacement = referentialIntegrity
+                ? this.referentialMapper.getLabel(det.text, det.entityType)
                 : `[${det.entityType}]`;
             redacted = redacted.slice(0, det.start) + replacement + redacted.slice(det.end);
         }

package/dist/esm/types.d.ts

@@ -1,7 +1,15 @@
 export declare enum EntityType {
     NAME = "NAME",
     ADDRESS = "ADDRESS",
-    IBAN = "IBAN",
+    /**
+     * Canonical name for a bank account identifier. `IBAN` is kept below as a
+     * legacy alias with the same value, so `EntityType.IBAN === EntityType.BANK_ACCOUNT`
+     * and existing code keeps working — but the emitted value, and therefore the
+     * `[BANK_ACCOUNT]` placeholder written into redacted text, is canonical.
+     */
+    BANK_ACCOUNT = "BANK_ACCOUNT",
+    /** @deprecated legacy alias of {@link EntityType.BANK_ACCOUNT} */
+    IBAN = "BANK_ACCOUNT",
     BIC = "BIC",
     CREDIT_CARD = "CREDIT_CARD",
     PHONE = "PHONE",
@@ -28,14 +36,21 @@ export declare enum EntityType {
     GPS_COORDINATES = "GPS_COORDINATES",
     UUID = "UUID",
     SOCIAL_HANDLE = "SOCIAL_HANDLE",
+    SECRET = "SECRET",
     OTHER = "OTHER"
 }
+/**
+ * Legacy type names accepted on input and mapped to their canonical form.
+ * Kept in sync with the pipeline canon (config/entity_types.json →
+ * legacy_aliases). The engine never *emits* these names.
+ */
+export declare const LEGACY_TYPE_ALIASES: Record<string, string>;
 export declare enum DetectionSource {
     RULES = "rules",
     CLOUD = "cloud"
 }
 export interface Detection {
-    entityType: EntityType;
+    entityType: EntityType | string;
     start: number;
     end: number;
     text: string;
@@ -50,7 +65,7 @@ export interface RedactResult {
     degraded: boolean;
 }
 export interface PatternDef {
-    entityType: EntityType;
+    entityType: EntityType | string;
     pattern: string;
     validator: string | null;
     description: string;

package/dist/esm/types.js

@@ -2,7 +2,15 @@ export var EntityType;
 (function (EntityType) {
     EntityType["NAME"] = "NAME";
     EntityType["ADDRESS"] = "ADDRESS";
-    EntityType["IBAN"] = "IBAN";
+    /**
+     * Canonical name for a bank account identifier. `IBAN` is kept below as a
+     * legacy alias with the same value, so `EntityType.IBAN === EntityType.BANK_ACCOUNT`
+     * and existing code keeps working — but the emitted value, and therefore the
+     * `[BANK_ACCOUNT]` placeholder written into redacted text, is canonical.
+     */
+    EntityType["BANK_ACCOUNT"] = "BANK_ACCOUNT";
+    /** @deprecated legacy alias of {@link EntityType.BANK_ACCOUNT} */
+    EntityType["IBAN"] = "BANK_ACCOUNT";
     EntityType["BIC"] = "BIC";
     EntityType["CREDIT_CARD"] = "CREDIT_CARD";
     EntityType["PHONE"] = "PHONE";
@@ -29,8 +37,17 @@ export var EntityType;
     EntityType["GPS_COORDINATES"] = "GPS_COORDINATES";
     EntityType["UUID"] = "UUID";
     EntityType["SOCIAL_HANDLE"] = "SOCIAL_HANDLE";
+    EntityType["SECRET"] = "SECRET";
     EntityType["OTHER"] = "OTHER";
 })(EntityType || (EntityType = {}));
+/**
+ * Legacy type names accepted on input and mapped to their canonical form.
+ * Kept in sync with the pipeline canon (config/entity_types.json →
+ * legacy_aliases). The engine never *emits* these names.
+ */
+export const LEGACY_TYPE_ALIASES = {
+    IBAN: "BANK_ACCOUNT",
+};
 export var DetectionSource;
 (function (DetectionSource) {
     DetectionSource["RULES"] = "rules";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "euredact",
-  "version": "0.1.0",
+  "version": "0.3.0",
   "description": "European PII redaction SDK -- fast, zero-dependency, rule-based detection and redaction of personal data across 31 European countries",
   "license": "Apache-2.0",
   "type": "module",
@@ -46,15 +46,19 @@
   ],
   "repository": {
     "type": "git",
-    "url": "https://github.com/jnjs/euredact"
+    "url": "git+https://github.com/euredact/euredact.git"
   },
   "scripts": {
     "build:esm": "tsc",
     "build:cjs": "tsc -p tsconfig.cjs.json",
     "postbuild:cjs": "echo '{\"type\":\"commonjs\"}' > dist/cjs/package.json",
     "build": "rm -rf dist && npm run build:esm && npm run build:cjs",
-    "test": "node --import tsx src/__tests__/run.ts",
-    "prepublishOnly": "npm run build"
+    "test": "npm run test:spec",
+    "benchmark": "node --import tsx src/__tests__/benchmark.ts",
+    "prepublishOnly": "npm run build",
+    "test:spec": "node --import tsx src/__tests__/spec.ts",
+    "test:eval": "node --import tsx src/__tests__/run.ts",
+    "test:eval:full": "node --max-old-space-size=8192 --import tsx src/__tests__/evalFull.ts"
   },
   "devDependencies": {
     "typescript": "^5.4.0",