euredact 0.3.5 → 0.3.7

package/dist/cjs/sdk.js

@@ -7,20 +7,44 @@ const engine_js_1 = require("./rules/engine.js");
 const evidence_js_1 = require("./rules/evidence.js");
 const types_js_1 = require("./types.js");
 const DATE_TYPES = new Set([types_js_1.EntityType.DOB, types_js_1.EntityType.DATE_OF_DEATH]);
+/** Entry count at which a one-time warning is emitted. */
+const MAPPING_WARN_THRESHOLD = 100_000;
+/**
+ * Maps real PII values to consistent labels within a session.
+ *
+ * The mapping is keyed on the **raw PII value** and is never evicted — evicting
+ * would hand a previously seen value a second label and quietly break
+ * referential integrity. Two consequences worth designing around:
+ *
+ * - It grows for as long as the process runs. Call `clear()` between workloads;
+ *   a warning is emitted once the mapping passes `MAPPING_WARN_THRESHOLD`.
+ * - Labels are shared by every caller of the same instance, including every
+ *   caller of the module-level `redact()`. A label repeated across two
+ *   documents reveals that they contain the same underlying value, so give each
+ *   tenant its own `EuRedact` instance rather than sharing the module-level one.
+ */
 class ReferentialMapper {
     counters = new Map();
     mapping = new Map();
+    warned = false;
     getLabel(text, entityType) {
         if (!this.mapping.has(text)) {
             const count = (this.counters.get(entityType) ?? 0) + 1;
             this.counters.set(entityType, count);
             this.mapping.set(text, `${entityType}_${count}`);
+            if (!this.warned && this.mapping.size > MAPPING_WARN_THRESHOLD) {
+                this.warned = true;
+                console.warn(`[euredact] Referential integrity mapping holds ${this.mapping.size} ` +
+                    `raw PII values and is never evicted. Call clear() between workloads ` +
+                    `to release them.`);
+            }
         }
         return this.mapping.get(text);
     }
     clear() {
         this.counters.clear();
         this.mapping.clear();
+        this.warned = false;
     }
 }
 const DEFAULT_MAX_INPUT_LENGTH = 10_485_760; // ~10 MB of text
@@ -110,14 +134,32 @@ class EuRedact {
             detections = detections.filter(d => !DATE_TYPES.has(d.entityType));
         }
         detections.sort((a, b) => a.start - b.start || b.end - a.end);
-        let redacted = text;
+        // Labels are resolved right-to-left because the referential mapper numbers
+        // each entity type in call order, and that order is part of the output
+        // contract. The string itself is then assembled in a single forward pass:
+        // rebuilding it per detection copied the whole document each time, which is
+        // O(document x detections) — 1.68 s of pure copying on a 1 MB document with
+        // 15,000 detections, versus 2 ms here.
+        const replacements = new Array(detections.length);
         for (let i = detections.length - 1; i >= 0; i--) {
             const det = detections[i];
-            const replacement = referentialIntegrity
+            replacements[i] = referentialIntegrity
                 ? this.referentialMapper.getLabel(det.text, det.entityType)
                 : `[${det.entityType}]`;
-            redacted = redacted.slice(0, det.start) + replacement + redacted.slice(det.end);
         }
+        const parts = [];
+        let pos = 0;
+        for (let i = 0; i < detections.length; i++) {
+            const det = detections[i];
+            // Spans reach here deduplicated and non-overlapping; one starting behind
+            // the cursor would splice into the middle of a label, so drop it.
+            if (det.start < pos)
+                continue;
+            parts.push(text.slice(pos, det.start), replacements[i]);
+            pos = det.end;
+        }
+        parts.push(text.slice(pos));
+        const redacted = parts.join("");
         // Report the inference so it can be audited. Spans in `evidence` are
         // offsets into the normalised text, matching `detections`.
         const inferredCountries = [...(0, evidence_js_1.weightsToRanking)(scores).entries()]

package/dist/esm/cache.d.ts

@@ -1,11 +1,15 @@
 import type { RedactResult } from "./types.js";
 export declare class ResultCache {
     private maxsize;
+    private maxChars;
     private enabled;
     private store;
-    constructor(maxsize?: number, enabled?: boolean);
+    private sizes;
+    private chars;
+    constructor(maxsize?: number, enabled?: boolean, maxChars?: number);
     key(text: string, countries: string[], mode: string): string;
     get(key: string): RedactResult | null;
     put(key: string, result: RedactResult): void;
+    private evict;
     clear(): void;
 }

package/dist/esm/cache.js

@@ -1,27 +1,41 @@
-// Simple hash function (FNV-1a) - no crypto dependency needed for cache keys
-function fnv1aHash(str) {
-    let hash = 0x811c9dc5;
-    for (let i = 0; i < str.length; i++) {
-        hash ^= str.charCodeAt(i);
-        hash = (hash * 0x01000193) >>> 0;
-    }
-    return hash.toString(16).padStart(8, "0");
+import { createHash } from "node:crypto";
+// Default ceiling on retained characters, not entries. A count-only cap of
+// 1024 entries says nothing about memory: 1024 cached 1 MB documents is a
+// gigabyte of PII-bearing text held live. ~16M characters is a few tens of MB.
+const DEFAULT_MAX_CHARS = 16_000_000;
+/** Approximate retained size of a cached result, in characters. */
+function resultChars(result) {
+    let chars = result.redactedText.length;
+    for (const d of result.detections)
+        chars += d.text.length;
+    return chars;
 }
 export class ResultCache {
     maxsize;
+    maxChars;
     enabled;
     store = new Map();
-    constructor(maxsize = 1024, enabled = true) {
+    sizes = new Map();
+    chars = 0;
+    constructor(maxsize = 1024, enabled = true, maxChars = DEFAULT_MAX_CHARS) {
         this.maxsize = maxsize;
+        this.maxChars = maxChars;
         this.enabled = enabled;
     }
     key(text, countries, mode) {
         const sorted = [...countries].sort();
-        const raw = `${text}|${sorted.join("|")}|${mode}`;
-        // Include raw length to reduce collision risk alongside dual FNV-1a hashes
-        const h1 = fnv1aHash(raw);
-        const h2 = fnv1aHash(raw.slice(Math.floor(raw.length / 2)) + raw.slice(0, Math.floor(raw.length / 2)));
-        return `${raw.length}:${h1}${h2}`;
+        // SHA-256, matching the Python implementation. The previous dual FNV-1a
+        // was non-cryptographic and collidable by construction: in a shared-process
+        // deployment an attacker who can guess a victim's document could submit a
+        // same-length colliding one and be served the victim's cached result,
+        // detections and raw matched PII included.
+        return createHash("sha256")
+            .update(text)
+            .update("|")
+            .update(sorted.join("|"))
+            .update("|")
+            .update(mode)
+            .digest("hex");
     }
     get(key) {
         if (!this.enabled)
@@ -38,15 +52,35 @@ export class ResultCache {
     put(key, result) {
         if (!this.enabled)
             return;
-        this.store.delete(key);
+        const size = resultChars(result);
+        // A single result bigger than the whole budget would evict everything else
+        // and still not fit; skip it rather than empty the cache for one document.
+        if (size > this.maxChars) {
+            this.evict(key);
+            return;
+        }
+        this.evict(key);
         this.store.set(key, result);
-        while (this.store.size > this.maxsize) {
+        this.sizes.set(key, size);
+        this.chars += size;
+        while (this.store.size > this.maxsize || this.chars > this.maxChars) {
             const first = this.store.keys().next().value;
-            if (first !== undefined)
-                this.store.delete(first);
+            if (first === undefined)
+                break;
+            this.evict(first);
         }
     }
+    evict(key) {
+        const size = this.sizes.get(key);
+        if (size !== undefined) {
+            this.chars -= size;
+            this.sizes.delete(key);
+        }
+        this.store.delete(key);
+    }
     clear() {
         this.store.clear();
+        this.sizes.clear();
+        this.chars = 0;
     }
 }

package/dist/esm/rules/context.d.ts

@@ -6,7 +6,7 @@
  * contain no IBAN, no `+CC` number and no ccTLD infers nothing at all, even
  * when the first page identified the document beyond doubt:
  *
- *     page 1   "Factuur — IBAN NL91ABNA0417164300, info@jansen.nl"
+ *     page 1   "Factuur — IBAN NL91ABNA0417164300, info@example.nl"
  *     page 7   "Telefoon 0612345678"        <- Dutch mobile, or Danish CPR?
  *
  * Scanned alone, page 7 has nothing to go on. Sharing a context across the two

package/dist/esm/rules/context.js

@@ -6,7 +6,7 @@
  * contain no IBAN, no `+CC` number and no ccTLD infers nothing at all, even
  * when the first page identified the document beyond doubt:
  *
- *     page 1   "Factuur — IBAN NL91ABNA0417164300, info@jansen.nl"
+ *     page 1   "Factuur — IBAN NL91ABNA0417164300, info@example.nl"
  *     page 7   "Telefoon 0612345678"        <- Dutch mobile, or Danish CPR?
  *
  * Scanned alone, page 7 has nothing to go on. Sharing a context across the two

package/dist/esm/rules/countries/index.js

@@ -157,7 +157,20 @@ const SHARED = {
         // --- Secret / API Key (assignment-based: KEY=value or KEY: value) ---
         p(EntityType.SECRET, String.raw `(?<=[:=] )\S{8,}|(?<=[:=])\S{8,}`, "high_entropy", "Assigned secret value", SECRET_CONTEXT, true),
         // --- Secret / API Key (entropy-based fallback for longer tokens) ---
-        p(EntityType.SECRET, String.raw `\b[A-Za-z0-9_\-+/]{24,}[A-Za-z0-9_\-+/=]*\b`, "high_entropy", "High-entropy token", SECRET_CONTEXT, true),
+        // Only the *trailing* \b is replaced. It could not match when the token run
+        // ended on "-", "+" or "/", so the engine backtracked across two
+        // overlapping quantified classes: an ordinary dashed rule line ("x-----...")
+        // cost O(n^2), 8.3 s for 80 KB. The lookahead simply asserts the run is
+        // over, which is what \b was there to express, and it cannot fail after a
+        // maximal run — so there is nothing to backtrack into.
+        //
+        // The leading \b stays. A class lookbehind there widened the span leftwards
+        // over any "/" in front, so an authorized_keys entry started "//...",
+        // matched the protocol-relative URL shape in the endpoint suppressor, and
+        // was dropped — leaving SSH key material in the clear. Base64 padding is
+        // now inside the span instead of trailing outside it; that is the only
+        // intended difference.
+        p(EntityType.SECRET, String.raw `\b[A-Za-z0-9_\-+/]{24,}[A-Za-z0-9_\-+/=]*(?![A-Za-z0-9_\-+/=])`, "high_entropy", "High-entropy token", SECRET_CONTEXT, true),
         p(EntityType.DOB, String.raw `\b(?:0[1-9]|[12][0-9]|3[01])[/.\-](?:0[1-9]|1[0-2])[/.\-](?:19|20)\d{2}\b`, null, "DD/MM/YYYY", DOB_CONTEXT, true),
         p(EntityType.DATE_OF_DEATH, String.raw `\b(?:0[1-9]|[12][0-9]|3[01])[/.\-](?:0[1-9]|1[0-2])[/.\-](?:19|20)\d{2}\b`, null, "", DATE_OF_DEATH_CONTEXT, true),
         p(EntityType.DOB, String.raw `\b(?:19|20)\d{2}[/.\-](?:0[1-9]|1[0-2])[/.\-](?:0[1-9]|[12][0-9]|3[01])\b`, null, "ISO YYYY-MM-DD", DOB_ISO_CONTEXT, true),

package/dist/esm/rules/engine.js

@@ -1,6 +1,6 @@
 import { DetectionSource, EntityType } from "../types.js";
 import { MultiPatternMatcher } from "./matchers.js";
-import { shouldSuppress } from "./suppressors.js";
+import { shouldSuppress, SuppressionScratch } from "./suppressors.js";
 import { detectStructuralDob } from "./structural.js";
 import { COUNTRY_CONFIGS } from "./countries/index.js";
 import * as evidenceMod from "./evidence.js";
@@ -273,6 +273,14 @@ export class RuleEngine {
         const allFailed = failedSpans
             .map(f => [f.start, f.end])
             .sort((a, b) => a[0] - b[0] || a[1] - b[1]);
+        // Running maximum of the end offsets, so the fragment test below can ask
+        // "does any span starting at or before me reach past my end?" in O(1)
+        // instead of walking the prefix per candidate.
+        const failedStarts = allFailed.map(f => f[0]);
+        const failedPrefixMax = new Array(allFailed.length + 1).fill(0);
+        for (let i = 0; i < allFailed.length; i++) {
+            failedPrefixMax[i + 1] = Math.max(failedPrefixMax[i], allFailed[i][1]);
+        }
         const mergedZones = new Map();
         for (const [etype, spans] of zonesByType) {
             spans.sort((a, b) => a[0] - b[0] || a[1] - b[1]);
@@ -295,8 +303,11 @@ export class RuleEngine {
             mergedZones.set(etype, { starts, ends });
         }
         const candidates = [];
+        // Whole-document facts a suppressor may need, derived at most once per call
+        // and discarded with it.
+        const scratch = new SuppressionScratch(text);
         for (const { match, hasValidValidator } of validated) {
-            if (shouldSuppress(text, match))
+            if (shouldSuppress(text, match, scratch))
                 continue;
             // A validator-less match inside a failed-validation span of its own type
             // is demoted below every other candidate rather than deleted. Demotion
@@ -324,13 +335,31 @@ export class RuleEngine {
             // default — the generic phone pattern kept claiming the tail of the
             // rejected national number.
             if (match.patternDef.validator === null && !demoted) {
+                // Upper bound of the spans starting at or before match.start.
+                let lo = 0, hi = failedStarts.length;
+                while (lo < hi) {
+                    const mid = (lo + hi) >>> 1;
+                    if (failedStarts[mid] <= match.start)
+                        lo = mid + 1;
+                    else
+                        hi = mid;
+                }
                 let fragment = false;
-                for (const [zs, ze] of allFailed) {
-                    if (zs > match.start)
-                        break;
-                    if (match.end <= ze && (match.start > zs || match.end < ze)) {
-                        fragment = true;
-                        break;
+                const reach = failedPrefixMax[lo];
+                if (reach > match.end) {
+                    // Something starting no later than us ends strictly later: we are a
+                    // proper subset of it.
+                    fragment = true;
+                }
+                else if (reach === match.end) {
+                    // Same end offset — a proper subset only if some span with that end
+                    // starts strictly earlier. Scanning backwards finds the enclosing
+                    // span first, and the equal case is rare.
+                    for (let i = lo - 1; i >= 0; i--) {
+                        if (allFailed[i][1] === match.end && allFailed[i][0] < match.start) {
+                            fragment = true;
+                            break;
+                        }
                     }
                 }
                 if (fragment)
@@ -381,6 +410,7 @@ export class RuleEngine {
                 length: match.end - match.start,
                 countryScore: countryScores.get(match.countryCode) ?? 0,
                 inScope: declared === null || declared.has(match.countryCode) ? 1 : 0,
+                tier: 0,
                 det: {
                     entityType: match.patternDef.entityType,
                     start: match.start,
@@ -395,7 +425,7 @@ export class RuleEngine {
         }
         for (const d of detectStructuralDob(text)) {
             candidates.push({ blindPriority: 1, cue: 0, priority: 1, length: d.end - d.start,
-                countryScore: 0, inScope: 1, det: d });
+                countryScore: 0, inScope: 1, tier: 0, det: d });
         }
         const detections = this.deduplicate(candidates);
         detections.sort((a, b) => a.start - b.start || b.end - a.end);
@@ -429,7 +459,10 @@ export class RuleEngine {
             if (seen === undefined || c.blindPriority > seen)
                 spanTier.set(key, c.blindPriority);
         }
-        const tierOf = (c) => spanTier.get(`${c.det.start}:${c.det.end}`);
+        // Resolved once per candidate rather than inside the comparator, which ran
+        // it O(n log n) times and rebuilt the key string on every comparison.
+        for (const c of candidates)
+            c.tier = spanTier.get(`${c.det.start}:${c.det.end}`);
         // Span length outranks *everything*, including validation.
         //
         // A shorter candidate winning re-cuts a longer one and leaves the
@@ -455,7 +488,7 @@ export class RuleEngine {
         // country-blind up to and including `start`, and (length, start) already
         // identifies a span uniquely.
         const sorted = [...candidates].sort((a, b) => b.length - a.length || // longest first     } country-blind
-            tierOf(b) - tierOf(a) || // best tier here    } — decides
+            b.tier - a.tier || // best tier here    } — decides
             a.det.start - b.det.start || // leftmost          } WHICH span
             b.cue - a.cue || // local cue         } country-aware
             b.priority - a.priority || // country priority  } — decides the

package/dist/esm/rules/suppressors.d.ts

@@ -6,4 +6,17 @@ export interface RawMatch {
     patternDef: PatternDef;
     countryCode: string;
 }
-export declare function shouldSuppress(text: string, match: RawMatch): boolean;
+/**
+ * Per-document scratch space for suppressors that would otherwise re-derive the
+ * same whole-document facts for every candidate.
+ *
+ * Built lazily and held only for the duration of one `detect()` call, so no
+ * document text outlives the call that supplied it.
+ */
+export declare class SuppressionScratch {
+    private readonly text;
+    private words;
+    constructor(text: string);
+    documentWords(): Set<string>;
+}
+export declare function shouldSuppress(text: string, match: RawMatch, scratch?: SuppressionScratch): boolean;