euredact 0.3.6 → 0.3.8

package/README.md

@@ -167,7 +167,7 @@ interface Detection {
   text: string;                    // The matched substring
   source: DetectionSource;         // "rules" or "cloud"
   country: string | null;          // ISO code or null for shared/custom patterns
-  confidence: string;              // Confidence level
+  confidence: string;              // "high" | "medium" | "low" — see below
   countryConfidence?: number;      // How strongly the document supports `country`,
                                    // in [0, 1]. 0 means the attribution rests on
                                    // a checksum alone.
@@ -176,6 +176,26 @@ interface Detection {
 }
 ```
 
+**`confidence`** describes how the *type* was arrived at. It never says anything
+about whether the span is masked — every detection is, at every level.
+
+| value | meaning |
+|---|---|
+| `"high"` | a pattern matched and, where one exists, its checksum passed |
+| `"medium"` | the type comes from a label touching the span, because no pattern of that type claimed it — `Αρ. Ταυτότητας: 00892341` is a `NATIONAL_ID` although nothing can checksum it |
+| `"low"` | a pattern matched, its checksum *failed*, and the document labels the span as that very type — `Rijksregisternummer: 85.03.19-284.73` is a national number with a bad check digit |
+
+Filter on it when you need only checksum-backed detections:
+
+```ts
+const strict = result.detections.filter(d => d.confidence === "high");
+```
+
+A `"low"` detection is the honest description of a mistyped, OCR'd or invented
+identifier: the shape and the label agree, the check digit does not. Earlier
+versions dropped these, which meant a redaction library printed in full an
+identifier it had recognised and rejected.
+
 #### `RedactResult`
 
 ```ts
@@ -205,9 +225,16 @@ PASSPORT          DRIVERS_LICENSE   RESIDENCE_PERMIT  LICENSE_PLATE
 VIN               VAT               POSTAL_CODE       IP_ADDRESS
 IPV6_ADDRESS      MAC_ADDRESS       HEALTH_INSURANCE  HEALTHCARE_PROVIDER
 CHAMBER_OF_COMMERCE  IMEI          GPS_COORDINATES   UUID
-SOCIAL_HANDLE     SECRET            OTHER
+SOCIAL_HANDLE     SECRET            INTERNAL_ID       OTHER
 ```
 
+`INTERNAL_ID` — an employee, badge or customer number tied to a person — is
+emitted **only** when an explicit label names it (`medarbejdernummer:`,
+`Personalnummer:`, `Employee No:`). There is no pattern for one, because there
+is no shape for one: without the label, a digit run is not distinguishable from
+any other. The type exists so that a labelled employee number is filed
+correctly instead of being claimed by the phone pattern.
+
 For custom patterns, `entityType` is a plain string (e.g. `"EMPLOYEE_ID"`).
 
 ## Country codes
@@ -313,7 +340,7 @@ for (const page of pages) {
   offset += page.length;
 }
 
-// page 1: "Factuur — IBAN NL91ABNA0417164300, info@jansen.nl"
+// page 1: "Factuur — IBAN NL91ABNA0417164300, info@example.nl"
 // page 7: "Telefoon 0612345678"  -> PHONE (NL), not NATIONAL_ID (DK)
 ```
 

package/dist/cjs/cache.js

@@ -1,30 +1,44 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.ResultCache = void 0;
-// Simple hash function (FNV-1a) - no crypto dependency needed for cache keys
-function fnv1aHash(str) {
-    let hash = 0x811c9dc5;
-    for (let i = 0; i < str.length; i++) {
-        hash ^= str.charCodeAt(i);
-        hash = (hash * 0x01000193) >>> 0;
-    }
-    return hash.toString(16).padStart(8, "0");
+const node_crypto_1 = require("node:crypto");
+// Default ceiling on retained characters, not entries. A count-only cap of
+// 1024 entries says nothing about memory: 1024 cached 1 MB documents is a
+// gigabyte of PII-bearing text held live. ~16M characters is a few tens of MB.
+const DEFAULT_MAX_CHARS = 16_000_000;
+/** Approximate retained size of a cached result, in characters. */
+function resultChars(result) {
+    let chars = result.redactedText.length;
+    for (const d of result.detections)
+        chars += d.text.length;
+    return chars;
 }
 class ResultCache {
     maxsize;
+    maxChars;
     enabled;
     store = new Map();
-    constructor(maxsize = 1024, enabled = true) {
+    sizes = new Map();
+    chars = 0;
+    constructor(maxsize = 1024, enabled = true, maxChars = DEFAULT_MAX_CHARS) {
         this.maxsize = maxsize;
+        this.maxChars = maxChars;
         this.enabled = enabled;
     }
     key(text, countries, mode) {
         const sorted = [...countries].sort();
-        const raw = `${text}|${sorted.join("|")}|${mode}`;
-        // Include raw length to reduce collision risk alongside dual FNV-1a hashes
-        const h1 = fnv1aHash(raw);
-        const h2 = fnv1aHash(raw.slice(Math.floor(raw.length / 2)) + raw.slice(0, Math.floor(raw.length / 2)));
-        return `${raw.length}:${h1}${h2}`;
+        // SHA-256, matching the Python implementation. The previous dual FNV-1a
+        // was non-cryptographic and collidable by construction: in a shared-process
+        // deployment an attacker who can guess a victim's document could submit a
+        // same-length colliding one and be served the victim's cached result,
+        // detections and raw matched PII included.
+        return (0, node_crypto_1.createHash)("sha256")
+            .update(text)
+            .update("|")
+            .update(sorted.join("|"))
+            .update("|")
+            .update(mode)
+            .digest("hex");
     }
     get(key) {
         if (!this.enabled)
@@ -41,16 +55,36 @@ class ResultCache {
     put(key, result) {
         if (!this.enabled)
             return;
-        this.store.delete(key);
+        const size = resultChars(result);
+        // A single result bigger than the whole budget would evict everything else
+        // and still not fit; skip it rather than empty the cache for one document.
+        if (size > this.maxChars) {
+            this.evict(key);
+            return;
+        }
+        this.evict(key);
         this.store.set(key, result);
-        while (this.store.size > this.maxsize) {
+        this.sizes.set(key, size);
+        this.chars += size;
+        while (this.store.size > this.maxsize || this.chars > this.maxChars) {
             const first = this.store.keys().next().value;
-            if (first !== undefined)
-                this.store.delete(first);
+            if (first === undefined)
+                break;
+            this.evict(first);
         }
     }
+    evict(key) {
+        const size = this.sizes.get(key);
+        if (size !== undefined) {
+            this.chars -= size;
+            this.sizes.delete(key);
+        }
+        this.store.delete(key);
+    }
     clear() {
         this.store.clear();
+        this.sizes.clear();
+        this.chars = 0;
     }
 }
 exports.ResultCache = ResultCache;

package/dist/cjs/rules/context.js

@@ -7,7 +7,7 @@
  * contain no IBAN, no `+CC` number and no ccTLD infers nothing at all, even
  * when the first page identified the document beyond doubt:
  *
- *     page 1   "Factuur — IBAN NL91ABNA0417164300, info@jansen.nl"
+ *     page 1   "Factuur — IBAN NL91ABNA0417164300, info@example.nl"
  *     page 7   "Telefoon 0612345678"        <- Dutch mobile, or Danish CPR?
  *
  * Scanned alone, page 7 has nothing to go on. Sharing a context across the two

package/dist/cjs/rules/countries/index.js

@@ -160,7 +160,20 @@ const SHARED = {
         // --- Secret / API Key (assignment-based: KEY=value or KEY: value) ---
         p(types_js_1.EntityType.SECRET, String.raw `(?<=[:=] )\S{8,}|(?<=[:=])\S{8,}`, "high_entropy", "Assigned secret value", SECRET_CONTEXT, true),
         // --- Secret / API Key (entropy-based fallback for longer tokens) ---
-        p(types_js_1.EntityType.SECRET, String.raw `\b[A-Za-z0-9_\-+/]{24,}[A-Za-z0-9_\-+/=]*\b`, "high_entropy", "High-entropy token", SECRET_CONTEXT, true),
+        // Only the *trailing* \b is replaced. It could not match when the token run
+        // ended on "-", "+" or "/", so the engine backtracked across two
+        // overlapping quantified classes: an ordinary dashed rule line ("x-----...")
+        // cost O(n^2), 8.3 s for 80 KB. The lookahead simply asserts the run is
+        // over, which is what \b was there to express, and it cannot fail after a
+        // maximal run — so there is nothing to backtrack into.
+        //
+        // The leading \b stays. A class lookbehind there widened the span leftwards
+        // over any "/" in front, so an authorized_keys entry started "//...",
+        // matched the protocol-relative URL shape in the endpoint suppressor, and
+        // was dropped — leaving SSH key material in the clear. Base64 padding is
+        // now inside the span instead of trailing outside it; that is the only
+        // intended difference.
+        p(types_js_1.EntityType.SECRET, String.raw `\b[A-Za-z0-9_\-+/]{24,}[A-Za-z0-9_\-+/=]*(?![A-Za-z0-9_\-+/=])`, "high_entropy", "High-entropy token", SECRET_CONTEXT, true),
         p(types_js_1.EntityType.DOB, String.raw `\b(?:0[1-9]|[12][0-9]|3[01])[/.\-](?:0[1-9]|1[0-2])[/.\-](?:19|20)\d{2}\b`, null, "DD/MM/YYYY", DOB_CONTEXT, true),
         p(types_js_1.EntityType.DATE_OF_DEATH, String.raw `\b(?:0[1-9]|[12][0-9]|3[01])[/.\-](?:0[1-9]|1[0-2])[/.\-](?:19|20)\d{2}\b`, null, "", DATE_OF_DEATH_CONTEXT, true),
         p(types_js_1.EntityType.DOB, String.raw `\b(?:19|20)\d{2}[/.\-](?:0[1-9]|1[0-2])[/.\-](?:0[1-9]|[12][0-9]|3[01])\b`, null, "ISO YYYY-MM-DD", DOB_ISO_CONTEXT, true),

package/dist/cjs/rules/cues.js

@@ -0,0 +1,168 @@
+"use strict";
+/**
+ * Local cues: what the document calls the value sitting next to it.
+ *
+ * A label touching the left edge of a span is the strongest statement a
+ * document makes about what that value *is*. "Phone: 0705237535" is a phone
+ * number even though those digits pass the Swedish personnummer checksum, and
+ * "Αρ. Ταυτότητας: 00892341" is an identity-card number even though nothing in
+ * the engine can checksum it.
+ *
+ * This table is the single source of that knowledge. Three callers read it, and
+ * none of them may move a span:
+ *
+ * 1. `localCueBonus` — ranks a candidate above its rivals on the *same span*
+ *    when the cue names its type. The oldest use, and why the table exists.
+ * 2. `detectWithEvidence` — re-admits a candidate whose checksum failed when the
+ *    cue names that same type. A declined identifier with its own label in front
+ *    of it is still that identifier, mistyped or truncated.
+ * 3. `retypedBy` — relabels a winning generic candidate whose type the cue
+ *    contradicts, when nothing of the cued type claimed the span.
+ *
+ * Consolidated here from two places that had drifted apart: `LOCAL_CUES` in
+ * `engine.ts` (typed, but missing most of Europe) and `ID_LABEL_BEFORE` in
+ * `suppressors.ts` (broader, but untyped — it could only *delete* a phone
+ * candidate, which left the value unmasked).
+ *
+ * Kept character-for-character in step with `euredact/rules/cues.py`. Two
+ * constructs are written the way they are so that both engines agree:
+ *
+ * * Boundaries are `(?<![A-Za-z0-9_])`, not `\b`. JavaScript's `\b` is
+ *   ASCII-only, so `\bΑΦΜ` never matches — a Greek letter is not a word
+ *   character there, and a space before it is not a transition. A boundary of
+ *   some kind is required regardless: without one the short alternatives match
+ *   the *tail* of an ordinary word. "NIS" matched "tālru**nis**", the Latvian
+ *   for telephone, and suppressed 653 real phone numbers; "iva" matched
+ *   "Pr**iva**t:" and read a personal e-mail address as a VAT number.
+ * * The qualifier class is spelled out rather than written `[^\W\d_]`, which
+ *   would reject "εταιρειας" here for the same ASCII reason.
+ *
+ * Every label ends in `(?:\w*|\s+<one word>)`: the label either runs on into a
+ * longer word ("telefoon" → "telefoonnummer") **or** takes one qualifier word
+ * naming whose identifier it is ("ΑΦΜ εταιρειας:", "Tel Nr:") — never both.
+ * Allowing both let "nie" grow into the surname "Nieminen" and then swallow the
+ * real label in "Matti Nieminen\nDOB: 08.05.1994", reading a date of birth as a
+ * Spanish NIE: 52 false positives across the corpus.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.CUE_TARGETS = exports.CUES = exports.CUE_WINDOW = void 0;
+exports.cuedType = cuedType;
+exports.retypedBy = retypedBy;
+const types_js_1 = require("../types.js");
+/**
+ * How far back a cue may sit. Short on purpose: a cue is only evidence about
+ * the value it introduces, which is the lesson of the postal-code year defect,
+ * where a cue 150 characters away licensed every number in the window.
+ */
+exports.CUE_WINDOW = 22;
+/**
+ * (entity type, label pattern). Ordered: the first match wins, so the narrower
+ * label comes first where two could read the same text.
+ */
+exports.CUES = [
+    // Phone. Ranking only — nothing is ever relabelled *to* PHONE: a phone cue in
+    // front of a span that no phone pattern matched says the document is laid out
+    // unusually, not that the value is a phone number.
+    [types_js_1.EntityType.PHONE,
+        /(?<![A-Za-z0-9_])(?:phone|tel|telephone|téléphone|telefoon|telefon|telefono|teléfono|tlf|mobil|mobile|gsm|handy|sími|puh|móvil|cell|tālrunis|τηλέφωνο|τηλ)(?:\w*|\s+[^\s:.\-\d,;()/]{2,20})\s*[:.\-]?\s*$/i],
+    // National identity number.
+    [types_js_1.EntityType.NATIONAL_ID,
+        /(?<![A-Za-z0-9_])(?:bsn|personnummer|rijksregisternummer|nationaal\s*nummer|numéro\s*national|national\s*(?:id|number)|identiteitsnummer|henkilötunnus|kennitala|cpr(?:-?nummer)?|nif|nie|dni|rr|nn|nir|insz|niss|nis|matricule|ausweisnummer|personalausweis|osobní\s*číslo|ЕГН|EGN)(?:\w*|\s+[^\s:.\-\d,;()/]{2,20})\s*[:.\-]?\s*$/i],
+    // Cyprus: "ID number" in both official languages, plus the Greek abbreviation
+    // ΑΔΤ (Αστυνομική Ταυτότητα). Neither has a phone reading.
+    [types_js_1.EntityType.NATIONAL_ID,
+        /(?<![A-Za-z0-9_])(?:αρ\.?\s*ταυτότητας|ταυτότητας|αδτ|kimlik\s*numaras[ıi])(?:\w*|\s+[^\s:.\-\d,;()/]{2,20})\s*[:.\-]?\s*$/i],
+    // Social security.
+    [types_js_1.EntityType.SSN,
+        /(?<![A-Za-z0-9_])(?:ssn|social\s*security|sozialversicherungsnummer|svnr|sofi)(?:\w*|\s+[^\s:.\-\d,;()/]{2,20})\s*[:.\-]?\s*$/i],
+    // Tax number. Greece's ΑΦΜ is issued by the tax authority; the identity-card
+    // number is the ΑΔΤ, above. The engine used to return NATIONAL_ID for a value
+    // cued ΑΦΜ, which is wrong independently of the phone problem.
+    [types_js_1.EntityType.TAX_ID,
+        /(?<![A-Za-z0-9_])(?:αφμ|α\.φ\.μ\.|steuer-?id|steuernummer|st\.?-?nr|stnr|tin|finanzamt\s+ist|tax\s*(?:id|no|number)|daňové\s*číslo|numer\s*podatkowy)(?:\w*|\s+[^\s:.\-\d,;()/]{2,20})\s*[:.\-]?\s*$/i],
+    // Health insurance.
+    [types_js_1.EntityType.HEALTH_INSURANCE,
+        /(?<![A-Za-z0-9_])(?:versichertennummer|krankenversicherung|kvnr|kv-?nr|mutualité|ziekenfonds|aansluitingsnummer)(?:\w*|\s+[^\s:.\-\d,;()/]{2,20})\s*[:.\-]?\s*$/i],
+    // Company / trade register.
+    [types_js_1.EntityType.CHAMBER_OF_COMMERCE,
+        /(?<![A-Za-z0-9_])(?:siren|siret|kbo|bce|kvk|ondernemingsnummer|ondernemingen\s+onder\s+nummer|kruispuntbank|numéro\s*d'entreprise|enterprise\s*number|handelsregister|company\s*(?:no|number|reg)|organisationsnummer|orgnr|i[čc]o|identifikační\s*číslo|ЕИК|eik|bulstat|cvr(?:-?nummer)?|virksomhedsnummer)(?:\w*|\s+[^\s:.\-\d,;()/]{2,20})\s*[:.\-]?\s*$/i],
+    // VAT.
+    [types_js_1.EntityType.VAT,
+        /(?<![A-Za-z0-9_])(?:vat|btw|tva|ust|iva|moms|alv|mwst|dič|vsk)(?:[\w.\-]*|\s+[^\s:.\-\d,;()/]{2,20})\s*[:.\-]?\s*$/i],
+    // BIC / SWIFT.
+    [types_js_1.EntityType.BIC,
+        /(?<![A-Za-z0-9_])(?:bic|swift)[\w/]*\s*[:.\-]?\s*\(?\s*$/i],
+    // Postal code. A postal code loses every contested span on priority (it is
+    // the weakest evidence the engine has), so without a cue entry
+    // "Cod postal: 040171" was handed to a German phone pattern.
+    [types_js_1.EntityType.POSTAL_CODE,
+        /(?<![A-Za-z0-9_])(?:cod\s*po[sș]tal|postnummer|postnr|postleitzahl|plz|postcode|code\s*postal|codice\s*postale|kod\s*pocztowy|ps[čc]|τ\.?κ\.?|пощенски\s*код)(?:\w*|\s+[^\s:.\-\d,;()/]{2,20})\s*[:.\-]?\s*$/i],
+    // Internal / employee identifier. The canon calls these INTERNAL_ID and
+    // annotates them with an LLM, so the engine has no pattern for them — but it
+    // does have a phone pattern that was happily claiming them. The cue is the
+    // only thing that identifies one, which is why the engine emits this type
+    // from here and nowhere else.
+    [types_js_1.EntityType.INTERNAL_ID,
+        /(?<![A-Za-z0-9_])(?:medarbejdernummer|ansattnummer|anställningsnummer|personalnummer|personeelsnummer|matricule\s*salarié|employee\s*(?:no|number|id)|badge\s*(?:no|number)|osobní\s*(?:číslo\s*)?zaměstnance)(?:\w*|\s+[^\s:.\-\d,;()/]{2,20})\s*[:.\-]?\s*$/i],
+];
+/**
+ * The entity type a label immediately before `start` names, or null.
+ *
+ * Reads at most `CUE_WINDOW` characters, and every pattern is anchored to the
+ * end of that window, so the cue must *touch* the span. A label further away is
+ * about some other value.
+ */
+function cuedType(text, start) {
+    const before = text.slice(Math.max(0, start - exports.CUE_WINDOW), start);
+    for (const [entityType, pattern] of exports.CUES) {
+        if (pattern.test(before))
+            return entityType;
+    }
+    return null;
+}
+/**
+ * Shapes so generic that a label in front is better evidence than the pattern
+ * that matched. A bare digit run is not a phone number because it *could* be
+ * dialled.
+ */
+const RETYPABLE = new Set([
+    types_js_1.EntityType.PHONE, types_js_1.EntityType.POSTAL_CODE, types_js_1.EntityType.LICENSE_PLATE,
+]);
+/**
+ * Types a label can assert on its own. Also the types eligible for the rescue
+ * in `engine.ts`, for the same reason read the other way — a label may vouch
+ * for one of these, so it may also vouch for a malformed one.
+ *
+ * PHONE is absent on purpose: "Tel:" in front of something no phone pattern
+ * matched means the document is laid out oddly, not that the value is a phone
+ * number. So are EMAIL, SECRET, DOB, VIN and the rest — those carry their own
+ * structure, and a nearby word is not entitled to overrule it.
+ *
+ * BIC's absence is what keeps the rescue honest. Its validator is a *registry
+ * lookup*, not a checksum, so a failure means "no such bank", which no label can
+ * talk you out of. Rescuing it read 34 unlisted codes behind a "BIC:" label as
+ * real ones. The same reasoning keeps CREDIT_CARD and BANK_ACCOUNT out: Luhn and
+ * mod-97 are strong enough that a failure really does mean "not one of these".
+ */
+exports.CUE_TARGETS = new Set([
+    types_js_1.EntityType.NATIONAL_ID, types_js_1.EntityType.SSN, types_js_1.EntityType.TAX_ID,
+    types_js_1.EntityType.HEALTH_INSURANCE, types_js_1.EntityType.CHAMBER_OF_COMMERCE,
+    types_js_1.EntityType.VAT, types_js_1.EntityType.POSTAL_CODE, types_js_1.EntityType.PASSPORT,
+    types_js_1.EntityType.INTERNAL_ID,
+]);
+/**
+ * The type a cue overrules `entityType` with, or null to leave it alone.
+ *
+ * Both gates are load-bearing. Measured over 2,000 corpus documents, re-typing
+ * without them fired 52 times and got roughly 30 of those wrong — EMAIL read as
+ * VAT, a date of birth as a phone number, a VIN as VAT — against 20 real fixes.
+ * With them: 14 re-typings, every one of them PHONE to a structured identifier.
+ */
+function retypedBy(text, start, entityType) {
+    if (!RETYPABLE.has(String(entityType)))
+        return null;
+    const cued = cuedType(text, start);
+    if (cued === null || cued === entityType || !exports.CUE_TARGETS.has(String(cued)))
+        return null;
+    return cued;
+}

package/dist/cjs/rules/engine.js

@@ -38,6 +38,7 @@ exports.resolveCountryCode = resolveCountryCode;
 const types_js_1 = require("../types.js");
 const matchers_js_1 = require("./matchers.js");
 const suppressors_js_1 = require("./suppressors.js");
+const cues_js_1 = require("./cues.js");
 const structural_js_1 = require("./structural.js");
 const index_js_1 = require("./countries/index.js");
 const evidenceMod = __importStar(require("./evidence.js"));
@@ -105,30 +106,13 @@ function warnUnknownCountry(code) {
 //
 // Measured: 263 phone numbers and 18 French SIREN were reported as national IDs,
 // each with an explicit cue in front of it.
-const LOCAL_CUES = [
-    [types_js_1.EntityType.PHONE,
-        /(?:phone|tel|telephone|téléphone|telefoon|telefon|telefono|teléfono|tlf|mobil|mobile|gsm|handy|sími|puh|móvil|cell)\w*\s*[:.\-]?\s*$/i],
-    [types_js_1.EntityType.CHAMBER_OF_COMMERCE,
-        /(?:siren|siret|kbo|bce|kvk|ondernemingsnummer|handelsregister|company\s*(?:no|number|reg)|organisationsnummer|orgnr)\w*\s*[:.\-]?\s*$/i],
-    [types_js_1.EntityType.VAT,
-        /(?:vat|btw|tva|ust|iva|moms|alv|mwst)[\w.\-]*\s*[:.\-]?\s*$/i],
-    [types_js_1.EntityType.BIC,
-        /(?:bic|swift)[\w/]*\s*[:.\-]?\s*\(?\s*$/i],
-    [types_js_1.EntityType.NATIONAL_ID,
-        /(?:bsn|personnummer|rijksregisternummer|national\s*id|identiteitsnummer|henkilötunnus|kennitala|cpr|nif|dni)\w*\s*[:.\-]?\s*$/i],
-];
-// How far back a cue may sit. Short on purpose: a cue is only evidence about the
-// value it introduces, which is the lesson of the postal-code year defect, where
-// a cue 150 characters away licensed every number in the window.
-const CUE_WINDOW = 22;
+//
+// The table itself lives in `cues.ts`, because two other steps read it: the
+// rescue of a declined identifier below, and the re-typing in deduplicate.
+// `CUE_WINDOW` is re-exported from there.
 /** 1 when a cue naming `entityType` sits immediately before the span. */
 function localCueBonus(text, start, entityType) {
-    const before = text.slice(Math.max(0, start - CUE_WINDOW), start);
-    for (const [cuedType, pattern] of LOCAL_CUES) {
-        if (cuedType === entityType && pattern.test(before))
-            return 1;
-    }
-    return 0;
+    return (0, cues_js_1.cuedType)(text, start) === entityType ? 1 : 0;
 }
 class RuleEngine {
     matcher = new matchers_js_1.MultiPatternMatcher();
@@ -235,7 +219,19 @@ class RuleEngine {
         // but failed the checksum is recorded with its country and entity type;
         // what it is allowed to demote is decided below, once the document's
         // countries are known.
+        //
+        // One kind of failure is rescued rather than recorded: a span the document
+        // itself labels as the very type whose checksum just failed.
+        // "Rijksregisternummer: 85.03.19-284.73" is a Belgian national number with a
+        // bad check digit — mistyped, OCR'd, or invented — and it is still a
+        // national number. Before this, the pattern declined, the generic phone rule
+        // was denied the span as a fragment, and the value was left in the clear: a
+        // redaction library printing an identifier it had recognised and rejected.
+        //
+        // Rescued candidates are still recorded as failed spans, so the demotion
+        // zones and the fragment rule below are unchanged.
         const validated = [];
+        const rescued = [];
         const failedSpans = [];
         for (const m of rawMatches) {
             if (this.matcher.validate(m)) {
@@ -244,6 +240,9 @@ class RuleEngine {
             else if (m.patternDef.validator !== null && !m.patternDef.requiresContext) {
                 failedSpans.push({ start: m.start, end: m.end, code: m.countryCode,
                     etype: String(m.patternDef.entityType) });
+                if (cues_js_1.CUE_TARGETS.has(String(m.patternDef.entityType))
+                    && (0, cues_js_1.cuedType)(text, m.start) === m.patternDef.entityType)
+                    rescued.push(m);
             }
         }
         // Evidence pass: work out which countries this document belongs to, from
@@ -310,6 +309,14 @@ class RuleEngine {
         const allFailed = failedSpans
             .map(f => [f.start, f.end])
             .sort((a, b) => a[0] - b[0] || a[1] - b[1]);
+        // Running maximum of the end offsets, so the fragment test below can ask
+        // "does any span starting at or before me reach past my end?" in O(1)
+        // instead of walking the prefix per candidate.
+        const failedStarts = allFailed.map(f => f[0]);
+        const failedPrefixMax = new Array(allFailed.length + 1).fill(0);
+        for (let i = 0; i < allFailed.length; i++) {
+            failedPrefixMax[i + 1] = Math.max(failedPrefixMax[i], allFailed[i][1]);
+        }
         const mergedZones = new Map();
         for (const [etype, spans] of zonesByType) {
             spans.sort((a, b) => a[0] - b[0] || a[1] - b[1]);
@@ -332,8 +339,11 @@ class RuleEngine {
             mergedZones.set(etype, { starts, ends });
         }
         const candidates = [];
+        // Whole-document facts a suppressor may need, derived at most once per call
+        // and discarded with it.
+        const scratch = new suppressors_js_1.SuppressionScratch(text);
         for (const { match, hasValidValidator } of validated) {
-            if ((0, suppressors_js_1.shouldSuppress)(text, match))
+            if ((0, suppressors_js_1.shouldSuppress)(text, match, scratch))
                 continue;
             // A validator-less match inside a failed-validation span of its own type
             // is demoted below every other candidate rather than deleted. Demotion
@@ -361,13 +371,31 @@ class RuleEngine {
             // default — the generic phone pattern kept claiming the tail of the
             // rejected national number.
             if (match.patternDef.validator === null && !demoted) {
+                // Upper bound of the spans starting at or before match.start.
+                let lo = 0, hi = failedStarts.length;
+                while (lo < hi) {
+                    const mid = (lo + hi) >>> 1;
+                    if (failedStarts[mid] <= match.start)
+                        lo = mid + 1;
+                    else
+                        hi = mid;
+                }
                 let fragment = false;
-                for (const [zs, ze] of allFailed) {
-                    if (zs > match.start)
-                        break;
-                    if (match.end <= ze && (match.start > zs || match.end < ze)) {
-                        fragment = true;
-                        break;
+                const reach = failedPrefixMax[lo];
+                if (reach > match.end) {
+                    // Something starting no later than us ends strictly later: we are a
+                    // proper subset of it.
+                    fragment = true;
+                }
+                else if (reach === match.end) {
+                    // Same end offset — a proper subset only if some span with that end
+                    // starts strictly earlier. Scanning backwards finds the enclosing
+                    // span first, and the equal case is rare.
+                    for (let i = lo - 1; i >= 0; i--) {
+                        if (allFailed[i][1] === match.end && allFailed[i][0] < match.start) {
+                            fragment = true;
+                            break;
+                        }
                     }
                 }
                 if (fragment)
@@ -409,8 +437,28 @@ class RuleEngine {
                 priority = 1;
                 blindPriority = 1;
             }
-            const country = (match.countryCode !== "SHARED" && match.countryCode !== "CUSTOM")
+            let entityType = match.patternDef.entityType;
+            let country = (match.countryCode !== "SHARED" && match.countryCode !== "CUSTOM")
                 ? match.countryCode : null;
+            let outOfScope = declared !== null && country !== null && !declared.has(match.countryCode);
+            let confidence = "high";
+            // The document labels this span as something else. Relabelling, not
+            // suppression: the span is found either way, so removing the claim would
+            // decide only whether the value is masked — and the answer was "no".
+            // "Rijksregisternummer: 85.03.19-284.73" produced no detection at all.
+            //
+            // The country came from the pattern that matched, and that pattern was
+            // just overruled: a Finnish phone rule saying "Finland" is worthless once
+            // the value is filed as a Cypriot identity number. Dropping it also clears
+            // outOfScope, which would otherwise claim the detection belongs to a
+            // country the caller did not declare while naming no country at all.
+            const retyped = (0, cues_js_1.retypedBy)(text, match.start, entityType);
+            if (retyped !== null) {
+                entityType = retyped;
+                country = null;
+                outOfScope = false;
+                confidence = "medium";
+            }
             candidates.push({
                 blindPriority,
                 cue: localCueBonus(text, match.start, match.patternDef.entityType),
@@ -418,6 +466,45 @@ class RuleEngine {
                 length: match.end - match.start,
                 countryScore: countryScores.get(match.countryCode) ?? 0,
                 inScope: declared === null || declared.has(match.countryCode) ? 1 : 0,
+                tier: 0,
+                det: {
+                    entityType,
+                    start: match.start,
+                    end: match.end,
+                    text: match.text,
+                    source: types_js_1.DetectionSource.RULES,
+                    country,
+                    confidence,
+                    outOfScope,
+                },
+            });
+        }
+        // A declined identifier the document itself labels. Cue-backed by
+        // construction, so `cue` is 1 — that is what lets it take a span from a
+        // foreign pattern whose checksum happened to pass, which is how
+        // "ΑΦΜ: 147382965" was reported as a Czech national ID.
+        //
+        // `priority` is 0: below every regex-only claim, so it wins only what
+        // nothing better wants. `blindPriority` is 1 so that span selection — the
+        // country-blind half of the sort — treats it like any other regex-strength
+        // claim and invariant I1 is untouched.
+        //
+        // Emitted as "low", which is the honest description: the shape and the label
+        // agree, the check digit does not. A caller wanting only checksum-backed
+        // detections filters on it.
+        for (const match of rescued) {
+            if ((0, suppressors_js_1.shouldSuppress)(text, match, scratch))
+                continue;
+            const country = (match.countryCode !== "SHARED" && match.countryCode !== "CUSTOM")
+                ? match.countryCode : null;
+            candidates.push({
+                blindPriority: 1,
+                cue: 1,
+                priority: 0,
+                length: match.end - match.start,
+                countryScore: countryScores.get(match.countryCode) ?? 0,
+                inScope: declared === null || declared.has(match.countryCode) ? 1 : 0,
+                tier: 0,
                 det: {
                     entityType: match.patternDef.entityType,
                     start: match.start,
@@ -425,14 +512,14 @@ class RuleEngine {
                     text: match.text,
                     source: types_js_1.DetectionSource.RULES,
                     country,
-                    confidence: "high",
+                    confidence: "low",
                     outOfScope: declared !== null && country !== null && !declared.has(match.countryCode),
                 },
             });
         }
         for (const d of (0, structural_js_1.detectStructuralDob)(text)) {
             candidates.push({ blindPriority: 1, cue: 0, priority: 1, length: d.end - d.start,
-                countryScore: 0, inScope: 1, det: d });
+                countryScore: 0, inScope: 1, tier: 0, det: d });
         }
         const detections = this.deduplicate(candidates);
         detections.sort((a, b) => a.start - b.start || b.end - a.end);
@@ -466,7 +553,10 @@ class RuleEngine {
             if (seen === undefined || c.blindPriority > seen)
                 spanTier.set(key, c.blindPriority);
         }
-        const tierOf = (c) => spanTier.get(`${c.det.start}:${c.det.end}`);
+        // Resolved once per candidate rather than inside the comparator, which ran
+        // it O(n log n) times and rebuilt the key string on every comparison.
+        for (const c of candidates)
+            c.tier = spanTier.get(`${c.det.start}:${c.det.end}`);
         // Span length outranks *everything*, including validation.
         //
         // A shorter candidate winning re-cuts a longer one and leaves the
@@ -492,7 +582,7 @@ class RuleEngine {
         // country-blind up to and including `start`, and (length, start) already
         // identifies a span uniquely.
         const sorted = [...candidates].sort((a, b) => b.length - a.length || // longest first     } country-blind
-            tierOf(b) - tierOf(a) || // best tier here    } — decides
+            b.tier - a.tier || // best tier here    } — decides
             a.det.start - b.det.start || // leftmost          } WHICH span
             b.cue - a.cue || // local cue         } country-aware
             b.priority - a.priority || // country priority  } — decides the