euredact 0.3.1 → 0.3.2

package/dist/esm/index.d.ts

@@ -1,5 +1,6 @@
 export { EntityType, DetectionSource, LEGACY_TYPE_ALIASES } from "./types.js";
-export type { Detection, RedactResult, PatternDef, CountryConfig } from "./types.js";
+export type { Detection, RedactResult, PatternDef, CountryConfig, CountryEvidence } from "./types.js";
+export { DocumentContext } from "./rules/context.js";
 export { EuRedact, type RedactOptions } from "./sdk.js";
 export { COUNTRY_CONFIGS } from "./rules/countries/index.js";
 export { setBicRegistry, getBicRegistry, SEED_BIC6_PREFIXES, type BicRegistryProvider } from "./rules/bicRegistry.js";

package/dist/esm/index.js

@@ -1,4 +1,5 @@
 export { EntityType, DetectionSource, LEGACY_TYPE_ALIASES } from "./types.js";
+export { DocumentContext } from "./rules/context.js";
 export { EuRedact } from "./sdk.js";
 export { COUNTRY_CONFIGS } from "./rules/countries/index.js";
 export { setBicRegistry, getBicRegistry, SEED_BIC6_PREFIXES } from "./rules/bicRegistry.js";

package/dist/esm/rules/context.d.ts

@@ -0,0 +1,31 @@
+/**
+ * Country evidence shared across the chunks of one document.
+ *
+ * A long document is usually redacted in pieces. Each piece is scanned
+ * independently, so each infers its own country — and a chunk that happens to
+ * contain no IBAN, no `+CC` number and no ccTLD infers nothing at all, even
+ * when the first page identified the document beyond doubt:
+ *
+ *     page 1   "Factuur — IBAN NL91ABNA0417164300, info@jansen.nl"
+ *     page 7   "Telefoon 0612345678"        <- Dutch mobile, or Danish CPR?
+ *
+ * Scanned alone, page 7 has nothing to go on. Sharing a context across the two
+ * carries page 1's evidence forward, and the number resolves as a Dutch phone.
+ *
+ * Spans are rebased by `chunkOffset` on the way in, so evidence collected from
+ * a chunk points at offsets in the whole document rather than the chunk.
+ *
+ * A context influences **scoring only**. It can change which national scheme is
+ * attributed to a value. It can never change which spans are found, so a
+ * context carried over from the wrong document cannot cause a miss.
+ */
+import type { CountryEvidence } from "../types.js";
+export declare class DocumentContext {
+    private seen;
+    private items;
+    /** Record evidence from a chunk starting at `chunkOffset`. */
+    add(evidence: CountryEvidence[], chunkOffset?: number): void;
+    /** Everything recorded so far, spans rebased to the whole document. */
+    evidence(): CountryEvidence[];
+    get size(): number;
+}

package/dist/esm/rules/context.js

@@ -0,0 +1,46 @@
+/**
+ * Country evidence shared across the chunks of one document.
+ *
+ * A long document is usually redacted in pieces. Each piece is scanned
+ * independently, so each infers its own country — and a chunk that happens to
+ * contain no IBAN, no `+CC` number and no ccTLD infers nothing at all, even
+ * when the first page identified the document beyond doubt:
+ *
+ *     page 1   "Factuur — IBAN NL91ABNA0417164300, info@jansen.nl"
+ *     page 7   "Telefoon 0612345678"        <- Dutch mobile, or Danish CPR?
+ *
+ * Scanned alone, page 7 has nothing to go on. Sharing a context across the two
+ * carries page 1's evidence forward, and the number resolves as a Dutch phone.
+ *
+ * Spans are rebased by `chunkOffset` on the way in, so evidence collected from
+ * a chunk points at offsets in the whole document rather than the chunk.
+ *
+ * A context influences **scoring only**. It can change which national scheme is
+ * attributed to a value. It can never change which spans are found, so a
+ * context carried over from the wrong document cannot cause a miss.
+ */
+export class DocumentContext {
+    // Deduped on the same key evidence.collect() uses, so re-running a chunk — a
+    // retry, an overlapping window — cannot let one entity vote twice and
+    // outweigh the rest of the document.
+    seen = new Set();
+    items = [];
+    /** Record evidence from a chunk starting at `chunkOffset`. */
+    add(evidence, chunkOffset = 0) {
+        for (const ev of evidence) {
+            const span = [ev.span[0] + chunkOffset, ev.span[1] + chunkOffset];
+            const key = `${span[0]}:${span[1]}:${ev.source}`;
+            if (this.seen.has(key))
+                continue;
+            this.seen.add(key);
+            this.items.push(chunkOffset === 0 ? ev : { ...ev, span });
+        }
+    }
+    /** Everything recorded so far, spans rebased to the whole document. */
+    evidence() {
+        return [...this.items];
+    }
+    get size() {
+        return this.items.length;
+    }
+}

package/dist/esm/rules/engine.d.ts

@@ -1,4 +1,5 @@
-import { type Detection } from "../types.js";
+import { type CountryEvidence, type Detection } from "../types.js";
+import { type RawMatch } from "./matchers.js";
 /**
  * The country modules use the EU/VAT convention (UK, EL) while the public
  * contract is ISO 3166-1 alpha-2 (GB, GR). Accept both spellings so a caller
@@ -12,11 +13,34 @@ export declare class RuleEngine {
     private loadedCountries;
     addCustomPattern(name: string, pattern: string): void;
     loadCountries(countryCodes: string[] | null): void;
-    detect(text: string, countryCodes: string[] | null): Detection[];
+    /**
+     * Generate candidate matches. Country-blind, by contract.
+     *
+     * This is the only caller of {@link MultiPatternMatcher.scan}, and it takes
+     * no country argument. Which country the caller declared may influence how a
+     * match is *scored*; it may never influence whether the match is *found*.
+     * Before this held, `countries: ["BE"]` made a valid Dutch BSN vanish
+     * entirely because the Dutch patterns were never run.
+     */
+    generateCandidates(text: string): RawMatch[];
+    detect(text: string, countryCodes: string[] | null, countryHint?: string[] | null): Detection[];
+    /**
+     * {@link RuleEngine.detect}, plus the country evidence it reasoned from.
+     *
+     * `countryCodes` is **scope**: it flags detections from elsewhere as
+     * `outOfScope` and, like `countryHint`, contributes a prior. `countryHint` is
+     * **only** a prior — it helps resolve ambiguity without marking anything out
+     * of scope. Neither gates what is looked for.
+     */
+    detectWithEvidence(text: string, countryCodes: string[] | null, countryHint?: string[] | null, priorEvidence?: CountryEvidence[] | null): {
+        detections: Detection[];
+        evidence: CountryEvidence[];
+        scores: Map<string, number>;
+    };
     /**
      * Remove overlapping detections with priority-aware resolution.
-     * Priority: validated (3) > custom (2) > regex-only (1).
-     * Within the same tier, longer span wins.
+     * Priority: validated (3) > custom (2) > regex-only (1) > postal (0) >
+     * demoted (-1). Within a tier, longer span wins, then country evidence.
      */
     private deduplicate;
     getLoadedCountries(): Set<string>;

package/dist/esm/rules/engine.js

@@ -3,6 +3,7 @@ import { MultiPatternMatcher } from "./matchers.js";
 import { shouldSuppress } from "./suppressors.js";
 import { detectStructuralDob } from "./structural.js";
 import { COUNTRY_CONFIGS } from "./countries/index.js";
+import * as evidenceMod from "./evidence.js";
 /**
  * Heuristic to detect nested quantifiers that cause catastrophic backtracking.
  * After stripping escaped chars and character classes, look for an unbounded
@@ -100,118 +101,248 @@ export class RuleEngine {
         if (newCountries)
             this.matcher.compile();
     }
-    detect(text, countryCodes) {
-        this.loadCountries(countryCodes);
-        // Scope the scan to requested countries for efficiency
-        // Resolve through the alias table so `countries: ["GB"]` actually scans
-        // the patterns registered under "UK".
-        let scanCodes = null;
+    /**
+     * Generate candidate matches. Country-blind, by contract.
+     *
+     * This is the only caller of {@link MultiPatternMatcher.scan}, and it takes
+     * no country argument. Which country the caller declared may influence how a
+     * match is *scored*; it may never influence whether the match is *found*.
+     * Before this held, `countries: ["BE"]` made a valid Dutch BSN vanish
+     * entirely because the Dutch patterns were never run.
+     */
+    generateCandidates(text) {
+        this.loadCountries(null);
+        return this.matcher.scan(text, null);
+    }
+    detect(text, countryCodes, countryHint) {
+        return this.detectWithEvidence(text, countryCodes, countryHint).detections;
+    }
+    /**
+     * {@link RuleEngine.detect}, plus the country evidence it reasoned from.
+     *
+     * `countryCodes` is **scope**: it flags detections from elsewhere as
+     * `outOfScope` and, like `countryHint`, contributes a prior. `countryHint` is
+     * **only** a prior — it helps resolve ambiguity without marking anything out
+     * of scope. Neither gates what is looked for.
+     */
+    detectWithEvidence(text, countryCodes, countryHint, priorEvidence) {
+        const rawMatches = this.generateCandidates(text);
+        // The declared countries are a scoring signal from here on: they rank
+        // candidates and flag out-of-scope detections. They do not gate.
+        let declared = null;
         if (countryCodes !== null) {
-            scanCodes = new Set();
+            declared = new Set();
             for (const c of countryCodes) {
                 const code = resolveCountryCode(c);
-                if (code !== null)
-                    scanCodes.add(code);
+                if (code === null)
+                    warnUnknownCountry(c);
+                else
+                    declared.add(code);
             }
-            scanCodes.add("SHARED");
-            scanCodes.add("CUSTOM");
+            declared.add("SHARED");
+            declared.add("CUSTOM");
+        }
+        // The prior fed to the evidence bus. Declared countries count too — a
+        // caller naming a country is asserting context, which is exactly what a
+        // prior is — but a hint never narrows scope, so it is kept out of
+        // `declared` and cannot flag anything out of scope.
+        const prior = new Set(declared ?? []);
+        for (const c of countryHint ?? []) {
+            const code = resolveCountryCode(c);
+            if (code === null)
+                warnUnknownCountry(c);
+            else
+                prior.add(code);
         }
-        let rawMatches = this.matcher.scan(text, scanCodes);
-        // Matches that have a validator but fail it create suppression zones:
-        // the span is recognisably a specific entity (e.g. IBAN-shaped) so
-        // overlapping regex-only matches (license plate, phone) are false
-        // positives and must be suppressed.
+        // Pass 1: checksum validation. A span that matched a checksummed pattern
+        // but failed the checksum is recorded with its country and entity type;
+        // what it is allowed to demote is decided below, once the document's
+        // countries are known.
         const validated = [];
-        const rawZones = [];
+        const failedSpans = [];
         for (const m of rawMatches) {
-            const isValid = this.matcher.validate(m);
-            if (isValid) {
+            if (this.matcher.validate(m)) {
                 validated.push({ match: m, hasValidValidator: m.patternDef.validator !== null });
             }
             else if (m.patternDef.validator !== null && !m.patternDef.requiresContext) {
-                rawZones.push([m.start, m.end]);
+                failedSpans.push({ start: m.start, end: m.end, code: m.countryCode,
+                    etype: String(m.patternDef.entityType) });
             }
         }
-        // Merge overlapping suppression zones for O(log n) containment checks
-        let zoneStarts = [];
-        let zoneEnds = [];
-        if (rawZones.length > 0) {
-            rawZones.sort((a, b) => a[0] - b[0] || a[1] - b[1]);
-            let [ms, me] = rawZones[0];
-            for (let i = 1; i < rawZones.length; i++) {
-                const [zs, ze] = rawZones[i];
-                if (zs <= me) {
+        // Evidence pass: work out which countries this document belongs to, from
+        // the entities that carry their country in the string.
+        //
+        // Every surviving candidate is eligible, not only checksum-backed ones. An
+        // earlier version required a validator here, which silently dropped the
+        // email ccTLD — the second-strongest signal at 4.50 log-odds and present in
+        // 98,949 of 152,300 corpus documents — because EMAIL has nothing to
+        // checksum. Candidates that failed a validator never reach `validated`, so
+        // this admits the unvalidatable, not the invalid.
+        const evidenceDets = [];
+        for (const { match } of validated) {
+            if (!evidenceMod.EVIDENCE_TYPES.has(String(match.patternDef.entityType)))
+                continue;
+            evidenceDets.push({
+                entityType: match.patternDef.entityType,
+                start: match.start, end: match.end, text: match.text,
+                source: DetectionSource.RULES,
+                country: (match.countryCode !== "SHARED" && match.countryCode !== "CUSTOM")
+                    ? match.countryCode : null,
+                confidence: "high",
+            });
+        }
+        const countryEvidence = evidenceMod.collect(evidenceDets);
+        const countryScores = evidenceMod.accumulate(priorEvidence && priorEvidence.length ? [...countryEvidence, ...priorEvidence] : countryEvidence, prior);
+        const hasScores = countryScores.size > 0;
+        const corroborated = (code) => !hasScores || code === "SHARED" || code === "CUSTOM" || countryScores.has(code);
+        // Now that the document's countries are known, decide which failed
+        // validators are entitled to demote anything. A checksum failure only means
+        // "not a valid X"; it says nothing about the span unless X was plausible
+        // here in the first place. A Belgian national-ID pattern failing on
+        // 0612345678 is not evidence against that span being a Dutch phone number.
+        //
+        // Zones are also per entity type. A failed checksum is evidence against
+        // *that type*, not against the span: Sweden's own personnummer checksum
+        // failing on 0708787668 used to demote the Swedish *phone* candidate for
+        // the same digits, handing the span to a Danish CPR that happened to
+        // validate. Worth 878 mistyped phone numbers per 30,000 documents.
+        const zonesByType = new Map();
+        for (const f of failedSpans) {
+            if (!corroborated(f.code))
+                continue;
+            const list = zonesByType.get(f.etype);
+            if (list)
+                list.push([f.start, f.end]);
+            else
+                zonesByType.set(f.etype, [[f.start, f.end]]);
+        }
+        const mergedZones = new Map();
+        for (const [etype, spans] of zonesByType) {
+            spans.sort((a, b) => a[0] - b[0] || a[1] - b[1]);
+            const starts = [];
+            const ends = [];
+            let [ms, me] = spans[0];
+            for (let i = 1; i < spans.length; i++) {
+                const [zs, ze] = spans[i];
+                if (zs <= me)
                     me = Math.max(me, ze);
-                }
                 else {
-                    zoneStarts.push(ms);
-                    zoneEnds.push(me);
+                    starts.push(ms);
+                    ends.push(me);
                     ms = zs;
                     me = ze;
                 }
             }
-            zoneStarts.push(ms);
-            zoneEnds.push(me);
+            starts.push(ms);
+            ends.push(me);
+            mergedZones.set(etype, { starts, ends });
         }
-        // Build candidates with priority: validated (3) > custom (2) > regex-only (1)
         const candidates = [];
         for (const { match, hasValidValidator } of validated) {
             if (shouldSuppress(text, match))
                 continue;
-            // O(log n) binary-search containment check on merged zones
-            if (match.patternDef.validator === null && zoneStarts.length > 0) {
-                let lo = 0, hi = zoneStarts.length;
-                while (lo < hi) {
-                    const mid = (lo + hi) >>> 1;
-                    if (zoneStarts[mid] <= match.start)
-                        lo = mid + 1;
-                    else
-                        hi = mid;
+            // A validator-less match inside a failed-validation span of its own type
+            // is demoted below every other candidate rather than deleted. Demotion
+            // cannot silence a detection, which is the property deletion lacked:
+            // measured across the corpus, deletion removed 454 detections of which
+            // 454 overlapped real labelled PII.
+            let demoted = false;
+            if (match.patternDef.validator === null) {
+                const zone = mergedZones.get(String(match.patternDef.entityType));
+                if (zone) {
+                    let lo = 0, hi = zone.starts.length;
+                    while (lo < hi) {
+                        const mid = (lo + hi) >>> 1;
+                        if (zone.starts[mid] <= match.start)
+                            lo = mid + 1;
+                        else
+                            hi = mid;
+                    }
+                    const idx = lo - 1;
+                    demoted = idx >= 0 && zone.ends[idx] >= match.end;
                 }
-                const idx = lo - 1;
-                if (idx >= 0 && zoneEnds[idx] >= match.end)
-                    continue;
             }
-            // A bare digit run is the weakest evidence in the engine and must never
-            // re-cut a span a structured detector claims (PHONE, SSN, NATIONAL_ID,
-            // BANK_ACCOUNT, VAT...). Postal codes take unclaimed spans only,
-            // whatever the span lengths.
-            const priority = hasValidValidator
-                ? 3
-                : match.countryCode === "CUSTOM"
-                    ? 2
-                    : match.patternDef.entityType === EntityType.POSTAL_CODE
-                        ? 0
-                        : 1;
-            const spanLength = match.end - match.start;
-            const score = priority * 1_000_000 + spanLength;
+            let priority;
+            if (demoted) {
+                priority = -1;
+            }
+            else if (hasValidValidator) {
+                // A passing checksum normally outranks everything. But a checksum only
+                // says the digits fit *some* national scheme, and weak ones fit by luck
+                // — a mod-11 scheme accepts a random number about one time in eleven.
+                // So a validated candidate from a country the document shows no trace
+                // of is treated as coincidence, not evidence.
+                //
+                // Concretely: the Dutch phone number 0612345678 also passes the Danish
+                // CPR checksum. Entities that carry their own country vouch for
+                // themselves — an IBAN emits evidence for its own country — so a
+                // foreign IBAN in a domestic invoice keeps its rank.
+                priority = corroborated(match.countryCode) ? 3 : 1;
+            }
+            else if (match.countryCode === "CUSTOM") {
+                priority = 2;
+            }
+            else if (match.patternDef.entityType === EntityType.POSTAL_CODE) {
+                // A bare digit run is the weakest evidence in the engine and must never
+                // re-cut a span a structured detector claims (PHONE, SSN, NATIONAL_ID,
+                // BANK_ACCOUNT, VAT...). Postal codes take unclaimed spans only,
+                // whatever the span lengths.
+                priority = 0;
+            }
+            else {
+                priority = 1;
+            }
+            const country = (match.countryCode !== "SHARED" && match.countryCode !== "CUSTOM")
+                ? match.countryCode : null;
             candidates.push({
+                priority,
+                length: match.end - match.start,
+                countryScore: countryScores.get(match.countryCode) ?? 0,
+                inScope: declared === null || declared.has(match.countryCode) ? 1 : 0,
                 det: {
                     entityType: match.patternDef.entityType,
                     start: match.start,
                     end: match.end,
                     text: match.text,
                     source: DetectionSource.RULES,
-                    country: (match.countryCode !== "SHARED" && match.countryCode !== "CUSTOM") ? match.countryCode : null,
+                    country,
                     confidence: "high",
+                    outOfScope: declared !== null && country !== null && !declared.has(match.countryCode),
                 },
-                score,
             });
         }
         for (const d of detectStructuralDob(text)) {
-            candidates.push({ det: d, score: 1_000_000 + (d.end - d.start) });
+            candidates.push({ priority: 1, length: d.end - d.start, countryScore: 0, inScope: 1, det: d });
+        }
+        const detections = this.deduplicate(candidates);
+        detections.sort((a, b) => a.start - b.start || b.end - a.end);
+        // Attach the confidence behind each country attribution, so a caller can
+        // tell "Danish, and the document is plainly Danish" from "Danish, on a
+        // checksum alone".
+        const ranking = evidenceMod.weightsToRanking(countryScores);
+        for (const d of detections) {
+            d.countryConfidence = (d.country !== null ? ranking.get(d.country) : undefined) ?? 0;
         }
-        return this.deduplicate(candidates);
+        return { detections, evidence: countryEvidence, scores: countryScores };
     }
     /**
      * Remove overlapping detections with priority-aware resolution.
-     * Priority: validated (3) > custom (2) > regex-only (1).
-     * Within the same tier, longer span wins.
+     * Priority: validated (3) > custom (2) > regex-only (1) > postal (0) >
+     * demoted (-1). Within a tier, longer span wins, then country evidence.
      */
     deduplicate(candidates) {
         if (candidates.length === 0)
             return [];
-        const sorted = [...candidates].sort((a, b) => b.score - a.score);
+        // Span length outranks scope: preferring the declared country over the
+        // longest match can truncate an entity. Measured: with countries ["BE"],
+        // the Belgian phone pattern claims 11 of the 14 characters of
+        // "06 12 34 56 78" and leaves three digits exposed. Scope only breaks ties
+        // between candidates of equal length, so it can change WHICH country is
+        // attributed, never WHAT is masked.
+        const sorted = [...candidates].sort((a, b) => b.priority - a.priority ||
+            b.length - a.length ||
+            b.countryScore - a.countryScore ||
+            b.inScope - a.inScope);
         const result = [];
         const occupied = new Set();
         for (const { det } of sorted) {

package/dist/esm/rules/evidence.d.ts

@@ -0,0 +1,63 @@
+/**
+ * Country evidence: what a document says about which country it belongs to.
+ *
+ * Some entities carry their country in the string — an IBAN's country code, a
+ * `+CC` dialling prefix, a VAT prefix, an email ccTLD. Those are used to work
+ * out which country a document belongs to, which in turn decides which
+ * national scheme owns an ambiguous value elsewhere in the same document.
+ * 36.6% of national-ID values in the corpus validate under more than one
+ * country's checksum, so without this the engine cannot tell a Dutch BSN from
+ * a Polish PESEL.
+ *
+ * Evidence influences **scoring only**. It ranks candidates that already exist
+ * and annotates the result; it never decides what is looked for. A wrong
+ * inference must not be able to cause a miss.
+ *
+ * Weights are derived, not chosen — measured as P(document country = X |
+ * signal claims X) across the corpus. Kept identical to the Python SDK's
+ * `rules/evidence.py`; re-derive there and port, rather than tuning by feel.
+ */
+import { type CountryEvidence, type Detection } from "../types.js";
+export declare const WEIGHTS: Record<string, number>;
+/**
+ * E.164 calling code -> country, for the countries this engine supports. Codes
+ * are unique within this set; a shared code (+1, +7) would have to emit
+ * evidence for every candidate country rather than one.
+ */
+export declare const DIALLING_CODES: Record<string, string>;
+/**
+ * ccTLD -> country. Deliberately excludes .eu/.com/.org/.net: those are not
+ * weak evidence, they are no evidence, and treating them as weak would let a
+ * .com address outvote a genuine national signal.
+ */
+export declare const CC_TLDS: Record<string, string>;
+/** Entity types that can carry country evidence. */
+export declare const EVIDENCE_TYPES: ReadonlySet<string>;
+/**
+ * Country evidence carried by *detections*, in document order.
+ *
+ * Deduplicated by (span, source): several countries' patterns can match the
+ * same IBAN, and counting that span once per pattern would let a single entity
+ * outvote the rest of the document.
+ */
+export declare function collect(detections: Detection[]): CountryEvidence[];
+/**
+ * Sum evidence into a per-country log-odds score.
+ *
+ * Returns a weighted set, never a single winner: a Belgian supplier invoicing a
+ * German customer is not "70% Belgian", both countries are genuinely present,
+ * and an ambiguous value must be resolvable against either.
+ */
+export declare function accumulate(evidence: CountryEvidence[], declared?: Iterable<string> | null): Map<string, number>;
+/**
+ * Convert per-country log-odds into confidences in [0, 1].
+ *
+ * Each country is scored independently — a logistic on its own log-odds —
+ * because document countries are not mutually exclusive. A Belgian supplier
+ * invoicing a German customer is genuinely both; softmaxing across countries
+ * would report the German side of that invoice at 0.00 and make cross-border
+ * documents indistinguishable from misattributions.
+ *
+ * Confidences therefore do not sum to 1, by design.
+ */
+export declare function weightsToRanking(scores: Map<string, number>): Map<string, number>;