euredact 0.3.1 → 0.3.2

package/dist/esm/rules/evidence.js

@@ -0,0 +1,195 @@
+/**
+ * Country evidence: what a document says about which country it belongs to.
+ *
+ * Some entities carry their country in the string — an IBAN's country code, a
+ * `+CC` dialling prefix, a VAT prefix, an email ccTLD. Those are used to work
+ * out which country a document belongs to, which in turn decides which
+ * national scheme owns an ambiguous value elsewhere in the same document.
+ * 36.6% of national-ID values in the corpus validate under more than one
+ * country's checksum, so without this the engine cannot tell a Dutch BSN from
+ * a Polish PESEL.
+ *
+ * Evidence influences **scoring only**. It ranks candidates that already exist
+ * and annotates the result; it never decides what is looked for. A wrong
+ * inference must not be able to cause a miss.
+ *
+ * Weights are derived, not chosen — measured as P(document country = X |
+ * signal claims X) across the corpus. Kept identical to the Python SDK's
+ * `rules/evidence.py`; re-derive there and port, rather than tuning by feel.
+ */
+import { EntityType } from "../types.js";
+// ── Derived weights ─────────────────────────────────────────────────────
+//
+// Measured on 152,300 documents:
+//
+//   signal          agree    total   P(doc=X|signal)   raw log-odds
+//   e164Prefix     41,402   41,402          0.9999             9.21
+//   bicCountry      2,588    2,588          0.9999             9.21
+//   emailTld       97,865   98,949          0.9890             4.50
+//   vatPrefix      19,022   20,136          0.9447             2.84
+//   ibanPrefix    110,572  126,428          0.8746             1.94
+//
+// The IBAN prefix being the *weakest* signal is real and worth knowing: a
+// Belgian IBAN in a Dutch invoice is ordinary, so the account's country is only
+// weak evidence about the document's.
+//
+// The perfect scores are capped. They are perfect because the generator emits
+// country-consistent values, not because a foreign mobile number never appears
+// in a real document. Treating a synthetic 1.0 as certainty would make a single
+// phone number unfalsifiable by any amount of contrary evidence.
+const MAX_LOG_ODDS = 4.0;
+export const WEIGHTS = {
+    e164Prefix: Math.min(9.21, MAX_LOG_ODDS),
+    bicCountry: Math.min(9.21, MAX_LOG_ODDS),
+    emailTld: Math.min(4.5, MAX_LOG_ODDS),
+    vatPrefix: 2.84,
+    ibanPrefix: 1.94,
+    declared: 2.0, // the caller's `countries`, a prior rather than a measurement
+};
+// ── Signal tables ───────────────────────────────────────────────────────
+/**
+ * E.164 calling code -> country, for the countries this engine supports. Codes
+ * are unique within this set; a shared code (+1, +7) would have to emit
+ * evidence for every candidate country rather than one.
+ */
+export const DIALLING_CODES = {
+    "43": "AT", "32": "BE", "359": "BG", "41": "CH", "357": "CY", "420": "CZ",
+    "49": "DE", "45": "DK", "372": "EE", "30": "EL", "34": "ES", "358": "FI",
+    "33": "FR", "385": "HR", "36": "HU", "353": "IE", "354": "IS", "39": "IT",
+    "370": "LT", "352": "LU", "371": "LV", "356": "MT", "31": "NL", "47": "NO",
+    "48": "PL", "351": "PT", "40": "RO", "46": "SE", "386": "SI", "421": "SK",
+    "44": "UK",
+};
+/**
+ * ccTLD -> country. Deliberately excludes .eu/.com/.org/.net: those are not
+ * weak evidence, they are no evidence, and treating them as weak would let a
+ * .com address outvote a genuine national signal.
+ */
+export const CC_TLDS = {
+    at: "AT", be: "BE", bg: "BG", ch: "CH", cy: "CY", cz: "CZ",
+    de: "DE", dk: "DK", ee: "EE", gr: "EL", es: "ES", fi: "FI",
+    fr: "FR", hr: "HR", hu: "HU", ie: "IE", is: "IS", it: "IT",
+    lt: "LT", lu: "LU", lv: "LV", mt: "MT", nl: "NL", no: "NO",
+    pl: "PL", pt: "PT", ro: "RO", se: "SE", si: "SI", sk: "SK",
+    uk: "UK",
+};
+/** VAT prefixes that are not the ISO code: Greece trades as EL, Northern
+ *  Ireland as XI. Naive alpha-2 handling gets both wrong. */
+const VAT_PREFIX_OVERRIDES = { EL: "EL", XI: "UK" };
+const emitIban = (det) => {
+    const code = det.text.slice(0, 2).toUpperCase();
+    if (!/^[A-Z]{2}$/.test(code))
+        return [];
+    return [{ country: code, source: "ibanPrefix", logOdds: WEIGHTS.ibanPrefix,
+            span: [det.start, det.end] }];
+};
+const emitE164 = (det) => {
+    if (!det.text.trimStart().startsWith("+"))
+        return [];
+    const digits = det.text.replace(/\D/g, "");
+    for (const len of [3, 2]) {
+        const country = DIALLING_CODES[digits.slice(0, len)];
+        if (country) {
+            return [{ country, source: "e164Prefix", logOdds: WEIGHTS.e164Prefix,
+                    span: [det.start, det.end] }];
+        }
+    }
+    return [];
+};
+const emitVat = (det) => {
+    const prefix = det.text.slice(0, 2).toUpperCase();
+    if (!/^[A-Z]{2}$/.test(prefix))
+        return [];
+    const country = VAT_PREFIX_OVERRIDES[prefix] ?? prefix;
+    return [{ country, source: "vatPrefix", logOdds: WEIGHTS.vatPrefix,
+            span: [det.start, det.end] }];
+};
+const emitBic = (det) => {
+    const clean = det.text.replace(/ /g, "").toUpperCase();
+    if (clean.length < 6)
+        return [];
+    return [{ country: clean.slice(4, 6), source: "bicCountry",
+            logOdds: WEIGHTS.bicCountry, span: [det.start, det.end] }];
+};
+const emitTld = (det) => {
+    const parts = det.text.split(".");
+    const country = CC_TLDS[parts[parts.length - 1].toLowerCase()];
+    if (!country)
+        return [];
+    return [{ country, source: "emailTld", logOdds: WEIGHTS.emailTld,
+            span: [det.start, det.end] }];
+};
+/** Dispatch by entity type, mirroring the suppressor tables. */
+const TYPE_EMITTERS = {
+    [EntityType.BANK_ACCOUNT]: [emitIban],
+    [EntityType.PHONE]: [emitE164],
+    [EntityType.VAT]: [emitVat],
+    [EntityType.BIC]: [emitBic],
+    [EntityType.EMAIL]: [emitTld],
+};
+/** Entity types that can carry country evidence. */
+export const EVIDENCE_TYPES = new Set(Object.keys(TYPE_EMITTERS));
+/**
+ * Country evidence carried by *detections*, in document order.
+ *
+ * Deduplicated by (span, source): several countries' patterns can match the
+ * same IBAN, and counting that span once per pattern would let a single entity
+ * outvote the rest of the document.
+ */
+export function collect(detections) {
+    const evidence = [];
+    const seen = new Set();
+    for (const det of detections) {
+        for (const emit of TYPE_EMITTERS[det.entityType] ?? []) {
+            for (const ev of emit(det)) {
+                const key = `${ev.span[0]}:${ev.span[1]}:${ev.source}`;
+                if (!seen.has(key)) {
+                    seen.add(key);
+                    evidence.push(ev);
+                }
+            }
+        }
+    }
+    // Document order, not pattern-registration order: this list is an audit
+    // trail, and a reader checking it against the text should not have to jump
+    // around. Also keeps the two SDKs byte-comparable.
+    evidence.sort((a, b) => a.span[0] - b.span[0] || a.span[1] - b.span[1]);
+    return evidence;
+}
+/**
+ * Sum evidence into a per-country log-odds score.
+ *
+ * Returns a weighted set, never a single winner: a Belgian supplier invoicing a
+ * German customer is not "70% Belgian", both countries are genuinely present,
+ * and an ambiguous value must be resolvable against either.
+ */
+export function accumulate(evidence, declared) {
+    const scores = new Map();
+    for (const ev of evidence) {
+        scores.set(ev.country, (scores.get(ev.country) ?? 0) + ev.logOdds);
+    }
+    for (const country of declared ?? []) {
+        if (country !== "SHARED" && country !== "CUSTOM") {
+            scores.set(country, (scores.get(country) ?? 0) + WEIGHTS.declared);
+        }
+    }
+    return scores;
+}
+/**
+ * Convert per-country log-odds into confidences in [0, 1].
+ *
+ * Each country is scored independently — a logistic on its own log-odds —
+ * because document countries are not mutually exclusive. A Belgian supplier
+ * invoicing a German customer is genuinely both; softmaxing across countries
+ * would report the German side of that invoice at 0.00 and make cross-border
+ * documents indistinguishable from misattributions.
+ *
+ * Confidences therefore do not sum to 1, by design.
+ */
+export function weightsToRanking(scores) {
+    const out = new Map();
+    for (const [country, score] of scores) {
+        out.set(country, 1 / (1 + Math.exp(-score)));
+    }
+    return out;
+}

package/dist/esm/rules/matchers.d.ts

@@ -7,6 +7,16 @@ export declare class MultiPatternMatcher {
     addPattern(patternDef: PatternDef, countryCode: string): void;
     addCountry(config: CountryConfig): void;
     compile(): void;
+    /**
+     * Scan text against patterns.
+     *
+     * `countryCodes` restricts which patterns run. **The engine never passes
+     * it.** Generation is country-blind by contract — see
+     * `RuleEngine.generateCandidates` and `src/__tests__/inference.ts` — because
+     * gating generation on the caller's declared country caused silent misses.
+     * The parameter is kept for a future opt-in fast mode, which would be the
+     * only caller permitted to trade recall for speed.
+     */
     scan(text: string, countryCodes?: Set<string> | null): RawMatch[];
     validate(match: RawMatch): boolean;
 }

package/dist/esm/rules/matchers.js

@@ -19,6 +19,16 @@ export class MultiPatternMatcher {
     compile() {
         this.compiled = true;
     }
+    /**
+     * Scan text against patterns.
+     *
+     * `countryCodes` restricts which patterns run. **The engine never passes
+     * it.** Generation is country-blind by contract — see
+     * `RuleEngine.generateCandidates` and `src/__tests__/inference.ts` — because
+     * gating generation on the caller's declared country caused silent misses.
+     * The parameter is kept for a future opt-in fast mode, which would be the
+     * only caller permitted to trade recall for speed.
+     */
     scan(text, countryCodes = null) {
         if (!this.compiled)
             this.compile();

package/dist/esm/sdk.d.ts

@@ -1,6 +1,27 @@
+import { DocumentContext } from "./rules/context.js";
 import { type RedactResult } from "./types.js";
 export interface RedactOptions {
+    /**
+     * Scope. Detections attributed elsewhere are flagged `outOfScope`, never
+     * dropped, and this also acts as a prior when resolving which national
+     * scheme owns an ambiguous value. It does **not** gate what is looked for.
+     */
     countries?: string[] | null;
+    /**
+     * A prior only. Helps resolve ambiguity without narrowing scope or flagging
+     * anything out of scope.
+     */
+    countryHint?: string[] | null;
+    /**
+     * Shares country evidence across the chunks of one document, so a chunk
+     * carrying no country signal of its own is still scored against what the
+     * rest of the document showed. Pass the same object for every chunk, with
+     * `chunkOffset` set to where the chunk starts in the whole document.
+     */
+    context?: DocumentContext | null;
+    /** Offset of this chunk within the document. Used only to rebase spans
+     *  recorded in `context`; returned detections are relative to `text`. */
+    chunkOffset?: number;
     mode?: string;
     referentialIntegrity?: boolean;
     detectDates?: boolean;

package/dist/esm/sdk.js

@@ -1,6 +1,7 @@
 import { ResultCache } from "./cache.js";
 import { normalize, mapOffsetToOriginal } from "./normalizer.js";
 import { RuleEngine } from "./rules/engine.js";
+import { weightsToRanking } from "./rules/evidence.js";
 import { EntityType } from "./types.js";
 const DATE_TYPES = new Set([EntityType.DOB, EntityType.DATE_OF_DEATH]);
 class ReferentialMapper {
@@ -44,12 +45,18 @@ export class EuRedact {
                 `(${this.maxInputLength.toLocaleString()} chars). Split the input or ` +
                 `increase maxInputLength when constructing EuRedact.`);
         }
-        const { countries = null, mode = "rules", referentialIntegrity = false, detectDates = false, cache = true, } = options;
+        const { countries = null, countryHint = null, context = null, chunkOffset = 0, mode = "rules", referentialIntegrity = false, detectDates = false, } = options;
+        // A context makes the result depend on evidence from other chunks, so the
+        // text no longer identifies the result. Caching is disabled rather than
+        // keyed on the context, whose contents change as chunks arrive.
+        const cache = context !== null ? false : (options.cache ?? true);
         const [normalizedText, offsetMapping] = normalize(text);
         const countriesTuple = countries
             ? countries.map(c => c.toUpperCase()).sort()
             : ["ALL"];
-        const cacheMode = `${mode}|dates=${detectDates}`;
+        // countryHint changes attribution, so it must key the cache too.
+        const hintKey = countryHint ? countryHint.map(c => c.toUpperCase()).sort().join(",") : "";
+        const cacheMode = `${mode}|dates=${detectDates}|hint=${hintKey}`;
         let cacheKey;
         if (cache) {
             cacheKey = this.cache.key(normalizedText, countriesTuple, cacheMode);
@@ -57,7 +64,10 @@ export class EuRedact {
             if (cached !== null)
                 return cached;
         }
-        let detections = this.engine.detect(normalizedText, countries);
+        const { detections: rawDetections, evidence, scores, } = this.engine.detectWithEvidence(normalizedText, countries, countryHint, context !== null ? context.evidence() : null);
+        if (context !== null)
+            context.add(evidence, chunkOffset);
+        let detections = rawDetections;
         if (offsetMapping !== null) {
             detections = detections.map(d => ({
                 ...d,
@@ -77,11 +87,18 @@ export class EuRedact {
                 : `[${det.entityType}]`;
             redacted = redacted.slice(0, det.start) + replacement + redacted.slice(det.end);
         }
+        // Report the inference so it can be audited. Spans in `evidence` are
+        // offsets into the normalised text, matching `detections`.
+        const inferredCountries = [...weightsToRanking(scores).entries()]
+            .sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]));
         const result = {
             redactedText: redacted,
             detections,
             source: "rules",
             degraded: false,
+            inferredCountries,
+            evidence,
+            detectionMode: countries && countries.length ? "declared" : "inferred",
         };
         if (cache && cacheKey) {
             this.cache.put(cacheKey, result);
@@ -89,8 +106,9 @@ export class EuRedact {
         return result;
     }
     redactBatch(texts, options = {}) {
-        const countries = options.countries ?? null;
-        this.engine.loadCountries(countries ? countries.map(c => c.toUpperCase()) : null);
+        // Generation is country-blind, so every country's patterns are loaded
+        // regardless of what the caller declared.
+        this.engine.loadCountries(null);
         return texts.map(text => this.redact(text, options));
     }
 }

package/dist/esm/types.d.ts

@@ -49,6 +49,24 @@ export declare enum DetectionSource {
     RULES = "rules",
     CLOUD = "cloud"
 }
+/**
+ * One reason to believe a document belongs to a country.
+ *
+ * Emitted by entities that carry their country in the string — an IBAN's
+ * country code, a `+CC` dialling prefix, a VAT prefix, an email ccTLD.
+ * Auditable by design: every inference traces back to the span and the source
+ * that produced it.
+ */
+export interface CountryEvidence {
+    /** ISO 3166-1 alpha-2. */
+    country: string;
+    /** Which signal produced this — "ibanPrefix", "e164Prefix", ... */
+    source: string;
+    /** Natural-log odds. Derived from the corpus, not hand-tuned. */
+    logOdds: number;
+    /** Offsets into the text that was scanned. */
+    span: [number, number];
+}
 export interface Detection {
     entityType: EntityType | string;
     start: number;
@@ -57,12 +75,47 @@ export interface Detection {
     source: DetectionSource;
     country: string | null;
     confidence: string;
+    /**
+     * How strongly the document supports {@link Detection.country}, in [0, 1].
+     *
+     * 0 when no country was attributed or nothing corroborated it — which is the
+     * signal that an attribution rests on a checksum alone.
+     */
+    countryConfidence?: number;
+    /**
+     * True when attributed to a country the caller did not declare.
+     *
+     * `countries` narrows *scoring*, never detection. An entity from outside the
+     * declared set is emitted and flagged, not dropped: callers assert context,
+     * they do not suppress evidence.
+     */
+    outOfScope?: boolean;
 }
 export interface RedactResult {
     redactedText: string;
     detections: Detection[];
     source: string;
     degraded: boolean;
+    /**
+     * Countries this document appears to belong to, strongest first, as
+     * `[country, confidence]` with confidence in [0, 1].
+     *
+     * Inferred from entities that carry their country in the string. Reported so
+     * the inference can be audited; it influences which national scheme is
+     * attributed to an ambiguous value, never which spans are found.
+     */
+    inferredCountries: Array<[string, number]>;
+    /** Every signal behind {@link RedactResult.inferredCountries}, with the span
+     *  that produced it. The audit trail for a country attribution. */
+    evidence: CountryEvidence[];
+    /**
+     * `"declared"` when the caller passed `countries`, `"inferred"` when the
+     * country was worked out from the content.
+     *
+     * Named `detectionMode` rather than `mode` because `redact({ mode })`
+     * already means the tier selector, which {@link RedactResult.source} reports.
+     */
+    detectionMode: string;
 }
 export interface PatternDef {
     entityType: EntityType | string;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "euredact",
-  "version": "0.3.1",
+  "version": "0.3.2",
   "description": "European PII redaction SDK -- fast, zero-dependency, rule-based detection and redaction of personal data across 31 European countries",
   "license": "Apache-2.0",
   "type": "module",
@@ -53,13 +53,14 @@
     "build:cjs": "tsc -p tsconfig.cjs.json",
     "postbuild:cjs": "echo '{\"type\":\"commonjs\"}' > dist/cjs/package.json",
     "build": "rm -rf dist && npm run build:esm && npm run build:cjs",
-    "test": "npm run test:spec && npm run test:conformance",
+    "test": "npm run test:spec && npm run test:inference && npm run test:conformance",
     "benchmark": "node --import tsx src/__tests__/benchmark.ts",
     "prepublishOnly": "npm run build",
     "test:spec": "node --import tsx src/__tests__/spec.ts",
     "test:eval": "node --import tsx src/__tests__/run.ts",
     "test:eval:full": "node --max-old-space-size=8192 --import tsx src/__tests__/evalFull.ts",
-    "test:conformance": "node --import tsx src/__tests__/conformance.ts"
+    "test:conformance": "node --import tsx src/__tests__/conformance.ts",
+    "test:inference": "node --import tsx src/__tests__/inference.ts"
   },
   "devDependencies": {
     "typescript": "^5.4.0",