euredact 0.3.0 → 0.3.2

package/dist/esm/rules/suppressors.js

@@ -1,5 +1,6 @@
 import { EntityType } from "../types.js";
 import { isRegisteredBic } from "./bicRegistry.js";
+import { DE_DISTRICT_CODES } from "./deDistricts.js";
 const CONTEXT_CHARS = 150;
 function getContext(text, start, end) {
     const ctxStart = Math.max(0, start - CONTEXT_CHARS);
@@ -117,6 +118,12 @@ function suppressPhoneDateOverlap(_text, match) {
         return false;
     return DATE_PATTERN_FULL.test(match.text.trim());
 }
+// Standards and classification prefixes that are plate-shaped once a letter and
+// digits follow: "ATC-N06", "ICD-O3". None is a German district code, but the
+// guard is still gated on the absence of a plate cue, so a genuine plate that
+// happens to collide is not lost.
+const STANDARDS_PREFIX = new Set(["ICD", "ISO", "DIN", "IEC", "RFC", "DSM", "ATC", "MDR"]);
+const PLATE_CUE_NEAR = /(?:Kennzeichen|Nummernschild|Kfz|Fahrzeug|amtliche[sn]?\s+Kennz|nummerplaat|plaque\s+d'immatriculation|license\s+plate|number\s+plate)/i;
 const NOT_CITY_CODES = new Set([
     "ID", "NR", "NO", "ST", "DR", "MR", "MS", "HR", "FR", "IM", "IN", "OR", "IF", "IS", "IT", "AT", "AD", "AG", "AV",
     "BE", "DE", "EU", "NL", "LU", "WS", "SS", "IP",
@@ -135,6 +142,13 @@ function suppressPlateInCompound(text, match) {
     }
     const matched = match.text.trim();
     const parts = matched.split(/[\s\-]+/);
+    // A standards or classification reference, not a plate — unless a plate cue
+    // nearby says otherwise.
+    if (parts.length > 0 && STANDARDS_PREFIX.has(parts[0].toUpperCase())) {
+        const [b, a] = getContext(text, match.start, match.end);
+        if (!PLATE_CUE_NEAR.test(b + a))
+            return true;
+    }
     if (parts.length > 0 && NOT_CITY_CODES.has(parts[0])) {
         const afterChar = match.end < text.length ? text[match.end] : "";
         const beforeChar = match.start > 0 ? text[match.start - 1] : "";
@@ -157,6 +171,31 @@ function suppressPlateInCompound(text, match) {
     }
     return false;
 }
+/**
+ * Reject a German plate whose district code is not a real one.
+ *
+ * The district-code set is closed, which makes it a whitelist rather than the
+ * open-ended blocklist of standards prefixes: "REF-A12", "SYS-B3", "KTO-A1"
+ * and every other document reference of that shape fail it without needing to
+ * be enumerated. Applied as a tier — an unknown code still emits when a plate
+ * cue is nearby, so a missing code costs recall only in the absence of any
+ * other evidence.
+ */
+function suppressDePlateUnknownDistrict(text, match) {
+    if (match.patternDef.entityType !== EntityType.LICENSE_PLATE)
+        return false;
+    if (match.countryCode !== "DE")
+        return false;
+    const parts = match.text.trim().split(/[\s\-]+/);
+    if (parts.length === 0 || !parts[0])
+        return false;
+    if (DE_DISTRICT_CODES.has(parts[0].toUpperCase()))
+        return false;
+    const [before, after] = getContext(text, match.start, match.end);
+    if (PLATE_CUE_NEAR.test(before + after))
+        return false;
+    return true;
+}
 function suppressNatidAsPassport(text, match) {
     if (match.patternDef.entityType !== EntityType.NATIONAL_ID)
         return false;
@@ -426,7 +465,7 @@ const TYPE_SUPPRESSORS = {
     [EntityType.POSTAL_CODE]: [suppressCurrency, suppressUnits, suppressMath, suppressLegal, suppressYearAsPostal, suppressPostalInsideIban, suppressPostalAsHouseNumber, suppressPostalInLongerIdentifier],
     [EntityType.BIC]: [suppressBicWithoutEvidence],
     [EntityType.BANK_ACCOUNT]: [suppressReference],
-    [EntityType.LICENSE_PLATE]: [suppressPlateInCompound],
+    [EntityType.LICENSE_PLATE]: [suppressPlateInCompound, suppressDePlateUnknownDistrict],
     [EntityType.CHAMBER_OF_COMMERCE]: [suppressReference],
 };
 export function shouldSuppress(text, match) {

package/dist/esm/sdk.d.ts

@@ -1,6 +1,27 @@
+import { DocumentContext } from "./rules/context.js";
 import { type RedactResult } from "./types.js";
 export interface RedactOptions {
+    /**
+     * Scope. Detections attributed elsewhere are flagged `outOfScope`, never
+     * dropped, and this also acts as a prior when resolving which national
+     * scheme owns an ambiguous value. It does **not** gate what is looked for.
+     */
     countries?: string[] | null;
+    /**
+     * A prior only. Helps resolve ambiguity without narrowing scope or flagging
+     * anything out of scope.
+     */
+    countryHint?: string[] | null;
+    /**
+     * Shares country evidence across the chunks of one document, so a chunk
+     * carrying no country signal of its own is still scored against what the
+     * rest of the document showed. Pass the same object for every chunk, with
+     * `chunkOffset` set to where the chunk starts in the whole document.
+     */
+    context?: DocumentContext | null;
+    /** Offset of this chunk within the document. Used only to rebase spans
+     *  recorded in `context`; returned detections are relative to `text`. */
+    chunkOffset?: number;
     mode?: string;
     referentialIntegrity?: boolean;
     detectDates?: boolean;

package/dist/esm/sdk.js

@@ -1,6 +1,7 @@
 import { ResultCache } from "./cache.js";
 import { normalize, mapOffsetToOriginal } from "./normalizer.js";
 import { RuleEngine } from "./rules/engine.js";
+import { weightsToRanking } from "./rules/evidence.js";
 import { EntityType } from "./types.js";
 const DATE_TYPES = new Set([EntityType.DOB, EntityType.DATE_OF_DEATH]);
 class ReferentialMapper {
@@ -44,12 +45,18 @@ export class EuRedact {
                 `(${this.maxInputLength.toLocaleString()} chars). Split the input or ` +
                 `increase maxInputLength when constructing EuRedact.`);
         }
-        const { countries = null, mode = "rules", referentialIntegrity = false, detectDates = false, cache = true, } = options;
+        const { countries = null, countryHint = null, context = null, chunkOffset = 0, mode = "rules", referentialIntegrity = false, detectDates = false, } = options;
+        // A context makes the result depend on evidence from other chunks, so the
+        // text no longer identifies the result. Caching is disabled rather than
+        // keyed on the context, whose contents change as chunks arrive.
+        const cache = context !== null ? false : (options.cache ?? true);
         const [normalizedText, offsetMapping] = normalize(text);
         const countriesTuple = countries
             ? countries.map(c => c.toUpperCase()).sort()
             : ["ALL"];
-        const cacheMode = `${mode}|dates=${detectDates}`;
+        // countryHint changes attribution, so it must key the cache too.
+        const hintKey = countryHint ? countryHint.map(c => c.toUpperCase()).sort().join(",") : "";
+        const cacheMode = `${mode}|dates=${detectDates}|hint=${hintKey}`;
         let cacheKey;
         if (cache) {
             cacheKey = this.cache.key(normalizedText, countriesTuple, cacheMode);
@@ -57,7 +64,10 @@ export class EuRedact {
             if (cached !== null)
                 return cached;
         }
-        let detections = this.engine.detect(normalizedText, countries);
+        const { detections: rawDetections, evidence, scores, } = this.engine.detectWithEvidence(normalizedText, countries, countryHint, context !== null ? context.evidence() : null);
+        if (context !== null)
+            context.add(evidence, chunkOffset);
+        let detections = rawDetections;
         if (offsetMapping !== null) {
             detections = detections.map(d => ({
                 ...d,
@@ -77,11 +87,18 @@ export class EuRedact {
                 : `[${det.entityType}]`;
             redacted = redacted.slice(0, det.start) + replacement + redacted.slice(det.end);
         }
+        // Report the inference so it can be audited. Spans in `evidence` are
+        // offsets into the normalised text, matching `detections`.
+        const inferredCountries = [...weightsToRanking(scores).entries()]
+            .sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]));
         const result = {
             redactedText: redacted,
             detections,
             source: "rules",
             degraded: false,
+            inferredCountries,
+            evidence,
+            detectionMode: countries && countries.length ? "declared" : "inferred",
         };
         if (cache && cacheKey) {
             this.cache.put(cacheKey, result);
@@ -89,8 +106,9 @@ export class EuRedact {
         return result;
     }
     redactBatch(texts, options = {}) {
-        const countries = options.countries ?? null;
-        this.engine.loadCountries(countries ? countries.map(c => c.toUpperCase()) : null);
+        // Generation is country-blind, so every country's patterns are loaded
+        // regardless of what the caller declared.
+        this.engine.loadCountries(null);
         return texts.map(text => this.redact(text, options));
     }
 }

package/dist/esm/types.d.ts

@@ -49,6 +49,24 @@ export declare enum DetectionSource {
     RULES = "rules",
     CLOUD = "cloud"
 }
+/**
+ * One reason to believe a document belongs to a country.
+ *
+ * Emitted by entities that carry their country in the string — an IBAN's
+ * country code, a `+CC` dialling prefix, a VAT prefix, an email ccTLD.
+ * Auditable by design: every inference traces back to the span and the source
+ * that produced it.
+ */
+export interface CountryEvidence {
+    /** ISO 3166-1 alpha-2. */
+    country: string;
+    /** Which signal produced this — "ibanPrefix", "e164Prefix", ... */
+    source: string;
+    /** Natural-log odds. Derived from the corpus, not hand-tuned. */
+    logOdds: number;
+    /** Offsets into the text that was scanned. */
+    span: [number, number];
+}
 export interface Detection {
     entityType: EntityType | string;
     start: number;
@@ -57,12 +75,47 @@ export interface Detection {
     source: DetectionSource;
     country: string | null;
     confidence: string;
+    /**
+     * How strongly the document supports {@link Detection.country}, in [0, 1].
+     *
+     * 0 when no country was attributed or nothing corroborated it — which is the
+     * signal that an attribution rests on a checksum alone.
+     */
+    countryConfidence?: number;
+    /**
+     * True when attributed to a country the caller did not declare.
+     *
+     * `countries` narrows *scoring*, never detection. An entity from outside the
+     * declared set is emitted and flagged, not dropped: callers assert context,
+     * they do not suppress evidence.
+     */
+    outOfScope?: boolean;
 }
 export interface RedactResult {
     redactedText: string;
     detections: Detection[];
     source: string;
     degraded: boolean;
+    /**
+     * Countries this document appears to belong to, strongest first, as
+     * `[country, confidence]` with confidence in [0, 1].
+     *
+     * Inferred from entities that carry their country in the string. Reported so
+     * the inference can be audited; it influences which national scheme is
+     * attributed to an ambiguous value, never which spans are found.
+     */
+    inferredCountries: Array<[string, number]>;
+    /** Every signal behind {@link RedactResult.inferredCountries}, with the span
+     *  that produced it. The audit trail for a country attribution. */
+    evidence: CountryEvidence[];
+    /**
+     * `"declared"` when the caller passed `countries`, `"inferred"` when the
+     * country was worked out from the content.
+     *
+     * Named `detectionMode` rather than `mode` because `redact({ mode })`
+     * already means the tier selector, which {@link RedactResult.source} reports.
+     */
+    detectionMode: string;
 }
 export interface PatternDef {
     entityType: EntityType | string;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "euredact",
-  "version": "0.3.0",
+  "version": "0.3.2",
   "description": "European PII redaction SDK -- fast, zero-dependency, rule-based detection and redaction of personal data across 31 European countries",
   "license": "Apache-2.0",
   "type": "module",
@@ -53,12 +53,14 @@
     "build:cjs": "tsc -p tsconfig.cjs.json",
     "postbuild:cjs": "echo '{\"type\":\"commonjs\"}' > dist/cjs/package.json",
     "build": "rm -rf dist && npm run build:esm && npm run build:cjs",
-    "test": "npm run test:spec",
+    "test": "npm run test:spec && npm run test:inference && npm run test:conformance",
     "benchmark": "node --import tsx src/__tests__/benchmark.ts",
     "prepublishOnly": "npm run build",
     "test:spec": "node --import tsx src/__tests__/spec.ts",
     "test:eval": "node --import tsx src/__tests__/run.ts",
-    "test:eval:full": "node --max-old-space-size=8192 --import tsx src/__tests__/evalFull.ts"
+    "test:eval:full": "node --max-old-space-size=8192 --import tsx src/__tests__/evalFull.ts",
+    "test:conformance": "node --import tsx src/__tests__/conformance.ts",
+    "test:inference": "node --import tsx src/__tests__/inference.ts"
   },
   "devDependencies": {
     "typescript": "^5.4.0",