euredact 0.3.1 → 0.3.3

package/dist/esm/rules/matchers.d.ts

@@ -7,6 +7,16 @@ export declare class MultiPatternMatcher {
     addPattern(patternDef: PatternDef, countryCode: string): void;
     addCountry(config: CountryConfig): void;
     compile(): void;
+    /**
+     * Scan text against patterns.
+     *
+     * `countryCodes` restricts which patterns run. **The engine never passes
+     * it.** Generation is country-blind by contract — see
+     * `RuleEngine.generateCandidates` and `src/__tests__/inference.ts` — because
+     * gating generation on the caller's declared country caused silent misses.
+     * The parameter is kept for a future opt-in fast mode, which would be the
+     * only caller permitted to trade recall for speed.
+     */
     scan(text: string, countryCodes?: Set<string> | null): RawMatch[];
     validate(match: RawMatch): boolean;
 }

package/dist/esm/rules/matchers.js

@@ -19,6 +19,16 @@ export class MultiPatternMatcher {
     compile() {
         this.compiled = true;
     }
+    /**
+     * Scan text against patterns.
+     *
+     * `countryCodes` restricts which patterns run. **The engine never passes
+     * it.** Generation is country-blind by contract — see
+     * `RuleEngine.generateCandidates` and `src/__tests__/inference.ts` — because
+     * gating generation on the caller's declared country caused silent misses.
+     * The parameter is kept for a future opt-in fast mode, which would be the
+     * only caller permitted to trade recall for speed.
+     */
     scan(text, countryCodes = null) {
         if (!this.compiled)
             this.compile();

package/dist/esm/sdk.d.ts

@@ -1,6 +1,27 @@
+import { DocumentContext } from "./rules/context.js";
 import { type RedactResult } from "./types.js";
 export interface RedactOptions {
+    /**
+     * Scope. Detections attributed elsewhere are flagged `outOfScope`, never
+     * dropped, and this also acts as a prior when resolving which national
+     * scheme owns an ambiguous value. It does **not** gate what is looked for.
+     */
     countries?: string[] | null;
+    /**
+     * A prior only. Helps resolve ambiguity without narrowing scope or flagging
+     * anything out of scope.
+     */
+    countryHint?: string[] | null;
+    /**
+     * Shares country evidence across the chunks of one document, so a chunk
+     * carrying no country signal of its own is still scored against what the
+     * rest of the document showed. Pass the same object for every chunk, with
+     * `chunkOffset` set to where the chunk starts in the whole document.
+     */
+    context?: DocumentContext | null;
+    /** Offset of this chunk within the document. Used only to rebase spans
+     *  recorded in `context`; returned detections are relative to `text`. */
+    chunkOffset?: number;
     mode?: string;
     referentialIntegrity?: boolean;
     detectDates?: boolean;

package/dist/esm/sdk.js

@@ -1,6 +1,7 @@
 import { ResultCache } from "./cache.js";
 import { normalize, mapOffsetToOriginal } from "./normalizer.js";
 import { RuleEngine } from "./rules/engine.js";
+import { weightsToRanking } from "./rules/evidence.js";
 import { EntityType } from "./types.js";
 const DATE_TYPES = new Set([EntityType.DOB, EntityType.DATE_OF_DEATH]);
 class ReferentialMapper {
@@ -20,6 +21,32 @@ class ReferentialMapper {
     }
 }
 const DEFAULT_MAX_INPUT_LENGTH = 10_485_760; // ~10 MB of text
+/**
+ * Reject a bare string where a list of country codes is expected.
+ *
+ * `countries: "NL"` is iterable, so it walks into the codes "N" and "L".
+ * Neither resolves, so nothing is declared — and every detection that *does*
+ * carry a country is then flagged `outOfScope`. A caller following the
+ * documented pattern of filtering on that field silently keeps none of them,
+ * while `redactedText` still looks perfectly correct. The failure direction is
+ * "no PII here", from a one-character typo.
+ *
+ * This already threw in Node, but by accident and only once something happened
+ * to call `.map` on it — deep in the call, with a message naming neither the
+ * argument nor the fix.
+ *
+ * A wrong *code* is data and only warns; raising there would invite callers to
+ * wrap redaction in try/catch and skip it. A wrong *type* is a programming
+ * error with no correct interpretation to fall back on.
+ */
+function checkCountryArg(value, param) {
+    if (typeof value === "string") {
+        throw new TypeError(`${param} must be an array of country codes, not a bare string. ` +
+            `Pass ${param}: ["${value}"] rather than ${param}: "${value}" — a string ` +
+            `is iterated character by character, which silently declares nothing and ` +
+            `flags every detection outOfScope.`);
+    }
+}
 export class EuRedact {
     engine = new RuleEngine();
     cache = new ResultCache();
@@ -39,17 +66,25 @@ export class EuRedact {
         this.referentialMapper.clear();
     }
     redact(text, options = {}) {
+        checkCountryArg(options.countries, "countries");
+        checkCountryArg(options.countryHint, "countryHint");
         if (text.length > this.maxInputLength) {
             throw new Error(`Input text length (${text.length.toLocaleString()} chars) exceeds the maximum ` +
                 `(${this.maxInputLength.toLocaleString()} chars). Split the input or ` +
                 `increase maxInputLength when constructing EuRedact.`);
         }
-        const { countries = null, mode = "rules", referentialIntegrity = false, detectDates = false, cache = true, } = options;
+        const { countries = null, countryHint = null, context = null, chunkOffset = 0, mode = "rules", referentialIntegrity = false, detectDates = false, } = options;
+        // A context makes the result depend on evidence from other chunks, so the
+        // text no longer identifies the result. Caching is disabled rather than
+        // keyed on the context, whose contents change as chunks arrive.
+        const cache = context !== null ? false : (options.cache ?? true);
         const [normalizedText, offsetMapping] = normalize(text);
         const countriesTuple = countries
             ? countries.map(c => c.toUpperCase()).sort()
             : ["ALL"];
-        const cacheMode = `${mode}|dates=${detectDates}`;
+        // countryHint changes attribution, so it must key the cache too.
+        const hintKey = countryHint ? countryHint.map(c => c.toUpperCase()).sort().join(",") : "";
+        const cacheMode = `${mode}|dates=${detectDates}|hint=${hintKey}`;
         let cacheKey;
         if (cache) {
             cacheKey = this.cache.key(normalizedText, countriesTuple, cacheMode);
@@ -57,7 +92,10 @@ export class EuRedact {
             if (cached !== null)
                 return cached;
         }
-        let detections = this.engine.detect(normalizedText, countries);
+        const { detections: rawDetections, evidence, scores, } = this.engine.detectWithEvidence(normalizedText, countries, countryHint, context !== null ? context.evidence : null);
+        if (context !== null)
+            context.add(evidence, chunkOffset);
+        let detections = rawDetections;
         if (offsetMapping !== null) {
             detections = detections.map(d => ({
                 ...d,
@@ -77,11 +115,18 @@ export class EuRedact {
                 : `[${det.entityType}]`;
             redacted = redacted.slice(0, det.start) + replacement + redacted.slice(det.end);
         }
+        // Report the inference so it can be audited. Spans in `evidence` are
+        // offsets into the normalised text, matching `detections`.
+        const inferredCountries = [...weightsToRanking(scores).entries()]
+            .sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]));
         const result = {
             redactedText: redacted,
             detections,
             source: "rules",
             degraded: false,
+            inferredCountries,
+            evidence,
+            detectionMode: countries && countries.length ? "declared" : "inferred",
         };
         if (cache && cacheKey) {
             this.cache.put(cacheKey, result);
@@ -89,8 +134,9 @@ export class EuRedact {
         return result;
     }
     redactBatch(texts, options = {}) {
-        const countries = options.countries ?? null;
-        this.engine.loadCountries(countries ? countries.map(c => c.toUpperCase()) : null);
+        // Generation is country-blind, so every country's patterns are loaded
+        // regardless of what the caller declared.
+        this.engine.loadCountries(null);
         return texts.map(text => this.redact(text, options));
     }
 }

package/dist/esm/types.d.ts

@@ -49,6 +49,24 @@ export declare enum DetectionSource {
     RULES = "rules",
     CLOUD = "cloud"
 }
+/**
+ * One reason to believe a document belongs to a country.
+ *
+ * Emitted by entities that carry their country in the string — an IBAN's
+ * country code, a `+CC` dialling prefix, a VAT prefix, an email ccTLD.
+ * Auditable by design: every inference traces back to the span and the source
+ * that produced it.
+ */
+export interface CountryEvidence {
+    /** ISO 3166-1 alpha-2. */
+    country: string;
+    /** Which signal produced this — "ibanPrefix", "e164Prefix", ... */
+    source: string;
+    /** Natural-log odds. Derived from the corpus, not hand-tuned. */
+    logOdds: number;
+    /** Offsets into the text that was scanned. */
+    span: [number, number];
+}
 export interface Detection {
     entityType: EntityType | string;
     start: number;
@@ -57,12 +75,47 @@ export interface Detection {
     source: DetectionSource;
     country: string | null;
     confidence: string;
+    /**
+     * How strongly the document supports {@link Detection.country}, in [0, 1].
+     *
+     * 0 when no country was attributed or nothing corroborated it — which is the
+     * signal that an attribution rests on a checksum alone.
+     */
+    countryConfidence?: number;
+    /**
+     * True when attributed to a country the caller did not declare.
+     *
+     * `countries` narrows *scoring*, never detection. An entity from outside the
+     * declared set is emitted and flagged, not dropped: callers assert context,
+     * they do not suppress evidence.
+     */
+    outOfScope?: boolean;
 }
 export interface RedactResult {
     redactedText: string;
     detections: Detection[];
     source: string;
     degraded: boolean;
+    /**
+     * Countries this document appears to belong to, strongest first, as
+     * `[country, confidence]` with confidence in [0, 1].
+     *
+     * Inferred from entities that carry their country in the string. Reported so
+     * the inference can be audited; it influences which national scheme is
+     * attributed to an ambiguous value, never which spans are found.
+     */
+    inferredCountries: Array<[string, number]>;
+    /** Every signal behind {@link RedactResult.inferredCountries}, with the span
+     *  that produced it. The audit trail for a country attribution. */
+    evidence: CountryEvidence[];
+    /**
+     * `"declared"` when the caller passed `countries`, `"inferred"` when the
+     * country was worked out from the content.
+     *
+     * Named `detectionMode` rather than `mode` because `redact({ mode })`
+     * already means the tier selector, which {@link RedactResult.source} reports.
+     */
+    detectionMode: string;
 }
 export interface PatternDef {
     entityType: EntityType | string;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "euredact",
-  "version": "0.3.1",
+  "version": "0.3.3",
   "description": "European PII redaction SDK -- fast, zero-dependency, rule-based detection and redaction of personal data across 31 European countries",
   "license": "Apache-2.0",
   "type": "module",
@@ -53,13 +53,15 @@
     "build:cjs": "tsc -p tsconfig.cjs.json",
     "postbuild:cjs": "echo '{\"type\":\"commonjs\"}' > dist/cjs/package.json",
     "build": "rm -rf dist && npm run build:esm && npm run build:cjs",
-    "test": "npm run test:spec && npm run test:conformance",
+    "test": "npm run test:spec && npm run test:inference && npm run test:conformance",
     "benchmark": "node --import tsx src/__tests__/benchmark.ts",
     "prepublishOnly": "npm run build",
     "test:spec": "node --import tsx src/__tests__/spec.ts",
     "test:eval": "node --import tsx src/__tests__/run.ts",
     "test:eval:full": "node --max-old-space-size=8192 --import tsx src/__tests__/evalFull.ts",
-    "test:conformance": "node --import tsx src/__tests__/conformance.ts"
+    "test:conformance": "node --import tsx src/__tests__/conformance.ts",
+    "test:inference": "node --import tsx src/__tests__/inference.ts",
+    "dump": "node --import tsx src/__tests__/dump.ts"
   },
   "devDependencies": {
     "typescript": "^5.4.0",