euredact 0.3.1 → 0.3.3

package/dist/cjs/sdk.js

@@ -4,6 +4,7 @@ exports.EuRedact = void 0;
 const cache_js_1 = require("./cache.js");
 const normalizer_js_1 = require("./normalizer.js");
 const engine_js_1 = require("./rules/engine.js");
+const evidence_js_1 = require("./rules/evidence.js");
 const types_js_1 = require("./types.js");
 const DATE_TYPES = new Set([types_js_1.EntityType.DOB, types_js_1.EntityType.DATE_OF_DEATH]);
 class ReferentialMapper {
@@ -23,6 +24,32 @@ class ReferentialMapper {
     }
 }
 const DEFAULT_MAX_INPUT_LENGTH = 10_485_760; // ~10 MB of text
+/**
+ * Reject a bare string where a list of country codes is expected.
+ *
+ * `countries: "NL"` is iterable, so it walks into the codes "N" and "L".
+ * Neither resolves, so nothing is declared — and every detection that *does*
+ * carry a country is then flagged `outOfScope`. A caller following the
+ * documented pattern of filtering on that field silently keeps none of them,
+ * while `redactedText` still looks perfectly correct. The failure direction is
+ * "no PII here", from a one-character typo.
+ *
+ * This already threw in Node, but by accident and only once something happened
+ * to call `.map` on it — deep in the call, with a message naming neither the
+ * argument nor the fix.
+ *
+ * A wrong *code* is data and only warns; raising there would invite callers to
+ * wrap redaction in try/catch and skip it. A wrong *type* is a programming
+ * error with no correct interpretation to fall back on.
+ */
+function checkCountryArg(value, param) {
+    if (typeof value === "string") {
+        throw new TypeError(`${param} must be an array of country codes, not a bare string. ` +
+            `Pass ${param}: ["${value}"] rather than ${param}: "${value}" — a string ` +
+            `is iterated character by character, which silently declares nothing and ` +
+            `flags every detection outOfScope.`);
+    }
+}
 class EuRedact {
     engine = new engine_js_1.RuleEngine();
     cache = new cache_js_1.ResultCache();
@@ -42,17 +69,25 @@ class EuRedact {
         this.referentialMapper.clear();
     }
     redact(text, options = {}) {
+        checkCountryArg(options.countries, "countries");
+        checkCountryArg(options.countryHint, "countryHint");
         if (text.length > this.maxInputLength) {
             throw new Error(`Input text length (${text.length.toLocaleString()} chars) exceeds the maximum ` +
                 `(${this.maxInputLength.toLocaleString()} chars). Split the input or ` +
                 `increase maxInputLength when constructing EuRedact.`);
         }
-        const { countries = null, mode = "rules", referentialIntegrity = false, detectDates = false, cache = true, } = options;
+        const { countries = null, countryHint = null, context = null, chunkOffset = 0, mode = "rules", referentialIntegrity = false, detectDates = false, } = options;
+        // A context makes the result depend on evidence from other chunks, so the
+        // text no longer identifies the result. Caching is disabled rather than
+        // keyed on the context, whose contents change as chunks arrive.
+        const cache = context !== null ? false : (options.cache ?? true);
         const [normalizedText, offsetMapping] = (0, normalizer_js_1.normalize)(text);
         const countriesTuple = countries
             ? countries.map(c => c.toUpperCase()).sort()
             : ["ALL"];
-        const cacheMode = `${mode}|dates=${detectDates}`;
+        // countryHint changes attribution, so it must key the cache too.
+        const hintKey = countryHint ? countryHint.map(c => c.toUpperCase()).sort().join(",") : "";
+        const cacheMode = `${mode}|dates=${detectDates}|hint=${hintKey}`;
         let cacheKey;
         if (cache) {
             cacheKey = this.cache.key(normalizedText, countriesTuple, cacheMode);
@@ -60,7 +95,10 @@ class EuRedact {
             if (cached !== null)
                 return cached;
         }
-        let detections = this.engine.detect(normalizedText, countries);
+        const { detections: rawDetections, evidence, scores, } = this.engine.detectWithEvidence(normalizedText, countries, countryHint, context !== null ? context.evidence : null);
+        if (context !== null)
+            context.add(evidence, chunkOffset);
+        let detections = rawDetections;
         if (offsetMapping !== null) {
             detections = detections.map(d => ({
                 ...d,
@@ -80,11 +118,18 @@ class EuRedact {
                 : `[${det.entityType}]`;
             redacted = redacted.slice(0, det.start) + replacement + redacted.slice(det.end);
         }
+        // Report the inference so it can be audited. Spans in `evidence` are
+        // offsets into the normalised text, matching `detections`.
+        const inferredCountries = [...(0, evidence_js_1.weightsToRanking)(scores).entries()]
+            .sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]));
         const result = {
             redactedText: redacted,
             detections,
             source: "rules",
             degraded: false,
+            inferredCountries,
+            evidence,
+            detectionMode: countries && countries.length ? "declared" : "inferred",
         };
         if (cache && cacheKey) {
             this.cache.put(cacheKey, result);
@@ -92,8 +137,9 @@ class EuRedact {
         return result;
     }
     redactBatch(texts, options = {}) {
-        const countries = options.countries ?? null;
-        this.engine.loadCountries(countries ? countries.map(c => c.toUpperCase()) : null);
+        // Generation is country-blind, so every country's patterns are loaded
+        // regardless of what the caller declared.
+        this.engine.loadCountries(null);
         return texts.map(text => this.redact(text, options));
     }
 }

package/dist/esm/index.d.ts

@@ -1,5 +1,6 @@
 export { EntityType, DetectionSource, LEGACY_TYPE_ALIASES } from "./types.js";
-export type { Detection, RedactResult, PatternDef, CountryConfig } from "./types.js";
+export type { Detection, RedactResult, PatternDef, CountryConfig, CountryEvidence } from "./types.js";
+export { DocumentContext } from "./rules/context.js";
 export { EuRedact, type RedactOptions } from "./sdk.js";
 export { COUNTRY_CONFIGS } from "./rules/countries/index.js";
 export { setBicRegistry, getBicRegistry, SEED_BIC6_PREFIXES, type BicRegistryProvider } from "./rules/bicRegistry.js";

package/dist/esm/index.js

@@ -1,4 +1,5 @@
 export { EntityType, DetectionSource, LEGACY_TYPE_ALIASES } from "./types.js";
+export { DocumentContext } from "./rules/context.js";
 export { EuRedact } from "./sdk.js";
 export { COUNTRY_CONFIGS } from "./rules/countries/index.js";
 export { setBicRegistry, getBicRegistry, SEED_BIC6_PREFIXES } from "./rules/bicRegistry.js";

package/dist/esm/rules/context.d.ts

@@ -0,0 +1,43 @@
+/**
+ * Country evidence shared across the chunks of one document.
+ *
+ * A long document is usually redacted in pieces. Each piece is scanned
+ * independently, so each infers its own country — and a chunk that happens to
+ * contain no IBAN, no `+CC` number and no ccTLD infers nothing at all, even
+ * when the first page identified the document beyond doubt:
+ *
+ *     page 1   "Factuur — IBAN NL91ABNA0417164300, info@jansen.nl"
+ *     page 7   "Telefoon 0612345678"        <- Dutch mobile, or Danish CPR?
+ *
+ * Scanned alone, page 7 has nothing to go on. Sharing a context across the two
+ * carries page 1's evidence forward, and the number resolves as a Dutch phone.
+ *
+ * Spans are rebased by `chunkOffset` on the way in, so evidence collected from
+ * a chunk points at offsets in the whole document rather than the chunk.
+ *
+ * A context influences **scoring only**. It can change which national scheme is
+ * attributed to a value. It can never change which spans are found, so a
+ * context carried over from the wrong document cannot cause a miss.
+ */
+import type { CountryEvidence } from "../types.js";
+export declare class DocumentContext {
+    private seen;
+    private items;
+    /** Record evidence from a chunk starting at `chunkOffset`. */
+    add(evidence: CountryEvidence[], chunkOffset?: number): void;
+    /**
+     * Everything recorded so far, spans rebased to the whole document.
+     *
+     * A getter, to match {@link DocumentContext.size}. These were a method and a
+     * getter on the same class, which is a trip hazard rather than a style
+     * question: `ctx.size()` and `ctx.evidence` both *succeed* and return
+     * something useless — a number is not callable, and a function object is
+     * truthy — so the mistake surfaces later as an empty result rather than as
+     * an error. Both are property reads now, and the wrong form throws.
+     *
+     * Returns a fresh copy each call, so mutating it cannot corrupt the context.
+     * (The Python SDK spells this `evidence()`, following its own idiom.)
+     */
+    get evidence(): CountryEvidence[];
+    get size(): number;
+}

package/dist/esm/rules/context.js

@@ -0,0 +1,58 @@
+/**
+ * Country evidence shared across the chunks of one document.
+ *
+ * A long document is usually redacted in pieces. Each piece is scanned
+ * independently, so each infers its own country — and a chunk that happens to
+ * contain no IBAN, no `+CC` number and no ccTLD infers nothing at all, even
+ * when the first page identified the document beyond doubt:
+ *
+ *     page 1   "Factuur — IBAN NL91ABNA0417164300, info@jansen.nl"
+ *     page 7   "Telefoon 0612345678"        <- Dutch mobile, or Danish CPR?
+ *
+ * Scanned alone, page 7 has nothing to go on. Sharing a context across the two
+ * carries page 1's evidence forward, and the number resolves as a Dutch phone.
+ *
+ * Spans are rebased by `chunkOffset` on the way in, so evidence collected from
+ * a chunk points at offsets in the whole document rather than the chunk.
+ *
+ * A context influences **scoring only**. It can change which national scheme is
+ * attributed to a value. It can never change which spans are found, so a
+ * context carried over from the wrong document cannot cause a miss.
+ */
+export class DocumentContext {
+    // Deduped on the same key evidence.collect() uses, so re-running a chunk — a
+    // retry, an overlapping window — cannot let one entity vote twice and
+    // outweigh the rest of the document.
+    seen = new Set();
+    items = [];
+    /** Record evidence from a chunk starting at `chunkOffset`. */
+    add(evidence, chunkOffset = 0) {
+        for (const ev of evidence) {
+            const span = [ev.span[0] + chunkOffset, ev.span[1] + chunkOffset];
+            const key = `${span[0]}:${span[1]}:${ev.source}`;
+            if (this.seen.has(key))
+                continue;
+            this.seen.add(key);
+            this.items.push(chunkOffset === 0 ? ev : { ...ev, span });
+        }
+    }
+    /**
+     * Everything recorded so far, spans rebased to the whole document.
+     *
+     * A getter, to match {@link DocumentContext.size}. These were a method and a
+     * getter on the same class, which is a trip hazard rather than a style
+     * question: `ctx.size()` and `ctx.evidence` both *succeed* and return
+     * something useless — a number is not callable, and a function object is
+     * truthy — so the mistake surfaces later as an empty result rather than as
+     * an error. Both are property reads now, and the wrong form throws.
+     *
+     * Returns a fresh copy each call, so mutating it cannot corrupt the context.
+     * (The Python SDK spells this `evidence()`, following its own idiom.)
+     */
+    get evidence() {
+        return [...this.items];
+    }
+    get size() {
+        return this.items.length;
+    }
+}

package/dist/esm/rules/engine.d.ts

@@ -1,4 +1,5 @@
-import { type Detection } from "../types.js";
+import { type CountryEvidence, type Detection } from "../types.js";
+import { type RawMatch } from "./matchers.js";
 /**
  * The country modules use the EU/VAT convention (UK, EL) while the public
  * contract is ISO 3166-1 alpha-2 (GB, GR). Accept both spellings so a caller
@@ -12,11 +13,34 @@ export declare class RuleEngine {
     private loadedCountries;
     addCustomPattern(name: string, pattern: string): void;
     loadCountries(countryCodes: string[] | null): void;
-    detect(text: string, countryCodes: string[] | null): Detection[];
+    /**
+     * Generate candidate matches. Country-blind, by contract.
+     *
+     * This is the only caller of {@link MultiPatternMatcher.scan}, and it takes
+     * no country argument. Which country the caller declared may influence how a
+     * match is *scored*; it may never influence whether the match is *found*.
+     * Before this held, `countries: ["BE"]` made a valid Dutch BSN vanish
+     * entirely because the Dutch patterns were never run.
+     */
+    generateCandidates(text: string): RawMatch[];
+    detect(text: string, countryCodes: string[] | null, countryHint?: string[] | null): Detection[];
+    /**
+     * {@link RuleEngine.detect}, plus the country evidence it reasoned from.
+     *
+     * `countryCodes` is **scope**: it flags detections from elsewhere as
+     * `outOfScope` and, like `countryHint`, contributes a prior. `countryHint` is
+     * **only** a prior — it helps resolve ambiguity without marking anything out
+     * of scope. Neither gates what is looked for.
+     */
+    detectWithEvidence(text: string, countryCodes: string[] | null, countryHint?: string[] | null, priorEvidence?: CountryEvidence[] | null): {
+        detections: Detection[];
+        evidence: CountryEvidence[];
+        scores: Map<string, number>;
+    };
     /**
      * Remove overlapping detections with priority-aware resolution.
-     * Priority: validated (3) > custom (2) > regex-only (1).
-     * Within the same tier, longer span wins.
+     * Priority: validated (3) > custom (2) > regex-only (1) > postal (0) >
+     * demoted (-1). Within a tier, longer span wins, then country evidence.
      */
     private deduplicate;
     getLoadedCountries(): Set<string>;