euredact 0.4.0 → 0.5.1

package/dist/cjs/sdk.js

@@ -33,13 +33,278 @@ var __importStar = (this && this.__importStar) || (function () {
     };
 })();
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.EuRedact = void 0;
+exports.EuRedact = exports.TokenMapper = exports.TOKEN_SUFFIX_LENGTH = exports.TOKEN_ALPHABET = void 0;
+exports.applyReplacements = applyReplacements;
+exports.restore = restore;
+const node_crypto_1 = require("node:crypto");
 const cache_js_1 = require("./cache.js");
 const normalizer_js_1 = require("./normalizer.js");
 const engine_js_1 = require("./rules/engine.js");
 const evidence_js_1 = require("./rules/evidence.js");
 const types_js_1 = require("./types.js");
 const DATE_TYPES = new Set([types_js_1.EntityType.DOB, types_js_1.EntityType.DATE_OF_DEATH]);
+/**
+ * Splice a label over every detection and return the masked text.
+ *
+ * `detections` must be sorted by `(start, -end)`.
+ *
+ * Labels are resolved right-to-left because the referential mapper numbers
+ * each entity type in call order, and that order is part of the output
+ * contract. The string itself is then assembled in a single forward pass:
+ * rebuilding it per detection copied the whole document each time, which is
+ * O(document x detections) — 1.68 s of pure copying on a 1 MB document with
+ * 15,000 detections, versus 2 ms here.
+ *
+ * Spans from the rule engine are deduplicated and non-overlapping. Spans from
+ * elsewhere (the cloud service) are only sorted, so a span may start behind
+ * the cursor. Its uncovered tail is still masked rather than dropped: dropping
+ * the span would leave those characters in the clear, and splicing it whole
+ * would corrupt the label already emitted over its head.
+ */
+function applyReplacements(text, detections, labelFor) {
+    const kept = [];
+    let pos = 0;
+    for (const det of detections) {
+        const start = Math.max(det.start, pos);
+        if (det.end <= start)
+            continue;
+        kept.push([det, start, det.end]);
+        pos = det.end;
+    }
+    const labels = new Array(kept.length);
+    for (let i = kept.length - 1; i >= 0; i--) {
+        const [det, start, end] = kept[i];
+        labels[i] = labelFor(det, text.slice(start, end));
+    }
+    const parts = [];
+    pos = 0;
+    for (let i = 0; i < kept.length; i++) {
+        const [, start, end] = kept[i];
+        parts.push(text.slice(pos, start), labels[i]);
+        pos = end;
+    }
+    parts.push(text.slice(pos));
+    return parts.join("");
+}
+/**
+ * Characters a token suffix is drawn from. No vowels, so a suffix never spells
+ * a word; no 0/1/I/O, so it survives being read back by a person; no
+ * underscore, so the type prefix stays unambiguous.
+ */
+exports.TOKEN_ALPHABET = "ABCDEFGHJKLMNPQRSTUVWXYZ23456789";
+exports.TOKEN_SUFFIX_LENGTH = 4;
+const TOKEN_MAX_DRAWS = 100;
+function escapeRegExp(s) {
+    return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}
+/**
+ * Maps each distinct PII value in one call to a reversible token.
+ *
+ * A token is `TYPE_XXXX`: the entity type, then `TOKEN_SUFFIX_LENGTH` random
+ * characters from `TOKEN_ALPHABET`. The same value gets the same token within
+ * the call, so relationships survive; across calls it gets a different one, so
+ * two tokenized documents never reveal a shared value.
+ *
+ * Nothing is kept beyond the call. The mapping goes to the caller in
+ * `RedactResult.tokens`, and only the caller can turn it back into text with
+ * `restore()`.
+ *
+ * Tokens are also kept clear of any token-shaped string already in the
+ * document. A caller that redacts an LLM's reply to a tokenized prompt has
+ * exactly such a document, and a fresh token colliding with an old one would
+ * make `restore()` put the wrong value back.
+ */
+class TokenMapper {
+    byValue = new Map();
+    taken = new Set();
+    /** Token -> original value, for `RedactResult.tokens`. */
+    tokens = {};
+    constructor(text, detections) {
+        const types = [...new Set(detections.map(d => String(d.entityType)))].sort();
+        if (types.length === 0)
+            return;
+        const shaped = new RegExp(`(?:${types.map(escapeRegExp).join("|")})_[${exports.TOKEN_ALPHABET}]{${exports.TOKEN_SUFFIX_LENGTH}}`, "g");
+        for (const m of text.matchAll(shaped))
+            this.taken.add(m[0]);
+    }
+    /** Return the token for `value`, minting one on first sight. */
+    getToken = (det, value) => {
+        let token = this.byValue.get(value);
+        if (token === undefined) {
+            const prefix = `${det.entityType}_`;
+            let draws = 0;
+            do {
+                if (draws++ >= TOKEN_MAX_DRAWS) {
+                    throw new Error(`could not mint a unique token with prefix ${JSON.stringify(prefix)} after ${TOKEN_MAX_DRAWS} draws`);
+                }
+                let suffix = "";
+                for (let i = 0; i < exports.TOKEN_SUFFIX_LENGTH; i++) {
+                    suffix += exports.TOKEN_ALPHABET[(0, node_crypto_1.randomInt)(exports.TOKEN_ALPHABET.length)];
+                }
+                token = prefix + suffix;
+            } while (this.taken.has(token));
+            this.taken.add(token);
+            this.byValue.set(value, token);
+            this.tokens[token] = value;
+        }
+        return token;
+    };
+}
+exports.TokenMapper = TokenMapper;
+/**
+ * Put the original values back into `text`.
+ *
+ * `tokens` is `RedactResult.tokens` from the `redact(text, { tokenize: true })`
+ * call that produced the text this one derives from — typically the reply an
+ * LLM wrote to the tokenized prompt. Every occurrence of every token is
+ * replaced; a token an LLM glued to other characters (`EMAIL_K7Q2s`) is still
+ * restored, since leaving a token behind is the worse failure.
+ */
+function restore(text, tokens) {
+    const keys = Object.keys(tokens);
+    if (keys.length === 0)
+        return text;
+    // Longest first so a token that is a prefix of another (custom pattern names
+    // allow it) cannot be matched short. The replacement is a function so `$&`
+    // and friends in the original values are literal.
+    keys.sort((a, b) => b.length - a.length);
+    const pattern = new RegExp(keys.map(escapeRegExp).join("|"), "g");
+    return text.replace(pattern, m => tokens[m]);
+}
+/**
+ * Types whose separators are presentational: the same identifier stays the
+ * same value however it is spaced, hyphenated or dotted, so an allowlisted
+ * IBAN is exempted whether the document writes NL91ABNA0417164300 or
+ * NL91 ABNA 0417 1643 00 (issue rules-engine#15).
+ *
+ * Free-text types are deliberately absent: folding separators in an address
+ * would make `jan.devries@acme.be` exempt `jandevries@acme.be`, a different
+ * mailbox at most providers.
+ */
+const SEPARATOR_INSENSITIVE = new Set([
+    types_js_1.EntityType.BANK_ACCOUNT, types_js_1.EntityType.BIC, types_js_1.EntityType.CREDIT_CARD,
+    types_js_1.EntityType.PHONE, types_js_1.EntityType.VAT, types_js_1.EntityType.NATIONAL_ID, types_js_1.EntityType.SSN,
+    types_js_1.EntityType.TAX_ID, types_js_1.EntityType.PASSPORT, types_js_1.EntityType.DRIVERS_LICENSE,
+    types_js_1.EntityType.RESIDENCE_PERMIT, types_js_1.EntityType.HEALTH_INSURANCE,
+    types_js_1.EntityType.CHAMBER_OF_COMMERCE, types_js_1.EntityType.IMEI, types_js_1.EntityType.VIN,
+]);
+/** Types carrying a domain an owner may want exempted wholesale. */
+const DOMAIN_BEARING = new Set([types_js_1.EntityType.EMAIL, types_js_1.EntityType.URL]);
+const SEPARATORS = /[\s.\-/()]+/g;
+/**
+ * Reject a bare string where a list of allowlisted values is expected.
+ *
+ * A string is iterable, so `allowlist: "ACME NV"` would become the
+ * one-character entries "A", "C", ... — none of which is a whole detection, so
+ * nothing is exempted and the caller's own name is redacted after all.
+ */
+function checkAllowlistArg(value, param = "allowlist") {
+    if (typeof value === "string") {
+        throw new TypeError(`${param} must be an array of values, not a bare string. ` +
+            `Pass ${param}: ["${value}"] rather than ${param}: "${value}".`);
+    }
+}
+/**
+ * The form an allowlist entry and a detected value are compared in. NFC
+ * because the rule engine matches on NFC-normalised text while the document
+ * may be NFD; lower-cased because an org name in a heading is the same org.
+ */
+function allowlistKey(value) {
+    return value.normalize("NFC").trim().toLowerCase();
+}
+/** `allowlistKey` with presentational separators removed. */
+function foldedKey(value) {
+    return allowlistKey(value).replace(SEPARATORS, "");
+}
+/** Comparison key -> the entry as the caller wrote it (for reporting). */
+function normalizeAllowlist(values) {
+    checkAllowlistArg(values);
+    const out = new Map();
+    for (const v of values ?? []) {
+        if (!v || !v.trim())
+            continue;
+        if (!out.has(allowlistKey(v)))
+            out.set(allowlistKey(v), v);
+        if (!out.has(foldedKey(v)))
+            out.set(foldedKey(v), v);
+    }
+    return out;
+}
+/** Domain key -> the entry as written. A leading `@` or `.` is ignored. */
+function normalizeDomains(values) {
+    checkAllowlistArg(values, "allowlistDomains");
+    const out = new Map();
+    for (const v of values ?? []) {
+        if (!v || !v.trim())
+            continue;
+        const k = allowlistKey(v).replace(/^[@.]+/, "");
+        if (k && !out.has(k))
+            out.set(k, v);
+    }
+    return out;
+}
+/** The domain an exemption rule would apply to, or null. */
+function domainOf(entityType, value) {
+    if (!DOMAIN_BEARING.has(entityType))
+        return null;
+    let v = allowlistKey(value);
+    if (v.includes("@"))
+        return v.slice(v.lastIndexOf("@") + 1).replace(/^[<[(]+|[>\])(),;:"']+$/g, "") || null;
+    v = v.replace(/^[a-z][a-z0-9+.-]*:\/\//, "");
+    return v.split("/")[0].split(":")[0].replace(/^[<[(]+|[>\])(),;:"']+$/g, "") || null;
+}
+/** The [rule, kind] exempting this detection, or null. */
+function matchingRule(det, slice, allowed, domains) {
+    for (const candidate of [slice, det.text]) {
+        if (!candidate)
+            continue;
+        const exact = allowed.get(allowlistKey(candidate));
+        if (exact !== undefined)
+            return [exact, "value"];
+        if (SEPARATOR_INSENSITIVE.has(det.entityType)) {
+            const folded = allowed.get(foldedKey(candidate));
+            if (folded !== undefined)
+                return [folded, "value"];
+        }
+        const host = domainOf(det.entityType, candidate);
+        if (host) {
+            for (const [owned, entry] of domains) {
+                if (host === owned || host.endsWith("." + owned))
+                    return [entry, "domain"];
+            }
+        }
+    }
+    return null;
+}
+/**
+ * Split detections into those to redact and those the caller exempted.
+ *
+ * Whole span only: `euredact.be` as a *value* does not exempt
+ * `joren@euredact.be`; that is what `allowlistDomains` is for.
+ */
+function applyAllowlist(text, detections, allowed, domains) {
+    if (allowed.size === 0 && domains.size === 0)
+        return [detections, []];
+    const kept = [];
+    const exempted = [];
+    for (const d of detections) {
+        const hit = matchingRule(d, text.slice(d.start, d.end), allowed, domains);
+        if (hit === null)
+            kept.push(d);
+        else
+            exempted.push({
+                entityType: d.entityType, start: d.start, end: d.end,
+                text: text.slice(d.start, d.end), rule: hit[0], ruleKind: hit[1],
+            });
+    }
+    return [kept, exempted];
+}
+/** Two label schemes for the same spans cannot both apply. */
+function checkLabelOptions(options) {
+    if (options.tokenize && options.referentialIntegrity) {
+        throw new Error("tokenize and referentialIntegrity are two label schemes for the same spans; pass one of them");
+    }
+}
 /** Entry count at which a one-time warning is emitted. */
 const MAPPING_WARN_THRESHOLD = 100_000;
 /**
@@ -81,40 +346,6 @@ class ReferentialMapper {
     }
 }
 const DEFAULT_MAX_INPUT_LENGTH = 10_485_760; // ~10 MB of text
-/**
- * Send a document to the cloud tier.
- *
- * Options the service cannot honour throw rather than being ignored. Silently
- * dropping one would mean returning a result that does not match what was
- * asked for — which, for anything that changes which spans come back, is
- * under-redaction wearing a plausible face.
- */
-async function redactViaCloud(text, options) {
-    const { CloudClient } = await Promise.resolve().then(() => __importStar(require("./cloud/client.js")));
-    const countries = options.countries ?? null;
-    if (!countries || countries.length !== 1) {
-        throw new Error('cloud mode needs exactly one country, e.g. { countries: ["BE"] }. The ' +
-            "model is trained and evaluated per country, so a multi-country request " +
-            "has no defined behaviour.");
-    }
-    if (options.countryHint) {
-        throw new Error("countryHint is not supported in cloud mode");
-    }
-    if (options.context || options.chunkOffset) {
-        throw new Error("context/chunkOffset are not supported in cloud mode: the model has never " +
-            "seen a chunk boundary, so the service rejects oversized input rather " +
-            "than splitting it");
-    }
-    if (options.referentialIntegrity) {
-        throw new Error("referentialIntegrity is not supported in cloud mode");
-    }
-    // detectDates is deliberately NOT forwarded. The service always runs its
-    // rules engine with dates on, because that is what the model was trained
-    // against; the caller's value cannot change that. It is the one ignored
-    // option that is safe to ignore — it can only cause MORE to be detected,
-    // never less, so it cannot produce under-redaction.
-    return new CloudClient().redact(text, { country: countries[0] });
-}
 /**
  * Reject a bare string where a list of country codes is expected.
  *
@@ -146,8 +377,18 @@ class EuRedact {
     cache = new cache_js_1.ResultCache();
     referentialMapper = new ReferentialMapper();
     maxInputLength;
+    allowlist;
+    allowlistDomains;
+    /**
+     * @param options.maxInputLength Longest document `redact` accepts, in characters.
+     * @param options.allowlist Values never to redact, for every call on this
+     *   instance — an organisation's own name, its own addresses. Merged with
+     *   the per-call `allowlist`. Matched whole, case-insensitively.
+     */
     constructor(options) {
         this.maxInputLength = options?.maxInputLength ?? DEFAULT_MAX_INPUT_LENGTH;
+        this.allowlist = normalizeAllowlist(options?.allowlist);
+        this.allowlistDomains = normalizeDomains(options?.allowlistDomains);
     }
     addCustomPattern(name, pattern) {
         this.engine.addCustomPattern(name, pattern);
@@ -159,9 +400,101 @@ class EuRedact {
         this.cache.clear();
         this.referentialMapper.clear();
     }
+    /** The instance allowlist merged with a call's, in comparison form. */
+    allowedFor(options) {
+        const allowed = new Map(this.allowlist);
+        for (const [k, v] of normalizeAllowlist(options.allowlist))
+            allowed.set(k, v);
+        return allowed;
+    }
+    /** The instance domain list merged with a call's. */
+    domainsFor(options) {
+        const domains = new Map(this.allowlistDomains);
+        for (const [k, v] of normalizeDomains(options.allowlistDomains))
+            domains.set(k, v);
+        return domains;
+    }
+    /** The label function for one call, given its output options. */
+    labelFor(referentialIntegrity, tokenMapper) {
+        if (tokenMapper !== null)
+            return tokenMapper.getToken;
+        if (referentialIntegrity) {
+            const mapper = this.referentialMapper;
+            return (det) => mapper.getLabel(det.text, det.entityType);
+        }
+        return (det) => `[${det.entityType}]`;
+    }
+    /**
+     * Send a document to the cloud tier.
+     *
+     * Options the service cannot honour throw rather than being ignored. Silently
+     * dropping one would mean returning a result that does not match what was
+     * asked for — which, for anything that changes which spans come back, is
+     * under-redaction wearing a plausible face.
+     */
+    async redactViaCloud(text, options) {
+        const { CloudClient } = await Promise.resolve().then(() => __importStar(require("./cloud/client.js")));
+        const countries = options.countries ?? null;
+        if (!countries || countries.length !== 1) {
+            throw new Error('cloud mode needs exactly one country, e.g. { countries: ["BE"] }. The ' +
+                "model is trained and evaluated per country, so a multi-country request " +
+                "has no defined behaviour.");
+        }
+        if (options.countryHint) {
+            throw new Error("countryHint is not supported in cloud mode");
+        }
+        if (options.context || options.chunkOffset) {
+            throw new Error("context/chunkOffset are not supported in cloud mode: the model has never " +
+                "seen a chunk boundary, so the service rejects oversized input rather " +
+                "than splitting it");
+        }
+        if (options.referentialIntegrity) {
+            throw new Error("referentialIntegrity is not supported in cloud mode");
+        }
+        // detectDates is deliberately NOT forwarded. The service always runs its
+        // rules engine with dates on, because that is what the model was trained
+        // against; the caller's value cannot change that. It is the one ignored
+        // option that is safe to ignore — it can only cause MORE to be detected,
+        // never less, so it cannot produce under-redaction.
+        const allowed = this.allowedFor(options);
+        const domains = this.domainsFor(options);
+        const result = await new CloudClient().redact(text, { country: countries[0] });
+        if (options.tokenize || allowed.size > 0 || domains.size > 0) {
+            await this.remaskCloudResult(result, text, { tokenize: options.tokenize ?? false, allowed, domains });
+        }
+        return result;
+    }
+    /**
+     * Rebuild the masked text from the service's spans, in place.
+     *
+     * The service masks with bracketed labels only. Any other output option is
+     * applied here, from its spans: the service builds its `redacted_text` from
+     * exactly the spans it returns (entities it reports but cannot place are
+     * listed separately and never applied), so nothing it masked is lost by
+     * masking again from the same spans.
+     *
+     * The service reports offsets in code points and this SDK slices UTF-16
+     * units, so the slice check is what stands between an emoji in the document
+     * and a rebuilt text with the wrong characters masked.
+     */
+    async remaskCloudResult(result, text, opts) {
+        const { CloudError } = await Promise.resolve().then(() => __importStar(require("./cloud/errors.js")));
+        for (const det of result.detections) {
+            if (text.slice(det.start, det.end) !== det.text) {
+                throw new CloudError("span offsets do not match the document; cannot apply tokenize/allowlist locally");
+            }
+        }
+        [result.detections, result.exempted] = applyAllowlist(text, result.detections, opts.allowed, opts.domains);
+        const tokenMapper = opts.tokenize ? new TokenMapper(text, result.detections) : null;
+        result.redactedText = applyReplacements(text, result.detections, this.labelFor(false, tokenMapper));
+        result.tokens = tokenMapper ? tokenMapper.tokens : {};
+    }
     redact(text, options = {}) {
         checkCountryArg(options.countries, "countries");
         checkCountryArg(options.countryHint, "countryHint");
+        checkLabelOptions(options);
+        const allowed = this.allowedFor(options);
+        const domains = this.domainsFor(options);
         const requestedMode = options.mode ?? "rules";
         if (requestedMode === "cloud") {
             // Cannot be served synchronously: the cloud tier is a network call, and
@@ -180,7 +513,7 @@ class EuRedact {
                 `(${this.maxInputLength.toLocaleString()} chars). Split the input or ` +
                 `increase maxInputLength when constructing EuRedact.`);
         }
-        const { countries = null, countryHint = null, context = null, chunkOffset = 0, mode = "rules", referentialIntegrity = false, detectDates = false, } = options;
+        const { countries = null, countryHint = null, context = null, chunkOffset = 0, mode = "rules", referentialIntegrity = false, tokenize = false, detectDates = false, } = options;
         // A context makes the result depend on evidence from other chunks, so the
         // text no longer identifies the result. Caching is disabled rather than
         // keyed on the context, whose contents change as chunks arrive.
@@ -191,7 +524,14 @@ class EuRedact {
             : ["ALL"];
         // countryHint changes attribution, so it must key the cache too.
         const hintKey = countryHint ? countryHint.map(c => c.toUpperCase()).sort().join(",") : "";
-        const cacheMode = `${mode}|dates=${detectDates}|hint=${hintKey}`;
+        // referentialIntegrity changes the labels, not the spans, so a cached
+        // bracketed result is the wrong answer for a labelled call on the same
+        // text — it has to key the cache too.
+        // JSON rather than a joined string: an entry may itself contain the
+        // separator, and two different lists must never share a key.
+        const allowKey = (allowed.size ? JSON.stringify([...allowed.keys()].sort()) : "")
+            + (domains.size ? "|dom=" + JSON.stringify([...domains.keys()].sort()) : "");
+        const cacheMode = `${mode}|dates=${detectDates}|hint=${hintKey}|ri=${referentialIntegrity}|tok=${tokenize}|allow=${allowKey}`;
         let cacheKey;
         if (cache) {
             cacheKey = this.cache.key(normalizedText, countriesTuple, cacheMode);
@@ -213,33 +553,11 @@ class EuRedact {
         if (!detectDates) {
             detections = detections.filter(d => !DATE_TYPES.has(d.entityType));
         }
+        let exempted;
+        [detections, exempted] = applyAllowlist(text, detections, allowed, domains);
         detections.sort((a, b) => a.start - b.start || b.end - a.end);
-        // Labels are resolved right-to-left because the referential mapper numbers
-        // each entity type in call order, and that order is part of the output
-        // contract. The string itself is then assembled in a single forward pass:
-        // rebuilding it per detection copied the whole document each time, which is
-        // O(document x detections) — 1.68 s of pure copying on a 1 MB document with
-        // 15,000 detections, versus 2 ms here.
-        const replacements = new Array(detections.length);
-        for (let i = detections.length - 1; i >= 0; i--) {
-            const det = detections[i];
-            replacements[i] = referentialIntegrity
-                ? this.referentialMapper.getLabel(det.text, det.entityType)
-                : `[${det.entityType}]`;
-        }
-        const parts = [];
-        let pos = 0;
-        for (let i = 0; i < detections.length; i++) {
-            const det = detections[i];
-            // Spans reach here deduplicated and non-overlapping; one starting behind
-            // the cursor would splice into the middle of a label, so drop it.
-            if (det.start < pos)
-                continue;
-            parts.push(text.slice(pos, det.start), replacements[i]);
-            pos = det.end;
-        }
-        parts.push(text.slice(pos));
-        const redacted = parts.join("");
+        const tokenMapper = tokenize ? new TokenMapper(text, detections) : null;
+        const redacted = applyReplacements(text, detections, this.labelFor(referentialIntegrity, tokenMapper));
         // Report the inference so it can be audited. Spans in `evidence` are
         // offsets into the normalised text, matching `detections`.
         const inferredCountries = [...(0, evidence_js_1.weightsToRanking)(scores).entries()]
@@ -252,6 +570,8 @@ class EuRedact {
             inferredCountries,
             evidence,
             detectionMode: countries && countries.length ? "declared" : "inferred",
+            tokens: tokenMapper ? tokenMapper.tokens : {},
+            exempted,
         };
         if (cache && cacheKey) {
             this.cache.put(cacheKey, result);
@@ -268,7 +588,8 @@ class EuRedact {
         const mode = options.mode ?? "rules";
         if (mode !== "cloud")
             return this.redact(text, options);
-        return redactViaCloud(text, options);
+        checkLabelOptions(options);
+        return this.redactViaCloud(text, options);
     }
     redactBatch(texts, options = {}) {
         // Generation is country-blind, so every country's patterns are loaded

package/dist/esm/cache.d.ts

@@ -1,4 +1,6 @@
 import type { RedactResult } from "./types.js";
+/** Approximate retained size of a cached result, in characters. */
+export declare function resultChars(result: RedactResult): number;
 export declare class ResultCache {
     private maxsize;
     private maxChars;

package/dist/esm/cache.js

@@ -4,10 +4,14 @@ import { createHash } from "node:crypto";
 // gigabyte of PII-bearing text held live. ~16M characters is a few tens of MB.
 const DEFAULT_MAX_CHARS = 16_000_000;
 /** Approximate retained size of a cached result, in characters. */
-function resultChars(result) {
+export function resultChars(result) {
     let chars = result.redactedText.length;
     for (const d of result.detections)
         chars += d.text.length;
+    for (const [token, value] of Object.entries(result.tokens))
+        chars += token.length + value.length;
+    for (const e of result.exempted)
+        chars += e.text.length + e.rule.length;
     return chars;
 }
 export class ResultCache {

package/dist/esm/cloud/client.js

@@ -71,11 +71,33 @@ function toEntityType(raw) {
     const known = Object.values(EntityType).includes(canonical);
     return known ? canonical : canonical;
 }
+/**
+ * Map the service's offsets, which count code points, onto this SDK's, which
+ * count UTF-16 units.
+ *
+ * The two agree until the first character outside the BMP — an emoji, say —
+ * after which every service offset is short by one unit per such character.
+ * Returns the identity when the text has none, which is the common case and
+ * costs one regex test.
+ */
+function codePointOffsets(text) {
+    if (!/[\uD800-\uDBFF]/.test(text))
+        return cp => cp;
+    const units = [];
+    let i = 0;
+    for (const ch of text) {
+        units.push(i);
+        i += ch.length;
+    }
+    units.push(i);
+    return cp => units[Math.min(cp, units.length - 1)];
+}
 function toResult(payload, text) {
+    const offset = codePointOffsets(text);
     const detections = (payload.entities ?? []).map(span => ({
         entityType: toEntityType(span.type),
-        start: span.start,
-        end: span.end,
+        start: offset(span.start),
+        end: offset(span.end),
         text: span.text,
         source: span.source === "model" ? DetectionSource.CLOUD : DetectionSource.RULES,
         country: null,
@@ -90,6 +112,8 @@ function toResult(payload, text) {
         inferredCountries: [],
         evidence: [],
         detectionMode: "declared",
+        tokens: {},
+        exempted: [],
     };
 }
 export class CloudClient {

package/dist/esm/index.d.ts

@@ -1,7 +1,7 @@
 export { EntityType, DetectionSource, LEGACY_TYPE_ALIASES, canonicalType } from "./types.js";
-export type { Detection, RedactResult, PatternDef, CountryConfig, CountryEvidence } from "./types.js";
+export type { Detection, Exemption, RedactResult, PatternDef, CountryConfig, CountryEvidence } from "./types.js";
 export { DocumentContext } from "./rules/context.js";
-export { EuRedact, type RedactOptions } from "./sdk.js";
+export { EuRedact, restore, type RedactOptions } from "./sdk.js";
 export { COUNTRY_CONFIGS } from "./rules/countries/index.js";
 export { setBicRegistry, getBicRegistry, SEED_BIC6_PREFIXES, type BicRegistryProvider } from "./rules/bicRegistry.js";
 export { configure, getConfig, CloudClient, CloudError, NotConfiguredError, QuotaExceededError, TooLargeError, type CloudConfig, type ConfigureOptions, type CloudRedactOptions, } from "./cloud/index.js";

package/dist/esm/index.js

@@ -1,6 +1,6 @@
 export { EntityType, DetectionSource, LEGACY_TYPE_ALIASES, canonicalType } from "./types.js";
 export { DocumentContext } from "./rules/context.js";
-export { EuRedact } from "./sdk.js";
+export { EuRedact, restore } from "./sdk.js";
 export { COUNTRY_CONFIGS } from "./rules/countries/index.js";
 export { setBicRegistry, getBicRegistry, SEED_BIC6_PREFIXES } from "./rules/bicRegistry.js";
 export { configure, getConfig, CloudClient, CloudError, NotConfiguredError, QuotaExceededError, TooLargeError, } from "./cloud/index.js";