euredact 0.4.0 → 0.5.0

package/dist/esm/sdk.js

@@ -1,9 +1,271 @@
+import { randomInt } from "node:crypto";
 import { ResultCache } from "./cache.js";
 import { normalize, mapOffsetToOriginal } from "./normalizer.js";
 import { RuleEngine } from "./rules/engine.js";
 import { weightsToRanking } from "./rules/evidence.js";
 import { EntityType } from "./types.js";
 const DATE_TYPES = new Set([EntityType.DOB, EntityType.DATE_OF_DEATH]);
+/**
+ * Splice a label over every detection and return the masked text.
+ *
+ * `detections` must be sorted by `(start, -end)`.
+ *
+ * Labels are resolved right-to-left because the referential mapper numbers
+ * each entity type in call order, and that order is part of the output
+ * contract. The string itself is then assembled in a single forward pass:
+ * rebuilding it per detection copied the whole document each time, which is
+ * O(document x detections) — 1.68 s of pure copying on a 1 MB document with
+ * 15,000 detections, versus 2 ms here.
+ *
+ * Spans from the rule engine are deduplicated and non-overlapping. Spans from
+ * elsewhere (the cloud service) are only sorted, so a span may start behind
+ * the cursor. Its uncovered tail is still masked rather than dropped: dropping
+ * the span would leave those characters in the clear, and splicing it whole
+ * would corrupt the label already emitted over its head.
+ */
+export function applyReplacements(text, detections, labelFor) {
+    const kept = [];
+    let pos = 0;
+    for (const det of detections) {
+        const start = Math.max(det.start, pos);
+        if (det.end <= start)
+            continue;
+        kept.push([det, start, det.end]);
+        pos = det.end;
+    }
+    const labels = new Array(kept.length);
+    for (let i = kept.length - 1; i >= 0; i--) {
+        const [det, start, end] = kept[i];
+        labels[i] = labelFor(det, text.slice(start, end));
+    }
+    const parts = [];
+    pos = 0;
+    for (let i = 0; i < kept.length; i++) {
+        const [, start, end] = kept[i];
+        parts.push(text.slice(pos, start), labels[i]);
+        pos = end;
+    }
+    parts.push(text.slice(pos));
+    return parts.join("");
+}
+/**
+ * Characters a token suffix is drawn from. No vowels, so a suffix never spells
+ * a word; no 0/1/I/O, so it survives being read back by a person; no
+ * underscore, so the type prefix stays unambiguous.
+ */
+export const TOKEN_ALPHABET = "ABCDEFGHJKLMNPQRSTUVWXYZ23456789";
+export const TOKEN_SUFFIX_LENGTH = 4;
+const TOKEN_MAX_DRAWS = 100;
+function escapeRegExp(s) {
+    return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}
+/**
+ * Maps each distinct PII value in one call to a reversible token.
+ *
+ * A token is `TYPE_XXXX`: the entity type, then `TOKEN_SUFFIX_LENGTH` random
+ * characters from `TOKEN_ALPHABET`. The same value gets the same token within
+ * the call, so relationships survive; across calls it gets a different one, so
+ * two tokenized documents never reveal a shared value.
+ *
+ * Nothing is kept beyond the call. The mapping goes to the caller in
+ * `RedactResult.tokens`, and only the caller can turn it back into text with
+ * `restore()`.
+ *
+ * Tokens are also kept clear of any token-shaped string already in the
+ * document. A caller that redacts an LLM's reply to a tokenized prompt has
+ * exactly such a document, and a fresh token colliding with an old one would
+ * make `restore()` put the wrong value back.
+ */
+export class TokenMapper {
+    byValue = new Map();
+    taken = new Set();
+    /** Token -> original value, for `RedactResult.tokens`. */
+    tokens = {};
+    constructor(text, detections) {
+        const types = [...new Set(detections.map(d => String(d.entityType)))].sort();
+        if (types.length === 0)
+            return;
+        const shaped = new RegExp(`(?:${types.map(escapeRegExp).join("|")})_[${TOKEN_ALPHABET}]{${TOKEN_SUFFIX_LENGTH}}`, "g");
+        for (const m of text.matchAll(shaped))
+            this.taken.add(m[0]);
+    }
+    /** Return the token for `value`, minting one on first sight. */
+    getToken = (det, value) => {
+        let token = this.byValue.get(value);
+        if (token === undefined) {
+            const prefix = `${det.entityType}_`;
+            let draws = 0;
+            do {
+                if (draws++ >= TOKEN_MAX_DRAWS) {
+                    throw new Error(`could not mint a unique token with prefix ${JSON.stringify(prefix)} after ${TOKEN_MAX_DRAWS} draws`);
+                }
+                let suffix = "";
+                for (let i = 0; i < TOKEN_SUFFIX_LENGTH; i++) {
+                    suffix += TOKEN_ALPHABET[randomInt(TOKEN_ALPHABET.length)];
+                }
+                token = prefix + suffix;
+            } while (this.taken.has(token));
+            this.taken.add(token);
+            this.byValue.set(value, token);
+            this.tokens[token] = value;
+        }
+        return token;
+    };
+}
+/**
+ * Put the original values back into `text`.
+ *
+ * `tokens` is `RedactResult.tokens` from the `redact(text, { tokenize: true })`
+ * call that produced the text this one derives from — typically the reply an
+ * LLM wrote to the tokenized prompt. Every occurrence of every token is
+ * replaced; a token an LLM glued to other characters (`EMAIL_K7Q2s`) is still
+ * restored, since leaving a token behind is the worse failure.
+ */
+export function restore(text, tokens) {
+    const keys = Object.keys(tokens);
+    if (keys.length === 0)
+        return text;
+    // Longest first so a token that is a prefix of another (custom pattern names
+    // allow it) cannot be matched short. The replacement is a function so `$&`
+    // and friends in the original values are literal.
+    keys.sort((a, b) => b.length - a.length);
+    const pattern = new RegExp(keys.map(escapeRegExp).join("|"), "g");
+    return text.replace(pattern, m => tokens[m]);
+}
+/**
+ * Types whose separators are presentational: the same identifier stays the
+ * same value however it is spaced, hyphenated or dotted, so an allowlisted
+ * IBAN is exempted whether the document writes NL91ABNA0417164300 or
+ * NL91 ABNA 0417 1643 00 (issue rules-engine#15).
+ *
+ * Free-text types are deliberately absent: folding separators in an address
+ * would make `jan.devries@acme.be` exempt `jandevries@acme.be`, a different
+ * mailbox at most providers.
+ */
+const SEPARATOR_INSENSITIVE = new Set([
+    EntityType.BANK_ACCOUNT, EntityType.BIC, EntityType.CREDIT_CARD,
+    EntityType.PHONE, EntityType.VAT, EntityType.NATIONAL_ID, EntityType.SSN,
+    EntityType.TAX_ID, EntityType.PASSPORT, EntityType.DRIVERS_LICENSE,
+    EntityType.RESIDENCE_PERMIT, EntityType.HEALTH_INSURANCE,
+    EntityType.CHAMBER_OF_COMMERCE, EntityType.IMEI, EntityType.VIN,
+]);
+/** Types carrying a domain an owner may want exempted wholesale. */
+const DOMAIN_BEARING = new Set([EntityType.EMAIL, EntityType.URL]);
+const SEPARATORS = /[\s.\-/()]+/g;
+/**
+ * Reject a bare string where a list of allowlisted values is expected.
+ *
+ * A string is iterable, so `allowlist: "ACME NV"` would become the
+ * one-character entries "A", "C", ... — none of which is a whole detection, so
+ * nothing is exempted and the caller's own name is redacted after all.
+ */
+function checkAllowlistArg(value, param = "allowlist") {
+    if (typeof value === "string") {
+        throw new TypeError(`${param} must be an array of values, not a bare string. ` +
+            `Pass ${param}: ["${value}"] rather than ${param}: "${value}".`);
+    }
+}
+/**
+ * The form an allowlist entry and a detected value are compared in. NFC
+ * because the rule engine matches on NFC-normalised text while the document
+ * may be NFD; lower-cased because an org name in a heading is the same org.
+ */
+function allowlistKey(value) {
+    return value.normalize("NFC").trim().toLowerCase();
+}
+/** `allowlistKey` with presentational separators removed. */
+function foldedKey(value) {
+    return allowlistKey(value).replace(SEPARATORS, "");
+}
+/** Comparison key -> the entry as the caller wrote it (for reporting). */
+function normalizeAllowlist(values) {
+    checkAllowlistArg(values);
+    const out = new Map();
+    for (const v of values ?? []) {
+        if (!v || !v.trim())
+            continue;
+        if (!out.has(allowlistKey(v)))
+            out.set(allowlistKey(v), v);
+        if (!out.has(foldedKey(v)))
+            out.set(foldedKey(v), v);
+    }
+    return out;
+}
+/** Domain key -> the entry as written. A leading `@` or `.` is ignored. */
+function normalizeDomains(values) {
+    checkAllowlistArg(values, "allowlistDomains");
+    const out = new Map();
+    for (const v of values ?? []) {
+        if (!v || !v.trim())
+            continue;
+        const k = allowlistKey(v).replace(/^[@.]+/, "");
+        if (k && !out.has(k))
+            out.set(k, v);
+    }
+    return out;
+}
+/** The domain an exemption rule would apply to, or null. */
+function domainOf(entityType, value) {
+    if (!DOMAIN_BEARING.has(entityType))
+        return null;
+    let v = allowlistKey(value);
+    if (v.includes("@"))
+        return v.slice(v.lastIndexOf("@") + 1).replace(/^[<[(]+|[>\])(),;:"']+$/g, "") || null;
+    v = v.replace(/^[a-z][a-z0-9+.-]*:\/\//, "");
+    return v.split("/")[0].split(":")[0].replace(/^[<[(]+|[>\])(),;:"']+$/g, "") || null;
+}
+/** The [rule, kind] exempting this detection, or null. */
+function matchingRule(det, slice, allowed, domains) {
+    for (const candidate of [slice, det.text]) {
+        if (!candidate)
+            continue;
+        const exact = allowed.get(allowlistKey(candidate));
+        if (exact !== undefined)
+            return [exact, "value"];
+        if (SEPARATOR_INSENSITIVE.has(det.entityType)) {
+            const folded = allowed.get(foldedKey(candidate));
+            if (folded !== undefined)
+                return [folded, "value"];
+        }
+        const host = domainOf(det.entityType, candidate);
+        if (host) {
+            for (const [owned, entry] of domains) {
+                if (host === owned || host.endsWith("." + owned))
+                    return [entry, "domain"];
+            }
+        }
+    }
+    return null;
+}
+/**
+ * Split detections into those to redact and those the caller exempted.
+ *
+ * Whole span only: `euredact.be` as a *value* does not exempt
+ * `joren@euredact.be`; that is what `allowlistDomains` is for.
+ */
+function applyAllowlist(text, detections, allowed, domains) {
+    if (allowed.size === 0 && domains.size === 0)
+        return [detections, []];
+    const kept = [];
+    const exempted = [];
+    for (const d of detections) {
+        const hit = matchingRule(d, text.slice(d.start, d.end), allowed, domains);
+        if (hit === null)
+            kept.push(d);
+        else
+            exempted.push({
+                entityType: d.entityType, start: d.start, end: d.end,
+                text: text.slice(d.start, d.end), rule: hit[0], ruleKind: hit[1],
+            });
+    }
+    return [kept, exempted];
+}
+/** Two label schemes for the same spans cannot both apply. */
+function checkLabelOptions(options) {
+    if (options.tokenize && options.referentialIntegrity) {
+        throw new Error("tokenize and referentialIntegrity are two label schemes for the same spans; pass one of them");
+    }
+}
 /** Entry count at which a one-time warning is emitted. */
 const MAPPING_WARN_THRESHOLD = 100_000;
 /**
@@ -45,40 +307,6 @@ class ReferentialMapper {
     }
 }
 const DEFAULT_MAX_INPUT_LENGTH = 10_485_760; // ~10 MB of text
-/**
- * Send a document to the cloud tier.
- *
- * Options the service cannot honour throw rather than being ignored. Silently
- * dropping one would mean returning a result that does not match what was
- * asked for — which, for anything that changes which spans come back, is
- * under-redaction wearing a plausible face.
- */
-async function redactViaCloud(text, options) {
-    const { CloudClient } = await import("./cloud/client.js");
-    const countries = options.countries ?? null;
-    if (!countries || countries.length !== 1) {
-        throw new Error('cloud mode needs exactly one country, e.g. { countries: ["BE"] }. The ' +
-            "model is trained and evaluated per country, so a multi-country request " +
-            "has no defined behaviour.");
-    }
-    if (options.countryHint) {
-        throw new Error("countryHint is not supported in cloud mode");
-    }
-    if (options.context || options.chunkOffset) {
-        throw new Error("context/chunkOffset are not supported in cloud mode: the model has never " +
-            "seen a chunk boundary, so the service rejects oversized input rather " +
-            "than splitting it");
-    }
-    if (options.referentialIntegrity) {
-        throw new Error("referentialIntegrity is not supported in cloud mode");
-    }
-    // detectDates is deliberately NOT forwarded. The service always runs its
-    // rules engine with dates on, because that is what the model was trained
-    // against; the caller's value cannot change that. It is the one ignored
-    // option that is safe to ignore — it can only cause MORE to be detected,
-    // never less, so it cannot produce under-redaction.
-    return new CloudClient().redact(text, { country: countries[0] });
-}
 /**
  * Reject a bare string where a list of country codes is expected.
  *
@@ -110,8 +338,18 @@ export class EuRedact {
     cache = new ResultCache();
     referentialMapper = new ReferentialMapper();
     maxInputLength;
+    allowlist;
+    allowlistDomains;
+    /**
+     * @param options.maxInputLength Longest document `redact` accepts, in characters.
+     * @param options.allowlist Values never to redact, for every call on this
+     *   instance — an organisation's own name, its own addresses. Merged with
+     *   the per-call `allowlist`. Matched whole, case-insensitively.
+     */
     constructor(options) {
         this.maxInputLength = options?.maxInputLength ?? DEFAULT_MAX_INPUT_LENGTH;
+        this.allowlist = normalizeAllowlist(options?.allowlist);
+        this.allowlistDomains = normalizeDomains(options?.allowlistDomains);
     }
     addCustomPattern(name, pattern) {
         this.engine.addCustomPattern(name, pattern);
@@ -123,9 +361,101 @@ export class EuRedact {
         this.cache.clear();
         this.referentialMapper.clear();
     }
+    /** The instance allowlist merged with a call's, in comparison form. */
+    allowedFor(options) {
+        const allowed = new Map(this.allowlist);
+        for (const [k, v] of normalizeAllowlist(options.allowlist))
+            allowed.set(k, v);
+        return allowed;
+    }
+    /** The instance domain list merged with a call's. */
+    domainsFor(options) {
+        const domains = new Map(this.allowlistDomains);
+        for (const [k, v] of normalizeDomains(options.allowlistDomains))
+            domains.set(k, v);
+        return domains;
+    }
+    /** The label function for one call, given its output options. */
+    labelFor(referentialIntegrity, tokenMapper) {
+        if (tokenMapper !== null)
+            return tokenMapper.getToken;
+        if (referentialIntegrity) {
+            const mapper = this.referentialMapper;
+            return (det) => mapper.getLabel(det.text, det.entityType);
+        }
+        return (det) => `[${det.entityType}]`;
+    }
+    /**
+     * Send a document to the cloud tier.
+     *
+     * Options the service cannot honour throw rather than being ignored. Silently
+     * dropping one would mean returning a result that does not match what was
+     * asked for — which, for anything that changes which spans come back, is
+     * under-redaction wearing a plausible face.
+     */
+    async redactViaCloud(text, options) {
+        const { CloudClient } = await import("./cloud/client.js");
+        const countries = options.countries ?? null;
+        if (!countries || countries.length !== 1) {
+            throw new Error('cloud mode needs exactly one country, e.g. { countries: ["BE"] }. The ' +
+                "model is trained and evaluated per country, so a multi-country request " +
+                "has no defined behaviour.");
+        }
+        if (options.countryHint) {
+            throw new Error("countryHint is not supported in cloud mode");
+        }
+        if (options.context || options.chunkOffset) {
+            throw new Error("context/chunkOffset are not supported in cloud mode: the model has never " +
+                "seen a chunk boundary, so the service rejects oversized input rather " +
+                "than splitting it");
+        }
+        if (options.referentialIntegrity) {
+            throw new Error("referentialIntegrity is not supported in cloud mode");
+        }
+        // detectDates is deliberately NOT forwarded. The service always runs its
+        // rules engine with dates on, because that is what the model was trained
+        // against; the caller's value cannot change that. It is the one ignored
+        // option that is safe to ignore — it can only cause MORE to be detected,
+        // never less, so it cannot produce under-redaction.
+        const allowed = this.allowedFor(options);
+        const domains = this.domainsFor(options);
+        const result = await new CloudClient().redact(text, { country: countries[0] });
+        if (options.tokenize || allowed.size > 0 || domains.size > 0) {
+            await this.remaskCloudResult(result, text, { tokenize: options.tokenize ?? false, allowed, domains });
+        }
+        return result;
+    }
+    /**
+     * Rebuild the masked text from the service's spans, in place.
+     *
+     * The service masks with bracketed labels only. Any other output option is
+     * applied here, from its spans: the service builds its `redacted_text` from
+     * exactly the spans it returns (entities it reports but cannot place are
+     * listed separately and never applied), so nothing it masked is lost by
+     * masking again from the same spans.
+     *
+     * The service reports offsets in code points and this SDK slices UTF-16
+     * units, so the slice check is what stands between an emoji in the document
+     * and a rebuilt text with the wrong characters masked.
+     */
+    async remaskCloudResult(result, text, opts) {
+        const { CloudError } = await import("./cloud/errors.js");
+        for (const det of result.detections) {
+            if (text.slice(det.start, det.end) !== det.text) {
+                throw new CloudError("span offsets do not match the document; cannot apply tokenize/allowlist locally");
+            }
+        }
+        [result.detections, result.exempted] = applyAllowlist(text, result.detections, opts.allowed, opts.domains);
+        const tokenMapper = opts.tokenize ? new TokenMapper(text, result.detections) : null;
+        result.redactedText = applyReplacements(text, result.detections, this.labelFor(false, tokenMapper));
+        result.tokens = tokenMapper ? tokenMapper.tokens : {};
+    }
     redact(text, options = {}) {
         checkCountryArg(options.countries, "countries");
         checkCountryArg(options.countryHint, "countryHint");
+        checkLabelOptions(options);
+        const allowed = this.allowedFor(options);
+        const domains = this.domainsFor(options);
         const requestedMode = options.mode ?? "rules";
         if (requestedMode === "cloud") {
             // Cannot be served synchronously: the cloud tier is a network call, and
@@ -144,7 +474,7 @@ export class EuRedact {
                 `(${this.maxInputLength.toLocaleString()} chars). Split the input or ` +
                 `increase maxInputLength when constructing EuRedact.`);
         }
-        const { countries = null, countryHint = null, context = null, chunkOffset = 0, mode = "rules", referentialIntegrity = false, detectDates = false, } = options;
+        const { countries = null, countryHint = null, context = null, chunkOffset = 0, mode = "rules", referentialIntegrity = false, tokenize = false, detectDates = false, } = options;
         // A context makes the result depend on evidence from other chunks, so the
         // text no longer identifies the result. Caching is disabled rather than
         // keyed on the context, whose contents change as chunks arrive.
@@ -155,7 +485,14 @@ export class EuRedact {
             : ["ALL"];
         // countryHint changes attribution, so it must key the cache too.
         const hintKey = countryHint ? countryHint.map(c => c.toUpperCase()).sort().join(",") : "";
-        const cacheMode = `${mode}|dates=${detectDates}|hint=${hintKey}`;
+        // referentialIntegrity changes the labels, not the spans, so a cached
+        // bracketed result is the wrong answer for a labelled call on the same
+        // text — it has to key the cache too.
+        // JSON rather than a joined string: an entry may itself contain the
+        // separator, and two different lists must never share a key.
+        const allowKey = (allowed.size ? JSON.stringify([...allowed.keys()].sort()) : "")
+            + (domains.size ? "|dom=" + JSON.stringify([...domains.keys()].sort()) : "");
+        const cacheMode = `${mode}|dates=${detectDates}|hint=${hintKey}|ri=${referentialIntegrity}|tok=${tokenize}|allow=${allowKey}`;
         let cacheKey;
         if (cache) {
             cacheKey = this.cache.key(normalizedText, countriesTuple, cacheMode);
@@ -177,33 +514,11 @@ export class EuRedact {
         if (!detectDates) {
             detections = detections.filter(d => !DATE_TYPES.has(d.entityType));
         }
+        let exempted;
+        [detections, exempted] = applyAllowlist(text, detections, allowed, domains);
         detections.sort((a, b) => a.start - b.start || b.end - a.end);
-        // Labels are resolved right-to-left because the referential mapper numbers
-        // each entity type in call order, and that order is part of the output
-        // contract. The string itself is then assembled in a single forward pass:
-        // rebuilding it per detection copied the whole document each time, which is
-        // O(document x detections) — 1.68 s of pure copying on a 1 MB document with
-        // 15,000 detections, versus 2 ms here.
-        const replacements = new Array(detections.length);
-        for (let i = detections.length - 1; i >= 0; i--) {
-            const det = detections[i];
-            replacements[i] = referentialIntegrity
-                ? this.referentialMapper.getLabel(det.text, det.entityType)
-                : `[${det.entityType}]`;
-        }
-        const parts = [];
-        let pos = 0;
-        for (let i = 0; i < detections.length; i++) {
-            const det = detections[i];
-            // Spans reach here deduplicated and non-overlapping; one starting behind
-            // the cursor would splice into the middle of a label, so drop it.
-            if (det.start < pos)
-                continue;
-            parts.push(text.slice(pos, det.start), replacements[i]);
-            pos = det.end;
-        }
-        parts.push(text.slice(pos));
-        const redacted = parts.join("");
+        const tokenMapper = tokenize ? new TokenMapper(text, detections) : null;
+        const redacted = applyReplacements(text, detections, this.labelFor(referentialIntegrity, tokenMapper));
         // Report the inference so it can be audited. Spans in `evidence` are
         // offsets into the normalised text, matching `detections`.
         const inferredCountries = [...weightsToRanking(scores).entries()]
@@ -216,6 +531,8 @@ export class EuRedact {
             inferredCountries,
             evidence,
             detectionMode: countries && countries.length ? "declared" : "inferred",
+            tokens: tokenMapper ? tokenMapper.tokens : {},
+            exempted,
         };
         if (cache && cacheKey) {
             this.cache.put(cacheKey, result);
@@ -232,7 +549,8 @@ export class EuRedact {
         const mode = options.mode ?? "rules";
         if (mode !== "cloud")
             return this.redact(text, options);
-        return redactViaCloud(text, options);
+        checkLabelOptions(options);
+        return this.redactViaCloud(text, options);
     }
     redactBatch(texts, options = {}) {
         // Generation is country-blind, so every country's patterns are loaded

package/dist/esm/types.d.ts

@@ -133,6 +133,24 @@ export interface Detection {
      */
     outOfScope?: boolean;
 }
+/**
+ * A detection the allowlist kept out of the redacted output.
+ *
+ * An exemption is a deliberate decision to leave a direct identifier in a
+ * document that otherwise claims to be redacted, so it is reported rather than
+ * silently dropped: without this, "never detected" and "detected and kept"
+ * look identical on the result.
+ */
+export interface Exemption {
+    entityType: EntityType | string;
+    start: number;
+    end: number;
+    text: string;
+    /** The allowlist entry that matched, as the caller wrote it. */
+    rule: string;
+    /** `"value"` for the literal allowlist, `"domain"` for allowlistDomains. */
+    ruleKind: "value" | "domain";
+}
 export interface RedactResult {
     redactedText: string;
     detections: Detection[];
@@ -158,6 +176,18 @@ export interface RedactResult {
      * already means the tier selector, which {@link RedactResult.source} reports.
      */
     detectionMode: string;
+    /**
+     * Token -> original text, populated by `redact(text, { tokenize: true })`.
+     *
+     * Hand it to `restore()` with the text that came back from wherever the
+     * redacted text went. Every token is unique to this call: the same value in
+     * another document gets a different token. Treat the mapping as read-only —
+     * a cached result shares it with every caller.
+     */
+    tokens: Record<string, string>;
+    /** Detections the allowlist kept out of the output, with the rule that
+     *  matched. Empty unless an allowlist was in force. */
+    exempted: Exemption[];
 }
 export interface PatternDef {
     entityType: EntityType | string;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "euredact",
-  "version": "0.4.0",
+  "version": "0.5.0",
   "description": "European PII redaction SDK -- fast, zero-dependency, rule-based detection and redaction of personal data across 31 European countries",
   "homepage": "https://euredact.dev",
   "license": "Apache-2.0",
@@ -54,7 +54,7 @@
     "build:cjs": "tsc -p tsconfig.cjs.json",
     "postbuild:cjs": "echo '{\"type\":\"commonjs\"}' > dist/cjs/package.json",
     "build": "rm -rf dist && npm run build:esm && npm run build:cjs",
-    "test": "npm run test:spec && npm run test:inference && npm run test:cloud && npm run test:conformance",
+    "test": "npm run test:spec && npm run test:inference && npm run test:cloud && npm run test:tokenize && npm run test:allowlist && npm run test:eval-scoring && npm run test:conformance",
     "benchmark": "node --import tsx src/__tests__/benchmark.ts",
     "prepublishOnly": "npm run build",
     "test:spec": "node --import tsx src/__tests__/spec.ts",
@@ -63,7 +63,10 @@
     "test:conformance": "node --import tsx src/__tests__/conformance.ts",
     "test:inference": "node --import tsx src/__tests__/inference.ts",
     "dump": "node --import tsx src/__tests__/dump.ts",
-    "test:cloud": "node --import tsx src/__tests__/cloud.ts"
+    "test:cloud": "node --import tsx src/__tests__/cloud.ts",
+    "test:tokenize": "node --import tsx src/__tests__/tokenize.ts",
+    "test:allowlist": "node --import tsx src/__tests__/allowlist.ts",
+    "test:eval-scoring": "node --import tsx src/__tests__/evalScoring.ts"
   },
   "devDependencies": {
     "@types/node": "22.10.2",