euredact 0.7.0 → 0.7.1

package/dist/esm/sdk.js

@@ -3,7 +3,7 @@ import { ResultCache } from "./cache.js";
 import { normalize, mapOffsetToOriginal } from "./normalizer.js";
 import { RuleEngine } from "./rules/engine.js";
 import { weightsToRanking } from "./rules/evidence.js";
-import { EntityType } from "./types.js";
+import { EntityType, canonicalType } from "./types.js";
 const DATE_TYPES = new Set([EntityType.DOB, EntityType.DATE_OF_DEATH]);
 /**
  * Splice a label over every detection and return the masked text.
@@ -353,6 +353,70 @@ function applyAllowlist(text, detections, allowed, domains) {
     }
     return [kept, exempted];
 }
+const KNOWN_TYPES = new Set(Object.values(EntityType));
+/**
+ * The entity types a caller asked to leave in the text, by canonical name.
+ *
+ * Names are matched case-insensitively and legacy names resolve (`IBAN` is
+ * `BANK_ACCOUNT`, `CREDENTIAL` is `SECRET`). A custom pattern's name counts
+ * once it is registered. Anything else throws: a typo in a type meant to be
+ * switched off would otherwise switch nothing off (rules-engine#125).
+ */
+export function normalizeExcludeTypes(values, customNames = new Set(), param = "excludeTypes") {
+    const out = new Set();
+    if (values === null || values === undefined)
+        return out;
+    checkAllowlistArg(values, param);
+    for (const v of values) {
+        if (typeof v !== "string") {
+            throw new TypeError(`${param} entries must be type names, not ${typeof v}`);
+        }
+        if (customNames.has(v)) {
+            out.add(v);
+            continue;
+        }
+        const name = canonicalType(v);
+        if (!KNOWN_TYPES.has(name)) {
+            throw new Error(`unknown entity type ${JSON.stringify(v)} in ${param}. Known types: ` +
+                [...KNOWN_TYPES].sort().join(", ") +
+                (customNames.size ? `; custom patterns: ${[...customNames].sort().join(", ")}` : ""));
+        }
+        out.add(name);
+    }
+    return out;
+}
+/**
+ * Split off the detections of the types the caller excluded. Applied to each
+ * span's final type, after ranking: dropping an excluded type's candidates
+ * earlier would let the next-best type claim the same span.
+ */
+function applyTypeExclusion(text, detections, excluded) {
+    if (excluded.size === 0)
+        return [detections, []];
+    const kept = [];
+    const exempted = [];
+    for (const d of detections) {
+        const name = String(d.entityType);
+        if (excluded.has(name)) {
+            exempted.push({
+                entityType: d.entityType, start: d.start, end: d.end,
+                text: text.slice(d.start, d.end), rule: name, ruleKind: "type",
+            });
+        }
+        else
+            kept.push(d);
+    }
+    return [kept, exempted];
+}
+/** Allowlist first, then excluded types; every exemption in text order. */
+function exempt(text, detections, allowed, domains, excluded) {
+    const [afterValues, byValue] = applyAllowlist(text, detections, allowed, domains);
+    const [kept, byType] = applyTypeExclusion(text, afterValues, excluded);
+    const exempted = [...byValue, ...byType];
+    if (byValue.length && byType.length)
+        exempted.sort((a, b) => a.start - b.start || b.end - a.end);
+    return [kept, exempted];
+}
 /** Two label schemes for the same spans cannot both apply. */
 function checkLabelOptions(options) {
     if (options.tokenize && options.referentialIntegrity) {
@@ -433,19 +497,26 @@ export class EuRedact {
     maxInputLength;
     allowlist;
     allowlistDomains;
+    customNames = new Set();
+    excludeTypes;
     /**
      * @param options.maxInputLength Longest document `redact` accepts, in characters.
      * @param options.allowlist Values never to redact, for every call on this
      *   instance — an organisation's own name, its own addresses. Merged with
      *   the per-call `allowlist`. Matched whole, case-insensitively.
+     * @param options.excludeTypes Entity types never reported or masked, for
+     *   every call on this instance, e.g. `["JOB_TITLE", "FINANCIAL_AMOUNT"]`.
+     *   Merged with the per-call value. Unknown names throw.
      */
     constructor(options) {
         this.maxInputLength = options?.maxInputLength ?? DEFAULT_MAX_INPUT_LENGTH;
         this.allowlist = normalizeAllowlist(options?.allowlist);
         this.allowlistDomains = normalizeDomains(options?.allowlistDomains);
+        this.excludeTypes = normalizeExcludeTypes(options?.excludeTypes);
     }
     addCustomPattern(name, pattern) {
         this.engine.addCustomPattern(name, pattern);
+        this.customNames.add(name);
         this.cache.clear();
     }
     /** Clear the result cache and referential integrity mappings.
@@ -461,6 +532,13 @@ export class EuRedact {
             allowed.set(k, v);
         return allowed;
     }
+    /** The instance's excluded types merged with a call's. */
+    excludedFor(options) {
+        const excluded = new Set(this.excludeTypes);
+        for (const t of normalizeExcludeTypes(options.excludeTypes, this.customNames))
+            excluded.add(t);
+        return excluded;
+    }
     /** The instance domain list merged with a call's. */
     domainsFor(options) {
         const domains = new Map(this.allowlistDomains);
@@ -517,14 +595,16 @@ export class EuRedact {
         }
         const allowed = this.allowedFor(options);
         const domains = this.domainsFor(options);
+        const excluded = this.excludedFor(options);
         const client = new CloudClient();
         // Dates are always on, whatever detectDates says: that is the rules output
         // the model was trained against, and a date of birth the rules can place
         // has no reason to travel. It is the one ignored option that is safe to
         // ignore — it can only cause MORE to be detected, never less.
         //
-        // No allowlist and no tokens on this pass. An exemption says what the
-        // caller wants back, not what may leave; both are applied below, after the
+        // No allowlist, no excluded types and no tokens on this pass. An exemption
+        // says what the caller wants back, not what may leave: an excluded EMAIL is
+        // still masked in the request. All three are applied below, after the
         // response, to the merged spans.
         const local = this.redactRules(text, { countries, detectDates: true, cache: options.cache }, new Map(), new Map());
         const [masked, labels] = maskForCloud(text, local.detections);
@@ -534,7 +614,7 @@ export class EuRedact {
             country: countries[0], idempotencyKey: options.idempotencyKey,
         });
         // A new array throughout: `local` may be the cached rules result.
-        const [detections, exempted] = applyAllowlist(text, [...local.detections, ...ontoOriginal(remote.detections, masked, text, labels, CloudError)], allowed, domains);
+        const [detections, exempted] = exempt(text, [...local.detections, ...ontoOriginal(remote.detections, masked, text, labels, CloudError)], allowed, domains, excluded);
         detections.sort((a, b) => a.start - b.start || b.end - a.end);
         const tokenMapper = options.tokenize ? new TokenMapper(text, detections) : null;
         return {
@@ -553,6 +633,7 @@ export class EuRedact {
         checkLabelOptions(options);
         const allowed = this.allowedFor(options);
         const domains = this.domainsFor(options);
+        const excluded = this.excludedFor(options);
         const requestedMode = options.mode ?? "rules";
         if (requestedMode === "cloud") {
             // Cannot be served synchronously: the cloud tier is a network call, and
@@ -569,10 +650,10 @@ export class EuRedact {
         if (options.idempotencyKey !== undefined) {
             throw new Error('idempotencyKey applies to mode: "cloud" only: a rules-only redaction makes no request to deduplicate');
         }
-        return this.redactRules(text, options, allowed, domains);
+        return this.redactRules(text, options, allowed, domains, excluded);
     }
     /** The local pipeline: normalise, detect, exempt, mask. */
-    redactRules(text, options, allowed, domains) {
+    redactRules(text, options, allowed, domains, excluded = new Set()) {
         if (text.length > this.maxInputLength) {
             throw new Error(`Input text length (${text.length.toLocaleString()} chars) exceeds the maximum ` +
                 `(${this.maxInputLength.toLocaleString()} chars). Split the input or ` +
@@ -595,7 +676,8 @@ export class EuRedact {
         // JSON rather than a joined string: an entry may itself contain the
         // separator, and two different lists must never share a key.
         const allowKey = (allowed.size ? JSON.stringify([...allowed.keys()].sort()) : "")
-            + (domains.size ? "|dom=" + JSON.stringify([...domains.keys()].sort()) : "");
+            + (domains.size ? "|dom=" + JSON.stringify([...domains.keys()].sort()) : "")
+            + (excluded.size ? "|excl=" + JSON.stringify([...excluded].sort()) : "");
         const cacheMode = `rules|dates=${detectDates}|hint=${hintKey}|ri=${referentialIntegrity}|tok=${tokenize}|allow=${allowKey}`;
         let cacheKey;
         if (cache) {
@@ -619,7 +701,7 @@ export class EuRedact {
             detections = detections.filter(d => !DATE_TYPES.has(d.entityType));
         }
         let exempted;
-        [detections, exempted] = applyAllowlist(text, detections, allowed, domains);
+        [detections, exempted] = exempt(text, detections, allowed, domains, excluded);
         detections.sort((a, b) => a.start - b.start || b.end - a.end);
         const tokenMapper = tokenize ? new TokenMapper(text, detections) : null;
         const redacted = applyReplacements(text, detections, this.labelFor(referentialIntegrity, tokenMapper));

package/dist/esm/types.d.ts

@@ -141,7 +141,7 @@ export interface Detection {
     outOfScope?: boolean;
 }
 /**
- * A detection the allowlist kept out of the redacted output.
+ * A detection the allowlist or `excludeTypes` kept out of the redacted output.
  *
  * An exemption is a deliberate decision to leave a direct identifier in a
  * document that otherwise claims to be redacted, so it is reported rather than
@@ -153,10 +153,12 @@ export interface Exemption {
     start: number;
     end: number;
     text: string;
-    /** The allowlist entry that matched, as the caller wrote it. */
+    /** The allowlist entry that matched, as the caller wrote it; for an
+     *  excluded type, the type's canonical name. */
     rule: string;
-    /** `"value"` for the literal allowlist, `"domain"` for allowlistDomains. */
-    ruleKind: "value" | "domain";
+    /** `"value"` for the literal allowlist, `"domain"` for allowlistDomains,
+     *  `"type"` for excludeTypes (rules-engine#125). */
+    ruleKind: "value" | "domain" | "type";
 }
 /**
  * One reason a cloud request cost what it did. `code` is one of the service's

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "euredact",
-  "version": "0.7.0",
+  "version": "0.7.1",
   "description": "European PII redaction SDK -- fast, zero-dependency, rule-based detection and redaction of personal data across 31 European countries",
   "homepage": "https://euredact.dev",
   "license": "Apache-2.0",
@@ -59,7 +59,7 @@
     "build:cjs": "tsc -p tsconfig.cjs.json",
     "postbuild:cjs": "echo '{\"type\":\"commonjs\"}' > dist/cjs/package.json",
     "build": "rm -rf dist && npm run build:esm && npm run build:cjs",
-    "test": "npm run test:spec && npm run test:inference && npm run test:cloud && npm run test:cloud-api && npm run test:tokenize && npm run test:allowlist && npm run test:eval-scoring && npm run test:platform && npm run test:batches && npm run test:conformance",
+    "test": "npm run test:spec && npm run test:inference && npm run test:cloud && npm run test:cloud-api && npm run test:tokenize && npm run test:allowlist && npm run test:exclude-types && npm run test:eval-scoring && npm run test:platform && npm run test:batches && npm run test:conformance",
     "benchmark": "node --import tsx src/__tests__/benchmark.ts",
     "prepublishOnly": "npm run build",
     "test:spec": "node --import tsx src/__tests__/spec.ts",
@@ -72,6 +72,7 @@
     "test:cloud-api": "node --import tsx src/__tests__/cloudApi.ts",
     "test:tokenize": "node --import tsx src/__tests__/tokenize.ts",
     "test:allowlist": "node --import tsx src/__tests__/allowlist.ts",
+    "test:exclude-types": "node --import tsx src/__tests__/excludeTypes.ts",
     "test:eval-scoring": "node --import tsx src/__tests__/evalScoring.ts",
     "test:platform": "node --import tsx src/__tests__/platform.ts",
     "test:batches": "node --import tsx src/__tests__/batches.ts"