euredact 0.4.0 → 0.5.0

package/dist/esm/rules/countries/index.js

@@ -96,7 +96,14 @@ const SHARED = {
         // is Unicode-aware) matched local parts like "petras_ž@..." while this one
         // did not. \p{L}\p{N} with the `u` flag restores parity; the leading guard
         // is a Unicode-safe word boundary.
-        p(EntityType.EMAIL, String.raw `(?<![\p{L}\p{N}._%+\-])[\p{L}\p{N}._%+\-]+@(?:[\p{L}\p{N}\-]+\.)+\p{L}{2,}\b`),
+        // The apostrophe is allowed only between word characters: it belongs in a
+        // local part, but putting it in the class would let the match absorb a
+        // quote from the surrounding text, while leaving it out made
+        // `johno'neill@outlook.ie` mask as `johno'[EMAIL]` (issue rules-engine#10).
+        // The second lookbehind refuses to start a match after `<word-char>'`,
+        // which is a match already in progress; without it every apostrophe was a
+        // fresh start offset and the scan went quadratic. Mirrors Python.
+        p(EntityType.EMAIL, String.raw `(?<![\p{L}\p{N}._%+\-])(?<![\p{L}\p{N}._%+\-]')[\p{L}\p{N}._%+\-]+(?:'[\p{L}\p{N}._%+\-]+)*@(?:[\p{L}\p{N}\-]+\.)+\p{L}{2,}\b`),
         // IBAN (any country) — structure + mod-97 only. Country-specific IBAN
         // patterns still exist and win on identical spans; this one guarantees an
         // IBAN is never missed just because its country was not requested.
@@ -106,7 +113,11 @@ const SHARED = {
         // "+43 664 8213 907" was invisible to the Austrian pattern, which expects
         // an unbroken subscriber number. A leading "+" is self-identifying, so
         // this is deliberately not country-gated.
-        p(EntityType.PHONE, String.raw `\+\d{1,3}(?:[\s\-/]?\(?\d{1,4}\)?){2,7}`, "e164", "International phone number (E.164) — any country"),
+        // Parentheses are matched as a pair inside one group: `\(?...\)?` let the
+        // closing parenthesis of the surrounding text join the number, so
+        // "Call (+32 475 12 34 56) today." also masked the ")" (issue
+        // rules-engine#3). Mirrors the Python pattern character for character.
+        p(EntityType.PHONE, String.raw `\+\d{1,3}(?:[\s\-/]?(?:\(\d{1,4}\)|\d{1,4})){2,7}`, "e164", "International phone number (E.164) — any country"),
         p(EntityType.BIC, String.raw `\b[A-Z]{4}[A-Z]{2}[A-Z0-9]{2}(?:[A-Z0-9]{3})?\b`, "bic"),
         // Space-separated form: "RZBA AT WW", "NICA BE BB". Written as a separate
         // pattern requiring a space between *every* group, rather than making the
@@ -117,7 +128,15 @@ const SHARED = {
         p(EntityType.CREDIT_CARD, String.raw `\b(?:4[0-9]{3}[\s\-]?[0-9]{4}[\s\-]?[0-9]{4}[\s\-]?[0-9]{4}|5[1-5][0-9]{2}[\s\-]?[0-9]{4}[\s\-]?[0-9]{4}[\s\-]?[0-9]{4}|3[47][0-9]{2}[\s\-]?[0-9]{6}[\s\-]?[0-9]{5})\b`, "luhn"),
         p(EntityType.VIN, String.raw `\b[A-HJ-NPR-Z0-9]{17}\b`, "vin"),
         p(EntityType.IP_ADDRESS, String.raw `\b(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\b`),
-        p(EntityType.IPV6_ADDRESS, String.raw `\b(?:[0-9a-fA-F]{1,4}:){7}[0-9a-fA-F]{1,4}\b|\b(?:[0-9a-fA-F]{1,4}:){1,7}:\b|\b(?:[0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}\b|\b::(?:[0-9a-fA-F]{1,4}:){0,5}[0-9a-fA-F]{1,4}\b`),
+        // Branch order is load-bearing: alternation is leftmost-first, not
+        // longest-match, so the form that consumes most is tried first, and the
+        // IPv4-embedded branches lead because every group of a dotted quad is
+        // also valid hex. `\b` is the wrong anchor around ":" (a non-word
+        // character), which is why `::1` and `2001:db8::` were not detected at
+        // all. A bare "::" is deliberately not matched. Mirrors the Python
+        // pattern character for character -- see the comment there
+        // (issue rules-engine#5).
+        p(EntityType.IPV6_ADDRESS, String.raw `(?<![0-9A-Za-z_:.])(?:(?:[0-9A-Fa-f]{1,4}:){6}(?:(?:25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])\.){3}(?:25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])|(?:[0-9A-Fa-f]{1,4}:){1,5}:(?:[0-9A-Fa-f]{1,4}:)*(?:(?:25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])\.){3}(?:25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])|::(?:[0-9A-Fa-f]{1,4}:)*(?:(?:25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])\.){3}(?:25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])|(?:[0-9A-Fa-f]{1,4}:){7}[0-9A-Fa-f]{1,4}|(?:[0-9A-Fa-f]{1,4}:){1,7}(?::[0-9A-Fa-f]{1,4}){1,7}|(?:[0-9A-Fa-f]{1,4}:){1,7}:|:(?::[0-9A-Fa-f]{1,4}){1,7})(?:%[0-9A-Za-z][0-9A-Za-z._-]*)?(?![0-9A-Za-z_:])`),
         p(EntityType.MAC_ADDRESS, String.raw `\b(?:[0-9a-fA-F]{2}[:\-]){5}[0-9a-fA-F]{2}\b`, null, "colon/dash"),
         p(EntityType.MAC_ADDRESS, String.raw `\b[0-9a-fA-F]{4}\.[0-9a-fA-F]{4}\.[0-9a-fA-F]{4}\b`, null, "Cisco"),
         p(EntityType.IMEI, String.raw `\b\d{15}\b`, "imei"),
@@ -221,12 +240,12 @@ const DE = {
     patterns: [
         p(EntityType.TAX_ID, String.raw `\b[1-9]\d{10}\b`, "german_tax_id"),
         p(EntityType.TAX_ID, String.raw `\b[1-9]\d[\s.]?\d{3}[\s.]?\d{3}[\s.]?\d{3}\b`, "german_tax_id"),
-        p(EntityType.NATIONAL_ID, String.raw `\b[CFGHJKLMNPRTVWXYZ][0-9A-Z]{8,9}\b`, null, "", ["Personalausweis", "Personalausweisnummer", "Ausweis", "Ausweisnummer", "Ausweis-Nr", "identity card", "Identitätskarte", "Perso", "PA-Nummer"], true),
+        p(EntityType.NATIONAL_ID, String.raw `\b[CFGHJKLMNPRTVWXYZ][0-9CFGHJKLMNPRTVWXYZ]{8}\d?\b`, null, "", ["Personalausweis", "Personalausweisnummer", "Ausweis", "Ausweisnummer", "Ausweis-Nr", "identity card", "Identitätskarte", "Perso", "PA-Nummer"], true),
         p(EntityType.IBAN, String.raw `\bDE\d{2}\s?\d{4}\s?\d{4}\s?\d{4}\s?\d{4}\s?\d{2}\b`, "iban"),
         p(EntityType.VAT, String.raw `\bDE\s?\d{9}\b`, "vat_de"),
         p(EntityType.PHONE, String.raw `\b0[1-9]\d{1,4}[\s/\-]?\d{3,8}\b`),
         p(EntityType.PHONE, String.raw `\+49\s?\d{2,5}[\s/\-]?\d{3,8}\b`),
-        p(EntityType.PASSPORT, String.raw `\b[CFGHJK][0-9A-Z]{8,9}\b`, null, "", ["Reisepass", "passport", "Passnummer", "Reisepassnummer", "Reisepass Nummer", "Reisepass-Nr", "Pass Nr"], true),
+        p(EntityType.PASSPORT, String.raw `\b[CFGHJK][0-9CFGHJKLMNPRTVWXYZ]{8}\d?\b`, null, "", ["Reisepass", "passport", "Passnummer", "Reisepassnummer", "Reisepass Nummer", "Reisepass-Nr", "Pass Nr"], true),
         // The separator after the city code is mandatory. When it was optional, a
         // contiguous letter run split across both groups and the hyphen was
         // consumed as the *second* separator, so any "LETTERS-DIGITS" token

package/dist/esm/sdk.d.ts

@@ -1,5 +1,69 @@
 import { DocumentContext } from "./rules/context.js";
-import { type RedactResult } from "./types.js";
+import { type Detection, type RedactResult } from "./types.js";
+/** Returns the label that replaces `det`; `slice` is the exact text it covers. */
+type LabelFor = (det: Detection, slice: string) => string;
+/**
+ * Splice a label over every detection and return the masked text.
+ *
+ * `detections` must be sorted by `(start, -end)`.
+ *
+ * Labels are resolved right-to-left because the referential mapper numbers
+ * each entity type in call order, and that order is part of the output
+ * contract. The string itself is then assembled in a single forward pass:
+ * rebuilding it per detection copied the whole document each time, which is
+ * O(document x detections) — 1.68 s of pure copying on a 1 MB document with
+ * 15,000 detections, versus 2 ms here.
+ *
+ * Spans from the rule engine are deduplicated and non-overlapping. Spans from
+ * elsewhere (the cloud service) are only sorted, so a span may start behind
+ * the cursor. Its uncovered tail is still masked rather than dropped: dropping
+ * the span would leave those characters in the clear, and splicing it whole
+ * would corrupt the label already emitted over its head.
+ */
+export declare function applyReplacements(text: string, detections: Detection[], labelFor: LabelFor): string;
+/**
+ * Characters a token suffix is drawn from. No vowels, so a suffix never spells
+ * a word; no 0/1/I/O, so it survives being read back by a person; no
+ * underscore, so the type prefix stays unambiguous.
+ */
+export declare const TOKEN_ALPHABET = "ABCDEFGHJKLMNPQRSTUVWXYZ23456789";
+export declare const TOKEN_SUFFIX_LENGTH = 4;
+/**
+ * Maps each distinct PII value in one call to a reversible token.
+ *
+ * A token is `TYPE_XXXX`: the entity type, then `TOKEN_SUFFIX_LENGTH` random
+ * characters from `TOKEN_ALPHABET`. The same value gets the same token within
+ * the call, so relationships survive; across calls it gets a different one, so
+ * two tokenized documents never reveal a shared value.
+ *
+ * Nothing is kept beyond the call. The mapping goes to the caller in
+ * `RedactResult.tokens`, and only the caller can turn it back into text with
+ * `restore()`.
+ *
+ * Tokens are also kept clear of any token-shaped string already in the
+ * document. A caller that redacts an LLM's reply to a tokenized prompt has
+ * exactly such a document, and a fresh token colliding with an old one would
+ * make `restore()` put the wrong value back.
+ */
+export declare class TokenMapper {
+    private byValue;
+    private taken;
+    /** Token -> original value, for `RedactResult.tokens`. */
+    readonly tokens: Record<string, string>;
+    constructor(text: string, detections: Detection[]);
+    /** Return the token for `value`, minting one on first sight. */
+    getToken: (det: Detection, value: string) => string;
+}
+/**
+ * Put the original values back into `text`.
+ *
+ * `tokens` is `RedactResult.tokens` from the `redact(text, { tokenize: true })`
+ * call that produced the text this one derives from — typically the reply an
+ * LLM wrote to the tokenized prompt. Every occurrence of every token is
+ * replaced; a token an LLM glued to other characters (`EMAIL_K7Q2s`) is still
+ * restored, since leaving a token behind is the worse failure.
+ */
+export declare function restore(text: string, tokens: Record<string, string>): string;
 export interface RedactOptions {
     /**
      * Scope. Detections attributed elsewhere are flagged `outOfScope`, never
@@ -24,6 +88,22 @@ export interface RedactOptions {
     chunkOffset?: number;
     mode?: string;
     referentialIntegrity?: boolean;
+    /**
+     * Replace each value with a reversible token (`EMAIL_K7Q2`) and return the
+     * token -> value mapping in `RedactResult.tokens`; see `restore()`. Tokens
+     * are unique to the call. Cannot be combined with `referentialIntegrity`.
+     */
+    tokenize?: boolean;
+    /**
+     * Values never to redact, matched whole and case-insensitively against each
+     * detection. Merged with the instance's allowlist.
+     */
+    allowlist?: string[] | null;
+    /**
+     * Domains whose addresses are never redacted, e.g. `["acme.be"]`. Applies to
+     * EMAIL and URL only, and covers subdomains. Merged with the instance list.
+     */
+    allowlistDomains?: string[] | null;
     detectDates?: boolean;
     cache?: boolean;
 }
@@ -32,13 +112,52 @@ export declare class EuRedact {
     private cache;
     private referentialMapper;
     private maxInputLength;
+    private allowlist;
+    private allowlistDomains;
+    /**
+     * @param options.maxInputLength Longest document `redact` accepts, in characters.
+     * @param options.allowlist Values never to redact, for every call on this
+     *   instance — an organisation's own name, its own addresses. Merged with
+     *   the per-call `allowlist`. Matched whole, case-insensitively.
+     */
     constructor(options?: {
         maxInputLength?: number;
+        allowlist?: string[] | null;
+        allowlistDomains?: string[] | null;
     });
     addCustomPattern(name: string, pattern: string): void;
     /** Clear the result cache and referential integrity mappings.
      *  Call this in long-running processes to free PII from memory. */
     clear(): void;
+    /** The instance allowlist merged with a call's, in comparison form. */
+    private allowedFor;
+    /** The instance domain list merged with a call's. */
+    private domainsFor;
+    /** The label function for one call, given its output options. */
+    private labelFor;
+    /**
+     * Send a document to the cloud tier.
+     *
+     * Options the service cannot honour throw rather than being ignored. Silently
+     * dropping one would mean returning a result that does not match what was
+     * asked for — which, for anything that changes which spans come back, is
+     * under-redaction wearing a plausible face.
+     */
+    private redactViaCloud;
+    /**
+     * Rebuild the masked text from the service's spans, in place.
+     *
+     * The service masks with bracketed labels only. Any other output option is
+     * applied here, from its spans: the service builds its `redacted_text` from
+     * exactly the spans it returns (entities it reports but cannot place are
+     * listed separately and never applied), so nothing it masked is lost by
+     * masking again from the same spans.
+     *
+     * The service reports offsets in code points and this SDK slices UTF-16
+     * units, so the slice check is what stands between an emoji in the document
+     * and a rebuilt text with the wrong characters masked.
+     */
+    private remaskCloudResult;
     redact(text: string, options?: RedactOptions): RedactResult;
     /**
      * Redact, awaiting the cloud tier when `mode: "cloud"` is asked for.
@@ -49,3 +168,4 @@ export declare class EuRedact {
     redactAsync(text: string, options?: RedactOptions): Promise<RedactResult>;
     redactBatch(texts: string[], options?: RedactOptions): RedactResult[];
 }
+export {};