euredact 0.4.0 → 0.5.0

package/README.md

@@ -42,6 +42,11 @@ console.log(result.detections);
   OpenAI, Slack, JWT, SendGrid, plus Shannon entropy-based detection for generic
   high-entropy tokens near context keywords
 - **Custom patterns:** register your own regex patterns for domain-specific PII
+- **Reversible tokenization:** `tokenize: true` swaps values for `EMAIL_K7Q2`-style
+  tokens and `restore()` puts them back — for prompts that go to an LLM and
+  come back
+- **Allowlist:** values that are never redacted, such as your own organisation's
+  name and addresses, per call or per instance
 - **Checksum validation:** IBAN mod-97, Luhn (credit cards/IMEI), and 30+
   country-specific validators
 - **Priority-aware deduplication:** validated (checksum, corroborated by the
@@ -66,6 +71,26 @@ console.log(result.detections);
 
 ## API Reference
 
+### Which option do I need?
+
+| I want to… | Use |
+|---|---|
+| redact a document with no further setup | `redact(text)` — all 31 countries, `[ENTITY_TYPE]` output |
+| restrict scope to the countries I operate in | `{ countries: ["NL", "BE"] }` |
+| keep detection wide but resolve ambiguity | `{ countryHint: ["DE"] }` |
+| send a prompt to an LLM and restore the reply | `{ tokenize: true }`, then [`restore()`](#restoretext-tokens) |
+| keep relationships visible across a whole session | `{ referentialIntegrity: true }` |
+| never redact my own company name or addresses | `{ allowlist: [...] }`, or `new EuRedact({ allowlist })` |
+| never redact anything at my own domain | `{ allowlistDomains: ["acme.be"] }` |
+| audit what the allowlist kept in the document | `result.exempted` |
+| catch person names, employers, job titles, diagnoses | `redactAsync(text, { mode: "cloud" })` — see [Cloud tier](#cloud-tier) |
+| include dates of birth | `{ detectDates: true }` |
+| redact a document too large for one call | `context` + `chunkOffset` — see [Chunked documents](#chunked-documents) |
+| detect an identifier the engine does not know | [`addCustomPattern()`](#addcustompatternname-pattern) |
+| isolate tenants from each other | one `new EuRedact()` each |
+| free PII held in memory | `clear()` |
+
+
 ### Module-Level Functions
 
 #### `redact(text, options?)`
@@ -78,25 +103,68 @@ Main entry point. Detects and redacts PII in the given text.
 
 ```ts
 interface RedactOptions {
-  countries?: string[] | null;    // Scope. Flags anything attributed elsewhere as
-                                  // outOfScope. Never gates what is looked for.
-  countryHint?: string[] | null;  // A prior only — resolves ambiguity without
-                                  // narrowing scope or flagging anything
-  context?: DocumentContext | null; // Share country evidence across the chunks
-                                  // of one document (see "Chunked documents")
-  chunkOffset?: number;           // Where this chunk starts in the document
-  referentialIntegrity?: boolean; // Replace with consistent labels (default: false)
-  detectDates?: boolean;          // Include DOB/date-of-death detections (default: false)
-  cache?: boolean;                // Enable result caching (default: true)
+  countries?: string[] | null;      // scope (see below); null = all 31
+  countryHint?: string[] | null;    // a prior only; does not narrow scope
+  context?: DocumentContext | null; // share evidence across chunks
+  chunkOffset?: number;             // where this chunk starts
+  mode?: string;                    // "rules" (default) | "cloud"
+  referentialIntegrity?: boolean;   // consistent labels, EMAIL_1
+  tokenize?: boolean;               // reversible tokens, EMAIL_K7Q2
+  allowlist?: string[] | null;      // values never redacted
+  allowlistDomains?: string[] | null; // domains never redacted (EMAIL/URL)
+  detectDates?: boolean;            // include DOB / date of death
+  cache?: boolean;                  // reuse results for identical input
 }
 ```
 
+**What is looked for**
+
 | Parameter | Default | Description |
 |---|---|---|
-| `countries` | `null` | ISO 3166-1 alpha-2 codes to restrict detection. `null` loads all 31 countries. |
-| `referentialIntegrity` | `false` | Replace PII with consistent labels instead of entity-type labels. |
-| `detectDates` | `false` | Include date-of-birth and date-of-death detections. Off by default. |
-| `cache` | `true` | Cache results for identical inputs. |
+| `text` | — | Input text to scan. Longer than `maxInputLength` throws. |
+| `countries` | `null` | ISO 3166-1 alpha-2 codes that define **scope**. `null` loads all 31. This never gates what is looked for: a detection attributed elsewhere is flagged `outOfScope`, never dropped. A bare string throws `TypeError`. See [Country codes](#country-codes). |
+| `countryHint` | `null` | A **prior only**. Resolves an ambiguous value without narrowing scope. See [Country Hints](#country-hints). |
+| `detectDates` | `false` | Include `DOB` and `DATE_OF_DEATH`. Off by default: a bare date without keyword or structural context is deferred to the cloud tier. |
+
+**How the output looks** — pick at most one of the first three.
+
+| Parameter | Default | Description |
+|---|---|---|
+| *(none)* | — | Default: each span becomes `[ENTITY_TYPE]`. |
+| `referentialIntegrity` | `false` | Consistent label per distinct value (`EMAIL_1`), persisting **on the instance** across calls. See [Referential Integrity](#referential-integrity). |
+| `tokenize` | `false` | Reversible token per value (`EMAIL_K7Q2`), with the mapping in `result.tokens` for [`restore()`](#restoretext-tokens). Unique to the **call**. Cannot be combined with `referentialIntegrity`. See [Reversible tokenization](#reversible-tokenization). |
+| `allowlist` | `null` | Values never to redact, whole-span and case-insensitive, merged with the instance's list. Structured identifiers (IBAN, phone, VAT, …) also match across spacing and hyphenation. Applies to cloud-tier types too. See [Allowlist](#allowlist). |
+| `allowlistDomains` | `null` | Domains whose addresses are never redacted, e.g. `["acme.be"]`. Applies to `EMAIL` and `URL` only, and covers subdomains. See [Allowlist](#allowlist). |
+
+**Long documents and tiers**
+
+| Parameter | Default | Description |
+|---|---|---|
+| `mode` | `"rules"` | `"rules"` runs locally and synchronously. `"cloud"` must go through [`redactAsync()`](#redactasynctext-options) — `redact()` is synchronous and a network call cannot be. See [Cloud tier](#cloud-tier). |
+| `context` | `null` | Share country evidence across the chunks of one document. Pass the same `DocumentContext` to every chunk. Disables the cache. See [Chunked documents](#chunked-documents). |
+| `chunkOffset` | `0` | Where this chunk starts in the whole document. Only rebases spans recorded in `context`; returned detections stay relative to `text`. |
+| `cache` | `true` | Reuse the result for an identical input and configuration. |
+
+TypeScript has no `coref` option; the Python SDK accepts one as a reserved
+no-op.
+
+#### `redactAsync(text, options?)`
+
+```ts
+function redactAsync(text: string, options?: RedactOptions): Promise<RedactResult>;
+```
+
+The asynchronous entry point, and the only way to reach the [cloud
+tier](#cloud-tier). With `mode: "rules"` (the default) it resolves immediately
+with exactly what `redact()` returns, so a caller that may or may not use the
+cloud tier can keep one code path:
+
+```ts
+const result = await redactAsync(text, { countries: ["BE"], mode: "cloud" });
+```
+
+Takes the same `RedactOptions` as `redact()`. Options the service cannot honour
+throw rather than being ignored — see [Cloud tier](#cloud-tier).
 
 #### `redactBatch(texts, options?)`
 
@@ -116,6 +184,16 @@ function addCustomPattern(name: string, pattern: string): void;
 Register a custom regex pattern. Matches are reported with `name` as the entity
 type. See [Custom Patterns](#custom-patterns) below.
 
+#### `restore(text, tokens)`
+
+```ts
+function restore(text: string, tokens: Record<string, string>): string;
+```
+
+Put the original values back into text that derives from a `tokenize: true`
+result — typically an LLM's reply to the tokenized prompt. `tokens` is
+`result.tokens`. See [Reversible tokenization](#reversible-tokenization).
+
 #### `availableCountries()`
 
 ```ts
@@ -142,8 +220,25 @@ console.log(result.redactedText);
 // "See [CASE_REF] for details"
 ```
 
-The `EuRedact` class exposes: `redact()`, `redactBatch()`, and
-`addCustomPattern()`.
+The `EuRedact` class exposes `redact()`, `redactAsync()`, `redactBatch()`,
+`clear()` and `addCustomPattern()`.
+
+```ts
+new EuRedact({
+  maxInputLength?: number,   // default 10_485_760 (~10 MB)
+  allowlist?: string[] | null,
+})
+```
+
+| Parameter | Default | Description |
+|---|---|---|
+| `maxInputLength` | `10_485_760` | Longest document `redact()` accepts, in characters. Above it, it throws — split the input or raise the ceiling. |
+| `allowlist` | `null` | Values never to redact, for every call on this instance. Merged with the per-call `allowlist`. See [Allowlist](#allowlist). |
+| `allowlistDomains` | `null` | Domains never to redact, for every call on this instance. |
+
+The result cache, referential-integrity labels and custom patterns are all per
+instance, which is what makes one instance per tenant the right default.
+`clear()` releases the cache and the label mapping.
 
 ### Return Types
 
@@ -155,6 +250,8 @@ interface RedactResult {
   detections: Detection[];    // All PII spans found
   source: string;             // Detection backend ("rules")
   degraded: boolean;          // True if the engine fell back to a simpler mode
+  tokens: Record<string, string>; // token -> original value; only with tokenize: true
+  exempted: Exemption[];          // spans the allowlist kept, with the rule that matched
 }
 ```
 
@@ -211,6 +308,7 @@ interface RedactResult {
   evidence: CountryEvidence[];                // every signal, with the span behind it
   detectionMode: string;                      // "declared" if countries was passed,
                                               // "inferred" otherwise
+  tokens: Record<string, string>;             // token -> original value; only with tokenize: true
 }
 ```
 
@@ -314,6 +412,8 @@ seen a chunk boundary.
 
 Options the service cannot honour reject rather than being ignored: multiple
 `countries`, `countryHint`, `context`/`chunkOffset` and `referentialIntegrity`.
+`tokenize`, `allowlist` and `allowlistDomains` are honoured: the SDK applies them to the spans the
+service returns and rebuilds the text from those.
 
 The package stays **zero-dependency** — the client uses the platform's own
 `fetch`. Node 18+ provides one; on Node 16 the rules engine is unaffected and a
@@ -617,6 +717,120 @@ console.log(result.redactedText);
 // "BSN NATIONAL_ID_1 en later weer NATIONAL_ID_1"
 ```
 
+## Reversible tokenization
+
+`tokenize: true` is for text that has to come back. Each value is replaced by a
+token that names its type and nothing else, and the result carries the mapping
+that turns tokens back into values:
+
+```ts
+import { redact, restore } from "euredact";
+
+const prompt = "Write an email to Joren at joren.janssens@euredact.be or call +32 475 12 34 56 about the invoice.";
+const result = redact(prompt, { countries: ["BE"], tokenize: true });
+console.log(result.redactedText);
+// "Write an email to Joren at EMAIL_K7Q2 or call PHONE_P4RT about the invoice."
+console.log(result.tokens);
+// { EMAIL_K7Q2: "joren.janssens@euredact.be", PHONE_P4RT: "+32 475 12 34 56" }
+
+const reply = await callYourLlm(result.redactedText);  // sees tokens, never the values
+console.log(restore(reply, result.tokens));
+// the reply, with the real address and number back in it
+```
+
+The suffixes are random; yours will differ. With the [cloud tier](#cloud-tier)
+the name is tokenized too (`PERSON_NAME_W3NB`), since person names have no
+shape for the rules tier to match on.
+
+Within one call the same value gets the same token, so a prompt that names
+someone twice still reads as one person. Across calls it gets a different
+token: nothing is retained on the instance, `result.tokens` is the only copy,
+and two tokenized documents never reveal that they share a value. That is the
+opposite retention model from `referentialIntegrity`, which is why the two
+cannot be combined. `redactBatch` tokenizes each text on its own.
+
+A token is `TYPE_` plus four characters from `ABCDEFGHJKLMNPQRSTUVWXYZ23456789`
+(no vowels, no `0`/`1`/`I`/`O`). Tokens are kept clear of any token-shaped
+string already in the document, so an LLM's reply to a tokenized prompt can
+itself be redacted without `restore()` putting the wrong value back.
+`restore()` replaces every occurrence, including a token an LLM glued to other
+characters (`EMAIL_P4RTs`) — leaving a token behind is the worse failure.
+
+Works in cloud mode via `redactAsync`: the SDK rebuilds the text from the spans
+the service returns, which is exactly what the service built its own output
+from.
+
+## Allowlist
+
+Values a caller declares are not PII to them — their own organisation's name,
+their own addresses. Set it per call, on the instance, or both; the two merge:
+
+```ts
+import { EuRedact } from "euredact";
+
+const sdk = new EuRedact({ allowlist: ["ACME NV", "info@acme.be"] });
+
+const text = "ACME NV: mail info@acme.be or jan@acme.be about IBAN NL91 ABNA 0417 1643 00.";
+console.log(sdk.redact(text, { countries: ["NL"] }).redactedText);
+// "ACME NV: mail info@acme.be or [EMAIL] about IBAN [BANK_ACCOUNT]."
+
+console.log(sdk.redact(text, { countries: ["NL"], allowlist: ["jan@acme.be"] }).redactedText);
+// "ACME NV: mail info@acme.be or jan@acme.be about IBAN [BANK_ACCOUNT]."
+```
+
+Matching is whole-span and case-insensitive. A bare string
+(`allowlist: "ACME NV"`) throws `TypeError` rather than being iterated into
+single letters that exempt nothing. Works in cloud mode via `redactAsync` —
+including on types only the model finds, such as `ORGANISATION_NAME` — and
+together with `tokenize`.
+
+### Spacing and punctuation
+
+For **structured identifiers** separators are presentational, so an allowlisted
+value matches however the document writes it:
+
+```ts
+const sdk = new EuRedact({ allowlist: ["NL91ABNA0417164300"] });
+sdk.redact("Pay to NL91 ABNA 0417 1643 00.", { countries: ["NL"] }).redactedText;
+// 'Pay to NL91 ABNA 0417 1643 00.'   — exempt, despite the spacing
+```
+
+Applies to `BANK_ACCOUNT`, `BIC`, `CREDIT_CARD`, `PHONE`, `VAT`, `NATIONAL_ID`,
+`SSN`, `TAX_ID`, `PASSPORT`, `DRIVERS_LICENSE`, `RESIDENCE_PERMIT`,
+`HEALTH_INSURANCE`, `CHAMBER_OF_COMMERCE`, `IMEI` and `VIN`. It widens the
+*spelling*, never the *scope*: a different account is still redacted. Free-text
+types stay literal, because `jan.devries@acme.be` and `jandevries@acme.be` are
+different mailboxes at most providers.
+
+### Exempting a whole domain
+
+```ts
+const sdk = new EuRedact({ allowlistDomains: ["acme.be"] });
+sdk.redact("Mail jan@acme.be or piet@acme.be", { countries: ["NL"] }).redactedText;
+// 'Mail jan@acme.be or piet@acme.be'
+```
+
+`EMAIL` and `URL` only, covering subdomains (`mail.acme.be`). The match is on a
+label boundary, so `acme.be` does **not** exempt `evilacme.be`. Entries may be
+written `acme.be`, `@acme.be` or `.acme.be`.
+
+There are deliberately **no wildcards**. The allowlist is the only option that
+turns redaction *off*, so an over-broad entry fails toward under-redaction and
+does so silently.
+
+### What was exempted
+
+```ts
+const r = redact("Mail jan@acme.be", { countries: ["NL"], allowlistDomains: ["acme.be"] });
+for (const e of r.exempted) console.log(e.entityType, e.text, e.rule, e.ruleKind);
+// EMAIL jan@acme.be acme.be domain
+```
+
+`Exemption` carries `entityType`, `start`, `end`, `text`, the `rule` that
+matched as you wrote it, and `ruleKind` (`"value"` or `"domain"`). An exempted
+span is absent from `detections`, so `exempted` is the only record that it was
+found at all.
+
 ## Architecture
 
 ```

package/dist/cjs/cache.js

@@ -1,6 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.ResultCache = void 0;
+exports.resultChars = resultChars;
 const node_crypto_1 = require("node:crypto");
 // Default ceiling on retained characters, not entries. A count-only cap of
 // 1024 entries says nothing about memory: 1024 cached 1 MB documents is a
@@ -11,6 +12,10 @@ function resultChars(result) {
     let chars = result.redactedText.length;
     for (const d of result.detections)
         chars += d.text.length;
+    for (const [token, value] of Object.entries(result.tokens))
+        chars += token.length + value.length;
+    for (const e of result.exempted)
+        chars += e.text.length + e.rule.length;
     return chars;
 }
 class ResultCache {

package/dist/cjs/cloud/client.js

@@ -74,11 +74,33 @@ function toEntityType(raw) {
     const known = Object.values(types_js_1.EntityType).includes(canonical);
     return known ? canonical : canonical;
 }
+/**
+ * Map the service's offsets, which count code points, onto this SDK's, which
+ * count UTF-16 units.
+ *
+ * The two agree until the first character outside the BMP — an emoji, say —
+ * after which every service offset is short by one unit per such character.
+ * Returns the identity when the text has none, which is the common case and
+ * costs one regex test.
+ */
+function codePointOffsets(text) {
+    if (!/[\uD800-\uDBFF]/.test(text))
+        return cp => cp;
+    const units = [];
+    let i = 0;
+    for (const ch of text) {
+        units.push(i);
+        i += ch.length;
+    }
+    units.push(i);
+    return cp => units[Math.min(cp, units.length - 1)];
+}
 function toResult(payload, text) {
+    const offset = codePointOffsets(text);
     const detections = (payload.entities ?? []).map(span => ({
         entityType: toEntityType(span.type),
-        start: span.start,
-        end: span.end,
+        start: offset(span.start),
+        end: offset(span.end),
         text: span.text,
         source: span.source === "model" ? types_js_1.DetectionSource.CLOUD : types_js_1.DetectionSource.RULES,
         country: null,
@@ -93,6 +115,8 @@ function toResult(payload, text) {
         inferredCountries: [],
         evidence: [],
         detectionMode: "declared",
+        tokens: {},
+        exempted: [],
     };
 }
 class CloudClient {

package/dist/cjs/index.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.TooLargeError = exports.QuotaExceededError = exports.NotConfiguredError = exports.CloudError = exports.CloudClient = exports.getConfig = exports.configure = exports.SEED_BIC6_PREFIXES = exports.getBicRegistry = exports.setBicRegistry = exports.COUNTRY_CONFIGS = exports.EuRedact = exports.DocumentContext = exports.canonicalType = exports.LEGACY_TYPE_ALIASES = exports.DetectionSource = exports.EntityType = void 0;
+exports.TooLargeError = exports.QuotaExceededError = exports.NotConfiguredError = exports.CloudError = exports.CloudClient = exports.getConfig = exports.configure = exports.SEED_BIC6_PREFIXES = exports.getBicRegistry = exports.setBicRegistry = exports.COUNTRY_CONFIGS = exports.restore = exports.EuRedact = exports.DocumentContext = exports.canonicalType = exports.LEGACY_TYPE_ALIASES = exports.DetectionSource = exports.EntityType = void 0;
 exports.availableCountries = availableCountries;
 exports.redact = redact;
 exports.redactAsync = redactAsync;
@@ -16,6 +16,7 @@ var context_js_1 = require("./rules/context.js");
 Object.defineProperty(exports, "DocumentContext", { enumerable: true, get: function () { return context_js_1.DocumentContext; } });
 var sdk_js_1 = require("./sdk.js");
 Object.defineProperty(exports, "EuRedact", { enumerable: true, get: function () { return sdk_js_1.EuRedact; } });
+Object.defineProperty(exports, "restore", { enumerable: true, get: function () { return sdk_js_1.restore; } });
 var index_js_1 = require("./rules/countries/index.js");
 Object.defineProperty(exports, "COUNTRY_CONFIGS", { enumerable: true, get: function () { return index_js_1.COUNTRY_CONFIGS; } });
 var bicRegistry_js_1 = require("./rules/bicRegistry.js");

package/dist/cjs/rules/countries/index.js

@@ -99,7 +99,14 @@ const SHARED = {
         // is Unicode-aware) matched local parts like "petras_ž@..." while this one
         // did not. \p{L}\p{N} with the `u` flag restores parity; the leading guard
         // is a Unicode-safe word boundary.
-        p(types_js_1.EntityType.EMAIL, String.raw `(?<![\p{L}\p{N}._%+\-])[\p{L}\p{N}._%+\-]+@(?:[\p{L}\p{N}\-]+\.)+\p{L}{2,}\b`),
+        // The apostrophe is allowed only between word characters: it belongs in a
+        // local part, but putting it in the class would let the match absorb a
+        // quote from the surrounding text, while leaving it out made
+        // `johno'neill@outlook.ie` mask as `johno'[EMAIL]` (issue rules-engine#10).
+        // The second lookbehind refuses to start a match after `<word-char>'`,
+        // which is a match already in progress; without it every apostrophe was a
+        // fresh start offset and the scan went quadratic. Mirrors Python.
+        p(types_js_1.EntityType.EMAIL, String.raw `(?<![\p{L}\p{N}._%+\-])(?<![\p{L}\p{N}._%+\-]')[\p{L}\p{N}._%+\-]+(?:'[\p{L}\p{N}._%+\-]+)*@(?:[\p{L}\p{N}\-]+\.)+\p{L}{2,}\b`),
         // IBAN (any country) — structure + mod-97 only. Country-specific IBAN
         // patterns still exist and win on identical spans; this one guarantees an
         // IBAN is never missed just because its country was not requested.
@@ -109,7 +116,11 @@ const SHARED = {
         // "+43 664 8213 907" was invisible to the Austrian pattern, which expects
         // an unbroken subscriber number. A leading "+" is self-identifying, so
         // this is deliberately not country-gated.
-        p(types_js_1.EntityType.PHONE, String.raw `\+\d{1,3}(?:[\s\-/]?\(?\d{1,4}\)?){2,7}`, "e164", "International phone number (E.164) — any country"),
+        // Parentheses are matched as a pair inside one group: `\(?...\)?` let the
+        // closing parenthesis of the surrounding text join the number, so
+        // "Call (+32 475 12 34 56) today." also masked the ")" (issue
+        // rules-engine#3). Mirrors the Python pattern character for character.
+        p(types_js_1.EntityType.PHONE, String.raw `\+\d{1,3}(?:[\s\-/]?(?:\(\d{1,4}\)|\d{1,4})){2,7}`, "e164", "International phone number (E.164) — any country"),
         p(types_js_1.EntityType.BIC, String.raw `\b[A-Z]{4}[A-Z]{2}[A-Z0-9]{2}(?:[A-Z0-9]{3})?\b`, "bic"),
         // Space-separated form: "RZBA AT WW", "NICA BE BB". Written as a separate
         // pattern requiring a space between *every* group, rather than making the
@@ -120,7 +131,15 @@ const SHARED = {
         p(types_js_1.EntityType.CREDIT_CARD, String.raw `\b(?:4[0-9]{3}[\s\-]?[0-9]{4}[\s\-]?[0-9]{4}[\s\-]?[0-9]{4}|5[1-5][0-9]{2}[\s\-]?[0-9]{4}[\s\-]?[0-9]{4}[\s\-]?[0-9]{4}|3[47][0-9]{2}[\s\-]?[0-9]{6}[\s\-]?[0-9]{5})\b`, "luhn"),
         p(types_js_1.EntityType.VIN, String.raw `\b[A-HJ-NPR-Z0-9]{17}\b`, "vin"),
         p(types_js_1.EntityType.IP_ADDRESS, String.raw `\b(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\b`),
-        p(types_js_1.EntityType.IPV6_ADDRESS, String.raw `\b(?:[0-9a-fA-F]{1,4}:){7}[0-9a-fA-F]{1,4}\b|\b(?:[0-9a-fA-F]{1,4}:){1,7}:\b|\b(?:[0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}\b|\b::(?:[0-9a-fA-F]{1,4}:){0,5}[0-9a-fA-F]{1,4}\b`),
+        // Branch order is load-bearing: alternation is leftmost-first, not
+        // longest-match, so the form that consumes most is tried first, and the
+        // IPv4-embedded branches lead because every group of a dotted quad is
+        // also valid hex. `\b` is the wrong anchor around ":" (a non-word
+        // character), which is why `::1` and `2001:db8::` were not detected at
+        // all. A bare "::" is deliberately not matched. Mirrors the Python
+        // pattern character for character -- see the comment there
+        // (issue rules-engine#5).
+        p(types_js_1.EntityType.IPV6_ADDRESS, String.raw `(?<![0-9A-Za-z_:.])(?:(?:[0-9A-Fa-f]{1,4}:){6}(?:(?:25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])\.){3}(?:25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])|(?:[0-9A-Fa-f]{1,4}:){1,5}:(?:[0-9A-Fa-f]{1,4}:)*(?:(?:25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])\.){3}(?:25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])|::(?:[0-9A-Fa-f]{1,4}:)*(?:(?:25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])\.){3}(?:25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])|(?:[0-9A-Fa-f]{1,4}:){7}[0-9A-Fa-f]{1,4}|(?:[0-9A-Fa-f]{1,4}:){1,7}(?::[0-9A-Fa-f]{1,4}){1,7}|(?:[0-9A-Fa-f]{1,4}:){1,7}:|:(?::[0-9A-Fa-f]{1,4}){1,7})(?:%[0-9A-Za-z][0-9A-Za-z._-]*)?(?![0-9A-Za-z_:])`),
         p(types_js_1.EntityType.MAC_ADDRESS, String.raw `\b(?:[0-9a-fA-F]{2}[:\-]){5}[0-9a-fA-F]{2}\b`, null, "colon/dash"),
         p(types_js_1.EntityType.MAC_ADDRESS, String.raw `\b[0-9a-fA-F]{4}\.[0-9a-fA-F]{4}\.[0-9a-fA-F]{4}\b`, null, "Cisco"),
         p(types_js_1.EntityType.IMEI, String.raw `\b\d{15}\b`, "imei"),
@@ -224,12 +243,12 @@ const DE = {
     patterns: [
         p(types_js_1.EntityType.TAX_ID, String.raw `\b[1-9]\d{10}\b`, "german_tax_id"),
         p(types_js_1.EntityType.TAX_ID, String.raw `\b[1-9]\d[\s.]?\d{3}[\s.]?\d{3}[\s.]?\d{3}\b`, "german_tax_id"),
-        p(types_js_1.EntityType.NATIONAL_ID, String.raw `\b[CFGHJKLMNPRTVWXYZ][0-9A-Z]{8,9}\b`, null, "", ["Personalausweis", "Personalausweisnummer", "Ausweis", "Ausweisnummer", "Ausweis-Nr", "identity card", "Identitätskarte", "Perso", "PA-Nummer"], true),
+        p(types_js_1.EntityType.NATIONAL_ID, String.raw `\b[CFGHJKLMNPRTVWXYZ][0-9CFGHJKLMNPRTVWXYZ]{8}\d?\b`, null, "", ["Personalausweis", "Personalausweisnummer", "Ausweis", "Ausweisnummer", "Ausweis-Nr", "identity card", "Identitätskarte", "Perso", "PA-Nummer"], true),
         p(types_js_1.EntityType.IBAN, String.raw `\bDE\d{2}\s?\d{4}\s?\d{4}\s?\d{4}\s?\d{4}\s?\d{2}\b`, "iban"),
         p(types_js_1.EntityType.VAT, String.raw `\bDE\s?\d{9}\b`, "vat_de"),
         p(types_js_1.EntityType.PHONE, String.raw `\b0[1-9]\d{1,4}[\s/\-]?\d{3,8}\b`),
         p(types_js_1.EntityType.PHONE, String.raw `\+49\s?\d{2,5}[\s/\-]?\d{3,8}\b`),
-        p(types_js_1.EntityType.PASSPORT, String.raw `\b[CFGHJK][0-9A-Z]{8,9}\b`, null, "", ["Reisepass", "passport", "Passnummer", "Reisepassnummer", "Reisepass Nummer", "Reisepass-Nr", "Pass Nr"], true),
+        p(types_js_1.EntityType.PASSPORT, String.raw `\b[CFGHJK][0-9CFGHJKLMNPRTVWXYZ]{8}\d?\b`, null, "", ["Reisepass", "passport", "Passnummer", "Reisepassnummer", "Reisepass Nummer", "Reisepass-Nr", "Pass Nr"], true),
         // The separator after the city code is mandatory. When it was optional, a
         // contiguous letter run split across both groups and the hyphen was
         // consumed as the *second* separator, so any "LETTERS-DIGITS" token