@doccloak/core 0.8.0 → 0.9.0

package/dist/regex/rules.data.js

@@ -275,6 +275,175 @@ export const RULES_DATA = [
                     "75008 Paris"
                 ],
                 "falsePositiveNotes": "May match non-address number+word sequences"
+            },
+            {
+                "id": "regex:universal:aws_access_key",
+                "entityType": "API_KEY",
+                "pattern": "\\b(?:AKIA|ASIA|ABIA|ACCA|AGPA|AIDA|AIPA|ANPA|ANVA|AROA)[0-9A-Z]{16}\\b",
+                "flags": "g",
+                "confidence": 0.95,
+                "domains": [
+                    "technical"
+                ],
+                "description": "AWS access key ID (AKIA/ASIA + 16 uppercase alphanumeric)",
+                "examples": [
+                    "AKIAIOSFODNN7EXAMPLE",
+                    "ASIAJEXAMPLEKEY12345"
+                ]
+            },
+            {
+                "id": "regex:universal:github_token",
+                "entityType": "API_KEY",
+                "pattern": "\\b(?:gh[pousr]_[A-Za-z0-9]{36,255}|github_pat_[A-Za-z0-9]{22}_[A-Za-z0-9]{59})\\b",
+                "flags": "g",
+                "confidence": 0.95,
+                "domains": [
+                    "technical"
+                ],
+                "description": "GitHub token (classic ghp_/gho_/ghu_/ghs_/ghr_ or fine-grained github_pat_)",
+                "examples": [
+                    "ghp_16C7e42F292c6912E7710c838347Ae178B4a",
+                    "github_pat_11ABCDEFG0123456789abc_abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456"
+                ]
+            },
+            {
+                "id": "regex:universal:slack_token",
+                "entityType": "API_KEY",
+                "pattern": "\\bxox[abprs]-[A-Za-z0-9-]{10,64}\\b",
+                "flags": "g",
+                "confidence": 0.95,
+                "domains": [
+                    "technical"
+                ],
+                "description": "Slack token (xoxb- bot, xoxp- user, xoxa-/xoxr-/xoxs- variants)",
+                "examples": [
+                    "xoxb-NotARealBotToken-DocCloakTestFixture",
+                    "xoxp-FakeUserToken-DocCloakTestFixture00"
+                ]
+            },
+            {
+                "id": "regex:universal:openai_api_key",
+                "entityType": "API_KEY",
+                "pattern": "\\bsk-(?:(?:proj|svcacct|admin)-[A-Za-z0-9_-]{40,}|[A-Za-z0-9]{40,})\\b",
+                "flags": "g",
+                "confidence": 0.92,
+                "domains": [
+                    "technical"
+                ],
+                "description": "OpenAI API key (legacy sk- + 40+ alphanumeric, or sk-proj-/sk-svcacct-/sk-admin- project keys)",
+                "examples": [
+                    "sk-proj-AbCdEfGhIjKlMnOpQrStUvWxYz0123456789AbCdEfGh",
+                    "sk-AbCdEfGhIjKlMnOpQrStUvWxYz0123456789AbCdEfGhIjKl"
+                ],
+                "falsePositiveNotes": "Anthropic sk-ant- keys never reach the alphanumeric-only branch (hyphen after ant stops it); the dedicated anthropic rule wins overlap resolution via its longer prefix"
+            },
+            {
+                "id": "regex:universal:anthropic_api_key",
+                "entityType": "API_KEY",
+                "pattern": "\\bsk-ant-[A-Za-z0-9_-]{24,}\\b",
+                "flags": "g",
+                "confidence": 0.95,
+                "domains": [
+                    "technical"
+                ],
+                "description": "Anthropic API key (sk-ant- prefix)",
+                "examples": [
+                    "sk-ant-api03-AbCdEfGhIjKlMnOpQrStUvWxYz0123456789-AbCdEfGh"
+                ]
+            },
+            {
+                "id": "regex:universal:google_api_key",
+                "entityType": "API_KEY",
+                "pattern": "\\bAIza[0-9A-Za-z_-]{35}\\b",
+                "flags": "g",
+                "confidence": 0.95,
+                "domains": [
+                    "technical"
+                ],
+                "description": "Google API key (AIza + 35 characters)",
+                "examples": [
+                    "AIzaSyD-9tSrke72PouQMnMX-a7eZSW0jkFMBWY"
+                ]
+            },
+            {
+                "id": "regex:universal:private_key_block",
+                "entityType": "SECRET",
+                "pattern": "-----BEGIN (?:[A-Z]+ )*PRIVATE KEY(?: BLOCK)?-----[\\s\\S]*?-----END (?:[A-Z]+ )*PRIVATE KEY(?: BLOCK)?-----",
+                "flags": "g",
+                "confidence": 0.99,
+                "domains": [
+                    "technical"
+                ],
+                "description": "PEM private key block (RSA/EC/DSA/OPENSSH/ENCRYPTED/PGP), including SSH private keys",
+                "examples": [
+                    "-----BEGIN RSA PRIVATE KEY-----\nMIIEowIBAAKCAQEA0Z3VS5JJcds3xfn\n-----END RSA PRIVATE KEY-----",
+                    "-----BEGIN OPENSSH PRIVATE KEY-----\nb3BlbnNzaC1rZXktdjEAAAAABG5vbmU\n-----END OPENSSH PRIVATE KEY-----"
+                ]
+            },
+            {
+                "id": "regex:universal:jwt",
+                "entityType": "SECRET",
+                "pattern": "\\beyJ[A-Za-z0-9_-]{8,}\\.[A-Za-z0-9_-]{8,}\\.[A-Za-z0-9_-]{8,}\\b",
+                "flags": "g",
+                "confidence": 0.95,
+                "domains": [
+                    "technical"
+                ],
+                "description": "JSON Web Token (three base64url segments, header starts with eyJ = base64 of '{\")",
+                "examples": [
+                    "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0.dozjgNryP4J3jVmNHl0w5N_XgL0n3I9PlFUP0THsR8U"
+                ]
+            },
+            {
+                "id": "regex:universal:connection_string_creds",
+                "entityType": "SECRET",
+                "pattern": "\\b[a-zA-Z][a-zA-Z0-9+.-]{1,30}:\\/\\/[^\\s:@\\/]{1,64}:[^\\s@]{1,128}@[^\\s\"']{1,256}",
+                "flags": "g",
+                "confidence": 0.9,
+                "domains": [
+                    "technical"
+                ],
+                "description": "Connection string with inline credentials (scheme://user:password@host)",
+                "examples": [
+                    "postgres://admin:S3cr3tPass@db.example.com:5432/mydb",
+                    "mongodb+srv://appuser:hunter2pass@cluster0.abc.mongodb.net/prod"
+                ],
+                "falsePositiveNotes": "Plain URLs without a user:password pair never match (the colon between user and password is required before the @)"
+            },
+            {
+                "id": "regex:universal:secret_assignment",
+                "entityType": "SECRET",
+                "pattern": "\\b[a-zA-Z0-9_-]{0,32}?(?:api[_-]?key|apikey|secret|token|passwd|password|credential|auth|access[_-]?key|client[_-]?secret|private[_-]?key|signing[_-]?key)[a-zA-Z0-9_-]{0,32}[\"']?\\s*[:=]>?\\s*[\"']?[A-Za-z0-9+\\/_=-]{16,}",
+                "flags": "gi",
+                "confidence": 0.75,
+                "domains": [
+                    "technical"
+                ],
+                "description": "Secret-ish keyword assignment (api_key = ..., password: ..., aws_secret_access_key: ...) with a 16+ char value; the value is Shannon-entropy gated by the secretAssignment validator",
+                "examples": [
+                    "api_key = \"9fQz3kX7Lm2Pw8Rt5Vy1Jh6Nd4Bg0Cs\"",
+                    "aws_secret_access_key: wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
+                    "session_token=a3f8c2d914b7e605d2c8a1f4b9e7d305"
+                ],
+                "falsePositiveNotes": "The keyword and separator are part of the match (lookbehind is banned for new rules), so the config key name is redacted along with the value. Low-entropy placeholder values (your_api_key_here, changeme) are rejected by the validator.",
+                "validate": "secretAssignment"
+            },
+            {
+                "id": "regex:universal:high_entropy_token",
+                "entityType": "SECRET",
+                "pattern": "\\b[A-Za-z0-9][A-Za-z0-9+\\/]{39,}={0,2}",
+                "flags": "g",
+                "confidence": 0.6,
+                "domains": [
+                    "technical"
+                ],
+                "description": "Bare high-entropy base64/alphanumeric token (40+ chars); the highEntropyToken validator requires mixed character classes and Shannon entropy >= 4.5 bits/char",
+                "examples": [
+                    "Zq7PmX2vR9kLtY4wNb8QcJ3fHd6GsA1uEi5oT0xVrKp",
+                    "kM3nP8qR2sT7uV1wX5yZ9aB4cD6eF0gH1jK2lM3nP4qR5sT6"
+                ],
+                "falsePositiveNotes": "Low confidence by design. Git SHAs and UUID fragments are single-case hex (rejected by the class-mix check); base64 data blobs exceed the 256 char cap; identifiers and prose fall below the entropy threshold.",
+                "validate": "highEntropyToken"
             }
         ]
     },
@@ -469,6 +638,23 @@ export const RULES_DATA = [
                     "Tesco Ltd",
                     "Deloitte LLP"
                 ]
+            },
+            {
+                "id": "regex:gb:driving_licence",
+                "entityType": "SSN",
+                "pattern": "\\b[A-Z9]{5}\\s?\\d{6}\\s?[A-Z9]{2}\\d[A-Z]{2}\\b",
+                "flags": "g",
+                "confidence": 0.8,
+                "domains": [
+                    "identity"
+                ],
+                "description": "UK driving licence number (16 chars: surname block, encoded date of birth, initials, check chars)",
+                "examples": [
+                    "MORGA657054SM9IJ",
+                    "SMITH 751125 JM9AB"
+                ],
+                "falsePositiveNotes": "Best-effort: other 16-char uppercase codes can match the shape; the validator checks the embedded date-of-birth digit block",
+                "validate": "ukDrivingLicence"
             }
         ]
     },

package/dist/regex/validators.d.ts

@@ -10,6 +10,13 @@
  * luhn_check -> luhn, nir_check -> nir, ipv4_check -> ipOctets.
  */
 export type RuleValidator = (match: string) => boolean;
+/**
+ * Shannon entropy of a string in bits per character. Exported for tests
+ * and threshold tuning; the secrets-tier validators below gate the
+ * generic candidate rules on it (detect-secrets / Prompt Armour
+ * precedent). Named patterns (AKIA..., ghp_...) need no entropy gate.
+ */
+export declare function shannonEntropy(value: string): number;
 /**
  * The registry. Keys are the names allowed in the `validate` field of
  * rules/*.json; loader.ts throws at module init on any unknown name.

package/dist/regex/validators.js

@@ -56,15 +56,20 @@ function ipOctets(ip) {
         return n >= 0 && n <= 255;
     });
 }
-/** UK National Insurance Number structural rules (prefix letter exclusions). */
+/**
+ * UK National Insurance Number structural rules (HMRC prefix spec):
+ * first letter is never D, F, I, Q, U, V; second letter is never
+ * D, F, I, O, Q, U, V; the prefixes BG, GB, NK, KN, TN, NT, ZZ are
+ * never allocated. T100 fixed the second-letter class, which was
+ * missing V (the Python twin in DocCloak.Cli validators.py still has
+ * the pre-fix class until the packs are re-vendored there).
+ */
 function nino(match) {
     const clean = match.replace(/\s/g, '').toUpperCase();
-    // NINO cannot start with D, F, I, Q, U, V; second letter cannot be D, F, I, O, Q, U, V
     if (/^[DFIQUV]/.test(clean))
         return false;
-    if (/^.[DFIOQU]/.test(clean))
+    if (/^.[DFIOQUV]/.test(clean))
         return false;
-    // Cannot be BG, GB, NK, KN, TN, NT, ZZ prefix
     if (/^(?:BG|GB|NK|KN|TN|NT|ZZ)/.test(clean))
         return false;
     return true;
@@ -85,6 +90,29 @@ function nhs(match) {
         return false; // invalid
     return checkDigit === parseInt(digits[9], 10);
 }
+/**
+ * UK driving licence driver number (T100, best-effort): after stripping
+ * spaces the 16-char code carries a 6-digit encoded date of birth at
+ * positions 6-11 (decade digit, month 01-12 or 51-62 for women, day
+ * 01-31, year-in-decade digit). There is no public checksum for the
+ * final check characters, so only the date block is validated.
+ */
+function ukDrivingLicence(match) {
+    const clean = match.replace(/\s/g, '').toUpperCase();
+    if (clean.length !== 16)
+        return false;
+    const dob = clean.slice(5, 11);
+    if (!/^\d{6}$/.test(dob))
+        return false;
+    const rawMonth = parseInt(dob.slice(1, 3), 10);
+    const month = rawMonth > 50 ? rawMonth - 50 : rawMonth;
+    if (month < 1 || month > 12)
+        return false;
+    const day = parseInt(dob.slice(3, 5), 10);
+    if (day < 1 || day > 31)
+        return false;
+    return true;
+}
 /** Polish PESEL weighted checksum. */
 function pesel(match) {
     const digits = match.replace(/\D/g, '');
@@ -456,6 +484,66 @@ function pps(match) {
     const expected = remainder === 0 ? 'W' : String.fromCharCode(64 + remainder);
     return checkChar === expected;
 }
+/**
+ * Shannon entropy of a string in bits per character. Exported for tests
+ * and threshold tuning; the secrets-tier validators below gate the
+ * generic candidate rules on it (detect-secrets / Prompt Armour
+ * precedent). Named patterns (AKIA..., ghp_...) need no entropy gate.
+ */
+export function shannonEntropy(value) {
+    if (value.length === 0)
+        return 0;
+    const counts = new Map();
+    for (const ch of value)
+        counts.set(ch, (counts.get(ch) ?? 0) + 1);
+    let entropy = 0;
+    for (const count of counts.values()) {
+        const p = count / value.length;
+        entropy -= p * Math.log2(p);
+    }
+    return entropy;
+}
+/**
+ * Generic "keyword = value" secret assignment. The rule includes the
+ * keyword and separator in the match (lookbehind is banned for new
+ * rules), so extract the value: everything after the first ':' or '=',
+ * with quotes/whitespace and base64 '=' padding stripped. Gate it on
+ * Shannon entropy: pure-hex values pass at >= 3.0 bits/char (random hex
+ * tops out at 4.0), anything else needs >= 4.0. Pure-digit values are
+ * rejected outright (IDs and phone numbers, not credentials).
+ * Thresholds tuned in tests/regex/secrets.test.ts.
+ */
+function secretAssignment(match) {
+    const sep = match.search(/[:=]/);
+    if (sep === -1)
+        return false;
+    const value = match.slice(sep + 1).replace(/^[>\s"']+/, '').replace(/=+$/, '');
+    if (value.length < 16)
+        return false;
+    if (/^\d+$/.test(value))
+        return false;
+    if (/^[0-9a-f]+$/i.test(value))
+        return shannonEntropy(value) >= 3.0;
+    return shannonEntropy(value) >= 4.0;
+}
+/**
+ * Bare high-entropy token gate for the generic base64/alphanumeric
+ * candidate rule. Requires the shape of machine-generated key material:
+ * 40-256 chars after stripping '=' padding (longer runs are data blobs
+ * like base64 images, not credentials), a mix of lowercase, uppercase,
+ * and digits (rejects git SHAs and UUIDs, which are single-case hex),
+ * and Shannon entropy >= 4.5 bits/char (rejects identifiers and prose;
+ * random 40+ char base64 sits near 4.8). Thresholds tuned in
+ * tests/regex/secrets.test.ts.
+ */
+function highEntropyToken(match) {
+    const value = match.replace(/=+$/, '');
+    if (value.length < 40 || value.length > 256)
+        return false;
+    if (!/[a-z]/.test(value) || !/[A-Z]/.test(value) || !/\d/.test(value))
+        return false;
+    return shannonEntropy(value) >= 4.5;
+}
 /**
  * The registry. Keys are the names allowed in the `validate` field of
  * rules/*.json; loader.ts throws at module init on any unknown name.
@@ -471,6 +559,7 @@ export const VALIDATORS = {
     dni,
     fodselsnummer,
     hetu,
+    highEntropyToken,
     ibanMod97,
     ipOctets,
     luhn,
@@ -487,6 +576,8 @@ export const VALIDATORS = {
     pps,
     regon,
     samordningsnummer,
+    secretAssignment,
     steuerId,
     svnr,
+    ukDrivingLicence,
 };

package/dist/restore-tokens.d.ts

@@ -0,0 +1,67 @@
+/**
+ * Tolerant restore-token matching (T098), extracted to a side-effect-free
+ * leaf module so the extension content bundle can import these VALUES
+ * without pulling in the session/surrogate machinery (see the T036 alias
+ * note in DocCloak.Extension/vite.content.config.ts).
+ */
+/**
+ * Bare typed token as an LLM writes it after dropping the brackets:
+ * STRICTLY uppercase segments ending in an underscore-digit run, on
+ * non-wordish boundaries (underscore excluded on both sides so an
+ * embedded "SOME_PERSON_1" identifier never yields a partial match).
+ * Case is deliberately NOT tolerated here: without brackets, only the
+ * exact uppercase shape is distinctive enough to touch ("person 1" in
+ * lowercase prose must stay prose). An immediately preceding '[' or '<'
+ * is also excluded: a dangling "[PERSON_1" (a token split mid-stream,
+ * possibly a truncated "[PERSON_11]") must never be partially restored -
+ * the complete bracketed form is the bracket scan's job.
+ * Global: callers reset lastIndex.
+ */
+export declare const BARE_TYPED_TOKEN_RE: RegExp;
+/**
+ * Canonical comparison form of a placeholder candidate or map key:
+ * markdown emphasis stripped, edge junk (brackets, punctuation, space)
+ * stripped, lowercased, separator runs collapsed to single underscores.
+ * "[PERSON_1]", "[person 1]", "**[PERSON-1]**" and "PERSON_1" all yield
+ * "person_1". Returns null when nothing substantive remains.
+ */
+export declare function canonicalPlaceholderForm(candidate: string): string | null;
+/**
+ * The type-family prefix of a canonical typed token ("person_1" ->
+ * "person", "credit_card_2" -> "credit_card"), or null for canonical
+ * forms that are not typed-token shaped (renamed labels like "ceo").
+ */
+export declare function canonicalTypePrefix(canonical: string): string | null;
+export interface TolerantTokenIndex {
+    /**
+     * Canonical form -> the ONE exact map key it identifies. Forms claimed
+     * by two different keys are ambiguous and absent by construction, so a
+     * lookup hit is always safe to restore.
+     */
+    byCanonical: ReadonlyMap<string, string>;
+    /**
+     * Canonical type prefixes of the typed keys in the map ("person",
+     * "redacted"). A candidate of a KNOWN family that resolves to no key is
+     * a DocCloak token the model mangled beyond recovery - callers may
+     * surface it as could-not-restore. Unknown families (site prose like
+     * "[citation_1]") are never DocCloak's to flag.
+     */
+    typePrefixes: ReadonlySet<string>;
+}
+/**
+ * Build the tolerant lookup for a map's keys. Only token-shaped keys
+ * participate: literal surrogate keys are natural-language text whose
+ * case/spacing variants collide with genuine prose, so they keep the
+ * exact-literal path only. Canonical collisions between two different
+ * keys are dropped entirely (never guess between [PERSON_1] and a
+ * renamed "[Person 1]"); exact matching still restores both.
+ */
+export declare function buildTolerantTokenIndex(keys: Iterable<string>): TolerantTokenIndex;
+/**
+ * Resolve one mangled candidate to its exact map key, or null when the
+ * candidate identifies no key or more than one (ambiguity never
+ * restores). Candidates come from bracketed/legacy token scans or from
+ * BARE_TYPED_TOKEN_RE; arbitrary prose canonicalizes to forms no key
+ * owns and falls through to null.
+ */
+export declare function resolveMangledToken(candidate: string, index: TolerantTokenIndex): string | null;

package/dist/restore-tokens.js

@@ -0,0 +1,120 @@
+/**
+ * Tolerant restore-token matching (T098), extracted to a side-effect-free
+ * leaf module so the extension content bundle can import these VALUES
+ * without pulling in the session/surrogate machinery (see the T036 alias
+ * note in DocCloak.Extension/vite.content.config.ts).
+ */
+// ── Tolerant restore matching (T098) ───────────────────────
+//
+// LLM replies mangle placeholders: case changes ("[person_1]"), separator
+// swaps ("[PERSON 1]", "[PERSON-1]"), markdown wrapping ("**[PERSON_1]**",
+// "[`PERSON_1`]", "`PERSON_1`"), dropped brackets ("PERSON_1") and stray
+// edge punctuation ("[PERSON_1.]"). The tolerant matcher reduces mapping
+// keys and reply candidates to one canonical form and restores only on an
+// UNAMBIGUOUS canonical hit: a canonical form claimed by two different map
+// keys is never matched, and a candidate that resolves to nothing is left
+// exactly as the model wrote it. A wrong restore silently corrupts the
+// user's document; a missed restore is visible and recoverable.
+/** A key shaped like one full token ([PERSON_1], <<REDACTED_1>>). */
+const TOKEN_SHAPED_KEY_RE = /^(?:\[[^[\]\n]+\]|<<[^<>\n]+>>)$/;
+/** Markdown emphasis characters LLMs wrap tokens in (bold/code/strike). */
+const MARKDOWN_EMPHASIS_RE = /[*`~]/g;
+/**
+ * Non-alphanumeric runs at either edge of a candidate: bracket/angle
+ * delimiters, stray punctuation and whitespace. Stripping edges (never
+ * the middle) keeps "[PERSON_1.]" and "PERSON_1" comparable while inner
+ * punctuation still distinguishes genuinely different labels.
+ */
+const EDGE_JUNK_RE = /^[^\p{L}\p{N}]+|[^\p{L}\p{N}]+$/gu;
+/** Separator runs unified to one '_': whitespace, underscores, hyphens. */
+const SEPARATOR_RUN_RE = /[\s_-]+/g;
+/**
+ * Bare typed token as an LLM writes it after dropping the brackets:
+ * STRICTLY uppercase segments ending in an underscore-digit run, on
+ * non-wordish boundaries (underscore excluded on both sides so an
+ * embedded "SOME_PERSON_1" identifier never yields a partial match).
+ * Case is deliberately NOT tolerated here: without brackets, only the
+ * exact uppercase shape is distinctive enough to touch ("person 1" in
+ * lowercase prose must stay prose). An immediately preceding '[' or '<'
+ * is also excluded: a dangling "[PERSON_1" (a token split mid-stream,
+ * possibly a truncated "[PERSON_11]") must never be partially restored -
+ * the complete bracketed form is the bracket scan's job.
+ * Global: callers reset lastIndex.
+ */
+export const BARE_TYPED_TOKEN_RE = /(?<![\p{L}\p{N}_[<])[A-Z][A-Z0-9]*(?:_[A-Z0-9]+)*_\d+(?![\p{L}\p{N}_])/gu;
+/**
+ * Canonical comparison form of a placeholder candidate or map key:
+ * markdown emphasis stripped, edge junk (brackets, punctuation, space)
+ * stripped, lowercased, separator runs collapsed to single underscores.
+ * "[PERSON_1]", "[person 1]", "**[PERSON-1]**" and "PERSON_1" all yield
+ * "person_1". Returns null when nothing substantive remains.
+ */
+export function canonicalPlaceholderForm(candidate) {
+    const stripped = candidate
+        .replace(MARKDOWN_EMPHASIS_RE, '')
+        .replace(EDGE_JUNK_RE, '')
+        .toLowerCase();
+    if (stripped.length === 0)
+        return null;
+    const parts = stripped.split(SEPARATOR_RUN_RE).filter((p) => p.length > 0);
+    if (parts.length === 0)
+        return null;
+    return parts.join('_');
+}
+/** Canonical typed-token shape: type prefix + '_' + counter digits. */
+const CANONICAL_TYPED_RE = /^([a-z][a-z0-9_]*)_\d+$/;
+/**
+ * The type-family prefix of a canonical typed token ("person_1" ->
+ * "person", "credit_card_2" -> "credit_card"), or null for canonical
+ * forms that are not typed-token shaped (renamed labels like "ceo").
+ */
+export function canonicalTypePrefix(canonical) {
+    const match = CANONICAL_TYPED_RE.exec(canonical);
+    return match ? match[1] : null;
+}
+/**
+ * Build the tolerant lookup for a map's keys. Only token-shaped keys
+ * participate: literal surrogate keys are natural-language text whose
+ * case/spacing variants collide with genuine prose, so they keep the
+ * exact-literal path only. Canonical collisions between two different
+ * keys are dropped entirely (never guess between [PERSON_1] and a
+ * renamed "[Person 1]"); exact matching still restores both.
+ */
+export function buildTolerantTokenIndex(keys) {
+    const byCanonical = new Map();
+    const ambiguous = new Set();
+    const typePrefixes = new Set();
+    for (const key of keys) {
+        if (!TOKEN_SHAPED_KEY_RE.test(key))
+            continue;
+        const canonical = canonicalPlaceholderForm(key);
+        if (canonical === null)
+            continue;
+        const prefix = canonicalTypePrefix(canonical);
+        if (prefix !== null)
+            typePrefixes.add(prefix);
+        if (ambiguous.has(canonical))
+            continue;
+        const existing = byCanonical.get(canonical);
+        if (existing !== undefined && existing !== key) {
+            byCanonical.delete(canonical);
+            ambiguous.add(canonical);
+            continue;
+        }
+        byCanonical.set(canonical, key);
+    }
+    return { byCanonical, typePrefixes };
+}
+/**
+ * Resolve one mangled candidate to its exact map key, or null when the
+ * candidate identifies no key or more than one (ambiguity never
+ * restores). Candidates come from bracketed/legacy token scans or from
+ * BARE_TYPED_TOKEN_RE; arbitrary prose canonicalizes to forms no key
+ * owns and falls through to null.
+ */
+export function resolveMangledToken(candidate, index) {
+    const canonical = canonicalPlaceholderForm(candidate);
+    if (canonical === null)
+        return null;
+    return index.byCanonical.get(canonical) ?? null;
+}