@discretion/core 0.1.0

package/dist/src/transforms/foldDigits.js

@@ -0,0 +1,135 @@
+/**
+ * Transform 5: DECIMAL DIGIT FOLDING.
+ *
+ * Every Unicode decimal digit is folded to its ASCII equivalent: Arabic-Indic
+ * ٠١٢, Extended Arabic-Indic ۰۱۲, Devanagari ०१२, Bengali ০১২, Thai ๐๑๒,
+ * Tamil, Telugu, Myanmar, Khmer, and the rest.
+ *
+ * WHY THIS EXISTS. NFKC folds FULLWIDTH digits, because those are
+ * compatibility variants of ASCII — but Arabic-Indic and Devanagari digits are
+ * not compatibility variants of anything. They are the ordinary digits of
+ * living scripts, and NFKC leaves them exactly as they are, correctly.
+ *
+ * The consequence for this pipeline was silent un-redaction. Every Stage 1
+ * detector matches `\d` or an explicit `[0-9]` class, so before this transform
+ * existed a Turkish national identity number written `٣٠٢١٤٥٦٤٦٤١٢`, an
+ * Iranian phone number in Extended Arabic-Indic, a Hindi Aadhaar number in
+ * Devanagari, a Thai national ID and a Bengali credit card were ALL reported
+ * as clean documents — measured, not assumed. That is the same class of
+ * failure as an identifier hidden inside HTML markup, and it landed on
+ * precisely the Arabic, Persian, Urdu, Hindi, Bengali and Thai users the
+ * multilingual work exists to serve.
+ *
+ * WHERE IT RUNS, and why last. After homoglyph folding, not before. Folding
+ * digits changes a token's script census — Arabic-Indic digits are Arabic,
+ * ASCII digits are not — so folding them earlier would alter the
+ * dominant-script decision that homoglyph folding depends on, and could flip
+ * a fold that D3 deliberately declines to make. Running afterwards leaves
+ * that decision on the original scripts. ASCII digits are NFKC-stable, so no
+ * further normalization pass is owed.
+ *
+ * THE USER'S TEXT IS NOT REWRITTEN. Folding happens in the NORMALIZED text
+ * only; masking edits the ORIGINAL through the offset map, so a Persian user
+ * sees their own digits back. That separation is the entire point of the
+ * offset-map contract.
+ */
+import { MappedTextBuilder } from '../offsetMap.js';
+/**
+ * The first code point of every Unicode decimal-digit block.
+ *
+ * Each block is ten consecutive code points whose first has value zero, so a
+ * digit's value is `codePoint - blockZero`. Generated by scanning the whole
+ * code space for `\p{Nd}` runs of ten; 72 blocks in the Unicode version
+ * shipped with the runtime. Sorted, which `blockZeroFor` relies on.
+ */
+const DIGIT_BLOCK_ZEROS = [
+    0x30, 0x660, 0x6f0, 0x7c0, 0x966, 0x9e6, 0xa66, 0xae6,
+    0xb66, 0xbe6, 0xc66, 0xce6, 0xd66, 0xde6, 0xe50, 0xed0,
+    0xf20, 0x1040, 0x1090, 0x17e0, 0x1810, 0x1946, 0x19d0, 0x1a80,
+    0x1a90, 0x1b50, 0x1bb0, 0x1c40, 0x1c50, 0xa620, 0xa8d0, 0xa900,
+    0xa9d0, 0xa9f0, 0xaa50, 0xabf0, 0xff10, 0x104a0, 0x10d30, 0x10d40,
+    0x11066, 0x110f0, 0x11136, 0x111d0, 0x112f0, 0x11450, 0x114d0, 0x11650,
+    0x116c0, 0x116d0, 0x11730, 0x118e0, 0x11950, 0x11bf0, 0x11c50, 0x11d50,
+    0x11da0, 0x11de0, 0x11f50, 0x16130, 0x16a60, 0x16ac0, 0x16b50, 0x16d70,
+    0x1ccf0, 0x1d7ce, 0x1e140, 0x1e2f0, 0x1e4f0, 0x1e5f1, 0x1e950, 0x1fbf0,
+];
+/** ASCII '0', whose block is skipped: it is already what we fold to. */
+const ASCII_ZERO = 0x30;
+/**
+ * The zero of the decimal-digit block containing `codePoint`, or undefined.
+ * Binary search over the sorted block list.
+ */
+function blockZeroFor(codePoint) {
+    let low = 0;
+    let high = DIGIT_BLOCK_ZEROS.length - 1;
+    while (low <= high) {
+        const mid = (low + high) >> 1;
+        const zero = DIGIT_BLOCK_ZEROS[mid];
+        if (codePoint < zero)
+            high = mid - 1;
+        else if (codePoint > zero + 9)
+            low = mid + 1;
+        else
+            return zero;
+    }
+    return undefined;
+}
+/** The ASCII digit for a code point, or undefined if it is not a decimal digit. */
+export function asciiDigitFor(codePoint) {
+    const zero = blockZeroFor(codePoint);
+    if (zero === undefined || zero === ASCII_ZERO)
+        return undefined;
+    return String.fromCharCode(ASCII_ZERO + (codePoint - zero));
+}
+/**
+ * Fold every non-ASCII decimal digit to ASCII.
+ *
+ * Most blocks are BMP and fold one code unit to one, leaving offsets
+ * untouched. A few are astral (Osmanya, Brahmi, mathematical digits) and fold
+ * a surrogate PAIR to a single character, so this is not a 1:1 transform and
+ * goes through `MappedTextBuilder` rather than assuming equal lengths — the
+ * same machinery NFKC's expansions use.
+ */
+export function foldDigits(text) {
+    // Fast path: the overwhelming majority of text has no non-ASCII digit, and
+    // must not pay for a builder or a map.
+    let first = -1;
+    for (let i = 0; i < text.length; i += 1) {
+        const cp = text.codePointAt(i);
+        if (cp > 0xffff)
+            i += 1;
+        if (cp !== ASCII_ZERO && asciiDigitFor(cp) !== undefined) {
+            first = i - (cp > 0xffff ? 1 : 0);
+            break;
+        }
+    }
+    if (first === -1)
+        return null;
+    const builder = new MappedTextBuilder();
+    const changes = [];
+    let cursor = 0;
+    for (let i = first; i < text.length; i += 1) {
+        const codePoint = text.codePointAt(i);
+        const width = codePoint > 0xffff ? 2 : 1;
+        const replacement = asciiDigitFor(codePoint);
+        if (replacement === undefined) {
+            i += width - 1;
+            continue;
+        }
+        builder.copyVerbatim(text, cursor, i);
+        builder.replaceRange(replacement, i, i + width);
+        changes.push({
+            kind: 'fold-digits',
+            start: i,
+            end: i + width,
+            before: text.slice(i, i + width),
+            after: replacement,
+        });
+        cursor = i + width;
+        i += width - 1;
+    }
+    builder.copyVerbatim(text, cursor, text.length);
+    const { text: out, map } = builder.finish(text.length);
+    return { text: out, map, changes };
+}
+//# sourceMappingURL=foldDigits.js.map

package/dist/src/transforms/foldDigits.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"foldDigits.js","sourceRoot":"","sources":["../../../src/transforms/foldDigits.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkCG;AAEH,OAAO,EAAE,iBAAiB,EAA6C,MAAM,iBAAiB,CAAC;AAE/F;;;;;;;GAOG;AACH,MAAM,iBAAiB,GAAsB;IAC3C,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK;IACrD,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK;IACtD,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM;IAC7D,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM;IAC9D,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO;IACjE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO;IACtE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO;IACtE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO;IACtE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO;CACvE,CAAC;AAEF,wEAAwE;AACxE,MAAM,UAAU,GAAG,IAAI,CAAC;AAExB;;;GAGG;AACH,SAAS,YAAY,CAAC,SAAiB;IACrC,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,IAAI,IAAI,GAAG,iBAAiB,CAAC,MAAM,GAAG,CAAC,CAAC;IACxC,OAAO,GAAG,IAAI,IAAI,EAAE,CAAC;QACnB,MAAM,GAAG,GAAG,CAAC,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC;QAC9B,MAAM,IAAI,GAAG,iBAAiB,CAAC,GAAG,CAAE,CAAC;QACrC,IAAI,SAAS,GAAG,IAAI;YAAE,IAAI,GAAG,GAAG,GAAG,CAAC,CAAC;aAChC,IAAI,SAAS,GAAG,IAAI,GAAG,CAAC;YAAE,GAAG,GAAG,GAAG,GAAG,CAAC,CAAC;;YACxC,OAAO,IAAI,CAAC;IACnB,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,mFAAmF;AACnF,MAAM,UAAU,aAAa,CAAC,SAAiB;IAC7C,MAAM,IAAI,GAAG,YAAY,CAAC,SAAS,CAAC,CAAC;IACrC,IAAI,IAAI,KAAK,SAAS,IAAI,IAAI,KAAK,UAAU;QAAE,OAAO,SAAS,CAAC;IAChE,OAAO,MAAM,CAAC,YAAY,CAAC,UAAU,GAAG,CAAC,SAAS,GAAG,IAAI,CAAC,CAAC,CAAC;AAC9D,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,UAAU,CAAC,IAAY;IACrC,2EAA2E;IAC3E,uCAAuC;IACvC,IAAI,KAAK,GAAG,CAAC,CAAC,CAAC;IACf,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;QACxC,MAAM,EAAE,GAAG,IAAI,CAAC,WAAW,CAAC,CAAC,CAAE,CAAC;QAChC,IAAI,EAAE,GAAG,MAAM;YAAE,CAAC,IAAI,CAAC,CAAC;QACxB,IAAI,EAAE,KAAK,UAAU,IAAI,aAAa,CAAC,EAAE,CAAC,KAAK,SAAS,EAAE,CAAC;YACzD,KAAK,GAAG,CAAC,GAAG,CAAC,EAAE,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAClC,MAAM;QACR,CAAC;IACH,CAAC;IACD,IAAI,KAAK,KAAK,CAAC,CAAC;QAAE,OAAO,IAAI,CAAC;IAE9B,MAAM,OAAO,GAAG,IAAI,iBAAiB,EAAE,CAAC;IACxC,MAAM,OAAO,GAAiB,EAAE,CAAC;IACjC,IAAI,MAAM,GAAG,CAAC,CAAC;IAEf,KAAK,IAAI,CAAC,GAAG,KAAK,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;QAC5C,MAAM,SAAS,GAAG,IAAI,CAAC,WAAW,CAAC,CAAC,CAAE,CAAC;QACvC,MAAM,KAAK,GAAG,SAAS,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACzC,MAAM,WAAW,GAAG,aAAa,CAAC,SAAS,CAAC,CAAC;QAC7C,IAAI,WAAW,KAAK,SAAS,EAAE,CAAC;YAC9B,CAAC,IAAI,KAAK,GAAG,CAAC,CAAC;YACf,SAAS;QACX,CAAC;QAED,OAAO,CAAC,YAAY,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC;QACtC,OAAO,CAAC,YAAY,CAAC,WAAW,EAAE,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,CAAC;QAChD,OAAO,CAAC,IAAI,CAAC;YACX,IAAI,EAAE,aAAa;YACnB,KAAK,EAAE,CAAC;YACR,GAAG,EAAE,CAAC,GAAG,KAAK;YACd,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC;YAChC,KAAK,EAAE,WAAW;SACnB,CAAC,CAAC;QACH,MAAM,GAAG,CAAC,GAAG,KAAK,CAAC;QACnB,CAAC,IAAI,KAAK,GAAG,CAAC,CAAC;IACjB,CAAC;IAED,OAAO,CAAC,YAAY,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;IAChD,MAAM,EAAE,IAAI,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACvD,OAAO,EAAE,IAAI,EAAE,GAAG,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC;AACrC,CAAC"}

package/dist/src/transforms/graphemes.d.ts

@@ -0,0 +1,6 @@
+/**
+ * Shared grapheme-cluster segmenter. Grapheme segmentation (UAX #29) is
+ * locale-independent, so one instance serves the whole library.
+ */
+export declare const graphemeSegmenter: Intl.Segmenter;
+//# sourceMappingURL=graphemes.d.ts.map

package/dist/src/transforms/graphemes.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"graphemes.d.ts","sourceRoot":"","sources":["../../../src/transforms/graphemes.ts"],"names":[],"mappings":"AAAA;;;GAGG;AACH,eAAO,MAAM,iBAAiB,gBAA6D,CAAC"}

package/dist/src/transforms/graphemes.js

@@ -0,0 +1,6 @@
+/**
+ * Shared grapheme-cluster segmenter. Grapheme segmentation (UAX #29) is
+ * locale-independent, so one instance serves the whole library.
+ */
+export const graphemeSegmenter = new Intl.Segmenter(undefined, { granularity: 'grapheme' });
+//# sourceMappingURL=graphemes.js.map

package/dist/src/transforms/graphemes.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"graphemes.js","sourceRoot":"","sources":["../../../src/transforms/graphemes.ts"],"names":[],"mappings":"AAAA;;;GAGG;AACH,MAAM,CAAC,MAAM,iBAAiB,GAAG,IAAI,IAAI,CAAC,SAAS,CAAC,SAAS,EAAE,EAAE,WAAW,EAAE,UAAU,EAAE,CAAC,CAAC"}

package/dist/src/transforms/homoglyphFold.d.ts

@@ -0,0 +1,5 @@
+import { type TransformStepResult } from '../offsetMap.js';
+/** Test hook: drop the memoized index (e.g. around mocked data). */
+export declare function resetSkeletonIndexForTests(): void;
+export declare function foldHomoglyphs(text: string): TransformStepResult | null;
+//# sourceMappingURL=homoglyphFold.d.ts.map

package/dist/src/transforms/homoglyphFold.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"homoglyphFold.d.ts","sourceRoot":"","sources":["../../../src/transforms/homoglyphFold.ts"],"names":[],"mappings":"AAwCA,OAAO,EAAqB,KAAK,mBAAmB,EAAmB,MAAM,iBAAiB,CAAC;AAkD/F,oEAAoE;AACpE,wBAAgB,0BAA0B,IAAI,IAAI,CAEjD;AA+FD,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,mBAAmB,GAAG,IAAI,CAgCvE"}

package/dist/src/transforms/homoglyphFold.js

@@ -0,0 +1,214 @@
+/**
+ * Transform 3: SELECTIVE HOMOGLYPH FOLDING.
+ *
+ * THE RULE — and why it is selective:
+ * Blanket folding would be a severe bug: folding every Cyrillic а to Latin a
+ * would turn an entire Russian document into unreadable Latin nonsense and
+ * destroy detection for every non-Latin user. A character is folded ONLY when
+ * it is script-anomalous within its own token:
+ *
+ *   1. Split the text into tokens on whitespace and punctuation.
+ *   2. Determine each token's dominant script (strict majority of its
+ *      letters; neutral characters don't count).
+ *   3. Fold a character only if its script differs from the token's dominant
+ *      script AND the confusables table gives it a mapping into that script.
+ *
+ * A token that is wholly Cyrillic / Greek / Arabic / … has no anomalous
+ * characters and is left completely untouched.
+ *
+ *   • "pаssword" (Cyrillic а in a Latin token) → folds to "password".
+ *   • "пароль"   (ordinary Cyrillic word)      → byte-identical.
+ *
+ * Mapping INTO the dominant script works in both directions:
+ *   - directly: the character's Unicode skeleton is in the dominant script
+ *     (Cyrillic а → skeleton "a", Latin) — replace with the skeleton;
+ *   - via equivalence: some character OF the dominant script shares the same
+ *     skeleton (Latin o inside a Greek token → Greek ο, whose skeleton is
+ *     also "o") — replace with that character.
+ *
+ * Replacements are restricted to letters/marks/digits: a replacement that
+ * introduced punctuation or whitespace would change tokenization on a second
+ * pass and break idempotency.
+ *
+ * This transform runs AFTER NFKC, so the confusables data's skeletons are
+ * NFKC-normalized at generation time; normalize() additionally re-runs NFKC
+ * after folding (see normalization.ts) so a folded character can compose
+ * with a following combining mark exactly as it would have in pass one.
+ */
+import { CONFUSABLES } from '@discretion/data';
+import { classifyCodePoint, scriptsCompatible } from '../scripts.js';
+import { MappedTextBuilder } from '../offsetMap.js';
+/** Tokens are maximal runs of characters that are neither whitespace nor punctuation. */
+const TOKEN = /[^\s\p{P}]+/gu;
+/** Replacements may only contain letters, combining marks, and digits. */
+const SAFE_REPLACEMENT = /^[\p{L}\p{M}\p{N}]+$/u;
+/**
+ * skeleton → script → representative character of that script sharing the
+ * skeleton. Built lazily on first use from the confusables table (plus each
+ * skeleton itself as the representative of its own script). Where several
+ * characters of one script share a skeleton, the lowest code point wins so
+ * regeneration of the data cannot reorder results.
+ */
+let skeletonIndex = null;
+function getSkeletonIndex() {
+    if (skeletonIndex)
+        return skeletonIndex;
+    const index = new Map();
+    const consider = (skeleton, script, repr) => {
+        if (script === 'other')
+            return;
+        // A representative becomes replacement text, so it must survive the rest
+        // of the pipeline unchanged: NFKC-stable (U+037A GREEK YPOGEGRAMMENI, for
+        // instance, decomposes to space + combining mark and would smuggle
+        // whitespace into a token) and free of punctuation/whitespace that would
+        // change tokenization on a second pass.
+        if (repr.normalize('NFKC') !== repr)
+            return;
+        if (!SAFE_REPLACEMENT.test(repr))
+            return;
+        let byScript = index.get(skeleton);
+        if (!byScript) {
+            byScript = new Map();
+            index.set(skeleton, byScript);
+        }
+        const existing = byScript.get(script);
+        if (existing === undefined || repr.codePointAt(0) < existing.codePointAt(0)) {
+            byScript.set(script, repr);
+        }
+    };
+    for (const [cp, entry] of CONFUSABLES) {
+        // The confusable character itself represents its script for this skeleton…
+        consider(entry.skeleton, entry.sourceScript, String.fromCodePoint(cp));
+        // …and the skeleton represents its own script (preferred when reachable:
+        // for single-char skeletons it is usually the lowest code point anyway).
+        consider(entry.skeleton, entry.skeletonScript, entry.skeleton);
+    }
+    skeletonIndex = index;
+    return index;
+}
+/** Test hook: drop the memoized index (e.g. around mocked data). */
+export function resetSkeletonIndexForTests() {
+    skeletonIndex = null;
+}
+/**
+ * The replacement for folding `cp` into `dominant`, or null when the
+ * confusables table offers no mapping into that script.
+ */
+function foldTarget(cp, dominant) {
+    const entry = CONFUSABLES.get(cp);
+    // A character absent from the table is its own skeleton: it can still be
+    // the skeleton OF characters in the dominant script — e.g. Latin 'o' has
+    // no entry, but inside a Greek token it folds to Greek omicron, whose
+    // skeleton is 'o' (the equivalence direction).
+    const skeleton = entry ? entry.skeleton : String.fromCodePoint(cp);
+    let replacement;
+    if (entry && entry.skeletonScript === dominant) {
+        replacement = entry.skeleton;
+    }
+    else {
+        replacement = getSkeletonIndex().get(skeleton)?.get(dominant);
+    }
+    if (replacement === undefined)
+        return null;
+    if (replacement === String.fromCodePoint(cp))
+        return null; // no-op
+    // Defense in depth (the index already filters its representatives, and
+    // generated skeletons are NFKC-normalized): a replacement must not change
+    // tokenization or decay under the NFKC re-pass, or folding would not be
+    // idempotent.
+    if (!SAFE_REPLACEMENT.test(replacement))
+        return null;
+    if (replacement.normalize('NFKC') !== replacement)
+        return null;
+    return replacement;
+}
+const ZWNJ = String.fromCharCode(0x200c);
+/** Compute the folds for one token; empty when the token is not mixed-script. */
+function computeFolds(token) {
+    // A ZWNJ surviving the strip stage marks joining-script morphology
+    // (Persian, Hindi, …). Folding inside such a token could change the script
+    // of a ZWNJ neighbor and flip the preservation decision on the next pass —
+    // so these tokens are never folding candidates.
+    if (token.includes(ZWNJ))
+        return [];
+    // First pass: per-script letter counts to find the token's dominant script.
+    const counts = new Map();
+    for (const char of token) {
+        const cls = classifyCodePoint(char.codePointAt(0));
+        if (cls === 'neutral' || cls === 'other')
+            continue;
+        counts.set(cls, (counts.get(cls) ?? 0) + 1);
+    }
+    let dominant = null;
+    let best = 0;
+    let tied = false;
+    for (const [script, count] of counts) {
+        if (count > best) {
+            best = count;
+            dominant = script;
+            tied = false;
+        }
+        else if (count === best) {
+            tied = true;
+        }
+    }
+    // No dominant script (all-neutral, unsupported-script, or tied token):
+    // be conservative and leave the token alone. The tie rule is ratified —
+    // an evenly split token gives no evidence of which script is "home".
+    if (dominant === null || tied)
+        return [];
+    // Second pass: fold script-anomalous characters that can map into `dominant`.
+    const folds = [];
+    let offset = 0;
+    for (const char of token) {
+        const cp = char.codePointAt(0);
+        const cls = classifyCodePoint(cp);
+        // Neutral characters (digits, marks) are never anomalous. Letters of the
+        // dominant script — or of a script COMPATIBLE with it (Han/Kana/Latin in
+        // Japanese text, Hangul/Han in Korean; see scriptsCompatible) — are at
+        // home. Everything else, including letters of unsupported scripts like
+        // Cherokee (classified 'other'), is anomalous and folds if mappable.
+        if (cls !== 'neutral' && !scriptsCompatible(cls, dominant)) {
+            const after = foldTarget(cp, dominant);
+            if (after !== null) {
+                folds.push({ offset, length: char.length, before: char, after });
+            }
+        }
+        offset += char.length;
+    }
+    return folds;
+}
+export function foldHomoglyphs(text) {
+    const builder = new MappedTextBuilder();
+    const changes = [];
+    let cursor = 0;
+    TOKEN.lastIndex = 0;
+    for (const match of text.matchAll(TOKEN)) {
+        const token = match[0];
+        const tokenStart = match.index;
+        const folds = computeFolds(token);
+        if (folds.length === 0)
+            continue;
+        // Copy everything between the previous fold and this token, then splice
+        // the folds in at their exact offsets.
+        for (const fold of folds) {
+            const foldStart = tokenStart + fold.offset;
+            builder.copyVerbatim(text, cursor, foldStart);
+            builder.replaceRange(fold.after, foldStart, foldStart + fold.length);
+            changes.push({
+                kind: 'homoglyph-fold',
+                start: foldStart,
+                end: foldStart + fold.length,
+                before: fold.before,
+                after: fold.after,
+            });
+            cursor = foldStart + fold.length;
+        }
+    }
+    if (changes.length === 0)
+        return null;
+    builder.copyVerbatim(text, cursor, text.length);
+    const { text: out, map } = builder.finish(text.length);
+    return { text: out, map, changes };
+}
+//# sourceMappingURL=homoglyphFold.js.map

package/dist/src/transforms/homoglyphFold.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"homoglyphFold.js","sourceRoot":"","sources":["../../../src/transforms/homoglyphFold.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoCG;AACH,OAAO,EAAE,WAAW,EAAwB,MAAM,kBAAkB,CAAC;AACrE,OAAO,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAErE,OAAO,EAAE,iBAAiB,EAA6C,MAAM,iBAAiB,CAAC;AAE/F,yFAAyF;AACzF,MAAM,KAAK,GAAG,eAAe,CAAC;AAE9B,0EAA0E;AAC1E,MAAM,gBAAgB,GAAG,uBAAuB,CAAC;AAEjD;;;;;;GAMG;AACH,IAAI,aAAa,GAAgD,IAAI,CAAC;AAEtE,SAAS,gBAAgB;IACvB,IAAI,aAAa;QAAE,OAAO,aAAa,CAAC;IACxC,MAAM,KAAK,GAAG,IAAI,GAAG,EAAmC,CAAC;IACzD,MAAM,QAAQ,GAAG,CAAC,QAAgB,EAAE,MAAkB,EAAE,IAAY,EAAQ,EAAE;QAC5E,IAAI,MAAM,KAAK,OAAO;YAAE,OAAO;QAC/B,yEAAyE;QACzE,0EAA0E;QAC1E,mEAAmE;QACnE,yEAAyE;QACzE,wCAAwC;QACxC,IAAI,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,KAAK,IAAI;YAAE,OAAO;QAC5C,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC;YAAE,OAAO;QACzC,IAAI,QAAQ,GAAG,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QACnC,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,QAAQ,GAAG,IAAI,GAAG,EAAsB,CAAC;YACzC,KAAK,CAAC,GAAG,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;QAChC,CAAC;QACD,MAAM,QAAQ,GAAG,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QACtC,IAAI,QAAQ,KAAK,SAAS,IAAI,IAAI,CAAC,WAAW,CAAC,CAAC,CAAE,GAAG,QAAQ,CAAC,WAAW,CAAC,CAAC,CAAE,EAAE,CAAC;YAC9E,QAAQ,CAAC,GAAG,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;QAC7B,CAAC;IACH,CAAC,CAAC;IACF,KAAK,MAAM,CAAC,EAAE,EAAE,KAAK,CAAC,IAAI,WAAW,EAAE,CAAC;QACtC,2EAA2E;QAC3E,QAAQ,CAAC,KAAK,CAAC,QAAQ,EAAE,KAAK,CAAC,YAAY,EAAE,MAAM,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC,CAAC;QACvE,yEAAyE;QACzE,yEAAyE;QACzE,QAAQ,CAAC,KAAK,CAAC,QAAQ,EAAE,KAAK,CAAC,cAAc,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC;IACjE,CAAC;IACD,aAAa,GAAG,KAAK,CAAC;IACtB,OAAO,KAAK,CAAC;AACf,CAAC;AAED,oEAAoE;AACpE,MAAM,UAAU,0BAA0B;IACxC,aAAa,GAAG,IAAI,CAAC;AACvB,CAAC;AAED;;;GAGG;AACH,SAAS,UAAU,CAAC,EAAU,EAAE,QAAoB;IAClD,MAAM,KAAK,GAAgC,WAAW,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAC/D,yEAAyE;IACzE,yEAAyE;IACzE,sEAAsE;IACtE,+CAA+C;IAC/C,MAAM,QAAQ,GAAG,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC;IACnE,IAAI,WAA+B,CAAC;IACpC,IAAI,KAAK,IAAI,KAAK,CAAC,cAAc,KAAK,QAAQ,EAAE,CAAC;QAC/C,WAAW,GAAG,KAAK,CAAC,QAAQ,CAAC;IAC/B,CAAC;SAAM,CAAC;QACN,WAAW,GAAG,gBAAgB,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,GAAG,CAAC,QAAQ,CAAC,CAAC;IAChE,CAAC;IACD,IAAI,WAAW,KAAK,SAAS;QAAE,OAAO,IAAI,CAAC;IAC3C,IAAI,WAAW,KAAK,MAAM,CAAC,aAAa,CAAC,EAAE,CAAC;QAAE,OAAO,IAAI,CAAC,CAAC,QAAQ;IACnE,uEAAuE;IACvE,0EAA0E;IAC1E,wEAAwE;IACxE,cAAc;IACd,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,WAAW,CAAC;QAAE,OAAO,IAAI,CAAC;IACrD,IAAI,WAAW,CAAC,SAAS,CAAC,MAAM,CAAC,KAAK,WAAW;QAAE,OAAO,IAAI,CAAC;IAC/D,OAAO,WAAW,CAAC;AACrB,CAAC;AAWD,MAAM,IAAI,GAAG,MAAM,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;AAEzC,iFAAiF;AACjF,SAAS,YAAY,CAAC,KAAa;IACjC,mEAAmE;IACnE,2EAA2E;IAC3E,2EAA2E;IAC3E,gDAAgD;IAChD,IAAI,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC;QAAE,OAAO,EAAE,CAAC;IAEpC,4EAA4E;IAC5E,MAAM,MAAM,GAAG,IAAI,GAAG,EAAsB,CAAC;IAC7C,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,GAAG,GAAG,iBAAiB,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAE,CAAC,CAAC;QACpD,IAAI,GAAG,KAAK,SAAS,IAAI,GAAG,KAAK,OAAO;YAAE,SAAS;QACnD,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC9C,CAAC;IACD,IAAI,QAAQ,GAAsB,IAAI,CAAC;IACvC,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,IAAI,IAAI,GAAG,KAAK,CAAC;IACjB,KAAK,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,IAAI,MAAM,EAAE,CAAC;QACrC,IAAI,KAAK,GAAG,IAAI,EAAE,CAAC;YACjB,IAAI,GAAG,KAAK,CAAC;YACb,QAAQ,GAAG,MAAM,CAAC;YAClB,IAAI,GAAG,KAAK,CAAC;QACf,CAAC;aAAM,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;YAC1B,IAAI,GAAG,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IACD,uEAAuE;IACvE,wEAAwE;IACxE,qEAAqE;IACrE,IAAI,QAAQ,KAAK,IAAI,IAAI,IAAI;QAAE,OAAO,EAAE,CAAC;IAEzC,8EAA8E;IAC9E,MAAM,KAAK,GAAW,EAAE,CAAC;IACzB,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,EAAE,GAAG,IAAI,CAAC,WAAW,CAAC,CAAC,CAAE,CAAC;QAChC,MAAM,GAAG,GAAG,iBAAiB,CAAC,EAAE,CAAC,CAAC;QAClC,yEAAyE;QACzE,yEAAyE;QACzE,uEAAuE;QACvE,uEAAuE;QACvE,qEAAqE;QACrE,IAAI,GAAG,KAAK,SAAS,IAAI,CAAC,iBAAiB,CAAC,GAAG,EAAE,QAAQ,CAAC,EAAE,CAAC;YAC3D,MAAM,KAAK,GAAG,UAAU,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC;YACvC,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;gBACnB,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;YACnE,CAAC;QACH,CAAC;QACD,MAAM,IAAI,IAAI,CAAC,MAAM,CAAC;IACxB,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,IAAY;IACzC,MAAM,OAAO,GAAG,IAAI,iBAAiB,EAAE,CAAC;IACxC,MAAM,OAAO,GAAiB,EAAE,CAAC;IACjC,IAAI,MAAM,GAAG,CAAC,CAAC;IAEf,KAAK,CAAC,SAAS,GAAG,CAAC,CAAC;IACpB,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;QACzC,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;QACvB,MAAM,UAAU,GAAG,KAAK,CAAC,KAAM,CAAC;QAChC,MAAM,KAAK,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC;QAClC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;YAAE,SAAS;QACjC,wEAAwE;QACxE,uCAAuC;QACvC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,MAAM,SAAS,GAAG,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC;YAC3C,OAAO,CAAC,YAAY,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC;YAC9C,OAAO,CAAC,YAAY,CAAC,IAAI,CAAC,KAAK,EAAE,SAAS,EAAE,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC;YACrE,OAAO,CAAC,IAAI,CAAC;gBACX,IAAI,EAAE,gBAAgB;gBACtB,KAAK,EAAE,SAAS;gBAChB,GAAG,EAAE,SAAS,GAAG,IAAI,CAAC,MAAM;gBAC5B,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,KAAK,EAAE,IAAI,CAAC,KAAK;aAClB,CAAC,CAAC;YACH,MAAM,GAAG,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC;QACnC,CAAC;IACH,CAAC;IACD,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IAEtC,OAAO,CAAC,YAAY,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;IAChD,MAAM,EAAE,IAAI,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACvD,OAAO,EAAE,IAAI,EAAE,GAAG,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC;AACrC,CAAC"}

package/dist/src/transforms/nfkc.d.ts

@@ -0,0 +1,20 @@
+/**
+ * Transform 2: UNICODE NFKC, applied PER GRAPHEME CLUSTER.
+ *
+ * Normalizing cluster-by-cluster is what makes an exact offset map possible:
+ * each cluster's output is recorded against that cluster's input range as we
+ * go. (Normalizing the whole string at once and reconstructing the alignment
+ * afterwards is guesswork — expansions, contractions and reorderings make the
+ * aligned diff ambiguous — so it is deliberately NOT done that way.)
+ *
+ * Cluster-local NFKC is sound because canonical composition only happens
+ * between a base and its combining marks, and UAX #29 never splits a base
+ * from its marks (nor a Hangul jamo sequence) across cluster boundaries.
+ *
+ * Expansions (fi → "fi", ½ → "1⁄2"): every output unit maps to the cluster
+ * start. Contractions (e + U+0301 → é): the single output unit maps to the
+ * cluster start and the sentinel/next cluster provides the end.
+ */
+import { type TransformStepResult } from '../offsetMap.js';
+export declare function nfkcByGrapheme(text: string): TransformStepResult | null;
+//# sourceMappingURL=nfkc.d.ts.map

package/dist/src/transforms/nfkc.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"nfkc.d.ts","sourceRoot":"","sources":["../../../src/transforms/nfkc.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AACH,OAAO,EAAqB,KAAK,mBAAmB,EAAmB,MAAM,iBAAiB,CAAC;AAG/F,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,mBAAmB,GAAG,IAAI,CA2BvE"}

package/dist/src/transforms/nfkc.js

@@ -0,0 +1,48 @@
+/**
+ * Transform 2: UNICODE NFKC, applied PER GRAPHEME CLUSTER.
+ *
+ * Normalizing cluster-by-cluster is what makes an exact offset map possible:
+ * each cluster's output is recorded against that cluster's input range as we
+ * go. (Normalizing the whole string at once and reconstructing the alignment
+ * afterwards is guesswork — expansions, contractions and reorderings make the
+ * aligned diff ambiguous — so it is deliberately NOT done that way.)
+ *
+ * Cluster-local NFKC is sound because canonical composition only happens
+ * between a base and its combining marks, and UAX #29 never splits a base
+ * from its marks (nor a Hangul jamo sequence) across cluster boundaries.
+ *
+ * Expansions (fi → "fi", ½ → "1⁄2"): every output unit maps to the cluster
+ * start. Contractions (e + U+0301 → é): the single output unit maps to the
+ * cluster start and the sentinel/next cluster provides the end.
+ */
+import { MappedTextBuilder } from '../offsetMap.js';
+import { graphemeSegmenter } from './graphemes.js';
+export function nfkcByGrapheme(text) {
+    // Fast path: already NFKC-normalized (covers pure ASCII instantly).
+    if (text.normalize('NFKC') === text)
+        return null;
+    const builder = new MappedTextBuilder();
+    const changes = [];
+    for (const { segment, index } of graphemeSegmenter.segment(text)) {
+        const segmentEnd = index + segment.length;
+        const normalized = segment.normalize('NFKC');
+        if (normalized === segment) {
+            builder.copyVerbatim(text, index, segmentEnd);
+        }
+        else {
+            builder.replaceRange(normalized, index, segmentEnd);
+            changes.push({
+                kind: 'nfkc',
+                start: index,
+                end: segmentEnd,
+                before: segment,
+                after: normalized,
+            });
+        }
+    }
+    if (changes.length === 0)
+        return null;
+    const { text: out, map } = builder.finish(text.length);
+    return { text: out, map, changes };
+}
+//# sourceMappingURL=nfkc.js.map

package/dist/src/transforms/nfkc.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"nfkc.js","sourceRoot":"","sources":["../../../src/transforms/nfkc.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AACH,OAAO,EAAE,iBAAiB,EAA6C,MAAM,iBAAiB,CAAC;AAC/F,OAAO,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;AAEnD,MAAM,UAAU,cAAc,CAAC,IAAY;IACzC,oEAAoE;IACpE,IAAI,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,KAAK,IAAI;QAAE,OAAO,IAAI,CAAC;IAEjD,MAAM,OAAO,GAAG,IAAI,iBAAiB,EAAE,CAAC;IACxC,MAAM,OAAO,GAAiB,EAAE,CAAC;IAEjC,KAAK,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,iBAAiB,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;QACjE,MAAM,UAAU,GAAG,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC;QAC1C,MAAM,UAAU,GAAG,OAAO,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;QAC7C,IAAI,UAAU,KAAK,OAAO,EAAE,CAAC;YAC3B,OAAO,CAAC,YAAY,CAAC,IAAI,EAAE,KAAK,EAAE,UAAU,CAAC,CAAC;QAChD,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,YAAY,CAAC,UAAU,EAAE,KAAK,EAAE,UAAU,CAAC,CAAC;YACpD,OAAO,CAAC,IAAI,CAAC;gBACX,IAAI,EAAE,MAAM;gBACZ,KAAK,EAAE,KAAK;gBACZ,GAAG,EAAE,UAAU;gBACf,MAAM,EAAE,OAAO;gBACf,KAAK,EAAE,UAAU;aAClB,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IACtC,MAAM,EAAE,IAAI,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACvD,OAAO,EAAE,IAAI,EAAE,GAAG,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC;AACrC,CAAC"}

package/dist/src/transforms/stripInvisibles.d.ts

@@ -0,0 +1,37 @@
+/**
+ * Transform 1: STRIP INVISIBLES.
+ *
+ * Removes zero-width and bidi control characters:
+ *   U+200B–U+200F  (ZWSP, ZWNJ, ZWJ, LRM, RLM)
+ *   U+202A–U+202E  (bidi embedding/override controls)
+ *   U+2060–U+2064  (word joiner, invisible operators)
+ *   U+FEFF         (BOM / zero-width no-break space)
+ *   U+00AD         (soft hyphen)
+ *
+ * These are pure noise for detection and are the classic obfuscation vector
+ * (a ZWSP dropped into the middle of "john@example.com"), so they go first —
+ * they interfere with every transform downstream.
+ *
+ * TWO CHARACTERS IN THE SET ARE SOMETIMES STRUCTURAL, NOT NOISE:
+ *
+ * THE EMOJI EXCEPTION — ZWJ (U+200D) inside an emoji sequence joins the
+ * sequence together; stripping it would corrupt 👨‍👩‍👧 into 👨👩👧. Per UAX #29,
+ * ZWJ and ZWNJ are the only characters in the strip set that can sit INSIDE
+ * a grapheme cluster (all the others have Grapheme_Cluster_Break=Control and
+ * form their own clusters), so the rule is: leave any cluster containing an
+ * Extended_Pictographic code point completely untouched.
+ *
+ * THE JOINING-SCRIPT EXCEPTION — ZWNJ (U+200C) is linguistically MEANINGFUL
+ * in scripts with cursive joining or conjunct formation: in Persian/Urdu and
+ * other Arabic-script languages it controls letter shaping and word
+ * segmentation (می‌خواهم), and in Indic scripts it suppresses conjuncts
+ * (क्‌ष renders dead consonant + ssa instead of the kṣa ligature). Stripping
+ * it there CORRUPTS the text. Rule (mirroring the emoji reasoning): preserve
+ * ZWNJ when its nearest substantive neighbors on BOTH sides — skipping other
+ * invisibles and combining marks — belong to a ZWNJ-using script. Everywhere
+ * else (Latin ligature suppression, obfuscation, stray controls) it is noise
+ * and is stripped.
+ */
+import { type TransformStepResult } from '../offsetMap.js';
+export declare function stripInvisibles(text: string): TransformStepResult | null;
+//# sourceMappingURL=stripInvisibles.d.ts.map

package/dist/src/transforms/stripInvisibles.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"stripInvisibles.d.ts","sourceRoot":"","sources":["../../../src/transforms/stripInvisibles.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;AACH,OAAO,EAAqB,KAAK,mBAAmB,EAAmB,MAAM,iBAAiB,CAAC;AA6E/F,wBAAgB,eAAe,CAAC,IAAI,EAAE,MAAM,GAAG,mBAAmB,GAAG,IAAI,CA8CxE"}