@elanlanguages/bridge-anonymization 0.1.0

package/dist/ner/tokenizer.js

@@ -0,0 +1,341 @@
+/**
+ * WordPiece Tokenizer
+ * Tokenizes text into subword tokens while maintaining character offset mapping
+ * Compatible with BERT-style models
+ */
+/**
+ * Default tokenizer configuration for BERT-style models
+ */
+export const DEFAULT_TOKENIZER_CONFIG = {
+    maxLength: 512,
+    unkToken: '[UNK]',
+    clsToken: '[CLS]',
+    sepToken: '[SEP]',
+    padToken: '[PAD]',
+    maskToken: '[MASK]',
+    doLowerCase: true,
+    stripAccents: true,
+};
+/**
+ * WordPiece Tokenizer implementation
+ */
+export class WordPieceTokenizer {
+    vocab;
+    inverseVocab;
+    config;
+    // Special token IDs
+    unkId;
+    clsId;
+    sepId;
+    padId;
+    constructor(vocab, config = {}) {
+        this.vocab = vocab;
+        this.config = { ...DEFAULT_TOKENIZER_CONFIG, ...config };
+        // Build inverse vocab
+        this.inverseVocab = new Map();
+        for (const [token, id] of vocab) {
+            this.inverseVocab.set(id, token);
+        }
+        // Get special token IDs
+        this.unkId = this.vocab.get(this.config.unkToken) ?? 0;
+        this.clsId = this.vocab.get(this.config.clsToken) ?? 101;
+        this.sepId = this.vocab.get(this.config.sepToken) ?? 102;
+        this.padId = this.vocab.get(this.config.padToken) ?? 0;
+    }
+    /**
+     * Tokenizes text into tokens with offset tracking
+     */
+    tokenize(text) {
+        const tokens = [];
+        const tokenToCharSpan = [];
+        // Add [CLS] token
+        tokens.push({
+            id: this.clsId,
+            token: this.config.clsToken,
+            start: 0,
+            end: 0,
+            isContinuation: false,
+            isSpecial: true,
+        });
+        tokenToCharSpan.push(null);
+        // Preprocess text
+        const processedText = this.preprocess(text);
+        // Split into words by whitespace
+        const wordSpans = this.splitIntoWords(processedText, text);
+        // Tokenize each word
+        for (const { word, start, end } of wordSpans) {
+            const wordTokens = this.tokenizeWord(word, start, end);
+            tokens.push(...wordTokens);
+            for (const t of wordTokens) {
+                tokenToCharSpan.push([t.start, t.end]);
+            }
+        }
+        // Add [SEP] token
+        tokens.push({
+            id: this.sepId,
+            token: this.config.sepToken,
+            start: text.length,
+            end: text.length,
+            isContinuation: false,
+            isSpecial: true,
+        });
+        tokenToCharSpan.push(null);
+        // Truncate if necessary
+        const maxTokens = this.config.maxLength;
+        if (tokens.length > maxTokens) {
+            tokens.length = maxTokens - 1;
+            tokenToCharSpan.length = maxTokens - 1;
+            // Add [SEP] at end
+            tokens.push({
+                id: this.sepId,
+                token: this.config.sepToken,
+                start: text.length,
+                end: text.length,
+                isContinuation: false,
+                isSpecial: true,
+            });
+            tokenToCharSpan.push(null);
+        }
+        // Build arrays
+        const inputIds = tokens.map((t) => t.id);
+        const attentionMask = tokens.map(() => 1);
+        const tokenTypeIds = tokens.map(() => 0);
+        return {
+            tokens,
+            inputIds,
+            attentionMask,
+            tokenTypeIds,
+            tokenToCharSpan,
+        };
+    }
+    /**
+     * Preprocesses text (lowercase, accent stripping)
+     */
+    preprocess(text) {
+        let processed = text;
+        if (this.config.doLowerCase) {
+            processed = processed.toLowerCase();
+        }
+        if (this.config.stripAccents) {
+            processed = this.stripAccents(processed);
+        }
+        return processed;
+    }
+    /**
+     * Strips accents from text
+     */
+    stripAccents(text) {
+        return text.normalize('NFD').replace(/[\u0300-\u036f]/g, '');
+    }
+    /**
+     * Splits text into words while tracking character offsets
+     */
+    splitIntoWords(processedText, originalText) {
+        const words = [];
+        // Split on whitespace and punctuation while keeping track of positions
+        const wordPattern = /\S+/g;
+        let match;
+        while ((match = wordPattern.exec(processedText)) !== null) {
+            // Find corresponding position in original text
+            // Since we may have lowercased, we need to map positions
+            const start = match.index;
+            const end = start + match[0].length;
+            words.push({
+                word: match[0],
+                start,
+                end,
+            });
+        }
+        return words;
+    }
+    /**
+     * Tokenizes a single word using WordPiece algorithm
+     */
+    tokenizeWord(word, startOffset, endOffset) {
+        const tokens = [];
+        // Handle punctuation separately
+        const subwords = this.splitWordIntoPieces(word);
+        let currentOffset = startOffset;
+        for (let i = 0; i < subwords.length; i++) {
+            let subword = subwords[i];
+            const isContinuation = i > 0;
+            // For continuation tokens, add ## prefix for vocab lookup
+            const vocabKey = isContinuation ? '##' + subword : subword;
+            // Look up in vocabulary
+            let tokenId = this.vocab.get(vocabKey);
+            // If not found, try to find longest matching prefix
+            if (tokenId === undefined) {
+                const { id, token } = this.findLongestMatch(subword, isContinuation);
+                tokenId = id;
+                subword = token;
+            }
+            const tokenLength = subword.length;
+            const tokenEnd = Math.min(currentOffset + tokenLength, endOffset);
+            tokens.push({
+                id: tokenId,
+                token: isContinuation ? '##' + subword : subword,
+                start: currentOffset,
+                end: tokenEnd,
+                isContinuation,
+                isSpecial: false,
+            });
+            currentOffset = tokenEnd;
+        }
+        return tokens;
+    }
+    /**
+     * Splits a word into pieces, handling punctuation
+     */
+    splitWordIntoPieces(word) {
+        const pieces = [];
+        let current = '';
+        for (const char of word) {
+            if (this.isPunctuation(char)) {
+                if (current.length > 0) {
+                    pieces.push(current);
+                    current = '';
+                }
+                pieces.push(char);
+            }
+            else {
+                current += char;
+            }
+        }
+        if (current.length > 0) {
+            pieces.push(current);
+        }
+        return pieces;
+    }
+    /**
+     * Checks if a character is punctuation
+     */
+    isPunctuation(char) {
+        const code = char.charCodeAt(0);
+        // ASCII punctuation and some Unicode punctuation
+        return ((code >= 33 && code <= 47) ||
+            (code >= 58 && code <= 64) ||
+            (code >= 91 && code <= 96) ||
+            (code >= 123 && code <= 126) ||
+            /[\u2000-\u206F]/.test(char) || // General punctuation
+            /[\u3000-\u303F]/.test(char) // CJK punctuation
+        );
+    }
+    /**
+     * Finds the longest matching token in vocabulary
+     */
+    findLongestMatch(word, isContinuation) {
+        const prefix = isContinuation ? '##' : '';
+        // Try progressively shorter substrings
+        for (let end = word.length; end > 0; end--) {
+            const subword = word.slice(0, end);
+            const vocabKey = prefix + subword;
+            const id = this.vocab.get(vocabKey);
+            if (id !== undefined) {
+                return { id, token: subword };
+            }
+        }
+        // Fall back to unknown token
+        return { id: this.unkId, token: word };
+    }
+    /**
+     * Decodes token IDs back to text
+     */
+    decode(tokenIds) {
+        const tokens = [];
+        for (const id of tokenIds) {
+            const token = this.inverseVocab.get(id);
+            if (token === undefined)
+                continue;
+            // Skip special tokens
+            if (token === this.config.clsToken ||
+                token === this.config.sepToken ||
+                token === this.config.padToken) {
+                continue;
+            }
+            // Handle continuation tokens
+            if (token.startsWith('##')) {
+                tokens.push(token.slice(2));
+            }
+            else {
+                if (tokens.length > 0) {
+                    tokens.push(' ');
+                }
+                tokens.push(token);
+            }
+        }
+        return tokens.join('');
+    }
+    /**
+     * Gets vocabulary size
+     */
+    get vocabSize() {
+        return this.vocab.size;
+    }
+    /**
+     * Gets a token ID by string
+     */
+    getTokenId(token) {
+        return this.vocab.get(token);
+    }
+    /**
+     * Gets a token string by ID
+     */
+    getToken(id) {
+        return this.inverseVocab.get(id);
+    }
+}
+/**
+ * Loads vocabulary from a text file (one token per line)
+ */
+export async function loadVocabFromFile(path) {
+    const fs = await import('fs/promises');
+    const content = await fs.readFile(path, 'utf-8');
+    return parseVocab(content);
+}
+/**
+ * Parses vocabulary from string content
+ */
+export function parseVocab(content) {
+    const vocab = new Map();
+    const lines = content.split('\n');
+    for (let i = 0; i < lines.length; i++) {
+        const token = lines[i]?.trim();
+        if (token !== undefined && token.length > 0) {
+            vocab.set(token, i);
+        }
+    }
+    return vocab;
+}
+/**
+ * Creates a minimal vocabulary for testing
+ */
+export function createTestVocab() {
+    const tokens = [
+        '[PAD]',
+        '[UNK]',
+        '[CLS]',
+        '[SEP]',
+        '[MASK]',
+        'the',
+        'a',
+        'is',
+        'was',
+        'john',
+        'smith',
+        'berlin',
+        'germany',
+        '##s',
+        '##ed',
+        '##ing',
+        ',',
+        '.',
+        '!',
+        '?',
+    ];
+    const vocab = new Map();
+    tokens.forEach((token, index) => {
+        vocab.set(token, index);
+    });
+    return vocab;
+}
+//# sourceMappingURL=tokenizer.js.map

package/dist/ner/tokenizer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"tokenizer.js","sourceRoot":"","sources":["../../src/ner/tokenizer.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AA8DH;;GAEG;AACH,MAAM,CAAC,MAAM,wBAAwB,GAAoB;IACvD,SAAS,EAAE,GAAG;IACd,QAAQ,EAAE,OAAO;IACjB,QAAQ,EAAE,OAAO;IACjB,QAAQ,EAAE,OAAO;IACjB,QAAQ,EAAE,OAAO;IACjB,SAAS,EAAE,QAAQ;IACnB,WAAW,EAAE,IAAI;IACjB,YAAY,EAAE,IAAI;CACnB,CAAC;AAEF;;GAEG;AACH,MAAM,OAAO,kBAAkB;IACrB,KAAK,CAAsB;IAC3B,YAAY,CAAsB;IAClC,MAAM,CAAkB;IAEhC,oBAAoB;IACZ,KAAK,CAAS;IACd,KAAK,CAAS;IACd,KAAK,CAAS;IACd,KAAK,CAAS;IAEtB,YAAY,KAA0B,EAAE,SAAmC,EAAE;QAC3E,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;QACnB,IAAI,CAAC,MAAM,GAAG,EAAE,GAAG,wBAAwB,EAAE,GAAG,MAAM,EAAE,CAAC;QAEzD,sBAAsB;QACtB,IAAI,CAAC,YAAY,GAAG,IAAI,GAAG,EAAE,CAAC;QAC9B,KAAK,MAAM,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,KAAK,EAAE,CAAC;YAChC,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,EAAE,EAAE,KAAK,CAAC,CAAC;QACnC,CAAC;QAED,wBAAwB;QACxB,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QACvD,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,GAAG,CAAC;QACzD,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,GAAG,CAAC;QACzD,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;IACzD,CAAC;IAED;;OAEG;IACH,QAAQ,CAAC,IAAY;QACnB,MAAM,MAAM,GAAY,EAAE,CAAC;QAC3B,MAAM,eAAe,GAAmC,EAAE,CAAC;QAE3D,kBAAkB;QAClB,MAAM,CAAC,IAAI,CAAC;YACV,EAAE,EAAE,IAAI,CAAC,KAAK;YACd,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,QAAQ;YAC3B,KAAK,EAAE,CAAC;YACR,GAAG,EAAE,CAAC;YACN,cAAc,EAAE,KAAK;YACrB,SAAS,EAAE,IAAI;SAChB,CAAC,CAAC;QACH,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAE3B,kBAAkB;QAClB,MAAM,aAAa,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;QAE5C,iCAAiC;QACjC,MAAM,SAAS,GAAG,IAAI,CAAC,cAAc,CAAC,aAAa,EAAE,IAAI,CAAC,CAAC;QAE3D,qBAAqB;QACrB,KAAK,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,IAAI,SAAS,EAAE,CAAC;YAC7C,MAAM,UAAU,GAAG,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,KAAK,EAAE,GAAG,CAAC,CAAC;YACvD,MAAM,CAAC,IAAI,CAAC,GAAG,UAAU,CAAC,CAAC;YAC3B,KAAK,MAAM,CAAC,IAAI,UAAU,EAAE,CAAC;gBAC3B,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;YACzC,CAAC;QACH,CAAC;QAED,kBAAkB;QAClB,MAAM,CAAC,IAAI,CAAC;YACV,EAAE,EAAE,IAAI,CAAC,KAAK;YACd,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,QAAQ;YAC3B,KAAK,EAAE,IAAI,CAAC,MAAM;YAClB,GAAG,EAAE,IAAI,CAAC,MAAM;YAChB,cAAc,EAAE,KAAK;YACrB,SAAS,EAAE,IAAI;SAChB,CAAC,CAAC;QACH,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAE3B,wBAAwB;QACxB,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC;QACxC,IAAI,MAAM,CAAC,MAAM,GAAG,SAAS,EAAE,CAAC;YAC9B,MAAM,CAAC,MAAM,GAAG,SAAS,GAAG,CAAC,CAAC;YAC9B,eAAe,CAAC,MAAM,GAAG,SAAS,GAAG,CAAC,CAAC;YACvC,mBAAmB;YACnB,MAAM,CAAC,IAAI,CAAC;gBACV,EAAE,EAAE,IAAI,CAAC,KAAK;gBACd,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,QAAQ;gBAC3B,KAAK,EAAE,IAAI,CAAC,MAAM;gBAClB,GAAG,EAAE,IAAI,CAAC,MAAM;gBAChB,cAAc,EAAE,KAAK;gBACrB,SAAS,EAAE,IAAI;aAChB,CAAC,CAAC;YACH,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC7B,CAAC;QAED,eAAe;QACf,MAAM,QAAQ,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QACzC,MAAM,aAAa,GAAG,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC;QAC1C,MAAM,YAAY,GAAG,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC;QAEzC,OAAO;YACL,MAAM;YACN,QAAQ;YACR,aAAa;YACb,YAAY;YACZ,eAAe;SAChB,CAAC;IACJ,CAAC;IAED;;OAEG;IACK,UAAU,CAAC,IAAY;QAC7B,IAAI,SAAS,GAAG,IAAI,CAAC;QAErB,IAAI,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;YAC5B,SAAS,GAAG,SAAS,CAAC,WAAW,EAAE,CAAC;QACtC,CAAC;QAED,IAAI,IAAI,CAAC,MAAM,CAAC,YAAY,EAAE,CAAC;YAC7B,SAAS,GAAG,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC;QAC3C,CAAC;QAED,OAAO,SAAS,CAAC;IACnB,CAAC;IAED;;OAEG;IACK,YAAY,CAAC,IAAY;QAC/B,OAAO,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,kBAAkB,EAAE,EAAE,CAAC,CAAC;IAC/D,CAAC;IAED;;OAEG;IACK,cAAc,CACpB,aAAqB,EACrB,YAAoB;QAEpB,MAAM,KAAK,GAAwD,EAAE,CAAC;QAEtE,uEAAuE;QACvE,MAAM,WAAW,GAAG,MAAM,CAAC;QAC3B,IAAI,KAA6B,CAAC;QAElC,OAAO,CAAC,KAAK,GAAG,WAAW,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;YAC1D,+CAA+C;YAC/C,yDAAyD;YACzD,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC;YAC1B,MAAM,GAAG,GAAG,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;YAEpC,KAAK,CAAC,IAAI,CAAC;gBACT,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC;gBACd,KAAK;gBACL,GAAG;aACJ,CAAC,CAAC;QACL,CAAC;QAED,OAAO,KAAK,CAAC;IACf,CAAC;IAED;;OAEG;IACK,YAAY,CAAC,IAAY,EAAE,WAAmB,EAAE,SAAiB;QACvE,MAAM,MAAM,GAAY,EAAE,CAAC;QAE3B,gCAAgC;QAChC,MAAM,QAAQ,GAAG,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,CAAC;QAEhD,IAAI,aAAa,GAAG,WAAW,CAAC;QAEhC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACzC,IAAI,OAAO,GAAG,QAAQ,CAAC,CAAC,CAAE,CAAC;YAC3B,MAAM,cAAc,GAAG,CAAC,GAAG,CAAC,CAAC;YAE7B,0DAA0D;YAC1D,MAAM,QAAQ,GAAG,cAAc,CAAC,CAAC,CAAC,IAAI,GAAG,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC;YAE3D,wBAAwB;YACxB,IAAI,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;YAEvC,oDAAoD;YACpD,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;gBAC1B,MAAM,EAAE,EAAE,EAAE,KAAK,EAAE,GAAG,IAAI,CAAC,gBAAgB,CAAC,OAAO,EAAE,cAAc,CAAC,CAAC;gBACrE,OAAO,GAAG,EAAE,CAAC;gBACb,OAAO,GAAG,KAAK,CAAC;YAClB,CAAC;YAED,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC;YACnC,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,aAAa,GAAG,WAAW,EAAE,SAAS,CAAC,CAAC;YAElE,MAAM,CAAC,IAAI,CAAC;gBACV,EAAE,EAAE,OAAO;gBACX,KAAK,EAAE,cAAc,CAAC,CAAC,CAAC,IAAI,GAAG,OAAO,CAAC,CAAC,CAAC,OAAO;gBAChD,KAAK,EAAE,aAAa;gBACpB,GAAG,EAAE,QAAQ;gBACb,cAAc;gBACd,SAAS,EAAE,KAAK;aACjB,CAAC,CAAC;YAEH,aAAa,GAAG,QAAQ,CAAC;QAC3B,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;OAEG;IACK,mBAAmB,CAAC,IAAY;QACtC,MAAM,MAAM,GAAa,EAAE,CAAC;QAC5B,IAAI,OAAO,GAAG,EAAE,CAAC;QAEjB,KAAK,MAAM,IAAI,IAAI,IAAI,EAAE,CAAC;YACxB,IAAI,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC7B,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACvB,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;oBACrB,OAAO,GAAG,EAAE,CAAC;gBACf,CAAC;gBACD,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACpB,CAAC;iBAAM,CAAC;gBACN,OAAO,IAAI,IAAI,CAAC;YAClB,CAAC;QACH,CAAC;QAED,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACvB,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACvB,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;OAEG;IACK,aAAa,CAAC,IAAY;QAChC,MAAM,IAAI,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;QAChC,iDAAiD;QACjD,OAAO,CACL,CAAC,IAAI,IAAI,EAAE,IAAI,IAAI,IAAI,EAAE,CAAC;YAC1B,CAAC,IAAI,IAAI,EAAE,IAAI,IAAI,IAAI,EAAE,CAAC;YAC1B,CAAC,IAAI,IAAI,EAAE,IAAI,IAAI,IAAI,EAAE,CAAC;YAC1B,CAAC,IAAI,IAAI,GAAG,IAAI,IAAI,IAAI,GAAG,CAAC;YAC5B,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,sBAAsB;YACtD,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,kBAAkB;SAChD,CAAC;IACJ,CAAC;IAED;;OAEG;IACK,gBAAgB,CACtB,IAAY,EACZ,cAAuB;QAEvB,MAAM,MAAM,GAAG,cAAc,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;QAE1C,uCAAuC;QACvC,KAAK,IAAI,GAAG,GAAG,IAAI,CAAC,MAAM,EAAE,GAAG,GAAG,CAAC,EAAE,GAAG,EAAE,EAAE,CAAC;YAC3C,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;YACnC,MAAM,QAAQ,GAAG,MAAM,GAAG,OAAO,CAAC;YAElC,MAAM,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;YACpC,IAAI,EAAE,KAAK,SAAS,EAAE,CAAC;gBACrB,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;YAChC,CAAC;QACH,CAAC;QAED,6BAA6B;QAC7B,OAAO,EAAE,EAAE,EAAE,IAAI,CAAC,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;IACzC,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,QAAkB;QACvB,MAAM,MAAM,GAAa,EAAE,CAAC;QAE5B,KAAK,MAAM,EAAE,IAAI,QAAQ,EAAE,CAAC;YAC1B,MAAM,KAAK,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YACxC,IAAI,KAAK,KAAK,SAAS;gBAAE,SAAS;YAElC,sBAAsB;YACtB,IACE,KAAK,KAAK,IAAI,CAAC,MAAM,CAAC,QAAQ;gBAC9B,KAAK,KAAK,IAAI,CAAC,MAAM,CAAC,QAAQ;gBAC9B,KAAK,KAAK,IAAI,CAAC,MAAM,CAAC,QAAQ,EAC9B,CAAC;gBACD,SAAS;YACX,CAAC;YAED,6BAA6B;YAC7B,IAAI,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC3B,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;YAC9B,CAAC;iBAAM,CAAC;gBACN,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACtB,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBACnB,CAAC;gBACD,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACrB,CAAC;QACH,CAAC;QAED,OAAO,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACzB,CAAC;IAED;;OAEG;IACH,IAAI,SAAS;QACX,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC;IACzB,CAAC;IAED;;OAEG;IACH,UAAU,CAAC,KAAa;QACtB,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;IAC/B,CAAC;IAED;;OAEG;IACH,QAAQ,CAAC,EAAU;QACjB,OAAO,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IACnC,CAAC;CACF;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,IAAY;IAClD,MAAM,EAAE,GAAG,MAAM,MAAM,CAAC,aAAa,CAAC,CAAC;IACvC,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IACjD,OAAO,UAAU,CAAC,OAAO,CAAC,CAAC;AAC7B,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,UAAU,CAAC,OAAe;IACxC,MAAM,KAAK,GAAG,IAAI,GAAG,EAAkB,CAAC;IACxC,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAElC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACtC,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC;QAC/B,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC5C,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;QACtB,CAAC;IACH,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,eAAe;IAC7B,MAAM,MAAM,GAAG;QACb,OAAO;QACP,OAAO;QACP,OAAO;QACP,OAAO;QACP,QAAQ;QACR,KAAK;QACL,GAAG;QACH,IAAI;QACJ,KAAK;QACL,MAAM;QACN,OAAO;QACP,QAAQ;QACR,SAAS;QACT,KAAK;QACL,MAAM;QACN,OAAO;QACP,GAAG;QACH,GAAG;QACH,GAAG;QACH,GAAG;KACJ,CAAC;IAEF,MAAM,KAAK,GAAG,IAAI,GAAG,EAAkB,CAAC;IACxC,MAAM,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE;QAC9B,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;IAC1B,CAAC,CAAC,CAAC;IAEH,OAAO,KAAK,CAAC;AACf,CAAC"}

package/dist/pipeline/index.d.ts

@@ -0,0 +1,9 @@
+/**
+ * Pipeline Module
+ * Exports all pipeline components
+ */
+export * from './prenormalize.js';
+export * from './resolver.js';
+export * from './tagger.js';
+export * from './validator.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/pipeline/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/pipeline/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,cAAc,mBAAmB,CAAC;AAClC,cAAc,eAAe,CAAC;AAC9B,cAAc,aAAa,CAAC;AAC5B,cAAc,gBAAgB,CAAC"}

package/dist/pipeline/index.js

@@ -0,0 +1,9 @@
+/**
+ * Pipeline Module
+ * Exports all pipeline components
+ */
+export * from './prenormalize.js';
+export * from './resolver.js';
+export * from './tagger.js';
+export * from './validator.js';
+//# sourceMappingURL=index.js.map

package/dist/pipeline/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/pipeline/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,cAAc,mBAAmB,CAAC;AAClC,cAAc,eAAe,CAAC;AAC9B,cAAc,aAAa,CAAC;AAC5B,cAAc,gBAAgB,CAAC"}

package/dist/pipeline/prenormalize.d.ts

@@ -0,0 +1,48 @@
+/**
+ * Pre-normalization
+ * Normalizes text before PII detection
+ */
+/**
+ * Pre-normalization options
+ */
+export interface PrenormalizeOptions {
+    /** Normalize line endings to \n */
+    normalizeLineEndings: boolean;
+    /** Apply Unicode NFKC normalization */
+    unicodeNormalize: boolean;
+    /** Trim leading/trailing whitespace */
+    trim: boolean;
+}
+/**
+ * Default pre-normalization options
+ */
+export declare const DEFAULT_PRENORMALIZE_OPTIONS: PrenormalizeOptions;
+/**
+ * Pre-normalizes text for PII detection
+ * Note: This currently only normalizes line endings to preserve character offsets
+ *
+ * @param text - Original input text
+ * @param options - Normalization options
+ * @returns Normalized text
+ */
+export declare function prenormalize(text: string, options?: Partial<PrenormalizeOptions>): string;
+/**
+ * Calculates offset adjustments when text is modified
+ * Used when prenormalization changes text length
+ */
+export interface OffsetMapping {
+    /** Map from original offset to normalized offset */
+    toNormalized: (originalOffset: number) => number;
+    /** Map from normalized offset to original offset */
+    toOriginal: (normalizedOffset: number) => number;
+}
+/**
+ * Creates an identity offset mapping (no changes)
+ */
+export declare function createIdentityMapping(): OffsetMapping;
+/**
+ * Creates offset mapping for line ending normalization
+ * This handles \r\n -> \n replacement
+ */
+export declare function createLineEndingMapping(originalText: string): OffsetMapping;
+//# sourceMappingURL=prenormalize.d.ts.map

package/dist/pipeline/prenormalize.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"prenormalize.d.ts","sourceRoot":"","sources":["../../src/pipeline/prenormalize.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAClC,mCAAmC;IACnC,oBAAoB,EAAE,OAAO,CAAC;IAC9B,uCAAuC;IACvC,gBAAgB,EAAE,OAAO,CAAC;IAC1B,uCAAuC;IACvC,IAAI,EAAE,OAAO,CAAC;CACf;AAED;;GAEG;AACH,eAAO,MAAM,4BAA4B,EAAE,mBAI1C,CAAC;AAEF;;;;;;;GAOG;AACH,wBAAgB,YAAY,CAC1B,IAAI,EAAE,MAAM,EACZ,OAAO,GAAE,OAAO,CAAC,mBAAmB,CAAM,GACzC,MAAM,CAqBR;AAED;;;GAGG;AACH,MAAM,WAAW,aAAa;IAC5B,oDAAoD;IACpD,YAAY,EAAE,CAAC,cAAc,EAAE,MAAM,KAAK,MAAM,CAAC;IACjD,oDAAoD;IACpD,UAAU,EAAE,CAAC,gBAAgB,EAAE,MAAM,KAAK,MAAM,CAAC;CAClD;AAED;;GAEG;AACH,wBAAgB,qBAAqB,IAAI,aAAa,CAKrD;AAED;;;GAGG;AACH,wBAAgB,uBAAuB,CAAC,YAAY,EAAE,MAAM,GAAG,aAAa,CA4C3E"}

package/dist/pipeline/prenormalize.js

@@ -0,0 +1,94 @@
+/**
+ * Pre-normalization
+ * Normalizes text before PII detection
+ */
+/**
+ * Default pre-normalization options
+ */
+export const DEFAULT_PRENORMALIZE_OPTIONS = {
+    normalizeLineEndings: true,
+    unicodeNormalize: false, // Disabled by default to preserve offsets
+    trim: false, // Disabled by default to preserve offsets
+};
+/**
+ * Pre-normalizes text for PII detection
+ * Note: This currently only normalizes line endings to preserve character offsets
+ *
+ * @param text - Original input text
+ * @param options - Normalization options
+ * @returns Normalized text
+ */
+export function prenormalize(text, options = {}) {
+    const opts = { ...DEFAULT_PRENORMALIZE_OPTIONS, ...options };
+    let result = text;
+    // Normalize line endings (\r\n -> \n, \r -> \n)
+    if (opts.normalizeLineEndings) {
+        result = result.replace(/\r\n/g, '\n').replace(/\r/g, '\n');
+    }
+    // Note: Unicode normalization (NFKC) can change string length
+    // We skip it by default to preserve character offsets
+    // If needed, implement offset mapping
+    if (opts.unicodeNormalize) {
+        result = result.normalize('NFKC');
+    }
+    if (opts.trim) {
+        result = result.trim();
+    }
+    return result;
+}
+/**
+ * Creates an identity offset mapping (no changes)
+ */
+export function createIdentityMapping() {
+    return {
+        toNormalized: (offset) => offset,
+        toOriginal: (offset) => offset,
+    };
+}
+/**
+ * Creates offset mapping for line ending normalization
+ * This handles \r\n -> \n replacement
+ */
+export function createLineEndingMapping(originalText) {
+    // Find all \r\n positions
+    const crlfPositions = [];
+    for (let i = 0; i < originalText.length - 1; i++) {
+        if (originalText[i] === '\r' && originalText[i + 1] === '\n') {
+            crlfPositions.push(i);
+        }
+    }
+    if (crlfPositions.length === 0) {
+        return createIdentityMapping();
+    }
+    return {
+        toNormalized(originalOffset) {
+            // Count how many \r\n pairs are before this offset
+            let adjustment = 0;
+            for (const pos of crlfPositions) {
+                if (pos < originalOffset) {
+                    adjustment++;
+                }
+                else {
+                    break;
+                }
+            }
+            return originalOffset - adjustment;
+        },
+        toOriginal(normalizedOffset) {
+            // Add back the removed \r characters
+            let adjustment = 0;
+            let currentNormalized = 0;
+            for (const pos of crlfPositions) {
+                if (currentNormalized + (pos - adjustment) <= normalizedOffset) {
+                    adjustment++;
+                    currentNormalized = pos - adjustment + 1;
+                }
+                else {
+                    break;
+                }
+            }
+            return normalizedOffset + adjustment;
+        },
+    };
+}
+//# sourceMappingURL=prenormalize.js.map

package/dist/pipeline/prenormalize.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"prenormalize.js","sourceRoot":"","sources":["../../src/pipeline/prenormalize.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAcH;;GAEG;AACH,MAAM,CAAC,MAAM,4BAA4B,GAAwB;IAC/D,oBAAoB,EAAE,IAAI;IAC1B,gBAAgB,EAAE,KAAK,EAAE,0CAA0C;IACnE,IAAI,EAAE,KAAK,EAAE,0CAA0C;CACxD,CAAC;AAEF;;;;;;;GAOG;AACH,MAAM,UAAU,YAAY,CAC1B,IAAY,EACZ,UAAwC,EAAE;IAE1C,MAAM,IAAI,GAAG,EAAE,GAAG,4BAA4B,EAAE,GAAG,OAAO,EAAE,CAAC;IAC7D,IAAI,MAAM,GAAG,IAAI,CAAC;IAElB,gDAAgD;IAChD,IAAI,IAAI,CAAC,oBAAoB,EAAE,CAAC;QAC9B,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;IAC9D,CAAC;IAED,8DAA8D;IAC9D,sDAAsD;IACtD,sCAAsC;IACtC,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;QAC1B,MAAM,GAAG,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;IACpC,CAAC;IAED,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;QACd,MAAM,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC;IACzB,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAaD;;GAEG;AACH,MAAM,UAAU,qBAAqB;IACnC,OAAO;QACL,YAAY,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM;QAChC,UAAU,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM;KAC/B,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,uBAAuB,CAAC,YAAoB;IAC1D,0BAA0B;IAC1B,MAAM,aAAa,GAAa,EAAE,CAAC;IACnC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QACjD,IAAI,YAAY,CAAC,CAAC,CAAC,KAAK,IAAI,IAAI,YAAY,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;YAC7D,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACxB,CAAC;IACH,CAAC;IAED,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC/B,OAAO,qBAAqB,EAAE,CAAC;IACjC,CAAC;IAED,OAAO;QACL,YAAY,CAAC,cAAsB;YACjC,mDAAmD;YACnD,IAAI,UAAU,GAAG,CAAC,CAAC;YACnB,KAAK,MAAM,GAAG,IAAI,aAAa,EAAE,CAAC;gBAChC,IAAI,GAAG,GAAG,cAAc,EAAE,CAAC;oBACzB,UAAU,EAAE,CAAC;gBACf,CAAC;qBAAM,CAAC;oBACN,MAAM;gBACR,CAAC;YACH,CAAC;YACD,OAAO,cAAc,GAAG,UAAU,CAAC;QACrC,CAAC;QAED,UAAU,CAAC,gBAAwB;YACjC,qCAAqC;YACrC,IAAI,UAAU,GAAG,CAAC,CAAC;YACnB,IAAI,iBAAiB,GAAG,CAAC,CAAC;YAE1B,KAAK,MAAM,GAAG,IAAI,aAAa,EAAE,CAAC;gBAChC,IAAI,iBAAiB,GAAG,CAAC,GAAG,GAAG,UAAU,CAAC,IAAI,gBAAgB,EAAE,CAAC;oBAC/D,UAAU,EAAE,CAAC;oBACb,iBAAiB,GAAG,GAAG,GAAG,UAAU,GAAG,CAAC,CAAC;gBAC3C,CAAC;qBAAM,CAAC;oBACN,MAAM;gBACR,CAAC;YACH,CAAC;YAED,OAAO,gBAAgB,GAAG,UAAU,CAAC;QACvC,CAAC;KACF,CAAC;AACJ,CAAC"}

package/dist/pipeline/resolver.d.ts

@@ -0,0 +1,56 @@
+/**
+ * Entity Resolver
+ * Merges, deduplicates, and resolves overlapping entity detections
+ */
+import { SpanMatch, AnonymizationPolicy } from '../types/index.js';
+/**
+ * Resolution strategy for overlapping entities
+ */
+export declare enum OverlapStrategy {
+    /** Regex matches always win over NER */
+    REGEX_PRIORITY = "REGEX_PRIORITY",
+    /** Longer span wins */
+    LONGER_SPAN = "LONGER_SPAN",
+    /** Higher confidence wins */
+    HIGHER_CONFIDENCE = "HIGHER_CONFIDENCE",
+    /** Use type priority from policy */
+    TYPE_PRIORITY = "TYPE_PRIORITY"
+}
+/**
+ * Entity resolver configuration
+ */
+export interface ResolverConfig {
+    /** Primary strategy for overlap resolution */
+    overlapStrategy: OverlapStrategy;
+    /** Whether regex matches always take precedence */
+    regexPriority: boolean;
+    /** Minimum confidence to keep an entity */
+    minConfidence: number;
+}
+/**
+ * Default resolver configuration
+ */
+export declare const DEFAULT_RESOLVER_CONFIG: ResolverConfig;
+/**
+ * Resolves and merges entity detections from regex and NER
+ */
+export declare function resolveEntities(regexMatches: SpanMatch[], nerMatches: SpanMatch[], policy: AnonymizationPolicy, originalText: string, config?: Partial<ResolverConfig>): SpanMatch[];
+/**
+ * Creates protected spans from regex matches
+ * Used to mask regex matches from NER to avoid double-detection
+ */
+export declare function createProtectedSpans(regexMatches: SpanMatch[]): Array<{
+    start: number;
+    end: number;
+}>;
+/**
+ * Checks if a span overlaps with any protected span
+ */
+export declare function isInProtectedSpan(span: {
+    start: number;
+    end: number;
+}, protectedSpans: Array<{
+    start: number;
+    end: number;
+}>): boolean;
+//# sourceMappingURL=resolver.d.ts.map

package/dist/pipeline/resolver.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"resolver.d.ts","sourceRoot":"","sources":["../../src/pipeline/resolver.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAEL,SAAS,EAET,mBAAmB,EAEpB,MAAM,mBAAmB,CAAC;AAG3B;;GAEG;AACH,oBAAY,eAAe;IACzB,wCAAwC;IACxC,cAAc,mBAAmB;IACjC,uBAAuB;IACvB,WAAW,gBAAgB;IAC3B,6BAA6B;IAC7B,iBAAiB,sBAAsB;IACvC,oCAAoC;IACpC,aAAa,kBAAkB;CAChC;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,8CAA8C;IAC9C,eAAe,EAAE,eAAe,CAAC;IACjC,mDAAmD;IACnD,aAAa,EAAE,OAAO,CAAC;IACvB,2CAA2C;IAC3C,aAAa,EAAE,MAAM,CAAC;CACvB;AAED;;GAEG;AACH,eAAO,MAAM,uBAAuB,EAAE,cAIrC,CAAC;AAEF;;GAEG;AACH,wBAAgB,eAAe,CAC7B,YAAY,EAAE,SAAS,EAAE,EACzB,UAAU,EAAE,SAAS,EAAE,EACvB,MAAM,EAAE,mBAAmB,EAC3B,YAAY,EAAE,MAAM,EACpB,MAAM,GAAE,OAAO,CAAC,cAAc,CAAM,GACnC,SAAS,EAAE,CAyBb;AAiOD;;;GAGG;AACH,wBAAgB,oBAAoB,CAClC,YAAY,EAAE,SAAS,EAAE,GACxB,KAAK,CAAC;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAA;CAAE,CAAC,CAEvC;AAED;;GAEG;AACH,wBAAgB,iBAAiB,CAC/B,IAAI,EAAE;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAA;CAAE,EACpC,cAAc,EAAE,KAAK,CAAC;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAA;CAAE,CAAC,GACpD,OAAO,CAET"}