@doccloak/core 0.8.0

package/dist/providers/bardsai.js

@@ -0,0 +1,393 @@
+/**
+ * BardS.ai EU PII Anonymization Provider
+ *
+ * XLM-RoBERTa-base fine-tuned for EU PII detection (24 EU languages, natively multilingual).
+ * 35 entity types, ~0.3B parameters, quantized ONNX (~279 MB).
+ * Model: bardsai/eu-pii-anonimization-multilang
+ * Runs in-browser via ONNX Runtime WebAssembly.
+ *
+ * Environment-agnostic: wasm paths, model bytes (blob cache + fetch) and the
+ * tokenizer all arrive through the injected CoreEnv. This module never touches
+ * import.meta, caches, navigator or @huggingface/transformers directly.
+ */
+import * as ort from 'onnxruntime-web';
+import { fetchModelBlob, retryAsync } from "../model-loader.js";
+// ── Model config ──────────────────────────────────────────
+const MODEL_URL = 'https://huggingface.co/bardsai/eu-pii-anonimization-multilang/resolve/main/onnx/model_quantized.onnx';
+const TOKENIZER_HF = 'bardsai/eu-pii-anonimization-multilang';
+const MODEL_NAME = 'BardS.ai EU PII';
+const DEFAULT_THRESHOLD = 0.5;
+const MAX_SEQ_LENGTH = 512;
+const CHUNK_OVERLAP = 30;
+/** id2label - 35 entity types from bardsai eu-pii-anonimization-multilang config.json (69 labels) */
+const ID2LABEL = {
+    0: 'O',
+    1: 'B-ACCOUNT_IDENTIFIER', 2: 'B-AUTH_SECRET', 3: 'B-BANK_ACCOUNT_IDENTIFIER',
+    4: 'B-BIOMETRIC_DATA', 5: 'B-CONTACT_HANDLE', 6: 'B-CRIMINAL_OFFENCE_DATA',
+    7: 'B-DATE_OF_BIRTH', 8: 'B-DEVICE_IDENTIFIER', 9: 'B-DOCUMENT_IDENTIFIER',
+    10: 'B-DOCUMENT_REFERENCE', 11: 'B-EMAIL_ADDRESS', 12: 'B-ETHNIC_ORIGIN',
+    13: 'B-FINANCIAL_AMOUNT', 14: 'B-GEO_LOCATION', 15: 'B-HEALTH_DATA',
+    16: 'B-IDENTIFYING_LINK', 17: 'B-IP_ADDRESS', 18: 'B-LOCATION',
+    19: 'B-ORGANIZATION_IDENTIFIER', 20: 'B-ORGANIZATION_NAME', 21: 'B-PAYMENT_CARD',
+    22: 'B-PAYMENT_CARD_SECURITY', 23: 'B-PERSON_ALIAS', 24: 'B-PERSON_ATTRIBUTE',
+    25: 'B-PERSON_IDENTIFIER', 26: 'B-PERSON_NAME', 27: 'B-PERSON_ROLE_OR_TITLE',
+    28: 'B-PHONE_NUMBER', 29: 'B-POLITICAL_OPINION', 30: 'B-POSTAL_ADDRESS',
+    31: 'B-PROPER_NAME', 32: 'B-RELIGION_OR_BELIEF', 33: 'B-SEXUAL_ORIENTATION',
+    34: 'B-TRADE_UNION_MEMBERSHIP', 35: 'B-VEHICLE_IDENTIFIER',
+    36: 'I-ACCOUNT_IDENTIFIER', 37: 'I-AUTH_SECRET', 38: 'I-BANK_ACCOUNT_IDENTIFIER',
+    39: 'I-BIOMETRIC_DATA', 40: 'I-CRIMINAL_OFFENCE_DATA', 41: 'I-DATE_OF_BIRTH',
+    42: 'I-DEVICE_IDENTIFIER', 43: 'I-DOCUMENT_IDENTIFIER', 44: 'I-DOCUMENT_REFERENCE',
+    45: 'I-EMAIL_ADDRESS', 46: 'I-ETHNIC_ORIGIN', 47: 'I-FINANCIAL_AMOUNT',
+    48: 'I-GEO_LOCATION', 49: 'I-HEALTH_DATA', 50: 'I-IDENTIFYING_LINK',
+    51: 'I-IP_ADDRESS', 52: 'I-LOCATION', 53: 'I-ORGANIZATION_IDENTIFIER',
+    54: 'I-ORGANIZATION_NAME', 55: 'I-PAYMENT_CARD', 56: 'I-PAYMENT_CARD_SECURITY',
+    57: 'I-PERSON_ATTRIBUTE', 58: 'I-PERSON_IDENTIFIER', 59: 'I-PERSON_NAME',
+    60: 'I-PERSON_ROLE_OR_TITLE', 61: 'I-PHONE_NUMBER', 62: 'I-POLITICAL_OPINION',
+    63: 'I-POSTAL_ADDRESS', 64: 'I-PROPER_NAME', 65: 'I-RELIGION_OR_BELIEF',
+    66: 'I-SEXUAL_ORIENTATION', 67: 'I-TRADE_UNION_MEMBERSHIP', 68: 'I-VEHICLE_IDENTIFIER',
+};
+/** Map bardsai entity labels → DocCloak EntityType */
+const LABEL_TO_ENTITY_TYPE = {
+    PERSON_NAME: 'PERSON',
+    PERSON_ALIAS: 'PERSON',
+    PROPER_NAME: 'PERSON',
+    EMAIL_ADDRESS: 'EMAIL',
+    CONTACT_HANDLE: 'EMAIL',
+    PHONE_NUMBER: 'PHONE',
+    PERSON_IDENTIFIER: 'SSN',
+    DATE_OF_BIRTH: 'DATE',
+    PAYMENT_CARD: 'CREDIT_CARD',
+    PAYMENT_CARD_SECURITY: 'CREDIT_CARD',
+    IP_ADDRESS: 'IP_ADDRESS',
+    BANK_ACCOUNT_IDENTIFIER: 'IBAN',
+    ACCOUNT_IDENTIFIER: 'IBAN',
+    POSTAL_ADDRESS: 'ADDRESS',
+    LOCATION: 'ADDRESS',
+    GEO_LOCATION: 'ADDRESS',
+    ORGANIZATION_NAME: 'COMPANY',
+    ORGANIZATION_IDENTIFIER: 'COMPANY',
+    FINANCIAL_AMOUNT: 'OTHER',
+    VEHICLE_IDENTIFIER: 'OTHER',
+    DOCUMENT_REFERENCE: 'OTHER',
+    DOCUMENT_IDENTIFIER: 'OTHER',
+    IDENTIFYING_LINK: 'OTHER',
+    PERSON_ROLE_OR_TITLE: 'OTHER',
+    PERSON_ATTRIBUTE: 'OTHER',
+    AUTH_SECRET: 'OTHER',
+    DEVICE_IDENTIFIER: 'OTHER',
+    BIOMETRIC_DATA: 'OTHER',
+    HEALTH_DATA: 'OTHER',
+    CRIMINAL_OFFENCE_DATA: 'OTHER',
+    ETHNIC_ORIGIN: 'OTHER',
+    POLITICAL_OPINION: 'OTHER',
+    RELIGION_OR_BELIEF: 'OTHER',
+    SEXUAL_ORIENTATION: 'OTHER',
+    TRADE_UNION_MEMBERSHIP: 'OTHER',
+};
+/** Extract the bare entity name from a BIO tag like "B-PERSON_NAME" → "PERSON_NAME" */
+function bioToEntity(tag) {
+    if (tag === 'O' || !tag.includes('-'))
+        return null;
+    return tag.split('-').slice(1).join('-');
+}
+export class BardsaiProvider {
+    get name() { return this._name; }
+    constructor(env) {
+        this._name = MODEL_NAME;
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        this.tokenizer = null;
+        this.session = null;
+        this.loading = false;
+        this.loadError = null;
+        this.loadWaiters = [];
+        this.progressCallback = null;
+        this.threshold = DEFAULT_THRESHOLD;
+        this.env = env;
+    }
+    isLoaded() {
+        return this.session !== null && this.tokenizer !== null;
+    }
+    isLoading() {
+        return this.loading;
+    }
+    onProgress(callback) {
+        this.progressCallback = callback;
+    }
+    setThreshold(value) {
+        this.threshold = Math.max(0.05, Math.min(0.95, value));
+    }
+    getThreshold() {
+        return this.threshold;
+    }
+    async load() {
+        if (this.isLoaded())
+            return;
+        if (this.loadError)
+            throw this.loadError;
+        if (this.loading) {
+            return new Promise((resolve, reject) => {
+                this.loadWaiters.push({ resolve, reject });
+            });
+        }
+        this.loading = true;
+        try {
+            ort.env.wasm.wasmPaths = this.env.wasm.paths;
+            // Default to single-threaded WASM. ORT's multi-threaded path hangs silently
+            // in some cross-origin-isolated browser contexts when spawning pthread
+            // workers (model downloads but session.create never resolves).
+            ort.env.wasm.numThreads = this.env.wasm.numThreads ?? 1;
+            const loaderEnv = {
+                cache: this.env.modelCache,
+                fetch: this.env.fetch,
+                persistStorage: this.env.persistStorage,
+            };
+            // Load tokenizer and model in parallel (skip tokenizer if already loaded from a prior session)
+            const tasks = [
+                fetchModelBlob(loaderEnv, MODEL_URL, (downloaded, total) => this.progressCallback?.(downloaded, total)),
+            ];
+            if (!this.tokenizer) {
+                tasks.push(retryAsync(() => this.env.loadTokenizer(TOKENIZER_HF), 'Tokenizer download').then((t) => {
+                    this.tokenizer = t;
+                }));
+            }
+            const [modelBlob] = await Promise.all(tasks);
+            const blobUrl = URL.createObjectURL(modelBlob);
+            try {
+                this.session = await ort.InferenceSession.create(blobUrl, {
+                    executionProviders: ['wasm'],
+                });
+            }
+            finally {
+                // ONNX Runtime has already read the data into the WASM heap
+                URL.revokeObjectURL(blobUrl);
+            }
+            console.info(`[DocCloak] Model loaded: ${this._name} | inputs: [${this.session.inputNames.join(', ')}] | outputs: [${this.session.outputNames.join(', ')}]`);
+            this.loading = false;
+            this.loadWaiters.forEach((w) => w.resolve());
+            this.loadWaiters.length = 0;
+        }
+        catch (err) {
+            this.session = null;
+            this.loadError = err instanceof Error ? err : new Error(String(err));
+            this.loading = false;
+            this.loadWaiters.forEach((w) => w.reject(this.loadError));
+            this.loadWaiters.length = 0;
+            throw this.loadError;
+        }
+    }
+    release() {
+        if (this.session) {
+            this.session.release();
+            this.session = null;
+        }
+        this.loading = false;
+        this.loadError = null;
+    }
+    async detect(text, onProgress) {
+        if (!this.isLoaded())
+            await this.load();
+        if (!this.session || !this.tokenizer)
+            return [];
+        if (!text.trim())
+            return [];
+        // Split into words with character offsets
+        const words = [];
+        const wordRegex = /\S+/g;
+        let match;
+        while ((match = wordRegex.exec(text)) !== null) {
+            words.push({ word: match[0], start: match.index, end: match.index + match[0].length });
+        }
+        if (words.length === 0)
+            return [];
+        // Tokenize full text once to count subtokens per word via ▁ prefix
+        onProgress?.(0);
+        const fullEncoded = await this.tokenizer(words.map((w) => w.word).join(' '), {
+            add_special_tokens: false,
+        });
+        const tokenIds = Array.from(fullEncoded.input_ids.data);
+        const tokenStrings = this.tokenizer.model.convert_ids_to_tokens(tokenIds.map(Number));
+        // Count subtokens per word: ▁ prefix marks word boundaries (SentencePiece)
+        const subtokenCounts = [];
+        let currentCount = 0;
+        for (const tok of tokenStrings) {
+            if (tok.startsWith('\u2581') && currentCount > 0) {
+                subtokenCounts.push(currentCount);
+                currentCount = 1;
+            }
+            else {
+                currentCount++;
+            }
+        }
+        if (currentCount > 0)
+            subtokenCounts.push(currentCount);
+        // Pad if mismatch (rare edge case)
+        while (subtokenCounts.length < words.length)
+            subtokenCounts.push(1);
+        onProgress?.(0.1);
+        // Count total chunks for progress tracking
+        const maxSubtokens = MAX_SEQ_LENGTH - 2;
+        const overlapWords = CHUNK_OVERLAP;
+        // Pre-count chunks
+        let totalChunks = 0;
+        {
+            let ci = 0;
+            while (ci < words.length) {
+                let sum = 0, ce = ci;
+                while (ce < words.length && sum + subtokenCounts[ce] <= maxSubtokens) {
+                    sum += subtokenCounts[ce];
+                    ce++;
+                }
+                if (ce === ci)
+                    ce = ci + 1;
+                totalChunks++;
+                const next = Math.max(ci + 1, ce - overlapWords);
+                if (ce >= words.length)
+                    break;
+                ci = next;
+            }
+        }
+        // Process chunks with progress
+        const allEntities = [];
+        let i = 0;
+        let chunksDone = 0;
+        onProgress?.(0.1); // tokenization done
+        while (i < words.length) {
+            let subtokenSum = 0;
+            let end = i;
+            while (end < words.length && subtokenSum + subtokenCounts[end] <= maxSubtokens) {
+                subtokenSum += subtokenCounts[end];
+                end++;
+            }
+            if (end === i)
+                end = i + 1;
+            const chunkWords = words.slice(i, end);
+            const chunkEntities = await this.inferChunk(chunkWords, text);
+            allEntities.push(...chunkEntities);
+            chunksDone++;
+            onProgress?.(0.1 + 0.9 * (chunksDone / totalChunks)); // inference = 10-100%
+            await new Promise((r) => setTimeout(r, 0));
+            const nextStart = Math.max(i + 1, end - overlapWords);
+            i = nextStart;
+            if (end >= words.length)
+                break;
+        }
+        return this.deduplicateSpans(allEntities);
+    }
+    async inferChunk(words, fullText) {
+        const chunkText = words.map((w) => w.word).join(' ');
+        const encoded = await this.tokenizer(chunkText, {
+            return_tensors: 'np',
+            truncation: true,
+            max_length: MAX_SEQ_LENGTH,
+            padding: true,
+        });
+        const inputIds = encoded.input_ids.data;
+        const attentionMask = encoded.attention_mask.data;
+        const seqLen = encoded.input_ids.dims[1];
+        // Build word→subtoken alignment via ▁ prefix (no per-word tokenization needed)
+        const chunkTokenIds = Array.from(inputIds);
+        const chunkTokenStrings = this.tokenizer.model.convert_ids_to_tokens(chunkTokenIds.map(Number));
+        const wordSubtokenStart = [];
+        // Skip <s> at position 0; first real token at position 1
+        for (let t = 1; t < seqLen - 1; t++) {
+            const tok = chunkTokenStrings[t];
+            if (!tok || tok === '</s>')
+                break;
+            if (tok.startsWith('\u2581')) {
+                wordSubtokenStart.push(t);
+            }
+        }
+        // XLM-RoBERTa: only input_ids + attention_mask (no token_type_ids)
+        const feeds = {
+            input_ids: new ort.Tensor('int64', inputIds, [1, seqLen]),
+            attention_mask: new ort.Tensor('int64', attentionMask, [1, seqLen]),
+        };
+        const results = await this.session.run(feeds);
+        const logits = results.logits.data;
+        const numLabels = results.logits.dims[2];
+        // Per-subtoken predictions: argmax + softmax confidence
+        const predictions = [];
+        for (let t = 0; t < seqLen; t++) {
+            const offset = t * numLabels;
+            let maxIdx = 0;
+            let maxVal = logits[offset];
+            for (let l = 1; l < numLabels; l++) {
+                if (logits[offset + l] > maxVal) {
+                    maxVal = logits[offset + l];
+                    maxIdx = l;
+                }
+            }
+            let sumExp = 0;
+            for (let l = 0; l < numLabels; l++) {
+                sumExp += Math.exp(logits[offset + l] - maxVal);
+            }
+            predictions.push({ labelIdx: maxIdx, confidence: 1 / sumExp });
+        }
+        // Aggregate subtoken predictions to word level (first subtoken wins)
+        const entities = [];
+        let currentEntity = null;
+        for (let wi = 0; wi < words.length; wi++) {
+            const stIdx = wordSubtokenStart[wi];
+            if (stIdx === undefined || stIdx >= seqLen - 1)
+                break;
+            const pred = predictions[stIdx];
+            const label = ID2LABEL[pred.labelIdx] || 'O';
+            const entityName = bioToEntity(label);
+            const isB = label.startsWith('B-');
+            const isI = label.startsWith('I-');
+            if (isB) {
+                if (currentEntity) {
+                    this.emitEntity(currentEntity, words, fullText, entities);
+                }
+                currentEntity = {
+                    entityName: entityName,
+                    entityType: LABEL_TO_ENTITY_TYPE[entityName] || 'OTHER',
+                    startWord: wi,
+                    endWord: wi,
+                    confidence: pred.confidence,
+                    tokenCount: 1,
+                };
+            }
+            else if (isI && currentEntity && entityName === currentEntity.entityName) {
+                currentEntity.endWord = wi;
+                currentEntity.confidence += pred.confidence;
+                currentEntity.tokenCount += 1;
+            }
+            else {
+                if (currentEntity) {
+                    this.emitEntity(currentEntity, words, fullText, entities);
+                    currentEntity = null;
+                }
+            }
+        }
+        if (currentEntity) {
+            this.emitEntity(currentEntity, words, fullText, entities);
+        }
+        return entities;
+    }
+    emitEntity(entity, words, fullText, entities) {
+        const avgConfidence = entity.confidence / entity.tokenCount;
+        if (avgConfidence < this.threshold)
+            return;
+        const charStart = words[entity.startWord].start;
+        const charEnd = words[entity.endWord].end;
+        entities.push({
+            type: entity.entityType,
+            value: fullText.slice(charStart, charEnd),
+            start: charStart,
+            end: charEnd,
+            confidence: avgConfidence,
+            detector: `bardsai:${entity.entityName}`,
+        });
+    }
+    deduplicateSpans(entities) {
+        const sorted = [...entities].sort((a, b) => b.confidence - a.confidence);
+        const result = [];
+        for (const e of sorted) {
+            const overlaps = result.some((existing) => e.start < existing.end && e.end > existing.start);
+            if (!overlaps)
+                result.push(e);
+        }
+        return result;
+    }
+}

package/dist/providers/gliner.d.ts

@@ -0,0 +1,52 @@
+/**
+ * GLiNER PII Edge provider.
+ *
+ * Environment-agnostic: wasm paths, custom-label persistence (KV), model
+ * bytes (blob cache + fetch) and the tokenizer all arrive through the
+ * injected CoreEnv. This module never touches import.meta, localStorage,
+ * caches, navigator or @huggingface/transformers directly.
+ */
+import type { DetectedEntity, DetectionProvider, ProgressCallback } from '../types.js';
+import type { CoreEnv } from '../env.js';
+export declare class GlinerProvider implements DetectionProvider {
+    private _name;
+    get name(): string;
+    private env;
+    private tokenizer;
+    private session;
+    private loading;
+    private loadError;
+    private loadWaiters;
+    private progressCallback;
+    private threshold;
+    private customLabels;
+    constructor(env: CoreEnv);
+    isLoaded(): boolean;
+    isLoading(): boolean;
+    onProgress(callback: ProgressCallback): void;
+    setThreshold(value: number): void;
+    getThreshold(): number;
+    /**
+     * Get the combined list of all active labels (built-in + custom).
+     */
+    private getActiveLabels;
+    /**
+     * Get user-defined custom labels.
+     */
+    getCustomLabels(): string[];
+    /**
+     * Set custom labels and persist through the injected KV store. The
+     * in-memory list updates synchronously (before the first await), so
+     * fire-and-forget callers keep the previous behavior.
+     */
+    setCustomLabels(labels: string[]): Promise<void>;
+    /**
+     * Restore custom labels from the injected KV store on startup.
+     */
+    restoreCustomLabels(): Promise<void>;
+    load(): Promise<void>;
+    release(): void;
+    detect(text: string, onProgress?: (progress: number) => void): Promise<DetectedEntity[]>;
+    private buildPromptTokens;
+    private inferChunk;
+}