@doccloak/core 0.9.0 → 0.10.0

package/dist/providers/gliner-base.js

@@ -0,0 +1,357 @@
+/**
+ * GLiNER PII Base provider (T122).
+ *
+ * knowledgator/gliner-pii-base-v1.0: deberta-v3-small backbone, zero-shot
+ * custom labels, span_mode "markerV0". Unlike the token_level small/edge
+ * models (gliner.ts), the ONNX graph takes span enumeration inputs
+ * (span_idx + span_mask) and emits span-level logits shaped
+ * [batch, num_words, max_width, num_classes] - one score per candidate
+ * (start word, width) pair per label. Verified against the pinned model in
+ * Node WASM: inputs [input_ids, attention_mask, words_mask, text_lengths,
+ * span_idx (int64), span_mask (bool)], output [logits].
+ *
+ * Environment-agnostic: wasm paths, custom-label persistence (KV), model
+ * bytes (blob cache + fetch) and the tokenizer all arrive through the
+ * injected CoreEnv. This module never touches import.meta, localStorage,
+ * caches, navigator or @huggingface/transformers directly.
+ */
+// ort 1.29: the /webgpu entry is the non-deprecated build (native WebGPU EP +
+// wasm fallback, asyncify binary). The bare entry resolves the deprecated JSEP
+// bundle whose wasm exceeds Cloudflare Pages' 25 MiB per-file limit.
+import * as ort from 'onnxruntime-web/webgpu';
+import { fetchModelBlob, retryAsync } from "../model-loader.js";
+import { DEFAULT_PII_LABELS, greedySelect, mapLabelToEntityType, sigmoid, splitWords, } from "./gliner.js";
+// ── Model config ──────────────────────────────────────────
+// Supply-chain pinning (T116): the model is fetched from an immutable
+// commit revision, never from mutable resolve/main, and the downloaded
+// blob is verified against a pinned SHA-256. A model update is a
+// deliberate act: bump GLINER_BASE_MODEL_REVISION and
+// GLINER_BASE_MODEL_SHA256 together in a reviewed commit and update
+// documentation/model-provenance.md.
+// Revision: main of knowledgator/gliner-pii-base-v1.0; pinned and hashed
+// 2026-08-26 (T122).
+export const GLINER_BASE_MODEL_REVISION = '61726e0ad791dcab3e29339bbec3ad42ded65641';
+/** SHA-256 of onnx/model_quint8.onnx (196,757,174 bytes) at GLINER_BASE_MODEL_REVISION. */
+export const GLINER_BASE_MODEL_SHA256 = '0514c8fd86d0513ce5351a3267f132b57d5bcd8f99a90d43cde1228092881d19';
+export const GLINER_BASE_MODEL_URL = `https://huggingface.co/knowledgator/gliner-pii-base-v1.0/resolve/${GLINER_BASE_MODEL_REVISION}/onnx/model_quint8.onnx`;
+const DEFAULT_MODEL_URL = GLINER_BASE_MODEL_URL;
+const DEFAULT_TOKENIZER_HF = 'knowledgator/gliner-pii-base-v1.0';
+const DEFAULT_MODEL_NAME = 'GLiNER PII Base';
+const CUSTOM_LABELS_STORAGE_KEY = 'doccloak-custom-labels';
+// Special token ids of the deberta-v3-small vocab (vocab_size 128003).
+// Verified against the repo's tokenizer_config.json at the pinned
+// revision - these differ from the ettin ids used by gliner.ts.
+const CLS_TOKEN_ID = 1; // [CLS]
+const SEP_TOKEN_ID = 2; // [SEP]
+const ENT_TOKEN_ID = 128001; // <<ENT>> (class_token_index in gliner_config.json)
+const SEP_PROMPT_TOKEN_ID = 128002; // <<SEP>> (prompt separator, not [SEP])
+/** max_width from the repo's gliner_config.json: spans cover 1..12 words. */
+export const GLINER_BASE_MAX_WIDTH = 12;
+const DEFAULT_THRESHOLD = 0.35;
+const MAX_WORDS_PER_CHUNK = 150;
+const CHUNK_OVERLAP = 40;
+/**
+ * Enumerate all candidate word spans exactly like the python GLiNER
+ * SpanProcessor: start-major, width-minor - for every start word i, the
+ * pairs (i, i+0) .. (i, i+maxWidth-1). Spans that would run past the end
+ * of the text are masked out and their indices zeroed (mirroring
+ * span_idx * span_mask in the python collate).
+ */
+export function buildSpanIndices(textLength, maxWidth = GLINER_BASE_MAX_WIDTH) {
+    const numSpans = textLength * maxWidth;
+    const spanIdx = new BigInt64Array(numSpans * 2);
+    const spanMask = new Uint8Array(numSpans);
+    for (let start = 0; start < textLength; start++) {
+        for (let width = 0; width < maxWidth; width++) {
+            const flat = start * maxWidth + width;
+            const end = start + width;
+            if (end < textLength) {
+                spanIdx[flat * 2] = BigInt(start);
+                spanIdx[flat * 2 + 1] = BigInt(end);
+                spanMask[flat] = 1;
+            }
+            // invalid spans stay (0, 0) with mask 0
+        }
+    }
+    return { spanIdx, spanMask, numSpans };
+}
+/**
+ * Decode markerV0 span logits (shape [1, numWords, maxWidth, numClasses])
+ * into raw character spans: sigmoid per (start, width, class) cell,
+ * threshold, then map word indices to character offsets. Non-overlap
+ * selection (greedySelect) happens at the caller across chunks.
+ */
+export function decodeSpanLogits(logits, dims, textLength, labels, threshold, charStarts, charEnds, fullText) {
+    const maxWidth = dims[2];
+    const numClasses = dims[3];
+    const spans = [];
+    for (let start = 0; start < textLength; start++) {
+        for (let width = 0; width < maxWidth; width++) {
+            const end = start + width;
+            if (end >= textLength)
+                break;
+            const base = (start * maxWidth + width) * numClasses;
+            for (let cls = 0; cls < numClasses && cls < labels.length; cls++) {
+                const prob = sigmoid(logits[base + cls]);
+                if (prob < threshold)
+                    continue;
+                const charStart = charStarts[start];
+                const charEnd = charEnds[end];
+                spans.push([fullText.slice(charStart, charEnd), charStart, charEnd, labels[cls], prob]);
+            }
+        }
+    }
+    return spans;
+}
+// ── Provider ──────────────────────────────────────────────
+export class GlinerBaseProvider {
+    get name() { return this._name; }
+    constructor(env) {
+        this._name = DEFAULT_MODEL_NAME;
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        this.tokenizer = null;
+        this.session = null;
+        this.loading = false;
+        this.loadError = null;
+        this.loadWaiters = [];
+        this.progressCallback = null;
+        this.threshold = DEFAULT_THRESHOLD;
+        this.customLabels = [];
+        this.verification = null;
+        this.env = env;
+    }
+    /**
+     * SHA-256 verification result of the served model blob (null until the
+     * model has been verified against the pinned hash). Surfaced so the
+     * model-status UI can show the positive "Model verified" state.
+     */
+    getVerification() {
+        return this.verification;
+    }
+    isLoaded() {
+        return this.session !== null && this.tokenizer !== null;
+    }
+    isLoading() {
+        return this.loading;
+    }
+    onProgress(callback) {
+        this.progressCallback = callback;
+    }
+    setThreshold(value) {
+        this.threshold = Math.max(0.05, Math.min(0.95, value));
+    }
+    getThreshold() {
+        return this.threshold;
+    }
+    /** Get the combined list of all active labels (built-in + custom). */
+    getActiveLabels() {
+        return [...DEFAULT_PII_LABELS, ...this.customLabels];
+    }
+    /** Get user-defined custom labels. */
+    getCustomLabels() {
+        return [...this.customLabels];
+    }
+    /**
+     * Set custom labels and persist through the injected KV store. The
+     * in-memory list updates synchronously (before the first await), so
+     * fire-and-forget callers keep the previous behavior.
+     */
+    async setCustomLabels(labels) {
+        this.customLabels = labels.filter((l) => l.trim().length > 0);
+        try {
+            await this.env.kv.set(CUSTOM_LABELS_STORAGE_KEY, JSON.stringify(this.customLabels));
+        }
+        catch { /* KV backend unavailable (e.g. storage-less Web Worker) */ }
+    }
+    /** Restore custom labels from the injected KV store on startup. */
+    async restoreCustomLabels() {
+        try {
+            const saved = await this.env.kv.get(CUSTOM_LABELS_STORAGE_KEY);
+            if (saved) {
+                this.customLabels = JSON.parse(saved);
+            }
+        }
+        catch { /* KV backend unavailable in Web Worker - labels are passed via message */ }
+    }
+    async load() {
+        if (this.isLoaded())
+            return;
+        if (this.loadError)
+            throw this.loadError;
+        if (this.loading) {
+            return new Promise((resolve, reject) => {
+                this.loadWaiters.push({ resolve, reject });
+            });
+        }
+        this.loading = true;
+        try {
+            ort.env.wasm.wasmPaths = this.env.wasm.paths;
+            // Default to single-threaded WASM. ORT's multi-threaded path hangs
+            // silently in some cross-origin-isolated browser contexts when spawning
+            // pthread workers (model downloads but session.create never resolves).
+            ort.env.wasm.numThreads = this.env.wasm.numThreads ?? 1;
+            const loaderEnv = {
+                cache: this.env.modelCache,
+                fetch: this.env.fetch,
+                persistStorage: this.env.persistStorage,
+            };
+            // Cache note (T122): the BardS.ai blob is deliberately NOT evicted -
+            // bardsai stays selectable as a legacy provider.
+            // Load tokenizer and model in parallel (skip tokenizer if already loaded from a prior session)
+            const tasks = [
+                fetchModelBlob(loaderEnv, DEFAULT_MODEL_URL, (downloaded, total) => this.progressCallback?.(downloaded, total), {
+                    sha256: GLINER_BASE_MODEL_SHA256,
+                    onVerified: (verification) => {
+                        this.verification = verification;
+                        console.info(`[DocCloak] Model verified: ${this._name} SHA-256 ${verification.sha256.slice(0, 12)}... matches the published hash (see documentation/model-provenance.md)`);
+                    },
+                }),
+            ];
+            if (!this.tokenizer) {
+                tasks.push(retryAsync(() => this.env.loadTokenizer(DEFAULT_TOKENIZER_HF), 'Tokenizer download').then((t) => {
+                    this.tokenizer = t;
+                }));
+            }
+            const [modelBlob] = await Promise.all(tasks);
+            const blobUrl = URL.createObjectURL(modelBlob);
+            try {
+                this.session = await ort.InferenceSession.create(blobUrl, {
+                    executionProviders: ['wasm'],
+                });
+            }
+            finally {
+                // ONNX Runtime has already read the data into the WASM heap
+                URL.revokeObjectURL(blobUrl);
+            }
+            console.info(`[DocCloak] Model loaded: ${this._name} | inputs: [${this.session.inputNames.join(', ')}] | outputs: [${this.session.outputNames.join(', ')}]`);
+            this.loading = false;
+            this.loadWaiters.forEach((w) => w.resolve());
+            this.loadWaiters.length = 0;
+        }
+        catch (err) {
+            this.session = null;
+            this.loadError = err instanceof Error ? err : new Error(String(err));
+            this.loading = false;
+            this.loadWaiters.forEach((w) => w.reject(this.loadError));
+            this.loadWaiters.length = 0;
+            throw this.loadError;
+        }
+    }
+    release() {
+        if (this.session) {
+            this.session.release();
+            this.session = null;
+        }
+        this.loading = false;
+        this.loadError = null;
+    }
+    async detect(text, onProgress) {
+        if (!this.isLoaded())
+            await this.load();
+        if (!this.session || !this.tokenizer)
+            return [];
+        const { words, starts, ends } = splitWords(text);
+        if (words.length === 0)
+            return [];
+        // Build prompt tokens once (shared across all chunks)
+        const activeLabels = this.getActiveLabels();
+        const promptTokens = this.buildPromptTokens(activeLabels);
+        const chunkSize = MAX_WORDS_PER_CHUNK;
+        const overlap = CHUNK_OVERLAP;
+        // Split into chunks if text is too long
+        const allSpans = [];
+        onProgress?.(0);
+        if (words.length <= chunkSize) {
+            // Single chunk - no splitting needed
+            const spans = await this.inferChunk(words, starts, ends, text, promptTokens, activeLabels);
+            allSpans.push(...spans);
+            onProgress?.(1);
+        }
+        else {
+            // Count total chunks
+            let totalChunks = 0;
+            for (let cs = 0; cs < words.length; cs += chunkSize - overlap) {
+                totalChunks++;
+                if (Math.min(cs + chunkSize, words.length) >= words.length)
+                    break;
+            }
+            // Multiple overlapping chunks
+            let chunksDone = 0;
+            for (let chunkStart = 0; chunkStart < words.length; chunkStart += chunkSize - overlap) {
+                const chunkEnd = Math.min(chunkStart + chunkSize, words.length);
+                const chunkWords = words.slice(chunkStart, chunkEnd);
+                const chunkStarts = starts.slice(chunkStart, chunkEnd);
+                const chunkEnds = ends.slice(chunkStart, chunkEnd);
+                const spans = await this.inferChunk(chunkWords, chunkStarts, chunkEnds, text, promptTokens, activeLabels);
+                allSpans.push(...spans);
+                chunksDone++;
+                onProgress?.(chunksDone / totalChunks);
+                await new Promise((r) => setTimeout(r, 0));
+                if (chunkEnd >= words.length)
+                    break;
+            }
+        }
+        // Greedy non-overlapping selection across all chunks
+        const selected = greedySelect(allSpans);
+        return selected
+            .filter(([spanText]) => spanText.trim().length >= 2)
+            .map(([spanText, start, end, label, score]) => ({
+            type: mapLabelToEntityType(label),
+            value: spanText,
+            start,
+            end,
+            confidence: score,
+            detector: `gliner-base:${label}`,
+        }));
+    }
+    buildPromptTokens(labels) {
+        const tokens = [];
+        for (const label of labels) {
+            tokens.push(ENT_TOKEN_ID);
+            for (const part of label.split(' ')) {
+                const subTokens = this.tokenizer.encode(part).slice(1, -1);
+                tokens.push(...subTokens);
+            }
+        }
+        tokens.push(SEP_PROMPT_TOKEN_ID);
+        return tokens;
+    }
+    async inferChunk(words, charStarts, charEnds, fullText, promptTokens, labels) {
+        if (!this.session)
+            return [];
+        // Build token sequence: [CLS] + prompt + text words + [SEP]
+        const inputIds = [CLS_TOKEN_ID, ...promptTokens];
+        const attentionMask = new Array(inputIds.length).fill(1);
+        const wordsMask = new Array(inputIds.length).fill(0);
+        let wordCounter = 1;
+        for (const word of words) {
+            const subTokens = this.tokenizer.encode(word).slice(1, -1);
+            for (let j = 0; j < subTokens.length; j++) {
+                inputIds.push(subTokens[j]);
+                attentionMask.push(1);
+                wordsMask.push(j === 0 ? wordCounter : 0);
+            }
+            wordCounter++;
+        }
+        inputIds.push(SEP_TOKEN_ID);
+        attentionMask.push(1);
+        wordsMask.push(0);
+        const seqLen = inputIds.length;
+        const textLength = words.length;
+        const { spanIdx, spanMask, numSpans } = buildSpanIndices(textLength);
+        const feeds = {
+            input_ids: new ort.Tensor('int64', BigInt64Array.from(inputIds.map(BigInt)), [1, seqLen]),
+            attention_mask: new ort.Tensor('int64', BigInt64Array.from(attentionMask.map(BigInt)), [1, seqLen]),
+            words_mask: new ort.Tensor('int64', BigInt64Array.from(wordsMask.map(BigInt)), [1, seqLen]),
+            text_lengths: new ort.Tensor('int64', BigInt64Array.from([BigInt(textLength)]), [1, 1]),
+            span_idx: new ort.Tensor('int64', spanIdx, [1, numSpans, 2]),
+            span_mask: new ort.Tensor('bool', spanMask, [1, numSpans]),
+        };
+        const results = await this.session.run(feeds);
+        const outputName = this.session.outputNames[0] || 'logits';
+        const output = results[outputName];
+        const logits = output.data;
+        // markerV0: logits shape [1, num_words, max_width, num_classes]
+        return decodeSpanLogits(logits, output.dims, textLength, labels, this.threshold, charStarts, charEnds, fullText);
+    }
+}

package/dist/providers/gliner.d.ts

@@ -1,18 +1,34 @@
 /**
- * GLiNER PII Edge provider.
+ * GLiNER PII Small provider.
  *
  * Environment-agnostic: wasm paths, custom-label persistence (KV), model
  * bytes (blob cache + fetch) and the tokenizer all arrive through the
  * injected CoreEnv. This module never touches import.meta, localStorage,
  * caches, navigator or @huggingface/transformers directly.
  */
-import type { DetectedEntity, DetectionProvider, ProgressCallback } from '../types.js';
+import type { DetectedEntity, EntityType, DetectionProvider, ProgressCallback } from '../types.js';
 import type { CoreEnv } from '../env.js';
 import type { ModelVerification } from '../model-loader.js';
-export declare const GLINER_MODEL_REVISION = "9b7f39b0a2da971a5beea78d35f1539d4009c891";
-/** SHA-256 of onnx/model_quint8.onnx (45,820,894 bytes) at GLINER_MODEL_REVISION. */
-export declare const GLINER_MODEL_SHA256 = "988acb03456b26e2d9f2521016d820310c2ed64deb4a846297d3289f0c2eb7e4";
-export declare const GLINER_MODEL_URL = "https://huggingface.co/knowledgator/gliner-pii-edge-v1.0/resolve/9b7f39b0a2da971a5beea78d35f1539d4009c891/onnx/model_quint8.onnx";
+export declare const GLINER_MODEL_REVISION = "d21aad5b4a7ec82b3d0970fd1ac74a12c087d85e";
+/** SHA-256 of onnx/model_quint8.onnx (82,680,500 bytes) at GLINER_MODEL_REVISION. */
+export declare const GLINER_MODEL_SHA256 = "891589426ee96f2748b16439f44fad8c3f97e198e002a6637e58dee989500216";
+export declare const GLINER_MODEL_URL = "https://huggingface.co/knowledgator/gliner-pii-small-v1.0/resolve/d21aad5b4a7ec82b3d0970fd1ac74a12c087d85e/onnx/model_quint8.onnx";
+/**
+ * Built-in zero-shot label set, shared with the GLiNER PII Base provider
+ * (gliner-base.ts) - the models are from the same zero-shot family, so the
+ * same prompt labels work for both.
+ */
+export declare const DEFAULT_PII_LABELS: string[];
+/** Shared with gliner-base.ts (same label set, same mapping). */
+export declare function mapLabelToEntityType(label: string): EntityType;
+export declare function splitWords(text: string): {
+    words: string[];
+    starts: number[];
+    ends: number[];
+};
+export declare function sigmoid(x: number): number;
+export type RawSpan = [string, number, number, string, number];
+export declare function greedySelect(spans: RawSpan[]): RawSpan[];
 export declare class GlinerProvider implements DetectionProvider {
     private _name;
     get name(): string;

package/dist/providers/gliner.js

@@ -1,12 +1,15 @@
 /**
- * GLiNER PII Edge provider.
+ * GLiNER PII Small provider.
  *
  * Environment-agnostic: wasm paths, custom-label persistence (KV), model
  * bytes (blob cache + fetch) and the tokenizer all arrive through the
  * injected CoreEnv. This module never touches import.meta, localStorage,
  * caches, navigator or @huggingface/transformers directly.
  */
-import * as ort from 'onnxruntime-web';
+// ort 1.29: the /webgpu entry is the non-deprecated build (native WebGPU EP +
+// wasm fallback, asyncify binary). The bare entry resolves the deprecated JSEP
+// bundle whose wasm exceeds Cloudflare Pages' 25 MiB per-file limit.
+import * as ort from 'onnxruntime-web/webgpu';
 import { evictModelFromCache, fetchModelBlob, retryAsync } from "../model-loader.js";
 // ── Model config ──────────────────────────────────────────
 // Supply-chain pinning (T116): the model is fetched from an immutable
@@ -15,17 +18,26 @@ import { evictModelFromCache, fetchModelBlob, retryAsync } from "../model-loader
 // deliberate act: bump GLINER_MODEL_REVISION and GLINER_MODEL_SHA256
 // together in a reviewed commit and update
 // documentation/model-provenance.md.
-// Revision: main of knowledgator/gliner-pii-edge-v1.0 as of its last
-// modification 2026-03-26; pinned and hashed 2026-08-10.
-export const GLINER_MODEL_REVISION = '9b7f39b0a2da971a5beea78d35f1539d4009c891';
-/** SHA-256 of onnx/model_quint8.onnx (45,820,894 bytes) at GLINER_MODEL_REVISION. */
-export const GLINER_MODEL_SHA256 = '988acb03456b26e2d9f2521016d820310c2ed64deb4a846297d3289f0c2eb7e4';
-export const GLINER_MODEL_URL = `https://huggingface.co/knowledgator/gliner-pii-edge-v1.0/resolve/${GLINER_MODEL_REVISION}/onnx/model_quint8.onnx`;
+// Revision: main of knowledgator/gliner-pii-small-v1.0; pinned and
+// hashed 2026-08-26 (T121 swap from gliner-pii-edge-v1.0, +1.3 F1
+// drop-in per documentation/model-research-2026-08.md).
+export const GLINER_MODEL_REVISION = 'd21aad5b4a7ec82b3d0970fd1ac74a12c087d85e';
+/** SHA-256 of onnx/model_quint8.onnx (82,680,500 bytes) at GLINER_MODEL_REVISION. */
+export const GLINER_MODEL_SHA256 = '891589426ee96f2748b16439f44fad8c3f97e198e002a6637e58dee989500216';
+export const GLINER_MODEL_URL = `https://huggingface.co/knowledgator/gliner-pii-small-v1.0/resolve/${GLINER_MODEL_REVISION}/onnx/model_quint8.onnx`;
 const DEFAULT_MODEL_URL = GLINER_MODEL_URL;
-/** Pre-pinning download URL; its cache entry is evicted best-effort on load. */
-const LEGACY_MODEL_URL = 'https://huggingface.co/knowledgator/gliner-pii-edge-v1.0/resolve/main/onnx/model_quint8.onnx';
-const DEFAULT_TOKENIZER_HF = 'knowledgator/gliner-pii-edge-v1.0';
-const DEFAULT_MODEL_NAME = 'GLiNER PII Edge';
+/**
+ * Superseded download URLs; their cache entries are evicted best-effort on
+ * load so stale ~46 MB copies do not linger against the origin quota:
+ * the pre-pinning edge resolve/main URL and the pinned edge URL replaced
+ * by gliner-pii-small-v1.0 (T121).
+ */
+const LEGACY_MODEL_URLS = [
+    'https://huggingface.co/knowledgator/gliner-pii-edge-v1.0/resolve/main/onnx/model_quint8.onnx',
+    'https://huggingface.co/knowledgator/gliner-pii-edge-v1.0/resolve/9b7f39b0a2da971a5beea78d35f1539d4009c891/onnx/model_quint8.onnx',
+];
+const DEFAULT_TOKENIZER_HF = 'knowledgator/gliner-pii-small-v1.0';
+const DEFAULT_MODEL_NAME = 'GLiNER PII Small';
 const CUSTOM_LABELS_STORAGE_KEY = 'doccloak-custom-labels';
 const CLS_TOKEN_ID = 50281;
 const SEP_TOKEN_ID = 50282;
@@ -34,7 +46,12 @@ const SEP_PROMPT_TOKEN_ID = 50369; // <<SEP>> (prompt separator, not [SEP])
 const DEFAULT_THRESHOLD = 0.35;
 const MAX_WORDS_PER_CHUNK = 150;
 const CHUNK_OVERLAP = 40;
-const DEFAULT_PII_LABELS = [
+/**
+ * Built-in zero-shot label set, shared with the GLiNER PII Base provider
+ * (gliner-base.ts) - the models are from the same zero-shot family, so the
+ * same prompt labels work for both.
+ */
+export const DEFAULT_PII_LABELS = [
     'person name',
     'calendar date',
     'email address', 'phone number', 'ip address',
@@ -46,7 +63,8 @@ const DEFAULT_PII_LABELS = [
     'company name',
 ];
 // ── Label → EntityType mapping ────────────────────────────
-function mapLabelToEntityType(label) {
+/** Shared with gliner-base.ts (same label set, same mapping). */
+export function mapLabelToEntityType(label) {
     const l = label.toLowerCase();
     if (l === 'person name')
         return 'PERSON';
@@ -73,8 +91,10 @@ function mapLabelToEntityType(label) {
     return 'OTHER';
 }
 // ── Word splitter ─────────────────────────────────────────
+// Mirrors GLiNER's WhitespaceTokenSplitter (words, plus standalone
+// punctuation); shared with gliner-base.ts (same words_splitter_type).
 const WORD_PATTERN = /[\p{L}\p{N}]+(?:[-_][\p{L}\p{N}]+)*|\S/gu;
-function splitWords(text) {
+export function splitWords(text) {
     const words = [];
     const starts = [];
     const ends = [];
@@ -88,10 +108,10 @@ function splitWords(text) {
     return { words, starts, ends };
 }
 // ── Sigmoid ───────────────────────────────────────────────
-function sigmoid(x) {
+export function sigmoid(x) {
     return 1 / (1 + Math.exp(-x));
 }
-function greedySelect(spans) {
+export function greedySelect(spans) {
     const sorted = spans.slice().sort((a, b) => b[4] - a[4]);
     const selected = [];
     for (const span of sorted) {
@@ -199,9 +219,12 @@ export class GlinerProvider {
                 fetch: this.env.fetch,
                 persistStorage: this.env.persistStorage,
             };
-            // Old cache entry from the pre-pinning resolve/main URL - drop it so
-            // a 45 MB stale copy does not linger against the origin quota.
-            void evictModelFromCache(loaderEnv, LEGACY_MODEL_URL);
+            // Old cache entries from superseded URLs (pre-pinning resolve/main
+            // and the retired edge pin) - drop them so stale copies do not
+            // linger against the origin quota.
+            for (const legacyUrl of LEGACY_MODEL_URLS) {
+                void evictModelFromCache(loaderEnv, legacyUrl);
+            }
             // Load tokenizer and model in parallel (skip tokenizer if already loaded from a prior session)
             const tasks = [
                 fetchModelBlob(loaderEnv, DEFAULT_MODEL_URL, (downloaded, total) => this.progressCallback?.(downloaded, total), {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@doccloak/core",
-  "version": "0.9.0",
+  "version": "0.10.0",
   "description": "Environment-agnostic document anonymization engine for the DocCloak ecosystem.",
   "license": "Apache-2.0",
   "author": "Witold Lojek",
@@ -62,6 +62,7 @@
     "rules:gen": "node scripts/generate-rules-module.mjs",
     "rules:check": "node scripts/generate-rules-module.mjs --check",
     "bench": "node --experimental-strip-types eval/run.mts",
+    "bench:multilang": "node --experimental-strip-types eval/multilang.mts",
     "bench:data": "node eval/download.mjs",
     "test": "vitest run",
     "test:watch": "vitest",
@@ -75,7 +76,7 @@
   "dependencies": {
     "cfb": "1.2.2",
     "jszip": "3.10.1",
-    "onnxruntime-web": "1.19.2",
+    "onnxruntime-web": "1.29.0",
     "tesseract.js": "7.0.0"
   }
 }