@doccloak/core 0.8.0 → 0.10.0

package/dist/model-loader.d.ts

@@ -14,6 +14,10 @@
  * - Serves the model from the injected BlobCache when available and prefers
  *   the cached (typically disk-backed) copy over the in-memory one to reduce
  *   peak RAM, which matters on iOS Safari where large tabs get killed.
+ * - Verifies downloads against a pinned SHA-256 when the caller provides one
+ *   (supply-chain hardening, T116): mismatches evict the cache entry and
+ *   reject with ModelIntegrityError; verified cache entries carry a marker
+ *   so they are not re-hashed on every startup.
  *
  * Environment-agnostic: all host capabilities (blob cache, fetch, persistent
  * storage request) are injected via ModelLoaderEnv. This module never touches
@@ -33,9 +37,47 @@ export interface ModelLoaderEnv {
     /** Best-effort persistent-storage request (navigator.storage.persist() on the web). */
     persistStorage?: () => Promise<boolean>;
 }
+/**
+ * A downloaded (or cached) model blob did not match the pinned SHA-256.
+ * Distinct error type so hosts can show a dedicated "integrity check
+ * failed" state instead of a generic download error. The offending cache
+ * entry is evicted before this is thrown; the regex/rules tier keeps
+ * working (existing degraded-mode behavior).
+ */
+export declare class ModelIntegrityError extends Error {
+    url: string;
+    expectedSha256: string;
+    actualSha256: string;
+    constructor(url: string, expectedSha256: string, actualSha256: string);
+}
+/** Successful SHA-256 verification of a model blob, surfaced to the model-status UI. */
+export interface ModelVerification {
+    url: string;
+    /** The verified SHA-256, lowercase hex - equals the pinned hash. */
+    sha256: string;
+}
+/**
+ * Cache key of the tiny "this URL was verified against this hash" marker.
+ * Uses a query parameter (not a URL fragment) because Cache Storage strips
+ * fragments, which would collide with the model entry itself. Keyed by
+ * url AND hash so bumping a pinned hash automatically invalidates markers
+ * written for the previous pin.
+ */
+export declare function verificationMarkerKey(url: string, sha256: string): string;
 export interface FetchModelOptions {
     maxAttempts?: number;
     retryBaseDelayMs?: number;
+    /**
+     * Pinned SHA-256 (lowercase hex) of the model blob. When set, every
+     * network download is hashed before use; a mismatch evicts the cache
+     * entry and rejects with ModelIntegrityError. Cached blobs that carry a
+     * verification marker for this hash skip re-hashing so startup cost is
+     * unchanged; cached blobs without a marker are hashed once and either
+     * marked verified or evicted and re-downloaded.
+     */
+    sha256?: string;
+    /** Fired once the served blob is known to match the pinned sha256. */
+    onVerified?: (verification: ModelVerification) => void;
 }
 /**
  * Download a model with progress reporting, resume and retry.
@@ -43,9 +85,10 @@ export interface FetchModelOptions {
  */
 export declare function fetchModelBlob(env: ModelLoaderEnv, url: string, onProgress?: DownloadProgress, options?: FetchModelOptions): Promise<Blob>;
 /**
- * Remove a single model from the cache (e.g. when its URL changes).
+ * Remove a single model from the cache (e.g. when its URL changes). When a
+ * pinned sha256 is given, the matching verification marker is removed too.
  */
-export declare function evictModelFromCache(env: ModelLoaderEnv, url: string): Promise<void>;
+export declare function evictModelFromCache(env: ModelLoaderEnv, url: string, sha256?: string): Promise<void>;
 /**
  * Retry an async operation with exponential backoff. Used for the smaller
  * companion downloads (tokenizer files) that go through libraries without

package/dist/model-loader.js

@@ -14,6 +14,10 @@
  * - Serves the model from the injected BlobCache when available and prefers
  *   the cached (typically disk-backed) copy over the in-memory one to reduce
  *   peak RAM, which matters on iOS Safari where large tabs get killed.
+ * - Verifies downloads against a pinned SHA-256 when the caller provides one
+ *   (supply-chain hardening, T116): mismatches evict the cache entry and
+ *   reject with ModelIntegrityError; verified cache entries carry a marker
+ *   so they are not re-hashed on every startup.
  *
  * Environment-agnostic: all host capabilities (blob cache, fetch, persistent
  * storage request) are injected via ModelLoaderEnv. This module never touches
@@ -30,6 +34,52 @@ class HttpError extends Error {
         this.status = status;
     }
 }
+/**
+ * A downloaded (or cached) model blob did not match the pinned SHA-256.
+ * Distinct error type so hosts can show a dedicated "integrity check
+ * failed" state instead of a generic download error. The offending cache
+ * entry is evicted before this is thrown; the regex/rules tier keeps
+ * working (existing degraded-mode behavior).
+ */
+export class ModelIntegrityError extends Error {
+    constructor(url, expectedSha256, actualSha256) {
+        super(`Model failed SHA-256 integrity check: expected ${expectedSha256.slice(0, 12)}..., ` +
+            `got ${actualSha256.slice(0, 12)}... (${url}). The downloaded copy was discarded.`);
+        this.name = 'ModelIntegrityError';
+        this.url = url;
+        this.expectedSha256 = expectedSha256;
+        this.actualSha256 = actualSha256;
+    }
+}
+/**
+ * Cache key of the tiny "this URL was verified against this hash" marker.
+ * Uses a query parameter (not a URL fragment) because Cache Storage strips
+ * fragments, which would collide with the model entry itself. Keyed by
+ * url AND hash so bumping a pinned hash automatically invalidates markers
+ * written for the previous pin.
+ */
+export function verificationMarkerKey(url, sha256) {
+    const sep = url.includes('?') ? '&' : '?';
+    return `${url}${sep}doccloak-sha256-verified=${sha256}`;
+}
+/**
+ * SHA-256 of a blob as lowercase hex. WebCrypto has no streaming digest,
+ * so the blob is hashed as one contiguous buffer: peak cost is one extra
+ * transient copy of the model, released before ONNX Runtime allocates its
+ * own copy.
+ */
+async function sha256Hex(blob) {
+    const buffer = await blob.arrayBuffer();
+    const digest = await crypto.subtle.digest('SHA-256', buffer);
+    return Array.from(new Uint8Array(digest), (b) => b.toString(16).padStart(2, '0')).join('');
+}
+/** Best-effort write of the verification marker (a 1-byte blob). */
+async function markVerified(cache, url, sha256) {
+    try {
+        await cache.put(verificationMarkerKey(url, sha256), new Blob(['1']));
+    }
+    catch { /* marker is an optimisation - next startup just re-hashes */ }
+}
 function isRetryable(err) {
     if (err instanceof HttpError) {
         return err.status === 408 || err.status === 429 || err.status >= 500;
@@ -80,9 +130,11 @@ async function tryCachePut(cache, url, blob) {
         if (!stored) {
             console.warn('[DocCloak] Model cache put refused (model still loaded in memory)');
         }
+        return stored;
     }
     catch (err) {
         console.warn('[DocCloak] Model cache put failed (model still loaded in memory):', err);
+        return false;
     }
 }
 /**
@@ -150,11 +202,33 @@ async function downloadAttempt(fetchFn, url, offset, onChunk, onHeaders) {
 export async function fetchModelBlob(env, url, onProgress, options) {
     const maxAttempts = options?.maxAttempts ?? MAX_ATTEMPTS;
     const retryBaseDelayMs = options?.retryBaseDelayMs ?? RETRY_BASE_DELAY_MS;
+    const expectedSha256 = options?.sha256?.toLowerCase();
     const cache = env.cache;
     const cachedBlob = await readFromCache(cache, url);
     if (cachedBlob) {
-        onProgress?.(cachedBlob.size, cachedBlob.size);
-        return cachedBlob;
+        if (!expectedSha256) {
+            onProgress?.(cachedBlob.size, cachedBlob.size);
+            return cachedBlob;
+        }
+        // Previously verified against this exact pin: skip the re-hash so
+        // startup cost is unchanged.
+        const marker = await readFromCache(cache, verificationMarkerKey(url, expectedSha256));
+        if (marker) {
+            onProgress?.(cachedBlob.size, cachedBlob.size);
+            options?.onVerified?.({ url, sha256: expectedSha256 });
+            return cachedBlob;
+        }
+        // No marker (cache predates pinning, or the pin changed): hash once.
+        const actualSha256 = await sha256Hex(cachedBlob);
+        if (actualSha256 === expectedSha256) {
+            await markVerified(cache, url, expectedSha256);
+            onProgress?.(cachedBlob.size, cachedBlob.size);
+            options?.onVerified?.({ url, sha256: expectedSha256 });
+            return cachedBlob;
+        }
+        // Stale or corrupt cached copy - evict and fall through to the network.
+        console.warn(`[DocCloak] Cached model failed SHA-256 check (expected ${expectedSha256.slice(0, 12)}..., got ${actualSha256.slice(0, 12)}...), re-downloading`);
+        await evictModelFromCache(env, url, expectedSha256);
     }
     await requestPersistentStorage(env);
     let lastProgressAt = 0;
@@ -213,22 +287,42 @@ export async function fetchModelBlob(env, url, onProgress, options) {
     reportProgress(downloaded, total || downloaded, true);
     let blob = new Blob(chunks);
     chunks = [];
-    await tryCachePut(cache, url, blob);
+    if (expectedSha256) {
+        // Verify the final assembled blob (covers both fresh and resumed
+        // downloads) BEFORE it is cached or handed to the runtime.
+        const actualSha256 = await sha256Hex(blob);
+        if (actualSha256 !== expectedSha256) {
+            await evictModelFromCache(env, url, expectedSha256);
+            throw new ModelIntegrityError(url, expectedSha256, actualSha256);
+        }
+    }
+    const stored = await tryCachePut(cache, url, blob);
+    if (expectedSha256 && stored)
+        await markVerified(cache, url, expectedSha256);
     // Prefer the cached (disk-backed) copy so the in-memory chunks can be
     // collected before ONNX Runtime allocates its own copy of the model.
     const diskBlob = await readFromCache(cache, url);
     if (diskBlob && diskBlob.size === blob.size)
         blob = diskBlob;
+    if (expectedSha256)
+        options?.onVerified?.({ url, sha256: expectedSha256 });
     return blob;
 }
 /**
- * Remove a single model from the cache (e.g. when its URL changes).
+ * Remove a single model from the cache (e.g. when its URL changes). When a
+ * pinned sha256 is given, the matching verification marker is removed too.
  */
-export async function evictModelFromCache(env, url) {
+export async function evictModelFromCache(env, url, sha256) {
     try {
         await env.cache.delete(url);
     }
     catch { /* ignore */ }
+    if (sha256) {
+        try {
+            await env.cache.delete(verificationMarkerKey(url, sha256.toLowerCase()));
+        }
+        catch { /* ignore */ }
+    }
 }
 /**
  * Retry an async operation with exponential backoff. Used for the smaller

package/dist/providers/bardsai.d.ts

@@ -12,6 +12,11 @@
  */
 import type { DetectedEntity, DetectionProvider, ProgressCallback } from '../types.js';
 import type { CoreEnv } from '../env.js';
+import type { ModelVerification } from '../model-loader.js';
+export declare const BARDSAI_MODEL_REVISION = "0e72e19f030ed4e661b1673e549af8e0dd176386";
+/** SHA-256 of onnx/model_quantized.onnx (278,736,360 bytes) at BARDSAI_MODEL_REVISION. */
+export declare const BARDSAI_MODEL_SHA256 = "8c9f555c743ed14eb7e505ff9d9c7785775a1671fd14031af33f424de8bd0e7b";
+export declare const BARDSAI_MODEL_URL = "https://huggingface.co/bardsai/eu-pii-anonimization-multilang/resolve/0e72e19f030ed4e661b1673e549af8e0dd176386/onnx/model_quantized.onnx";
 export declare class BardsaiProvider implements DetectionProvider {
     private _name;
     get name(): string;
@@ -23,7 +28,14 @@ export declare class BardsaiProvider implements DetectionProvider {
     private loadWaiters;
     private progressCallback;
     private threshold;
+    private verification;
     constructor(env: CoreEnv);
+    /**
+     * SHA-256 verification result of the served model blob (null until the
+     * model has been verified against the pinned hash). Surfaced so the
+     * model-status UI can show the positive "Model verified" state.
+     */
+    getVerification(): ModelVerification | null;
     isLoaded(): boolean;
     isLoading(): boolean;
     onProgress(callback: ProgressCallback): void;

package/dist/providers/bardsai.js

@@ -10,10 +10,27 @@
  * tokenizer all arrive through the injected CoreEnv. This module never touches
  * import.meta, caches, navigator or @huggingface/transformers directly.
  */
-import * as ort from 'onnxruntime-web';
-import { fetchModelBlob, retryAsync } from "../model-loader.js";
+// ort 1.29: the /webgpu entry is the non-deprecated build (native WebGPU EP +
+// wasm fallback, asyncify binary). The bare entry resolves the deprecated JSEP
+// bundle whose wasm exceeds Cloudflare Pages' 25 MiB per-file limit.
+import * as ort from 'onnxruntime-web/webgpu';
+import { evictModelFromCache, fetchModelBlob, retryAsync } from "../model-loader.js";
 // ── Model config ──────────────────────────────────────────
-const MODEL_URL = 'https://huggingface.co/bardsai/eu-pii-anonimization-multilang/resolve/main/onnx/model_quantized.onnx';
+// Supply-chain pinning (T116): the model is fetched from an immutable
+// commit revision, never from mutable resolve/main, and the downloaded
+// blob is verified against a pinned SHA-256. A model update is a
+// deliberate act: bump BARDSAI_MODEL_REVISION and BARDSAI_MODEL_SHA256
+// together in a reviewed commit and update
+// documentation/model-provenance.md.
+// Revision: main of bardsai/eu-pii-anonimization-multilang as of its last
+// modification 2026-05-13; pinned and hashed 2026-08-10.
+export const BARDSAI_MODEL_REVISION = '0e72e19f030ed4e661b1673e549af8e0dd176386';
+/** SHA-256 of onnx/model_quantized.onnx (278,736,360 bytes) at BARDSAI_MODEL_REVISION. */
+export const BARDSAI_MODEL_SHA256 = '8c9f555c743ed14eb7e505ff9d9c7785775a1671fd14031af33f424de8bd0e7b';
+export const BARDSAI_MODEL_URL = `https://huggingface.co/bardsai/eu-pii-anonimization-multilang/resolve/${BARDSAI_MODEL_REVISION}/onnx/model_quantized.onnx`;
+const MODEL_URL = BARDSAI_MODEL_URL;
+/** Pre-pinning download URL; its cache entry is evicted best-effort on load. */
+const LEGACY_MODEL_URL = 'https://huggingface.co/bardsai/eu-pii-anonimization-multilang/resolve/main/onnx/model_quantized.onnx';
 const TOKENIZER_HF = 'bardsai/eu-pii-anonimization-multilang';
 const MODEL_NAME = 'BardS.ai EU PII';
 const DEFAULT_THRESHOLD = 0.5;
@@ -102,8 +119,17 @@ export class BardsaiProvider {
         this.loadWaiters = [];
         this.progressCallback = null;
         this.threshold = DEFAULT_THRESHOLD;
+        this.verification = null;
         this.env = env;
     }
+    /**
+     * SHA-256 verification result of the served model blob (null until the
+     * model has been verified against the pinned hash). Surfaced so the
+     * model-status UI can show the positive "Model verified" state.
+     */
+    getVerification() {
+        return this.verification;
+    }
     isLoaded() {
         return this.session !== null && this.tokenizer !== null;
     }
@@ -141,9 +167,18 @@ export class BardsaiProvider {
                 fetch: this.env.fetch,
                 persistStorage: this.env.persistStorage,
             };
+            // Old cache entry from the pre-pinning resolve/main URL - drop it so
+            // a 279 MB stale copy does not linger against the origin quota.
+            void evictModelFromCache(loaderEnv, LEGACY_MODEL_URL);
             // Load tokenizer and model in parallel (skip tokenizer if already loaded from a prior session)
             const tasks = [
-                fetchModelBlob(loaderEnv, MODEL_URL, (downloaded, total) => this.progressCallback?.(downloaded, total)),
+                fetchModelBlob(loaderEnv, MODEL_URL, (downloaded, total) => this.progressCallback?.(downloaded, total), {
+                    sha256: BARDSAI_MODEL_SHA256,
+                    onVerified: (verification) => {
+                        this.verification = verification;
+                        console.info(`[DocCloak] Model verified: ${this._name} SHA-256 ${verification.sha256.slice(0, 12)}... matches the published hash (see documentation/model-provenance.md)`);
+                    },
+                }),
             ];
             if (!this.tokenizer) {
                 tasks.push(retryAsync(() => this.env.loadTokenizer(TOKENIZER_HF), 'Tokenizer download').then((t) => {
@@ -199,13 +234,11 @@ export class BardsaiProvider {
         }
         if (words.length === 0)
             return [];
-        // Tokenize full text once to count subtokens per word via ▁ prefix
+        // Tokenize full text once to count subtokens per word via ▁ prefix.
+        // transformers.js 4.x removed tokenizer.model.convert_ids_to_tokens;
+        // tokenize() returns the same SentencePiece token strings directly.
         onProgress?.(0);
-        const fullEncoded = await this.tokenizer(words.map((w) => w.word).join(' '), {
-            add_special_tokens: false,
-        });
-        const tokenIds = Array.from(fullEncoded.input_ids.data);
-        const tokenStrings = this.tokenizer.model.convert_ids_to_tokens(tokenIds.map(Number));
+        const tokenStrings = this.tokenizer.tokenize(words.map((w) => w.word).join(' '));
         // Count subtokens per word: ▁ prefix marks word boundaries (SentencePiece)
         const subtokenCounts = [];
         let currentCount = 0;
@@ -285,16 +318,15 @@ export class BardsaiProvider {
         const attentionMask = encoded.attention_mask.data;
         const seqLen = encoded.input_ids.dims[1];
         // Build word→subtoken alignment via ▁ prefix (no per-word tokenization needed)
-        const chunkTokenIds = Array.from(inputIds);
-        const chunkTokenStrings = this.tokenizer.model.convert_ids_to_tokens(chunkTokenIds.map(Number));
+        // transformers.js 4.x removed tokenizer.model.convert_ids_to_tokens;
+        // tokenize() returns the token strings without specials, so positions are
+        // offset by 1 for the leading <s> in the encoded sequence and the seqLen
+        // guard stops at the truncation boundary.
+        const chunkTokenStrings = this.tokenizer.tokenize(chunkText);
         const wordSubtokenStart = [];
-        // Skip <s> at position 0; first real token at position 1
-        for (let t = 1; t < seqLen - 1; t++) {
-            const tok = chunkTokenStrings[t];
-            if (!tok || tok === '</s>')
-                break;
-            if (tok.startsWith('\u2581')) {
-                wordSubtokenStart.push(t);
+        for (let i = 0; i < chunkTokenStrings.length && i + 1 < seqLen - 1; i++) {
+            if (chunkTokenStrings[i].startsWith('\u2581')) {
+                wordSubtokenStart.push(i + 1);
             }
         }
         // XLM-RoBERTa: only input_ids + attention_mask (no token_type_ids)

package/dist/providers/gliner-base.d.ts

@@ -0,0 +1,92 @@
+/**
+ * GLiNER PII Base provider (T122).
+ *
+ * knowledgator/gliner-pii-base-v1.0: deberta-v3-small backbone, zero-shot
+ * custom labels, span_mode "markerV0". Unlike the token_level small/edge
+ * models (gliner.ts), the ONNX graph takes span enumeration inputs
+ * (span_idx + span_mask) and emits span-level logits shaped
+ * [batch, num_words, max_width, num_classes] - one score per candidate
+ * (start word, width) pair per label. Verified against the pinned model in
+ * Node WASM: inputs [input_ids, attention_mask, words_mask, text_lengths,
+ * span_idx (int64), span_mask (bool)], output [logits].
+ *
+ * Environment-agnostic: wasm paths, custom-label persistence (KV), model
+ * bytes (blob cache + fetch) and the tokenizer all arrive through the
+ * injected CoreEnv. This module never touches import.meta, localStorage,
+ * caches, navigator or @huggingface/transformers directly.
+ */
+import type { DetectedEntity, DetectionProvider, ProgressCallback } from '../types.js';
+import type { CoreEnv } from '../env.js';
+import type { ModelVerification } from '../model-loader.js';
+import { type RawSpan } from './gliner.js';
+export declare const GLINER_BASE_MODEL_REVISION = "61726e0ad791dcab3e29339bbec3ad42ded65641";
+/** SHA-256 of onnx/model_quint8.onnx (196,757,174 bytes) at GLINER_BASE_MODEL_REVISION. */
+export declare const GLINER_BASE_MODEL_SHA256 = "0514c8fd86d0513ce5351a3267f132b57d5bcd8f99a90d43cde1228092881d19";
+export declare const GLINER_BASE_MODEL_URL = "https://huggingface.co/knowledgator/gliner-pii-base-v1.0/resolve/61726e0ad791dcab3e29339bbec3ad42ded65641/onnx/model_quint8.onnx";
+/** max_width from the repo's gliner_config.json: spans cover 1..12 words. */
+export declare const GLINER_BASE_MAX_WIDTH = 12;
+export interface SpanIndices {
+    /** Flattened (start, end) word pairs, shape [1, numSpans, 2]. */
+    spanIdx: BigInt64Array;
+    /** 1 for valid spans (end < textLength), 0 for padding, shape [1, numSpans]. */
+    spanMask: Uint8Array;
+    numSpans: number;
+}
+/**
+ * Enumerate all candidate word spans exactly like the python GLiNER
+ * SpanProcessor: start-major, width-minor - for every start word i, the
+ * pairs (i, i+0) .. (i, i+maxWidth-1). Spans that would run past the end
+ * of the text are masked out and their indices zeroed (mirroring
+ * span_idx * span_mask in the python collate).
+ */
+export declare function buildSpanIndices(textLength: number, maxWidth?: number): SpanIndices;
+/**
+ * Decode markerV0 span logits (shape [1, numWords, maxWidth, numClasses])
+ * into raw character spans: sigmoid per (start, width, class) cell,
+ * threshold, then map word indices to character offsets. Non-overlap
+ * selection (greedySelect) happens at the caller across chunks.
+ */
+export declare function decodeSpanLogits(logits: Float32Array, dims: readonly number[], textLength: number, labels: string[], threshold: number, charStarts: number[], charEnds: number[], fullText: string): RawSpan[];
+export declare class GlinerBaseProvider implements DetectionProvider {
+    private _name;
+    get name(): string;
+    private env;
+    private tokenizer;
+    private session;
+    private loading;
+    private loadError;
+    private loadWaiters;
+    private progressCallback;
+    private threshold;
+    private customLabels;
+    private verification;
+    constructor(env: CoreEnv);
+    /**
+     * SHA-256 verification result of the served model blob (null until the
+     * model has been verified against the pinned hash). Surfaced so the
+     * model-status UI can show the positive "Model verified" state.
+     */
+    getVerification(): ModelVerification | null;
+    isLoaded(): boolean;
+    isLoading(): boolean;
+    onProgress(callback: ProgressCallback): void;
+    setThreshold(value: number): void;
+    getThreshold(): number;
+    /** Get the combined list of all active labels (built-in + custom). */
+    private getActiveLabels;
+    /** Get user-defined custom labels. */
+    getCustomLabels(): string[];
+    /**
+     * Set custom labels and persist through the injected KV store. The
+     * in-memory list updates synchronously (before the first await), so
+     * fire-and-forget callers keep the previous behavior.
+     */
+    setCustomLabels(labels: string[]): Promise<void>;
+    /** Restore custom labels from the injected KV store on startup. */
+    restoreCustomLabels(): Promise<void>;
+    load(): Promise<void>;
+    release(): void;
+    detect(text: string, onProgress?: (progress: number) => void): Promise<DetectedEntity[]>;
+    private buildPromptTokens;
+    private inferChunk;
+}