@doccloak/core 0.10.0 → 0.12.0

package/dist/pipeline.js

@@ -1,40 +1,93 @@
 /**
  * Pure entity pipeline (T005).
  *
- * Extracted verbatim from the web app's detection.worker.ts: text and
- * entities in, entities out. No worker messaging, no providers, no
- * environment access. The worker (and any other host) composes these
- * functions around its own ML provider call.
+ * Extracted from the web app's detection.worker.ts: text and entities in,
+ * entities out. No worker messaging, no providers, no environment access.
+ * The worker (and any other host) composes these functions around its own
+ * ML provider call.
  *
- * Function bodies are byte-identical to the originals by design; only the
- * signatures of detectWithRegex (regexRegion becomes a parameter instead of
- * worker module state) and detectEntities (ML results are passed in instead
- * of fetched from a provider) differ.
+ * Security pass 2026-09:
+ *  - T180: detectWithRegex runs every rule line by line (rules flagged
+ *    `multiline` see the whole text), so quadratic patterns cost O(line^2)
+ *    instead of O(document^2).
+ *  - T181: filterFalsePositives is script-aware (CJK/Thai and PERSON values
+ *    of two characters survive) and propagateEntities uses Unicode letter/
+ *    digit boundaries, exact offsets on the original text and a common-word
+ *    guard for lowercase PERSON matches.
  */
 import { ALL_REGEX_RULES } from "./regex/index.js";
+/**
+ * Split on '\n' keeping offsets. '\r' stays inside the line (a CRLF line then
+ * ends with '\r', exactly as it did when the whole text was matched at once).
+ * Empty lines are skipped: no rule can match the empty string.
+ */
+export function segmentLines(text) {
+    const segments = [];
+    let lineStart = 0;
+    while (lineStart <= text.length) {
+        const newline = text.indexOf('\n', lineStart);
+        const lineEnd = newline === -1 ? text.length : newline;
+        if (lineEnd > lineStart) {
+            segments.push({ text: text.slice(lineStart, lineEnd), offset: lineStart });
+        }
+        if (newline === -1)
+            break;
+        lineStart = newline + 1;
+    }
+    return segments;
+}
+function runRule(rule, segment, out) {
+    rule.pattern.lastIndex = 0;
+    let match;
+    while ((match = rule.pattern.exec(segment.text)) !== null) {
+        if (match[0].length === 0) {
+            // Defensive: a zero-width match would otherwise loop forever.
+            rule.pattern.lastIndex++;
+            continue;
+        }
+        if (rule.validate && !rule.validate(match[0]))
+            continue;
+        out.push({
+            type: rule.type,
+            value: match[0],
+            start: segment.offset + match.index,
+            end: segment.offset + match.index + match[0].length,
+            confidence: rule.confidence,
+            detector: rule.detector,
+        });
+    }
+}
+/**
+ * Run the regex rules of a region over the text.
+ *
+ * Rules without `multiline` run on each line separately (offsets are added
+ * back, so spans are document-absolute); rules with `multiline: true` run on
+ * the whole text. Output order is rule-major, then position, as before.
+ *
+ * Complexity note (T180): several street/company rules are quadratic in the
+ * length of their input. Per-line matching bounds that to the longest line,
+ * so 100 KB of ordinary prose is fast; a single 100 KB line without newlines
+ * is still slow until the patterns themselves are bounded (T198).
+ */
 export function detectWithRegex(text, regexRegion = 'all') {
     const entities = [];
     const rules = regexRegion === 'all'
         ? ALL_REGEX_RULES
         : ALL_REGEX_RULES.filter((r) => r.region === 'universal' || r.region === regexRegion);
+    const whole = { text, offset: 0 };
+    const lines = segmentLines(text);
     for (const rule of rules) {
-        rule.pattern.lastIndex = 0;
-        let match;
-        while ((match = rule.pattern.exec(text)) !== null) {
-            if (rule.validate && !rule.validate(match[0]))
-                continue;
-            entities.push({
-                type: rule.type,
-                value: match[0],
-                start: match.index,
-                end: match.index + match[0].length,
-                confidence: rule.confidence,
-                detector: rule.detector,
-            });
+        if (rule.multiline) {
+            runRule(rule, whole, entities);
+        }
+        else {
+            for (const line of lines)
+                runRule(rule, line, entities);
         }
     }
     return entities;
 }
+// ── resolveOverlaps ────────────────────────────────────────
 export function resolveOverlaps(entities) {
     const sorted = [...entities].sort((a, b) => {
         if (a.start !== b.start)
@@ -54,13 +107,152 @@ export function resolveOverlaps(entities) {
     }
     return resolved;
 }
+// ── trimSpanPunctuation ────────────────────────────────────
+const LEADING_PUNCT = /^[\s(\[{"'«“‘„]+/u;
+const TRAILING_PUNCT = /[\s,.;:!?)\]}"'»”’]+$/u;
+/**
+ * Word-level providers (BardS.ai splits on whitespace) return spans that
+ * carry the punctuation glued to the last word: "Anna Nowak," or
+ * "jan@example.com.". The redacted text then loses that punctuation and the
+ * mapping stores it as part of the value. Trim a fixed set of opening and
+ * closing punctuation at both ends of every span (T204); a span that would
+ * become empty is dropped. Regex spans are not passed through this: their
+ * patterns already stop at the identifier.
+ */
+export function trimSpanPunctuation(entities) {
+    const out = [];
+    for (const entity of entities) {
+        const value = entity.value;
+        const lead = LEADING_PUNCT.exec(value)?.[0].length ?? 0;
+        const trail = TRAILING_PUNCT.exec(value.slice(lead))?.[0].length ?? 0;
+        if (lead === 0 && trail === 0) {
+            out.push(entity);
+            continue;
+        }
+        const trimmed = value.slice(lead, value.length - trail);
+        if (trimmed.length === 0)
+            continue;
+        out.push({ ...entity, value: trimmed, start: entity.start + lead, end: entity.end - trail });
+    }
+    return out;
+}
+// ── filterFalsePositives ───────────────────────────────────
+/**
+ * Scripts whose names are routinely one or two characters long (and whose
+ * words are not space-delimited). A value containing any of these keeps the
+ * lower two-character threshold for every entity type.
+ */
+const CJK_THAI = /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Hangul}\p{Script=Thai}]/u;
+/**
+ * Two-character PERSON values that are titles, suffixes or abbreviations,
+ * never names. Compared against the lowercased trimmed value (T181, R2).
+ */
+export const PERSON_SHORT_STOPLIST = new Set([
+    'mr', 'ms', 'dr', 'jr', 'sr', 'st', 'ii', 'iv', 'vi', 'no', 're', 'cc',
+]);
+function codePointLength(value) {
+    let n = 0;
+    for (const _ of value)
+        n++;
+    return n;
+}
+/**
+ * Drop ML spans that are too short to be real entities (R2):
+ *  - values containing CJK or Thai characters: at least 2 code points, any type
+ *  - PERSON: at least 2 characters, unless the value is a known 2-letter
+ *    title/suffix (PERSON_SHORT_STOPLIST); initials like "J." pass
+ *  - every other type: at least 3 characters (unchanged)
+ */
 export function filterFalsePositives(entities) {
     return entities.filter((e) => {
-        if (e.value.trim().length < 3)
+        const value = e.value.trim();
+        if (value.length === 0)
             return false;
-        return true;
+        if (CJK_THAI.test(value))
+            return codePointLength(value) >= 2;
+        if (e.type === 'PERSON') {
+            if (value.length < 2)
+                return false;
+            return !PERSON_SHORT_STOPLIST.has(value.toLowerCase());
+        }
+        return value.length >= 3;
     });
 }
+// ── propagateEntities ──────────────────────────────────────
+/**
+ * Words that are also given names or surnames, per language (T181, R15).
+ * A PERSON term found in the text with DIFFERENT casing than the detected
+ * entity, and NOT starting with an uppercase letter, is skipped when its
+ * lowercase form is listed here: "Bill" the person never turns "the bill is
+ * due" into "[PERSON_1] is due". Capitalised and ALL-CAPS mentions always
+ * propagate, and lowercase mentions of ordinary names ("thanks john") are
+ * still caught (golden prop-03-case-insensitive).
+ *
+ * Only words that appear in lowercase in running prose belong here. German
+ * nouns are always capitalised in prose, so the DE list holds adjectives and
+ * month names that double as surnames or first names; the PL list holds
+ * nouns (animals, months, plants) that are frequent Polish surnames and
+ * first names.
+ */
+export const COMMON_WORD_NAMES = {
+    en: [
+        'bill', 'mark', 'will', 'may', 'art', 'grace', 'hope', 'rose', 'jack', 'sue',
+        'pat', 'chase', 'dawn', 'june', 'carol', 'guy', 'rich', 'frank', 'ray', 'don',
+        'bob', 'jay', 'gene',
+    ],
+    pl: [
+        'lis', 'wilk', 'wrona', 'sowa', 'kruk', 'baran', 'kot', 'mucha', 'kowal', 'sroka',
+        'wróbel', 'dudek', 'mróz', 'zając', 'sobota', 'piątek', 'maj', 'marzec', 'kwiecień',
+        'lipiec', 'róża', 'jagoda', 'malina', 'kalina', 'cebula',
+    ],
+    de: [
+        'lang', 'klein', 'groß', 'gross', 'weiß', 'weiss', 'schwarz', 'braun', 'roth', 'kurz',
+        'jung', 'alt', 'neu', 'ernst', 'reich', 'stark', 'wild', 'frei', 'gut', 'voll',
+        'kühn', 'fromm', 'hell', 'klug', 'treu', 'lieb', 'mai',
+    ],
+};
+const COMMON_WORD_NAME_SET = new Set(Object.values(COMMON_WORD_NAMES).flat());
+/** A character that continues a word: a Unicode letter or digit. */
+const WORD_CHAR = /[\p{L}\p{N}]/u;
+/**
+ * Scripts written without spaces between words. A term edge in one of these
+ * scripts cannot demand a non-letter neighbour (a Chinese name is normally
+ * glued to the next character), mirroring the restore-side rule that CJK
+ * keys carry no boundaries.
+ */
+const UNSPACED_SCRIPT = /^[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Hangul}\p{Script=Thai}]$/u;
+function escapeRegExp(value) {
+    return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+}
+function codePointBefore(text, index) {
+    if (index <= 0)
+        return '';
+    const pair = Array.from(text.slice(Math.max(0, index - 2), index));
+    return pair[pair.length - 1] ?? '';
+}
+function codePointAt(text, index) {
+    if (index >= text.length)
+        return '';
+    const cp = text.codePointAt(index);
+    return cp === undefined ? '' : String.fromCodePoint(cp);
+}
+function startsUppercase(value) {
+    const first = codePointAt(value, 0);
+    return first !== '' && first !== first.toLowerCase();
+}
+/**
+ * Find every further occurrence of each detected value (and of the words of
+ * multi-word values) in the text and emit it as a 'propagated' entity.
+ *
+ * Terms: whole values of at least 2 characters; words of multi-word values
+ * of at least 4 characters (trailing punctuation stripped). Terms of at most
+ * 2 characters match with exact case; longer terms match case-insensitively
+ * (Unicode case folding via the 'iu' flags) on the ORIGINAL text, so offsets
+ * are exact even after characters whose lowercase form has a different
+ * length (Turkish dotted İ). A hit needs a non-letter, non-digit neighbour
+ * on both sides ([\p{L}\p{N}] complement), except on a side where the term
+ * itself ends in an unspaced script (CJK, Thai).
+ */
 export function propagateEntities(text, entities) {
     const propagated = [];
     const seen = new Set();
@@ -71,7 +263,7 @@ export function propagateEntities(text, entities) {
     const addedTerms = new Set();
     for (const e of entities) {
         const val = e.value.trim();
-        if (val.length >= 3 && !addedTerms.has(val.toLowerCase())) {
+        if (codePointLength(val) >= 2 && !addedTerms.has(val.toLowerCase())) {
             addedTerms.add(val.toLowerCase());
             terms.push({ term: val, type: e.type });
         }
@@ -86,41 +278,48 @@ export function propagateEntities(text, entities) {
         }
     }
     for (const { term, type } of terms) {
-        let searchFrom = 0;
-        const lowerText = text.toLowerCase();
-        const lowerTerm = term.toLowerCase();
-        while (searchFrom < text.length) {
-            const idx = lowerText.indexOf(lowerTerm, searchFrom);
-            if (idx === -1)
-                break;
-            const end = idx + term.length;
+        const exactCase = codePointLength(term) <= 2;
+        const pattern = new RegExp(escapeRegExp(term), exactCase ? 'gu' : 'giu');
+        const needsBoundaryBefore = !UNSPACED_SCRIPT.test(codePointAt(term, 0));
+        const needsBoundaryAfter = !UNSPACED_SCRIPT.test(codePointBefore(term, term.length));
+        let match;
+        while ((match = pattern.exec(text)) !== null) {
+            const idx = match.index;
+            const found = match[0];
+            const end = idx + found.length;
+            // Same overlap semantics as the original indexOf loop (searchFrom = idx + 1).
+            pattern.lastIndex = idx + 1;
             const key = `${idx}:${end}`;
-            if (!seen.has(key)) {
-                const charBefore = idx > 0 ? text[idx - 1] : ' ';
-                const charAfter = end < text.length ? text[end] : ' ';
-                const isBoundaryBefore = /[\s.,;:!?()"'„"\-–—/]/.test(charBefore);
-                const isBoundaryAfter = /[\s.,;:!?()"'„"\-–—/]/.test(charAfter);
-                if (isBoundaryBefore && isBoundaryAfter) {
-                    seen.add(key);
-                    propagated.push({
-                        type,
-                        value: text.slice(idx, end),
-                        start: idx,
-                        end,
-                        confidence: 0.95,
-                        detector: 'propagated',
-                    });
-                }
+            if (seen.has(key))
+                continue;
+            if (needsBoundaryBefore && WORD_CHAR.test(codePointBefore(text, idx)))
+                continue;
+            if (needsBoundaryAfter && WORD_CHAR.test(codePointAt(text, end)))
+                continue;
+            if (type === 'PERSON'
+                && found !== term
+                && !startsUppercase(found)
+                && COMMON_WORD_NAME_SET.has(found.toLowerCase())) {
+                continue;
             }
-            searchFrom = idx + 1;
+            seen.add(key);
+            propagated.push({
+                type,
+                value: found,
+                start: idx,
+                end,
+                confidence: 0.95,
+                detector: 'propagated',
+            });
         }
     }
     return propagated;
 }
+// ── detectEntities ─────────────────────────────────────────
 export function detectEntities(text, mlEntities, regexEntities) {
     if (!text.trim())
         return [];
-    const mlResults = filterFalsePositives(mlEntities);
+    const mlResults = filterFalsePositives(trimSpanPunctuation(mlEntities));
     const regexResults = regexEntities;
     const initial = resolveOverlaps([...mlResults, ...regexResults]);
     const propagated = propagateEntities(text, initial);

package/dist/providers/bardsai.d.ts

@@ -12,11 +12,18 @@
  */
 import type { DetectedEntity, DetectionProvider, ProgressCallback } from '../types.js';
 import type { CoreEnv } from '../env.js';
-import type { ModelVerification } from '../model-loader.js';
+import type { ModelVerification, TokenizerFiles } from '../model-loader.js';
 export declare const BARDSAI_MODEL_REVISION = "0e72e19f030ed4e661b1673e549af8e0dd176386";
 /** SHA-256 of onnx/model_quantized.onnx (278,736,360 bytes) at BARDSAI_MODEL_REVISION. */
 export declare const BARDSAI_MODEL_SHA256 = "8c9f555c743ed14eb7e505ff9d9c7785775a1671fd14031af33f424de8bd0e7b";
 export declare const BARDSAI_MODEL_URL = "https://huggingface.co/bardsai/eu-pii-anonimization-multilang/resolve/0e72e19f030ed4e661b1673e549af8e0dd176386/onnx/model_quantized.onnx";
+/**
+ * Tokenizer files at the same pinned commit (T185): fetched through
+ * fetchModelBlob with SHA-256 + size, cached next to the model and handed to
+ * CoreEnv.buildTokenizer. Hashes re-measured against the resolve/<commit>
+ * URLs on 2026-09-24 (documentation/model-provenance.md).
+ */
+export declare const BARDSAI_TOKENIZER_FILES: TokenizerFiles;
 export declare class BardsaiProvider implements DetectionProvider {
     private _name;
     get name(): string;

package/dist/providers/bardsai.js

@@ -14,7 +14,7 @@
 // wasm fallback, asyncify binary). The bare entry resolves the deprecated JSEP
 // bundle whose wasm exceeds Cloudflare Pages' 25 MiB per-file limit.
 import * as ort from 'onnxruntime-web/webgpu';
-import { evictModelFromCache, fetchModelBlob, retryAsync } from "../model-loader.js";
+import { evictModelFromCache, fetchModelBlob, loadPinnedTokenizer } from "../model-loader.js";
 // ── Model config ──────────────────────────────────────────
 // Supply-chain pinning (T116): the model is fetched from an immutable
 // commit revision, never from mutable resolve/main, and the downloaded
@@ -31,6 +31,26 @@ export const BARDSAI_MODEL_URL = `https://huggingface.co/bardsai/eu-pii-anonimiz
 const MODEL_URL = BARDSAI_MODEL_URL;
 /** Pre-pinning download URL; its cache entry is evicted best-effort on load. */
 const LEGACY_MODEL_URL = 'https://huggingface.co/bardsai/eu-pii-anonimization-multilang/resolve/main/onnx/model_quantized.onnx';
+/**
+ * Tokenizer files at the same pinned commit (T185): fetched through
+ * fetchModelBlob with SHA-256 + size, cached next to the model and handed to
+ * CoreEnv.buildTokenizer. Hashes re-measured against the resolve/<commit>
+ * URLs on 2026-09-24 (documentation/model-provenance.md).
+ */
+export const BARDSAI_TOKENIZER_FILES = [
+    {
+        url: `https://huggingface.co/bardsai/eu-pii-anonimization-multilang/resolve/${BARDSAI_MODEL_REVISION}/tokenizer.json`,
+        sha256: '2464f9721707cb3d5edcf9a3d73454b13e8a7b3bb8fdba94b3de3d843f30e946',
+        size: 16781584,
+    },
+    {
+        url: `https://huggingface.co/bardsai/eu-pii-anonimization-multilang/resolve/${BARDSAI_MODEL_REVISION}/tokenizer_config.json`,
+        sha256: 'c019e3e4f7f901adf680dde303c7d964de86675a13e089bcc0f614d3fce75333',
+        size: 314,
+    },
+];
+const TOKENIZER_FILES = BARDSAI_TOKENIZER_FILES;
+/** Hugging Face id for the deprecated CoreEnv.loadTokenizer fallback only. */
 const TOKENIZER_HF = 'bardsai/eu-pii-anonimization-multilang';
 const MODEL_NAME = 'BardS.ai EU PII';
 const DEFAULT_THRESHOLD = 0.5;
@@ -181,7 +201,7 @@ export class BardsaiProvider {
                 }),
             ];
             if (!this.tokenizer) {
-                tasks.push(retryAsync(() => this.env.loadTokenizer(TOKENIZER_HF), 'Tokenizer download').then((t) => {
+                tasks.push(loadPinnedTokenizer(this.env, TOKENIZER_FILES, TOKENIZER_HF).then((t) => {
                     this.tokenizer = t;
                 }));
             }

package/dist/providers/gliner-base.d.ts

@@ -17,12 +17,19 @@
  */
 import type { DetectedEntity, DetectionProvider, ProgressCallback } from '../types.js';
 import type { CoreEnv } from '../env.js';
-import type { ModelVerification } from '../model-loader.js';
+import type { ModelVerification, TokenizerFiles } from '../model-loader.js';
 import { type RawSpan } from './gliner.js';
 export declare const GLINER_BASE_MODEL_REVISION = "61726e0ad791dcab3e29339bbec3ad42ded65641";
 /** SHA-256 of onnx/model_quint8.onnx (196,757,174 bytes) at GLINER_BASE_MODEL_REVISION. */
 export declare const GLINER_BASE_MODEL_SHA256 = "0514c8fd86d0513ce5351a3267f132b57d5bcd8f99a90d43cde1228092881d19";
 export declare const GLINER_BASE_MODEL_URL = "https://huggingface.co/knowledgator/gliner-pii-base-v1.0/resolve/61726e0ad791dcab3e29339bbec3ad42ded65641/onnx/model_quint8.onnx";
+/**
+ * Tokenizer files at the same pinned commit (T185): fetched through
+ * fetchModelBlob with SHA-256 + size, cached next to the model and handed to
+ * CoreEnv.buildTokenizer. Hashes re-measured against the resolve/<commit>
+ * URLs on 2026-09-24 (documentation/model-provenance.md).
+ */
+export declare const GLINER_BASE_TOKENIZER_FILES: TokenizerFiles;
 /** max_width from the repo's gliner_config.json: spans cover 1..12 words. */
 export declare const GLINER_BASE_MAX_WIDTH = 12;
 export interface SpanIndices {

package/dist/providers/gliner-base.js

@@ -19,7 +19,7 @@
 // wasm fallback, asyncify binary). The bare entry resolves the deprecated JSEP
 // bundle whose wasm exceeds Cloudflare Pages' 25 MiB per-file limit.
 import * as ort from 'onnxruntime-web/webgpu';
-import { fetchModelBlob, retryAsync } from "../model-loader.js";
+import { fetchModelBlob, loadPinnedTokenizer } from "../model-loader.js";
 import { DEFAULT_PII_LABELS, greedySelect, mapLabelToEntityType, sigmoid, splitWords, } from "./gliner.js";
 // ── Model config ──────────────────────────────────────────
 // Supply-chain pinning (T116): the model is fetched from an immutable
@@ -35,6 +35,26 @@ export const GLINER_BASE_MODEL_REVISION = '61726e0ad791dcab3e29339bbec3ad42ded65
 export const GLINER_BASE_MODEL_SHA256 = '0514c8fd86d0513ce5351a3267f132b57d5bcd8f99a90d43cde1228092881d19';
 export const GLINER_BASE_MODEL_URL = `https://huggingface.co/knowledgator/gliner-pii-base-v1.0/resolve/${GLINER_BASE_MODEL_REVISION}/onnx/model_quint8.onnx`;
 const DEFAULT_MODEL_URL = GLINER_BASE_MODEL_URL;
+/**
+ * Tokenizer files at the same pinned commit (T185): fetched through
+ * fetchModelBlob with SHA-256 + size, cached next to the model and handed to
+ * CoreEnv.buildTokenizer. Hashes re-measured against the resolve/<commit>
+ * URLs on 2026-09-24 (documentation/model-provenance.md).
+ */
+export const GLINER_BASE_TOKENIZER_FILES = [
+    {
+        url: `https://huggingface.co/knowledgator/gliner-pii-base-v1.0/resolve/${GLINER_BASE_MODEL_REVISION}/tokenizer.json`,
+        sha256: 'ee028763434d18611c1c36356ea1d050e90a9fa94ede57fac48b39f85f818ad1',
+        size: 8649232,
+    },
+    {
+        url: `https://huggingface.co/knowledgator/gliner-pii-base-v1.0/resolve/${GLINER_BASE_MODEL_REVISION}/tokenizer_config.json`,
+        sha256: '3ec8a90d8758fbc56d50831990c3a3a65660f020c5b06534adf43b04091ffa9e',
+        size: 1691,
+    },
+];
+const TOKENIZER_FILES = GLINER_BASE_TOKENIZER_FILES;
+/** Hugging Face id for the deprecated CoreEnv.loadTokenizer fallback only. */
 const DEFAULT_TOKENIZER_HF = 'knowledgator/gliner-pii-base-v1.0';
 const DEFAULT_MODEL_NAME = 'GLiNER PII Base';
 const CUSTOM_LABELS_STORAGE_KEY = 'doccloak-custom-labels';
@@ -208,7 +228,7 @@ export class GlinerBaseProvider {
                 }),
             ];
             if (!this.tokenizer) {
-                tasks.push(retryAsync(() => this.env.loadTokenizer(DEFAULT_TOKENIZER_HF), 'Tokenizer download').then((t) => {
+                tasks.push(loadPinnedTokenizer(this.env, TOKENIZER_FILES, DEFAULT_TOKENIZER_HF).then((t) => {
                     this.tokenizer = t;
                 }));
             }

package/dist/providers/gliner.d.ts

@@ -8,11 +8,18 @@
  */
 import type { DetectedEntity, EntityType, DetectionProvider, ProgressCallback } from '../types.js';
 import type { CoreEnv } from '../env.js';
-import type { ModelVerification } from '../model-loader.js';
+import type { ModelVerification, TokenizerFiles } from '../model-loader.js';
 export declare const GLINER_MODEL_REVISION = "d21aad5b4a7ec82b3d0970fd1ac74a12c087d85e";
 /** SHA-256 of onnx/model_quint8.onnx (82,680,500 bytes) at GLINER_MODEL_REVISION. */
 export declare const GLINER_MODEL_SHA256 = "891589426ee96f2748b16439f44fad8c3f97e198e002a6637e58dee989500216";
 export declare const GLINER_MODEL_URL = "https://huggingface.co/knowledgator/gliner-pii-small-v1.0/resolve/d21aad5b4a7ec82b3d0970fd1ac74a12c087d85e/onnx/model_quint8.onnx";
+/**
+ * Tokenizer files at the same pinned commit (T185): fetched through
+ * fetchModelBlob with SHA-256 + size, cached next to the model and handed to
+ * CoreEnv.buildTokenizer. Hashes re-measured against the resolve/<commit>
+ * URLs on 2026-09-24 (documentation/model-provenance.md).
+ */
+export declare const GLINER_TOKENIZER_FILES: TokenizerFiles;
 /**
  * Built-in zero-shot label set, shared with the GLiNER PII Base provider
  * (gliner-base.ts) - the models are from the same zero-shot family, so the

package/dist/providers/gliner.js

@@ -10,7 +10,7 @@
 // wasm fallback, asyncify binary). The bare entry resolves the deprecated JSEP
 // bundle whose wasm exceeds Cloudflare Pages' 25 MiB per-file limit.
 import * as ort from 'onnxruntime-web/webgpu';
-import { evictModelFromCache, fetchModelBlob, retryAsync } from "../model-loader.js";
+import { evictModelFromCache, fetchModelBlob, loadPinnedTokenizer } from "../model-loader.js";
 // ── Model config ──────────────────────────────────────────
 // Supply-chain pinning (T116): the model is fetched from an immutable
 // commit revision, never from mutable resolve/main, and the downloaded
@@ -26,6 +26,25 @@ export const GLINER_MODEL_REVISION = 'd21aad5b4a7ec82b3d0970fd1ac74a12c087d85e';
 export const GLINER_MODEL_SHA256 = '891589426ee96f2748b16439f44fad8c3f97e198e002a6637e58dee989500216';
 export const GLINER_MODEL_URL = `https://huggingface.co/knowledgator/gliner-pii-small-v1.0/resolve/${GLINER_MODEL_REVISION}/onnx/model_quint8.onnx`;
 const DEFAULT_MODEL_URL = GLINER_MODEL_URL;
+/**
+ * Tokenizer files at the same pinned commit (T185): fetched through
+ * fetchModelBlob with SHA-256 + size, cached next to the model and handed to
+ * CoreEnv.buildTokenizer. Hashes re-measured against the resolve/<commit>
+ * URLs on 2026-09-24 (documentation/model-provenance.md).
+ */
+export const GLINER_TOKENIZER_FILES = [
+    {
+        url: `https://huggingface.co/knowledgator/gliner-pii-small-v1.0/resolve/${GLINER_MODEL_REVISION}/tokenizer.json`,
+        sha256: '84b3a9b18f04a0ccd03b72d9f871b7e0bec40fd7021ef50bc30a7c3693c11205',
+        size: 3583593,
+    },
+    {
+        url: `https://huggingface.co/knowledgator/gliner-pii-small-v1.0/resolve/${GLINER_MODEL_REVISION}/tokenizer_config.json`,
+        sha256: '3398f6d1ad4b4c4f9874d390d060a75c58cad5e5ce9b22841b3e40643b4ada27',
+        size: 21214,
+    },
+];
+const TOKENIZER_FILES = GLINER_TOKENIZER_FILES;
 /**
  * Superseded download URLs; their cache entries are evicted best-effort on
  * load so stale ~46 MB copies do not linger against the origin quota:
@@ -36,6 +55,7 @@ const LEGACY_MODEL_URLS = [
     'https://huggingface.co/knowledgator/gliner-pii-edge-v1.0/resolve/main/onnx/model_quint8.onnx',
     'https://huggingface.co/knowledgator/gliner-pii-edge-v1.0/resolve/9b7f39b0a2da971a5beea78d35f1539d4009c891/onnx/model_quint8.onnx',
 ];
+/** Hugging Face id for the deprecated CoreEnv.loadTokenizer fallback only. */
 const DEFAULT_TOKENIZER_HF = 'knowledgator/gliner-pii-small-v1.0';
 const DEFAULT_MODEL_NAME = 'GLiNER PII Small';
 const CUSTOM_LABELS_STORAGE_KEY = 'doccloak-custom-labels';
@@ -236,7 +256,7 @@ export class GlinerProvider {
                 }),
             ];
             if (!this.tokenizer) {
-                tasks.push(retryAsync(() => this.env.loadTokenizer(DEFAULT_TOKENIZER_HF), 'Tokenizer download').then((t) => {
+                tasks.push(loadPinnedTokenizer(this.env, TOKENIZER_FILES, DEFAULT_TOKENIZER_HF).then((t) => {
                     this.tokenizer = t;
                 }));
             }

package/dist/regex/loader.d.ts

@@ -31,6 +31,8 @@ export interface RegexRuleJson {
     falsePositiveNotes?: string;
     /** Name of a post-match validation function in the validator registry */
     validate?: string;
+    /** Run against the whole document instead of line by line (default false) */
+    multiline?: boolean;
 }
 /** JSON shape of a whole rules/<region>.json file. */
 export interface RegionRulesJson {

package/dist/regex/loader.js

@@ -20,6 +20,8 @@ function compileRule(region, json) {
         rule.examples = json.examples;
     if (json.falsePositiveNotes !== undefined)
         rule.falsePositiveNotes = json.falsePositiveNotes;
+    if (json.multiline !== undefined)
+        rule.multiline = json.multiline;
     if (json.validate !== undefined) {
         const validate = VALIDATORS[json.validate];
         if (validate === undefined) {