@doccloak/core 0.11.0 → 0.12.1

package/dist/dom/xlsx.js

@@ -15,18 +15,28 @@
  * - worksheet header/footer definitions
  * - legacy cell comments (xl/comments*.xml) and threaded comments
  *
- * Deliberately out of scope in phase 1 (documented for T-later): drawing/chart
- * text parts (xl/drawings, xl/charts), data connection definitions
- * (xl/connections.xml) and defined-name formulas. Metadata scrubbing follows
- * the docx module's policy via the shared OPC helpers.
+ * - drawing text boxes and chart titles / string caches (xl/drawings,
+ *   xl/charts) through the DrawingML units (T175)
+ *
+ * Deliberately out of scope (T176): sheet names, defined-name formulas, table
+ * column names, pivot caches; layer zero scrubs known values there. Data
+ * connections and external links are reported as unredactable (T177).
+ * Metadata scrubbing follows the docx module's policy via the shared OPC
+ * helpers.
  */
 import JSZip from 'jszip';
-import { applyTextReplacements } from "./docx.js";
-import { readXmlPart, writeXmlPart, scrubDocPropsParts, removePackageThumbnail, scrubExternalRelTargets, replaceSensitiveValues, } from "./opc.js";
+import { applyTextReplacements, assertUnredactableAllowed, unredactableWarnings } from "./docx.js";
+import { drawingUnits } from "./drawingml.js";
+import { XLSX_DRAWING_TEXT_PARTS, MAX_UNPACKED_BYTES, assertUnpackedSize, collectUnredactableParts, } from "./package-policy.js";
+import { OOXML_NS, isElementIn, elementsByLocalName, foreignNamespaces, readXmlPart, writeXmlPart, scrubDocPropsParts, normalizeZipEntries, removePackageThumbnail, scrubExternalRelTargets, replaceSensitiveValues, scrubPackageValues, } from "./opc.js";
 import { getFileExtension } from "./docx.js";
-const SS_NS = 'http://schemas.openxmlformats.org/spreadsheetml/2006/main';
-function isSpreadsheetEl(el, localName) {
-    return el.namespaceURI === SS_NS && el.localName === localName;
+import { UnsupportedDocumentError } from "./errors.js";
+/** True for a SpreadsheetML element (transitional or strict namespace, T172). */
+export function isSpreadsheetEl(el, localName) {
+    return isElementIn(el, OOXML_NS.ss, localName);
+}
+function isSpreadsheetNs(el) {
+    return el.namespaceURI !== null && OOXML_NS.ss.has(el.namespaceURI);
 }
 /** Collect descendant <t> elements (spreadsheet namespace) in document order. */
 function collectTextLeaves(root) {
@@ -41,12 +51,51 @@ function collectTextLeaves(root) {
 /** String units in xl/sharedStrings.xml: one per <si>. */
 function sharedStringUnits(xmlDoc) {
     const units = [];
-    const sis = xmlDoc.getElementsByTagNameNS(SS_NS, 'si');
-    for (let i = 0; i < sis.length; i++) {
-        units.push(collectTextLeaves(sis[i]));
+    for (const si of elementsByLocalName(xmlDoc, OOXML_NS.ss, 'si')) {
+        units.push(collectTextLeaves(si));
     }
     return units;
 }
+/** A cell number that can be an identifier: an integer of seven or more digits. */
+const NUMERIC_IDENTIFIER = /^[+-]?\d{7,}$/;
+/** What SpreadsheetML accepts as a numeric <v>. */
+const NUMERIC_VALUE = /^[+-]?(\d+\.?\d*|\.\d+)([eE][+-]?\d+)?$/;
+/**
+ * After replacement, a numeric cell whose <v> no longer holds a number
+ * (it now carries a placeholder) would make Excel repair the workbook. Turn
+ * it into an inline string cell and drop its formula, which would otherwise
+ * recompute the original number (T176).
+ */
+function normalizeReplacedNumericCells(xmlDoc) {
+    for (const cell of elementsByLocalName(xmlDoc, OOXML_NS.ss, 'c')) {
+        const type = cell.getAttribute('t');
+        if (type !== null && type !== 'n')
+            continue;
+        let v = null;
+        let f = null;
+        for (let c = 0; c < cell.children.length; c++) {
+            const child = cell.children[c];
+            if (isSpreadsheetEl(child, 'v'))
+                v = child;
+            else if (isSpreadsheetEl(child, 'f'))
+                f = child;
+        }
+        if (!v)
+            continue;
+        const text = v.textContent ?? '';
+        if (text.length === 0 || NUMERIC_VALUE.test(text))
+            continue;
+        cell.setAttribute('t', 'inlineStr');
+        if (f)
+            cell.removeChild(f);
+        const is = xmlDoc.createElementNS(cell.namespaceURI, 'is');
+        const t = xmlDoc.createElementNS(cell.namespaceURI, 't');
+        t.setAttribute('xml:space', 'preserve');
+        t.textContent = text;
+        is.appendChild(t);
+        cell.replaceChild(is, v);
+    }
+}
 const HEADER_FOOTER_NAMES = new Set([
     'oddHeader', 'oddFooter', 'evenHeader', 'evenFooter', 'firstHeader', 'firstFooter',
 ]);
@@ -59,7 +108,7 @@ function worksheetUnits(xmlDoc) {
     const all = xmlDoc.getElementsByTagName('*');
     for (let i = 0; i < all.length; i++) {
         const el = all[i];
-        if (el.namespaceURI !== SS_NS)
+        if (!isSpreadsheetNs(el))
             continue;
         if (el.localName === 'c') {
             const type = el.getAttribute('t');
@@ -76,6 +125,19 @@ function worksheetUnits(xmlDoc) {
                         units.push([child]);
                 }
             }
+            else if (type === null || type === 'n') {
+                // Numeric cells (T176): a PESEL, phone or account number typed into
+                // a cell is stored as a number. Values of at least seven digits are
+                // extracted so the detectors see them; a replaced cell is turned
+                // into an inline string on write (normalizeReplacedNumericCells).
+                // Shorter numbers (amounts, counts, dates as serials) are not
+                // identifiers and stay out of the text.
+                for (let c = 0; c < el.children.length; c++) {
+                    const child = el.children[c];
+                    if (isSpreadsheetEl(child, 'v') && NUMERIC_IDENTIFIER.test(child.textContent ?? ''))
+                        units.push([child]);
+                }
+            }
         }
         else if (HEADER_FOOTER_NAMES.has(el.localName)) {
             units.push([el]);
@@ -92,9 +154,8 @@ function commentUnits(xmlDoc) {
     const units = [];
     const root = xmlDoc.documentElement;
     if (root.localName === 'comments') {
-        const texts = xmlDoc.getElementsByTagNameNS(SS_NS, 'text');
-        for (let i = 0; i < texts.length; i++) {
-            const leaves = collectTextLeaves(texts[i]);
+        for (const text of elementsByLocalName(xmlDoc, OOXML_NS.ss, 'text')) {
+            const leaves = collectTextLeaves(text);
             if (leaves.length > 0)
                 units.push(leaves);
         }
@@ -115,16 +176,19 @@ function commentUnits(xmlDoc) {
 /**
  * Read a .xlsx file and extract its text content with position mapping.
  */
-export async function readXlsx(file) {
+export async function readXlsx(file, options = {}) {
     const arrayBuffer = await file.arrayBuffer();
     const zip = await JSZip.loadAsync(arrayBuffer);
+    assertUnpackedSize(zip, options.maxUnpackedBytes ?? MAX_UNPACKED_BYTES);
     if (!zip.file('xl/workbook.xml')) {
         throw new Error('Invalid .xlsx file: missing xl/workbook.xml');
     }
     // Deterministic part order so load-time and export-time extractions produce
-    // identical offsets: sharedStrings, then worksheets, then comment parts.
+    // identical offsets: sharedStrings, then worksheets, then comment parts,
+    // then drawing text boxes and charts (T175: a:p paragraphs, string caches).
     const worksheetPaths = [];
     const commentPaths = [];
+    const drawingPaths = [];
     zip.forEach((relativePath) => {
         if (/^xl\/worksheets\/[^/]+\.xml$/.test(relativePath)) {
             worksheetPaths.push(relativePath);
@@ -133,20 +197,37 @@ export async function readXlsx(file) {
             /^xl\/threadedComments\/[^/]+\.xml$/.test(relativePath)) {
             commentPaths.push(relativePath);
         }
+        else if (XLSX_DRAWING_TEXT_PARTS.test(relativePath)) {
+            drawingPaths.push(relativePath);
+        }
     });
     worksheetPaths.sort();
     commentPaths.sort();
+    drawingPaths.sort();
     const partPlans = [
         { path: 'xl/sharedStrings.xml', unitsOf: sharedStringUnits },
         ...worksheetPaths.map((path) => ({ path, unitsOf: worksheetUnits })),
         ...commentPaths.map((path) => ({ path, unitsOf: commentUnits })),
+        ...drawingPaths.map((path) => ({ path, unitsOf: drawingUnits })),
     ];
     let flatText = '';
     const contentParts = [];
+    // Fail closed (H1): worksheets whose sheetData is not SpreadsheetML in a
+    // namespace we read (transitional or strict) while foreign elements exist
+    // are refused rather than copied through.
+    let sheetDataFound = false;
+    const foreign = new Set();
     for (const plan of partPlans) {
         const xmlDoc = await readXmlPart(zip, plan.path);
         if (!xmlDoc)
             continue;
+        if (plan.unitsOf === worksheetUnits) {
+            if (elementsByLocalName(xmlDoc, OOXML_NS.ss, 'sheetData').length > 0)
+                sheetDataFound = true;
+            else
+                for (const ns of foreignNamespaces(xmlDoc))
+                    foreign.add(ns);
+        }
         const textNodes = [];
         for (const unit of plan.unitsOf(xmlDoc)) {
             const nodes = [];
@@ -172,11 +253,20 @@ export async function readXlsx(file) {
         }
         contentParts.push({ path: plan.path, xmlDoc, textNodes });
     }
+    if (worksheetPaths.length > 0 && !sheetDataFound && foreign.size > 0) {
+        throw new UnsupportedDocumentError('unrecognized-namespace', `Unsupported .xlsx: no SpreadsheetML sheetData in any worksheet, found elements in ${[...foreign].join(', ')}`, [...foreign]);
+    }
+    // Package policy (T177): parts the writer would copy verbatim are
+    // reported; the writer refuses them until the host passes allowUnredactable.
+    const unredactable = await collectUnredactableParts(zip, 'xlsx');
     return {
         plainText: flatText,
         zip,
         contentParts,
         textNodes: contentParts.flatMap((part) => part.textNodes),
+        empty: flatText.trim().length === 0,
+        unredactable,
+        warnings: [],
     };
 }
 /**
@@ -259,17 +349,37 @@ async function sanitizeXlsxMetadata(zip, valueReplacements) {
  * Apply text replacements to the xlsx XML, preserving formatting, then scrub
  * metadata. Returns a new .xlsx file as a Blob; the input file is untouched.
  */
-export async function writeAnonymizedXlsx(extraction, replacements, valueReplacements = []) {
+export async function writeAnonymizedXlsx(extraction, replacements, valueReplacements = [], options = {}) {
+    const result = await writeAnonymizedXlsxWithReport(extraction, replacements, valueReplacements, options);
+    return result.blob;
+}
+/**
+ * Same as writeAnonymizedXlsx, with the write report. After the content
+ * parts and metadata, layer zero (T172) scrubs every known value from every
+ * other XML/rels/vml part: pivot cache records, drawings, charts, tables,
+ * defined names in the workbook, docProps.
+ */
+export async function writeAnonymizedXlsxWithReport(extraction, replacements, valueReplacements = [], options = {}) {
+    assertUnredactableAllowed(extraction.unredactable, options.allowUnredactable);
     applyTextReplacements(extraction.textNodes, replacements);
+    const contentPaths = new Set();
     for (const part of extraction.contentParts) {
+        if (/^xl\/worksheets\//.test(part.path))
+            normalizeReplacedNumericCells(part.xmlDoc);
         sanitizeXlsxPart(part.xmlDoc, valueReplacements);
         writeXmlPart(extraction.zip, part.path, part.xmlDoc);
+        contentPaths.add(part.path);
     }
     await sanitizeXlsxMetadata(extraction.zip, valueReplacements);
-    return extraction.zip.generateAsync({
+    const scrub = await scrubPackageValues(extraction.zip, valueReplacements, { skip: contentPaths });
+    // Container normalisation (T179): fixed entry dates, no comments.
+    normalizeZipEntries(extraction.zip);
+    const blob = await extraction.zip.generateAsync({
         type: 'blob',
         mimeType: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
+        comment: '',
     });
+    return { blob, warnings: [...scrub.warnings, ...unredactableWarnings(extraction.unredactable)] };
 }
 /**
  * Check if a file is a supported Excel workbook (.xlsx only; the legacy

package/dist/engine.d.ts

@@ -28,6 +28,24 @@ export type ProviderFactory = (env: CoreEnv) => DetectionProvider;
 export interface EngineOptions {
     /** Override provider construction (tests inject deterministic stubs). */
     providers?: Partial<Record<ProviderId, ProviderFactory>>;
+    /**
+     * Whether detect() may download and initialise the model on its own
+     * (T185, security report S9). Default true (previous behaviour). With
+     * false the engine never touches the network implicitly: detect() before
+     * a completed preload() (or switchProvider()) rejects with
+     * ModelNotLoadedError, while a preload that is still in flight is awaited.
+     * Hosts that gate model downloads behind user consent should set this to
+     * false and call preload() from the consent flow.
+     */
+    autoLoad?: boolean;
+}
+/**
+ * detect() was called while the active provider's model is not loaded and
+ * createEngine was given `autoLoad: false`. Call engine.preload() first.
+ */
+export declare class ModelNotLoadedError extends Error {
+    providerId: ProviderId;
+    constructor(providerId: ProviderId);
 }
 /**
  * env.kv keys, byte-identical to the localStorage keys the web app used

package/dist/engine.js

@@ -36,6 +36,17 @@ export const PROVIDERS = [
         description: 'Best multilingual accuracy, 24+ EU languages, 35 PII types (~279 MB)',
     },
 ];
+/**
+ * detect() was called while the active provider's model is not loaded and
+ * createEngine was given `autoLoad: false`. Call engine.preload() first.
+ */
+export class ModelNotLoadedError extends Error {
+    constructor(providerId) {
+        super(`Model for provider "${providerId}" is not loaded; call engine.preload() before detect() (createEngine autoLoad is off)`);
+        this.name = 'ModelNotLoadedError';
+        this.providerId = providerId;
+    }
+}
 // ── Settings ───────────────────────────────────────────────
 /**
  * env.kv keys, byte-identical to the localStorage keys the web app used
@@ -95,10 +106,13 @@ export function createEngine(env, initial, options) {
         bardsai: (e) => new BardsaiProvider(e),
         ...options?.providers,
     };
+    const autoLoad = options?.autoLoad ?? true;
     const settings = {
         providerId: initial?.providerId ?? pickDefaultProvider(env.hardware),
         threshold: 0,
-        regexEnabled: initial?.regexEnabled ?? false,
+        // Regex rules are on unless the host or a saved setting turns them off
+        // (T203): structured identifiers must not depend on the model alone.
+        regexEnabled: initial?.regexEnabled ?? true,
         regexRegion: initial?.regexRegion ?? 'all',
         customLabels: [...(initial?.customLabels ?? [])],
     };
@@ -175,14 +189,16 @@ export function createEngine(env, initial, options) {
             await p.setCustomLabels(settings.customLabels);
         }
     }
-    async function preload() {
-        await ready;
+    function preload() {
+        // inflightLoad is assigned synchronously so a detect() issued right
+        // after preload() (without awaiting it) sees the load in flight.
         if (inflightLoad)
             return inflightLoad;
-        const p = ensureProvider();
-        if (p.isLoaded())
-            return;
-        inflightLoad = (async () => {
+        const run = (async () => {
+            await ready;
+            const p = ensureProvider();
+            if (p.isLoaded())
+                return;
             // A previous attempt may have failed: release() clears the cached load
             // error so load() can retry (model bytes are served from the blob
             // cache when available). Mirrors the old worker init path.
@@ -194,12 +210,11 @@ export function createEngine(env, initial, options) {
             if (supportsCustomLabels(p))
                 settings.customLabels = p.getCustomLabels();
         })();
-        try {
-            await inflightLoad;
-        }
-        finally {
-            inflightLoad = null;
-        }
+        inflightLoad = run;
+        return run.finally(() => {
+            if (inflightLoad === run)
+                inflightLoad = null;
+        });
     }
     async function switchProvider(id) {
         await ready;
@@ -254,6 +269,15 @@ export function createEngine(env, initial, options) {
         if (!text.trim())
             return [];
         const p = ensureProvider();
+        if (!autoLoad && !p.isLoaded()) {
+            // Explicit-preload mode: never download from detect(). A preload that
+            // is already running is awaited; otherwise the host must call it.
+            if (!inflightLoad)
+                throw new ModelNotLoadedError(settings.providerId);
+            await inflightLoad;
+            if (!p.isLoaded())
+                throw new ModelNotLoadedError(settings.providerId);
+        }
         const mlResults = await p.detect(text, (progress) => {
             onProgress?.(progress);
             for (const cb of detectionListeners)

package/dist/env.d.ts

@@ -31,7 +31,27 @@ export interface CoreEnv {
         paths: string;
         numThreads?: number;
     };
-    loadTokenizer(hfModelId: string): Promise<unknown>;
+    /**
+     * Build a tokenizer from the parsed contents of the provider's pinned
+     * `tokenizer.json` and `tokenizer_config.json` (T185). Core fetches both
+     * files itself through fetchModelBlob (immutable resolve/<commit> URL,
+     * SHA-256 + size check, blob cache, resume and retry) and only hands the
+     * verified, parsed objects here; the host never contacts the network for
+     * tokenizers. With @huggingface/transformers this is
+     * `new PreTrainedTokenizer(tokenizerJson, tokenizerConfig)` (or the class
+     * named by `tokenizerConfig.tokenizer_class`). May return a promise.
+     *
+     * Optional in 0.12.0 only so hosts still on loadTokenizer keep compiling;
+     * required from 0.13.0.
+     */
+    buildTokenizer?(tokenizerJson: unknown, tokenizerConfig: unknown): unknown | Promise<unknown>;
+    /**
+     * @deprecated since 0.12.0, removed in 0.13.0. Used only when
+     * buildTokenizer is absent. Loading by Hugging Face model id lets the
+     * host library probe mutable `resolve/main` and keep an unverified
+     * second cache (security report S1/S3); implement buildTokenizer.
+     */
+    loadTokenizer?(hfModelId: string): Promise<unknown>;
     hardware?: HardwareHints;
     persistStorage?: () => Promise<boolean>;
 }

package/dist/index.d.ts

@@ -4,21 +4,23 @@
 export type { KVStore, BlobCache, HardwareHints, CoreEnv, MemoryBlobCacheOptions, } from './env.js';
 export { memoryKV, memoryBlobCache } from './env.js';
 export declare const CORE_PACKAGE_NAME = "@doccloak/core";
-export { fetchModelBlob, evictModelFromCache, retryAsync, ModelIntegrityError, verificationMarkerKey } from './model-loader.js';
-export type { ModelLoaderEnv, DownloadProgress, FetchModelOptions, ModelVerification } from './model-loader.js';
+export { fetchModelBlob, evictModelFromCache, retryAsync, ModelIntegrityError, verificationMarkerKey, verificationMarkerValue, parseVerificationMarker, fetchPinnedJson, loadPinnedTokenizer, } from './model-loader.js';
+export type { ModelLoaderEnv, DownloadProgress, FetchModelOptions, ModelVerification, TokenizerFileSpec, TokenizerFiles, PinnedTokenizerOptions, } from './model-loader.js';
 export { ALL_REGEX_RULES, REGEX_REGIONS } from './regex/index.js';
 export type { RegexRule, PiiDomain, RegionCode, RegexRegionId } from './regex/index.js';
 export * from './types.js';
 export * from './session.js';
 export * from './doc.js';
+export { UnsupportedDocumentError, isUnsupportedDocumentError } from './dom/errors.js';
+export type { UnsupportedDocumentCode } from './dom/errors.js';
 export { generateSurrogate, generateUniqueSurrogate, generateSessionSalt, sessionDayOffset, FAKE_EMAIL_DOMAINS, } from './surrogates.js';
 export type { SurrogateContext } from './surrogates.js';
-export { GlinerProvider, GLINER_MODEL_URL, GLINER_MODEL_REVISION, GLINER_MODEL_SHA256 } from './providers/gliner.js';
-export { GlinerBaseProvider, GLINER_BASE_MODEL_URL, GLINER_BASE_MODEL_REVISION, GLINER_BASE_MODEL_SHA256, GLINER_BASE_MAX_WIDTH, buildSpanIndices, decodeSpanLogits, } from './providers/gliner-base.js';
+export { GlinerProvider, GLINER_MODEL_URL, GLINER_MODEL_REVISION, GLINER_MODEL_SHA256, GLINER_TOKENIZER_FILES, } from './providers/gliner.js';
+export { GlinerBaseProvider, GLINER_BASE_MODEL_URL, GLINER_BASE_MODEL_REVISION, GLINER_BASE_MODEL_SHA256, GLINER_BASE_TOKENIZER_FILES, GLINER_BASE_MAX_WIDTH, buildSpanIndices, decodeSpanLogits, } from './providers/gliner-base.js';
 export type { SpanIndices } from './providers/gliner-base.js';
-export { BardsaiProvider, BARDSAI_MODEL_URL, BARDSAI_MODEL_REVISION, BARDSAI_MODEL_SHA256 } from './providers/bardsai.js';
+export { BardsaiProvider, BARDSAI_MODEL_URL, BARDSAI_MODEL_REVISION, BARDSAI_MODEL_SHA256, BARDSAI_TOKENIZER_FILES, } from './providers/bardsai.js';
 export { detectWithRegex, resolveOverlaps, filterFalsePositives, propagateEntities, detectEntities, } from './pipeline.js';
-export { createEngine, PROVIDERS, pickDefaultProvider, defaultThresholdFor, clampThreshold, ENGINE_SETTINGS_KEYS, } from './engine.js';
+export { createEngine, ModelNotLoadedError, PROVIDERS, pickDefaultProvider, defaultThresholdFor, clampThreshold, ENGINE_SETTINGS_KEYS, } from './engine.js';
 export type { ProviderId, ProviderEntry, ProviderFactory, EngineOptions, EngineSettings, DocCloakEngine, } from './engine.js';
 export { serveEngine, connectEngine } from './worker-protocol.js';
 export type { PortLike, EngineClient, EngineRequest, EngineResponse, } from './worker-protocol.js';

package/dist/index.js

@@ -4,7 +4,7 @@
 export { memoryKV, memoryBlobCache } from "./env.js";
 export const CORE_PACKAGE_NAME = '@doccloak/core';
 // T006: model loader
-export { fetchModelBlob, evictModelFromCache, retryAsync, ModelIntegrityError, verificationMarkerKey } from "./model-loader.js";
+export { fetchModelBlob, evictModelFromCache, retryAsync, ModelIntegrityError, verificationMarkerKey, verificationMarkerValue, parseVerificationMarker, fetchPinnedJson, loadPinnedTokenizer, } from "./model-loader.js";
 // end T006
 // T004: regex rules (REGEX_REGIONS moved here from the web engine in T009)
 export { ALL_REGEX_RULES, REGEX_REGIONS } from "./regex/index.js";
@@ -14,22 +14,27 @@ export * from "./types.js";
 export * from "./session.js";
 export * from "./doc.js";
 // end T003
+// T178: readDocText / writeAnonymizedDoc refuse fast-saved and encrypted .doc
+// files with this error (also exported from '@doccloak/core/dom'); re-exported
+// here so hosts using only the main entry can switch on `code`.
+export { UnsupportedDocumentError, isUnsupportedDocumentError } from "./dom/errors.js";
+// end T178
 // T043: surrogate generator (realistic, shape-preserving replacements)
 export { generateSurrogate, generateUniqueSurrogate, generateSessionSalt, sessionDayOffset, FAKE_EMAIL_DOMAINS, } from "./surrogates.js";
 // end T043
 // T007: gliner provider
-export { GlinerProvider, GLINER_MODEL_URL, GLINER_MODEL_REVISION, GLINER_MODEL_SHA256 } from "./providers/gliner.js";
+export { GlinerProvider, GLINER_MODEL_URL, GLINER_MODEL_REVISION, GLINER_MODEL_SHA256, GLINER_TOKENIZER_FILES, } from "./providers/gliner.js";
 // end T007
 // T122: gliner base provider (markerV0 span mode)
-export { GlinerBaseProvider, GLINER_BASE_MODEL_URL, GLINER_BASE_MODEL_REVISION, GLINER_BASE_MODEL_SHA256, GLINER_BASE_MAX_WIDTH, buildSpanIndices, decodeSpanLogits, } from "./providers/gliner-base.js";
+export { GlinerBaseProvider, GLINER_BASE_MODEL_URL, GLINER_BASE_MODEL_REVISION, GLINER_BASE_MODEL_SHA256, GLINER_BASE_TOKENIZER_FILES, GLINER_BASE_MAX_WIDTH, buildSpanIndices, decodeSpanLogits, } from "./providers/gliner-base.js";
 // end T122
 // T008: bardsai provider
-export { BardsaiProvider, BARDSAI_MODEL_URL, BARDSAI_MODEL_REVISION, BARDSAI_MODEL_SHA256 } from "./providers/bardsai.js";
+export { BardsaiProvider, BARDSAI_MODEL_URL, BARDSAI_MODEL_REVISION, BARDSAI_MODEL_SHA256, BARDSAI_TOKENIZER_FILES, } from "./providers/bardsai.js";
 // end T008
 // T005: pipeline
 export { detectWithRegex, resolveOverlaps, filterFalsePositives, propagateEntities, detectEntities, } from "./pipeline.js";
 // end T005
 // T009: engine assembly + worker protocol
-export { createEngine, PROVIDERS, pickDefaultProvider, defaultThresholdFor, clampThreshold, ENGINE_SETTINGS_KEYS, } from "./engine.js";
+export { createEngine, ModelNotLoadedError, PROVIDERS, pickDefaultProvider, defaultThresholdFor, clampThreshold, ENGINE_SETTINGS_KEYS, } from "./engine.js";
 export { serveEngine, connectEngine } from "./worker-protocol.js";
 // end T009

package/dist/model-loader.d.ts

@@ -17,13 +17,18 @@
  * - Verifies downloads against a pinned SHA-256 when the caller provides one
  *   (supply-chain hardening, T116): mismatches evict the cache entry and
  *   reject with ModelIntegrityError; verified cache entries carry a marker
- *   so they are not re-hashed on every startup.
+ *   so they are not re-hashed on every startup. The marker records the
+ *   verified byte size (T185): a cache hit whose blob size no longer matches
+ *   the marker is re-hashed instead of trusted.
+ * - Fetches the pinned tokenizer files (tokenizer.json, tokenizer_config.json)
+ *   through the same path (loadPinnedTokenizer, T185) so tokenizers get the
+ *   same resume, retry, cache and integrity treatment as the model.
  *
  * Environment-agnostic: all host capabilities (blob cache, fetch, persistent
  * storage request) are injected via ModelLoaderEnv. This module never touches
  * caches, navigator or global fetch directly.
  */
-import type { BlobCache } from './env.js';
+import type { BlobCache, CoreEnv } from './env.js';
 export type DownloadProgress = (downloaded: number, total: number) => void;
 /**
  * Host capabilities the loader needs. Hosts pass the relevant slice of their
@@ -64,6 +69,19 @@ export interface ModelVerification {
  * written for the previous pin.
  */
 export declare function verificationMarkerKey(url: string, sha256: string): string;
+/**
+ * Value stored in a verification marker: `${sha256}:${size}`, binding the
+ * marker to the byte size of the blob that was actually hashed (T185, S4).
+ * A cache hit is only trusted without re-hashing when the cached blob's
+ * size equals the recorded one. Markers written by 0.11.0 hold the single
+ * byte '1' (size unknown): they are upgraded by hashing once and rewriting.
+ */
+export declare function verificationMarkerValue(sha256: string, size: number): string;
+/**
+ * Size recorded in a marker value for the given hash, or null when the
+ * marker is the legacy 0.11.0 '1' or otherwise unreadable (size unknown).
+ */
+export declare function parseVerificationMarker(value: string, sha256: string): number | null;
 export interface FetchModelOptions {
     maxAttempts?: number;
     retryBaseDelayMs?: number;
@@ -76,6 +94,13 @@ export interface FetchModelOptions {
      * marked verified or evicted and re-downloaded.
      */
     sha256?: string;
+    /**
+     * Pinned byte size of the file. With sha256 it is cross-checked against
+     * every download (a size mismatch is an integrity failure like a hash
+     * mismatch). It also serves as the progress total when the server sends
+     * no content-length. Without sha256 it is only a progress hint.
+     */
+    size?: number;
     /** Fired once the served blob is known to match the pinned sha256. */
     onVerified?: (verification: ModelVerification) => void;
 }
@@ -90,9 +115,39 @@ export declare function fetchModelBlob(env: ModelLoaderEnv, url: string, onProgr
  */
 export declare function evictModelFromCache(env: ModelLoaderEnv, url: string, sha256?: string): Promise<void>;
 /**
- * Retry an async operation with exponential backoff. Used for the smaller
- * companion downloads (tokenizer files) that go through libraries without
- * their own retry handling - one dropped request on a flaky connection
- * should not abort the whole model load.
+ * Retry an async operation with exponential backoff. Kept for the
+ * deprecated CoreEnv.loadTokenizer path (a host library without its own
+ * retry handling) and for hosts composing their own downloads; the pinned
+ * tokenizer files now go through fetchModelBlob via loadPinnedTokenizer.
  */
 export declare function retryAsync<T>(operation: () => Promise<T>, label: string, options?: FetchModelOptions): Promise<T>;
+/** One pinned companion file: immutable resolve/<commit> URL, SHA-256 and byte size. */
+export interface TokenizerFileSpec {
+    url: string;
+    /** Lowercase hex SHA-256 of the file at the pinned commit. */
+    sha256: string;
+    /** Byte size of the file at the pinned commit. */
+    size: number;
+}
+/** The two files a provider pins for its tokenizer: tokenizer.json, then tokenizer_config.json. */
+export type TokenizerFiles = readonly [tokenizerJson: TokenizerFileSpec, tokenizerConfig: TokenizerFileSpec];
+export type PinnedTokenizerOptions = Pick<FetchModelOptions, 'maxAttempts' | 'retryBaseDelayMs' | 'onVerified'>;
+/**
+ * Download (or serve from cache) one pinned JSON file through fetchModelBlob
+ * with its sha256 and size, then parse it. Integrity failures surface as
+ * ModelIntegrityError with the entry evicted, like the model itself.
+ */
+export declare function fetchPinnedJson(env: ModelLoaderEnv, file: TokenizerFileSpec, options?: PinnedTokenizerOptions): Promise<unknown>;
+/**
+ * Resolve a provider's tokenizer through the host env.
+ *
+ * Preferred path: fetch both pinned files via fetchPinnedJson (resume,
+ * retry, blob cache, verification marker, ModelIntegrityError) and hand the
+ * parsed objects to CoreEnv.buildTokenizer. No library probes mutable
+ * resolve/main, no second cache bucket, and a warm cache works offline.
+ *
+ * Fallback (deprecated, removed in 0.13.0): hosts that only implement
+ * CoreEnv.loadTokenizer(hfModelId) are still served through it with the
+ * previous retry wrapper.
+ */
+export declare function loadPinnedTokenizer(env: CoreEnv, files: TokenizerFiles, legacyHfModelId: string, options?: PinnedTokenizerOptions): Promise<unknown>;