@doccloak/core 0.11.0 → 0.12.1

package/dist/model-loader.js

@@ -17,7 +17,12 @@
  * - Verifies downloads against a pinned SHA-256 when the caller provides one
  *   (supply-chain hardening, T116): mismatches evict the cache entry and
  *   reject with ModelIntegrityError; verified cache entries carry a marker
- *   so they are not re-hashed on every startup.
+ *   so they are not re-hashed on every startup. The marker records the
+ *   verified byte size (T185): a cache hit whose blob size no longer matches
+ *   the marker is re-hashed instead of trusted.
+ * - Fetches the pinned tokenizer files (tokenizer.json, tokenizer_config.json)
+ *   through the same path (loadPinnedTokenizer, T185) so tokenizers get the
+ *   same resume, retry, cache and integrity treatment as the model.
  *
  * Environment-agnostic: all host capabilities (blob cache, fetch, persistent
  * storage request) are injected via ModelLoaderEnv. This module never touches
@@ -62,6 +67,26 @@ export function verificationMarkerKey(url, sha256) {
     const sep = url.includes('?') ? '&' : '?';
     return `${url}${sep}doccloak-sha256-verified=${sha256}`;
 }
+/**
+ * Value stored in a verification marker: `${sha256}:${size}`, binding the
+ * marker to the byte size of the blob that was actually hashed (T185, S4).
+ * A cache hit is only trusted without re-hashing when the cached blob's
+ * size equals the recorded one. Markers written by 0.11.0 hold the single
+ * byte '1' (size unknown): they are upgraded by hashing once and rewriting.
+ */
+export function verificationMarkerValue(sha256, size) {
+    return `${sha256.toLowerCase()}:${size}`;
+}
+/**
+ * Size recorded in a marker value for the given hash, or null when the
+ * marker is the legacy 0.11.0 '1' or otherwise unreadable (size unknown).
+ */
+export function parseVerificationMarker(value, sha256) {
+    const match = /^([0-9a-f]{64}):(\d{1,15})$/.exec(value.trim());
+    if (!match || match[1] !== sha256.toLowerCase())
+        return null;
+    return Number(match[2]);
+}
 /**
  * SHA-256 of a blob as lowercase hex. WebCrypto has no streaming digest,
  * so the blob is hashed as one contiguous buffer: peak cost is one extra
@@ -73,13 +98,25 @@ async function sha256Hex(blob) {
     const digest = await crypto.subtle.digest('SHA-256', buffer);
     return Array.from(new Uint8Array(digest), (b) => b.toString(16).padStart(2, '0')).join('');
 }
-/** Best-effort write of the verification marker (a 1-byte blob). */
-async function markVerified(cache, url, sha256) {
+/** Best-effort write of the verification marker (`${sha256}:${size}`). */
+async function markVerified(cache, url, sha256, size) {
     try {
-        await cache.put(verificationMarkerKey(url, sha256), new Blob(['1']));
+        await cache.put(verificationMarkerKey(url, sha256), new Blob([verificationMarkerValue(sha256, size)]));
     }
     catch { /* marker is an optimisation - next startup just re-hashes */ }
 }
+/** Size recorded by the marker for this url + hash; null if absent, legacy or unreadable. */
+async function readMarkerSize(cache, url, sha256) {
+    const marker = await readFromCache(cache, verificationMarkerKey(url, sha256));
+    if (!marker)
+        return { present: false, size: null };
+    try {
+        return { present: true, size: parseVerificationMarker(await marker.text(), sha256) };
+    }
+    catch {
+        return { present: true, size: null };
+    }
+}
 function isRetryable(err) {
     if (err instanceof HttpError) {
         return err.status === 408 || err.status === 429 || err.status >= 500;
@@ -203,6 +240,7 @@ export async function fetchModelBlob(env, url, onProgress, options) {
     const maxAttempts = options?.maxAttempts ?? MAX_ATTEMPTS;
     const retryBaseDelayMs = options?.retryBaseDelayMs ?? RETRY_BASE_DELAY_MS;
     const expectedSha256 = options?.sha256?.toLowerCase();
+    const pinnedSize = options?.size;
     const cache = env.cache;
     const cachedBlob = await readFromCache(cache, url);
     if (cachedBlob) {
@@ -210,18 +248,24 @@ export async function fetchModelBlob(env, url, onProgress, options) {
             onProgress?.(cachedBlob.size, cachedBlob.size);
             return cachedBlob;
         }
-        // Previously verified against this exact pin: skip the re-hash so
+        // Previously verified against this exact pin AND the cached blob still
+        // has the size recorded at verification time: skip the re-hash so
         // startup cost is unchanged.
-        const marker = await readFromCache(cache, verificationMarkerKey(url, expectedSha256));
-        if (marker) {
+        const marker = await readMarkerSize(cache, url, expectedSha256);
+        if (marker.present && marker.size === cachedBlob.size) {
             onProgress?.(cachedBlob.size, cachedBlob.size);
             options?.onVerified?.({ url, sha256: expectedSha256 });
             return cachedBlob;
         }
-        // No marker (cache predates pinning, or the pin changed): hash once.
+        if (marker.present && marker.size !== null) {
+            console.warn(`[DocCloak] Cached model size ${cachedBlob.size} differs from its verification marker (${marker.size}), re-hashing`);
+        }
+        // No marker (cache predates pinning, or the pin changed), a legacy
+        // 0.11.0 marker without a size, or a size mismatch: hash once and
+        // (re)write the marker with the verified size.
         const actualSha256 = await sha256Hex(cachedBlob);
         if (actualSha256 === expectedSha256) {
-            await markVerified(cache, url, expectedSha256);
+            await markVerified(cache, url, expectedSha256, cachedBlob.size);
             onProgress?.(cachedBlob.size, cachedBlob.size);
             options?.onVerified?.({ url, sha256: expectedSha256 });
             return cachedBlob;
@@ -237,7 +281,8 @@ export async function fetchModelBlob(env, url, onProgress, options) {
         if (!force && now - lastProgressAt < PROGRESS_INTERVAL_MS)
             return;
         lastProgressAt = now;
-        onProgress?.(downloaded, total);
+        // The pinned size stands in for the total when the server sends none.
+        onProgress?.(downloaded, total || pinnedSize || 0);
     };
     let chunks = [];
     let downloaded = 0;
@@ -289,16 +334,17 @@ export async function fetchModelBlob(env, url, onProgress, options) {
     chunks = [];
     if (expectedSha256) {
         // Verify the final assembled blob (covers both fresh and resumed
-        // downloads) BEFORE it is cached or handed to the runtime.
+        // downloads) BEFORE it is cached or handed to the runtime. A pinned
+        // size that disagrees is an integrity failure too.
         const actualSha256 = await sha256Hex(blob);
-        if (actualSha256 !== expectedSha256) {
+        if (actualSha256 !== expectedSha256 || (pinnedSize !== undefined && blob.size !== pinnedSize)) {
             await evictModelFromCache(env, url, expectedSha256);
             throw new ModelIntegrityError(url, expectedSha256, actualSha256);
         }
     }
     const stored = await tryCachePut(cache, url, blob);
     if (expectedSha256 && stored)
-        await markVerified(cache, url, expectedSha256);
+        await markVerified(cache, url, expectedSha256, blob.size);
     // Prefer the cached (disk-backed) copy so the in-memory chunks can be
     // collected before ONNX Runtime allocates its own copy of the model.
     const diskBlob = await readFromCache(cache, url);
@@ -325,10 +371,10 @@ export async function evictModelFromCache(env, url, sha256) {
     }
 }
 /**
- * Retry an async operation with exponential backoff. Used for the smaller
- * companion downloads (tokenizer files) that go through libraries without
- * their own retry handling - one dropped request on a flaky connection
- * should not abort the whole model load.
+ * Retry an async operation with exponential backoff. Kept for the
+ * deprecated CoreEnv.loadTokenizer path (a host library without its own
+ * retry handling) and for hosts composing their own downloads; the pinned
+ * tokenizer files now go through fetchModelBlob via loadPinnedTokenizer.
  */
 export async function retryAsync(operation, label, options) {
     const maxAttempts = options?.maxAttempts ?? MAX_ATTEMPTS;
@@ -345,3 +391,62 @@ export async function retryAsync(operation, label, options) {
         }
     }
 }
+/**
+ * Download (or serve from cache) one pinned JSON file through fetchModelBlob
+ * with its sha256 and size, then parse it. Integrity failures surface as
+ * ModelIntegrityError with the entry evicted, like the model itself.
+ */
+export async function fetchPinnedJson(env, file, options) {
+    const blob = await fetchModelBlob(env, file.url, undefined, {
+        ...options,
+        sha256: file.sha256,
+        size: file.size,
+    });
+    const text = await blob.text();
+    try {
+        return JSON.parse(text);
+    }
+    catch (err) {
+        throw new Error(`Pinned file is not valid JSON (${file.url}): ${err instanceof Error ? err.message : String(err)}`);
+    }
+}
+let warnedDeprecatedLoadTokenizer = false;
+/**
+ * Resolve a provider's tokenizer through the host env.
+ *
+ * Preferred path: fetch both pinned files via fetchPinnedJson (resume,
+ * retry, blob cache, verification marker, ModelIntegrityError) and hand the
+ * parsed objects to CoreEnv.buildTokenizer. No library probes mutable
+ * resolve/main, no second cache bucket, and a warm cache works offline.
+ *
+ * Fallback (deprecated, removed in 0.13.0): hosts that only implement
+ * CoreEnv.loadTokenizer(hfModelId) are still served through it with the
+ * previous retry wrapper.
+ */
+export async function loadPinnedTokenizer(env, files, legacyHfModelId, options) {
+    if (typeof env.buildTokenizer === 'function') {
+        const loaderEnv = {
+            cache: env.modelCache,
+            fetch: env.fetch,
+            persistStorage: env.persistStorage,
+        };
+        const [tokenizerJson, tokenizerConfig] = await Promise.all([
+            fetchPinnedJson(loaderEnv, files[0], options),
+            fetchPinnedJson(loaderEnv, files[1], options),
+        ]);
+        const tokenizer = await env.buildTokenizer(tokenizerJson, tokenizerConfig);
+        if (tokenizer === null || tokenizer === undefined) {
+            throw new Error('CoreEnv.buildTokenizer returned no tokenizer');
+        }
+        return tokenizer;
+    }
+    if (typeof env.loadTokenizer === 'function') {
+        if (!warnedDeprecatedLoadTokenizer) {
+            warnedDeprecatedLoadTokenizer = true;
+            console.warn('[DocCloak] CoreEnv.loadTokenizer is deprecated and will be removed in @doccloak/core 0.13.0; implement buildTokenizer(tokenizerJson, tokenizerConfig) instead');
+        }
+        // Call through env so host implementations keep their receiver.
+        return retryAsync(() => env.loadTokenizer(legacyHfModelId), 'Tokenizer download', options);
+    }
+    throw new Error('CoreEnv must implement buildTokenizer(tokenizerJson, tokenizerConfig); loadTokenizer(hfModelId) is deprecated');
+}

package/dist/pipeline.d.ts

@@ -1,19 +1,98 @@
 /**
  * Pure entity pipeline (T005).
  *
- * Extracted verbatim from the web app's detection.worker.ts: text and
- * entities in, entities out. No worker messaging, no providers, no
- * environment access. The worker (and any other host) composes these
- * functions around its own ML provider call.
+ * Extracted from the web app's detection.worker.ts: text and entities in,
+ * entities out. No worker messaging, no providers, no environment access.
+ * The worker (and any other host) composes these functions around its own
+ * ML provider call.
  *
- * Function bodies are byte-identical to the originals by design; only the
- * signatures of detectWithRegex (regexRegion becomes a parameter instead of
- * worker module state) and detectEntities (ML results are passed in instead
- * of fetched from a provider) differ.
+ * Security pass 2026-09:
+ *  - T180: detectWithRegex runs every rule line by line (rules flagged
+ *    `multiline` see the whole text), so quadratic patterns cost O(line^2)
+ *    instead of O(document^2).
+ *  - T181: filterFalsePositives is script-aware (CJK/Thai and PERSON values
+ *    of two characters survive) and propagateEntities uses Unicode letter/
+ *    digit boundaries, exact offsets on the original text and a common-word
+ *    guard for lowercase PERSON matches.
  */
 import type { DetectedEntity } from './types.js';
+/** One newline-free slice of the input plus its offset in the original text. */
+interface TextSegment {
+    text: string;
+    offset: number;
+}
+/**
+ * Split on '\n' keeping offsets. '\r' stays inside the line (a CRLF line then
+ * ends with '\r', exactly as it did when the whole text was matched at once).
+ * Empty lines are skipped: no rule can match the empty string.
+ */
+export declare function segmentLines(text: string): TextSegment[];
+/**
+ * Run the regex rules of a region over the text.
+ *
+ * Rules without `multiline` run on each line separately (offsets are added
+ * back, so spans are document-absolute); rules with `multiline: true` run on
+ * the whole text. Output order is rule-major, then position, as before.
+ *
+ * Complexity note (T180): several street/company rules are quadratic in the
+ * length of their input. Per-line matching bounds that to the longest line,
+ * so 100 KB of ordinary prose is fast; a single 100 KB line without newlines
+ * is still slow until the patterns themselves are bounded (T198).
+ */
 export declare function detectWithRegex(text: string, regexRegion?: string): DetectedEntity[];
 export declare function resolveOverlaps(entities: DetectedEntity[]): DetectedEntity[];
+/**
+ * Word-level providers (BardS.ai splits on whitespace) return spans that
+ * carry the punctuation glued to the last word: "Anna Nowak," or
+ * "jan@example.com.". The redacted text then loses that punctuation and the
+ * mapping stores it as part of the value. Trim a fixed set of opening and
+ * closing punctuation at both ends of every span (T204); a span that would
+ * become empty is dropped. Regex spans are not passed through this: their
+ * patterns already stop at the identifier.
+ */
+export declare function trimSpanPunctuation(entities: DetectedEntity[]): DetectedEntity[];
+/**
+ * Two-character PERSON values that are titles, suffixes or abbreviations,
+ * never names. Compared against the lowercased trimmed value (T181, R2).
+ */
+export declare const PERSON_SHORT_STOPLIST: ReadonlySet<string>;
+/**
+ * Drop ML spans that are too short to be real entities (R2):
+ *  - values containing CJK or Thai characters: at least 2 code points, any type
+ *  - PERSON: at least 2 characters, unless the value is a known 2-letter
+ *    title/suffix (PERSON_SHORT_STOPLIST); initials like "J." pass
+ *  - every other type: at least 3 characters (unchanged)
+ */
 export declare function filterFalsePositives(entities: DetectedEntity[]): DetectedEntity[];
+/**
+ * Words that are also given names or surnames, per language (T181, R15).
+ * A PERSON term found in the text with DIFFERENT casing than the detected
+ * entity, and NOT starting with an uppercase letter, is skipped when its
+ * lowercase form is listed here: "Bill" the person never turns "the bill is
+ * due" into "[PERSON_1] is due". Capitalised and ALL-CAPS mentions always
+ * propagate, and lowercase mentions of ordinary names ("thanks john") are
+ * still caught (golden prop-03-case-insensitive).
+ *
+ * Only words that appear in lowercase in running prose belong here. German
+ * nouns are always capitalised in prose, so the DE list holds adjectives and
+ * month names that double as surnames or first names; the PL list holds
+ * nouns (animals, months, plants) that are frequent Polish surnames and
+ * first names.
+ */
+export declare const COMMON_WORD_NAMES: Readonly<Record<'en' | 'pl' | 'de', readonly string[]>>;
+/**
+ * Find every further occurrence of each detected value (and of the words of
+ * multi-word values) in the text and emit it as a 'propagated' entity.
+ *
+ * Terms: whole values of at least 2 characters; words of multi-word values
+ * of at least 4 characters (trailing punctuation stripped). Terms of at most
+ * 2 characters match with exact case; longer terms match case-insensitively
+ * (Unicode case folding via the 'iu' flags) on the ORIGINAL text, so offsets
+ * are exact even after characters whose lowercase form has a different
+ * length (Turkish dotted İ). A hit needs a non-letter, non-digit neighbour
+ * on both sides ([\p{L}\p{N}] complement), except on a side where the term
+ * itself ends in an unspaced script (CJK, Thai).
+ */
 export declare function propagateEntities(text: string, entities: DetectedEntity[]): DetectedEntity[];
 export declare function detectEntities(text: string, mlEntities: DetectedEntity[], regexEntities: DetectedEntity[]): DetectedEntity[];
+export {};