gheim 0.1.5 → 0.1.7

package/README.md

@@ -41,14 +41,15 @@ The core package itself has no runtime dependencies.
 (transformers.js). The package's default model is
 [`joelbarmettler/gheim-ch-560m`](https://huggingface.co/joelbarmettler/gheim-ch-560m)
 — a 560M xlm-roberta-large fine-tune optimised for Swiss-market PII
-(strict-span F1 0.916 on Swiss text, see
+(test strict F1 0.910, char F1 0.946 on Swiss text, see
 [MODEL_CARD.md](https://github.com/joelbarmettlerUZH/gheim/blob/main/MODEL_CARD.md)).
 Any HuggingFace token-classification model that emits the same 33-class
 BIOES schema can be substituted via `model:`.
 
 | Model | Best for | Parameters | Notes |
 |---|---|---:|---|
-| [`joelbarmettler/gheim-ch-560m`](https://huggingface.co/joelbarmettler/gheim-ch-560m) **(default)** | Swiss-market text (de_CH, fr_CH, it_CH, rm, en) with CH-format account numbers (IBAN, AHV, VAT-CHE) | 560M | Apache 2.0. Test F1 0.916. Hub repo ships `onnx/model_quantized.onnx` only — load with `dtype: "q8"`. |
+| [`joelbarmettler/gheim-ch-560m`](https://huggingface.co/joelbarmettler/gheim-ch-560m) **(default)** | Production / commercial. Swiss court / parliament / web text with CH-format account numbers (IBAN, AHV, VAT-CHE) | 560M | Apache 2.0. Test strict F1 0.910, char F1 0.946. Hub repo ships both fp32 and `onnx/model_quantized.onnx` — load with `dtype: "q8"` for browser deployment. |
+| [`joelbarmettler/gheim-ch-560m-research`](https://huggingface.co/joelbarmettler/gheim-ch-560m-research) | Research / non-commercial. Stronger cross-domain transfer on Swiss-news text (swissner PER char F1 0.90 vs 0.70 on the default) | 560M | **CC BY-NC-SA 4.0 + Reuters research-only rider.** safetensors only — no browser-targeted ONNX. |
 | [`openai/privacy-filter`](https://huggingface.co/openai/privacy-filter) | English-first or general use, long-context (up to 128k tokens) | 1.4B (50M active, MoE) | Apache 2.0. Wider language coverage, larger weights. |
 
 ```ts
@@ -58,6 +59,9 @@ import { LocalDetector } from "gheim";
 // the browser and falls back to WASM; in Node it's WASM directly.
 const det = new LocalDetector({ dtype: "q8" });
 
+// Stronger cross-domain transfer (research, non-commercial; safetensors only):
+const detResearch = new LocalDetector({ model: "joelbarmettler/gheim-ch-560m-research" });
+
 // Alternative for English or general use:
 const detEn = new LocalDetector({ model: "openai/privacy-filter" });
 ```

package/dist/cjs/index.cjs

@@ -157,8 +157,8 @@ class Session {
     }
     out.push(text.slice(cursor));
     let redacted = out.join("");
-    redacted = recoverDuplicateOccurrences(redacted, this.mapping);
-    return { redacted, merged };
+    const recovered = recoverDuplicateOccurrences(text, redacted, merged, this.mapping);
+    return { redacted: recovered.redacted, merged: recovered.merged };
   }
   restore(text) {
     if (Object.keys(this.mapping).length === 0)
@@ -217,14 +217,36 @@ function mergeAdjacent(text, spans) {
 function escapeRegex(s) {
   return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
 }
-function recoverDuplicateOccurrences(redacted, mapping) {
+function recoverDuplicateOccurrences(text, redacted, merged, mapping) {
+  const surfaceToLabel = new Map;
+  for (const m of merged)
+    surfaceToLabel.set(text.slice(m.start, m.end), m.label);
+  const covered = merged.map((m) => [m.start, m.end]);
+  const overlaps = (s, e) => covered.some(([cs, ce]) => s < ce && e > cs);
   const entries = Object.entries(mapping).filter(([, surface]) => surface && surface.length > 0).sort((a, b) => b[1].length - a[1].length);
   let result = redacted;
+  const additional = [];
   for (const [sentinel, surface] of entries) {
-    const pattern = new RegExp(`(?<![\\p{L}\\p{N}_])${escapeRegex(surface)}(?![\\p{L}\\p{N}_])`, "gu");
-    result = result.replace(pattern, sentinel);
+    const boundaryPattern = `(?<![\\p{L}\\p{N}_])${escapeRegex(surface)}(?![\\p{L}\\p{N}_])`;
+    result = result.replace(new RegExp(boundaryPattern, "gu"), sentinel);
+    const label = surfaceToLabel.get(surface);
+    if (!label)
+      continue;
+    const origRe = new RegExp(boundaryPattern, "gu");
+    let match;
+    while ((match = origRe.exec(text)) !== null) {
+      const start = match.index;
+      const end = start + surface.length;
+      if (overlaps(start, end))
+        continue;
+      additional.push({ start, end, label });
+      covered.push([start, end]);
+    }
   }
-  return result;
+  if (additional.length === 0)
+    return { redacted: result, merged };
+  const augmented = [...merged, ...additional].sort((a, b) => a.start - b.start);
+  return { redacted: result, merged: augmented };
 }
 // src/core/stream.ts
 var SENTINEL_AT = /<([A-Z][A-Z0-9_]*)_(\d+)>/y;
@@ -369,13 +391,14 @@ class LocalDetector {
   chunkOverlapTokens;
   batchSize;
   actualDevice = null;
+  actualDtype = null;
   _pipeline = null;
   _loadPromise = null;
   _onProgress;
   constructor(opts = {}) {
     this.model = opts.model ?? DEFAULT_MODEL2;
     this.device = opts.device ?? "auto";
-    this.dtype = opts.dtype ?? "fp32";
+    this.dtype = opts.dtype ?? "auto";
     this.aggregationStrategy = opts.aggregationStrategy ?? "simple";
     this.chunkTokens = opts.chunkTokens ?? DEFAULT_CHUNK_TOKENS;
     this.chunkOverlapTokens = opts.chunkOverlapTokens ?? DEFAULT_OVERLAP_TOKENS;
@@ -387,8 +410,14 @@ class LocalDetector {
     if (opts.pipelineInstance) {
       this._pipeline = opts.pipelineInstance;
       this.actualDevice = this.device === "auto" ? null : this.device;
+      this.actualDtype = this.dtype === "auto" ? null : this.dtype;
     }
   }
+  _resolveDtype(device) {
+    if (this.dtype !== "auto")
+      return this.dtype;
+    return device === "webgpu" ? "fp16" : "q8";
+  }
   async load() {
     if (this._pipeline)
       return this._pipeline;
@@ -400,14 +429,16 @@ class LocalDetector {
       let lastErr;
       for (let i = 0;i < sequence.length; i++) {
         const dev = sequence[i] ?? "wasm";
+        const resolvedDtype = this._resolveDtype(dev);
         try {
           const pipe = await mod.pipeline("token-classification", this.model, {
             device: dev,
-            dtype: this.dtype,
+            dtype: resolvedDtype,
             progress_callback: this._onProgress ? (raw) => this._emitProgress(raw, dev) : undefined
           });
           this._pipeline = pipe;
           this.actualDevice = dev;
+          this.actualDtype = resolvedDtype;
           return pipe;
         } catch (e) {
           lastErr = e;
@@ -423,6 +454,7 @@ class LocalDetector {
   async _planDeviceSequence() {
     if (this.device !== "auto")
       return [this.device];
+    const isBrowser = typeof globalThis.window !== "undefined";
     const seq = [];
     const nav = typeof navigator !== "undefined" ? navigator : null;
     if (nav?.gpu?.requestAdapter) {
@@ -432,7 +464,7 @@ class LocalDetector {
           seq.push("webgpu");
       } catch {}
     }
-    seq.push("wasm");
+    seq.push(isBrowser ? "wasm" : "cpu");
     return seq;
   }
   _emitProgress(raw, device) {
@@ -864,6 +896,7 @@ class CalibratedDetector {
   chunkOverlapTokens;
   trimWhitespace;
   actualDevice = null;
+  actualDtype = null;
   _model = null;
   _tokenizer = null;
   _id2label = {};
@@ -874,7 +907,7 @@ class CalibratedDetector {
     this.model = opts.model ?? DEFAULT_MODEL3;
     this.oBias = opts.oBias ?? 0.5;
     this.device = opts.device ?? "auto";
-    this.dtype = opts.dtype ?? "fp32";
+    this.dtype = opts.dtype ?? "auto";
     this.chunkTokens = opts.chunkTokens ?? DEFAULT_CHUNK_TOKENS2;
     this.chunkOverlapTokens = opts.chunkOverlapTokens ?? DEFAULT_OVERLAP_TOKENS2;
     this.trimWhitespace = opts.trimWhitespace ?? true;
@@ -892,8 +925,9 @@ class CalibratedDetector {
       const mod = await loadTransformersJs2();
       let preferredDevice = this.device;
       if (this.device === "auto") {
+        const isBrowser = typeof globalThis.window !== "undefined";
         const nav = typeof navigator !== "undefined" ? navigator : null;
-        preferredDevice = "wasm";
+        preferredDevice = isBrowser ? "wasm" : "cpu";
         if (nav?.gpu?.requestAdapter) {
           try {
             const adapter = await nav.gpu.requestAdapter();
@@ -902,14 +936,16 @@ class CalibratedDetector {
           } catch {}
         }
       }
+      const resolvedDtype = this.dtype === "auto" ? preferredDevice === "webgpu" ? "fp16" : "q8" : this.dtype;
       const tokenizer = await mod.AutoTokenizer.from_pretrained(this.model);
       const model = await mod.AutoModelForTokenClassification.from_pretrained(this.model, {
         device: preferredDevice,
-        dtype: this.dtype,
+        dtype: resolvedDtype,
         progress_callback: this._onProgress ? (raw) => this._emitProgress(raw, preferredDevice) : undefined
       });
       this._tokenizer = tokenizer;
       this._model = model;
+      this.actualDtype = resolvedDtype;
       this._id2label = model.config.id2label;
       const label2id = model.config.label2id;
       this._oId = label2id?.["O"] ?? 0;
@@ -1168,4 +1204,4 @@ async function* deanonymizeStream(chunks, session) {
 }
 
 // src/index.ts
-var VERSION = "0.1.5";
+var VERSION = "0.1.6";

package/dist/cjs/openai/index.cjs

@@ -277,13 +277,14 @@ class LocalDetector {
   chunkOverlapTokens;
   batchSize;
   actualDevice = null;
+  actualDtype = null;
   _pipeline = null;
   _loadPromise = null;
   _onProgress;
   constructor(opts = {}) {
     this.model = opts.model ?? DEFAULT_MODEL2;
     this.device = opts.device ?? "auto";
-    this.dtype = opts.dtype ?? "fp32";
+    this.dtype = opts.dtype ?? "auto";
     this.aggregationStrategy = opts.aggregationStrategy ?? "simple";
     this.chunkTokens = opts.chunkTokens ?? DEFAULT_CHUNK_TOKENS;
     this.chunkOverlapTokens = opts.chunkOverlapTokens ?? DEFAULT_OVERLAP_TOKENS;
@@ -295,8 +296,14 @@ class LocalDetector {
     if (opts.pipelineInstance) {
       this._pipeline = opts.pipelineInstance;
       this.actualDevice = this.device === "auto" ? null : this.device;
+      this.actualDtype = this.dtype === "auto" ? null : this.dtype;
     }
   }
+  _resolveDtype(device) {
+    if (this.dtype !== "auto")
+      return this.dtype;
+    return device === "webgpu" ? "fp16" : "q8";
+  }
   async load() {
     if (this._pipeline)
       return this._pipeline;
@@ -308,14 +315,16 @@ class LocalDetector {
       let lastErr;
       for (let i = 0;i < sequence.length; i++) {
         const dev = sequence[i] ?? "wasm";
+        const resolvedDtype = this._resolveDtype(dev);
         try {
           const pipe = await mod.pipeline("token-classification", this.model, {
             device: dev,
-            dtype: this.dtype,
+            dtype: resolvedDtype,
             progress_callback: this._onProgress ? (raw) => this._emitProgress(raw, dev) : undefined
           });
           this._pipeline = pipe;
           this.actualDevice = dev;
+          this.actualDtype = resolvedDtype;
           return pipe;
         } catch (e) {
           lastErr = e;
@@ -331,6 +340,7 @@ class LocalDetector {
   async _planDeviceSequence() {
     if (this.device !== "auto")
       return [this.device];
+    const isBrowser = typeof globalThis.window !== "undefined";
     const seq = [];
     const nav = typeof navigator !== "undefined" ? navigator : null;
     if (nav?.gpu?.requestAdapter) {
@@ -340,7 +350,7 @@ class LocalDetector {
           seq.push("webgpu");
       } catch {}
     }
-    seq.push("wasm");
+    seq.push(isBrowser ? "wasm" : "cpu");
     return seq;
   }
   _emitProgress(raw, device) {
@@ -772,6 +782,7 @@ class CalibratedDetector {
   chunkOverlapTokens;
   trimWhitespace;
   actualDevice = null;
+  actualDtype = null;
   _model = null;
   _tokenizer = null;
   _id2label = {};
@@ -782,7 +793,7 @@ class CalibratedDetector {
     this.model = opts.model ?? DEFAULT_MODEL3;
     this.oBias = opts.oBias ?? 0.5;
     this.device = opts.device ?? "auto";
-    this.dtype = opts.dtype ?? "fp32";
+    this.dtype = opts.dtype ?? "auto";
     this.chunkTokens = opts.chunkTokens ?? DEFAULT_CHUNK_TOKENS2;
     this.chunkOverlapTokens = opts.chunkOverlapTokens ?? DEFAULT_OVERLAP_TOKENS2;
     this.trimWhitespace = opts.trimWhitespace ?? true;
@@ -800,8 +811,9 @@ class CalibratedDetector {
       const mod = await loadTransformersJs2();
       let preferredDevice = this.device;
       if (this.device === "auto") {
+        const isBrowser = typeof globalThis.window !== "undefined";
         const nav = typeof navigator !== "undefined" ? navigator : null;
-        preferredDevice = "wasm";
+        preferredDevice = isBrowser ? "wasm" : "cpu";
         if (nav?.gpu?.requestAdapter) {
           try {
             const adapter = await nav.gpu.requestAdapter();
@@ -810,14 +822,16 @@ class CalibratedDetector {
           } catch {}
         }
       }
+      const resolvedDtype = this.dtype === "auto" ? preferredDevice === "webgpu" ? "fp16" : "q8" : this.dtype;
       const tokenizer = await mod.AutoTokenizer.from_pretrained(this.model);
       const model = await mod.AutoModelForTokenClassification.from_pretrained(this.model, {
         device: preferredDevice,
-        dtype: this.dtype,
+        dtype: resolvedDtype,
         progress_callback: this._onProgress ? (raw) => this._emitProgress(raw, preferredDevice) : undefined
       });
       this._tokenizer = tokenizer;
       this._model = model;
+      this.actualDtype = resolvedDtype;
       this._id2label = model.config.id2label;
       const label2id = model.config.label2id;
       this._oId = label2id?.["O"] ?? 0;
@@ -1086,8 +1100,8 @@ class Session {
     }
     out.push(text.slice(cursor));
     let redacted = out.join("");
-    redacted = recoverDuplicateOccurrences(redacted, this.mapping);
-    return { redacted, merged };
+    const recovered = recoverDuplicateOccurrences(text, redacted, merged, this.mapping);
+    return { redacted: recovered.redacted, merged: recovered.merged };
   }
   restore(text) {
     if (Object.keys(this.mapping).length === 0)
@@ -1146,14 +1160,36 @@ function mergeAdjacent(text, spans) {
 function escapeRegex(s) {
   return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
 }
-function recoverDuplicateOccurrences(redacted, mapping) {
+function recoverDuplicateOccurrences(text, redacted, merged, mapping) {
+  const surfaceToLabel = new Map;
+  for (const m of merged)
+    surfaceToLabel.set(text.slice(m.start, m.end), m.label);
+  const covered = merged.map((m) => [m.start, m.end]);
+  const overlaps = (s, e) => covered.some(([cs, ce]) => s < ce && e > cs);
   const entries = Object.entries(mapping).filter(([, surface]) => surface && surface.length > 0).sort((a, b) => b[1].length - a[1].length);
   let result = redacted;
+  const additional = [];
   for (const [sentinel, surface] of entries) {
-    const pattern = new RegExp(`(?<![\\p{L}\\p{N}_])${escapeRegex(surface)}(?![\\p{L}\\p{N}_])`, "gu");
-    result = result.replace(pattern, sentinel);
+    const boundaryPattern = `(?<![\\p{L}\\p{N}_])${escapeRegex(surface)}(?![\\p{L}\\p{N}_])`;
+    result = result.replace(new RegExp(boundaryPattern, "gu"), sentinel);
+    const label = surfaceToLabel.get(surface);
+    if (!label)
+      continue;
+    const origRe = new RegExp(boundaryPattern, "gu");
+    let match;
+    while ((match = origRe.exec(text)) !== null) {
+      const start = match.index;
+      const end = start + surface.length;
+      if (overlaps(start, end))
+        continue;
+      additional.push({ start, end, label });
+      covered.push([start, end]);
+    }
   }
-  return result;
+  if (additional.length === 0)
+    return { redacted: result, merged };
+  const augmented = [...merged, ...additional].sort((a, b) => a.start - b.start);
+  return { redacted: result, merged: augmented };
 }
 
 // src/openai/_base.ts

package/dist/esm/index.js

@@ -100,8 +100,8 @@ class Session {
     }
     out.push(text.slice(cursor));
     let redacted = out.join("");
-    redacted = recoverDuplicateOccurrences(redacted, this.mapping);
-    return { redacted, merged };
+    const recovered = recoverDuplicateOccurrences(text, redacted, merged, this.mapping);
+    return { redacted: recovered.redacted, merged: recovered.merged };
   }
   restore(text) {
     if (Object.keys(this.mapping).length === 0)
@@ -160,14 +160,36 @@ function mergeAdjacent(text, spans) {
 function escapeRegex(s) {
   return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
 }
-function recoverDuplicateOccurrences(redacted, mapping) {
+function recoverDuplicateOccurrences(text, redacted, merged, mapping) {
+  const surfaceToLabel = new Map;
+  for (const m of merged)
+    surfaceToLabel.set(text.slice(m.start, m.end), m.label);
+  const covered = merged.map((m) => [m.start, m.end]);
+  const overlaps = (s, e) => covered.some(([cs, ce]) => s < ce && e > cs);
   const entries = Object.entries(mapping).filter(([, surface]) => surface && surface.length > 0).sort((a, b) => b[1].length - a[1].length);
   let result = redacted;
+  const additional = [];
   for (const [sentinel, surface] of entries) {
-    const pattern = new RegExp(`(?<![\\p{L}\\p{N}_])${escapeRegex(surface)}(?![\\p{L}\\p{N}_])`, "gu");
-    result = result.replace(pattern, sentinel);
+    const boundaryPattern = `(?<![\\p{L}\\p{N}_])${escapeRegex(surface)}(?![\\p{L}\\p{N}_])`;
+    result = result.replace(new RegExp(boundaryPattern, "gu"), sentinel);
+    const label = surfaceToLabel.get(surface);
+    if (!label)
+      continue;
+    const origRe = new RegExp(boundaryPattern, "gu");
+    let match;
+    while ((match = origRe.exec(text)) !== null) {
+      const start = match.index;
+      const end = start + surface.length;
+      if (overlaps(start, end))
+        continue;
+      additional.push({ start, end, label });
+      covered.push([start, end]);
+    }
   }
-  return result;
+  if (additional.length === 0)
+    return { redacted: result, merged };
+  const augmented = [...merged, ...additional].sort((a, b) => a.start - b.start);
+  return { redacted: result, merged: augmented };
 }
 // src/core/stream.ts
 var SENTINEL_AT = /<([A-Z][A-Z0-9_]*)_(\d+)>/y;
@@ -312,13 +334,14 @@ class LocalDetector {
   chunkOverlapTokens;
   batchSize;
   actualDevice = null;
+  actualDtype = null;
   _pipeline = null;
   _loadPromise = null;
   _onProgress;
   constructor(opts = {}) {
     this.model = opts.model ?? DEFAULT_MODEL2;
     this.device = opts.device ?? "auto";
-    this.dtype = opts.dtype ?? "fp32";
+    this.dtype = opts.dtype ?? "auto";
     this.aggregationStrategy = opts.aggregationStrategy ?? "simple";
     this.chunkTokens = opts.chunkTokens ?? DEFAULT_CHUNK_TOKENS;
     this.chunkOverlapTokens = opts.chunkOverlapTokens ?? DEFAULT_OVERLAP_TOKENS;
@@ -330,8 +353,14 @@ class LocalDetector {
     if (opts.pipelineInstance) {
       this._pipeline = opts.pipelineInstance;
       this.actualDevice = this.device === "auto" ? null : this.device;
+      this.actualDtype = this.dtype === "auto" ? null : this.dtype;
     }
   }
+  _resolveDtype(device) {
+    if (this.dtype !== "auto")
+      return this.dtype;
+    return device === "webgpu" ? "fp16" : "q8";
+  }
   async load() {
     if (this._pipeline)
       return this._pipeline;
@@ -343,14 +372,16 @@ class LocalDetector {
       let lastErr;
       for (let i = 0;i < sequence.length; i++) {
         const dev = sequence[i] ?? "wasm";
+        const resolvedDtype = this._resolveDtype(dev);
         try {
           const pipe = await mod.pipeline("token-classification", this.model, {
             device: dev,
-            dtype: this.dtype,
+            dtype: resolvedDtype,
             progress_callback: this._onProgress ? (raw) => this._emitProgress(raw, dev) : undefined
           });
           this._pipeline = pipe;
           this.actualDevice = dev;
+          this.actualDtype = resolvedDtype;
           return pipe;
         } catch (e) {
           lastErr = e;
@@ -366,6 +397,7 @@ class LocalDetector {
   async _planDeviceSequence() {
     if (this.device !== "auto")
       return [this.device];
+    const isBrowser = typeof globalThis.window !== "undefined";
     const seq = [];
     const nav = typeof navigator !== "undefined" ? navigator : null;
     if (nav?.gpu?.requestAdapter) {
@@ -375,7 +407,7 @@ class LocalDetector {
           seq.push("webgpu");
       } catch {}
     }
-    seq.push("wasm");
+    seq.push(isBrowser ? "wasm" : "cpu");
     return seq;
   }
   _emitProgress(raw, device) {
@@ -807,6 +839,7 @@ class CalibratedDetector {
   chunkOverlapTokens;
   trimWhitespace;
   actualDevice = null;
+  actualDtype = null;
   _model = null;
   _tokenizer = null;
   _id2label = {};
@@ -817,7 +850,7 @@ class CalibratedDetector {
     this.model = opts.model ?? DEFAULT_MODEL3;
     this.oBias = opts.oBias ?? 0.5;
     this.device = opts.device ?? "auto";
-    this.dtype = opts.dtype ?? "fp32";
+    this.dtype = opts.dtype ?? "auto";
     this.chunkTokens = opts.chunkTokens ?? DEFAULT_CHUNK_TOKENS2;
     this.chunkOverlapTokens = opts.chunkOverlapTokens ?? DEFAULT_OVERLAP_TOKENS2;
     this.trimWhitespace = opts.trimWhitespace ?? true;
@@ -835,8 +868,9 @@ class CalibratedDetector {
       const mod = await loadTransformersJs2();
       let preferredDevice = this.device;
       if (this.device === "auto") {
+        const isBrowser = typeof globalThis.window !== "undefined";
         const nav = typeof navigator !== "undefined" ? navigator : null;
-        preferredDevice = "wasm";
+        preferredDevice = isBrowser ? "wasm" : "cpu";
         if (nav?.gpu?.requestAdapter) {
           try {
             const adapter = await nav.gpu.requestAdapter();
@@ -845,14 +879,16 @@ class CalibratedDetector {
           } catch {}
         }
       }
+      const resolvedDtype = this.dtype === "auto" ? preferredDevice === "webgpu" ? "fp16" : "q8" : this.dtype;
       const tokenizer = await mod.AutoTokenizer.from_pretrained(this.model);
       const model = await mod.AutoModelForTokenClassification.from_pretrained(this.model, {
         device: preferredDevice,
-        dtype: this.dtype,
+        dtype: resolvedDtype,
         progress_callback: this._onProgress ? (raw) => this._emitProgress(raw, preferredDevice) : undefined
       });
       this._tokenizer = tokenizer;
       this._model = model;
+      this.actualDtype = resolvedDtype;
       this._id2label = model.config.id2label;
       const label2id = model.config.label2id;
       this._oId = label2id?.["O"] ?? 0;
@@ -1111,7 +1147,7 @@ async function* deanonymizeStream(chunks, session) {
 }
 
 // src/index.ts
-var VERSION = "0.1.5";
+var VERSION = "0.1.6";
 export {
   renderSentinel,
   parseSentinel,

package/dist/esm/openai/index.js

@@ -234,13 +234,14 @@ class LocalDetector {
   chunkOverlapTokens;
   batchSize;
   actualDevice = null;
+  actualDtype = null;
   _pipeline = null;
   _loadPromise = null;
   _onProgress;
   constructor(opts = {}) {
     this.model = opts.model ?? DEFAULT_MODEL2;
     this.device = opts.device ?? "auto";
-    this.dtype = opts.dtype ?? "fp32";
+    this.dtype = opts.dtype ?? "auto";
     this.aggregationStrategy = opts.aggregationStrategy ?? "simple";
     this.chunkTokens = opts.chunkTokens ?? DEFAULT_CHUNK_TOKENS;
     this.chunkOverlapTokens = opts.chunkOverlapTokens ?? DEFAULT_OVERLAP_TOKENS;
@@ -252,8 +253,14 @@ class LocalDetector {
     if (opts.pipelineInstance) {
       this._pipeline = opts.pipelineInstance;
       this.actualDevice = this.device === "auto" ? null : this.device;
+      this.actualDtype = this.dtype === "auto" ? null : this.dtype;
     }
   }
+  _resolveDtype(device) {
+    if (this.dtype !== "auto")
+      return this.dtype;
+    return device === "webgpu" ? "fp16" : "q8";
+  }
   async load() {
     if (this._pipeline)
       return this._pipeline;
@@ -265,14 +272,16 @@ class LocalDetector {
       let lastErr;
       for (let i = 0;i < sequence.length; i++) {
         const dev = sequence[i] ?? "wasm";
+        const resolvedDtype = this._resolveDtype(dev);
         try {
           const pipe = await mod.pipeline("token-classification", this.model, {
             device: dev,
-            dtype: this.dtype,
+            dtype: resolvedDtype,
             progress_callback: this._onProgress ? (raw) => this._emitProgress(raw, dev) : undefined
           });
           this._pipeline = pipe;
           this.actualDevice = dev;
+          this.actualDtype = resolvedDtype;
           return pipe;
         } catch (e) {
           lastErr = e;
@@ -288,6 +297,7 @@ class LocalDetector {
   async _planDeviceSequence() {
     if (this.device !== "auto")
       return [this.device];
+    const isBrowser = typeof globalThis.window !== "undefined";
     const seq = [];
     const nav = typeof navigator !== "undefined" ? navigator : null;
     if (nav?.gpu?.requestAdapter) {
@@ -297,7 +307,7 @@ class LocalDetector {
           seq.push("webgpu");
       } catch {}
     }
-    seq.push("wasm");
+    seq.push(isBrowser ? "wasm" : "cpu");
     return seq;
   }
   _emitProgress(raw, device) {
@@ -729,6 +739,7 @@ class CalibratedDetector {
   chunkOverlapTokens;
   trimWhitespace;
   actualDevice = null;
+  actualDtype = null;
   _model = null;
   _tokenizer = null;
   _id2label = {};
@@ -739,7 +750,7 @@ class CalibratedDetector {
     this.model = opts.model ?? DEFAULT_MODEL3;
     this.oBias = opts.oBias ?? 0.5;
     this.device = opts.device ?? "auto";
-    this.dtype = opts.dtype ?? "fp32";
+    this.dtype = opts.dtype ?? "auto";
     this.chunkTokens = opts.chunkTokens ?? DEFAULT_CHUNK_TOKENS2;
     this.chunkOverlapTokens = opts.chunkOverlapTokens ?? DEFAULT_OVERLAP_TOKENS2;
     this.trimWhitespace = opts.trimWhitespace ?? true;
@@ -757,8 +768,9 @@ class CalibratedDetector {
       const mod = await loadTransformersJs2();
       let preferredDevice = this.device;
       if (this.device === "auto") {
+        const isBrowser = typeof globalThis.window !== "undefined";
         const nav = typeof navigator !== "undefined" ? navigator : null;
-        preferredDevice = "wasm";
+        preferredDevice = isBrowser ? "wasm" : "cpu";
         if (nav?.gpu?.requestAdapter) {
           try {
             const adapter = await nav.gpu.requestAdapter();
@@ -767,14 +779,16 @@ class CalibratedDetector {
           } catch {}
         }
       }
+      const resolvedDtype = this.dtype === "auto" ? preferredDevice === "webgpu" ? "fp16" : "q8" : this.dtype;
       const tokenizer = await mod.AutoTokenizer.from_pretrained(this.model);
       const model = await mod.AutoModelForTokenClassification.from_pretrained(this.model, {
         device: preferredDevice,
-        dtype: this.dtype,
+        dtype: resolvedDtype,
         progress_callback: this._onProgress ? (raw) => this._emitProgress(raw, preferredDevice) : undefined
       });
       this._tokenizer = tokenizer;
       this._model = model;
+      this.actualDtype = resolvedDtype;
       this._id2label = model.config.id2label;
       const label2id = model.config.label2id;
       this._oId = label2id?.["O"] ?? 0;
@@ -1043,8 +1057,8 @@ class Session {
     }
     out.push(text.slice(cursor));
     let redacted = out.join("");
-    redacted = recoverDuplicateOccurrences(redacted, this.mapping);
-    return { redacted, merged };
+    const recovered = recoverDuplicateOccurrences(text, redacted, merged, this.mapping);
+    return { redacted: recovered.redacted, merged: recovered.merged };
   }
   restore(text) {
     if (Object.keys(this.mapping).length === 0)
@@ -1103,14 +1117,36 @@ function mergeAdjacent(text, spans) {
 function escapeRegex(s) {
   return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
 }
-function recoverDuplicateOccurrences(redacted, mapping) {
+function recoverDuplicateOccurrences(text, redacted, merged, mapping) {
+  const surfaceToLabel = new Map;
+  for (const m of merged)
+    surfaceToLabel.set(text.slice(m.start, m.end), m.label);
+  const covered = merged.map((m) => [m.start, m.end]);
+  const overlaps = (s, e) => covered.some(([cs, ce]) => s < ce && e > cs);
   const entries = Object.entries(mapping).filter(([, surface]) => surface && surface.length > 0).sort((a, b) => b[1].length - a[1].length);
   let result = redacted;
+  const additional = [];
   for (const [sentinel, surface] of entries) {
-    const pattern = new RegExp(`(?<![\\p{L}\\p{N}_])${escapeRegex(surface)}(?![\\p{L}\\p{N}_])`, "gu");
-    result = result.replace(pattern, sentinel);
+    const boundaryPattern = `(?<![\\p{L}\\p{N}_])${escapeRegex(surface)}(?![\\p{L}\\p{N}_])`;
+    result = result.replace(new RegExp(boundaryPattern, "gu"), sentinel);
+    const label = surfaceToLabel.get(surface);
+    if (!label)
+      continue;
+    const origRe = new RegExp(boundaryPattern, "gu");
+    let match;
+    while ((match = origRe.exec(text)) !== null) {
+      const start = match.index;
+      const end = start + surface.length;
+      if (overlaps(start, end))
+        continue;
+      additional.push({ start, end, label });
+      covered.push([start, end]);
+    }
   }
-  return result;
+  if (additional.length === 0)
+    return { redacted: result, merged };
+  const augmented = [...merged, ...additional].sort((a, b) => a.start - b.start);
+  return { redacted: result, merged: augmented };
 }
 
 // src/openai/_base.ts

package/dist/types/core/session.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"session.d.ts","sourceRoot":"","sources":["../../../src/core/session.ts"],"names":[],"mappings":"AAAA,kDAAkD;AAClD,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,sBAAsB,CAAC;AAGjD;;;gEAGgE;AAChE,MAAM,MAAM,UAAU,GAAG,CAAC,CAAC,EAAE,MAAM,KAAK,MAAM,GAAG,IAAI,CAAC;AAkBtD,uCAAuC;AACvC,MAAM,WAAW,cAAc;IAC7B;;;;;;;;;;OAUG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;CAC1C;AAED,qBAAa,OAAO;IAClB,8DAA8D;IAC9D,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAM;IACrC,OAAO,CAAC,QAAQ,CAAkC;IAClD,OAAO,CAAC,QAAQ,CAAkC;IAClD,OAAO,CAAC,mBAAmB,CAAkC;gBAEjD,IAAI,GAAE,cAAmB;IAQrC,QAAQ,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,MAAM;IAiBhD,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS,CAAC,IAAI,GAAG,UAAU,CAAC,EAAE,GAAG,MAAM;IAIvE;;;;;;;OAOG;IACH,kBAAkB,CAChB,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE,SAAS,CAAC,IAAI,GAAG,UAAU,CAAC,EAAE,GACpC;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,UAAU,EAAE,CAAA;KAAE;IA+B7C,OAAO,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM;IAK7B,MAAM,IAAI;QAAE,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;KAAE;IAI7C,MAAM,CAAC,QAAQ,CACb,GAAG,EAAE;QAAE,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;KAAE,EACxC,IAAI,GAAE,cAAmB,GACxB,OAAO;CAqBX;AAID,8EAA8E;AAC9E,MAAM,WAAW,UAAU;IACzB,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;;;;;;GAOG;AACH,wBAAgB,aAAa,CAC3B,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE,SAAS,CAAC,IAAI,GAAG,UAAU,CAAC,EAAE,GACpC,UAAU,EAAE,CAqBd"}
+{"version":3,"file":"session.d.ts","sourceRoot":"","sources":["../../../src/core/session.ts"],"names":[],"mappings":"AAAA,kDAAkD;AAClD,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,sBAAsB,CAAC;AAGjD;;;gEAGgE;AAChE,MAAM,MAAM,UAAU,GAAG,CAAC,CAAC,EAAE,MAAM,KAAK,MAAM,GAAG,IAAI,CAAC;AAkBtD,uCAAuC;AACvC,MAAM,WAAW,cAAc;IAC7B;;;;;;;;;;OAUG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;CAC1C;AAED,qBAAa,OAAO;IAClB,8DAA8D;IAC9D,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAM;IACrC,OAAO,CAAC,QAAQ,CAAkC;IAClD,OAAO,CAAC,QAAQ,CAAkC;IAClD,OAAO,CAAC,mBAAmB,CAAkC;gBAEjD,IAAI,GAAE,cAAmB;IAQrC,QAAQ,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,MAAM;IAiBhD,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS,CAAC,IAAI,GAAG,UAAU,CAAC,EAAE,GAAG,MAAM;IAIvE;;;;;;;OAOG;IACH,kBAAkB,CAChB,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE,SAAS,CAAC,IAAI,GAAG,UAAU,CAAC,EAAE,GACpC;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,UAAU,EAAE,CAAA;KAAE;IA8B7C,OAAO,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM;IAK7B,MAAM,IAAI;QAAE,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;KAAE;IAI7C,MAAM,CAAC,QAAQ,CACb,GAAG,EAAE;QAAE,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;KAAE,EACxC,IAAI,GAAE,cAAmB,GACxB,OAAO;CAqBX;AAID,8EAA8E;AAC9E,MAAM,WAAW,UAAU;IACzB,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;;;;;;GAOG;AACH,wBAAgB,aAAa,CAC3B,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE,SAAS,CAAC,IAAI,GAAG,UAAU,CAAC,EAAE,GACpC,UAAU,EAAE,CAqBd"}

package/dist/types/detectors/calibrated.d.ts

@@ -1,19 +1,27 @@
 /**
- * Calibrated detector — fixes the v1 model's "single-category mode"
- * bug at inference time by subtracting a constant `oBias` from the O
- * class logit before argmax.
+ * Calibrated detector — subtracts a constant `oBias` from the O-class
+ * logit before argmax. Originally introduced to recover the v1 model's
+ * "single-category mode" pathology (Müller + IBAN co-occurrence flipped
+ * the person prediction to O); the v3 / gheim-ch-560m training pipeline
+ * largely resolved that pathology, but the calibration still buys a
+ * small precision/recall improvement and remains the default.
  *
- * Empirical justification (see eval/calibration_sweep.json): on a
- * 237-case multi-entity pathology suite the v1 model emits literally
- * zero non-O tokens for "Müller" / "März" when an IBAN follows them —
- * but the per-token softmax shows person at 47% vs O at 51%. A small
- * additive bias on O flips those marginal cases without inducing false
- * positives on PII-free text.
+ * Sweep on gheim-ch-560m (see eval/calibration_sweep.json):
+ * `oBias=0.5` is the Pareto-clean default — probe perfect-rate is flat
+ * at 91.5% across bias 0.0 and 0.5, and test char-F1 lifts by 0.27pp.
+ * Higher biases (1.0–1.5) trade test F1 for marginal probe gains.
  *
- * Sweet spot: oBias=0.5:
- *   - pathology full-coverage 81% → 94.5% (+13.5pp)
- *   - test_v1 strict-F1 0.872 → 0.868 (−0.4pp)
- *   - ~0.05 extra FP per chunk on real text
+ * A separate sweep on the int8 ONNX path via `transformers.js`
+ * (eval/calibration_sweep_q8.json) showed a much larger calibration
+ * effect (+1.4pp probe, +0.43pp char-F1 at 0.5). That gap is not a
+ * property of the int8 quantisation — Python `onnxruntime` on the same
+ * int8 file gives 91.0% probe perfect-rate, essentially fp32-equivalent
+ * — but of a runtime divergence inside `onnxruntime-web` / WASM that
+ * the calibration partially papers over. The reliable fix is to use the
+ * `model_fp16.onnx` variant when WebGPU is available (which is what
+ * LocalDetector's default `dtype: "auto"` does); int8 should be treated
+ * as a low-RAM fallback. See eval/q8_quality_report.md for the
+ * diagnostic.
  *
  * `defaultDetector()` returns this with oBias=0.5, so users get the
  * better behavior automatically.
@@ -26,14 +34,13 @@ export interface CalibratedDetectorOptions {
     model?: string;
     /**
      * Constant subtracted from the O-class logit before argmax. Default
-     * `0.5` is the Pareto-clean point per the full-scale bias sweep
-     * (eval/calibration_sweep.json):
+     * `0.5` is the Pareto-clean point on the reference (fp32/fp16) path:
      *
      *   - oBias=0.0  uncalibrated (matches the raw model)
-     *   - oBias=0.5  recovers ~95% of multi-entity pathology cases for
-     *               ~0.4pp test_v1 strict-F1 cost
-     *   - oBias=1.0  more aggressive: 96.6% pathology, 1.1pp test_v1 cost
-     *   - oBias=1.5+ trades too much precision; not recommended
+     *   - oBias=0.5  +0.27pp test char-F1, no probe cost (91.5% perfect)
+     *   - oBias=1.0+ marginal probe gains, test F1 starts to fall off
+     *
+     * See eval/calibration_sweep.json.
      */
     oBias?: number;
     /** Backend selector. See {@link LocalDetectorOptions.device}. */
@@ -60,6 +67,9 @@ export declare class CalibratedDetector implements Detector {
     readonly trimWhitespace: boolean;
     /** Backend actually used after `load()` resolves; `null` before. */
     actualDevice: string | null;
+    /** Dtype actually used after `load()` resolves; `null` before. Differs
+     *  from {@link dtype} only when `dtype: "auto"` was requested. */
+    actualDtype: string | null;
     private _model;
     private _tokenizer;
     private _id2label;

package/dist/types/detectors/calibrated.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"calibrated.d.ts","sourceRoot":"","sources":["../../../src/detectors/calibrated.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AACH,OAAO,KAAK,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAChD,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,YAAY,CAAC;AAMzD,0DAA0D;AAC1D,MAAM,WAAW,yBAAyB;IACxC,kEAAkE;IAClE,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;;;;;;;;OAUG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,iEAAiE;IACjE,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,kEAAkE;IAClE,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,yDAAyD;IACzD,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,wDAAwD;IACxD,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,8EAA8E;IAC9E,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,4EAA4E;IAC5E,UAAU,CAAC,EAAE,CAAC,KAAK,EAAE,sBAAsB,KAAK,IAAI,CAAC;CACtD;AA4CD,uDAAuD;AACvD,qBAAa,kBAAmB,YAAW,QAAQ;IACjD,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAC;IACpC,QAAQ,CAAC,cAAc,EAAE,OAAO,CAAC;IACjC,oEAAoE;IACpE,YAAY,EAAE,MAAM,GAAG,IAAI,CAAQ;IACnC,OAAO,CAAC,MAAM,CAA8B;IAC5C,OAAO,CAAC,UAAU,CAAkC;IACpD,OAAO,CAAC,SAAS,CAA8B;IAC/C,OAAO,CAAC,IAAI,CAAK;IACjB,OAAO,CAAC,YAAY,CAA8B;IAClD,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC,CAA0C;gBAE3D,IAAI,GAAE,yBAA8B;IAiB1C,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAuCrB,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE;QAAE,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;YAYxD,cAAc;YA0Bd,aAAa;IA6F3B,OAAO,CAAC,aAAa;CAiBtB"}
+{"version":3,"file":"calibrated.d.ts","sourceRoot":"","sources":["../../../src/detectors/calibrated.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,OAAO,KAAK,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAChD,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,YAAY,CAAC;AAMzD,0DAA0D;AAC1D,MAAM,WAAW,yBAAyB;IACxC,kEAAkE;IAClE,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;;;;;;;OASG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,iEAAiE;IACjE,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,kEAAkE;IAClE,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,yDAAyD;IACzD,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,wDAAwD;IACxD,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,8EAA8E;IAC9E,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,4EAA4E;IAC5E,UAAU,CAAC,EAAE,CAAC,KAAK,EAAE,sBAAsB,KAAK,IAAI,CAAC;CACtD;AA4CD,uDAAuD;AACvD,qBAAa,kBAAmB,YAAW,QAAQ;IACjD,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAC;IACpC,QAAQ,CAAC,cAAc,EAAE,OAAO,CAAC;IACjC,oEAAoE;IACpE,YAAY,EAAE,MAAM,GAAG,IAAI,CAAQ;IACnC;sEACkE;IAClE,WAAW,EAAE,MAAM,GAAG,IAAI,CAAQ;IAClC,OAAO,CAAC,MAAM,CAA8B;IAC5C,OAAO,CAAC,UAAU,CAAkC;IACpD,OAAO,CAAC,SAAS,CAA8B;IAC/C,OAAO,CAAC,IAAI,CAAK;IACjB,OAAO,CAAC,YAAY,CAA8B;IAClD,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC,CAA0C;gBAE3D,IAAI,GAAE,yBAA8B;IAiB1C,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAoDrB,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE;QAAE,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;YAYxD,cAAc;YA0Bd,aAAa;IA6F3B,OAAO,CAAC,aAAa;CAiBtB"}

package/dist/types/detectors/index.d.ts

@@ -11,8 +11,8 @@ import type { Detector } from "./base.ts";
 /** Pick a reasonable default: remote when `GHEIM_API_KEY` is set in env,
  *  else a calibrated local model.
  *
- *  The calibrated default (`oBias=0.5`) recovers ~95% of the
- *  multi-entity-letter cases that the raw v1 model fails on, with
- *  ~0.4pp test_v1 strict-F1 cost. See eval/calibration_sweep.json. */
+ *  The calibrated default (`oBias=0.5`) is Pareto-clean on gheim-ch-560m:
+ *  +0.27pp test char-F1 with no probe cost (91.5% perfect at both 0.0
+ *  and 0.5). See eval/calibration_sweep.json. */
 export declare function defaultDetector(): Detector;
 //# sourceMappingURL=index.d.ts.map

package/dist/types/detectors/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/detectors/index.ts"],"names":[],"mappings":"AAAA,YAAY,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAChD,OAAO,EACL,gBAAgB,EAChB,aAAa,IAAI,oBAAoB,EACrC,cAAc,GACf,MAAM,aAAa,CAAC;AACrB,YAAY,EAAE,qBAAqB,EAAE,MAAM,aAAa,CAAC;AACzD,OAAO,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAC3C,YAAY,EAAE,oBAAoB,EAAE,sBAAsB,EAAE,MAAM,YAAY,CAAC;AAC/E,OAAO,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;AACnD,YAAY,EAAE,wBAAwB,EAAE,MAAM,gBAAgB,CAAC;AAC/D,OAAO,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAC;AACrD,YAAY,EAAE,yBAAyB,EAAE,MAAM,iBAAiB,CAAC;AAEjE,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAI1C;;;;;sEAKsE;AACtE,wBAAgB,eAAe,IAAI,QAAQ,CAI1C"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/detectors/index.ts"],"names":[],"mappings":"AAAA,YAAY,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAChD,OAAO,EACL,gBAAgB,EAChB,aAAa,IAAI,oBAAoB,EACrC,cAAc,GACf,MAAM,aAAa,CAAC;AACrB,YAAY,EAAE,qBAAqB,EAAE,MAAM,aAAa,CAAC;AACzD,OAAO,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAC3C,YAAY,EAAE,oBAAoB,EAAE,sBAAsB,EAAE,MAAM,YAAY,CAAC;AAC/E,OAAO,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;AACnD,YAAY,EAAE,wBAAwB,EAAE,MAAM,gBAAgB,CAAC;AAC/D,OAAO,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAC;AACrD,YAAY,EAAE,yBAAyB,EAAE,MAAM,iBAAiB,CAAC;AAEjE,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAI1C;;;;;iDAKiD;AACjD,wBAAgB,eAAe,IAAI,QAAQ,CAI1C"}

package/dist/types/detectors/local.d.ts

@@ -29,7 +29,15 @@ export interface LocalDetectorOptions {
      * (no probe, no fallback — your choice is respected and may throw).
      */
     device?: string;
-    /** Quantization dtype: "fp32" | "fp16" | "q8" | "q4" | "auto" */
+    /**
+     * Quantization dtype: `"fp32" | "fp16" | "q8" | "q4" | "auto"`. Defaults
+     * to `"auto"`, which picks based on the resolved backend: `"fp16"` when
+     * WebGPU was selected (large but byte-equivalent to fp32 on the forensic
+     * probe), `"q8"` otherwise (~550 MB; safe everywhere, but degrades on
+     * the `onnxruntime-web` WASM kernel on Swiss edge cases — see
+     * eval/q8_quality_report.md). Set explicitly to opt out of this
+     * trade-off.
+     */
     dtype?: string;
     /** Aggregation strategy passed to the pipeline call. */
     aggregationStrategy?: string;
@@ -98,10 +106,21 @@ export declare class LocalDetector implements Detector {
      *  the model is loaded. Different from {@link device} only when
      *  `device: "auto"` was requested. */
     actualDevice: string | null;
+    /** Dtype that ended up being used after `load()` resolves. `null` until
+     *  the model is loaded. Different from {@link dtype} only when
+     *  `dtype: "auto"` was requested — in that case it resolves to `"fp16"`
+     *  on WebGPU and `"q8"` everywhere else. */
+    actualDtype: string | null;
     private _pipeline;
     private _loadPromise;
     private readonly _onProgress?;
     constructor(opts?: LocalDetectorOptions);
+    /** Resolve `dtype: "auto"` for a concrete device: fp16 on WebGPU (byte-
+     *  equivalent to fp32 on the forensic probe), q8 on WASM/CPU. The
+     *  WASM-q8 path is known to degrade on Swiss edge cases via
+     *  `onnxruntime-web`, but it's the small-file fallback for low-RAM
+     *  contexts where WebGPU isn't available. Explicit dtype passes through. */
+    private _resolveDtype;
     /**
      * Load the model. Idempotent; concurrent calls share a single load.
      * With `device: "auto"`, probes `navigator.gpu.requestAdapter()` and
@@ -109,7 +128,15 @@ export declare class LocalDetector implements Detector {
      * if construction throws (and reporting both attempts via `onProgress`).
      */
     load(): Promise<PipelineCallable>;
-    /** Build the ordered list of backends to attempt. */
+    /** Build the ordered list of backends to attempt.
+     *
+     * In a browser we try WebGPU (if a real adapter is reachable) and fall
+     * back to WASM. In Node, transformers.js uses onnxruntime-node, which
+     * speaks "cpu" / "cuda" / "webgpu" — never "wasm" — so the fallback
+     * has to be "cpu" instead. We detect "browser-ness" via `typeof window`;
+     * Node 21+ now ships a global `navigator`, so that's no longer a
+     * reliable discriminator on its own.
+     */
     private _planDeviceSequence;
     /** Translate a raw transformers.js progress callback into our normalized event. */
     private _emitProgress;

package/dist/types/detectors/local.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"local.d.ts","sourceRoot":"","sources":["../../../src/detectors/local.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AACH,OAAO,KAAK,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAEhD,eAAO,MAAM,aAAa,iCAAiC,CAAC;AAQ5D,0EAA0E;AAC1E,MAAM,WAAW,oBAAoB;IACnC,mEAAmE;IACnE,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;;;;;;;OASG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,iEAAiE;IACjE,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,wDAAwD;IACxD,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,+EAA+E;IAC/E,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;;OAGG;IACH,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B;;;OAGG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,6DAA6D;IAC7D,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B;;;;;OAKG;IACH,UAAU,CAAC,EAAE,CAAC,KAAK,EAAE,sBAAsB,KAAK,IAAI,CAAC;CACtD;AAED,uFAAuF;AACvF,MAAM,WAAW,sBAAsB;IACrC,yBAAyB;IACzB,KAAK,EAAE,UAAU,GAAG,UAAU,GAAG,UAAU,GAAG,OAAO,GAAG,MAAM,CAAC;IAC/D,6CAA6C;IAC7C,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,6DAA6D;IAC7D,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,4DAA4D;IAC5D,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;wEACoE;IACpE,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;+EAE2E;IAC3E,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,UAAU,aAAa;IACrB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,KAAK,gBAAgB,GAAG,CACtB,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,EACvB,IAAI,CAAC,EAAE;IAAE,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAAC,UAAU,CAAC,EAAE,MAAM,CAAA;CAAE,KAC1D,OAAO,CAAC,aAAa,EAAE,GAAG,aAAa,EAAE,EAAE,CAAC,CAAC;AAwBlD,qBAAa,aAAc,YAAW,QAAQ;IAC5C,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,kFAAkF;IAClF,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,mBAAmB,EAAE,MAAM,CAAC;IACrC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAC;IACpC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B;;0CAEsC;IACtC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAQ;IACnC,OAAO,CAAC,SAAS,CAAiC;IAClD,OAAO,CAAC,YAAY,CAA0C;IAC9D,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC,CAA0C;gBAE3D,IAAI,GAAE,oBAAyB;IAsB3C;;;;;OAKG;IACG,IAAI,IAAI,OAAO,CAAC,gBAAgB,CAAC;IAkCvC,qDAAqD;YACvC,mBAAmB;IAmBjC,mFAAmF;IACnF,OAAO,CAAC,aAAa;IAqBf,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE;QAAE,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;IAYtE;;;;;OAKG;IACG,WAAW,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,IAAI,CAAC,EAAE;QAAE,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC;YAyFlE,SAAS;CAkBxB"}
+{"version":3,"file":"local.d.ts","sourceRoot":"","sources":["../../../src/detectors/local.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AACH,OAAO,KAAK,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAEhD,eAAO,MAAM,aAAa,iCAAiC,CAAC;AAQ5D,0EAA0E;AAC1E,MAAM,WAAW,oBAAoB;IACnC,mEAAmE;IACnE,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;;;;;;;OASG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;;;;;;;OAQG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,wDAAwD;IACxD,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,+EAA+E;IAC/E,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;;OAGG;IACH,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B;;;OAGG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,6DAA6D;IAC7D,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B;;;;;OAKG;IACH,UAAU,CAAC,EAAE,CAAC,KAAK,EAAE,sBAAsB,KAAK,IAAI,CAAC;CACtD;AAED,uFAAuF;AACvF,MAAM,WAAW,sBAAsB;IACrC,yBAAyB;IACzB,KAAK,EAAE,UAAU,GAAG,UAAU,GAAG,UAAU,GAAG,OAAO,GAAG,MAAM,CAAC;IAC/D,6CAA6C;IAC7C,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,6DAA6D;IAC7D,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,4DAA4D;IAC5D,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;wEACoE;IACpE,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;+EAE2E;IAC3E,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,UAAU,aAAa;IACrB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,KAAK,gBAAgB,GAAG,CACtB,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,EACvB,IAAI,CAAC,EAAE;IAAE,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAAC,UAAU,CAAC,EAAE,MAAM,CAAA;CAAE,KAC1D,OAAO,CAAC,aAAa,EAAE,GAAG,aAAa,EAAE,EAAE,CAAC,CAAC;AAwBlD,qBAAa,aAAc,YAAW,QAAQ;IAC5C,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,kFAAkF;IAClF,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,mBAAmB,EAAE,MAAM,CAAC;IACrC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAC;IACpC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B;;0CAEsC;IACtC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAQ;IACnC;;;gDAG4C;IAC5C,WAAW,EAAE,MAAM,GAAG,IAAI,CAAQ;IAClC,OAAO,CAAC,SAAS,CAAiC;IAClD,OAAO,CAAC,YAAY,CAA0C;IAC9D,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC,CAA0C;gBAE3D,IAAI,GAAE,oBAAyB;IAuB3C;;;;gFAI4E;IAC5E,OAAO,CAAC,aAAa;IAKrB;;;;;OAKG;IACG,IAAI,IAAI,OAAO,CAAC,gBAAgB,CAAC;IAoCvC;;;;;;;;OAQG;YACW,mBAAmB;IAsBjC,mFAAmF;IACnF,OAAO,CAAC,aAAa;IAqBf,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE;QAAE,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;IAYtE;;;;;OAKG;IACG,WAAW,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,IAAI,CAAC,EAAE;QAAE,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC;YAyFlE,SAAS;CAkBxB"}

package/dist/types/index.d.ts

@@ -17,7 +17,7 @@ export { DEFAULT_BASE_URL, REMOTE_DEFAULT_MODEL, RemoteDetector, LocalDetector,
 export type { RemoteDetectorOptions, LocalDetectorOptions, LocalDetectorLoadEvent, CompositeDetectorOptions, CalibratedDetectorOptions, } from "./detectors/index.ts";
 export { anonymizeMessages, anonymizeText, deanonymizeStream, deanonymizeText, } from "./plain.ts";
 export type { ChatMessage } from "./plain.ts";
-export declare const VERSION = "0.1.5";
+export declare const VERSION = "0.1.6";
 export { mergeAdjacent } from "./core/session.ts";
 export type { MergedSpan } from "./core/session.ts";
 //# sourceMappingURL=index.d.ts.map

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "gheim",
-  "version": "0.1.5",
+  "version": "0.1.7",
   "description": "PII round-trip for LLM APIs: anonymize before the request, de-anonymize the stream on the way back.",
   "license": "Apache-2.0",
   "author": "Joel Barmettler <joel.barmettler@gmail.com>",