gheim 0.1.1 → 0.1.3

package/README.md

@@ -38,33 +38,38 @@ The core package itself has no runtime dependencies.
 
 `LocalDetector` loads a token-classification model via
 [`@huggingface/transformers`](https://www.npmjs.com/package/@huggingface/transformers)
-(transformers.js). Two models are recommended depending on the
-deployment context. Both implement the same 33-class BIOES output
-schema and are interchangeable.
+(transformers.js). The package's default model is
+[`joelbarmettler/gheim-ch-560m`](https://huggingface.co/joelbarmettler/gheim-ch-560m)
+— a 560M xlm-roberta-large fine-tune optimised for Swiss-market PII
+(strict-span F1 0.916 on Swiss text, see
+[MODEL_CARD.md](https://github.com/joelbarmettlerUZH/gheim/blob/main/MODEL_CARD.md)).
+Any HuggingFace token-classification model that emits the same 33-class
+BIOES schema can be substituted via `model:`.
 
 | Model | Best for | Parameters | Notes |
 |---|---|---:|---|
-| [`joelbarmettler/gheim-ch-560m`](https://huggingface.co/joelbarmettler/gheim-ch-560m) | Swiss-market text (de_CH, fr_CH, it_CH, rm, en) with CH-format account numbers (IBAN, AHV, VAT-CHE) | 560M | Apache 2.0. Test F1 = 0.916 on Swiss text. ONNX export ships in the `onnx/` subfolder. |
+| [`joelbarmettler/gheim-ch-560m`](https://huggingface.co/joelbarmettler/gheim-ch-560m) **(default)** | Swiss-market text (de_CH, fr_CH, it_CH, rm, en) with CH-format account numbers (IBAN, AHV, VAT-CHE) | 560M | Apache 2.0. Test F1 0.916. Hub repo ships `onnx/model_quantized.onnx` only — load with `dtype: "q8"`. |
 | [`openai/privacy-filter`](https://huggingface.co/openai/privacy-filter) | English-first or general use, long-context (up to 128k tokens) | 1.4B (50M active, MoE) | Apache 2.0. Wider language coverage, larger weights. |
 
 ```ts
 import { LocalDetector } from "gheim";
 
-// Recommended for Swiss-market text:
-const det = new LocalDetector({ model: "joelbarmettler/gheim-ch-560m" });
+// Default — Swiss-tuned, q8 ONNX. `device: "auto"` probes WebGPU in
+// the browser and falls back to WASM; in Node it's WASM directly.
+const det = new LocalDetector({ dtype: "q8" });
 
 // Alternative for English or general use:
 const detEn = new LocalDetector({ model: "openai/privacy-filter" });
 ```
 
-The package default `model` is `openai/privacy-filter`. Pass `model`
-explicitly to opt into the Swiss model.
-
 ## Drop-in OpenAI client
 
 ```ts
 import { OpenAI } from "gheim/openai";
 
+// Same constructor shape as `openai`. `apiKey`, `baseURL`, etc. work
+// at the top level (forwarded to the inner client). gheim-specific
+// keys: `gheimDetector`, `gheimStrict`, `openaiClient`, `clientOptions`.
 const client = new OpenAI();
 const r = await client.chat.completions.create({
   model: "gpt-4o",
@@ -74,6 +79,17 @@ const r = await client.chat.completions.create({
 // OpenAI only ever saw "<PERSON_1>".
 ```
 
+Custom endpoint or key (e.g. OpenRouter, local vLLM):
+
+```ts
+import { OpenAI } from "gheim/openai";
+
+const client = new OpenAI({
+  apiKey: process.env.OPENROUTER_API_KEY,
+  baseURL: "https://openrouter.ai/api/v1",
+});
+```
+
 Streaming:
 
 ```ts
@@ -104,9 +120,9 @@ All non-chat OpenAI resources are on `client.raw` once `await client.ready()` re
 import { Session, LocalDetector, anonymizeText, deanonymizeText } from "gheim";
 
 const session = new Session();
-(session as any).detector = new LocalDetector({
-  model: "joelbarmettler/gheim-ch-560m",
-});
+(session as any).detector = new LocalDetector({ dtype: "q8" });
+//                                 ^ defaults to joelbarmettler/gheim-ch-560m
+
 const clean = await anonymizeText("Hi, my name is Joel", session);
 // ... call any LLM with clean ...
 const final = deanonymizeText(responseText, session);
@@ -155,12 +171,18 @@ client.raw.beta.assistants.create(...);  // always works regardless of strict mo
 ```ts
 import { LocalDetector, RemoteDetector, defaultDetector } from "gheim";
 
-// Local: via @huggingface/transformers, on Node and Bun.
+// Local: via @huggingface/transformers, on Node, Bun, and browsers.
+// `device: "auto"` (default) probes navigator.gpu in the browser and
+// falls back to WASM if WebGPU isn't usable; in Node it's WASM only.
 const local = new LocalDetector({
-  model: "joelbarmettler/gheim-ch-560m",
   device: "auto",
-  dtype: "fp32",
+  dtype: "q8",
+  onProgress: (e) => {
+    if (e.fraction != null) console.log(`loading: ${(e.fraction * 100) | 0}%`);
+  },
 });
+await local.load();
+console.log("backend in use:", local.actualDevice);  // "webgpu" or "wasm"
 
 // Remote: against your own gheim-server or api.gheim.ch.
 const remote = new RemoteDetector({
@@ -174,10 +196,9 @@ const auto = defaultDetector();
 
 ## Runtime support
 
-Tested on Node 18+ and Bun 1.1+ with `device: "cpu"` (the default). The
-underlying transformers.js library also supports `device: "webgpu"` in
-modern browsers; the WebGPU path is not exercised in CI and should be
-considered best-effort.
+Tested on Node 18+, Bun 1.1+, and modern browsers (Chrome 113+, Edge,
+Safari 17+, Firefox via flag). Browser builds use WebGPU when available
+and fall back to WebAssembly automatically.
 
 For Node servers, installing `onnxruntime-node` alongside
 `@huggingface/transformers` enables the native ONNX backend, which is

package/dist/cjs/index.cjs

@@ -41,6 +41,7 @@ var exports_src = {};
 __export(exports_src, {
   renderSentinel: () => renderSentinel,
   parseSentinel: () => parseSentinel,
+  mergeAdjacent: () => mergeAdjacent,
   labelTag: () => labelTag,
   isPossibleSentinelPrefix: () => isPossibleSentinelPrefix,
   defaultDetector: () => defaultDetector,
@@ -56,7 +57,8 @@ __export(exports_src, {
   REMOTE_DEFAULT_MODEL: () => DEFAULT_MODEL,
   LocalDetector: () => LocalDetector,
   LABEL_TO_TAG: () => LABEL_TO_TAG,
-  DEFAULT_BASE_URL: () => DEFAULT_BASE_URL
+  DEFAULT_BASE_URL: () => DEFAULT_BASE_URL,
+  CompositeDetector: () => CompositeDetector
 });
 module.exports = __toCommonJS(exports_src);
 
@@ -96,98 +98,6 @@ function isPossibleSentinelPrefix(buffer) {
   return PARTIAL_SENTINEL_RE.test(buffer);
 }
 
-// src/normalizers.ts
-var import_node_module = require("node:module");
-var _require = null;
-var _requireResolved = false;
-function _getRequire() {
-  if (_requireResolved)
-    return _require;
-  _requireResolved = true;
-  try {
-    _require = import_node_module.createRequire("file:///home/runner/work/gheim/gheim/packages/gheim-js/src/normalizers.ts");
-  } catch {
-    _require = null;
-  }
-  return _require;
-}
-function loadSync(name, errMsg) {
-  const req = _getRequire();
-  if (!req)
-    throw new Error(errMsg);
-  try {
-    return req(name);
-  } catch {
-    throw new Error(errMsg);
-  }
-}
-function loadPhoneLib() {
-  return loadSync("libphonenumber-js", "the e164 normalizer requires libphonenumber-js. " + "Install with: bun add libphonenumber-js");
-}
-function e164(opts = {}) {
-  const lib = loadPhoneLib();
-  return (s) => {
-    try {
-      const parsed = lib.parsePhoneNumberFromString(s, opts.region);
-      if (!parsed || !parsed.isValid())
-        return null;
-      return parsed.format("E.164");
-    } catch {
-      return null;
-    }
-  };
-}
-function loadChrono(opts) {
-  const mod = loadSync("chrono-node", "the isoDate normalizer requires chrono-node. " + "Install with: bun add chrono-node");
-  if (opts.locale) {
-    const localeParser = mod[opts.locale];
-    if (localeParser)
-      return localeParser;
-  }
-  return mod;
-}
-function isoDate(opts = {}) {
-  const parser = loadChrono(opts);
-  return (s) => {
-    const d = parser.parseDate(s);
-    if (!d)
-      return null;
-    return d.toISOString().slice(0, 10);
-  };
-}
-function germanTransliteration() {
-  const table = {
-    ä: "ae",
-    Ä: "ae",
-    ö: "oe",
-    Ö: "oe",
-    ü: "ue",
-    Ü: "ue",
-    ß: "ss"
-  };
-  return (s) => {
-    let out = "";
-    for (const ch of s.normalize("NFKC")) {
-      out += table[ch] ?? ch;
-    }
-    return out.toLowerCase().split(/\s+/).filter(Boolean).join(" ");
-  };
-}
-var BUILTINS = {
-  e164: () => e164(),
-  iso_date: () => isoDate(),
-  german: () => germanTransliteration()
-};
-function resolveNormalizer(spec) {
-  if (typeof spec === "function")
-    return spec;
-  const factory = BUILTINS[spec];
-  if (!factory) {
-    throw new Error(`unknown built-in normalizer: ${JSON.stringify(spec)}. ` + `Available: ${Object.keys(BUILTINS).join(", ")}, or pass a callable.`);
-  }
-  return factory();
-}
-
 // src/core/session.ts
 function normalizeSurface(s) {
   return s.normalize("NFKC").toLowerCase().split(/\s+/).filter(Boolean).join(" ");
@@ -200,8 +110,8 @@ class Session {
   resolvedNormalizers = {};
   constructor(opts = {}) {
     if (opts.normalizers) {
-      for (const [label, spec] of Object.entries(opts.normalizers)) {
-        this.resolvedNormalizers[label] = resolveNormalizer(spec);
+      for (const [label, fn] of Object.entries(opts.normalizers)) {
+        this.resolvedNormalizers[label] = fn;
       }
     }
   }
@@ -225,8 +135,11 @@ class Session {
     return sentinel;
   }
   applySpans(text, spans) {
+    return this.applySpansDetailed(text, spans).redacted;
+  }
+  applySpansDetailed(text, spans) {
     if (spans.length === 0)
-      return text;
+      return { redacted: text, merged: [] };
     const sorted = [...spans].sort((a, b) => a.start - b.start || b.end - b.start - (a.end - a.start));
     const merged = mergeAdjacent(text, sorted);
     const out = [];
@@ -242,7 +155,7 @@ class Session {
       cursor = end;
     }
     out.push(text.slice(cursor));
-    return out.join("");
+    return { redacted: out.join(""), merged };
   }
   restore(text) {
     if (Object.keys(this.mapping).length === 0)
@@ -440,8 +353,10 @@ class LocalDetector {
   chunkTokens;
   chunkOverlapTokens;
   batchSize;
+  actualDevice = null;
   _pipeline = null;
   _loadPromise = null;
+  _onProgress;
   constructor(opts = {}) {
     this.model = opts.model ?? DEFAULT_MODEL2;
     this.device = opts.device ?? "auto";
@@ -450,11 +365,13 @@ class LocalDetector {
     this.chunkTokens = opts.chunkTokens ?? DEFAULT_CHUNK_TOKENS;
     this.chunkOverlapTokens = opts.chunkOverlapTokens ?? DEFAULT_OVERLAP_TOKENS;
     this.batchSize = opts.batchSize ?? DEFAULT_BATCH_SIZE;
+    this._onProgress = opts.onProgress;
     if (this.chunkOverlapTokens >= this.chunkTokens) {
       throw new Error(`chunkOverlapTokens (${this.chunkOverlapTokens}) must be smaller than chunkTokens (${this.chunkTokens})`);
     }
     if (opts.pipelineInstance) {
       this._pipeline = opts.pipelineInstance;
+      this.actualDevice = this.device === "auto" ? null : this.device;
     }
   }
   async load() {
@@ -463,24 +380,70 @@ class LocalDetector {
     if (this._loadPromise)
       return this._loadPromise;
     this._loadPromise = (async () => {
-      let mod;
-      try {
-        mod = await import("@huggingface/transformers");
-      } catch (e) {
-        throw new Error("LocalDetector requires `@huggingface/transformers`. Install with: bun add @huggingface/transformers");
+      const mod = await loadTransformersJs();
+      const sequence = await this._planDeviceSequence();
+      let lastErr;
+      for (let i = 0;i < sequence.length; i++) {
+        const dev = sequence[i] ?? "wasm";
+        try {
+          const pipe = await mod.pipeline("token-classification", this.model, {
+            device: dev,
+            dtype: this.dtype,
+            progress_callback: this._onProgress ? (raw) => this._emitProgress(raw, dev) : undefined
+          });
+          this._pipeline = pipe;
+          this.actualDevice = dev;
+          return pipe;
+        } catch (e) {
+          lastErr = e;
+          if (i < sequence.length - 1)
+            continue;
+          throw e;
+        }
       }
-      const pipe = await mod.pipeline("token-classification", this.model, {
-        device: this.device,
-        dtype: this.dtype
-      });
-      this._pipeline = pipe;
-      return pipe;
+      throw lastErr ?? new Error("LocalDetector.load(): no candidate backends");
     })();
     return this._loadPromise;
   }
+  async _planDeviceSequence() {
+    if (this.device !== "auto")
+      return [this.device];
+    const seq = [];
+    const nav = typeof navigator !== "undefined" ? navigator : null;
+    if (nav?.gpu?.requestAdapter) {
+      try {
+        const adapter = await nav.gpu.requestAdapter();
+        if (adapter)
+          seq.push("webgpu");
+      } catch {}
+    }
+    seq.push("wasm");
+    return seq;
+  }
+  _emitProgress(raw, device) {
+    if (!this._onProgress)
+      return;
+    const event = {
+      phase: raw.status ?? "progress",
+      device
+    };
+    if (raw.file)
+      event.file = raw.file;
+    if (raw.status === "progress") {
+      if (raw.loaded != null)
+        event.loadedBytes = raw.loaded;
+      if (raw.total != null)
+        event.totalBytes = raw.total;
+      if (raw.progress != null) {
+        const norm = raw.progress > 1 ? raw.progress / 100 : raw.progress;
+        event.fraction = Math.max(0, Math.min(1, norm));
+      }
+    }
+    this._onProgress(event);
+  }
   async detect(text, opts) {
     if (opts?.model && opts.model !== this.model) {
-      throw new Error(`LocalDetector was constructed with model='${this.model}'; per-call model override ('${opts.model}') not supported.`);
+      throw new Error(`LocalDetector was constructed with model='${this.model}'; ` + `per-call model override ('${opts.model}') not supported.`);
     }
     if (!text)
       return [];
@@ -489,7 +452,7 @@ class LocalDetector {
   }
   async detectBatch(texts, opts) {
     if (opts?.model && opts.model !== this.model) {
-      throw new Error(`LocalDetector was constructed with model='${this.model}'; per-call model override ('${opts.model}') not supported.`);
+      throw new Error(`LocalDetector was constructed with model='${this.model}'; ` + `per-call model override ('${opts.model}') not supported.`);
     }
     if (texts.length === 0)
       return [];
@@ -650,6 +613,221 @@ function chunkByChars(text, chunkChars, overlapChars) {
 function normalizeWord(word) {
   return word.replace(/##/g, "").trim();
 }
+async function loadTransformersJs() {
+  try {
+    return await import("@huggingface/transformers");
+  } catch {
+    throw new Error("LocalDetector requires `@huggingface/transformers`. Install with: bun add @huggingface/transformers");
+  }
+}
+// src/detectors/composite.ts
+function luhnOk(s) {
+  const digits = [];
+  for (const c of s) {
+    if (c >= "0" && c <= "9")
+      digits.push(Number(c));
+  }
+  if (digits.length < 13 || digits.length > 19)
+    return false;
+  let total = 0;
+  for (let i = 0;i < digits.length; i++) {
+    let d = digits[digits.length - 1 - i] ?? 0;
+    if (i % 2 === 1) {
+      d *= 2;
+      if (d > 9)
+        d -= 9;
+    }
+    total += d;
+  }
+  return total % 10 === 0;
+}
+function ibanChOk(raw) {
+  const s = raw.replace(/[\s-]/g, "");
+  if (s.length !== 21 || !s.startsWith("CH") || !/^\d+$/.test(s.slice(2))) {
+    return false;
+  }
+  const rearranged = s.slice(4) + s.slice(0, 4);
+  let numeric = "";
+  for (const c of rearranged) {
+    numeric += c >= "A" && c <= "Z" ? String(c.charCodeAt(0) - 55) : c;
+  }
+  let rem = 0;
+  for (const ch of numeric) {
+    rem = (rem * 10 + Number(ch)) % 97;
+  }
+  return rem === 1;
+}
+function ahvOk(s) {
+  const digits = [];
+  for (const c of s) {
+    if (c >= "0" && c <= "9")
+      digits.push(Number(c));
+  }
+  if (digits.length !== 13)
+    return false;
+  if (digits[0] !== 7 || digits[1] !== 5 || digits[2] !== 6)
+    return false;
+  let total = 0;
+  for (let i = 0;i < 12; i++) {
+    total += (digits[i] ?? 0) * (i % 2 === 0 ? 1 : 3);
+  }
+  const check = (10 - total % 10) % 10;
+  return digits[12] === check;
+}
+function vatCheOk(s) {
+  const digits = [];
+  for (const c of s) {
+    if (c >= "0" && c <= "9")
+      digits.push(Number(c));
+  }
+  if (digits.length < 9)
+    return false;
+  const weights = [5, 4, 3, 2, 7, 6, 5, 4];
+  let total = 0;
+  for (let i = 0;i < 8; i++) {
+    total += (digits[i] ?? 0) * (weights[i] ?? 0);
+  }
+  let expected = 11 - total % 11;
+  if (expected === 11)
+    expected = 0;
+  else if (expected === 10)
+    return false;
+  return digits[8] === expected;
+}
+var REGEXES = [
+  {
+    label: "private_email",
+    pattern: /\b[\w._%+\-]+@[\w.\-]+\.[A-Za-z]{2,}\b/g
+  },
+  {
+    label: "private_url",
+    pattern: /https?:\/\/[^\s)>\]]+/g
+  },
+  {
+    label: "private_phone",
+    pattern: /(?:\+41|0041|0)\s?(?:\(0\)\s?)?\d{2}[\s\-/.]?\d{3}[\s\-/.]?\d{2}[\s\-/.]?\d{2}\b/g
+  },
+  {
+    label: "account_number",
+    pattern: /\bCH\d{2}(?:[\s-]?\d){17}\b/g,
+    validator: ibanChOk
+  },
+  {
+    label: "account_number",
+    pattern: /\b756\.\d{4}\.\d{4}\.\d{2}\b/g,
+    validator: ahvOk
+  },
+  {
+    label: "account_number",
+    pattern: /\b756\d{10}\b/g,
+    validator: ahvOk
+  },
+  {
+    label: "account_number",
+    pattern: /\bCHE-\d{3}\.\d{3}\.\d{3}(?:\s+(?:MWST|TVA|IVA))?\b/g,
+    validator: vatCheOk
+  },
+  {
+    label: "account_number",
+    pattern: /\b(?:\d[\s-]?){15,16}\d\b/g,
+    validator: luhnOk
+  },
+  {
+    label: "private_date",
+    pattern: /\b(?:\d{1,2}[./]\d{1,2}[./]\d{4}|\d{4}[./-]\d{1,2}[./-]\d{1,2})\b/g
+  },
+  {
+    label: "secret",
+    pattern: /\bsk-(?:proj-)?[A-Za-z0-9_\-]{20,}\b/g
+  },
+  {
+    label: "secret",
+    pattern: /\bghp_[A-Za-z0-9]{36}\b/g
+  },
+  {
+    label: "secret",
+    pattern: /\bAKIA[A-Z0-9]{16}\b/g
+  }
+];
+var URL_TRAIL_TRIM = '.,;:!?")>]';
+function trimUrl(text, start, end) {
+  while (end > start && URL_TRAIL_TRIM.includes(text[end - 1] ?? ""))
+    end--;
+  return [start, end];
+}
+function findRegexSpans(text) {
+  const candidates = [];
+  for (const { label, pattern, validator } of REGEXES) {
+    pattern.lastIndex = 0;
+    let m;
+    while ((m = pattern.exec(text)) !== null) {
+      let start = m.index;
+      let end = m.index + m[0].length;
+      if (label === "private_url")
+        [start, end] = trimUrl(text, start, end);
+      const value = text.slice(start, end);
+      if (!value.trim())
+        continue;
+      if (validator && !validator(value))
+        continue;
+      candidates.push({
+        start,
+        end,
+        label,
+        text: value,
+        score: 1
+      });
+    }
+  }
+  candidates.sort((a, b) => a.start !== b.start ? a.start - b.start : b.end - b.start - (a.end - a.start));
+  const out = [];
+  let lastEnd = -1;
+  for (const sp of candidates) {
+    if (sp.start < lastEnd)
+      continue;
+    out.push(sp);
+    lastEnd = sp.end;
+  }
+  return out;
+}
+function maskSpans(text, spans) {
+  if (spans.length === 0)
+    return text;
+  const chars = [...text];
+  for (const sp of spans) {
+    for (let i = sp.start;i < sp.end && i < chars.length; i++) {
+      chars[i] = " ";
+    }
+  }
+  return chars.join("");
+}
+
+class CompositeDetector {
+  model;
+  constructor(opts) {
+    if (!opts || !opts.model) {
+      throw new Error("CompositeDetector requires { model: Detector } — pass any object with " + "a .detect(text) method (e.g. new LocalDetector({...})).");
+    }
+    this.model = opts.model;
+  }
+  async detect(text, opts) {
+    if (!text)
+      return [];
+    const regexSpans = findRegexSpans(text);
+    const masked = maskSpans(text, regexSpans);
+    const modelSpans = await this.model.detect(masked, opts);
+    const out = [...regexSpans];
+    const regexRanges = regexSpans.map((s) => [s.start, s.end]);
+    for (const sp of modelSpans) {
+      const overlaps = regexRanges.some(([rs, re]) => sp.start < re && rs < sp.end);
+      if (overlaps)
+        continue;
+      out.push(sp);
+    }
+    out.sort((a, b) => a.start !== b.start ? a.start - b.start : b.end - b.start - (a.end - a.start));
+    return out;
+  }
+}
 // src/detectors/index.ts
 function defaultDetector() {
   const hasKey = typeof process !== "undefined" && Boolean(process.env?.GHEIM_API_KEY);
@@ -704,4 +882,4 @@ async function* deanonymizeStream(chunks, session) {
 }
 
 // src/index.ts
-var VERSION = "0.1.0";
+var VERSION = "0.1.3";